tcg/wasm32: add opt-in compiled-vs-TCI per-TB differential checker

Runs authoritative TCI first, snapshots its result, rewinds guest state,
runs the compiled TB as a disposable shadow, compares GPRs/DRAM, then
restores TCI. Opt-in via wasm_diff_enable(); normal execution unchanged.

Valid comparison required restoring all state a single TB can touch:
- full CPUNegativeOffsetState (the env-8 interrupt check sits before env)
- deep CPUTLB.f[].table / d[].fulltlb per MMU mode
- and skipping async cpu_exit() force-exit races.

Only side-effect-safe candidates compared (store TB, no general helper).
Result: 160k+ store-TB comparisons during reader-open, zero divergence.
Per-TB store codegen is exonerated; corruption is emergent across TBs.

accel/tcg/cpu-exec.c

@@ -41,6 +41,9 @@
 #include "tb-context.h"
 #include "internal-common.h"
 #include "internal-target.h"
+#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER)
+#include "../../tcg/wasm32.h"
+#endif
 
 /* -icount align implementation. */
 
@@ -173,6 +176,13 @@ uint32_t curr_cflags(CPUState *cpu)
         cflags |= CF_NO_GOTO_TB;
     }
 
+#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER)
+    /* Differential mode runs one TB per dispatch so wasm_cur_tb_pc stays exact. */
+    if (wasm_diff_on()) {
+        cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR;
+    }
+#endif
+
     return cflags;
 }
 
@@ -455,6 +465,9 @@ cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit)
     }
 
     qemu_thread_jit_execute();
+#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER)
+    wasm_cur_tb_pc = itb->pc;
+#endif
     ret = tcg_qemu_tb_exec(cpu_env(cpu), tb_ptr);
     cpu->neg.can_do_io = true;
     qemu_plugin_disable_mem_helpers(cpu);

tcg/tcg.c

@@ -6607,6 +6607,9 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
 #endif
 
 #if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER)
+    wasm_diff_register_tb(tb->tc.ptr, tcg_splitwx_to_rx(tb->tc.ptr), pc_start,
+                          wasm_tci_only_tb,
+                          wasm_tci_only_tb && !wasm_tb_had_helper);
     if (wasm_tci_only_tb) {
         for (int i = 0; i < get_core_nums(); i++) {
             export_vec_base[i] = WASM_TCI_ONLY_ENTRY;

tcg/wasm32.c

@@ -22,7 +22,9 @@
 #if !defined(CONFIG_TCG_INTERPRETER) && defined(EMSCRIPTEN)
 
 #include "qemu/osdep.h"
+#include "cpu.h"
 #include "exec/cpu_ldst.h"
+#include "exec/cpu-common.h"
 #include "tcg/tcg-op.h"
 #include "tcg/tcg-ldst.h"
 #include <string.h>
@@ -590,7 +592,7 @@ __thread tcg_target_ulong regs[TCG_TARGET_NB_REGS];
 
 static inline uintptr_t tcg_qemu_tb_exec_tci(CPUArchState *env)
 {
-    uint32_t *tb_ptr = (uint8_t*)ctx.tb_ptr + *(uint32_t*)ctx.tb_ptr;
+    uint32_t *tb_ptr = (uint8_t *)ctx.tb_ptr + *(uint32_t *)ctx.tb_ptr;
     uint64_t *stack = ctx.stack;
 
     regs[TCG_AREG0] = (tcg_target_ulong)env;
@@ -1170,7 +1172,7 @@ static inline uintptr_t tcg_qemu_tb_exec_tci(CPUArchState *env)
                     // enter to wasm TB
                     return 0;
                 }
-                tb_ptr = (uint8_t*)tb_ptr + *(uint32_t*)tb_ptr;
+                tb_ptr = (uint8_t *)tb_ptr + *(uint32_t *)tb_ptr;
             }
             break;
 
@@ -1193,7 +1195,7 @@ static inline uintptr_t tcg_qemu_tb_exec_tci(CPUArchState *env)
                 // enter to wasm TB
                 return 0;
             }
-            tb_ptr = (uint8_t*)tb_ptr + *(uint32_t*)tb_ptr;
+            tb_ptr = (uint8_t *)tb_ptr + *(uint32_t *)tb_ptr;
 
             break;
 
@@ -1258,16 +1260,302 @@ static inline uintptr_t tcg_qemu_tb_exec_tci(CPUArchState *env)
     }
 }
 
+/*
+ * Compiled-vs-TCI differential (debug).
+ *
+ * TCI stays the live/authoritative path (never crashes). For a TB whose guest PC
+ * is in the inflate window we additionally run the compiled blob as a shadow
+ * against a DRAM + register snapshot, then compare the resulting registers and
+ * DRAM against the authoritative TCI run. The first mismatch pinpoints the TB
+ * (and the first divergent guest register / DRAM word) whose compiled codegen is
+ * wrong — the corruption only manifests through a store, so a divergent DRAM word
+ * is the miscompiled write. Opt-in via wasm_diff_enable(); no effect otherwise.
+ */
+#define DIFF_DRAM_BASE   0x3fc80000u
+#define DIFF_DRAM_SIZE   0x60000u
+#define DIFF_PC_LO       0x42000000ull
+#define DIFF_PC_HI       0x421a0000ull
+
+__thread uint64_t wasm_cur_tb_pc;
+static int wasm_diff_enabled;
+static GHashTable *diff_store_tbs;
+static GHashTable *diff_safe_tbs;
+static __thread int diff_reported;
+static __thread int diff_announced;
+static __thread int diff_started;
+static __thread int diff_skipped_slow;
+static __thread uint32_t diff_compared;
+static __thread uint8_t *diff_dram_pre, *diff_dram_c, *diff_dram_t;
+static __thread uint8_t diff_env_pre[sizeof(CPUArchState)];
+static __thread uint8_t diff_env_c[sizeof(CPUArchState)];
+static __thread uint8_t diff_env_t[sizeof(CPUArchState)];
+
+/*
+ * TB Entry Reads CPUNegativeOffsetState - The exit/interrupt check at TB start
+ * loads env-8 (icount_decr/exit flag), which lives before CPUArchState. The
+ * shadow must see the same pre-state here or it wrongly takes the early-exit
+ * path, so snapshot a window immediately below env too.
+ */
+#define DIFF_NEG sizeof(CPUNegativeOffsetState)
+static __thread uint8_t diff_neg_pre[DIFF_NEG];
+static __thread uint8_t diff_neg_t[DIFF_NEG];
+static __thread uint8_t *diff_tlb_pre[NB_MMU_MODES];
+static __thread uint8_t *diff_fulltlb_pre[NB_MMU_MODES];
+static __thread size_t diff_tlb_size[NB_MMU_MODES];
+static __thread size_t diff_fulltlb_size[NB_MMU_MODES];
+static __thread uint32_t diff_compiled_ret;
+static __thread int diff_tci_iters;
+
+EMSCRIPTEN_KEEPALIVE void wasm_diff_enable(void)
+{
+    wasm_diff_enabled = 1;
+}
+
+int wasm_diff_on(void)
+{
+    return wasm_diff_enabled;
+}
+
+void wasm_diff_register_tb(const void *rw_ptr, const void *rx_ptr, uint64_t pc,
+                           bool has_store, bool safe_shadow)
+{
+    if (!wasm_diff_on()) {
+        return;
+    }
+    if (!diff_store_tbs) {
+        diff_store_tbs = g_hash_table_new(g_direct_hash, g_direct_equal);
+        diff_safe_tbs = g_hash_table_new(g_direct_hash, g_direct_equal);
+    }
+    g_hash_table_remove(diff_store_tbs, rw_ptr);
+    g_hash_table_remove(diff_store_tbs, rx_ptr);
+    g_hash_table_remove(diff_safe_tbs, rw_ptr);
+    g_hash_table_remove(diff_safe_tbs, rx_ptr);
+    if (!has_store) {
+        return;
+    }
+    gpointer value = GUINT_TO_POINTER((uint32_t)pc);
+    g_hash_table_insert(diff_store_tbs, (void *)rw_ptr, value);
+    g_hash_table_insert(diff_store_tbs, (void *)rx_ptr, value);
+    if (safe_shadow) {
+        g_hash_table_insert(diff_safe_tbs, (void *)rw_ptr, GINT_TO_POINTER(1));
+        g_hash_table_insert(diff_safe_tbs, (void *)rx_ptr, GINT_TO_POINTER(1));
+    }
+}
+
+bool wasm_diff_lookup_store_tb(const void *tb_ptr, uint64_t *pc)
+{
+    gpointer value = diff_store_tbs ? g_hash_table_lookup(diff_store_tbs, tb_ptr) : NULL;
+    if (!value) {
+        return false;
+    }
+    if (pc) {
+        *pc = GPOINTER_TO_UINT(value);
+    }
+    return true;
+}
+
+bool wasm_diff_lookup_safe_tb(const void *tb_ptr)
+{
+    return diff_safe_tbs && g_hash_table_contains(diff_safe_tbs, tb_ptr);
+}
+
+static void diff_report(uint64_t pc)
+{
+    uint32_t pre_ra, pre_sp, stack_m4 = 0, stack_0 = 0, stack_4 = 0;
+    memcpy(&pre_ra, diff_env_pre + 4, 4);
+    memcpy(&pre_sp, diff_env_pre + 8, 4);
+    if (pre_sp >= DIFF_DRAM_BASE + 4 && pre_sp + 8 <= DIFF_DRAM_BASE + DIFF_DRAM_SIZE) {
+        uint32_t off = pre_sp - DIFF_DRAM_BASE;
+        memcpy(&stack_m4, diff_dram_pre + off - 4, 4);
+        memcpy(&stack_0, diff_dram_pre + off, 4);
+        memcpy(&stack_4, diff_dram_pre + off + 4, 4);
+    }
+    /* Compare guest integer registers (env offset 0..127 = gpr[0..31]). */
+    for (int i = 0; i < 32; i++) {
+        uint32_t c, t;
+        memcpy(&c, diff_env_c + i * 4, 4);
+        memcpy(&t, diff_env_t + i * 4, 4);
+        if (c != t) {
+            uint32_t neg8_pre, neg8_t;
+            memcpy(&neg8_pre, diff_neg_pre + DIFF_NEG - 8, 4);
+            memcpy(&neg8_t, diff_neg_t + DIFF_NEG - 8, 4);
+            printf("TCGDIFF pc=0x%llx gpr[%d] compiled=0x%08x tci=0x%08x "
+                   "pre-ra=0x%08x pre-sp=0x%08x cret=0x%08x neg8_pre=0x%08x neg8_t=0x%08x tci_iters=%d\n",
+                   (unsigned long long)pc, i, c, t, pre_ra, pre_sp,
+                   diff_compiled_ret, neg8_pre, neg8_t, diff_tci_iters);
+            fflush(stdout);
+            diff_reported = 1;
+            return;
+        }
+    }
+    /* Compare DRAM word-by-word; first divergent word is the miscompiled store. */
+    for (uint32_t off = 0; off < DIFF_DRAM_SIZE; off += 4) {
+        uint32_t c, t;
+        memcpy(&c, diff_dram_c + off, 4);
+        memcpy(&t, diff_dram_t + off, 4);
+        if (c != t) {
+            printf("TCGDIFF pc=0x%llx dram[0x%08x] compiled=0x%08x tci=0x%08x\n",
+                   (unsigned long long)pc, DIFF_DRAM_BASE + off, c, t);
+            fflush(stdout);
+            diff_reported = 1;
+            return;
+        }
+    }
+}
+
+static void diff_snapshot_pre(void)
+{
+    if (!diff_dram_pre) {
+        diff_dram_pre = malloc(DIFF_DRAM_SIZE);
+        diff_dram_c = malloc(DIFF_DRAM_SIZE);
+        diff_dram_t = malloc(DIFF_DRAM_SIZE);
+    }
+    CPUNegativeOffsetState *neg = (void *)((uint8_t *)ctx.env - DIFF_NEG);
+    memcpy(diff_env_pre, ctx.env, sizeof(CPUArchState));
+    memcpy(diff_neg_pre, neg, DIFF_NEG);
+    for (int i = 0; i < NB_MMU_MODES; i++) {
+        size_t tlb_size = neg->tlb.f[i].mask + (1 << CPU_TLB_ENTRY_BITS);
+        size_t n = (neg->tlb.f[i].mask >> CPU_TLB_ENTRY_BITS) + 1;
+        size_t fulltlb_size = n * sizeof(CPUTLBEntryFull);
+        diff_tlb_pre[i] = g_realloc(diff_tlb_pre[i], tlb_size);
+        diff_fulltlb_pre[i] = g_realloc(diff_fulltlb_pre[i], fulltlb_size);
+        diff_tlb_size[i] = tlb_size;
+        diff_fulltlb_size[i] = fulltlb_size;
+        memcpy(diff_tlb_pre[i], neg->tlb.f[i].table, tlb_size);
+        memcpy(diff_fulltlb_pre[i], neg->tlb.d[i].fulltlb, fulltlb_size);
+    }
+    cpu_physical_memory_read(DIFF_DRAM_BASE, diff_dram_pre, DIFF_DRAM_SIZE);
+}
+
+static bool diff_restore_pre(CPUArchState *env)
+{
+    CPUNegativeOffsetState *neg = (void *)((uint8_t *)env - DIFF_NEG);
+    CPUNegativeOffsetState *pre = (void *)diff_neg_pre;
+    for (int i = 0; i < NB_MMU_MODES; i++) {
+        if (neg->tlb.f[i].table != pre->tlb.f[i].table ||
+            neg->tlb.d[i].fulltlb != pre->tlb.d[i].fulltlb) {
+            return false;
+        }
+    }
+    cpu_physical_memory_write(DIFF_DRAM_BASE, diff_dram_pre, DIFF_DRAM_SIZE);
+    memcpy(env, diff_env_pre, sizeof(CPUArchState));
+    memcpy(neg, diff_neg_pre, DIFF_NEG);
+    for (int i = 0; i < NB_MMU_MODES; i++) {
+        memcpy(neg->tlb.f[i].table, diff_tlb_pre[i], diff_tlb_size[i]);
+        memcpy(neg->tlb.d[i].fulltlb, diff_fulltlb_pre[i], diff_fulltlb_size[i]);
+    }
+    return true;
+}
+
+static void diff_ensure_compiled(const void *tb_ptr)
+{
+    ctx.tb_ptr = (uint32_t *)tb_ptr;
+    int slot_off = (uint32_t)ctx.tb_ptr + export_vec_off;
+    if (*(int32_t *)slot_off <= 0) {
+        instantiate_wasm();
+    }
+}
+
+static bool diff_run_compiled(const void *tb_ptr)
+{
+    ctx.tb_ptr = (uint32_t *)tb_ptr;
+    ctx.do_init = 1;
+    ctx.done_flag = 2;
+    int slot_off = (uint32_t)ctx.tb_ptr + export_vec_off;
+    g_assert(*(int32_t *)slot_off > 0);
+    diff_compiled_ret = ((wasm_func_ptr)(*(uint32_t *)slot_off))(&ctx);
+    memcpy(diff_env_c, ctx.env, sizeof(CPUArchState));
+    cpu_physical_memory_read(DIFF_DRAM_BASE, diff_dram_c, DIFF_DRAM_SIZE);
+    return ctx.done_flag == 2 && (uint32_t)ctx.tb_ptr == 0;
+}
+
 uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env,
                                             const void *v_tb_ptr)
 {
     ctx.env = env;
     ctx.tb_ptr = (uint32_t*)v_tb_ptr;
     ctx.do_init = 1;
+
+    if (wasm_diff_on()) {
+        if (!diff_started) {
+            printf("TCGDIFF enabled first-pc=0x%llx tb-meta=0x%08x\n",
+                   (unsigned long long)wasm_cur_tb_pc, *(uint32_t *)ctx.tb_ptr);
+            fflush(stdout);
+            diff_started = 1;
+        }
+        uint64_t pc = 0;
+        bool is_store_tb = wasm_diff_lookup_store_tb(v_tb_ptr, &pc);
+        int eligible = !diff_reported && is_store_tb &&
+                       wasm_diff_lookup_safe_tb(v_tb_ptr) &&
+                       pc >= DIFF_PC_LO && pc < DIFF_PC_HI;
+        if (eligible) {
+            if (!diff_announced) {
+                printf("TCGDIFF active first-store-pc=0x%llx\n", (unsigned long long)pc);
+                fflush(stdout);
+                diff_announced = 1;
+            }
+
+            diff_ensure_compiled(v_tb_ptr);
+            struct wasmContext pre_ctx = ctx;
+            diff_snapshot_pre();
+
+            /* Run and retain the authoritative TCI result first. */
+            uint32_t res;
+            int tci_iters = 0;
+            do {
+                res = tcg_qemu_tb_exec_tci(env);
+                tci_iters++;
+            } while ((uint32_t)ctx.tb_ptr != 0);
+            diff_tci_iters = tci_iters;
+            struct wasmContext tci_ctx = ctx;
+            memcpy(diff_env_t, env, sizeof(CPUArchState));
+            memcpy(diff_neg_t, (uint8_t *)env - DIFF_NEG, DIFF_NEG);
+            cpu_physical_memory_read(DIFF_DRAM_BASE, diff_dram_t, DIFF_DRAM_SIZE);
+
+            /* Rewind guest state, run compiled as a shadow, then restore TCI. */
+            bool restored = diff_restore_pre(env);
+            ctx = pre_ctx;
+            bool compiled_complete = restored && diff_run_compiled(v_tb_ptr);
+            cpu_physical_memory_write(DIFF_DRAM_BASE, diff_dram_t, DIFF_DRAM_SIZE);
+            memcpy(env, diff_env_t, sizeof(CPUArchState));
+            memcpy((uint8_t *)env - DIFF_NEG, diff_neg_t, DIFF_NEG);
+            ctx = tci_ctx;
+
+            /* Skip Async-Exit Races - If the IO thread forced a CPU exit
+             * (icount_decr.high == -1) around the TCI run, TCI took the TB-entry
+             * interrupt exit while the restored compiled shadow saw a cleared
+             * flag and ran fully. That divergence is a harness race, not codegen. */
+            uint32_t neg8_t_flag;
+            memcpy(&neg8_t_flag, diff_neg_t + DIFF_NEG - 8, 4);
+            if (neg8_t_flag & 0xffff0000) {
+                compiled_complete = false;
+            }
+
+            if (compiled_complete && !diff_reported) {
+                diff_compared++;
+                if (diff_compared == 1 || diff_compared % 10000 == 0) {
+                    printf("TCGDIFF compared=%u pc=0x%llx\n", diff_compared,
+                           (unsigned long long)pc);
+                    fflush(stdout);
+                }
+                diff_report(pc);
+            } else if (!diff_skipped_slow) {
+                printf("TCGDIFF skip-slow-path pc=0x%llx\n", (unsigned long long)pc);
+                fflush(stdout);
+                diff_skipped_slow = 1;
+            }
+            return res;
+        }
+    }
+
     while (true) {
         int tb_entry_ptr = (uint32_t)ctx.tb_ptr + export_vec_off;
         uint32_t res;
-        if (*(int32_t*)tb_entry_ptr > 0) {
+        bool diff_store_tci = wasm_diff_on() &&
+                              wasm_diff_lookup_store_tb(ctx.tb_ptr, NULL);
+        if (diff_store_tci) {
+            res = tcg_qemu_tb_exec_tci(env);
+        } else if (*(int32_t*)tb_entry_ptr > 0) {
             res = ((wasm_func_ptr)(*(uint32_t*)tb_entry_ptr))(&ctx);
         } else if (*(int32_t *)tb_entry_ptr == WASM_TCI_ONLY_ENTRY) {
             res = tcg_qemu_tb_exec_tci(env);

tcg/wasm32.h

@@ -43,6 +43,16 @@ void init_wasm32();
 
 void wasm_dump_module(const uint8_t *p, int len, const char *name);
 
+/* TCG compiled-vs-TCI differential (debug, opt-in via wasm_diff_enable()).
+ * wasm_cur_tb_pc is the guest PC of the TB about to run, set by cpu_tb_exec. */
+extern __thread uint64_t wasm_cur_tb_pc;
+void wasm_diff_enable(void);
+int wasm_diff_on(void);
+void wasm_diff_register_tb(const void *rw_ptr, const void *rx_ptr, uint64_t pc,
+                           bool has_store, bool safe_shadow);
+bool wasm_diff_lookup_store_tb(const void *tb_ptr, uint64_t *pc);
+bool wasm_diff_lookup_safe_tb(const void *tb_ptr);
+
 extern __thread bool wasm_tci_only_tb;
 
 #define INSTANTIATE_NUM 1500

tcg/wasm32/tcg-target.c.inc

@@ -3366,6 +3366,7 @@ static uint8_t tcg_tci_out_qemu_ldst(TCGContext *s, TCGOpcode opc, const TCGArg
  * module for static analysis. No effect on emitted code. */
 __thread bool wasm_tb_had_store;
 __thread bool wasm_tb_had_load;
+__thread bool wasm_tb_had_helper;
 static void tcg_out_qemu_ld(TCGContext *s, TCGOpcode opc, const TCGArg *args, bool is_64)
 {
     wasm_tb_had_load = true;
@@ -3508,6 +3509,7 @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
                          const TCGHelperInfo *info)
 {
+    wasm_tb_had_helper = true;
     tcg_tci_out_call(s, target, info);
     tcg_wasm_out_call(s, target, info);
 }
@@ -3787,6 +3789,9 @@ void tcg_out_init() {
     current_label_pos = 0;
     env_cached = false;
     wasm_tci_only_tb = false;
+    wasm_tb_had_store = false;
+    wasm_tb_had_load = false;
+    wasm_tb_had_helper = false;
 }
 
 /* Test if a constant matches the constraint. */