diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index 8163295f34..7cce2e3eb3 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -41,6 +41,9 @@ #include "tb-context.h" #include "internal-common.h" #include "internal-target.h" +#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) +#include "../../tcg/wasm32.h" +#endif /* -icount align implementation. */ @@ -173,6 +176,13 @@ uint32_t curr_cflags(CPUState *cpu) cflags |= CF_NO_GOTO_TB; } +#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) + /* Differential mode runs one TB per dispatch so wasm_cur_tb_pc stays exact. */ + if (wasm_diff_on()) { + cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR; + } +#endif + return cflags; } @@ -455,6 +465,9 @@ cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit) } qemu_thread_jit_execute(); +#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) + wasm_cur_tb_pc = itb->pc; +#endif ret = tcg_qemu_tb_exec(cpu_env(cpu), tb_ptr); cpu->neg.can_do_io = true; qemu_plugin_disable_mem_helpers(cpu); diff --git a/tcg/tcg.c b/tcg/tcg.c index fc8546de0f..df7c175423 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -6607,6 +6607,9 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) #endif #if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) + wasm_diff_register_tb(tb->tc.ptr, tcg_splitwx_to_rx(tb->tc.ptr), pc_start, + wasm_tci_only_tb, + wasm_tci_only_tb && !wasm_tb_had_helper); if (wasm_tci_only_tb) { for (int i = 0; i < get_core_nums(); i++) { export_vec_base[i] = WASM_TCI_ONLY_ENTRY; diff --git a/tcg/wasm32.c b/tcg/wasm32.c index ab88ac611c..7f502a5a87 100644 --- a/tcg/wasm32.c +++ b/tcg/wasm32.c @@ -22,7 +22,9 @@ #if !defined(CONFIG_TCG_INTERPRETER) && defined(EMSCRIPTEN) #include "qemu/osdep.h" +#include "cpu.h" #include "exec/cpu_ldst.h" +#include "exec/cpu-common.h" #include "tcg/tcg-op.h" #include "tcg/tcg-ldst.h" #include @@ -590,7 +592,7 @@ __thread tcg_target_ulong regs[TCG_TARGET_NB_REGS]; static inline uintptr_t tcg_qemu_tb_exec_tci(CPUArchState *env) { - uint32_t *tb_ptr = (uint8_t*)ctx.tb_ptr + *(uint32_t*)ctx.tb_ptr; + uint32_t *tb_ptr = (uint8_t *)ctx.tb_ptr + *(uint32_t *)ctx.tb_ptr; uint64_t *stack = ctx.stack; regs[TCG_AREG0] = (tcg_target_ulong)env; @@ -1170,7 +1172,7 @@ static inline uintptr_t tcg_qemu_tb_exec_tci(CPUArchState *env) // enter to wasm TB return 0; } - tb_ptr = (uint8_t*)tb_ptr + *(uint32_t*)tb_ptr; + tb_ptr = (uint8_t *)tb_ptr + *(uint32_t *)tb_ptr; } break; @@ -1193,7 +1195,7 @@ static inline uintptr_t tcg_qemu_tb_exec_tci(CPUArchState *env) // enter to wasm TB return 0; } - tb_ptr = (uint8_t*)tb_ptr + *(uint32_t*)tb_ptr; + tb_ptr = (uint8_t *)tb_ptr + *(uint32_t *)tb_ptr; break; @@ -1258,16 +1260,302 @@ static inline uintptr_t tcg_qemu_tb_exec_tci(CPUArchState *env) } } +/* + * Compiled-vs-TCI differential (debug). + * + * TCI stays the live/authoritative path (never crashes). For a TB whose guest PC + * is in the inflate window we additionally run the compiled blob as a shadow + * against a DRAM + register snapshot, then compare the resulting registers and + * DRAM against the authoritative TCI run. The first mismatch pinpoints the TB + * (and the first divergent guest register / DRAM word) whose compiled codegen is + * wrong — the corruption only manifests through a store, so a divergent DRAM word + * is the miscompiled write. Opt-in via wasm_diff_enable(); no effect otherwise. + */ +#define DIFF_DRAM_BASE 0x3fc80000u +#define DIFF_DRAM_SIZE 0x60000u +#define DIFF_PC_LO 0x42000000ull +#define DIFF_PC_HI 0x421a0000ull + +__thread uint64_t wasm_cur_tb_pc; +static int wasm_diff_enabled; +static GHashTable *diff_store_tbs; +static GHashTable *diff_safe_tbs; +static __thread int diff_reported; +static __thread int diff_announced; +static __thread int diff_started; +static __thread int diff_skipped_slow; +static __thread uint32_t diff_compared; +static __thread uint8_t *diff_dram_pre, *diff_dram_c, *diff_dram_t; +static __thread uint8_t diff_env_pre[sizeof(CPUArchState)]; +static __thread uint8_t diff_env_c[sizeof(CPUArchState)]; +static __thread uint8_t diff_env_t[sizeof(CPUArchState)]; + +/* + * TB Entry Reads CPUNegativeOffsetState - The exit/interrupt check at TB start + * loads env-8 (icount_decr/exit flag), which lives before CPUArchState. The + * shadow must see the same pre-state here or it wrongly takes the early-exit + * path, so snapshot a window immediately below env too. + */ +#define DIFF_NEG sizeof(CPUNegativeOffsetState) +static __thread uint8_t diff_neg_pre[DIFF_NEG]; +static __thread uint8_t diff_neg_t[DIFF_NEG]; +static __thread uint8_t *diff_tlb_pre[NB_MMU_MODES]; +static __thread uint8_t *diff_fulltlb_pre[NB_MMU_MODES]; +static __thread size_t diff_tlb_size[NB_MMU_MODES]; +static __thread size_t diff_fulltlb_size[NB_MMU_MODES]; +static __thread uint32_t diff_compiled_ret; +static __thread int diff_tci_iters; + +EMSCRIPTEN_KEEPALIVE void wasm_diff_enable(void) +{ + wasm_diff_enabled = 1; +} + +int wasm_diff_on(void) +{ + return wasm_diff_enabled; +} + +void wasm_diff_register_tb(const void *rw_ptr, const void *rx_ptr, uint64_t pc, + bool has_store, bool safe_shadow) +{ + if (!wasm_diff_on()) { + return; + } + if (!diff_store_tbs) { + diff_store_tbs = g_hash_table_new(g_direct_hash, g_direct_equal); + diff_safe_tbs = g_hash_table_new(g_direct_hash, g_direct_equal); + } + g_hash_table_remove(diff_store_tbs, rw_ptr); + g_hash_table_remove(diff_store_tbs, rx_ptr); + g_hash_table_remove(diff_safe_tbs, rw_ptr); + g_hash_table_remove(diff_safe_tbs, rx_ptr); + if (!has_store) { + return; + } + gpointer value = GUINT_TO_POINTER((uint32_t)pc); + g_hash_table_insert(diff_store_tbs, (void *)rw_ptr, value); + g_hash_table_insert(diff_store_tbs, (void *)rx_ptr, value); + if (safe_shadow) { + g_hash_table_insert(diff_safe_tbs, (void *)rw_ptr, GINT_TO_POINTER(1)); + g_hash_table_insert(diff_safe_tbs, (void *)rx_ptr, GINT_TO_POINTER(1)); + } +} + +bool wasm_diff_lookup_store_tb(const void *tb_ptr, uint64_t *pc) +{ + gpointer value = diff_store_tbs ? g_hash_table_lookup(diff_store_tbs, tb_ptr) : NULL; + if (!value) { + return false; + } + if (pc) { + *pc = GPOINTER_TO_UINT(value); + } + return true; +} + +bool wasm_diff_lookup_safe_tb(const void *tb_ptr) +{ + return diff_safe_tbs && g_hash_table_contains(diff_safe_tbs, tb_ptr); +} + +static void diff_report(uint64_t pc) +{ + uint32_t pre_ra, pre_sp, stack_m4 = 0, stack_0 = 0, stack_4 = 0; + memcpy(&pre_ra, diff_env_pre + 4, 4); + memcpy(&pre_sp, diff_env_pre + 8, 4); + if (pre_sp >= DIFF_DRAM_BASE + 4 && pre_sp + 8 <= DIFF_DRAM_BASE + DIFF_DRAM_SIZE) { + uint32_t off = pre_sp - DIFF_DRAM_BASE; + memcpy(&stack_m4, diff_dram_pre + off - 4, 4); + memcpy(&stack_0, diff_dram_pre + off, 4); + memcpy(&stack_4, diff_dram_pre + off + 4, 4); + } + /* Compare guest integer registers (env offset 0..127 = gpr[0..31]). */ + for (int i = 0; i < 32; i++) { + uint32_t c, t; + memcpy(&c, diff_env_c + i * 4, 4); + memcpy(&t, diff_env_t + i * 4, 4); + if (c != t) { + uint32_t neg8_pre, neg8_t; + memcpy(&neg8_pre, diff_neg_pre + DIFF_NEG - 8, 4); + memcpy(&neg8_t, diff_neg_t + DIFF_NEG - 8, 4); + printf("TCGDIFF pc=0x%llx gpr[%d] compiled=0x%08x tci=0x%08x " + "pre-ra=0x%08x pre-sp=0x%08x cret=0x%08x neg8_pre=0x%08x neg8_t=0x%08x tci_iters=%d\n", + (unsigned long long)pc, i, c, t, pre_ra, pre_sp, + diff_compiled_ret, neg8_pre, neg8_t, diff_tci_iters); + fflush(stdout); + diff_reported = 1; + return; + } + } + /* Compare DRAM word-by-word; first divergent word is the miscompiled store. */ + for (uint32_t off = 0; off < DIFF_DRAM_SIZE; off += 4) { + uint32_t c, t; + memcpy(&c, diff_dram_c + off, 4); + memcpy(&t, diff_dram_t + off, 4); + if (c != t) { + printf("TCGDIFF pc=0x%llx dram[0x%08x] compiled=0x%08x tci=0x%08x\n", + (unsigned long long)pc, DIFF_DRAM_BASE + off, c, t); + fflush(stdout); + diff_reported = 1; + return; + } + } +} + +static void diff_snapshot_pre(void) +{ + if (!diff_dram_pre) { + diff_dram_pre = malloc(DIFF_DRAM_SIZE); + diff_dram_c = malloc(DIFF_DRAM_SIZE); + diff_dram_t = malloc(DIFF_DRAM_SIZE); + } + CPUNegativeOffsetState *neg = (void *)((uint8_t *)ctx.env - DIFF_NEG); + memcpy(diff_env_pre, ctx.env, sizeof(CPUArchState)); + memcpy(diff_neg_pre, neg, DIFF_NEG); + for (int i = 0; i < NB_MMU_MODES; i++) { + size_t tlb_size = neg->tlb.f[i].mask + (1 << CPU_TLB_ENTRY_BITS); + size_t n = (neg->tlb.f[i].mask >> CPU_TLB_ENTRY_BITS) + 1; + size_t fulltlb_size = n * sizeof(CPUTLBEntryFull); + diff_tlb_pre[i] = g_realloc(diff_tlb_pre[i], tlb_size); + diff_fulltlb_pre[i] = g_realloc(diff_fulltlb_pre[i], fulltlb_size); + diff_tlb_size[i] = tlb_size; + diff_fulltlb_size[i] = fulltlb_size; + memcpy(diff_tlb_pre[i], neg->tlb.f[i].table, tlb_size); + memcpy(diff_fulltlb_pre[i], neg->tlb.d[i].fulltlb, fulltlb_size); + } + cpu_physical_memory_read(DIFF_DRAM_BASE, diff_dram_pre, DIFF_DRAM_SIZE); +} + +static bool diff_restore_pre(CPUArchState *env) +{ + CPUNegativeOffsetState *neg = (void *)((uint8_t *)env - DIFF_NEG); + CPUNegativeOffsetState *pre = (void *)diff_neg_pre; + for (int i = 0; i < NB_MMU_MODES; i++) { + if (neg->tlb.f[i].table != pre->tlb.f[i].table || + neg->tlb.d[i].fulltlb != pre->tlb.d[i].fulltlb) { + return false; + } + } + cpu_physical_memory_write(DIFF_DRAM_BASE, diff_dram_pre, DIFF_DRAM_SIZE); + memcpy(env, diff_env_pre, sizeof(CPUArchState)); + memcpy(neg, diff_neg_pre, DIFF_NEG); + for (int i = 0; i < NB_MMU_MODES; i++) { + memcpy(neg->tlb.f[i].table, diff_tlb_pre[i], diff_tlb_size[i]); + memcpy(neg->tlb.d[i].fulltlb, diff_fulltlb_pre[i], diff_fulltlb_size[i]); + } + return true; +} + +static void diff_ensure_compiled(const void *tb_ptr) +{ + ctx.tb_ptr = (uint32_t *)tb_ptr; + int slot_off = (uint32_t)ctx.tb_ptr + export_vec_off; + if (*(int32_t *)slot_off <= 0) { + instantiate_wasm(); + } +} + +static bool diff_run_compiled(const void *tb_ptr) +{ + ctx.tb_ptr = (uint32_t *)tb_ptr; + ctx.do_init = 1; + ctx.done_flag = 2; + int slot_off = (uint32_t)ctx.tb_ptr + export_vec_off; + g_assert(*(int32_t *)slot_off > 0); + diff_compiled_ret = ((wasm_func_ptr)(*(uint32_t *)slot_off))(&ctx); + memcpy(diff_env_c, ctx.env, sizeof(CPUArchState)); + cpu_physical_memory_read(DIFF_DRAM_BASE, diff_dram_c, DIFF_DRAM_SIZE); + return ctx.done_flag == 2 && (uint32_t)ctx.tb_ptr == 0; +} + uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, const void *v_tb_ptr) { ctx.env = env; ctx.tb_ptr = (uint32_t*)v_tb_ptr; ctx.do_init = 1; + + if (wasm_diff_on()) { + if (!diff_started) { + printf("TCGDIFF enabled first-pc=0x%llx tb-meta=0x%08x\n", + (unsigned long long)wasm_cur_tb_pc, *(uint32_t *)ctx.tb_ptr); + fflush(stdout); + diff_started = 1; + } + uint64_t pc = 0; + bool is_store_tb = wasm_diff_lookup_store_tb(v_tb_ptr, &pc); + int eligible = !diff_reported && is_store_tb && + wasm_diff_lookup_safe_tb(v_tb_ptr) && + pc >= DIFF_PC_LO && pc < DIFF_PC_HI; + if (eligible) { + if (!diff_announced) { + printf("TCGDIFF active first-store-pc=0x%llx\n", (unsigned long long)pc); + fflush(stdout); + diff_announced = 1; + } + + diff_ensure_compiled(v_tb_ptr); + struct wasmContext pre_ctx = ctx; + diff_snapshot_pre(); + + /* Run and retain the authoritative TCI result first. */ + uint32_t res; + int tci_iters = 0; + do { + res = tcg_qemu_tb_exec_tci(env); + tci_iters++; + } while ((uint32_t)ctx.tb_ptr != 0); + diff_tci_iters = tci_iters; + struct wasmContext tci_ctx = ctx; + memcpy(diff_env_t, env, sizeof(CPUArchState)); + memcpy(diff_neg_t, (uint8_t *)env - DIFF_NEG, DIFF_NEG); + cpu_physical_memory_read(DIFF_DRAM_BASE, diff_dram_t, DIFF_DRAM_SIZE); + + /* Rewind guest state, run compiled as a shadow, then restore TCI. */ + bool restored = diff_restore_pre(env); + ctx = pre_ctx; + bool compiled_complete = restored && diff_run_compiled(v_tb_ptr); + cpu_physical_memory_write(DIFF_DRAM_BASE, diff_dram_t, DIFF_DRAM_SIZE); + memcpy(env, diff_env_t, sizeof(CPUArchState)); + memcpy((uint8_t *)env - DIFF_NEG, diff_neg_t, DIFF_NEG); + ctx = tci_ctx; + + /* Skip Async-Exit Races - If the IO thread forced a CPU exit + * (icount_decr.high == -1) around the TCI run, TCI took the TB-entry + * interrupt exit while the restored compiled shadow saw a cleared + * flag and ran fully. That divergence is a harness race, not codegen. */ + uint32_t neg8_t_flag; + memcpy(&neg8_t_flag, diff_neg_t + DIFF_NEG - 8, 4); + if (neg8_t_flag & 0xffff0000) { + compiled_complete = false; + } + + if (compiled_complete && !diff_reported) { + diff_compared++; + if (diff_compared == 1 || diff_compared % 10000 == 0) { + printf("TCGDIFF compared=%u pc=0x%llx\n", diff_compared, + (unsigned long long)pc); + fflush(stdout); + } + diff_report(pc); + } else if (!diff_skipped_slow) { + printf("TCGDIFF skip-slow-path pc=0x%llx\n", (unsigned long long)pc); + fflush(stdout); + diff_skipped_slow = 1; + } + return res; + } + } + while (true) { int tb_entry_ptr = (uint32_t)ctx.tb_ptr + export_vec_off; uint32_t res; - if (*(int32_t*)tb_entry_ptr > 0) { + bool diff_store_tci = wasm_diff_on() && + wasm_diff_lookup_store_tb(ctx.tb_ptr, NULL); + if (diff_store_tci) { + res = tcg_qemu_tb_exec_tci(env); + } else if (*(int32_t*)tb_entry_ptr > 0) { res = ((wasm_func_ptr)(*(uint32_t*)tb_entry_ptr))(&ctx); } else if (*(int32_t *)tb_entry_ptr == WASM_TCI_ONLY_ENTRY) { res = tcg_qemu_tb_exec_tci(env); diff --git a/tcg/wasm32.h b/tcg/wasm32.h index a4384fc267..4b31459294 100644 --- a/tcg/wasm32.h +++ b/tcg/wasm32.h @@ -43,6 +43,16 @@ void init_wasm32(); void wasm_dump_module(const uint8_t *p, int len, const char *name); +/* TCG compiled-vs-TCI differential (debug, opt-in via wasm_diff_enable()). + * wasm_cur_tb_pc is the guest PC of the TB about to run, set by cpu_tb_exec. */ +extern __thread uint64_t wasm_cur_tb_pc; +void wasm_diff_enable(void); +int wasm_diff_on(void); +void wasm_diff_register_tb(const void *rw_ptr, const void *rx_ptr, uint64_t pc, + bool has_store, bool safe_shadow); +bool wasm_diff_lookup_store_tb(const void *tb_ptr, uint64_t *pc); +bool wasm_diff_lookup_safe_tb(const void *tb_ptr); + extern __thread bool wasm_tci_only_tb; #define INSTANTIATE_NUM 1500 diff --git a/tcg/wasm32/tcg-target.c.inc b/tcg/wasm32/tcg-target.c.inc index 414a34054c..f7bac72f6e 100644 --- a/tcg/wasm32/tcg-target.c.inc +++ b/tcg/wasm32/tcg-target.c.inc @@ -3366,6 +3366,7 @@ static uint8_t tcg_tci_out_qemu_ldst(TCGContext *s, TCGOpcode opc, const TCGArg * module for static analysis. No effect on emitted code. */ __thread bool wasm_tb_had_store; __thread bool wasm_tb_had_load; +__thread bool wasm_tb_had_helper; static void tcg_out_qemu_ld(TCGContext *s, TCGOpcode opc, const TCGArg *args, bool is_64) { wasm_tb_had_load = true; @@ -3508,6 +3509,7 @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, const TCGHelperInfo *info) { + wasm_tb_had_helper = true; tcg_tci_out_call(s, target, info); tcg_wasm_out_call(s, target, info); } @@ -3787,6 +3789,9 @@ void tcg_out_init() { current_label_pos = 0; env_cached = false; wasm_tci_only_tb = false; + wasm_tb_had_store = false; + wasm_tb_had_load = false; + wasm_tb_had_helper = false; } /* Test if a constant matches the constraint. */