diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index b76a4eac4e..a673eb2eb7 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -225,7 +225,7 @@ static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast, rate = desc->window_max_entries * 100 / old_size; if (rate > 70) { - new_size = MIN(old_size << 1, 1 << CPU_TLB_DYN_MAX_BITS); + new_size = MIN(old_size << 1, (uint64_t)1 << CPU_TLB_DYN_MAX_BITS); } else if (rate < 30 && window_expired) { size_t ceil = pow2ceil(desc->window_max_entries); size_t expected_rate = desc->window_max_entries * 100 / ceil; diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c index 440cb211d8..575be0e359 100644 --- a/accel/tcg/tb-maint.c +++ b/accel/tcg/tb-maint.c @@ -879,10 +879,18 @@ static inline void tb_jmp_unlink(TranslationBlock *dest) qemu_spin_unlock(&dest->jmp_lock); } +#if !defined(CONFIG_TCG_INTERPRETER) && defined(EMSCRIPTEN) +#include "../../tcg/wasm32.h" +#endif + static void tb_jmp_cache_inval_tb(TranslationBlock *tb) { CPUState *cpu; +#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) + remove_tb(tb->tc.ptr); +#endif + if (tb_cflags(tb) & CF_PCREL) { /* A TB may be at any virtual address */ CPU_FOREACH(cpu) { diff --git a/accel/tcg/tcg-accel-ops-mttcg.c b/accel/tcg/tcg-accel-ops-mttcg.c index 49814ec4af..0715610d37 100644 --- a/accel/tcg/tcg-accel-ops-mttcg.c +++ b/accel/tcg/tcg-accel-ops-mttcg.c @@ -61,12 +61,18 @@ static void mttcg_force_rcu(Notifier *notify, void *data) * variable current_cpu can be used deep in the code to find the * current CPUState for a given thread. */ - +#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) +#include "../../tcg/wasm32.h" +#endif static void *mttcg_cpu_thread_fn(void *arg) { MttcgForceRcuNotifier force_rcu; CPUState *cpu = arg; +#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) + init_wasm32(); +#endif + assert(tcg_enabled()); g_assert(!icount_enabled()); diff --git a/include/tcg/helper-info.h b/include/tcg/helper-info.h index 909fe73afa..c5064473ac 100644 --- a/include/tcg/helper-info.h +++ b/include/tcg/helper-info.h @@ -9,7 +9,7 @@ #ifndef TCG_HELPER_INFO_H #define TCG_HELPER_INFO_H -#ifdef CONFIG_TCG_INTERPRETER +#if defined(CONFIG_TCG_INTERPRETER) || defined(EMSCRIPTEN) #include #endif #include "tcg-target-reg-bits.h" @@ -48,11 +48,14 @@ struct TCGHelperInfo { const char *name; /* Used with g_once_init_enter. */ -#ifdef CONFIG_TCG_INTERPRETER +#if defined(CONFIG_TCG_INTERPRETER) && !defined(EMSCRIPTEN) ffi_cif *cif; #else uintptr_t init; #endif +#if defined(EMSCRIPTEN) + ffi_cif *cif; +#endif unsigned typemask : 32; unsigned flags : 8; diff --git a/include/tcg/tcg-opc.h b/include/tcg/tcg-opc.h index 546eb49c11..834757c3a4 100644 --- a/include/tcg/tcg-opc.h +++ b/include/tcg/tcg-opc.h @@ -305,7 +305,7 @@ DEF(last_generic, 0, 0, 0, TCG_OPF_NOT_PRESENT) #include "tcg-target.opc.h" #endif -#ifdef TCG_TARGET_INTERPRETER +#if defined(TCG_TARGET_INTERPRETER) || defined(EMSCRIPTEN) /* These opcodes are only for use between the tci generator and interpreter. */ DEF(tci_movi, 1, 0, 1, TCG_OPF_NOT_PRESENT) DEF(tci_movl, 1, 0, 1, TCG_OPF_NOT_PRESENT) diff --git a/target/arm/helper.c b/target/arm/helper.c index fcb13fe87e..de7cd7d3f8 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -219,7 +219,7 @@ static void count_cpreg(gpointer key, gpointer opaque) } } -static gint cpreg_key_compare(gconstpointer a, gconstpointer b) +static gint cpreg_key_compare(gconstpointer a, gconstpointer b, void* d) { uint64_t aidx = cpreg_to_kvm_id((uintptr_t)a); uint64_t bidx = cpreg_to_kvm_id((uintptr_t)b); @@ -243,7 +243,7 @@ void init_cpreg_list(ARMCPU *cpu) int arraylen; keys = g_hash_table_get_keys(cpu->cp_regs); - keys = g_list_sort(keys, cpreg_key_compare); + keys = g_list_sort(keys, (GCompareFunc)cpreg_key_compare); cpu->cpreg_array_len = 0; diff --git a/tcg/meson.build b/tcg/meson.build index 7eb99c807e..0894a577c2 100644 --- a/tcg/meson.build +++ b/tcg/meson.build @@ -22,6 +22,11 @@ if get_option('tcg_interpreter') method: 'pkg-config') tcg_ss.add(libffi) tcg_ss.add(files('tci.c')) +elif cpu == 'wasm32' + libffi = dependency('libffi', version: '>=3.0', required: true, + method: 'pkg-config') + specific_ss.add(libffi) + specific_ss.add(files('wasm32.c')) endif tcg_ss.add(when: libdw, if_true: files('debuginfo.c')) @@ -31,10 +36,6 @@ endif tcg_ss = tcg_ss.apply({}) -if cpu == 'wasm32' - specific_ss.add(files('wasm32.c')) -endif - libtcg_user = static_library('tcg_user', tcg_ss.sources() + genh, dependencies: tcg_ss.dependencies(), diff --git a/tcg/tcg.c b/tcg/tcg.c index d470dfc321..214e66e9d6 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -429,6 +429,35 @@ tlb_mask_table_ofs(TCGContext *s, int which) #if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) +#define SUB_BUF_MAX 100000 +__thread uint8_t sub_buf[SUB_BUF_MAX]; +__thread uint8_t *sub_buf_ptr; + +static inline void tcg_sub_out8(TCGContext *s, uint8_t v) +{ + *sub_buf_ptr++ = v; + if ((sub_buf_ptr - sub_buf) > SUB_BUF_MAX) { + printf("buffer too small"); fflush(stdout); + exit(1); + } +} + +static inline void tcg_sub_out32(TCGContext *s, uint32_t v) +{ + memcpy(sub_buf_ptr, &v, sizeof(v)); + sub_buf_ptr += 4; +} + +static inline void* cur_sub_buf_ptr() +{ + return sub_buf_ptr; +} + +static inline int cur_sub_buf_off_rel() +{ + return (int)sub_buf_ptr - (int)sub_buf; +} + #define LABEL_MAX 200 struct label_placeholder { @@ -525,12 +554,12 @@ static void wasm_add_label_context(TCGContext *s, int label, int block) label_to_block[label] = block; } -static void wasm_add_label_block_ptr_placeholder(TCGContext *s, int label) +static void wasm_add_label_block_ptr_placeholder(int label, uintptr_t code_ptr) { int i = block_ptr_placeholder_idx_pos++; tcg_debug_assert(i <= LABEL_MAX); block_ptr_placeholder[i].label = label; - block_ptr_placeholder[i].ptr = (uintptr_t)s->code_ptr;; + block_ptr_placeholder[i].ptr = code_ptr; } #endif @@ -869,7 +898,7 @@ static const TCGTargetOpDef constraint_sets[] = { #include "tcg-target.c.inc" -#ifndef CONFIG_TCG_INTERPRETER +#if !defined(CONFIG_TCG_INTERPRETER) && !defined(EMSCRIPTEN) /* Validate CPUTLBDescFast placement. */ QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) - sizeof(CPUNegativeOffsetState)) @@ -1052,7 +1081,7 @@ static TCGHelperInfo info_helper_st128_mmu = { | dh_typemask(ptr, 5) /* uintptr_t ra */ }; -#if defined(CONFIG_TCG_INTERPRETER) +#if defined(EMSCRIPTEN) || defined(CONFIG_TCG_INTERPRETER) static ffi_type *typecode_to_ffi(int argmask) { /* @@ -6110,11 +6139,11 @@ static const uint8_t mod_header_d[] = { 0x41, 0x00, // i32.const 0 0x36, 0x00, DO_INIT_OFF, // i32.store do_init 0x42, 0x00, // i64.const 0 - 0x24, 24, // global.set $block_ptr + 0x24, 16, // global.set $block_ptr 0x0b, // end 0x03, 0x40, // loop - 0x23, 24, // global.get $block_ptr + 0x23, 16, // global.get $block_ptr 0x50, // i64.eqz 0x04, 0x40, // if }; @@ -6445,6 +6474,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) s->data_gen_ptr = NULL; #if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) + sub_buf_ptr = sub_buf; tcg_out_init(); num_helper_funcs = 0; wasm_block_idx = 0; @@ -6453,9 +6483,19 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) memset(label_to_block, -1, LABEL_MAX); memset(target_helper_funcs, -1, WASM_NUM_HELPER_FUNCS_MAX); - s->code_ptr += 4; // placeholder for export vector offset + uint32_t *tci_code_off = (uint32_t*)s->code_ptr; + s->code_ptr += 4; + uint32_t *size_base = (uint32_t*)s->code_ptr; + s->code_ptr += 4; + uint32_t *export_vec_base = (uint32_t*)s->code_ptr; + int export_size = get_core_nums() * 4; + memset(s->code_ptr, 0, export_size); + s->code_ptr += export_size; + *size_base = export_size; + uint8_t *code_begin = s->code_ptr; s->code_ptr += 4; // placeholder for size + *tci_code_off = s->code_ptr - s->code_buf; #endif #ifdef TCG_TARGET_NEED_LDST_LABELS @@ -6567,10 +6607,10 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) #endif #if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) - tcg_out8(s, 0x0b); //end if - tcg_out8(s, 0x0b); //end loop - tcg_out8(s, 0x0); // unreachable - tcg_out8(s, 0x0b); //end func + tcg_sub_out8(s, 0x0b); //end if + tcg_sub_out8(s, 0x0b); //end loop + tcg_sub_out8(s, 0x0); // unreachable + tcg_sub_out8(s, 0x0b); //end func // fill blocks for (int i = 0; i < block_ptr_placeholder_idx_pos; i++) { @@ -6578,12 +6618,16 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) int ph = block_ptr_placeholder[i].ptr; int blk = label_to_block[label]; tcg_debug_assert(blk >= 0); + *(uint8_t*)ph = 0x80; fill_uint32_leb128(ph, blk); } int code_size = (uint32_t)((uintptr_t)s->code_ptr - (uintptr_t)code_begin - 4); *(uint32_t *)code_begin = code_size; + int sub_buf_len = sub_buf_ptr - sub_buf; + int wasm_body_size = sub_buf_len; + // write header uint8_t *wasm_blob_ptr = s->code_ptr; uint8_t *wasm_blob_ptr_base = s->code_ptr; @@ -6611,28 +6655,27 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) uint8_t *header_d_ptr = wasm_blob_ptr; memcpy(wasm_blob_ptr, mod_header_d, sizeof(mod_header_d)); wasm_blob_ptr += sizeof(mod_header_d); - write_wasm_code_size(s, header_d_ptr, code_size, 1); - - // write header size - *(uint32_t *)wasm_blob_ptr_base = wasm_blob_ptr - wasm_blob_ptr_base - 4; + write_wasm_code_size(s, header_d_ptr, wasm_body_size, 1); s->code_ptr = wasm_blob_ptr; + // write body + memcpy(s->code_ptr, sub_buf, sub_buf_len); + s->code_ptr += sub_buf_len; + + // write blob size + if (sub_buf_len > SUB_BUF_MAX) { + printf("sub too large sub_buf_len: %d\n", sub_buf_len); fflush(stdout); + exit(1); + } + *(uint32_t *)wasm_blob_ptr_base = s->code_ptr - wasm_blob_ptr_base - 4; + // record importing helper functions - uint32_t *size_base = (uint32_t*)s->code_ptr; + size_base = (uint32_t*)s->code_ptr; s->code_ptr += 4; memcpy(s->code_ptr, target_helper_funcs, num_helper_funcs * 4); s->code_ptr += num_helper_funcs * 4; *size_base = num_helper_funcs * 4; - // init exporting functions with zeros - size_base = (uint32_t*)s->code_ptr; - s->code_ptr += 4; - *(uint32_t*)(s->code_buf) = (uint32_t)(s->code_ptr - s->code_buf); - int export_size = get_core_nums() * 4; - memset(s->code_ptr, 0, export_size); - s->code_ptr += export_size; - *size_base = export_size; - if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { return -1; } diff --git a/tcg/wasm32.c b/tcg/wasm32.c index 45136035ee..dfcbf1493b 100644 --- a/tcg/wasm32.c +++ b/tcg/wasm32.c @@ -1,3 +1,24 @@ +/* + * Tiny Code Generator for QEMU + * + * Wasm integration + ported TCI interpreter from tci.c + * + * Copyright (c) 2009, 2011, 2016 Stefan Weil + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + #if !defined(CONFIG_TCG_INTERPRETER) && defined(EMSCRIPTEN) #include "qemu/osdep.h" @@ -17,95 +38,65 @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) return 0; //nop } -EM_JS(int, instantiate_wasm, (int cur_core_num, int all_cores_num, int wasm_body_begin, int wasm_body_size, int wasm_header_begin, int wasm_header_size, int import_vec_begin, int import_vec_size, int export_vec_begin, int export_vec_size, int *to_remove_ptr, int *to_remove_num), { +EM_JS(int, instantiate_wasm, (), { const memory_v = new DataView(HEAP8.buffer); - var wasm = new Uint8Array(wasm_header_size + wasm_body_size); - wasm.set(HEAP8.subarray(wasm_header_begin, wasm_header_begin + wasm_header_size)); - wasm.set(HEAP8.subarray(wasm_body_begin, wasm_body_begin + wasm_body_size), wasm_header_size); - const mod = new WebAssembly.Module(wasm); + + const tb_ptr = memory_v.getInt32(Module.__wasm32_tb.tb_ptr_ptr, true); + const export_vec_size = memory_v.getInt32(tb_ptr + 4, true); + const export_vec_begin = tb_ptr + 4 + 4; + + const tmp_body_size = memory_v.getInt32(export_vec_begin + export_vec_size, true); + const tmp_body_begin = export_vec_begin + export_vec_size + 4; + const wasm_size = memory_v.getInt32(tmp_body_begin + tmp_body_size, true); + const wasm_begin = tmp_body_begin + tmp_body_size + 4; + const import_vec_size = memory_v.getInt32(wasm_begin + wasm_size, true); + const import_vec_begin = wasm_begin + wasm_size + 4; + + const wasm = HEAP8.subarray(wasm_begin, wasm_begin + wasm_size); var helper = {}; for (var i = 0; i < import_vec_size / 4; i++) { helper[i] = wasmTable.get(memory_v.getInt32(import_vec_begin + i * 4, true)); } + const mod = new WebAssembly.Module(wasm); const inst = new WebAssembly.Instance(mod, { - "env": { - "buffer": wasmMemory, - }, - "helper": helper, - }); - var ptr = export_vec_begin + 4 * cur_core_num; + "env": { + "buffer": wasmMemory, + }, + "helper": helper, + }); + var ptr = export_vec_begin + 4 * Module.__wasm32_tb.cur_core_num; const fidx = addFunction(inst.exports.start, 'ii'); memory_v.setUint32(ptr, fidx, true); - ptr += 4 * all_cores_num; - const remove_n = memory_v.getInt32(to_remove_num, true); + const remove_n = memory_v.getInt32(Module.__wasm32_tb.to_remove_instance_idx_ptr, true); if (remove_n > 500) { - for (var i = 0; i < remove_n * 4; i += 4) { - removeFunction(memory_v.getInt32(to_remove_ptr + i, true)); - } - memory_v.setInt32(to_remove_num, 0, true); + for (var i = 0; i < remove_n * 4; i += 4) { + removeFunction(memory_v.getInt32(Module.__wasm32_tb.to_remove_instance_ptr + i, true)); + } + memory_v.setInt32(Module.__wasm32_tb.to_remove_instance_idx_ptr, 0, true); } - return fidx; + return 0; }); +__thread bool initdone = false; +__thread int cur_core_num = -1; +__thread int export_vec_off = -1; +__thread int all_cores_num = -1; +int cur_core_num_max = 0; + #define TO_REMOVE_INSTANCE_SIZE 50000 __thread static int to_remove_instance[TO_REMOVE_INSTANCE_SIZE]; __thread static int to_remove_instance_idx = 0; -#define ACTIVE_TBS_MAX 60000 -#define REMOVE_TBS_NUM 10000 -__thread static int active_tbs[ACTIVE_TBS_MAX]; -__thread static int active_tbs_begin = 0; -__thread static int active_tbs_end = 0; -__thread static int active_tbs_lim; -__thread static int active_tbs_rmv; - -static int active_tbs_len() -{ - if (active_tbs_begin == active_tbs_end) { - return 0; - } - if (active_tbs_begin < active_tbs_end) { - return active_tbs_end - active_tbs_begin; - } - if (active_tbs_begin > active_tbs_end) { - return (active_tbs_lim - active_tbs_begin) + active_tbs_end; - } - g_assert_not_reached(); -} - -static void prepare_wasm(void *tb_ptr, int cur_core_num, int all_cores_num) -{ - uint32_t code_size = *(uint32_t*)((uint32_t)tb_ptr + 4); - uint32_t code_begin = (uint32_t)tb_ptr + 4 + 4; - uint32_t header_size = *(uint32_t*)(code_begin + code_size); - uint32_t header_begin = code_begin + code_size + 4; - uint32_t import_vec_size = *(uint32_t*)(header_begin + header_size); - uint32_t import_vec_begin = header_begin + header_size + 4; - uint32_t export_vec_size = *(uint32_t*)(import_vec_begin + import_vec_size); - uint32_t export_vec_begin = import_vec_begin + import_vec_size + 4; - - if (active_tbs_len() == (active_tbs_lim-1)) { - int idx = active_tbs_begin; - for (int i = 0; i < active_tbs_rmv; i++) { - uint8_t *p = (uint8_t*)active_tbs[idx]; - int f = *(uint32_t*)((uint8_t*)p + *(uint32_t*)p + cur_core_num * 4); - *(uint32_t*)((uint8_t*)p + *(uint32_t*)p + cur_core_num * 4) = 0; - to_remove_instance[to_remove_instance_idx++] = f; - idx++; - if (idx >= active_tbs_lim) { - idx = 0; - } - } - active_tbs_begin = idx; - } - - int fidx = instantiate_wasm(cur_core_num, all_cores_num, code_begin, code_size, header_begin, header_size, import_vec_begin, import_vec_size, export_vec_begin, export_vec_size, to_remove_instance, &to_remove_instance_idx); - active_tbs[active_tbs_end++] = (int)tb_ptr; - if (active_tbs_end >= active_tbs_lim) { - active_tbs_end = 0; +void remove_tb(void *tb_ptr) { + int32_t *slot = (int32_t*)((uint8_t*)tb_ptr + export_vec_off); + int32_t f = *slot; + if (f <= 0) { + return; } + *slot = 0; + to_remove_instance[to_remove_instance_idx++] = f; } __thread struct wasmContext ctx = { @@ -129,40 +120,1107 @@ void set_unwinding_flag() typedef uint32_t (*wasm_func_ptr)(struct wasmContext*); -__thread bool initdone = false; -__thread int cur_core_num = -1; -__thread int all_cores_num = -1; -int cur_core_num_max = 0; - int get_core_nums() { return emscripten_num_logical_cores(); } -extern unsigned int tcg_max_ctxs; +EM_JS(void, init_wasm32_js, (int tb_ptr_ptr, int cur_core_num, int to_remove_instance_ptr, int to_remove_instance_idx_ptr), { + Module.__wasm32_tb = { + tb_ptr_ptr: tb_ptr_ptr, + cur_core_num: cur_core_num, + to_remove_instance_ptr: to_remove_instance_ptr, + to_remove_instance_idx_ptr: to_remove_instance_idx_ptr, + }; +}); -uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, const void *v_tb_ptr) +void init_wasm32() { if (!initdone) { - active_tbs_lim = ACTIVE_TBS_MAX / tcg_max_ctxs; - active_tbs_rmv = REMOVE_TBS_NUM / tcg_max_ctxs; cur_core_num = qatomic_fetch_inc(&cur_core_num_max); + export_vec_off = 4 + 4 + cur_core_num * 4; all_cores_num = get_core_nums(); - ctx.stack = (uint64_t*)malloc((TCG_STATIC_CALL_ARGS_SIZE + TCG_STATIC_FRAME_SIZE) / sizeof(uint64_t)); - ctx.stack128 = (uint64_t*)malloc((TCG_STATIC_CALL_ARGS_SIZE + TCG_STATIC_FRAME_SIZE) / sizeof(uint64_t)); + ctx.stack = (uint64_t*)malloc(TCG_STATIC_CALL_ARGS_SIZE + TCG_STATIC_FRAME_SIZE); + ctx.stack128 = (uint64_t*)malloc(TCG_STATIC_CALL_ARGS_SIZE + TCG_STATIC_FRAME_SIZE); + ctx.tci_tb_ptr = (uint32_t*)&tci_tb_ptr; + init_wasm32_js((int)&ctx.tb_ptr, cur_core_num, (int)to_remove_instance, (int)&to_remove_instance_idx); initdone = true; } +} + +__thread uintptr_t tci_tb_ptr; + +static void tci_write_reg64(tcg_target_ulong *regs, uint32_t high_index, + uint32_t low_index, uint64_t value) +{ + regs[low_index] = (uint32_t)value; + regs[high_index] = value >> 32; +} + +/* Create a 64 bit value from two 32 bit values. */ +static uint64_t tci_uint64(uint32_t high, uint32_t low) +{ + return ((uint64_t)high << 32) + low; +} + +static void tci_args_ldst(uint32_t insn, TCGReg *r0, TCGReg *r1, MemOpIdx *m2, const void *tb_ptr, void **l0) +{ + int diff = sextract32(insn, 12, 20); + *l0 = diff ? (uint8_t *)tb_ptr + diff : NULL; + + uint64_t *data64 = (uint64_t*)*l0; + *r0 = (TCGReg)data64[0]; + *r1 = (TCGReg)data64[1]; + *m2 = (MemOpIdx)data64[2]; +} + +/* + * Load sets of arguments all at once. The naming convention is: + * tci_args_ + * where arguments is a sequence of + * + * b = immediate (bit position) + * c = condition (TCGCond) + * i = immediate (uint32_t) + * I = immediate (tcg_target_ulong) + * l = label or pointer + * m = immediate (MemOpIdx) + * n = immediate (call return length) + * r = register + * s = signed ldst offset + */ + +static void tci_args_l(uint32_t insn, const void *tb_ptr, void **l0) +{ + int diff = sextract32(insn, 12, 20); + *l0 = diff ? (uint8_t *)tb_ptr + diff : NULL; +} + +static void tci_args_r(uint32_t insn, TCGReg *r0) +{ + *r0 = extract32(insn, 8, 4); +} + +static void tci_args_nl(uint32_t insn, const void *tb_ptr, + uint8_t *n0, void **l1) +{ + *n0 = extract32(insn, 8, 4); + *l1 = sextract32(insn, 12, 20) + (void *)tb_ptr; +} + +static void tci_args_rl(uint32_t insn, const void *tb_ptr, + TCGReg *r0, void **l1) +{ + *r0 = extract32(insn, 8, 4); + *l1 = sextract32(insn, 12, 20) + (void *)tb_ptr; +} + +static void tci_args_rr(uint32_t insn, TCGReg *r0, TCGReg *r1) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); +} + +static void tci_args_ri(uint32_t insn, TCGReg *r0, tcg_target_ulong *i1) +{ + *r0 = extract32(insn, 8, 4); + *i1 = sextract32(insn, 12, 20); +} + +static void tci_args_rrr(uint32_t insn, TCGReg *r0, TCGReg *r1, TCGReg *r2) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); +} + +static void tci_args_rrs(uint32_t insn, TCGReg *r0, TCGReg *r1, int32_t *i2) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *i2 = sextract32(insn, 16, 16); +} + +static void tci_args_rrbb(uint32_t insn, TCGReg *r0, TCGReg *r1, + uint8_t *i2, uint8_t *i3) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *i2 = extract32(insn, 16, 6); + *i3 = extract32(insn, 22, 6); +} + +static void tci_args_rrrc(uint32_t insn, + TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGCond *c3) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *c3 = extract32(insn, 20, 4); +} + +static void tci_args_rrrbb(uint32_t insn, TCGReg *r0, TCGReg *r1, + TCGReg *r2, uint8_t *i3, uint8_t *i4) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *i3 = extract32(insn, 20, 6); + *i4 = extract32(insn, 26, 6); +} + +static void tci_args_rrrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, + TCGReg *r2, TCGReg *r3, TCGReg *r4) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *r3 = extract32(insn, 20, 4); + *r4 = extract32(insn, 24, 4); +} + +static void tci_args_rrrr(uint32_t insn, + TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGReg *r3) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *r3 = extract32(insn, 20, 4); +} + +static void tci_args_rrrrrc(uint32_t insn, TCGReg *r0, TCGReg *r1, + TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGCond *c5) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *r3 = extract32(insn, 20, 4); + *r4 = extract32(insn, 24, 4); + *c5 = extract32(insn, 28, 4); +} + +static void tci_args_rrrrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, + TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGReg *r5) +{ + *r0 = extract32(insn, 8, 4); + *r1 = extract32(insn, 12, 4); + *r2 = extract32(insn, 16, 4); + *r3 = extract32(insn, 20, 4); + *r4 = extract32(insn, 24, 4); + *r5 = extract32(insn, 28, 4); +} + +static bool tci_compare32(uint32_t u0, uint32_t u1, TCGCond condition) +{ + bool result = false; + int32_t i0 = u0; + int32_t i1 = u1; + switch (condition) { + case TCG_COND_EQ: + result = (u0 == u1); + break; + case TCG_COND_NE: + result = (u0 != u1); + break; + case TCG_COND_LT: + result = (i0 < i1); + break; + case TCG_COND_GE: + result = (i0 >= i1); + break; + case TCG_COND_LE: + result = (i0 <= i1); + break; + case TCG_COND_GT: + result = (i0 > i1); + break; + case TCG_COND_LTU: + result = (u0 < u1); + break; + case TCG_COND_GEU: + result = (u0 >= u1); + break; + case TCG_COND_LEU: + result = (u0 <= u1); + break; + case TCG_COND_GTU: + result = (u0 > u1); + break; + default: + g_assert_not_reached(); + } + return result; +} + +static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) +{ + bool result = false; + int64_t i0 = u0; + int64_t i1 = u1; + switch (condition) { + case TCG_COND_EQ: + result = (u0 == u1); + break; + case TCG_COND_NE: + result = (u0 != u1); + break; + case TCG_COND_LT: + result = (i0 < i1); + break; + case TCG_COND_GE: + result = (i0 >= i1); + break; + case TCG_COND_LE: + result = (i0 <= i1); + break; + case TCG_COND_GT: + result = (i0 > i1); + break; + case TCG_COND_LTU: + result = (u0 < u1); + break; + case TCG_COND_GEU: + result = (u0 >= u1); + break; + case TCG_COND_LEU: + result = (u0 <= u1); + break; + case TCG_COND_GTU: + result = (u0 > u1); + break; + default: + g_assert_not_reached(); + } + return result; +} + +static uint64_t tlb_load(CPUArchState *env, uint64_t taddr, MemOp mop, uint64_t* ptr, bool is_ld) +{ + uint64_t *data64 = (uint64_t*)ptr; + unsigned a_mask = (unsigned)data64[3]; + int mask_ofs = (int)data64[4]; + int8_t page_bits = (int8_t)data64[5]; + uint64_t page_mask = (uint64_t)data64[6]; + int table_ofs = (uint64_t)data64[7]; + + unsigned s_bits = mop & MO_SIZE; + unsigned s_mask = (1u << s_bits) - 1; + tcg_target_long compare_mask; + int add_off = offsetof(CPUTLBEntry, addend); + + uint64_t tmp0 = taddr >> (page_bits - CPU_TLB_ENTRY_BITS); + uint64_t tmp2 = *(uint64_t*)((uint8_t*)env + mask_ofs); + uint64_t tmp2_b = *(uint64_t*)((uint8_t*)env + table_ofs); + uint64_t tmp3 = (tmp0 & tmp2) + tmp2_b; + int off = off = is_ld ? offsetof(CPUTLBEntry, addr_read) + : offsetof(CPUTLBEntry, addr_write); + uint64_t target = *(uint64_t*)((uint8_t*)tmp3 + off); + uint64_t c_addr = taddr; + if (a_mask < s_mask) { + c_addr += s_mask - a_mask; + } + compare_mask = page_mask | a_mask; + c_addr &= compare_mask; + + if (c_addr == target) { + int32_t addend = *(uint32_t*)((uint8_t*)tmp3 + add_off); + uint64_t target_addr = taddr + addend; + return target_addr; + } + return 0; +} + +static uint64_t tci_qemu_ld(CPUArchState *env, uint64_t taddr, + MemOpIdx oi, const void *tb_ptr, uint64_t* ptr) +{ + MemOp mop = get_memop(oi); + uintptr_t ra = (uintptr_t)tb_ptr; + + uint64_t target_addr = tlb_load(env, taddr, mop, ptr, true); + if (target_addr != 0) { + switch (mop & MO_SSIZE) { + case MO_UB: + return *(uint8_t*)target_addr; + case MO_SB: + return *(int8_t*)target_addr; + case MO_UW: + return *(uint16_t*)target_addr; + case MO_SW: + return *(int16_t*)target_addr; + case MO_UL: + return *(uint32_t*)target_addr; + case MO_SL: + return *(int32_t*)target_addr; + case MO_UQ: + return *(uint64_t*)target_addr; + default: + g_assert_not_reached(); + } + } + + switch (mop & MO_SSIZE) { + case MO_UB: + return helper_ldub_mmu(env, taddr, oi, ra); + case MO_SB: + return helper_ldsb_mmu(env, taddr, oi, ra); + case MO_UW: + return helper_lduw_mmu(env, taddr, oi, ra); + case MO_SW: + return helper_ldsw_mmu(env, taddr, oi, ra); + case MO_UL: + return helper_ldul_mmu(env, taddr, oi, ra); + case MO_SL: + return helper_ldsl_mmu(env, taddr, oi, ra); + case MO_UQ: + return helper_ldq_mmu(env, taddr, oi, ra); + default: + g_assert_not_reached(); + } +} + +static void tci_qemu_st(CPUArchState *env, uint64_t taddr, uint64_t val, + MemOpIdx oi, const void *tb_ptr, uint64_t* ptr) +{ + MemOp mop = get_memop(oi); + uintptr_t ra = (uintptr_t)tb_ptr; + + uint64_t target_addr = tlb_load(env, taddr, mop, ptr, false); + if (target_addr != 0) { + switch (mop & MO_SIZE) { + case MO_UB: + *(uint8_t*)target_addr = (uint8_t)val; + break; + case MO_UW: + *(uint16_t*)target_addr = (uint16_t)val; + break; + case MO_UL: + *(uint32_t*)target_addr = (uint32_t)val; + break; + case MO_UQ: + *(uint64_t*)target_addr = (uint64_t)val; + break; + default: + g_assert_not_reached(); + } + return; + } + + switch (mop & MO_SIZE) { + case MO_UB: + helper_stb_mmu(env, taddr, val, oi, ra); + break; + case MO_UW: + helper_stw_mmu(env, taddr, val, oi, ra); + break; + case MO_UL: + helper_stl_mmu(env, taddr, val, oi, ra); + break; + case MO_UQ: + helper_stq_mmu(env, taddr, val, oi, ra); + break; + default: + g_assert_not_reached(); + } +} + +#if TCG_TARGET_REG_BITS == 64 +# define CASE_32_64(x) \ + case glue(glue(INDEX_op_, x), _i64): \ + case glue(glue(INDEX_op_, x), _i32): +# define CASE_64(x) \ + case glue(glue(INDEX_op_, x), _i64): +#else +# define CASE_32_64(x) \ + case glue(glue(INDEX_op_, x), _i32): +# define CASE_64(x) +#endif + +__thread tcg_target_ulong regs[TCG_TARGET_NB_REGS]; + +static inline uintptr_t tcg_qemu_tb_exec_tci(CPUArchState *env) +{ + uint32_t *tb_ptr = (uint8_t*)ctx.tb_ptr + *(uint32_t*)ctx.tb_ptr; + uint64_t *stack = ctx.stack; + + regs[TCG_AREG0] = (tcg_target_ulong)env; + regs[TCG_REG_CALL_STACK] = (uintptr_t)stack; + + for (;;) { + uint32_t insn; + TCGOpcode opc; + TCGReg r0, r1, r2, r3, r4, r5; + tcg_target_ulong t1; + TCGCond condition; + uint8_t pos, len; + uint32_t tmp32; + uint64_t tmp64, taddr; + uint64_t T1, T2; + MemOpIdx oi; + int32_t ofs; + void *ptr; + + uint32_t *savep = tb_ptr; + insn = *tb_ptr++; + opc = extract32(insn, 0, 8); + + TCGOpDef *def = &tcg_op_defs[opc]; + + switch (opc) { + case INDEX_op_call: + { + void *call_slots[MAX_CALL_IARGS]; + ffi_cif *cif; + void *func; + unsigned i, s, n; + + tci_args_nl(insn, tb_ptr, &len, &ptr); + uint64_t *data64 = (uint64_t*)ptr; + func = (void*)data64[0]; + cif = (void*)data64[1]; + + int reg_iarg_base = 8; + if ((uint32_t)func == (uint32_t)helper_lookup_tb_ptr) { + regs[TCG_REG_R0] = (uint32_t)helper_lookup_tb_ptr((CPUArchState *)regs[reg_iarg_base]); + break; + } + + int reg_idx = 0; + int reg_idx_end = 5; // NUM_OF_IARG_REGS + int stack_idx = 0; + n = cif->nargs; + for (i = s = 0; i < n; ++i) { + ffi_type *t = cif->arg_types[i]; + if (reg_idx < reg_idx_end) { + call_slots[i] = ®s[reg_iarg_base + reg_idx]; + reg_idx += DIV_ROUND_UP(t->size, 8); + } else { + call_slots[i] = &stack[stack_idx]; + stack_idx += DIV_ROUND_UP(t->size, 8); + } + } + + /* Helper functions may need to access the "return address" */ + tci_tb_ptr = (uintptr_t)tb_ptr; + ffi_call(cif, func, stack, call_slots); + } + + switch (len) { + case 0: /* void */ + break; + case 1: /* uint32_t */ + /* + * The result winds up "left-aligned" in the stack[0] slot. + * Note that libffi has an odd special case in that it will + * always widen an integral result to ffi_arg. + */ + if (sizeof(ffi_arg) == 8) { + regs[TCG_REG_R0] = (uint32_t)stack[0]; + } else { + regs[TCG_REG_R0] = *(uint32_t *)stack; + } + break; + case 2: /* uint64_t */ + /* + * For TCG_TARGET_REG_BITS == 32, the register pair + * must stay in host memory order. + */ + memcpy(®s[TCG_REG_R0], stack, 8); + break; + case 3: /* Int128 */ + memcpy(®s[TCG_REG_R0], stack, 16); + break; + default: + g_assert_not_reached(); + } + break; + + case INDEX_op_br: + tci_args_l(insn, tb_ptr, &ptr); + tb_ptr = ptr; + continue; + case INDEX_op_setcond_i32: + tci_args_rrrc(insn, &r0, &r1, &r2, &condition); + regs[r0] = tci_compare32(regs[r1], regs[r2], condition); + break; + case INDEX_op_movcond_i32: + tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); + tmp32 = tci_compare32(regs[r1], regs[r2], condition); + regs[r0] = regs[tmp32 ? r3 : r4]; + break; +#if TCG_TARGET_REG_BITS == 32 + case INDEX_op_setcond2_i32: + tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); + T1 = tci_uint64(regs[r2], regs[r1]); + T2 = tci_uint64(regs[r4], regs[r3]); + regs[r0] = tci_compare64(T1, T2, condition); + break; +#elif TCG_TARGET_REG_BITS == 64 + case INDEX_op_setcond_i64: + tci_args_rrrc(insn, &r0, &r1, &r2, &condition); + regs[r0] = tci_compare64(regs[r1], regs[r2], condition); + break; + case INDEX_op_movcond_i64: + tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); + tmp32 = tci_compare64(regs[r1], regs[r2], condition); + regs[r0] = regs[tmp32 ? r3 : r4]; + break; +#endif + CASE_32_64(mov) + tci_args_rr(insn, &r0, &r1); + regs[r0] = regs[r1]; + break; + case INDEX_op_tci_movi: + tci_args_ri(insn, &r0, &t1); + regs[r0] = t1; + break; + case INDEX_op_tci_movl: + tci_args_rl(insn, tb_ptr, &r0, &ptr); + regs[r0] = *(tcg_target_ulong *)ptr; + break; + + /* Load/store operations (32 bit). */ + + CASE_32_64(ld8u) + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + regs[r0] = *(uint8_t *)ptr; + break; + CASE_32_64(ld8s) + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + regs[r0] = *(int8_t *)ptr; + break; + CASE_32_64(ld16u) + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + regs[r0] = *(uint16_t *)ptr; + break; + CASE_32_64(ld16s) + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + regs[r0] = *(int16_t *)ptr; + break; + case INDEX_op_ld_i32: + CASE_64(ld32u) + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + regs[r0] = *(uint32_t *)ptr; + break; + CASE_32_64(st8) + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + *(uint8_t *)ptr = regs[r0]; + break; + CASE_32_64(st16) + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + *(uint16_t *)ptr = regs[r0]; + break; + case INDEX_op_st_i32: + CASE_64(st32) + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + *(uint32_t *)ptr = regs[r0]; + break; + + /* Arithmetic operations (mixed 32/64 bit). */ + + CASE_32_64(add) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] + regs[r2]; + break; + CASE_32_64(sub) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] - regs[r2]; + break; + CASE_32_64(mul) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] * regs[r2]; + break; + CASE_32_64(and) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] & regs[r2]; + break; + CASE_32_64(or) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] | regs[r2]; + break; + CASE_32_64(xor) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] ^ regs[r2]; + break; +#if TCG_TARGET_HAS_andc_i32 || TCG_TARGET_HAS_andc_i64 + CASE_32_64(andc) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] & ~regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_orc_i32 || TCG_TARGET_HAS_orc_i64 + CASE_32_64(orc) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] | ~regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_eqv_i32 || TCG_TARGET_HAS_eqv_i64 + CASE_32_64(eqv) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = ~(regs[r1] ^ regs[r2]); + break; +#endif +#if TCG_TARGET_HAS_nand_i32 || TCG_TARGET_HAS_nand_i64 + CASE_32_64(nand) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = ~(regs[r1] & regs[r2]); + break; +#endif +#if TCG_TARGET_HAS_nor_i32 || TCG_TARGET_HAS_nor_i64 + CASE_32_64(nor) + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = ~(regs[r1] | regs[r2]); + break; +#endif + + /* Arithmetic operations (32 bit). */ + + case INDEX_op_div_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (int32_t)regs[r1] / (int32_t)regs[r2]; + break; + case INDEX_op_divu_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (uint32_t)regs[r1] / (uint32_t)regs[r2]; + break; + case INDEX_op_rem_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (int32_t)regs[r1] % (int32_t)regs[r2]; + break; + case INDEX_op_remu_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (uint32_t)regs[r1] % (uint32_t)regs[r2]; + break; +#if TCG_TARGET_HAS_clz_i32 + case INDEX_op_clz_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + tmp32 = regs[r1]; + regs[r0] = tmp32 ? clz32(tmp32) : regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_ctz_i32 + case INDEX_op_ctz_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + tmp32 = regs[r1]; + regs[r0] = tmp32 ? ctz32(tmp32) : regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_ctpop_i32 + case INDEX_op_ctpop_i32: + tci_args_rr(insn, &r0, &r1); + regs[r0] = ctpop32(regs[r1]); + break; +#endif + + /* Shift/rotate operations (32 bit). */ + + case INDEX_op_shl_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (uint32_t)regs[r1] << (regs[r2] & 31); + break; + case INDEX_op_shr_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (uint32_t)regs[r1] >> (regs[r2] & 31); + break; + case INDEX_op_sar_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (int32_t)regs[r1] >> (regs[r2] & 31); + break; +#if TCG_TARGET_HAS_rot_i32 + case INDEX_op_rotl_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = rol32(regs[r1], regs[r2] & 31); + break; + case INDEX_op_rotr_i32: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = ror32(regs[r1], regs[r2] & 31); + break; +#endif +#if TCG_TARGET_HAS_deposit_i32 + case INDEX_op_deposit_i32: + tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); + regs[r0] = deposit32(regs[r1], pos, len, regs[r2]); + break; +#endif +#if TCG_TARGET_HAS_extract_i32 + case INDEX_op_extract_i32: + tci_args_rrbb(insn, &r0, &r1, &pos, &len); + regs[r0] = extract32(regs[r1], pos, len); + break; +#endif +#if TCG_TARGET_HAS_sextract_i32 + case INDEX_op_sextract_i32: + tci_args_rrbb(insn, &r0, &r1, &pos, &len); + regs[r0] = sextract32(regs[r1], pos, len); + break; +#endif + case INDEX_op_brcond_i32: + tci_args_rl(insn, tb_ptr, &r0, &ptr); + if ((uint32_t)regs[r0]) { + tb_ptr = ptr; + } + break; +#if TCG_TARGET_REG_BITS == 32 || TCG_TARGET_HAS_add2_i32 + case INDEX_op_add2_i32: + tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); + T1 = tci_uint64(regs[r3], regs[r2]); + T2 = tci_uint64(regs[r5], regs[r4]); + tci_write_reg64(regs, r1, r0, T1 + T2); + break; +#endif +#if TCG_TARGET_REG_BITS == 32 || TCG_TARGET_HAS_sub2_i32 + case INDEX_op_sub2_i32: + tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); + T1 = tci_uint64(regs[r3], regs[r2]); + T2 = tci_uint64(regs[r5], regs[r4]); + tci_write_reg64(regs, r1, r0, T1 - T2); + break; +#endif +#if TCG_TARGET_HAS_mulu2_i32 + case INDEX_op_mulu2_i32: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + tmp64 = (uint64_t)(uint32_t)regs[r2] * (uint32_t)regs[r3]; + tci_write_reg64(regs, r1, r0, tmp64); + break; +#endif +#if TCG_TARGET_HAS_muls2_i32 + case INDEX_op_muls2_i32: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + tmp64 = (int64_t)(int32_t)regs[r2] * (int32_t)regs[r3]; + tci_write_reg64(regs, r1, r0, tmp64); + break; +#endif +#if TCG_TARGET_HAS_ext8s_i32 || TCG_TARGET_HAS_ext8s_i64 + CASE_32_64(ext8s) + tci_args_rr(insn, &r0, &r1); + regs[r0] = (int8_t)regs[r1]; + break; +#endif +#if TCG_TARGET_HAS_ext16s_i32 || TCG_TARGET_HAS_ext16s_i64 || \ + TCG_TARGET_HAS_bswap16_i32 || TCG_TARGET_HAS_bswap16_i64 + CASE_32_64(ext16s) + tci_args_rr(insn, &r0, &r1); + regs[r0] = (int16_t)regs[r1]; + break; +#endif +#if TCG_TARGET_HAS_ext8u_i32 || TCG_TARGET_HAS_ext8u_i64 + CASE_32_64(ext8u) + tci_args_rr(insn, &r0, &r1); + regs[r0] = (uint8_t)regs[r1]; + break; +#endif +#if TCG_TARGET_HAS_ext16u_i32 || TCG_TARGET_HAS_ext16u_i64 + CASE_32_64(ext16u) + tci_args_rr(insn, &r0, &r1); + regs[r0] = (uint16_t)regs[r1]; + break; +#endif +#if TCG_TARGET_HAS_bswap16_i32 || TCG_TARGET_HAS_bswap16_i64 + CASE_32_64(bswap16) + tci_args_rr(insn, &r0, &r1); + regs[r0] = bswap16(regs[r1]); + break; +#endif +#if TCG_TARGET_HAS_bswap32_i32 || TCG_TARGET_HAS_bswap32_i64 + CASE_32_64(bswap32) + tci_args_rr(insn, &r0, &r1); + regs[r0] = bswap32(regs[r1]); + break; +#endif +#if TCG_TARGET_HAS_not_i32 || TCG_TARGET_HAS_not_i64 + CASE_32_64(not) + tci_args_rr(insn, &r0, &r1); + regs[r0] = ~regs[r1]; + break; +#endif +#if TCG_TARGET_HAS_neg_i32 || TCG_TARGET_HAS_neg_i64 + CASE_32_64(neg) + tci_args_rr(insn, &r0, &r1); + regs[r0] = -regs[r1]; + break; +#endif +#if TCG_TARGET_REG_BITS == 64 + /* Load/store operations (64 bit). */ + + case INDEX_op_ld32s_i64: + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + regs[r0] = *(int32_t *)ptr; + break; + case INDEX_op_ld_i64: + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + regs[r0] = *(uint64_t *)ptr; + break; + case INDEX_op_st_i64: + tci_args_rrs(insn, &r0, &r1, &ofs); + ptr = (void *)(regs[r1] + ofs); + *(uint64_t *)ptr = regs[r0]; + break; + + /* Arithmetic operations (64 bit). */ + + case INDEX_op_div_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (int64_t)regs[r1] / (int64_t)regs[r2]; + break; + case INDEX_op_divu_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (uint64_t)regs[r1] / (uint64_t)regs[r2]; + break; + case INDEX_op_rem_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (int64_t)regs[r1] % (int64_t)regs[r2]; + break; + case INDEX_op_remu_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (uint64_t)regs[r1] % (uint64_t)regs[r2]; + break; +#if TCG_TARGET_HAS_clz_i64 + case INDEX_op_clz_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] ? clz64(regs[r1]) : regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_ctz_i64 + case INDEX_op_ctz_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] ? ctz64(regs[r1]) : regs[r2]; + break; +#endif +#if TCG_TARGET_HAS_ctpop_i64 + case INDEX_op_ctpop_i64: + tci_args_rr(insn, &r0, &r1); + regs[r0] = ctpop64(regs[r1]); + break; +#endif +#if TCG_TARGET_HAS_mulu2_i64 + case INDEX_op_mulu2_i64: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + mulu64(®s[r0], ®s[r1], regs[r2], regs[r3]); + break; +#endif +#if TCG_TARGET_HAS_muls2_i64 + case INDEX_op_muls2_i64: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + muls64(®s[r0], ®s[r1], regs[r2], regs[r3]); + break; +#endif +#if TCG_TARGET_HAS_add2_i64 + case INDEX_op_add2_i64: + tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); + T1 = regs[r2] + regs[r4]; + T2 = regs[r3] + regs[r5] + (T1 < regs[r2]); + regs[r0] = T1; + regs[r1] = T2; + break; +#endif +#if TCG_TARGET_HAS_add2_i64 + case INDEX_op_sub2_i64: + tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); + T1 = regs[r2] - regs[r4]; + T2 = regs[r3] - regs[r5] - (regs[r2] < regs[r4]); + regs[r0] = T1; + regs[r1] = T2; + break; +#endif + + /* Shift/rotate operations (64 bit). */ + + case INDEX_op_shl_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] << (regs[r2] & 63); + break; + case INDEX_op_shr_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] >> (regs[r2] & 63); + break; + case INDEX_op_sar_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = (int64_t)regs[r1] >> (regs[r2] & 63); + break; +#if TCG_TARGET_HAS_rot_i64 + case INDEX_op_rotl_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = rol64(regs[r1], regs[r2] & 63); + break; + case INDEX_op_rotr_i64: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = ror64(regs[r1], regs[r2] & 63); + break; +#endif +#if TCG_TARGET_HAS_deposit_i64 + case INDEX_op_deposit_i64: + tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); + regs[r0] = deposit64(regs[r1], pos, len, regs[r2]); + break; +#endif +#if TCG_TARGET_HAS_extract_i64 + case INDEX_op_extract_i64: + tci_args_rrbb(insn, &r0, &r1, &pos, &len); + regs[r0] = extract64(regs[r1], pos, len); + break; +#endif +#if TCG_TARGET_HAS_sextract_i64 + case INDEX_op_sextract_i64: + tci_args_rrbb(insn, &r0, &r1, &pos, &len); + regs[r0] = sextract64(regs[r1], pos, len); + break; +#endif + case INDEX_op_brcond_i64: + tci_args_rl(insn, tb_ptr, &r0, &ptr); + if (regs[r0]) { + tb_ptr = ptr; + } + break; + case INDEX_op_ext32s_i64: + case INDEX_op_ext_i32_i64: + tci_args_rr(insn, &r0, &r1); + regs[r0] = (int32_t)regs[r1]; + break; + case INDEX_op_ext32u_i64: + case INDEX_op_extu_i32_i64: + tci_args_rr(insn, &r0, &r1); + regs[r0] = (uint32_t)regs[r1]; + break; +#if TCG_TARGET_HAS_bswap64_i64 + case INDEX_op_bswap64_i64: + tci_args_rr(insn, &r0, &r1); + regs[r0] = bswap64(regs[r1]); + break; +#endif +#endif /* TCG_TARGET_REG_BITS == 64 */ + + /* QEMU specific operations. */ + + case INDEX_op_exit_tb: + tci_args_l(insn, tb_ptr, &ptr); + ctx.tb_ptr = 0; + return (uintptr_t)ptr; + + case INDEX_op_goto_tb: + tci_args_l(insn, tb_ptr, &ptr); + if (*(uint32_t **)ptr != 0) { + tb_ptr = *(uint32_t **)ptr; + ctx.tb_ptr = tb_ptr; + int tb_entry_ptr = (uint32_t)tb_ptr + export_vec_off; + if ((*(int32_t*)tb_entry_ptr <= 0) && (*(int32_t*)tb_entry_ptr > (-1 * INSTANTIATE_NUM))) { + *(int32_t*)tb_entry_ptr -= 1; + } else { + // enter to wasm TB + return 0; + } + tb_ptr = (uint8_t*)tb_ptr + *(uint32_t*)tb_ptr; + } + break; + + case INDEX_op_goto_ptr: + tci_args_r(insn, &r0); + ptr = (void *)regs[r0]; + if (!ptr) { + ctx.tb_ptr = 0; + return 0; + } + tb_ptr = ptr; + + ctx.tb_ptr = tb_ptr; + int tb_entry_ptr = (uint32_t)tb_ptr + export_vec_off; + if ((*(int32_t*)tb_entry_ptr <= 0) && (*(int32_t*)tb_entry_ptr > (-1 * INSTANTIATE_NUM))) { + *(int32_t*)tb_entry_ptr -= 1; + } else { + // enter to wasm TB + return 0; + } + tb_ptr = (uint8_t*)tb_ptr + *(uint32_t*)tb_ptr; + + break; + + case INDEX_op_qemu_ld_a32_i32: + tci_args_ldst(insn, &r0, &r1, &oi, tb_ptr, &ptr); + taddr = (uint32_t)regs[r1]; + goto do_ld_i32; + case INDEX_op_qemu_ld_a64_i32: + tci_args_ldst(insn, &r0, &r1, &oi, tb_ptr, &ptr); + taddr = regs[r1]; + do_ld_i32: + regs[r0] = tci_qemu_ld(env, taddr, oi, tb_ptr, ptr); + break; + + case INDEX_op_qemu_ld_a32_i64: + tci_args_ldst(insn, &r0, &r1, &oi, tb_ptr, &ptr); + taddr = (uint32_t)regs[r1]; + goto do_ld_i64; + case INDEX_op_qemu_ld_a64_i64: + tci_args_ldst(insn, &r0, &r1, &oi, tb_ptr, &ptr); + taddr = regs[r1]; + do_ld_i64: + tmp64 = tci_qemu_ld(env, taddr, oi, tb_ptr, ptr); + if (TCG_TARGET_REG_BITS == 32) { + tci_write_reg64(regs, r1, r0, tmp64); + } else { + regs[r0] = tmp64; + } + break; + + case INDEX_op_qemu_st_a32_i32: + tci_args_ldst(insn, &r0, &r1, &oi, tb_ptr, &ptr); + taddr = (uint32_t)regs[r1]; + goto do_st_i32; + case INDEX_op_qemu_st_a64_i32: + tci_args_ldst(insn, &r0, &r1, &oi, tb_ptr, &ptr); + taddr = regs[r1]; + do_st_i32: + tci_qemu_st(env, taddr, regs[r0], oi, tb_ptr, ptr); + break; + + case INDEX_op_qemu_st_a32_i64: + tci_args_ldst(insn, &r0, &r1, &oi, tb_ptr, &ptr); + tmp64 = regs[r0]; + taddr = (uint32_t)regs[r1]; + goto do_st_i64; + case INDEX_op_qemu_st_a64_i64: + tci_args_ldst(insn, &r0, &r1, &oi, tb_ptr, &ptr); + tmp64 = regs[r0]; + taddr = regs[r1]; + do_st_i64: + tci_qemu_st(env, taddr, tmp64, oi, tb_ptr, ptr); + break; + + case INDEX_op_mb: + /* Ensure ordering for all kinds */ + smp_mb(); + break; + default: + g_assert_not_reached(); + } + } +} + +uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, + const void *v_tb_ptr) +{ ctx.env = env; ctx.tb_ptr = (uint32_t*)v_tb_ptr; ctx.do_init = 1; - ctx.tci_tb_ptr = (uint32_t*)&tci_tb_ptr; - uint32_t prev_tb_ptr = 0; while (true) { - int tb_entry_ptr = (uint32_t)ctx.tb_ptr + *(uint32_t*)ctx.tb_ptr + cur_core_num * 4; - if (*(uint32_t*)tb_entry_ptr == 0) { - prepare_wasm(ctx.tb_ptr, cur_core_num, all_cores_num); + int tb_entry_ptr = (uint32_t)ctx.tb_ptr + export_vec_off; + uint32_t res; + if (*(int32_t*)tb_entry_ptr > 0) { + res = ((wasm_func_ptr)(*(uint32_t*)tb_entry_ptr))(&ctx); + } else if (*(int32_t*)tb_entry_ptr > (-1 * INSTANTIATE_NUM)) { + *(int32_t*)tb_entry_ptr -= 1; + res = tcg_qemu_tb_exec_tci(env); + } else { + instantiate_wasm(); + res = ((wasm_func_ptr)(*(uint32_t*)tb_entry_ptr))(&ctx); } - uint32_t res = ((wasm_func_ptr)(*(uint32_t*)tb_entry_ptr))(&ctx); if ((uint32_t)ctx.tb_ptr == 0) { return res; } diff --git a/tcg/wasm32.h b/tcg/wasm32.h index 3278f2541f..1861510f0b 100644 --- a/tcg/wasm32.h +++ b/tcg/wasm32.h @@ -35,4 +35,10 @@ void set_unwinding_flag(); int get_core_nums(); +void remove_tb(void *tb_ptr); + +void init_wasm32(); + +#define INSTANTIATE_NUM 1500 + #endif diff --git a/tcg/wasm32/tcg-target.c.inc b/tcg/wasm32/tcg-target.c.inc index 7217d872c0..3c85588be9 100644 --- a/tcg/wasm32/tcg-target.c.inc +++ b/tcg/wasm32/tcg-target.c.inc @@ -30,6 +30,8 @@ #include "../wasm32.h" #include +#include +#include "../tcg-pool.c.inc" static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { @@ -190,7 +192,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) } } -static const int tcg_target_reg_alloc_order[] = { +static const int tcg_target_reg_alloc_order[TCG_TARGET_NB_REGS] = { TCG_REG_R0, TCG_REG_R1, TCG_REG_R2, @@ -207,35 +209,22 @@ static const int tcg_target_reg_alloc_order[] = { TCG_REG_R13, TCG_REG_R14, TCG_REG_R15, - - // Arguments - TCG_REG_A0, - TCG_REG_A1, - TCG_REG_A2, - TCG_REG_A3, - TCG_REG_A4, - TCG_REG_A5, - TCG_REG_A6, - TCG_REG_A7, }; -#define NUM_OF_IARG_REGS 8 +#define NUM_OF_IARG_REGS 5 static const int tcg_target_call_iarg_regs[NUM_OF_IARG_REGS] = { - TCG_REG_A0, - TCG_REG_A1, - TCG_REG_A2, - TCG_REG_A3, - TCG_REG_A4, - TCG_REG_A5, - TCG_REG_A6, - TCG_REG_A7, + TCG_REG_R8, + TCG_REG_R9, + TCG_REG_R10, + TCG_REG_R11, + TCG_REG_R12, }; static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) { tcg_debug_assert(kind == TCG_CALL_RET_NORMAL); tcg_debug_assert(slot >= 0 && slot < 128 / TCG_TARGET_REG_BITS); - return TCG_REG_A0 + slot; + return TCG_REG_R0 + slot; } #ifdef CONFIG_DEBUG_TCG @@ -256,18 +245,10 @@ static const char *const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { "r13", "r14", "r15", - "a0", - "a1", - "a2", - "a3", - "a4", - "a5", - "a6", - "a7", }; #endif -#define REG_INDEX_IARG_BASE 16 +#define REG_INDEX_IARG_BASE 8 static const uint8_t tcg_target_reg_index[TCG_TARGET_NB_REGS] = { 0, // TCG_REG_R0 1, // TCG_REG_R1 @@ -285,18 +266,9 @@ static const uint8_t tcg_target_reg_index[TCG_TARGET_NB_REGS] = { 13, // TCG_REG_R13 14, // TCG_REG_R14 15, // TCG_REG_R15 - - 16, // TCG_REG_A0 - 17, // TCG_REG_A1 - 18, // TCG_REG_A2 - 19, // TCG_REG_A3 - 20, // TCG_REG_A4 - 21, // TCG_REG_A5 - 22, // TCG_REG_A6 - 23, // TCG_REG_A7 }; -#define BLOCK_PTR_IDX 24 +#define BLOCK_PTR_IDX 16 #define CTX_IDX 0 #define TMP32_LOCAL_ENV_IDX 1 @@ -318,7 +290,17 @@ __thread bool env_cached = false; // table index #define HELPER_TABLE_IDX 0 -static void tcg_out_leb128_sint32_t(TCGContext *s, int32_t v) { +static inline void tcg_wasm_out8(TCGContext *s, uint32_t v) +{ + tcg_sub_out8(s, v); +} + +static uintptr_t cur_wasm_ptr(TCGContext *s) +{ + return (uintptr_t)cur_sub_buf_ptr(); +} + +static void tcg_wasm_out_leb128_sint32_t(TCGContext *s, int32_t v) { bool more = true; bool negative = (v < 0); uint8_t b; @@ -333,11 +315,11 @@ static void tcg_out_leb128_sint32_t(TCGContext *s, int32_t v) { more = false; else b |= 0x80; - tcg_out8(s, b); + tcg_wasm_out8(s, b); } } -static void tcg_out_leb128_sint64_t(TCGContext *s, int64_t v) { +static void tcg_wasm_out_leb128_sint64_t(TCGContext *s, int64_t v) { bool more = true; bool negative = (v < 0); uint8_t b; @@ -352,11 +334,11 @@ static void tcg_out_leb128_sint64_t(TCGContext *s, int64_t v) { more = false; else b |= 0x80; - tcg_out8(s, b); + tcg_wasm_out8(s, b); } } -static void tcg_out_leb128_uint32_t(TCGContext *s, uint32_t v) { +static void tcg_wasm_out_leb128_uint32_t(TCGContext *s, uint32_t v) { uint32_t low7 = 0x7f; uint8_t b; do { @@ -364,332 +346,292 @@ static void tcg_out_leb128_uint32_t(TCGContext *s, uint32_t v) { v >>= 7; if (v != 0) b |= 0x80; - tcg_out8(s, b); + tcg_wasm_out8(s, b); } while (v != 0); } -static void tcg_out_op_br(TCGContext *s, int i) +static void tcg_wasm_out_op_br(TCGContext *s, int i) { - tcg_out8(s, 0x0c); - tcg_out8(s, i); + tcg_wasm_out8(s, 0x0c); + tcg_wasm_out8(s, i); } -static void tcg_out_op_if_noret(TCGContext *s) +static void tcg_wasm_out_op_if_noret(TCGContext *s) { - tcg_out8(s, 0x04); - tcg_out8(s, 0x40); + tcg_wasm_out8(s, 0x04); + tcg_wasm_out8(s, 0x40); } -static void tcg_out_op_if_ret_i64(TCGContext *s) +static void tcg_wasm_out_op_if_ret_i64(TCGContext *s) { - tcg_out8(s, 0x04); - tcg_out8(s, 0x7e); + tcg_wasm_out8(s, 0x04); + tcg_wasm_out8(s, 0x7e); } -static void tcg_out_op_if_ret_i32(TCGContext *s) +static void tcg_wasm_out_op_if_ret_i32(TCGContext *s) { - tcg_out8(s, 0x04); - tcg_out8(s, 0x7f); + tcg_wasm_out8(s, 0x04); + tcg_wasm_out8(s, 0x7f); } -static void tcg_out_op_else(TCGContext *s) +static void tcg_wasm_out_op_else(TCGContext *s) { - tcg_out8(s, 0x05); + tcg_wasm_out8(s, 0x05); } -static void tcg_out_op_end(TCGContext *s) +static void tcg_wasm_out_op_end(TCGContext *s) { - tcg_out8(s, 0x0b); + tcg_wasm_out8(s, 0x0b); } -static void tcg_out_op_i32_eqz(TCGContext *s){ tcg_out8(s, 0x45); } -static void tcg_out_op_i32_eq(TCGContext *s){ tcg_out8(s, 0x46); } -static void tcg_out_op_i32_and(TCGContext *s){ tcg_out8(s, 0x71); } -static void tcg_out_op_i32_or(TCGContext *s){ tcg_out8(s, 0x72); } -//static void tcg_out_op_i32_xor(TCGContext *s){ tcg_out8(s, 0x73); } -static void tcg_out_op_i32_shl(TCGContext *s){ tcg_out8(s, 0x74); } -static void tcg_out_op_i32_shr_s(TCGContext *s){ tcg_out8(s, 0x75); } -static void tcg_out_op_i32_shr_u(TCGContext *s){ tcg_out8(s, 0x76); } -static void tcg_out_op_i32_rotl(TCGContext *s){ tcg_out8(s, 0x77); } -static void tcg_out_op_i32_rotr(TCGContext *s){ tcg_out8(s, 0x78); } -static void tcg_out_op_i32_clz(TCGContext *s){ tcg_out8(s, 0x67); } -static void tcg_out_op_i32_ctz(TCGContext *s){ tcg_out8(s, 0x68); } -//static void tcg_out_op_i32_popcnt(TCGContext *s){ tcg_out8(s, 0x69); } -static void tcg_out_op_i32_add(TCGContext *s){ tcg_out8(s, 0x6a); } -//static void tcg_out_op_i32_sub(TCGContext *s){ tcg_out8(s, 0x6b); } -//static void tcg_out_op_i32_mul(TCGContext *s){ tcg_out8(s, 0x6c); } -//static void tcg_out_op_i32_div_s(TCGContext *s){ tcg_out8(s, 0x6d); } -//static void tcg_out_op_i32_div_u(TCGContext *s){ tcg_out8(s, 0x6e); } -//static void tcg_out_op_i32_rem_s(TCGContext *s){ tcg_out8(s, 0x6f); } -//static void tcg_out_op_i32_rem_u(TCGContext *s){ tcg_out8(s, 0x70); } -static void tcg_out_op_i32_ne(TCGContext *s){ tcg_out8(s, 0x47); } -//static void tcg_out_op_i32_le_u(TCGContext *s){ tcg_out8(s, 0x4d); } +static void tcg_wasm_out_op_i32_eqz(TCGContext *s){ tcg_wasm_out8(s, 0x45); } +static void tcg_wasm_out_op_i32_eq(TCGContext *s){ tcg_wasm_out8(s, 0x46); } +static void tcg_wasm_out_op_i32_and(TCGContext *s){ tcg_wasm_out8(s, 0x71); } +static void tcg_wasm_out_op_i32_or(TCGContext *s){ tcg_wasm_out8(s, 0x72); } +//static void tcg_wasm_out_op_i32_xor(TCGContext *s){ tcg_wasm_out8(s, 0x73); } +static void tcg_wasm_out_op_i32_shl(TCGContext *s){ tcg_wasm_out8(s, 0x74); } +static void tcg_wasm_out_op_i32_shr_s(TCGContext *s){ tcg_wasm_out8(s, 0x75); } +static void tcg_wasm_out_op_i32_shr_u(TCGContext *s){ tcg_wasm_out8(s, 0x76); } +static void tcg_wasm_out_op_i32_rotl(TCGContext *s){ tcg_wasm_out8(s, 0x77); } +static void tcg_wasm_out_op_i32_rotr(TCGContext *s){ tcg_wasm_out8(s, 0x78); } +static void tcg_wasm_out_op_i32_clz(TCGContext *s){ tcg_wasm_out8(s, 0x67); } +static void tcg_wasm_out_op_i32_ctz(TCGContext *s){ tcg_wasm_out8(s, 0x68); } +static void tcg_wasm_out_op_i32_popcnt(TCGContext *s){ tcg_wasm_out8(s, 0x69); } +static void tcg_wasm_out_op_i32_add(TCGContext *s){ tcg_wasm_out8(s, 0x6a); } +//static void tcg_wasm_out_op_i32_sub(TCGContext *s){ tcg_wasm_out8(s, 0x6b); } +//static void tcg_wasm_out_op_i32_mul(TCGContext *s){ tcg_wasm_out8(s, 0x6c); } +//static void tcg_wasm_out_op_i32_div_s(TCGContext *s){ tcg_wasm_out8(s, 0x6d); } +//static void tcg_wasm_out_op_i32_div_u(TCGContext *s){ tcg_wasm_out8(s, 0x6e); } +//static void tcg_wasm_out_op_i32_rem_s(TCGContext *s){ tcg_wasm_out8(s, 0x6f); } +//static void tcg_wasm_out_op_i32_rem_u(TCGContext *s){ tcg_wasm_out8(s, 0x70); } +static void tcg_wasm_out_op_i32_ne(TCGContext *s){ tcg_wasm_out8(s, 0x47); } +//static void tcg_wasm_out_op_i32_le_u(TCGContext *s){ tcg_wasm_out8(s, 0x4d); } -static void tcg_out_op_i64_eqz(TCGContext *s){ tcg_out8(s, 0x50); } -static void tcg_out_op_i64_eq(TCGContext *s){ tcg_out8(s, 0x51); } -static void tcg_out_op_i64_and(TCGContext *s){ tcg_out8(s, 0x83); } -static void tcg_out_op_i64_or(TCGContext *s){ tcg_out8(s, 0x84); } -static void tcg_out_op_i64_xor(TCGContext *s){ tcg_out8(s, 0x85); } -static void tcg_out_op_i64_shl(TCGContext *s){ tcg_out8(s, 0x86); } -static void tcg_out_op_i64_shr_s(TCGContext *s){ tcg_out8(s, 0x87); } -static void tcg_out_op_i64_shr_u(TCGContext *s){ tcg_out8(s, 0x88); } -static void tcg_out_op_i64_rotl(TCGContext *s){ tcg_out8(s, 0x89); } -static void tcg_out_op_i64_rotr(TCGContext *s){ tcg_out8(s, 0x8a); } -static void tcg_out_op_i64_clz(TCGContext *s){ tcg_out8(s, 0x79); } -static void tcg_out_op_i64_ctz(TCGContext *s){ tcg_out8(s, 0x7a); } -static void tcg_out_op_i64_popcnt(TCGContext *s){ tcg_out8(s, 0x7b); } -static void tcg_out_op_i64_add(TCGContext *s){ tcg_out8(s, 0x7c); } -static void tcg_out_op_i64_sub(TCGContext *s){ tcg_out8(s, 0x7d); } -static void tcg_out_op_i64_mul(TCGContext *s){ tcg_out8(s, 0x7e); } -static void tcg_out_op_i64_div_s(TCGContext *s){ tcg_out8(s, 0x7f); } -static void tcg_out_op_i64_div_u(TCGContext *s){ tcg_out8(s, 0x80); } -static void tcg_out_op_i64_rem_s(TCGContext *s){ tcg_out8(s, 0x81); } -static void tcg_out_op_i64_rem_u(TCGContext *s){ tcg_out8(s, 0x82); } -//static void tcg_out_op_i64_ne(TCGContext *s){ tcg_out8(s, 0x52); } -static void tcg_out_op_i64_le_u(TCGContext *s){ tcg_out8(s, 0x58); } -static void tcg_out_op_i64_lt_u(TCGContext *s){ tcg_out8(s, 0x54); } -static void tcg_out_op_i64_gt_u(TCGContext *s){ tcg_out8(s, 0x56); } +static void tcg_wasm_out_op_i64_eqz(TCGContext *s){ tcg_wasm_out8(s, 0x50); } +static void tcg_wasm_out_op_i64_eq(TCGContext *s){ tcg_wasm_out8(s, 0x51); } +static void tcg_wasm_out_op_i64_and(TCGContext *s){ tcg_wasm_out8(s, 0x83); } +static void tcg_wasm_out_op_i64_or(TCGContext *s){ tcg_wasm_out8(s, 0x84); } +static void tcg_wasm_out_op_i64_xor(TCGContext *s){ tcg_wasm_out8(s, 0x85); } +static void tcg_wasm_out_op_i64_shl(TCGContext *s){ tcg_wasm_out8(s, 0x86); } +static void tcg_wasm_out_op_i64_shr_s(TCGContext *s){ tcg_wasm_out8(s, 0x87); } +static void tcg_wasm_out_op_i64_shr_u(TCGContext *s){ tcg_wasm_out8(s, 0x88); } +static void tcg_wasm_out_op_i64_rotl(TCGContext *s){ tcg_wasm_out8(s, 0x89); } +static void tcg_wasm_out_op_i64_rotr(TCGContext *s){ tcg_wasm_out8(s, 0x8a); } +static void tcg_wasm_out_op_i64_clz(TCGContext *s){ tcg_wasm_out8(s, 0x79); } +static void tcg_wasm_out_op_i64_ctz(TCGContext *s){ tcg_wasm_out8(s, 0x7a); } +static void tcg_wasm_out_op_i64_popcnt(TCGContext *s){ tcg_wasm_out8(s, 0x7b); } +static void tcg_wasm_out_op_i64_add(TCGContext *s){ tcg_wasm_out8(s, 0x7c); } +static void tcg_wasm_out_op_i64_sub(TCGContext *s){ tcg_wasm_out8(s, 0x7d); } +static void tcg_wasm_out_op_i64_mul(TCGContext *s){ tcg_wasm_out8(s, 0x7e); } +static void tcg_wasm_out_op_i64_div_s(TCGContext *s){ tcg_wasm_out8(s, 0x7f); } +static void tcg_wasm_out_op_i64_div_u(TCGContext *s){ tcg_wasm_out8(s, 0x80); } +static void tcg_wasm_out_op_i64_rem_s(TCGContext *s){ tcg_wasm_out8(s, 0x81); } +static void tcg_wasm_out_op_i64_rem_u(TCGContext *s){ tcg_wasm_out8(s, 0x82); } +//static void tcg_wasm_out_op_i64_ne(TCGContext *s){ tcg_wasm_out8(s, 0x52); } +static void tcg_wasm_out_op_i64_le_u(TCGContext *s){ tcg_wasm_out8(s, 0x58); } +static void tcg_wasm_out_op_i64_lt_u(TCGContext *s){ tcg_wasm_out8(s, 0x54); } +static void tcg_wasm_out_op_i64_gt_u(TCGContext *s){ tcg_wasm_out8(s, 0x56); } -static void tcg_out_op_i32_wrap_i64(TCGContext *s){ tcg_out8(s, 0xa7); } +static void tcg_wasm_out_op_i32_wrap_i64(TCGContext *s){ tcg_wasm_out8(s, 0xa7); } -static void tcg_out_op_var(TCGContext *s, uint8_t instr, uint8_t i) +static void tcg_wasm_out_op_var(TCGContext *s, uint8_t instr, uint8_t i) { - tcg_out8(s, instr); - tcg_out8(s, i); + tcg_wasm_out8(s, instr); + tcg_wasm_out8(s, i); } -static void tcg_out_op_local_get(TCGContext *s, uint8_t i) +static void tcg_wasm_out_op_local_get(TCGContext *s, uint8_t i) { - tcg_out_op_var(s, 0x20, i); + tcg_wasm_out_op_var(s, 0x20, i); } -static void tcg_out_op_local_set(TCGContext *s, uint8_t i) +static void tcg_wasm_out_op_local_set(TCGContext *s, uint8_t i) { - tcg_out_op_var(s, 0x21, i); + tcg_wasm_out_op_var(s, 0x21, i); } -static void tcg_out_op_local_tee(TCGContext *s, uint8_t i) +static void tcg_wasm_out_op_local_tee(TCGContext *s, uint8_t i) { - tcg_out_op_var(s, 0x22, i); + tcg_wasm_out_op_var(s, 0x22, i); } -static void tcg_out_op_global_get(TCGContext *s, uint8_t i) +static void tcg_wasm_out_op_global_get(TCGContext *s, uint8_t i) { - tcg_out_op_var(s, 0x23, i); + tcg_wasm_out_op_var(s, 0x23, i); } -static void tcg_out_op_global_set(TCGContext *s, uint8_t i) +static void tcg_wasm_out_op_global_set(TCGContext *s, uint8_t i) { if (i == tcg_target_reg_index[TCG_REG_R14]) { env_cached = false; } - tcg_out_op_var(s, 0x24, i); + tcg_wasm_out_op_var(s, 0x24, i); } -static void tcg_out_op_global_get_r_i32(TCGContext *s, TCGReg r0) +static void tcg_wasm_out_op_global_get_r_i32(TCGContext *s, TCGReg r0) { if (r0 == TCG_REG_R14) { if (!env_cached) { - tcg_out_op_global_get(s, tcg_target_reg_index[r0]); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_local_tee(s, TMP32_LOCAL_ENV_IDX); + tcg_wasm_out_op_global_get(s, tcg_target_reg_index[r0]); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_local_tee(s, TMP32_LOCAL_ENV_IDX); env_cached = true; } else { - tcg_out_op_local_get(s, TMP32_LOCAL_ENV_IDX); + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_ENV_IDX); } return; } - tcg_out_op_global_get(s, tcg_target_reg_index[r0]); - tcg_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get(s, tcg_target_reg_index[r0]); + tcg_wasm_out_op_i32_wrap_i64(s); } -static void tcg_out_op_global_get_r(TCGContext *s, TCGReg r0) +static void tcg_wasm_out_op_global_get_r(TCGContext *s, TCGReg r0) { - tcg_out_op_global_get(s, tcg_target_reg_index[r0]); + tcg_wasm_out_op_global_get(s, tcg_target_reg_index[r0]); } -static void tcg_out_op_global_set_r(TCGContext *s, TCGReg r0) +static void tcg_wasm_out_op_global_set_r(TCGContext *s, TCGReg r0) { - tcg_out_op_global_set(s, tcg_target_reg_index[r0]); + tcg_wasm_out_op_global_set(s, tcg_target_reg_index[r0]); } -static void tcg_out_op_i32_const(TCGContext *s, int32_t v) +static void tcg_wasm_out_op_i32_const(TCGContext *s, int32_t v) { - tcg_out8(s, 0x41); - tcg_out_leb128_sint32_t(s, v); + tcg_wasm_out8(s, 0x41); + tcg_wasm_out_leb128_sint32_t(s, v); } -static void tcg_out_op_i64_const(TCGContext *s, int64_t v) +static void tcg_wasm_out_op_i64_const(TCGContext *s, int64_t v) { - tcg_out8(s, 0x42); - tcg_out_leb128_sint64_t(s, v); + tcg_wasm_out8(s, 0x42); + tcg_wasm_out_leb128_sint64_t(s, v); } -static void tcg_out_op_loadstore(TCGContext *s, uint8_t instr, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_loadstore(TCGContext *s, uint8_t instr, uint32_t a, uint32_t o) { - tcg_out8(s, instr); - tcg_out_leb128_uint32_t(s, a); - tcg_out_leb128_uint32_t(s, o); + tcg_wasm_out8(s, instr); + tcg_wasm_out_leb128_uint32_t(s, a); + tcg_wasm_out_leb128_uint32_t(s, o); } -static void tcg_out_op_i64_store(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_store(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x37, a, o); + tcg_wasm_out_op_loadstore(s, 0x37, a, o); } -static void tcg_out_op_i32_store(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i32_store(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x36, a, o); + tcg_wasm_out_op_loadstore(s, 0x36, a, o); } -static void tcg_out_op_i64_store8(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_store8(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x3c, a, o); + tcg_wasm_out_op_loadstore(s, 0x3c, a, o); } - -//static void tcg_out_op_i32_store8(TCGContext *s, uint32_t a, uint32_t o) -//{ -// tcg_out_op_loadstore(s, 0x3a, a, o); -//} -static void tcg_out_op_i64_store16(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_store16(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x3d, a, o); + tcg_wasm_out_op_loadstore(s, 0x3d, a, o); } -//static void tcg_out_op_i32_store16(TCGContext *s, uint32_t a, uint32_t o) -//{ -// tcg_out_op_loadstore(s, 0x3b, a, o); -//} - -static void tcg_out_op_i64_store32(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_store32(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x3e, a, o); + tcg_wasm_out_op_loadstore(s, 0x3e, a, o); } -static void tcg_out_op_i64_load(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_load(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x29, a, o); + tcg_wasm_out_op_loadstore(s, 0x29, a, o); } -static void tcg_out_op_i32_load(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i32_load(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x28, a, o); + tcg_wasm_out_op_loadstore(s, 0x28, a, o); } - static void tcg_out_op_i64_load8_s(TCGContext *s, uint32_t a, uint32_t o) + static void tcg_wasm_out_op_i64_load8_s(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x30, a, o); + tcg_wasm_out_op_loadstore(s, 0x30, a, o); } -//static void tcg_out_op_i32_load8_s(TCGContext *s, uint32_t a, uint32_t o) -//{ -// tcg_out_op_loadstore(s, 0x2c, a, o); -//} -// -static void tcg_out_op_i64_load8_u(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_load8_u(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x31, a, o); + tcg_wasm_out_op_loadstore(s, 0x31, a, o); } -//static void tcg_out_op_i32_load8_u(TCGContext *s, uint32_t a, uint32_t o) -//{ -// tcg_out_op_loadstore(s, 0x2d, a, o); -//} - -static void tcg_out_op_i64_load16_s(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_load16_s(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x32, a, o); + tcg_wasm_out_op_loadstore(s, 0x32, a, o); } -//static void tcg_out_op_i32_load16_s(TCGContext *s, uint32_t a, uint32_t o) -//{ -// tcg_out_op_loadstore(s, 0x2e, a, o); -//} - -static void tcg_out_op_i64_load16_u(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_load16_u(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x33, a, o); + tcg_wasm_out_op_loadstore(s, 0x33, a, o); } -//static void tcg_out_op_i32_load16_u(TCGContext *s, uint32_t a, uint32_t o) -//{ -// tcg_out_op_loadstore(s, 0x2f, a, o); -//} - -static void tcg_out_op_i64_load32_u(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_load32_u(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x35, a, o); + tcg_wasm_out_op_loadstore(s, 0x35, a, o); } -static void tcg_out_op_i64_load32_s(TCGContext *s, uint32_t a, uint32_t o) +static void tcg_wasm_out_op_i64_load32_s(TCGContext *s, uint32_t a, uint32_t o) { - tcg_out_op_loadstore(s, 0x34, a, o); + tcg_wasm_out_op_loadstore(s, 0x34, a, o); } -static void tcg_out_op_return(TCGContext *s) +static void tcg_wasm_out_op_return(TCGContext *s) { - tcg_out8(s, 0x0f); + tcg_wasm_out8(s, 0x0f); } -static void tcg_out_op_call(TCGContext *s, uint32_t func_idx) +static void tcg_wasm_out_op_call(TCGContext *s, uint32_t func_idx) { - tcg_out8(s, 0x10); - tcg_out_leb128_uint32_t(s, func_idx); + tcg_wasm_out8(s, 0x10); + tcg_wasm_out_leb128_uint32_t(s, func_idx); } -static void tcg_out_op_i64_extend_i32_u(TCGContext *s) +static void tcg_wasm_out_op_i64_extend_i32_u(TCGContext *s) { - tcg_out8(s, 0xad); + tcg_wasm_out8(s, 0xad); } -static void tcg_out_op_i64_extend_i32_s(TCGContext *s) +static void tcg_wasm_out_op_i64_extend_i32_s(TCGContext *s) { - tcg_out8(s, 0xac); + tcg_wasm_out8(s, 0xac); } -static void tcg_out_op_i64_extend8_s(TCGContext *s) +static void tcg_wasm_out_op_i64_extend8_s(TCGContext *s) { - tcg_out8(s, 0xc2); + tcg_wasm_out8(s, 0xc2); } -//static void tcg_out_op_i32_extend8_s(TCGContext *s) -//{ -// tcg_out8(s, 0xc0); -//} - -static void tcg_out_op_i64_extend16_s(TCGContext *s) +static void tcg_wasm_out_op_i64_extend16_s(TCGContext *s) { - tcg_out8(s, 0xc3); + tcg_wasm_out8(s, 0xc3); } -//static void tcg_out_op_i32_extend16_s(TCGContext *s) -//{ -// tcg_out8(s, 0xc1); -//} - -static void tcg_out_op_not(TCGContext *s){ - tcg_out_op_i64_const(s, -1); - tcg_out_op_i64_xor(s); +static void tcg_wasm_out_op_not(TCGContext *s){ + tcg_wasm_out_op_i64_const(s, -1); + tcg_wasm_out_op_i64_xor(s); } -static void tcg_out_op_set_r_as_i64(TCGContext *s, TCGReg al, TCGReg ah) +static void tcg_wasm_out_op_set_r_as_i64(TCGContext *s, TCGReg al, TCGReg ah) { - tcg_out_op_local_set(s, TMP64_4_IDX); + tcg_wasm_out_op_local_set(s, TMP64_4_IDX); // set lower bits - tcg_out_op_local_get(s, TMP64_4_IDX); - tcg_out_op_i64_const(s, 0xffffffff); - tcg_out_op_i64_and(s); - tcg_out_op_global_set_r(s, al); + tcg_wasm_out_op_local_get(s, TMP64_4_IDX); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_set_r(s, al); // set higher bits - tcg_out_op_local_get(s, TMP64_4_IDX); - tcg_out_op_i64_const(s, 32); - tcg_out_op_i64_shr_u(s); - tcg_out_op_i64_const(s, 0xffffffff); - tcg_out_op_i64_and(s); - tcg_out_op_global_set_r(s, ah); + tcg_wasm_out_op_local_get(s, TMP64_4_IDX); + tcg_wasm_out_op_i64_const(s, 32); + tcg_wasm_out_op_i64_shr_u(s); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_set_r(s, ah); } static const struct { @@ -708,440 +650,591 @@ static const struct { [TCG_COND_GTU] = { 0x4b /* i32.gt_u */ , 0x56 /* i64.gt_u */} }; -static void tcg_out_op_cond_i64(TCGContext *s, TCGCond cond, TCGReg arg1, TCGReg arg2) +static void tcg_wasm_out_op_cond_i64(TCGContext *s, TCGCond cond, TCGReg arg1, TCGReg arg2) { uint8_t op = tcg_cond_to_inst[cond].i64; - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out8(s, op); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out8(s, op); } -#define tcg_out_i64_calc(op) \ - static void tcg_out_i64_calc_##op(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ \ - tcg_out_op_global_get_r(s, arg1); \ - tcg_out_op_global_get_r(s, arg2); \ - tcg_out_op_i64_##op(s); \ - tcg_out_op_global_set_r(s, ret); \ +static void tcg_wasm_out_op_cond_i32(TCGContext *s, TCGCond cond, TCGReg arg1, TCGReg arg2) +{ + uint8_t op = tcg_cond_to_inst[cond].i32; + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out8(s, op); +} + +#define tcg_wasm_out_i64_calc(op) \ + static void tcg_wasm_out_i64_calc_##op(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ \ + tcg_wasm_out_op_global_get_r(s, arg1); \ + tcg_wasm_out_op_global_get_r(s, arg2); \ + tcg_wasm_out_op_i64_##op(s); \ + tcg_wasm_out_op_global_set_r(s, ret); \ } -tcg_out_i64_calc(and); -tcg_out_i64_calc(or); -tcg_out_i64_calc(xor); -tcg_out_i64_calc(shl); -tcg_out_i64_calc(shr_s); -tcg_out_i64_calc(shr_u); -tcg_out_i64_calc(rotl); -tcg_out_i64_calc(rotr); -tcg_out_i64_calc(add); -tcg_out_i64_calc(sub); -tcg_out_i64_calc(mul); -tcg_out_i64_calc(div_s); -tcg_out_i64_calc(div_u); -tcg_out_i64_calc(rem_s); -tcg_out_i64_calc(rem_u); +tcg_wasm_out_i64_calc(and); +tcg_wasm_out_i64_calc(or); +tcg_wasm_out_i64_calc(xor); +tcg_wasm_out_i64_calc(shl); +tcg_wasm_out_i64_calc(shr_s); +tcg_wasm_out_i64_calc(shr_u); +tcg_wasm_out_i64_calc(rotl); +tcg_wasm_out_i64_calc(rotr); +tcg_wasm_out_i64_calc(add); +tcg_wasm_out_i64_calc(sub); +tcg_wasm_out_i64_calc(mul); +tcg_wasm_out_i64_calc(div_s); +tcg_wasm_out_i64_calc(div_u); +tcg_wasm_out_i64_calc(rem_s); +tcg_wasm_out_i64_calc(rem_u); -static void tcg_out_i32_rotl(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i32_rotl(s); - tcg_out_op_i64_extend_i32_s(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_rem_s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) { + switch (type) { + case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_i64_rem_s(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_rem_s(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + default: + g_assert_not_reached(); + } } -static void tcg_out_i32_rotr(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i32_rotr(s); - tcg_out_op_i64_extend_i32_s(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_rem_u(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) { + switch (type) { + case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_rem_u(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_rem_u(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + default: + g_assert_not_reached(); + } } -static void tcg_out_clz64(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i64_eqz(s); - tcg_out_op_if_ret_i64(s); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_else(s); - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i64_clz(s); - tcg_out_op_end(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_div_s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) { + switch (type) { + case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_i64_div_s(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_div_s(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + default: + g_assert_not_reached(); + } } -static void tcg_out_clz32(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i64_eqz(s); - tcg_out_op_if_ret_i32(s); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_else(s); - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i32_clz(s); - tcg_out_op_end(s); - tcg_out_op_i64_extend_i32_s(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_div_u(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) { + switch (type) { + case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_div_u(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_div_u(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + default: + g_assert_not_reached(); + } } -static void tcg_out_ctz64(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i64_eqz(s); - tcg_out_op_if_ret_i64(s); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_else(s); - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i64_ctz(s); - tcg_out_op_end(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_shl(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2){ + switch (type) { + case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_const(s, 31); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_shl(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_shl(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + default: + g_assert_not_reached(); + } } -static void tcg_out_ctz32(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i64_eqz(s); - tcg_out_op_if_ret_i32(s); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_else(s); - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i32_ctz(s); - tcg_out_op_end(s); - tcg_out_op_i64_extend_i32_s(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_shr_u(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2){ + switch (type) { + case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_const(s, 31); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_shr_u(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_shr_u(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + default: + g_assert_not_reached(); + } } -static void tcg_out_not(TCGContext *s, TCGReg ret, TCGReg arg){ - tcg_out_op_global_get_r(s, arg); - tcg_out_op_not(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_shr_s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2){ + switch (type) { + case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_const(s, 31); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_shr_s(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_shr_s(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; + default: + g_assert_not_reached(); + } +} +static void tcg_wasm_out_i32_rotl(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i32_rotl(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_andc(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_not(s); - tcg_out_op_i64_and(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_i32_rotr(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i32_rotr(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_orc(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_not(s); - tcg_out_op_i64_or(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_clz64(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_eqz(s); + tcg_wasm_out_op_if_ret_i64(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_else(s); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_clz(s); + tcg_wasm_out_op_end(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_eqv(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i64_xor(s); - tcg_out_op_not(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_clz32(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_eqz(s); + tcg_wasm_out_op_if_ret_i32(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_else(s); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i32_clz(s); + tcg_wasm_out_op_end(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_nand(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i64_and(s); - tcg_out_op_not(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_ctz64(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_eqz(s); + tcg_wasm_out_op_if_ret_i64(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_else(s); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_ctz(s); + tcg_wasm_out_op_end(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_nor(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i64_or(s); - tcg_out_op_not(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_ctz32(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_eqz(s); + tcg_wasm_out_op_if_ret_i32(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_else(s); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i32_ctz(s); + tcg_wasm_out_op_end(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_neg(TCGContext *s, TCGReg ret, TCGReg arg){ - tcg_out_op_global_get_r(s, arg); - tcg_out_op_not(s); - tcg_out_op_i64_const(s, 1); - tcg_out_op_i64_add(s); - tcg_out_op_global_set_r(s, ret); +static void tcg_wasm_out_not(TCGContext *s, TCGReg ret, TCGReg arg){ + tcg_wasm_out_op_global_get_r(s, arg); + tcg_wasm_out_op_not(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static bool patch_reloc(tcg_insn_unit *code_ptr, int type, - intptr_t value, intptr_t addend) +static void tcg_wasm_out_andc(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_not(s); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_set_r(s, ret); +} + +static void tcg_wasm_out_orc(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_not(s); + tcg_wasm_out_op_i64_or(s); + tcg_wasm_out_op_global_set_r(s, ret); +} + +static void tcg_wasm_out_eqv(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_xor(s); + tcg_wasm_out_op_not(s); + tcg_wasm_out_op_global_set_r(s, ret); +} + +static void tcg_wasm_out_nand(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_not(s); + tcg_wasm_out_op_global_set_r(s, ret); +} + +static void tcg_wasm_out_nor(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2){ + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_or(s); + tcg_wasm_out_op_not(s); + tcg_wasm_out_op_global_set_r(s, ret); +} + +static void tcg_wasm_out_neg(TCGContext *s, TCGReg ret, TCGReg arg){ + tcg_wasm_out_op_global_get_r(s, arg); + tcg_wasm_out_op_not(s); + tcg_wasm_out_op_i64_const(s, 1); + tcg_wasm_out_op_i64_add(s); + tcg_wasm_out_op_global_set_r(s, ret); +} + +static void tcg_wasm_out_ld(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_i64_load32_u(s, 0, (uint32_t)offset); + tcg_wasm_out_op_global_set_r(s, val); + break; + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_i64_load(s, 0, (uint32_t)offset); + tcg_wasm_out_op_global_set_r(s, val); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_ld8s(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_i64_load8_s(s, 0, (uint32_t)offset); + tcg_wasm_out_op_global_set_r(s, val); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_ld8u(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_i64_load8_u(s, 0, (uint32_t)offset); + tcg_wasm_out_op_global_set_r(s, val); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_ld16s(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_i64_load16_s(s, 0, (uint32_t)offset); + tcg_wasm_out_op_global_set_r(s, val); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_ld16u(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_i64_load16_u(s, 0, (uint32_t)offset); + tcg_wasm_out_op_global_set_r(s, val); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_ld32s(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_i64_load32_s(s, 0, (uint32_t)offset); + tcg_wasm_out_op_global_set_r(s, val); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_ld32u(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_i64_load32_u(s, 0, (uint32_t)offset); + tcg_wasm_out_op_global_set_r(s, val); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_st(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_global_get_r(s, val); + tcg_wasm_out_op_i64_store32(s, 0, (uint32_t)offset); + break; + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_global_get_r(s, val); + tcg_wasm_out_op_i64_store(s, 0, (uint32_t)offset); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_st8(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_global_get_r(s, val); + tcg_wasm_out_op_i64_store8(s, 0, (uint32_t)offset); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_st16(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_global_get_r(s, val); + tcg_wasm_out_op_i64_store16(s, 0, (uint32_t)offset); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_wasm_out_st32(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_wasm_out_op_global_get_r_i32(s, base); + if ((int32_t)offset < 0) { + tcg_wasm_out_op_i32_const(s, (int32_t)offset); + tcg_wasm_out_op_i32_add(s); + offset = 0; + } + tcg_wasm_out_op_global_get_r(s, val); + tcg_wasm_out_op_i64_store32(s, 0, (uint32_t)offset); + break; + default: + g_assert_not_reached(); + } +} + +static inline bool tcg_wasm_out_sti(TCGContext *s, TCGType type, TCGArg val, + TCGReg base, intptr_t offset) { return false; } -static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_i64_load32_u(s, 0, (uint32_t)offset); - tcg_out_op_global_set_r(s, val); - break; - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_i64_load(s, 0, (uint32_t)offset); - tcg_out_op_global_set_r(s, val); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_ld8s(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_i64_load8_s(s, 0, (uint32_t)offset); - tcg_out_op_global_set_r(s, val); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_ld8u(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_i64_load8_u(s, 0, (uint32_t)offset); - tcg_out_op_global_set_r(s, val); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_ld16s(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_i64_load16_s(s, 0, (uint32_t)offset); - tcg_out_op_global_set_r(s, val); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_ld16u(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_i64_load16_u(s, 0, (uint32_t)offset); - tcg_out_op_global_set_r(s, val); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_ld32s(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_i64_load32_s(s, 0, (uint32_t)offset); - tcg_out_op_global_set_r(s, val); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_ld32u(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_i64_load32_u(s, 0, (uint32_t)offset); - tcg_out_op_global_set_r(s, val); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_st(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_global_get_r(s, val); - tcg_out_op_i64_store32(s, 0, (uint32_t)offset); - break; - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_global_get_r(s, val); - tcg_out_op_i64_store(s, 0, (uint32_t)offset); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_st8(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_global_get_r(s, val); - tcg_out_op_i64_store8(s, 0, (uint32_t)offset); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_st16(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_global_get_r(s, val); - tcg_out_op_i64_store16(s, 0, (uint32_t)offset); - break; - default: - g_assert_not_reached(); - } -} - -static void tcg_out_st32(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) -{ - switch (type) { - case TCG_TYPE_I32: - case TCG_TYPE_I64: - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - tcg_out_op_global_get_r(s, val); - tcg_out_op_i64_store32(s, 0, (uint32_t)offset); - break; - default: - g_assert_not_reached(); - } -} - -static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, - TCGReg base, intptr_t offset) -{ - tcg_out_op_global_get_r_i32(s, base); - if ((int32_t)offset < 0) { - tcg_out_op_i32_const(s, (int32_t)offset); - tcg_out_op_i32_add(s); - offset = 0; - } - switch (type) { - case TCG_TYPE_I32: - tcg_out_op_i64_const(s, (int32_t)val); - tcg_out_op_i64_store32(s, 0, (uint32_t)offset); - break; - case TCG_TYPE_I64: - tcg_out_op_i64_const(s, val); - tcg_out_op_i64_store(s, 0, (uint32_t)offset); - break; - default: - g_assert_not_reached(); - } - return true; -} - -static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) +static bool tcg_wasm_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) { switch (type) { case TCG_TYPE_I32: + tcg_wasm_out_op_global_get_r(s, arg); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_extend_i32_u(s); + tcg_wasm_out_op_global_set_r(s, ret); + break; case TCG_TYPE_I64: - tcg_out_op_global_get_r(s, arg); - tcg_out_op_global_set_r(s, ret); + tcg_wasm_out_op_global_get_r(s, arg); + tcg_wasm_out_op_global_set_r(s, ret); break; default: g_assert_not_reached(); @@ -1149,276 +1242,315 @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) return true; } -static void tcg_out_movi(TCGContext *s, TCGType type, +static void tcg_wasm_out_movi(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg) { switch (type) { case TCG_TYPE_I32: - tcg_out_op_i64_const(s, (int32_t)arg); + tcg_wasm_out_op_i64_const(s, (int32_t)arg); break; case TCG_TYPE_I64: - tcg_out_op_i64_const(s, arg); + tcg_wasm_out_op_i64_const(s, arg); break; default: g_assert_not_reached(); } - tcg_out_op_global_set_r(s, ret); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) { switch (type) { case TCG_TYPE_I32: case TCG_TYPE_I64: - tcg_out_op_global_get_r(s, rs); - tcg_out_op_i64_extend8_s(s); - tcg_out_op_global_set_r(s, rd); + tcg_wasm_out_op_global_get_r(s, rs); + tcg_wasm_out_op_i64_extend8_s(s); + tcg_wasm_out_op_global_set_r(s, rd); break; default: g_assert_not_reached(); } } -static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs) { - tcg_out_op_global_get_r(s, rs); - tcg_out_op_i64_const(s, 0xff); - tcg_out_op_i64_and(s); - tcg_out_op_global_set_r(s, rd); + tcg_wasm_out_op_global_get_r(s, rs); + tcg_wasm_out_op_i64_const(s, 0xff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_set_r(s, rd); } -static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) { switch (type) { case TCG_TYPE_I32: case TCG_TYPE_I64: - tcg_out_op_global_get_r(s, rs); - tcg_out_op_i64_extend16_s(s); - tcg_out_op_global_set_r(s, rd); + tcg_wasm_out_op_global_get_r(s, rs); + tcg_wasm_out_op_i64_extend16_s(s); + tcg_wasm_out_op_global_set_r(s, rd); break; default: g_assert_not_reached(); } } -static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs) { - tcg_out_op_global_get_r(s, rs); - tcg_out_op_i64_const(s, 0xffff); - tcg_out_op_i64_and(s); - tcg_out_op_global_set_r(s, rd); + tcg_wasm_out_op_global_get_r(s, rs); + tcg_wasm_out_op_i64_const(s, 0xffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_set_r(s, rd); } -static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs) { - tcg_out_op_global_get_r(s, rs); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i64_extend_i32_s(s); - tcg_out_op_global_set_r(s, rd); + tcg_wasm_out_op_global_get_r(s, rs); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_global_set_r(s, rd); } -static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs) { - tcg_out_op_global_get_r(s, rs); - tcg_out_op_i64_const(s, 0xffffffff); - tcg_out_op_i64_and(s); - tcg_out_op_global_set_r(s, rd); + tcg_wasm_out_op_global_get_r(s, rs); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_set_r(s, rd); } -static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs) { - tcg_out_ext32s(s, rd, rs); + tcg_wasm_out_ext32s(s, rd, rs); } -static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs) { - tcg_out_ext32u(s, rd, rs); + tcg_wasm_out_ext32u(s, rd, rs); } -static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs) { - tcg_out_op_global_get_r(s, rs); - tcg_out_op_i64_const(s, 0xffffffff); - tcg_out_op_i64_and(s); - tcg_out_op_global_set_r(s, rd); + tcg_wasm_out_op_global_get_r(s, rs); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_set_r(s, rd); } -static void tcg_out_extrh_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs) +static void tcg_wasm_out_extrh_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs) { - tcg_out_op_global_get_r(s, rs); - tcg_out_op_i64_const(s, 32); - tcg_out_op_i64_shr_u(s); - tcg_out_op_global_set_r(s, rd); + tcg_wasm_out_op_global_get_r(s, rs); + tcg_wasm_out_op_i64_const(s, 32); + tcg_wasm_out_op_i64_shr_u(s); + tcg_wasm_out_op_global_set_r(s, rd); } -static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2) -{ - return false; -} - -static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, - tcg_target_long imm) -{ - /* This function is only used for passing structs by reference. */ - g_assert_not_reached(); -} - -static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, +static void tcg_wasm_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg arg1, TCGReg arg2) { - tcg_out_op_cond_i64(s, cond, arg1, arg2); - tcg_out_op_i64_extend_i32_u(s); - tcg_out_op_global_set_r(s, ret); + tcg_wasm_out_op_cond_i32(s, cond, arg1, arg2); + tcg_wasm_out_op_i64_extend_i32_u(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, +static void tcg_wasm_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg arg1, TCGReg arg2) +{ + tcg_wasm_out_op_cond_i64(s, cond, arg1, arg2); + tcg_wasm_out_op_i64_extend_i32_u(s); + tcg_wasm_out_op_global_set_r(s, ret); +} + +static void tcg_wasm_out_movcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg c1, TCGReg c2, TCGReg v1, TCGReg v2) { - tcg_out_op_cond_i64(s, cond, c1, c2); - tcg_out_op_if_ret_i64(s); - tcg_out_op_global_get_r(s, v1); - tcg_out_op_else(s); - tcg_out_op_global_get_r(s, v2); - tcg_out_op_end(s); - tcg_out_op_global_set_r(s, ret); + tcg_wasm_out_op_cond_i32(s, cond, c1, c2); + tcg_wasm_out_op_if_ret_i64(s); + tcg_wasm_out_op_global_get_r(s, v1); + tcg_wasm_out_op_else(s); + tcg_wasm_out_op_global_get_r(s, v2); + tcg_wasm_out_op_end(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_add2(TCGContext *s, TCGReg retl, TCGReg reth, +static void tcg_wasm_out_movcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg c1, TCGReg c2, TCGReg v1, TCGReg v2) +{ + tcg_wasm_out_op_cond_i64(s, cond, c1, c2); + tcg_wasm_out_op_if_ret_i64(s); + tcg_wasm_out_op_global_get_r(s, v1); + tcg_wasm_out_op_else(s); + tcg_wasm_out_op_global_get_r(s, v2); + tcg_wasm_out_op_end(s); + tcg_wasm_out_op_global_set_r(s, ret); +} + +static void tcg_wasm_out_add2(TCGContext *s, TCGReg retl, TCGReg reth, TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) { // add higer - tcg_out_op_global_get_r(s, ah); - tcg_out_op_global_get_r(s, bh); - tcg_out_op_i64_add(s); + tcg_wasm_out_op_global_get_r(s, ah); + tcg_wasm_out_op_global_get_r(s, bh); + tcg_wasm_out_op_i64_add(s); // add lower - tcg_out_op_global_get_r(s, al); - tcg_out_op_global_get_r(s, bl); - tcg_out_op_i64_add(s); + tcg_wasm_out_op_global_get_r(s, al); + tcg_wasm_out_op_global_get_r(s, bl); + tcg_wasm_out_op_i64_add(s); // get carry if ((al == retl) && (bl == retl)) { - tcg_out_op_local_set(s, TMP64_0_IDX); - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_global_get_r(s, al); - tcg_out_op_i64_lt_u(s); - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_global_set_r(s, retl); + tcg_wasm_out_op_local_set(s, TMP64_0_IDX); + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_global_get_r(s, al); + tcg_wasm_out_op_i64_lt_u(s); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_global_set_r(s, retl); } else { - tcg_out_op_global_set_r(s, retl); - tcg_out_op_global_get_r(s, retl); + tcg_wasm_out_op_global_set_r(s, retl); + tcg_wasm_out_op_global_get_r(s, retl); if (al == retl) { - tcg_out_op_global_get_r(s, bl); + tcg_wasm_out_op_global_get_r(s, bl); } else { - tcg_out_op_global_get_r(s, al); + tcg_wasm_out_op_global_get_r(s, al); } - tcg_out_op_i64_lt_u(s); + tcg_wasm_out_op_i64_lt_u(s); + tcg_wasm_out_op_i64_extend_i32_s(s); } // add carry to higher - tcg_out_op_i64_add(s); - tcg_out_op_global_set_r(s, reth); + tcg_wasm_out_op_i64_add(s); + tcg_wasm_out_op_global_set_r(s, reth); } -static void tcg_out_sub2(TCGContext *s, TCGReg retl, TCGReg reth, +static void tcg_wasm_out_sub2(TCGContext *s, TCGReg retl, TCGReg reth, TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) { // sub higher - tcg_out_op_global_get_r(s, ah); - tcg_out_op_global_get_r(s, bh); - tcg_out_op_i64_sub(s); + tcg_wasm_out_op_global_get_r(s, ah); + tcg_wasm_out_op_global_get_r(s, bh); + tcg_wasm_out_op_i64_sub(s); // sub lower - tcg_out_op_global_get_r(s, al); - tcg_out_op_global_get_r(s, bl); - tcg_out_op_i64_sub(s); + tcg_wasm_out_op_global_get_r(s, al); + tcg_wasm_out_op_global_get_r(s, bl); + tcg_wasm_out_op_i64_sub(s); // get underflow if (al == retl) { - tcg_out_op_local_set(s, TMP64_0_IDX); + tcg_wasm_out_op_local_set(s, TMP64_0_IDX); - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_global_get_r(s, al); - tcg_out_op_i64_gt_u(s); + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_global_get_r(s, al); + tcg_wasm_out_op_i64_gt_u(s); - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_global_set_r(s, retl); + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_global_set_r(s, retl); } else { - tcg_out_op_global_set_r(s, retl); + tcg_wasm_out_op_global_set_r(s, retl); - tcg_out_op_global_get_r(s, retl); - tcg_out_op_global_get_r(s, al); - tcg_out_op_i64_gt_u(s); + tcg_wasm_out_op_global_get_r(s, retl); + tcg_wasm_out_op_global_get_r(s, al); + tcg_wasm_out_op_i64_gt_u(s); } - tcg_out_op_i64_sub(s); - tcg_out_op_global_set_r(s, reth); + tcg_wasm_out_op_i64_sub(s); + tcg_wasm_out_op_global_set_r(s, reth); } -static void tcg_out_mulu2_i32(TCGContext *s, TCGReg retl, TCGReg reth, TCGReg arg1, TCGReg arg2) +static void tcg_wasm_out_mulu2_i32(TCGContext *s, TCGReg retl, TCGReg reth, TCGReg arg1, TCGReg arg2) { - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i64_mul(s); - tcg_out_op_set_r_as_i64(s, retl, reth); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_mul(s); + tcg_wasm_out_op_set_r_as_i64(s, retl, reth); } -static void tcg_out_muls2_i32(TCGContext *s, TCGReg retl, TCGReg reth, TCGReg arg1, TCGReg arg2) +static void tcg_wasm_out_muls2_i32(TCGContext *s, TCGReg retl, TCGReg reth, TCGReg arg1, TCGReg arg2) { - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i64_mul(s); - tcg_out_op_set_r_as_i64(s, retl, reth); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_mul(s); + tcg_wasm_out_op_set_r_as_i64(s, retl, reth); } -static void tcg_out_muluh_i32(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2) +static void tcg_wasm_out_muluh_i32(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2) { - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i64_mul(s); - tcg_out_op_i64_const(s, 32); - tcg_out_op_i64_shr_u(s); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_set_r(s, ret); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_mul(s); + tcg_wasm_out_op_i64_const(s, 32); + tcg_wasm_out_op_i64_shr_u(s); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_mulsh_i32(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2) +static void tcg_wasm_out_mulsh_i32(TCGContext *s, TCGReg ret, TCGReg arg1, TCGReg arg2) { - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i64_mul(s); - tcg_out_op_i64_const(s, 32); - tcg_out_op_i64_shr_u(s); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_set_r(s, ret); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_mul(s); + tcg_wasm_out_op_i64_const(s, 32); + tcg_wasm_out_op_i64_shr_u(s); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_set_r(s, ret); } -static void tcg_out_ctpop(TCGContext *s, TCGReg dest, TCGReg src) +static void tcg_wasm_out_ctpop_i32(TCGContext *s, TCGReg dest, TCGReg src) { - tcg_out_op_global_get_r(s, src); - tcg_out_op_i64_popcnt(s); - tcg_out_op_global_set_r(s, dest); + tcg_wasm_out_op_global_get_r(s, src); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i32_popcnt(s); + tcg_wasm_out_op_global_set_r(s, dest); } -static void tcg_out_deposit(TCGContext *s, TCGReg dest, TCGReg arg1, TCGReg arg2, int pos, int len) +static void tcg_wasm_out_ctpop_i64(TCGContext *s, TCGReg dest, TCGReg src) { - int mask = ((1< 0) { - tcg_out_op_i64_const(s, sl); - tcg_out_op_i64_shl(s); + tcg_wasm_out_op_i64_const(s, sl); + tcg_wasm_out_op_i64_shl(s); } - tcg_out_op_i64_const(s, rs); + tcg_wasm_out_op_i64_const(s, rs); if (sign) { - tcg_out_op_i64_shr_s(s); + tcg_wasm_out_op_i64_shr_s(s); } else { - tcg_out_op_i64_shr_u(s); + tcg_wasm_out_op_i64_shr_u(s); } - tcg_out_op_global_set_r(s, dest); + tcg_wasm_out_op_global_set_r(s, dest); } -static void tcg_out_extract2_i32(TCGContext *s, TCGReg dest, TCGReg arg1, TCGReg arg2, int pos) +static void tcg_wasm_out_extract2_i32(TCGContext *s, TCGReg dest, TCGReg arg1, TCGReg arg2, int pos) { - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i64_const(s, 32-pos); - tcg_out_op_i64_shl(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_const(s, 32-pos); + tcg_wasm_out_op_i64_shl(s); - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i64_const(s, 0xffffffff); - tcg_out_op_i64_and(s); - tcg_out_op_i64_const(s, pos); - tcg_out_op_i64_shr_u(s); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_const(s, pos); + tcg_wasm_out_op_i64_shr_u(s); - tcg_out_op_i64_or(s); + tcg_wasm_out_op_i64_or(s); - tcg_out_op_i64_const(s, 0xffffffff); - tcg_out_op_i64_and(s); - tcg_out_op_global_set_r(s, dest); + tcg_wasm_out_op_i64_const(s, 0xffffffff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_global_set_r(s, dest); } -static void tcg_out_extract2_i64(TCGContext *s, TCGReg dest, TCGReg arg1, TCGReg arg2, int pos) +static void tcg_wasm_out_extract2_i64(TCGContext *s, TCGReg dest, TCGReg arg1, TCGReg arg2, int pos) { - tcg_out_op_global_get_r(s, arg2); - tcg_out_op_i64_const(s, 64-pos); - tcg_out_op_i64_shl(s); + tcg_wasm_out_op_global_get_r(s, arg2); + tcg_wasm_out_op_i64_const(s, 64-pos); + tcg_wasm_out_op_i64_shl(s); - tcg_out_op_global_get_r(s, arg1); - tcg_out_op_i64_const(s, pos); - tcg_out_op_i64_shr_u(s); + tcg_wasm_out_op_global_get_r(s, arg1); + tcg_wasm_out_op_i64_const(s, pos); + tcg_wasm_out_op_i64_shr_u(s); - tcg_out_op_i64_or(s); - tcg_out_op_global_set_r(s, dest); + tcg_wasm_out_op_i64_or(s); + tcg_wasm_out_op_global_set_r(s, dest); } -static void tcg_out_bswap64(TCGContext *s, TCGReg dest, TCGReg src, int flags) +static void tcg_wasm_out_bswap64(TCGContext *s, TCGReg dest, TCGReg src, int flags) { - tcg_out_op_global_get_r(s, src); // ABCDEFGH - tcg_out_op_i64_const(s, 32); - tcg_out_op_i64_rotr(s); - tcg_out_op_local_set(s, TMP64_0_IDX); // EFGHABCD + tcg_wasm_out_op_global_get_r(s, src); // ABCDEFGH + tcg_wasm_out_op_i64_const(s, 32); + tcg_wasm_out_op_i64_rotr(s); + tcg_wasm_out_op_local_set(s, TMP64_0_IDX); // EFGHABCD - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_i64_const(s, 0xf000f000); - tcg_out_op_i64_and(s); - tcg_out_op_i64_const(s, 12); - tcg_out_op_i64_shr_u(s); // ___E___A + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_i64_const(s, 0xff000000ff000000); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_const(s, 24); + tcg_wasm_out_op_i64_shr_u(s); // ___E___A - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_i64_const(s, 0x0f000f00); - tcg_out_op_i64_and(s); - tcg_out_op_i64_const(s, 4); - tcg_out_op_i64_shr_u(s); // __F___B_ + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_i64_const(s, 0x00ff000000ff0000); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_const(s, 8); + tcg_wasm_out_op_i64_shr_u(s); // __F___B_ - tcg_out_op_i64_or(s); + tcg_wasm_out_op_i64_or(s); - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_i64_const(s, 0x00f000f0); - tcg_out_op_i64_and(s); - tcg_out_op_i64_const(s, 4); - tcg_out_op_i64_shl(s); // _G___C__ + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_i64_const(s, 0x0000ff000000ff00); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_const(s, 8); + tcg_wasm_out_op_i64_shl(s); // _G___C__ - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_i64_const(s, 0x000f000f); - tcg_out_op_i64_and(s); - tcg_out_op_i64_const(s, 12); - tcg_out_op_i64_shl(s); // H___D___ + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_i64_const(s, 0x000000ff000000ff); + tcg_wasm_out_op_i64_and(s); + tcg_wasm_out_op_i64_const(s, 24); + tcg_wasm_out_op_i64_shl(s); // H___D___ - tcg_out_op_i64_or(s); + tcg_wasm_out_op_i64_or(s); - tcg_out_op_i64_or(s); // HGFEDCBA - tcg_out_op_global_set_r(s, dest); + tcg_wasm_out_op_i64_or(s); // HGFEDCBA + tcg_wasm_out_op_global_set_r(s, dest); } -static void tcg_out_bswap32(TCGContext *s, TCGReg dest, TCGReg src, int flags) +static void tcg_wasm_out_bswap32(TCGContext *s, TCGReg dest, TCGReg src, int flags) { - tcg_out_op_global_get_r(s, src); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_local_set(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_global_get_r(s, src); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_local_set(s, TMP32_LOCAL_0_IDX); - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); // ABCD - tcg_out_op_i32_const(s, 16); - tcg_out_op_i32_rotr(s); - tcg_out_op_local_set(s, TMP32_LOCAL_0_IDX); // CDAB + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); // ABCD + tcg_wasm_out_op_i32_const(s, 16); + tcg_wasm_out_op_i32_rotr(s); + tcg_wasm_out_op_local_set(s, TMP32_LOCAL_0_IDX); // CDAB - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); - tcg_out_op_i32_const(s, 0xff00ff00); - tcg_out_op_i32_and(s); - tcg_out_op_i32_const(s, 8); - tcg_out_op_i32_shr_u(s); // _C_A + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_i32_const(s, 0xff00ff00); + tcg_wasm_out_op_i32_and(s); + tcg_wasm_out_op_i32_const(s, 8); + tcg_wasm_out_op_i32_shr_u(s); // _C_A - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); - tcg_out_op_i32_const(s, 0x00ff00ff); - tcg_out_op_i32_and(s); - tcg_out_op_i32_const(s, 8); - tcg_out_op_i32_shl(s); // D_B_ + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_i32_const(s, 0x00ff00ff); + tcg_wasm_out_op_i32_and(s); + tcg_wasm_out_op_i32_const(s, 8); + tcg_wasm_out_op_i32_shl(s); // D_B_ - tcg_out_op_i32_or(s); // DCBA - tcg_out_op_i64_extend_i32_u(s); - tcg_out_op_global_set_r(s, dest); + tcg_wasm_out_op_i32_or(s); // DCBA + tcg_wasm_out_op_i64_extend_i32_u(s); + tcg_wasm_out_op_global_set_r(s, dest); } -static void tcg_out_bswap16(TCGContext *s, TCGReg dest, TCGReg src, int flags) +static void tcg_wasm_out_bswap16(TCGContext *s, TCGReg dest, TCGReg src, int flags) { - tcg_out_op_global_get_r(s, src); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_local_set(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_global_get_r(s, src); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_local_set(s, TMP32_LOCAL_0_IDX); - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); // __AB - tcg_out_op_i32_const(s, 8); - tcg_out_op_i32_rotr(s); - tcg_out_op_local_set(s, TMP32_LOCAL_0_IDX); // B__A + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); // __AB + tcg_wasm_out_op_i32_const(s, 8); + tcg_wasm_out_op_i32_rotr(s); + tcg_wasm_out_op_local_set(s, TMP32_LOCAL_0_IDX); // B__A - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); - tcg_out_op_i32_const(s, 0x000000ff); - tcg_out_op_i32_and(s); // ___A + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_i32_const(s, 0x000000ff); + tcg_wasm_out_op_i32_and(s); // ___A - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); - tcg_out_op_i32_const(s, 0xff000000); - tcg_out_op_i32_and(s); - tcg_out_op_i32_const(s, 24); + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_i32_const(s, 0xff000000); + tcg_wasm_out_op_i32_and(s); + tcg_wasm_out_op_i32_const(s, 16); if (flags & TCG_BSWAP_OS) { - tcg_out_op_i32_shr_s(s); // SSB_ + tcg_wasm_out_op_i32_shr_s(s); // SSB_ } else { - tcg_out_op_i32_shr_u(s); // 00B_ + tcg_wasm_out_op_i32_shr_u(s); // 00B_ } - tcg_out_op_i32_or(s); // **BA - tcg_out_op_i64_extend_i32_u(s); - tcg_out_op_global_set_r(s, dest); + tcg_wasm_out_op_i32_or(s); // **BA + tcg_wasm_out_op_i64_extend_i32_u(s); + tcg_wasm_out_op_global_set_r(s, dest); } -static void tcg_out_ctx_i32_store_const(TCGContext *s, int off, int32_t v) +static void tcg_wasm_out_ctx_i32_store_const(TCGContext *s, int off, int32_t v) { - tcg_out_op_local_get(s, CTX_IDX); - tcg_out_op_i32_const(s, v); - tcg_out_op_i32_store(s, 0, off); + tcg_wasm_out_op_local_get(s, CTX_IDX); + tcg_wasm_out_op_i32_const(s, v); + tcg_wasm_out_op_i32_store(s, 0, off); } -static void tcg_out_ctx_i32_store_r(TCGContext *s, int off, TCGReg r0) +static void tcg_wasm_out_ctx_i32_store_r(TCGContext *s, int off, TCGReg r0) { - tcg_out_op_local_get(s, CTX_IDX); - tcg_out_op_global_get_r(s, r0); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i32_store(s, 0, off); + tcg_wasm_out_op_local_get(s, CTX_IDX); + tcg_wasm_out_op_global_get_r(s, r0); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i32_store(s, 0, off); } -static void tcg_out_ctx_i32_load(TCGContext *s, int off) +static void tcg_wasm_out_ctx_i32_load(TCGContext *s, int off) { - tcg_out_op_local_get(s, CTX_IDX); - tcg_out_op_i32_load(s, 0, off); + tcg_wasm_out_op_local_get(s, CTX_IDX); + tcg_wasm_out_op_i32_load(s, 0, off); } -static void tcg_out_label_idx(TCGContext *s, int label) +static void tcg_wasm_out_label_idx(TCGContext *s, int label) { int block_idx = wasm_alloc_block_idx(s); wasm_add_label_context(s, label, block_idx); - tcg_out_op_end(s); // end if of the previous block + tcg_wasm_out_op_end(s); // end if of the previous block // following block - tcg_out_op_global_get(s, BLOCK_PTR_IDX); - tcg_out_op_i64_const(s, block_idx); - tcg_out_op_i64_le_u(s); - tcg_out_op_if_noret(s); + tcg_wasm_out_op_global_get(s, BLOCK_PTR_IDX); + tcg_wasm_out_op_i64_const(s, block_idx); + tcg_wasm_out_op_i64_le_u(s); + tcg_wasm_out_op_if_noret(s); env_cached = false; } @@ -1619,24 +1751,24 @@ static void tcg_out_label_cb(TCGContext *s, TCGLabel *l) { current_label[current_label_pos++] = l->id; tcg_debug_assert(current_label_pos < 100); - tcg_out_label_idx(s, l->id + 1); + tcg_wasm_out_label_idx(s, l->id + 1); } -static void tcg_out_op_br_to_label(TCGContext *s, TCGLabel *l, bool br_if) +static void tcg_wasm_out_op_br_to_label(TCGContext *s, TCGLabel *l, bool br_if) { int toploop_depth = 1; if (br_if) { - tcg_out_op_if_noret(s); + tcg_wasm_out_op_if_noret(s); toploop_depth++; } - tcg_out8(s, 0x42); // i64.const - wasm_add_label_block_ptr_placeholder(s, l->id + 1); - tcg_out8(s, 0x80); // filled before instantiation - tcg_out8(s, 0x80); - tcg_out8(s, 0x80); - tcg_out8(s, 0x80); - tcg_out8(s, 0x00); - tcg_out_op_global_set(s, BLOCK_PTR_IDX); + tcg_wasm_out8(s, 0x42); // i64.const + wasm_add_label_block_ptr_placeholder(l->id + 1, cur_wasm_ptr(s)); + tcg_wasm_out8(s, 0x80); // filled before instantiation + tcg_wasm_out8(s, 0x80); + tcg_wasm_out8(s, 0x80); + tcg_wasm_out8(s, 0x80); + tcg_wasm_out8(s, 0x00); + tcg_wasm_out_op_global_set(s, BLOCK_PTR_IDX); bool found = false; for (int i = 0; i < current_label_pos; i++) { if (current_label[i] == l->id) { @@ -1645,93 +1777,99 @@ static void tcg_out_op_br_to_label(TCGContext *s, TCGLabel *l, bool br_if) } } if (found) { - tcg_out_op_br(s, toploop_depth); // br to the top of loop + tcg_wasm_out_op_br(s, toploop_depth); // br to the top of loop } else { - tcg_out_op_br(s, toploop_depth - 1); // br to the end of the current block + tcg_wasm_out_op_br(s, toploop_depth - 1); // br to the end of the current block } if (br_if) { - tcg_out_op_end(s); + tcg_wasm_out_op_end(s); } } -static void tcg_out_br(TCGContext *s, TCGLabel *l) +static void tcg_wasm_out_br(TCGContext *s, TCGLabel *l) { - tcg_out_op_br_to_label(s, l, false); + tcg_wasm_out_op_br_to_label(s, l, false); } -static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, +static void tcg_wasm_out_brcond_i32(TCGContext *s, TCGCond cond, TCGReg arg1, TCGReg arg2, TCGLabel *l) { - tcg_out_op_cond_i64(s, cond, arg1, arg2); - tcg_out_op_br_to_label(s, l, true); + tcg_wasm_out_op_cond_i32(s, cond, arg1, arg2); + tcg_wasm_out_op_br_to_label(s, l, true); } -static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) +static void tcg_wasm_out_brcond_i64(TCGContext *s, TCGCond cond, TCGReg arg1, + TCGReg arg2, TCGLabel *l) { - tcg_out_ctx_i32_store_const(s, TB_PTR_OFF, 0); - tcg_out_op_i32_const(s, (int32_t)arg); - tcg_out_op_return(s); + tcg_wasm_out_op_cond_i64(s, cond, arg1, arg2); + tcg_wasm_out_op_br_to_label(s, l, true); } -static void tcg_out_goto_ptr(TCGContext *s, TCGReg arg) +static void tcg_wasm_out_exit_tb(TCGContext *s, uintptr_t arg) { - tcg_out_op_global_get_r(s, arg); - tcg_out_op_i32_wrap_i64(s); - tcg_out_ctx_i32_load(s, TB_PTR_OFF); - tcg_out_op_i32_eq(s); - tcg_out_op_if_noret(s); - tcg_out_op_i64_const(s, 0); - tcg_out_op_global_set(s, BLOCK_PTR_IDX); - tcg_out_op_br(s, 2); // br to the top of loop - tcg_out_op_end(s); - - tcg_out_ctx_i32_store_r(s, TB_PTR_OFF, arg); - tcg_out_ctx_i32_store_const(s, DO_INIT_OFF, 1); - tcg_out_op_i32_const(s, 0); - tcg_out_op_return(s); + tcg_wasm_out_ctx_i32_store_const(s, TB_PTR_OFF, 0); + tcg_wasm_out_op_i32_const(s, (int32_t)arg); + tcg_wasm_out_op_return(s); } -static void tcg_out_goto_tb(TCGContext *s, int which) +static void tcg_wasm_out_goto_ptr(TCGContext *s, TCGReg arg) { - tcg_out_op_i32_const(s, (int32_t)get_jmp_target_addr(s, which)); - tcg_out_op_i32_load(s, 0, 0); - tcg_out_op_local_set(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_global_get_r(s, arg); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_ctx_i32_load(s, TB_PTR_OFF); + tcg_wasm_out_op_i32_eq(s); + tcg_wasm_out_op_if_noret(s); + tcg_wasm_out_op_i64_const(s, 0); + tcg_wasm_out_op_global_set(s, BLOCK_PTR_IDX); + tcg_wasm_out_op_br(s, 2); // br to the top of loop + tcg_wasm_out_op_end(s); - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); - tcg_out_op_i32_const(s, 0); - tcg_out_op_i32_ne(s); - tcg_out_op_if_noret(s); + tcg_wasm_out_ctx_i32_store_r(s, TB_PTR_OFF, arg); + tcg_wasm_out_ctx_i32_store_const(s, DO_INIT_OFF, 1); + tcg_wasm_out_op_i32_const(s, 0); + tcg_wasm_out_op_return(s); +} - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); - tcg_out_ctx_i32_load(s, TB_PTR_OFF); - tcg_out_op_i32_eq(s); - tcg_out_op_if_noret(s); - tcg_out_op_i64_const(s, 0); - tcg_out_op_global_set(s, BLOCK_PTR_IDX); - tcg_out_op_br(s, 3); // br to the top of loop - tcg_out_op_end(s); +static void tcg_wasm_out_goto_tb(TCGContext *s, int which) +{ + tcg_wasm_out_op_i32_const(s, (int32_t)get_jmp_target_addr(s, which)); + tcg_wasm_out_op_i32_load(s, 0, 0); + tcg_wasm_out_op_local_set(s, TMP32_LOCAL_0_IDX); + + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_i32_const(s, 0); + tcg_wasm_out_op_i32_ne(s); + tcg_wasm_out_op_if_noret(s); + + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_ctx_i32_load(s, TB_PTR_OFF); + tcg_wasm_out_op_i32_eq(s); + tcg_wasm_out_op_if_noret(s); + tcg_wasm_out_op_i64_const(s, 0); + tcg_wasm_out_op_global_set(s, BLOCK_PTR_IDX); + tcg_wasm_out_op_br(s, 3); // br to the top of loop + tcg_wasm_out_op_end(s); // store jmp target address to buf - tcg_out_op_local_get(s, CTX_IDX); - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); - tcg_out_op_i32_store(s, 0, TB_PTR_OFF); - tcg_out_ctx_i32_store_const(s, DO_INIT_OFF, 1); + tcg_wasm_out_op_local_get(s, CTX_IDX); + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_i32_store(s, 0, TB_PTR_OFF); + tcg_wasm_out_ctx_i32_store_const(s, DO_INIT_OFF, 1); - tcg_out_op_i32_const(s, 0); - tcg_out_op_return(s); - tcg_out_op_end(s); - - set_jmp_reset_offset(s, which); + tcg_wasm_out_op_i32_const(s, 0); + tcg_wasm_out_op_return(s); + tcg_wasm_out_op_end(s); } static void push_arg_i64(TCGContext *s, int *reg_idx, int *stack_offset) { if (*reg_idx < NUM_OF_IARG_REGS) { - tcg_out_op_global_get_r(s, REG_INDEX_IARG_BASE + *reg_idx); // arg register + tcg_wasm_out_op_global_get_r(s, REG_INDEX_IARG_BASE + *reg_idx); // arg register int addend = 1; *reg_idx = *reg_idx + addend; } else { - tcg_out_op_global_get_r(s, TCG_REG_CALL_STACK); - tcg_out_op_i64_load(s, 0, *stack_offset); + tcg_wasm_out_op_global_get_r(s, TCG_REG_CALL_STACK); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load(s, 0, *stack_offset); int addend = 8; *stack_offset = *stack_offset + addend; } @@ -1745,7 +1883,8 @@ void gen_func_wrapper_code(TCGContext *s, const tcg_insn_unit *func, const TCGHe if (rettype == dh_typecode_i128) { // receive 128bit return value via the stack buffer - tcg_out_op_global_get_r(s, TCG_REG_CALL_STACK); + tcg_wasm_out_op_global_get_r(s, TCG_REG_CALL_STACK); + tcg_wasm_out_op_i32_wrap_i64(s); } nargs = 32 - clz32(typemask >> 3); @@ -1753,7 +1892,7 @@ void gen_func_wrapper_code(TCGContext *s, const tcg_insn_unit *func, const TCGHe int stack_offset = 0; int reg_idx = 0; int stack128_base = 0; - bool cached_128base = true; + bool cached_128base = false; for (int j = 0; j < nargs; ++j) { int typecode = extract32(typemask, (j + 1) * 3, 3); if (typecode == dh_typecode_void) { @@ -1764,36 +1903,37 @@ void gen_func_wrapper_code(TCGContext *s, const tcg_insn_unit *func, const TCGHe case dh_typecode_s32: case dh_typecode_ptr: push_arg_i64(s, ®_idx, &stack_offset); - tcg_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i32_wrap_i64(s); break; case dh_typecode_i64: case dh_typecode_s64: push_arg_i64(s, ®_idx, &stack_offset); break; case dh_typecode_i128: - // push current 128stack pointer - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_i32_const(s, stack128_base); - tcg_out_op_i32_add(s); - // copy data to 128stack if (!cached_128base) { - tcg_out_ctx_i32_load(s, STACK128_OFF); - tcg_out_op_i64_extend_i32_s(s); - tcg_out_op_local_set(s, TMP64_0_IDX); + tcg_wasm_out_ctx_i32_load(s, STACK128_OFF); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_local_set(s, TMP64_0_IDX); cached_128base = true; } - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_i32_wrap_i64(s); + // push current 128stack pointer + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i32_const(s, stack128_base); + tcg_wasm_out_op_i32_add(s); + + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_i32_wrap_i64(s); push_arg_i64(s, ®_idx, &stack_offset); - tcg_out_op_i64_store(s, 0, stack128_base); + tcg_wasm_out_op_i64_store(s, 0, stack128_base); stack128_base += 8; - tcg_out_op_local_get(s, TMP64_0_IDX); - tcg_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_local_get(s, TMP64_0_IDX); + tcg_wasm_out_op_i32_wrap_i64(s); push_arg_i64(s, ®_idx, &stack_offset); - tcg_out_op_i64_store(s, 0, stack128_base); + tcg_wasm_out_op_i64_store(s, 0, stack128_base); stack128_base += 8; break; default: @@ -1801,7 +1941,7 @@ void gen_func_wrapper_code(TCGContext *s, const tcg_insn_unit *func, const TCGHe } } - tcg_out_op_call(s, func_idx); + tcg_wasm_out_op_call(s, func_idx); stack_offset = 0; if (rettype != dh_typecode_void) { @@ -1809,22 +1949,24 @@ void gen_func_wrapper_code(TCGContext *s, const tcg_insn_unit *func, const TCGHe case dh_typecode_i32: case dh_typecode_s32: case dh_typecode_ptr: - tcg_out_op_i64_extend_i32_s(s); - tcg_out_op_global_set_r(s, TCG_REG_A0); + tcg_wasm_out_op_i64_extend_i32_s(s); + tcg_wasm_out_op_global_set_r(s, TCG_REG_R0); break; case dh_typecode_i64: case dh_typecode_s64: - tcg_out_op_global_set_r(s, TCG_REG_A0); + tcg_wasm_out_op_global_set_r(s, TCG_REG_R0); break; case dh_typecode_i128: - tcg_out_op_global_get_r(s, TCG_REG_CALL_STACK); - tcg_out_op_i64_load(s, 0, stack_offset); - tcg_out_op_global_set_r(s, TCG_REG_A0); + tcg_wasm_out_op_global_get_r(s, TCG_REG_CALL_STACK); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load(s, 0, stack_offset); + tcg_wasm_out_op_global_set_r(s, TCG_REG_R0); stack_offset += 8; - - tcg_out_op_global_get_r(s, TCG_REG_CALL_STACK); - tcg_out_op_i64_load(s, 0, stack_offset); - tcg_out_op_global_set_r(s, TCG_REG_A1); + + tcg_wasm_out_op_global_get_r(s, TCG_REG_CALL_STACK); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load(s, 0, stack_offset); + tcg_wasm_out_op_global_set_r(s, TCG_REG_R1); stack_offset += 8; break; default: @@ -1959,13 +2101,14 @@ static void gen_func_type_qemu_st(TCGContext *s, uint32_t oi) wasm_add_helper_types_pos(s, sz); } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func, +static void tcg_wasm_out_call(TCGContext *s, const tcg_insn_unit *func, const TCGHelperInfo *info) { // set return position - tcg_out_ctx_i32_load(s, HELPER_RET_TB_PTR_OFF); - tcg_out_op_i32_const(s, (int32_t)s->code_buf + tcg_current_code_size(s)); - tcg_out_op_i32_store(s, 0, 0); + tcg_wasm_out_ctx_i32_load(s, HELPER_RET_TB_PTR_OFF); + tcg_wasm_out_op_i32_const(s, (int32_t)s->code_ptr); + + tcg_wasm_out_op_i32_store(s, 0, 0); int func_idx = get_wasm_helper_idx(s, (int)func); if (func_idx < 0) { @@ -1976,26 +2119,26 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func, } int target_block_idx = wasm_block_current_idx(s) + 1; - tcg_out_op_i64_const(s, target_block_idx); - tcg_out_op_global_set(s, BLOCK_PTR_IDX); - tcg_out_op_end(s); // close if of this block (rewind skips this) + tcg_wasm_out_op_i64_const(s, target_block_idx); + tcg_wasm_out_op_global_set(s, BLOCK_PTR_IDX); + tcg_wasm_out_op_end(s); // close if of this block (rewind skips this) env_cached = false; int block_idx = wasm_alloc_block_idx(s); - tcg_out_op_global_get(s, BLOCK_PTR_IDX); - tcg_out_op_i64_const(s, block_idx); - tcg_out_op_i64_le_u(s); - tcg_out_op_if_noret(s); - tcg_out_ctx_i32_store_const(s, UNWINDING_OFF, 0); + tcg_wasm_out_op_global_get(s, BLOCK_PTR_IDX); + tcg_wasm_out_op_i64_const(s, block_idx); + tcg_wasm_out_op_i64_le_u(s); + tcg_wasm_out_op_if_noret(s); + tcg_wasm_out_ctx_i32_store_const(s, UNWINDING_OFF, 0); gen_func_wrapper_code(s, func, info, func_idx); - tcg_out_op_i32_const(s, 1); - tcg_out_ctx_i32_load(s, UNWINDING_OFF); - tcg_out_op_i32_eq(s); - tcg_out_op_if_noret(s); - tcg_out_op_i32_const(s, 0); - tcg_out_op_return(s); - tcg_out_op_end(s); + tcg_wasm_out_op_i32_const(s, 1); + tcg_wasm_out_ctx_i32_load(s, UNWINDING_OFF); + tcg_wasm_out_op_i32_eq(s); + tcg_wasm_out_op_if_noret(s); + tcg_wasm_out_op_i32_const(s, 0); + tcg_wasm_out_op_return(s); + tcg_wasm_out_op_end(s); } void tb_target_set_jmp_target(const TranslationBlock *tb, int n, @@ -2004,7 +2147,7 @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, /* Always indirect, nothing to do */ } -static uint8_t tcg_out_tlb_load(TCGContext *s, TCGReg addr, MemOpIdx oi, bool is_ld) +static uint8_t tcg_wasm_out_tlb_load(TCGContext *s, TCGReg addr, MemOpIdx oi, bool is_ld) { MemOp opc = get_memop(oi); TCGAtomAlign aa; @@ -2019,124 +2162,130 @@ static uint8_t tcg_out_tlb_load(TCGContext *s, TCGReg addr, MemOpIdx oi, bool is int mem_index = get_mmuidx(oi); int fast_ofs = tlb_mask_table_ofs(s, mem_index); int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); + int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); int add_off = offsetof(CPUTLBEntry, addend); - tcg_out_op_global_get_r(s, addr); - tcg_out_op_i64_const(s, s->page_bits - CPU_TLB_ENTRY_BITS); - tcg_out_op_i64_shr_u(s); + tcg_wasm_out_op_global_get_r(s, addr); + tcg_wasm_out_op_i64_const(s, s->page_bits - CPU_TLB_ENTRY_BITS); + tcg_wasm_out_op_i64_shr_u(s); - tcg_out_op_global_get_r_i32(s, TCG_AREG0); + tcg_wasm_out_op_global_get_r_i32(s, TCG_AREG0); int off = mask_ofs; if ((int64_t)off < 0) { - tcg_out_op_i32_const(s, (int64_t)off); - tcg_out_op_i32_add(s); + tcg_wasm_out_op_i32_const(s, (int64_t)off); + tcg_wasm_out_op_i32_add(s); off = 0; } - tcg_out_op_i64_load(s, 0, (uint64_t)off); - tcg_out_op_local_set(s, TMP64_2_IDX); + tcg_wasm_out_op_i64_load(s, 0, (uint64_t)off); + tcg_wasm_out_op_local_set(s, TMP64_2_IDX); - tcg_out_op_local_get(s, TMP64_2_IDX); + tcg_wasm_out_op_local_get(s, TMP64_2_IDX); - tcg_out_op_i64_and(s); + tcg_wasm_out_op_i64_and(s); - tcg_out_op_local_get(s, TMP64_2_IDX); - tcg_out_op_i64_const(s, 32); - tcg_out_op_i64_shr_u(s); - tcg_out_op_i64_add(s); + tcg_wasm_out_op_global_get_r_i32(s, TCG_AREG0); + off = table_ofs; + if ((int64_t)off < 0) { + tcg_wasm_out_op_i32_const(s, (int64_t)off); + tcg_wasm_out_op_i32_add(s); + off = 0; + } + tcg_wasm_out_op_i64_load(s, 0, (uint64_t)off); + tcg_wasm_out_op_i64_add(s); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_local_tee(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_local_tee(s, TMP32_LOCAL_0_IDX); off = is_ld ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write); if ((int64_t)off < 0) { - tcg_out_op_i32_const(s, (int64_t)off); - tcg_out_op_i32_add(s); + tcg_wasm_out_op_i32_const(s, (int64_t)off); + tcg_wasm_out_op_i32_add(s); off = 0; } - tcg_out_op_i64_load(s, 0, (uint64_t)off); + tcg_wasm_out_op_i64_load(s, 0, (uint64_t)off); - tcg_out_op_global_get_r(s, addr); + tcg_wasm_out_op_global_get_r(s, addr); if (a_mask < s_mask) { - tcg_out_op_i64_const(s, s_mask - a_mask); - tcg_out_op_i64_add(s); + tcg_wasm_out_op_i64_const(s, s_mask - a_mask); + tcg_wasm_out_op_i64_add(s); } compare_mask = (uint64_t)s->page_mask | a_mask; - tcg_out_op_i64_const(s, compare_mask); - tcg_out_op_i64_and(s); + tcg_wasm_out_op_i64_const(s, compare_mask); + tcg_wasm_out_op_i64_and(s); - tcg_out_op_i64_eq(s); + tcg_wasm_out_op_i64_eq(s); - tcg_out_op_i64_const(s, 0); - tcg_out_op_local_set(s, TMP64_0_IDX); + tcg_wasm_out_op_i64_const(s, 0); + tcg_wasm_out_op_local_set(s, TMP64_0_IDX); - tcg_out_op_if_noret(s); - tcg_out_op_local_get(s, TMP32_LOCAL_0_IDX); + tcg_wasm_out_op_if_noret(s); + tcg_wasm_out_op_local_get(s, TMP32_LOCAL_0_IDX); off = add_off; if ((int64_t)off < 0) { - tcg_out_op_i32_const(s, (int64_t)off); - tcg_out_op_i32_add(s); + tcg_wasm_out_op_i32_const(s, (int64_t)off); + tcg_wasm_out_op_i32_add(s); off = 0; } - tcg_out_op_i32_load(s, 0, (uint64_t)off); - tcg_out_op_i64_extend_i32_u(s); - tcg_out_op_global_get_r(s, addr); - tcg_out_op_i64_add(s); - tcg_out_op_local_set(s, TMP64_0_IDX); + tcg_wasm_out_op_i32_load(s, 0, (uint64_t)off); + tcg_wasm_out_op_i64_extend_i32_u(s); + tcg_wasm_out_op_global_get_r(s, addr); + tcg_wasm_out_op_i64_add(s); + tcg_wasm_out_op_local_set(s, TMP64_0_IDX); - tcg_out_op_end(s); + tcg_wasm_out_op_end(s); return TMP64_0_IDX; } -static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg r, uint8_t base, MemOp opc) +static void tcg_wasm_out_qemu_ld_direct(TCGContext *s, TCGReg r, uint8_t base, MemOp opc) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & (MO_SSIZE)) { case MO_UB: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i64_load8_u(s, 0, 0); - tcg_out_op_global_set_r(s, r); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load8_u(s, 0, 0); + tcg_wasm_out_op_global_set_r(s, r); break; case MO_SB: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i64_load8_s(s, 0, 0); - tcg_out_op_global_set_r(s, r); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load8_s(s, 0, 0); + tcg_wasm_out_op_global_set_r(s, r); break; case MO_UW: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i64_load16_u(s, 0, 0); - tcg_out_op_global_set_r(s, r); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load16_u(s, 0, 0); + tcg_wasm_out_op_global_set_r(s, r); break; case MO_SW: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i64_load16_s(s, 0, 0); - tcg_out_op_global_set_r(s, r); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load16_s(s, 0, 0); + tcg_wasm_out_op_global_set_r(s, r); break; case MO_UL: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i64_load32_u(s, 0, 0); - tcg_out_op_global_set_r(s, r); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load32_u(s, 0, 0); + tcg_wasm_out_op_global_set_r(s, r); break; case MO_SL: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i64_load32_s(s, 0, 0); - tcg_out_op_global_set_r(s, r); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load32_s(s, 0, 0); + tcg_wasm_out_op_global_set_r(s, r); break; case MO_UQ: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_i64_load(s, 0, 0); - tcg_out_op_global_set_r(s, r); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_i64_load(s, 0, 0); + tcg_wasm_out_op_global_set_r(s, r); break; default: g_assert_not_reached(); @@ -2166,7 +2315,7 @@ static void* qemu_ld_helper_ptr(uint32_t oi) } } -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_wasm_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg addr_reg; TCGReg data_reg; @@ -2178,11 +2327,11 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) oi = *args++; opc = get_memop(oi); - uint8_t base = tcg_out_tlb_load(s, addr_reg, oi, true); + uint8_t base = tcg_wasm_out_tlb_load(s, addr_reg, oi, true); - tcg_out_op_local_get(s, base); - tcg_out_op_i64_eqz(s); - tcg_out_op_if_noret(s); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i64_eqz(s); + tcg_wasm_out_op_if_noret(s); // path for miss case int helper_func_idx = (uint32_t)qemu_ld_helper_ptr(oi); @@ -2195,80 +2344,81 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) } // save function pointer - tcg_out_ctx_i32_store_const(s, DONE_FLAG_OFF, 0); + tcg_wasm_out_ctx_i32_store_const(s, DONE_FLAG_OFF, 0); - tcg_out_op_else(s); + tcg_wasm_out_op_else(s); // fast path - tcg_out_qemu_ld_direct(s, data_reg, base, opc); + tcg_wasm_out_qemu_ld_direct(s, data_reg, base, opc); - tcg_out_op_end(s); + tcg_wasm_out_op_end(s); int target_block_idx = wasm_block_current_idx(s) + 1; - tcg_out_op_i64_const(s, target_block_idx); - tcg_out_op_global_set(s, BLOCK_PTR_IDX); + tcg_wasm_out_op_i64_const(s, target_block_idx); + tcg_wasm_out_op_global_set(s, BLOCK_PTR_IDX); - tcg_out_op_end(s); // close if of this block (rewind skips this) + tcg_wasm_out_op_end(s); // close if of this block (rewind skips this) env_cached = false; // block for calling helper (+1) int block_idx = wasm_alloc_block_idx(s); - tcg_out_op_global_get(s, BLOCK_PTR_IDX); - tcg_out_op_i64_const(s, block_idx); - tcg_out_op_i64_le_u(s); - tcg_out_op_if_noret(s); + tcg_wasm_out_op_global_get(s, BLOCK_PTR_IDX); + tcg_wasm_out_op_i64_const(s, block_idx); + tcg_wasm_out_op_i64_le_u(s); + tcg_wasm_out_op_if_noret(s); - tcg_out_op_local_get(s, base); - tcg_out_op_i64_eqz(s); - tcg_out_op_if_noret(s); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i64_eqz(s); + tcg_wasm_out_op_if_noret(s); // call helper - tcg_out_op_global_get_r(s, TCG_AREG0); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_get_r(s, addr_reg); - tcg_out_op_i32_const(s, oi); - tcg_out_op_i32_const(s, (int32_t)s->code_buf + tcg_current_code_size(s)); - tcg_out_op_call(s, func_idx); - tcg_out_op_global_set_r(s, data_reg); - tcg_out_ctx_i32_load(s, DONE_FLAG_OFF); - tcg_out_op_i32_eqz(s); + tcg_wasm_out_op_global_get_r(s, TCG_AREG0); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, addr_reg); + tcg_wasm_out_op_i32_const(s, oi); + tcg_wasm_out_op_i32_const(s, (int32_t)s->code_ptr); - tcg_out_op_if_noret(s); - tcg_out_op_i32_const(s, 0); - tcg_out_op_return(s); - tcg_out_op_end(s); - tcg_out_op_end(s); + tcg_wasm_out_op_call(s, func_idx); + tcg_wasm_out_op_global_set_r(s, data_reg); + tcg_wasm_out_ctx_i32_load(s, DONE_FLAG_OFF); + tcg_wasm_out_op_i32_eqz(s); + + tcg_wasm_out_op_if_noret(s); + tcg_wasm_out_op_i32_const(s, 0); + tcg_wasm_out_op_return(s); + tcg_wasm_out_op_end(s); + tcg_wasm_out_op_end(s); } -static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, uint8_t base, MemOp opc) +static void tcg_wasm_out_qemu_st_direct(TCGContext *s, TCGReg lo, uint8_t base, MemOp opc) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & (MO_SSIZE)) { case MO_8: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_get_r(s, lo); - tcg_out_op_i64_store8(s, 0, 0); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, lo); + tcg_wasm_out_op_i64_store8(s, 0, 0); break; case MO_16: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_get_r(s, lo); - tcg_out_op_i64_store16(s, 0, 0); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, lo); + tcg_wasm_out_op_i64_store16(s, 0, 0); break; case MO_32: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_get_r(s, lo); - tcg_out_op_i64_store32(s, 0, 0); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, lo); + tcg_wasm_out_op_i64_store32(s, 0, 0); break; case MO_64: - tcg_out_op_local_get(s, base); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_get_r(s, lo); - tcg_out_op_i64_store(s, 0, 0); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, lo); + tcg_wasm_out_op_i64_store(s, 0, 0); break; default: g_assert_not_reached(); @@ -2279,20 +2429,22 @@ static void* qemu_st_helper_ptr(uint32_t oi) { MemOp mop = get_memop(oi); switch (mop & MO_SIZE) { - case MO_UB: + case MO_8: return helper_stb_mmu; - case MO_UW: + case MO_16: return helper_stw_mmu; - case MO_UL: + case MO_32: return helper_stl_mmu; - case MO_UQ: + case MO_64: return helper_stq_mmu; + case MO_128: + return helper_st16_mmu; default: g_assert_not_reached(); } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_wasm_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg addr_reg; TCGReg data_reg; @@ -2304,11 +2456,11 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) oi = *args++; opc = get_memop(oi); - uint8_t base = tcg_out_tlb_load(s, addr_reg, oi, false); + uint8_t base = tcg_wasm_out_tlb_load(s, addr_reg, oi, false); - tcg_out_op_local_get(s, base); - tcg_out_op_i64_eqz(s); - tcg_out_op_if_noret(s); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i64_eqz(s); + tcg_wasm_out_op_if_noret(s); // path for miss case int helper_func_idx = (uint32_t)qemu_st_helper_ptr(oi); @@ -2321,58 +2473,1030 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) } // save function pointer - tcg_out_ctx_i32_store_const(s, DONE_FLAG_OFF, 0); + tcg_wasm_out_ctx_i32_store_const(s, DONE_FLAG_OFF, 0); - tcg_out_op_else(s); + tcg_wasm_out_op_else(s); // fast path - tcg_out_qemu_st_direct(s, data_reg, base, opc); + tcg_wasm_out_qemu_st_direct(s, data_reg, base, opc); - tcg_out_op_end(s); + tcg_wasm_out_op_end(s); int target_block_idx = wasm_block_current_idx(s) + 1; - tcg_out_op_i64_const(s, target_block_idx); - tcg_out_op_global_set(s, BLOCK_PTR_IDX); + tcg_wasm_out_op_i64_const(s, target_block_idx); + tcg_wasm_out_op_global_set(s, BLOCK_PTR_IDX); - tcg_out_op_end(s); // close if of this block (rewind skips this) + tcg_wasm_out_op_end(s); // close if of this block (rewind skips this) env_cached = false; // block for calling helper (+1) int block_idx = wasm_alloc_block_idx(s); - tcg_out_op_global_get(s, BLOCK_PTR_IDX); - tcg_out_op_i64_const(s, block_idx); - tcg_out_op_i64_le_u(s); - tcg_out_op_if_noret(s); + tcg_wasm_out_op_global_get(s, BLOCK_PTR_IDX); + tcg_wasm_out_op_i64_const(s, block_idx); + tcg_wasm_out_op_i64_le_u(s); + tcg_wasm_out_op_if_noret(s); - tcg_out_op_local_get(s, base); - tcg_out_op_i64_eqz(s); - tcg_out_op_if_noret(s); + tcg_wasm_out_op_local_get(s, base); + tcg_wasm_out_op_i64_eqz(s); + tcg_wasm_out_op_if_noret(s); // call helper - tcg_out_op_global_get_r(s, TCG_AREG0); - tcg_out_op_i32_wrap_i64(s); - tcg_out_op_global_get_r(s, addr_reg); + tcg_wasm_out_op_global_get_r(s, TCG_AREG0); + tcg_wasm_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, addr_reg); MemOp mop = get_memop(oi); switch (mop & MO_SSIZE) { case MO_UQ: - tcg_out_op_global_get_r(s, data_reg); + tcg_wasm_out_op_global_get_r(s, data_reg); break; default: - tcg_out_op_global_get_r(s, data_reg); - tcg_out_op_i32_wrap_i64(s); + tcg_wasm_out_op_global_get_r(s, data_reg); + tcg_wasm_out_op_i32_wrap_i64(s); break; } - tcg_out_op_i32_const(s, oi); - tcg_out_op_i32_const(s, (int32_t)s->code_buf + tcg_current_code_size(s)); - tcg_out_op_call(s, func_idx); + tcg_wasm_out_op_i32_const(s, oi); + tcg_wasm_out_op_i32_const(s, (int32_t)s->code_ptr); - tcg_out_ctx_i32_load(s, DONE_FLAG_OFF); - tcg_out_op_i32_eqz(s); - tcg_out_op_if_noret(s); - tcg_out_op_i32_const(s, 0); - tcg_out_op_return(s); - tcg_out_op_end(s); - tcg_out_op_end(s); + tcg_wasm_out_op_call(s, func_idx); + + tcg_wasm_out_ctx_i32_load(s, DONE_FLAG_OFF); + tcg_wasm_out_op_i32_eqz(s); + tcg_wasm_out_op_if_noret(s); + tcg_wasm_out_op_i32_const(s, 0); + tcg_wasm_out_op_return(s); + tcg_wasm_out_op_end(s); + tcg_wasm_out_op_end(s); +} + +static bool patch_reloc(tcg_insn_unit *code_ptr_i, int type, + intptr_t value, intptr_t addend) +{ + int32_t *code_ptr = (int32_t*)code_ptr_i; + intptr_t diff = value - (intptr_t)(code_ptr + 1); + + tcg_debug_assert(addend == 0); + tcg_debug_assert(type == 20); + + if (diff == sextract32(diff, 0, type)) { + tcg_patch32(code_ptr, deposit32(*code_ptr, 32 - type, type, diff)); + return true; + } + return false; +} + +static void stack_bounds_check(TCGReg base, intptr_t offset) +{ + if (base == TCG_REG_CALL_STACK) { + tcg_debug_assert(offset >= 0); + tcg_debug_assert(offset < (TCG_STATIC_CALL_ARGS_SIZE + + TCG_STATIC_FRAME_SIZE)); + } +} + +static inline void tcg_tci_out32(TCGContext *s, uint32_t v) +{ + tcg_out32(s, v); +} + +static uintptr_t cur_tci_ptr(TCGContext *s) +{ + return (uintptr_t)s->code_ptr; +} + +static void tcg_tci_out_op_l(TCGContext *s, TCGOpcode op, TCGLabel *l0) +{ + uint32_t insn = 0; + + tcg_out_reloc(s, (void*)cur_tci_ptr(s), 20, l0, 0); + insn = deposit32(insn, 0, 8, op); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_p(TCGContext *s, TCGOpcode op, void *p0) +{ + uint32_t insn = 0; + intptr_t diff; + + /* Special case for exit_tb: map null -> 0. */ + if (p0 == NULL) { + diff = 0; + } else { + diff = p0 - (void *)((uint8_t*)cur_tci_ptr(s) + 4); + tcg_debug_assert(diff != 0); + if (diff != sextract32(diff, 0, 20)) { + tcg_raise_tb_overflow(s); + } + } + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 12, 20, diff); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_r(TCGContext *s, TCGOpcode op, TCGReg r0) +{ + uint32_t insn = 0; + + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_v(TCGContext *s, TCGOpcode op) +{ + tcg_tci_out32(s, (uint8_t)op); +} + +static void tcg_tci_out_op_ri(TCGContext *s, TCGOpcode op, TCGReg r0, int32_t i1) +{ + uint32_t insn = 0; + + tcg_debug_assert(i1 == sextract32(i1, 0, 20)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 20, i1); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rl(TCGContext *s, TCGOpcode op, TCGReg r0, TCGLabel *l1) +{ + uint32_t insn = 0; + + tcg_out_reloc(s, (void*)cur_tci_ptr(s), 20, l1, 0); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rr(TCGContext *s, TCGOpcode op, TCGReg r0, TCGReg r1) +{ + uint32_t insn = 0; + + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrm(TCGContext *s, TCGOpcode op, + TCGReg r0, TCGReg r1, TCGArg m2) +{ + uint32_t insn = 0; + + tcg_debug_assert(m2 == extract32(m2, 0, 16)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 16, m2); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrr(TCGContext *s, TCGOpcode op, + TCGReg r0, TCGReg r1, TCGReg r2) +{ + uint32_t insn = 0; + + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrs(TCGContext *s, TCGOpcode op, + TCGReg r0, TCGReg r1, intptr_t i2) +{ + uint32_t insn = 0; + + tcg_debug_assert(i2 == sextract32(i2, 0, 16)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 16, i2); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrbb(TCGContext *s, TCGOpcode op, TCGReg r0, + TCGReg r1, uint8_t b2, uint8_t b3) +{ + uint32_t insn = 0; + + tcg_debug_assert(b2 == extract32(b2, 0, 6)); + tcg_debug_assert(b3 == extract32(b3, 0, 6)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 6, b2); + insn = deposit32(insn, 22, 6, b3); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrrc(TCGContext *s, TCGOpcode op, + TCGReg r0, TCGReg r1, TCGReg r2, TCGCond c3) +{ + uint32_t insn = 0; + + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, c3); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrrbb(TCGContext *s, TCGOpcode op, TCGReg r0, + TCGReg r1, TCGReg r2, uint8_t b3, uint8_t b4) +{ + uint32_t insn = 0; + + tcg_debug_assert(b3 == extract32(b3, 0, 6)); + tcg_debug_assert(b4 == extract32(b4, 0, 6)); + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 6, b3); + insn = deposit32(insn, 26, 6, b4); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrrrr(TCGContext *s, TCGOpcode op, TCGReg r0, + TCGReg r1, TCGReg r2, TCGReg r3, TCGReg r4) +{ + uint32_t insn = 0; + + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, r3); + insn = deposit32(insn, 24, 4, r4); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrrr(TCGContext *s, TCGOpcode op, + TCGReg r0, TCGReg r1, TCGReg r2, TCGReg r3) +{ + uint32_t insn = 0; + + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, r3); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrrrrc(TCGContext *s, TCGOpcode op, + TCGReg r0, TCGReg r1, TCGReg r2, + TCGReg r3, TCGReg r4, TCGCond c5) +{ + uint32_t insn = 0; + + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, r3); + insn = deposit32(insn, 24, 4, r4); + insn = deposit32(insn, 28, 4, c5); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_op_rrrrrr(TCGContext *s, TCGOpcode op, + TCGReg r0, TCGReg r1, TCGReg r2, + TCGReg r3, TCGReg r4, TCGReg r5) +{ + uint32_t insn = 0; + + insn = deposit32(insn, 0, 8, op); + insn = deposit32(insn, 8, 4, r0); + insn = deposit32(insn, 12, 4, r1); + insn = deposit32(insn, 16, 4, r2); + insn = deposit32(insn, 20, 4, r3); + insn = deposit32(insn, 24, 4, r4); + insn = deposit32(insn, 28, 4, r5); + tcg_tci_out32(s, insn); +} + +static void tcg_tci_out_movi(TCGContext *s, TCGType type, + TCGReg ret, tcg_target_long arg) +{ + switch (type) { + case TCG_TYPE_I32: +#if TCG_TARGET_REG_BITS == 64 + arg = (int32_t)arg; + /* fall through */ + case TCG_TYPE_I64: +#endif + break; + default: + g_assert_not_reached(); + } + + if (arg == sextract32(arg, 0, 20)) { + tcg_tci_out_op_ri(s, INDEX_op_tci_movi, ret, arg); + } else { + uint32_t insn = 0; + + new_pool_label(s, arg, 20, (void*)cur_tci_ptr(s), 0); + insn = deposit32(insn, 0, 8, INDEX_op_tci_movl); + insn = deposit32(insn, 8, 4, ret); + tcg_tci_out32(s, insn); + } +} + +static void tcg_tci_out_ldst(TCGContext *s, TCGOpcode op, TCGReg val, + TCGReg base, intptr_t offset) +{ + stack_bounds_check(base, offset); + if (offset != sextract32(offset, 0, 16)) { + tcg_tci_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, offset); + tcg_tci_out_op_rrr(s, (TCG_TARGET_REG_BITS == 32 + ? INDEX_op_add_i32 : INDEX_op_add_i64), + TCG_REG_TMP, TCG_REG_TMP, base); + base = TCG_REG_TMP; + offset = 0; + } + tcg_tci_out_op_rrs(s, op, val, base, offset); +} + +static bool tcg_tci_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) +{ + switch (type) { + case TCG_TYPE_I32: + tcg_tci_out_op_rr(s, INDEX_op_mov_i32, ret, arg); + break; +#if TCG_TARGET_REG_BITS == 64 + case TCG_TYPE_I64: + tcg_tci_out_op_rr(s, INDEX_op_mov_i64, ret, arg); + break; +#endif + default: + g_assert_not_reached(); + } + return true; +} + +static void tcg_tci_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) +{ + switch (type) { + case TCG_TYPE_I32: + tcg_debug_assert(TCG_TARGET_HAS_ext8s_i32); + tcg_tci_out_op_rr(s, INDEX_op_ext8s_i32, rd, rs); + break; +#if TCG_TARGET_REG_BITS == 64 + case TCG_TYPE_I64: + tcg_debug_assert(TCG_TARGET_HAS_ext8s_i64); + tcg_tci_out_op_rr(s, INDEX_op_ext8s_i64, rd, rs); + break; +#endif + default: + g_assert_not_reached(); + } +} + +static void tcg_tci_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs) +{ + if (TCG_TARGET_REG_BITS == 64) { + tcg_debug_assert(TCG_TARGET_HAS_ext8u_i64); + tcg_tci_out_op_rr(s, INDEX_op_ext8u_i64, rd, rs); + } else { + tcg_debug_assert(TCG_TARGET_HAS_ext8u_i32); + tcg_tci_out_op_rr(s, INDEX_op_ext8u_i32, rd, rs); + } +} + +static void tcg_tci_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) +{ + switch (type) { + case TCG_TYPE_I32: + tcg_debug_assert(TCG_TARGET_HAS_ext16s_i32); + tcg_tci_out_op_rr(s, INDEX_op_ext16s_i32, rd, rs); + break; +#if TCG_TARGET_REG_BITS == 64 + case TCG_TYPE_I64: + tcg_debug_assert(TCG_TARGET_HAS_ext16s_i64); + tcg_tci_out_op_rr(s, INDEX_op_ext16s_i64, rd, rs); + break; +#endif + default: + g_assert_not_reached(); + } +} + +static void tcg_tci_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs) +{ + if (TCG_TARGET_REG_BITS == 64) { + tcg_debug_assert(TCG_TARGET_HAS_ext16u_i64); + tcg_tci_out_op_rr(s, INDEX_op_ext16u_i64, rd, rs); + } else { + tcg_debug_assert(TCG_TARGET_HAS_ext16u_i32); + tcg_tci_out_op_rr(s, INDEX_op_ext16u_i32, rd, rs); + } +} + +static void tcg_tci_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs) +{ + tcg_debug_assert(TCG_TARGET_REG_BITS == 64); + tcg_debug_assert(TCG_TARGET_HAS_ext32s_i64); + tcg_tci_out_op_rr(s, INDEX_op_ext32s_i64, rd, rs); +} + +static void tcg_tci_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs) +{ + tcg_debug_assert(TCG_TARGET_REG_BITS == 64); + tcg_debug_assert(TCG_TARGET_HAS_ext32u_i64); + tcg_tci_out_op_rr(s, INDEX_op_ext32u_i64, rd, rs); +} + +static void tcg_tci_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs) +{ + tcg_tci_out_ext32s(s, rd, rs); +} + +static void tcg_tci_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs) +{ + tcg_tci_out_ext32u(s, rd, rs); +} + +static void tcg_tci_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs) +{ + tcg_debug_assert(TCG_TARGET_REG_BITS == 64); + tcg_tci_out_mov(s, TCG_TYPE_I32, rd, rs); +} + +static void tcg_tci_out_call(TCGContext *s, const tcg_insn_unit *func, + const TCGHelperInfo *info) +{ + ffi_cif *cif = info->cif; + uint32_t insn = 0; + uint8_t which; + + if (cif->rtype == &ffi_type_void) { + which = 0; + } else { + tcg_debug_assert(cif->rtype->size == 4 || + cif->rtype->size == 8 || + cif->rtype->size == 16); + which = ctz32(cif->rtype->size) - 1; + } + new_pool_l2(s, 20, (void*)cur_tci_ptr(s), 0, (uintptr_t)func, (uintptr_t)cif); + insn = deposit32(insn, 0, 8, INDEX_op_call); + insn = deposit32(insn, 8, 4, which); + tcg_tci_out32(s, insn); +} + +#if TCG_TARGET_REG_BITS == 64 +# define CASE_32_64(x) \ + case glue(glue(INDEX_op_, x), _i64): \ + case glue(glue(INDEX_op_, x), _i32): +# define CASE_64(x) \ + case glue(glue(INDEX_op_, x), _i64): +#else +# define CASE_32_64(x) \ + case glue(glue(INDEX_op_, x), _i32): +# define CASE_64(x) +#endif + +static void tcg_tci_out_exit_tb(TCGContext *s, uintptr_t arg) +{ + tcg_tci_out_op_p(s, INDEX_op_exit_tb, (void *)arg); +} + +static void tcg_tci_out_goto_tb(TCGContext *s, int which) +{ + /* indirect jump method. */ + tcg_tci_out_op_p(s, INDEX_op_goto_tb, (void *)get_jmp_target_addr(s, which)); + set_jmp_reset_offset(s, which); +} + +static void tcg_out_nop_fill(tcg_insn_unit *p, int count) +{ + int32_t *p2 = (int32_t*)p; + memset(p2, 0, sizeof(*p2) * count); +} + +static void tcg_out_goto_ptr(TCGContext *s, TCGOpcode opc, TCGReg arg) +{ + tcg_tci_out_op_r(s, opc, arg); + tcg_wasm_out_goto_ptr(s, arg); +} +static void tcg_out_br(TCGContext *s, TCGOpcode opc, TCGLabel *l) +{ + tcg_tci_out_op_l(s, opc, l); + tcg_wasm_out_br(s, l); +} +static void tcg_out_setcond_i32(TCGContext *s, TCGOpcode opc, TCGCond cond, TCGReg ret, + TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrrc(s, opc, ret, arg1, arg2, cond); + tcg_wasm_out_setcond_i32(s, cond, ret, arg1, arg2); +} +static void tcg_out_setcond_i64(TCGContext *s, TCGOpcode opc, TCGCond cond, TCGReg ret, + TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrrc(s, opc, ret, arg1, arg2, cond); + tcg_wasm_out_setcond_i64(s, cond, ret, arg1, arg2); +} +static void tcg_out_movcond_i32(TCGContext *s, TCGOpcode opc, TCGCond cond, TCGReg ret, + TCGReg c1, TCGReg c2, TCGReg v1, TCGReg v2) +{ + tcg_tci_out_op_rrrrrc(s, opc, ret, c1, c2, v1, v2, cond); + tcg_wasm_out_movcond_i32(s, cond, ret, c1, c2, v1, v2); +} +static void tcg_out_movcond_i64(TCGContext *s, TCGOpcode opc, TCGCond cond, TCGReg ret, + TCGReg c1, TCGReg c2, TCGReg v1, TCGReg v2) +{ + tcg_tci_out_op_rrrrrc(s, opc, ret, c1, c2, v1, v2, cond); + tcg_wasm_out_movcond_i64(s, cond, ret, c1, c2, v1, v2); +} +static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + tcg_tci_out_ldst(s, INDEX_op_ld_i32, val, base, offset); + break; + case TCG_TYPE_I64: + tcg_tci_out_ldst(s, INDEX_op_ld_i64, val, base, offset); + break; + default: + g_assert_not_reached(); + } + tcg_wasm_out_ld(s, type, val, base, offset); +} +static void tcg_out_ld8s(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + tcg_tci_out_ldst(s, opc, val, base, offset); + tcg_wasm_out_ld8s(s, type, val, base, offset); +} +static void tcg_out_ld8u(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + tcg_tci_out_ldst(s, opc, val, base, offset); + tcg_wasm_out_ld8u(s, type, val, base, offset); +} +static void tcg_out_ld16s(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + tcg_tci_out_ldst(s, opc, val, base, offset); + tcg_wasm_out_ld16s(s, type, val, base, offset); +} +static void tcg_out_ld16u(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + tcg_tci_out_ldst(s, opc, val, base, offset); + tcg_wasm_out_ld16u(s, type, val, base, offset); +} +static void tcg_out_ld32s(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + tcg_tci_out_ldst(s, opc, val, base, offset); + tcg_wasm_out_ld32s(s, type, val, base, offset); +} +static void tcg_out_ld32u(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + tcg_tci_out_ldst(s, opc, val, base, offset); + tcg_wasm_out_ld32u(s, type, val, base, offset); +} +static void tcg_out_st(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + switch (type) { + case TCG_TYPE_I32: + tcg_tci_out_ldst(s, INDEX_op_st_i32, val, base, offset); + break; + case TCG_TYPE_I64: + tcg_tci_out_ldst(s, INDEX_op_st_i64, val, base, offset); + break; + default: + g_assert_not_reached(); + } + tcg_wasm_out_st(s, type, val, base, offset); +} +static void tcg_out_st8(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + tcg_tci_out_ldst(s, opc, val, base, offset); + tcg_wasm_out_st8(s, type, val, base, offset); +} +static void tcg_out_st16(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + tcg_tci_out_ldst(s, opc, val, base, offset); + tcg_wasm_out_st16(s, type, val, base, offset); +} +static void tcg_out_st32(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + tcg_tci_out_ldst(s, opc, val, base, offset); + tcg_wasm_out_st32(s, type, val, base, offset); +} +static void tcg_out_i64_calc_add(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_add(s, ret, arg1, arg2); +} +static void tcg_out_i64_calc_sub(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_sub(s, ret, arg1, arg2); +} +static void tcg_out_i64_calc_mul(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_mul(s, ret, arg1, arg2); +} +static void tcg_out_i64_calc_and(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_and(s, ret, arg1, arg2); +} +static void tcg_out_i64_calc_or(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_or(s, ret, arg1, arg2); +} +static void tcg_out_i64_calc_xor(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_xor(s, ret, arg1, arg2); +} +static void tcg_out_shl(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_shl(s, type, ret, arg1, arg2); +} +static void tcg_out_i64_calc_shl(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_shl(s, ret, arg1, arg2); +} +static void tcg_out_shr_u(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_shr_u(s, type, ret, arg1, arg2); +} +static void tcg_out_i64_calc_shr_u(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_shr_u(s, ret, arg1, arg2); +} +static void tcg_out_shr_s(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_shr_s(s, type, ret, arg1, arg2); +} +static void tcg_out_i64_calc_shr_s(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_shr_s(s, ret, arg1, arg2); +} +static void tcg_out_i64_calc_rotl(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_rotl(s, ret, arg1, arg2); +} +static void tcg_out_i32_rotl(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i32_rotl(s, ret, arg1, arg2); +} +static void tcg_out_i32_rotr(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i32_rotr(s, ret, arg1, arg2); +} +static void tcg_out_i64_calc_rotr(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_i64_calc_rotr(s, ret, arg1, arg2); +} +static void tcg_out_i64_calc_div_s(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_div_s(s, type, ret, arg1, arg2); +} +static void tcg_out_i64_calc_div_u(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_div_u(s, type, ret, arg1, arg2); +} +static void tcg_out_i64_calc_rem_s(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_rem_s(s, type, ret, arg1, arg2); +} +static void tcg_out_i64_calc_rem_u(TCGContext *s, TCGOpcode opc, TCGType type, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_rem_u(s, type, ret, arg1, arg2); +} +static void tcg_out_andc(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_andc(s, ret, arg1, arg2); +} +static void tcg_out_orc(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_orc(s, ret, arg1, arg2); +} +static void tcg_out_eqv(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_eqv(s, ret, arg1, arg2); +} +static void tcg_out_nand(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_nand(s, ret, arg1, arg2); +} +static void tcg_out_nor(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_nor(s, ret, arg1, arg2); +} +static void tcg_out_clz32(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_clz32(s, ret, arg1, arg2); +} +static void tcg_out_clz64(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_clz64(s, ret, arg1, arg2); +} +static void tcg_out_ctz32(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_ctz32(s, ret, arg1, arg2); +} +static void tcg_out_ctz64(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrr(s, opc, ret, arg1, arg2); + tcg_wasm_out_ctz64(s, ret, arg1, arg2); +} +static void tcg_out_brcond_i32(TCGContext *s, TCGOpcode opc, TCGCond cond, TCGReg arg1, + TCGReg arg2, TCGLabel *l) +{ + tcg_tci_out_op_rrrc(s, (opc == INDEX_op_brcond_i32 ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64), TCG_REG_TMP, arg1, arg2, cond); + tcg_tci_out_op_rl(s, opc, TCG_REG_TMP, l); + tcg_wasm_out_brcond_i32(s, cond, arg1, arg2, l); + +} +static void tcg_out_brcond_i64(TCGContext *s, TCGOpcode opc, TCGCond cond, TCGReg arg1, + TCGReg arg2, TCGLabel *l) +{ + tcg_tci_out_op_rrrc(s, (opc == INDEX_op_brcond_i32 ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64), TCG_REG_TMP, arg1, arg2, cond); + tcg_tci_out_op_rl(s, opc, TCG_REG_TMP, l); + tcg_wasm_out_brcond_i64(s, cond, arg1, arg2, l); + +} +static void tcg_out_neg(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_op_rr(s, opc, ret, arg); + tcg_wasm_out_neg(s, ret, arg); +} +static void tcg_out_not(TCGContext *s, TCGOpcode opc, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_op_rr(s, opc, ret, arg); + tcg_wasm_out_not(s, ret, arg); +} +static void tcg_out_ctpop_i32(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg src) +{ + tcg_tci_out_op_rr(s, opc, dest, src); + tcg_wasm_out_ctpop_i32(s, dest, src); +} +static void tcg_out_ctpop_i64(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg src) +{ + tcg_tci_out_op_rr(s, opc, dest, src); + tcg_wasm_out_ctpop_i64(s, dest, src); +} +static void tcg_out_add2(TCGContext *s, TCGOpcode opc, TCGReg retl, TCGReg reth, + TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) +{ + tcg_tci_out_op_rrrrrr(s, opc, retl, reth, al, ah, bl, bh); + tcg_wasm_out_add2(s, retl, reth, al, ah, bl, bh); +} +static void tcg_out_sub2(TCGContext *s, TCGOpcode opc, TCGReg retl, TCGReg reth, + TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) +{ + tcg_tci_out_op_rrrrrr(s, opc, retl, reth, al, ah, bl, bh); + tcg_wasm_out_sub2(s, retl, reth, al, ah, bl, bh); +} +static void tcg_out_mulu2_i32(TCGContext *s, TCGOpcode opc, TCGReg retl, TCGReg reth, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrrr(s, opc, retl, reth, arg1, arg2); + tcg_wasm_out_mulu2_i32(s, retl, reth, arg1, arg2); +} +static void tcg_out_muls2_i32(TCGContext *s, TCGOpcode opc, TCGReg retl, TCGReg reth, TCGReg arg1, TCGReg arg2) +{ + tcg_tci_out_op_rrrr(s, opc, retl, reth, arg1, arg2); + tcg_wasm_out_muls2_i32(s, retl, reth, arg1, arg2); +} + +static void tcg_out_bswap16_i32(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg src, int flags) +{ + tcg_tci_out_op_rr(s, opc, dest, src); + if (flags & TCG_BSWAP_OS) { + tcg_tci_out_op_rr(s, INDEX_op_ext16s_i32, dest, dest); + } + tcg_wasm_out_bswap16(s, dest, src, flags); + if (flags & TCG_BSWAP_OS) { + tcg_wasm_out_ext16s(s, TCG_TYPE_I32, dest, dest); + } +} +static void tcg_out_bswap16_i64(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg src, int flags) +{ + tcg_tci_out_op_rr(s, opc, dest, src); + if (flags & TCG_BSWAP_OS) { + tcg_tci_out_op_rr(s, INDEX_op_ext16s_i64, dest, dest); + } + tcg_wasm_out_bswap16(s, dest, src, flags); + if (flags & TCG_BSWAP_OS) { + tcg_wasm_out_ext16s(s, TCG_TYPE_I64, dest, dest); + } +} + +static void tcg_out_bswap32_i32(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg src, int flags) +{ + tcg_tci_out_op_rr(s, opc, dest, src); + tcg_wasm_out_bswap32(s, dest, src, flags); +} +static void tcg_out_bswap32_i64(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg src, int flags) +{ + tcg_tci_out_op_rr(s, opc, dest, src); + if (flags & TCG_BSWAP_OS) { + tcg_tci_out_op_rr(s, INDEX_op_ext32s_i64, dest, dest); + } + tcg_wasm_out_bswap32(s, dest, src, flags); + if (flags & TCG_BSWAP_OS) { + tcg_wasm_out_ext32s(s, dest, dest); + } +} +static void tcg_out_bswap64_i64(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg src, int flags) +{ + tcg_tci_out_op_rr(s, opc, dest, src); + tcg_wasm_out_bswap64(s, dest, src, flags); +} +static uint8_t tcg_tci_out_qemu_ldst(TCGContext *s, TCGOpcode opc, const TCGArg *args) +{ + TCGReg addr_reg = args[1]; + MemOpIdx oi = args[2]; + + MemOp mopc = get_memop(oi); + TCGAtomAlign aa = atom_and_align_for_opc(s, mopc, MO_ATOM_IFALIGN, false); + unsigned a_mask = (1u << aa.align) - 1; + + int mem_index = get_mmuidx(oi); + int fast_ofs = tlb_mask_table_ofs(s, mem_index); + int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); + int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); + + new_pool_l8(s, 20, (void*)cur_tci_ptr(s), 0, + (TCGReg)args[0], (TCGReg)args[1], (TCGArg)args[2], (int32_t)a_mask, (int32_t)mask_ofs, (uint64_t)s->page_bits, s->page_mask, table_ofs); + + uint32_t insn = 0; + insn = deposit32(insn, 0, 8, opc); + tcg_tci_out32(s, insn); +} +static void tcg_out_qemu_ld(TCGContext *s, TCGOpcode opc, const TCGArg *args, bool is_64) +{ + tcg_tci_out_qemu_ldst(s, opc, args); + tcg_wasm_out_qemu_ld(s, args, is_64); +} +static void tcg_out_qemu_st(TCGContext *s, TCGOpcode opc, const TCGArg *args, bool is_64) +{ + tcg_tci_out_qemu_ldst(s, opc, args); + tcg_wasm_out_qemu_st(s, args, is_64); +} +static void tcg_out_deposit_i32(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg arg1, TCGReg arg2, int pos, int len) +{ + TCGArg max = opc == INDEX_op_deposit_i32 ? 32 : 64; + tcg_tci_out_op_rrrbb(s, opc, dest, arg1, arg2, pos, len); + tcg_wasm_out_deposit_i32(s, dest, arg1, arg2, pos, len); +} +static void tcg_out_deposit_i64(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg arg1, TCGReg arg2, int pos, int len) +{ + TCGArg max = opc == INDEX_op_deposit_i32 ? 32 : 64; + tcg_tci_out_op_rrrbb(s, opc, dest, arg1, arg2, pos, len); + tcg_wasm_out_deposit_i64(s, dest, arg1, arg2, pos, len); +} +static void tcg_out_extract_i32(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg arg1, int pos, int len) +{ + TCGArg max = tcg_op_defs[opc].flags & TCG_OPF_64BIT ? 64 : 32; + tcg_tci_out_op_rrbb(s, opc, dest, arg1, pos, len); + tcg_wasm_out_extract(s, dest, arg1, pos, len, TCG_TYPE_I32, false); +} +static void tcg_out_extract_i64(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg arg1, int pos, int len) +{ + TCGArg max = tcg_op_defs[opc].flags & TCG_OPF_64BIT ? 64 : 32; + tcg_tci_out_op_rrbb(s, opc, dest, arg1, pos, len); + tcg_wasm_out_extract(s, dest, arg1, pos, len, TCG_TYPE_I64, false); +} +static void tcg_out_sextract_i32(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg arg1, int pos, int len) +{ + TCGArg max = tcg_op_defs[opc].flags & TCG_OPF_64BIT ? 64 : 32; + tcg_tci_out_op_rrbb(s, opc, dest, arg1, pos, len); + tcg_wasm_out_extract(s, dest, arg1, pos, len, TCG_TYPE_I32, true); +} +static void tcg_out_sextract_i64(TCGContext *s, TCGOpcode opc, TCGReg dest, TCGReg arg1, int pos, int len) +{ + TCGArg max = tcg_op_defs[opc].flags & TCG_OPF_64BIT ? 64 : 32; + tcg_tci_out_op_rrbb(s, opc, dest, arg1, pos, len); + tcg_wasm_out_extract(s, dest, arg1, pos, len, TCG_TYPE_I64, true); +} +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_ext8s(s, type, ret, arg); + tcg_wasm_out_ext8s(s, type, ret, arg); +} +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_ext16s(s, type, ret, arg); + tcg_wasm_out_ext16s(s, type, ret, arg); +} +static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_ext8u(s, ret, arg); + tcg_wasm_out_ext8u(s, ret, arg); +} +static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_ext16u(s, ret, arg); + tcg_wasm_out_ext16u(s, ret, arg); +} +static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_ext32s(s, ret, arg); + tcg_wasm_out_ext32s(s, ret, arg); +} +static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_ext32u(s, ret, arg); + tcg_wasm_out_ext32u(s, ret, arg); +} +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_exts_i32_i64(s, ret, arg); + tcg_wasm_out_exts_i32_i64(s, ret, arg); +} +static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_extu_i32_i64(s, ret, arg); + tcg_wasm_out_extu_i32_i64(s, ret, arg); +} + +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs) +{ + tcg_tci_out_extrl_i64_i32(s, rd, rs); + tcg_wasm_out_extrl_i64_i32(s, rd, rs); +} + +static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) +{ + tcg_tci_out_mov(s, type, ret, arg); + tcg_wasm_out_mov(s, type, ret, arg); + return true; +} +static void tcg_out_movi(TCGContext *s, TCGType type, + TCGReg ret, tcg_target_long arg) +{ + tcg_tci_out_movi(s, type, ret, arg); + tcg_wasm_out_movi(s, type, ret, arg); +} +static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, + tcg_target_long imm) +{ + g_assert_not_reached(); +} +static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2) +{ + return false; +} +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) +{ + tcg_tci_out_exit_tb(s, arg); + tcg_wasm_out_exit_tb(s, arg); +} +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + tcg_tci_out_goto_tb(s, which); + tcg_wasm_out_goto_tb(s, which); +} +static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, + TCGReg base, intptr_t ofs) +{ + return false; +} +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, + const TCGHelperInfo *info) +{ + tcg_tci_out_call(s, target, info); + tcg_wasm_out_call(s, target, info); } static void tcg_out_op(TCGContext *s, TCGOpcode opc, @@ -2381,252 +3505,269 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, { switch (opc) { case INDEX_op_goto_ptr: - tcg_out_goto_ptr(s, args[0]); + tcg_out_goto_ptr(s, opc, args[0]); break; case INDEX_op_br: - tcg_out_br(s, arg_label(args[0])); + tcg_out_br(s, opc, arg_label(args[0])); break; case INDEX_op_setcond_i32: + tcg_out_setcond_i32(s, opc, args[3], args[0], args[1], args[2]);// + break; case INDEX_op_setcond_i64: - tcg_out_setcond(s, args[3], args[0], args[1], args[2]);// + tcg_out_setcond_i64(s, opc, args[3], args[0], args[1], args[2]);// break; case INDEX_op_movcond_i32: + tcg_out_movcond_i32(s, opc, args[5], args[0], args[1], args[2], args[3], args[4]);// + break; case INDEX_op_movcond_i64: - tcg_out_movcond(s, args[5], args[0], args[1], args[2], args[3], args[4]);// + tcg_out_movcond_i64(s, opc, args[5], args[0], args[1], args[2], args[3], args[4]);// break; case INDEX_op_ld_i64: tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_ld8s_i32: case INDEX_op_ld8s_i64: - tcg_out_ld8s(s, TCG_TYPE_I64, args[0], args[1], args[2]); + tcg_out_ld8s(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_ld8u_i32: case INDEX_op_ld8u_i64: - tcg_out_ld8u(s, TCG_TYPE_I64, args[0], args[1], args[2]); + tcg_out_ld8u(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_ld16s_i32: case INDEX_op_ld16s_i64: - tcg_out_ld16s(s, TCG_TYPE_I64, args[0], args[1], args[2]); + tcg_out_ld16s(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_ld16u_i32: case INDEX_op_ld16u_i64: - tcg_out_ld16u(s, TCG_TYPE_I64, args[0], args[1], args[2]); + tcg_out_ld16u(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_ld32u_i64: - tcg_out_ld32u(s, TCG_TYPE_I64, args[0], args[1], args[2]); + tcg_out_ld32u(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_ld_i32: case INDEX_op_ld32s_i64: - tcg_out_ld32s(s, TCG_TYPE_I64, args[0], args[1], args[2]); + tcg_out_ld32s(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_st_i64: tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_st8_i32: case INDEX_op_st8_i64: - tcg_out_st8(s, TCG_TYPE_I64, args[0], args[1], args[2]); + tcg_out_st8(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_st16_i32: case INDEX_op_st16_i64: - tcg_out_st16(s, TCG_TYPE_I64, args[0], args[1], args[2]); + tcg_out_st16(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_st_i32: case INDEX_op_st32_i64: - tcg_out_st32(s, TCG_TYPE_I64, args[0], args[1], args[2]); + tcg_out_st32(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_add_i32: case INDEX_op_add_i64: - tcg_out_i64_calc_add(s, args[0], args[1], args[2]); + tcg_out_i64_calc_add(s, opc, args[0], args[1], args[2]); break; case INDEX_op_sub_i32: case INDEX_op_sub_i64: - tcg_out_i64_calc_sub(s, args[0], args[1], args[2]); + tcg_out_i64_calc_sub(s, opc, args[0], args[1], args[2]); break; case INDEX_op_mul_i32: case INDEX_op_mul_i64: - tcg_out_i64_calc_mul(s, args[0], args[1], args[2]); + tcg_out_i64_calc_mul(s, opc, args[0], args[1], args[2]); break; case INDEX_op_and_i32: case INDEX_op_and_i64: - tcg_out_i64_calc_and(s, args[0], args[1], args[2]); + tcg_out_i64_calc_and(s, opc, args[0], args[1], args[2]); break; case INDEX_op_or_i32: case INDEX_op_or_i64: - tcg_out_i64_calc_or(s, args[0], args[1], args[2]); + tcg_out_i64_calc_or(s, opc, args[0], args[1], args[2]); break; case INDEX_op_xor_i32: case INDEX_op_xor_i64: - tcg_out_i64_calc_xor(s, args[0], args[1], args[2]); + tcg_out_i64_calc_xor(s, opc, args[0], args[1], args[2]); break; case INDEX_op_shl_i32: + tcg_out_shl(s, opc, TCG_TYPE_I32, args[0], args[1], args[2]); + break; case INDEX_op_shl_i64: - tcg_out_i64_calc_shl(s, args[0], args[1], args[2]); + tcg_out_shl(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_shr_i32: + tcg_out_shr_u(s, opc, TCG_TYPE_I32, args[0], args[1], args[2]); + break; case INDEX_op_shr_i64: - tcg_out_i64_calc_shr_u(s, args[0], args[1], args[2]); + tcg_out_shr_u(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_sar_i32: + tcg_out_shr_s(s, opc, TCG_TYPE_I32, args[0], args[1], args[2]); + break; case INDEX_op_sar_i64: - tcg_out_i64_calc_shr_s(s, args[0], args[1], args[2]); + tcg_out_shr_s(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_rotl_i32: - tcg_out_i32_rotl(s, args[0], args[1], args[2]); + tcg_out_i32_rotl(s, opc, args[0], args[1], args[2]); break; case INDEX_op_rotl_i64: - tcg_out_i64_calc_rotl(s, args[0], args[1], args[2]); + tcg_out_i64_calc_rotl(s, opc, args[0], args[1], args[2]); break; case INDEX_op_rotr_i32: - tcg_out_i32_rotr(s, args[0], args[1], args[2]); + tcg_out_i32_rotr(s, opc, args[0], args[1], args[2]); break; case INDEX_op_rotr_i64: - tcg_out_i64_calc_rotr(s, args[0], args[1], args[2]); + tcg_out_i64_calc_rotr(s, opc, args[0], args[1], args[2]); break; case INDEX_op_div_i32: + tcg_out_i64_calc_div_s(s, opc, TCG_TYPE_I32, args[0], args[1], args[2]); + break; case INDEX_op_div_i64: - tcg_out_i64_calc_div_s(s, args[0], args[1], args[2]); + tcg_out_i64_calc_div_s(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_divu_i32: + tcg_out_i64_calc_div_u(s, opc, TCG_TYPE_I32, args[0], args[1], args[2]); + break; case INDEX_op_divu_i64: - tcg_out_i64_calc_div_u(s, args[0], args[1], args[2]); + tcg_out_i64_calc_div_u(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_rem_i32: + tcg_out_i64_calc_rem_s(s, opc, TCG_TYPE_I32, args[0], args[1], args[2]); + break; case INDEX_op_rem_i64: - tcg_out_i64_calc_rem_s(s, args[0], args[1], args[2]); + tcg_out_i64_calc_rem_s(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_remu_i32: + tcg_out_i64_calc_rem_u(s, opc, TCG_TYPE_I32, args[0], args[1], args[2]); + break; case INDEX_op_remu_i64: - tcg_out_i64_calc_rem_u(s, args[0], args[1], args[2]); + tcg_out_i64_calc_rem_u(s, opc, TCG_TYPE_I64, args[0], args[1], args[2]); break; case INDEX_op_andc_i32: case INDEX_op_andc_i64: - tcg_out_andc(s, args[0], args[1], args[2]); + tcg_out_andc(s, opc, args[0], args[1], args[2]); break; case INDEX_op_orc_i32: case INDEX_op_orc_i64: - tcg_out_orc(s, args[0], args[1], args[2]); + tcg_out_orc(s, opc, args[0], args[1], args[2]); break; case INDEX_op_eqv_i32: case INDEX_op_eqv_i64: - tcg_out_eqv(s, args[0], args[1], args[2]); + tcg_out_eqv(s, opc, args[0], args[1], args[2]); break; case INDEX_op_nand_i32: case INDEX_op_nand_i64: - tcg_out_nand(s, args[0], args[1], args[2]); + tcg_out_nand(s, opc, args[0], args[1], args[2]); break; case INDEX_op_nor_i32: case INDEX_op_nor_i64: - tcg_out_nor(s, args[0], args[1], args[2]); + tcg_out_nor(s, opc, args[0], args[1], args[2]); break; case INDEX_op_clz_i32: - tcg_out_clz32(s, args[0], args[1], args[2]); + tcg_out_clz32(s, opc, args[0], args[1], args[2]); break; case INDEX_op_clz_i64: - tcg_out_clz64(s, args[0], args[1], args[2]); + tcg_out_clz64(s, opc, args[0], args[1], args[2]); break; case INDEX_op_ctz_i32: - tcg_out_ctz32(s, args[0], args[1], args[2]); + tcg_out_ctz32(s, opc, args[0], args[1], args[2]); break; case INDEX_op_ctz_i64: - tcg_out_ctz64(s, args[0], args[1], args[2]); + tcg_out_ctz64(s, opc, args[0], args[1], args[2]); break; case INDEX_op_brcond_i32: + tcg_out_brcond_i32(s, opc, args[2], args[0], args[1], arg_label(args[3])); + break; case INDEX_op_brcond_i64: - tcg_out_brcond(s, args[2], args[0], args[1], arg_label(args[3])); + tcg_out_brcond_i64(s, opc, args[2], args[0], args[1], arg_label(args[3])); break; case INDEX_op_neg_i32: case INDEX_op_neg_i64: - tcg_out_neg(s, args[0], args[1]); + tcg_out_neg(s, opc, args[0], args[1]); break; case INDEX_op_not_i32: case INDEX_op_not_i64: - tcg_out_not(s, args[0], args[1]); + tcg_out_not(s, opc, args[0], args[1]); break; case INDEX_op_ctpop_i32: + tcg_out_ctpop_i32(s, opc, args[1], args[2]); + break; case INDEX_op_ctpop_i64: - tcg_out_ctpop(s, args[1], args[2]); + tcg_out_ctpop_i64(s, opc, args[1], args[2]); break; case INDEX_op_add2_i32: case INDEX_op_add2_i64: - tcg_out_add2(s, args[0], args[1], args[2], args[3], args[4], args[5]); + tcg_out_add2(s, opc, args[0], args[1], args[2], args[3], args[4], args[5]); break; case INDEX_op_sub2_i32: case INDEX_op_sub2_i64: - tcg_out_sub2(s, args[0], args[1], args[2], args[3], args[4], args[5]); - break; - case INDEX_op_muls2_i32: - tcg_out_muls2_i32(s, args[0], args[1], args[2], args[3]); - break; - case INDEX_op_mulu2_i32: - tcg_out_mulu2_i32(s, args[0], args[1], args[2], args[3]); - break; - case INDEX_op_mulsh_i32: - tcg_out_mulsh_i32(s, args[0], args[1], args[2]); - break; - case INDEX_op_muluh_i32: - tcg_out_muluh_i32(s, args[0], args[1], args[2]); - break; - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - tcg_out_bswap16(s, args[0], args[1], args[2]); - break; - case INDEX_op_bswap32_i32: - case INDEX_op_bswap32_i64: - tcg_out_bswap32(s, args[0], args[1], args[2]); - break; - case INDEX_op_bswap64_i64: - tcg_out_bswap64(s, args[0], args[1], args[2]); + tcg_out_sub2(s, opc, args[0], args[1], args[2], args[3], args[4], args[5]); break; case INDEX_op_qemu_ld_a32_i32: case INDEX_op_qemu_ld_a64_i32: - tcg_out_qemu_ld(s, args, false); + tcg_out_qemu_ld(s, opc, args, false); break; case INDEX_op_qemu_st_a32_i32: case INDEX_op_qemu_st_a64_i32: - tcg_out_qemu_st(s, args, false); + tcg_out_qemu_st(s, opc, args, false); break; case INDEX_op_qemu_ld_a32_i64: case INDEX_op_qemu_ld_a64_i64: - tcg_out_qemu_ld(s, args, true); + tcg_out_qemu_ld(s, opc, args, true); break; case INDEX_op_qemu_st_a32_i64: case INDEX_op_qemu_st_a64_i64: - tcg_out_qemu_st(s, args, true); - break; - case INDEX_op_mb: - break; - case INDEX_op_deposit_i32: - case INDEX_op_deposit_i64: - tcg_out_deposit(s, args[0], args[1], args[2], args[3], args[4]); - break; - case INDEX_op_extract_i32: - tcg_out_extract(s, args[0], args[1], args[2], args[3], TCG_TYPE_I32, false); - break; - case INDEX_op_extract_i64: - tcg_out_extract(s, args[0], args[1], args[2], args[3], TCG_TYPE_I64, false); - break; - case INDEX_op_sextract_i32: - tcg_out_extract(s, args[0], args[1], args[2], args[3], TCG_TYPE_I32, true); - break; - case INDEX_op_sextract_i64: - tcg_out_extract(s, args[0], args[1], args[2], args[3], TCG_TYPE_I64, true); + tcg_out_qemu_st(s, opc, args, true); break; case INDEX_op_extrl_i64_i32: tcg_out_extrl_i64_i32(s, args[0], args[1]); break; - case INDEX_op_extrh_i64_i32: - tcg_out_extrh_i64_i32(s, args[0], args[1]); + case INDEX_op_mb: + tcg_tci_out_op_v(s, opc); + tcg_wasm_out8(s, 0x01); // nop break; - case INDEX_op_extract2_i32: - tcg_out_extract2_i32(s, args[0], args[1], args[2], args[3]); + case INDEX_op_extract_i32: + tcg_out_extract_i32(s, opc, args[0], args[1], args[2], args[3]); break; - case INDEX_op_extract2_i64: - tcg_out_extract2_i64(s, args[0], args[1], args[2], args[3]); + case INDEX_op_extract_i64: + tcg_out_extract_i64(s, opc, args[0], args[1], args[2], args[3]); + break; + case INDEX_op_sextract_i32: + tcg_out_sextract_i32(s, opc, args[0], args[1], args[2], args[3]); + break; + case INDEX_op_sextract_i64: + tcg_out_sextract_i64(s, opc, args[0], args[1], args[2], args[3]); + break; + case INDEX_op_deposit_i32: + tcg_out_deposit_i32(s, opc, args[0], args[1], args[2], args[3], args[4]); + break; + case INDEX_op_deposit_i64: + tcg_out_deposit_i64(s, opc, args[0], args[1], args[2], args[3], args[4]); + break; + case INDEX_op_bswap16_i32: + tcg_out_bswap16_i32(s, opc, args[0], args[1], args[2]); + break; + case INDEX_op_bswap16_i64: + tcg_out_bswap16_i64(s, opc, args[0], args[1], args[2]); + break; + case INDEX_op_bswap32_i32: + tcg_out_bswap32_i32(s, opc, args[0], args[1], args[2]); + break; + case INDEX_op_bswap32_i64: + tcg_out_bswap32_i64(s, opc, args[0], args[1], args[2]); + break; + case INDEX_op_bswap64_i64: + tcg_out_bswap64_i64(s, opc, args[0], args[1], args[2]); + break; + case INDEX_op_muls2_i32: + tcg_out_muls2_i32(s, opc, args[0], args[1], args[2], args[3]); + break; + case INDEX_op_mulu2_i32: + tcg_out_mulu2_i32(s, opc, args[0], args[1], args[2], args[3]); break; default: g_assert_not_reached(); + break; } + return; } void tcg_out_init() { @@ -2635,11 +3776,16 @@ void tcg_out_init() { } /* Test if a constant matches the constraint. */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { return ct & TCG_CT_CONST; } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + bool tcg_target_has_memory_bswap(MemOp memop) { return false; @@ -2651,8 +3797,8 @@ static void tcg_target_init(TCGContext *s) tcg_debug_assert(tcg_op_defs_max <= UINT8_MAX); /* Registers available for 32 bit operations. */ - tcg_target_available_regs[TCG_TYPE_I64] = 0xffffff; - tcg_target_available_regs[TCG_TYPE_I32] = 0xffffff; + tcg_target_available_regs[TCG_TYPE_I64] = BIT(TCG_TARGET_NB_REGS) - 1; + tcg_target_available_regs[TCG_TYPE_I32] = BIT(TCG_TARGET_NB_REGS) - 1; /* * The interpreter "registers" are in the local stack frame and * cannot be clobbered by the called helper functions. However, @@ -2660,7 +3806,7 @@ static void tcg_target_init(TCGContext *s) * the return value registers. */ tcg_target_call_clobber_regs = - MAKE_64BIT_MASK(TCG_REG_A0, 64 / TCG_TARGET_REG_BITS); + MAKE_64BIT_MASK(TCG_REG_R0, 128 / TCG_TARGET_REG_BITS); s->reserved_regs = 0; tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP); diff --git a/tcg/wasm32/tcg-target.h b/tcg/wasm32/tcg-target.h index b6c9f395be..37ad4b93c8 100644 --- a/tcg/wasm32/tcg-target.h +++ b/tcg/wasm32/tcg-target.h @@ -45,7 +45,7 @@ #define TCG_TARGET_HAS_deposit_i32 1 #define TCG_TARGET_HAS_extract_i32 1 #define TCG_TARGET_HAS_sextract_i32 1 -#define TCG_TARGET_HAS_extract2_i32 1 +#define TCG_TARGET_HAS_extract2_i32 0 #define TCG_TARGET_HAS_eqv_i32 1 #define TCG_TARGET_HAS_nand_i32 1 #define TCG_TARGET_HAS_nor_i32 1 @@ -57,32 +57,31 @@ #define TCG_TARGET_HAS_orc_i32 1 #define TCG_TARGET_HAS_rot_i32 1 #define TCG_TARGET_HAS_movcond_i32 1 +#define TCG_TARGET_HAS_negsetcond_i32 0 #define TCG_TARGET_HAS_muls2_i32 1 -#define TCG_TARGET_HAS_muluh_i32 1 -#define TCG_TARGET_HAS_mulsh_i32 1 +#define TCG_TARGET_HAS_muluh_i32 0 +#define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_qemu_st8_i32 0 -#define TCG_TARGET_HAS_mulu2_i32 1 -#define TCG_TARGET_HAS_add2_i32 1 -#define TCG_TARGET_HAS_sub2_i32 1 +#define TCG_TARGET_HAS_extr_i64_i32 0 +#define TCG_TARGET_HAS_extrl_i64_i32 1 +#define TCG_TARGET_HAS_extrh_i64_i32 0 #define TCG_TARGET_HAS_bswap16_i64 1 #define TCG_TARGET_HAS_bswap32_i64 1 #define TCG_TARGET_HAS_bswap64_i64 1 +#define TCG_TARGET_HAS_deposit_i64 1 +#define TCG_TARGET_HAS_extract_i64 1 +#define TCG_TARGET_HAS_sextract_i64 1 +#define TCG_TARGET_HAS_extract2_i64 0 #define TCG_TARGET_HAS_div_i64 1 #define TCG_TARGET_HAS_rem_i64 1 #define TCG_TARGET_HAS_ext8s_i64 1 #define TCG_TARGET_HAS_ext16s_i64 1 +#define TCG_TARGET_HAS_ext32s_i64 1 #define TCG_TARGET_HAS_ext8u_i64 1 #define TCG_TARGET_HAS_ext16u_i64 1 #define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_ext32s_i64 1 #define TCG_TARGET_HAS_andc_i64 1 -#define TCG_TARGET_HAS_deposit_i64 1 -#define TCG_TARGET_HAS_extract_i64 1 -#define TCG_TARGET_HAS_sextract_i64 1 -#define TCG_TARGET_HAS_extract2_i64 1 -#define TCG_TARGET_HAS_extrl_i64_i32 1 -#define TCG_TARGET_HAS_extrh_i64_i32 1 #define TCG_TARGET_HAS_eqv_i64 1 #define TCG_TARGET_HAS_nand_i64 1 #define TCG_TARGET_HAS_nor_i64 1 @@ -93,19 +92,22 @@ #define TCG_TARGET_HAS_not_i64 1 #define TCG_TARGET_HAS_orc_i64 1 #define TCG_TARGET_HAS_rot_i64 1 +#define TCG_TARGET_HAS_negsetcond_i64 0 #define TCG_TARGET_HAS_movcond_i64 1 #define TCG_TARGET_HAS_muls2_i64 0 +#define TCG_TARGET_HAS_add2_i32 1 +#define TCG_TARGET_HAS_sub2_i32 1 +#define TCG_TARGET_HAS_mulu2_i32 1 +#define TCG_TARGET_HAS_add2_i64 1 +#define TCG_TARGET_HAS_sub2_i64 1 #define TCG_TARGET_HAS_mulu2_i64 0 #define TCG_TARGET_HAS_muluh_i64 0 #define TCG_TARGET_HAS_mulsh_i64 0 -#define TCG_TARGET_HAS_qemu_st8_i64 0 -#define TCG_TARGET_HAS_add2_i64 1 -#define TCG_TARGET_HAS_sub2_i64 1 #define TCG_TARGET_HAS_qemu_ldst_i128 0 /* Number of registers available. */ -#define TCG_TARGET_NB_REGS 24 +#define TCG_TARGET_NB_REGS 16 /* List of registers which are used by TCG. */ typedef enum { @@ -125,16 +127,6 @@ typedef enum { TCG_REG_R13, TCG_REG_R14, TCG_REG_R15, - - // Arguments - TCG_REG_A0, - TCG_REG_A1, - TCG_REG_A2, - TCG_REG_A3, - TCG_REG_A4, - TCG_REG_A5, - TCG_REG_A6, - TCG_REG_A7, TCG_REG_TMP = TCG_REG_R13, TCG_AREG0 = TCG_REG_R14, @@ -150,6 +142,7 @@ typedef enum { #define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL #define HAVE_TCG_QEMU_TB_EXEC +#define TCG_TARGET_NEED_POOL_LABELS #define TCG_TARGET_DEFAULT_MO (0) diff --git a/util/coroutine-fiber.c b/util/coroutine-fiber.c index 868b9a2d2b..2ce1575362 100644 --- a/util/coroutine-fiber.c +++ b/util/coroutine-fiber.c @@ -86,7 +86,9 @@ CoroutineAction qemu_coroutine_switch(Coroutine *from_, Coroutine *to_, CoroutineEmscripten *from = DO_UPCAST(CoroutineEmscripten, base, from_); CoroutineEmscripten *to = DO_UPCAST(CoroutineEmscripten, base, to_); +#if defined(EMSCRIPTEN) && !defined(CONFIG_TCG_INTERPRETER) set_unwinding_flag(); +#endif set_current(to_); to->action = action;