From af7d9824459c65d91bf1fed2411c665a57a992bc Mon Sep 17 00:00:00 2001 From: victoryang00 Date: Mon, 11 Mar 2024 23:18:16 -0700 Subject: [PATCH] Merge branch 'Multi-V-VM-main' --- build-scripts/config_common.cmake | 4 + core/config.h | 5 + core/iwasm/aot/aot_reloc.h | 1 + core/iwasm/aot/aot_runtime.c | 87 +- core/iwasm/aot/aot_runtime.h | 3 + core/iwasm/common/wasm_c_api.c | 28 +- core/iwasm/common/wasm_exec_env.c | 6 + core/iwasm/common/wasm_exec_env.h | 6 + core/iwasm/common/wasm_memory.c | 60 +- core/iwasm/common/wasm_runtime_common.c | 38 +- core/iwasm/common/wasm_runtime_common.h | 8 + core/iwasm/common/wasm_shared_memory.c | 69 +- core/iwasm/compilation/aot_compiler.c | 826 +++++++++++++++++- core/iwasm/compilation/aot_compiler.h | 12 + core/iwasm/compilation/aot_emit_aot_file.c | 44 +- core/iwasm/compilation/aot_emit_control.c | 43 +- core/iwasm/compilation/aot_emit_exception.c | 53 ++ core/iwasm/compilation/aot_emit_exception.h | 2 + core/iwasm/compilation/aot_emit_function.c | 61 +- core/iwasm/compilation/aot_emit_function.h | 5 +- core/iwasm/compilation/aot_llvm.c | 107 ++- core/iwasm/compilation/aot_llvm.h | 21 + core/iwasm/compilation/aot_llvm_extra.cpp | 116 ++- core/iwasm/interpreter/wasm_interp.h | 5 + core/iwasm/interpreter/wasm_interp_classic.c | 122 ++- core/iwasm/interpreter/wasm_runtime.c | 47 +- core/iwasm/interpreter/wasm_runtime.h | 9 +- .../libraries/ckpt-restore/ckpt-restore.cmake | 12 + .../libraries/ckpt-restore/ckpt_restore.h | 93 ++ .../lib-pthread/lib_pthread_wrapper.c | 79 +- .../lib-socket/inc/wasi_socket_ext.h | 6 +- .../lib-socket/src/wasi/wasi_socket_ext.c | 124 +++ .../lib_wasi_threads_wrapper.c | 21 +- .../libraries/libc-wasi/libc_wasi_wrapper.c | 583 +++++++++++- .../sandboxed-system-primitives/src/posix.c | 11 +- .../libraries/thread-mgr/thread_manager.c | 65 +- doc/build_wamr.md | 19 + wamr-compiler/main.c | 21 + 38 files changed, 2608 insertions(+), 214 deletions(-) create mode 100644 core/iwasm/libraries/ckpt-restore/ckpt-restore.cmake create mode 100755 core/iwasm/libraries/ckpt-restore/ckpt_restore.h diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 773ff28375..d97ad82227 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -304,6 +304,10 @@ if (WAMR_BUILD_AOT_STACK_FRAME EQUAL 1) add_definitions (-DWASM_ENABLE_AOT_STACK_FRAME=1) message (" AOT stack frame enabled") endif () +if (WAMR_BUILD_CHECKPOINT_RESTORE EQUAL 1) + add_definitions (-DWASM_ENABLE_CHECKPOINT_RESTORE=1) + message (" Checkpoint Restore enabled") +endif () if (WAMR_BUILD_MEMORY_PROFILING EQUAL 1) add_definitions (-DWASM_ENABLE_MEMORY_PROFILING=1) message (" Memory profiling enabled") diff --git a/core/config.h b/core/config.h index be9ebfc3a2..7cb770de19 100644 --- a/core/config.h +++ b/core/config.h @@ -335,6 +335,11 @@ #define WASM_ENABLE_AOT_STACK_FRAME 0 #endif +/* Checkpoint Restore */ +#ifndef WASM_ENABLE_CHECKPOINT_RESTORE +#define WASM_ENABLE_CHECKPOINT_RESTORE 0 +#endif + /* Heap verification */ #ifndef BH_ENABLE_GC_VERIFY #define BH_ENABLE_GC_VERIFY 0 diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index c250f20a72..ef37088a7f 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -55,6 +55,7 @@ typedef struct { #if WASM_ENABLE_AOT_STACK_FRAME != 0 #define REG_AOT_TRACE_SYM() \ + REG_SYM(aot_raise), \ REG_SYM(aot_alloc_frame), \ REG_SYM(aot_free_frame), \ REG_SYM(aot_frame_update_profile_info), diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index cc5d7fd000..a3b3a00841 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -16,6 +16,12 @@ #include "../libraries/thread-mgr/thread_manager.h" #endif +#include "wasm_interp.h" +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#include "wamr_export.h" +extern int counter_; +#endif + /* * Note: These offsets need to match the values hardcoded in * AoT compilation code: aot_create_func_context, check_suspend_flags. @@ -72,6 +78,11 @@ bh_static_assert(offsetof(AOTFrame, sp) == sizeof(uintptr_t) * 5); bh_static_assert(offsetof(AOTFrame, frame_ref) == sizeof(uintptr_t) * 6); bh_static_assert(offsetof(AOTFrame, lp) == sizeof(uintptr_t) * 7); +bh_static_assert(offsetof(AOTFrame, ip_offset) == sizeof(uintptr_t) * 4); +bh_static_assert(offsetof(AOTFrame, sp) == sizeof(uintptr_t) * 5); +bh_static_assert(offsetof(AOTFrame, frame_ref) == sizeof(uintptr_t) * 6); +bh_static_assert(offsetof(AOTFrame, lp) == sizeof(uintptr_t) * 7); + static void set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) { @@ -804,6 +815,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, bh_assert(memory_idx == 0); bh_assert(parent->memory_count > memory_idx); shared_memory_instance = parent->memories[memory_idx]; + shared_memory_instance->ref_count++; shared_memory_inc_reference(shared_memory_instance); return shared_memory_instance; } @@ -992,6 +1004,9 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, AOTMemoryInstance *memories, *memory_inst; AOTMemInitData *data_seg; uint64 total_size; +#if WASM_ENABLE_SHARED_MEMORY != 0 + bool is_shared_memory; +#endif module_inst->memory_count = memory_count; total_size = sizeof(AOTMemoryInstance *) * (uint64)memory_count; @@ -1019,6 +1034,15 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, return true; } +#if WASM_ENABLE_SHARED_MEMORY != 0 + /* Currently we have only one memory instance */ + is_shared_memory = module->memories[0].memory_flags & 0x02 ? true : false; + if (is_shared_memory && parent != NULL) { + /* Ignore setting memory init data if the memory has been initialized */ + return true; + } +#endif + for (i = 0; i < module->mem_init_data_count; i++) { data_seg = module->mem_init_data_list[i]; #if WASM_ENABLE_BULK_MEMORY != 0 @@ -1860,6 +1884,7 @@ destroy_c_api_frames(Vector *frames) void aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) { +#if WASM_ENABLE_CHECKPOINT_RESTORE == 0 WASMModuleInstanceExtraCommon *common = &((AOTModuleInstanceExtra *)module_inst->e)->common; if (module_inst->exec_env_singleton) { @@ -1934,6 +1959,7 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) #endif wasm_runtime_free(module_inst); +#endif } AOTFunctionInstance * @@ -2225,6 +2251,7 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, while (exec_env->cur_frame != prev_frame) aot_free_frame(exec_env); #endif + // checkpoint if (!ret) { if (argv1 != argv1_buf) wasm_runtime_free(argv1); @@ -2789,6 +2816,17 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, } tbl_elem_val = ((table_elem_type_t *)tbl_inst->elems)[table_elem_idx]; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (exec_env->is_restore && exec_env->restore_call_chain) { + struct AOTFrame *rcc = *(exec_env->restore_call_chain); + while (rcc->prev_frame) { + rcc = rcc->prev_frame; + } + LOG_DEBUG("func_idx: %d instead of %d of thread %ld\n", rcc->func_index, + func_idx, exec_env->handle); + func_idx = rcc->func_index; + } +#endif if (tbl_elem_val == NULL_REF) { aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT); goto fail; @@ -3525,9 +3563,17 @@ aot_free_frame(WASMExecEnv *exec_env) #else /* else of WASM_ENABLE_GC == 0 */ +int +aot_raise(WASMExecEnv *exec_env, int sig) +{ + raise(sig); +} bool aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) { +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("aot_alloc_frame %u thread %d\n", func_index, exec_env->handle); +#endif AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; AOTModule *module = (AOTModule *)module_inst->module; #if WASM_ENABLE_PERF_PROFILING != 0 @@ -3537,6 +3583,27 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) AOTFrame *frame; uint32 max_local_cell_num, max_stack_cell_num, all_cell_num; uint32 aot_func_idx, frame_size; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (exec_env->restore_call_chain) { + frame = exec_env->restore_call_chain[exec_env->call_chain_size - 1]; + LOG_DEBUG("frame restored, func idx %zu\n", frame->func_index); + exec_env->call_chain_size--; + frame->prev_frame = (AOTFrame *)exec_env->cur_frame; + exec_env->cur_frame = (struct WASMInterpFrame *)frame; + if (exec_env->call_chain_size == 0) { + // TODO: fix memory leak + exec_env->restore_call_chain = NULL; + } + LOG_DEBUG("restore call chain %zu==%u, %p, %p, %d\n", + ((AOTFrame *)exec_env->cur_frame)->func_index, func_index, + exec_env, exec_env->restore_call_chain, exec_env->handle); + if (((AOTFrame *)exec_env->cur_frame)->func_index != func_index) { + LOG_DEBUG("NOT MATCH!!!\n"); + exit(1); + } + return true; + } +#endif if (func_index >= module->import_func_count) { aot_func_idx = func_index - module->import_func_count; @@ -3568,6 +3635,11 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) frame->time_started = (uintptr_t)os_time_thread_cputime_us(); frame->func_perf_prof_info = func_perf_prof; #endif + frame->ip_offset = 0; + frame->sp = frame->lp + max_local_cell_num; +#if WASM_ENABLE_GC != 0 + frame->frame_ref = frame->sp + max_stack_cell_num; +#endif #if WASM_ENABLE_GC != 0 frame->sp = frame->lp + max_local_cell_num; @@ -3584,6 +3656,10 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) static inline void aot_free_frame_internal(WASMExecEnv *exec_env) { +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + int func_index = ((AOTFrame *)exec_env->cur_frame)->func_index; + LOG_DEBUG("aot_free_frame %zu %d\n", func_index, exec_env->handle); +#endif AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame; AOTFrame *prev_frame = cur_frame->prev_frame; @@ -3598,9 +3674,18 @@ aot_free_frame_internal(WASMExecEnv *exec_env) if (prev_frame) prev_frame->func_perf_prof_info->children_exec_time += time_elapsed; #endif - wasm_exec_env_free_wasm_frame(exec_env, cur_frame); exec_env->cur_frame = (struct WASMInterpFrame *)prev_frame; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (func_index == stop_func_index && is_debug) { + LOG_DEBUG("cur_func_count %d %d\n", cur_func_count, + stop_func_threshold); + if (cur_func_count == stop_func_threshold) { + serialize_to_file(exec_env); + } + cur_func_count++; + } +#endif } void diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 71baeb1716..93d43ba4ba 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -673,6 +673,9 @@ aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, bool aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index); +int +aot_raise(WASMExecEnv *exec_env, int exception); + void aot_free_frame(WASMExecEnv *exec_env); diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index bffa870cf7..4b0847b1c7 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -26,7 +26,9 @@ #if WASM_ENABLE_THREAD_MGR != 0 #include "thread_manager.h" #endif - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#include "ckpt_restore.h" +#endif /* * Thread Model: * - Only one wasm_engine_t in one process @@ -518,6 +520,12 @@ static bool search_thread_local_store_num(Vector *stores_by_tid, korp_tid tid, thread_local_stores *out_ts, unsigned *out_i) { +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + korp_tid temp; + if((temp = wamr_get_new_korp_tid(tid))){ + tid = temp; + } +#endif unsigned i; for (i = 0; i < stores_by_tid->num_elems; i++) { @@ -538,6 +546,12 @@ search_thread_local_store_num(Vector *stores_by_tid, korp_tid tid, static unsigned retrive_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) { +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + korp_tid temp; + if((temp = wamr_get_new_korp_tid(tid))){ + tid = temp; + } +#endif #ifndef os_thread_local_attribute unsigned i = 0; thread_local_stores ts = { 0 }; @@ -568,6 +582,12 @@ retrive_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) static bool increase_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) { +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + korp_tid temp; + if((temp = wamr_get_new_korp_tid(tid))){ + tid = temp; + } +#endif #ifndef os_thread_local_attribute unsigned i = 0; thread_local_stores ts = { 0 }; @@ -614,6 +634,12 @@ increase_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) static bool decrease_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) { +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + korp_tid temp; + if((temp = wamr_get_new_korp_tid(tid))){ + tid = temp; + } +#endif #ifndef os_thread_local_attribute unsigned i = 0; thread_local_stores ts = { 0 }; diff --git a/core/iwasm/common/wasm_exec_env.c b/core/iwasm/common/wasm_exec_env.c index 0b3778e60a..ef881b6a29 100644 --- a/core/iwasm/common/wasm_exec_env.c +++ b/core/iwasm/common/wasm_exec_env.c @@ -72,6 +72,7 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, exec_env->wasm_stack.top_boundary = exec_env->wasm_stack.bottom + stack_size; exec_env->wasm_stack.top = exec_env->wasm_stack.bottom; + exec_env->is_checkpoint = false; #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { @@ -85,6 +86,10 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, wasm_runtime_dump_exec_env_mem_consumption(exec_env); #endif + exec_env->is_checkpoint = false; + exec_env->is_restore = false; + exec_env->call_chain_size = 0; + exec_env->restore_call_chain = NULL; return exec_env; #ifdef OS_ENABLE_HW_BOUND_CHECK @@ -174,6 +179,7 @@ wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst, } #endif + #if WASM_ENABLE_THREAD_MGR != 0 /* Create a new cluster for this exec_env */ if (!(cluster = wasm_cluster_create(exec_env))) { diff --git a/core/iwasm/common/wasm_exec_env.h b/core/iwasm/common/wasm_exec_env.h index 4f93493efe..4447c3e50c 100644 --- a/core/iwasm/common/wasm_exec_env.h +++ b/core/iwasm/common/wasm_exec_env.h @@ -158,6 +158,12 @@ typedef struct WASMExecEnv { /* The WASM stack size */ uint32 wasm_stack_size; + /* Whether is checkpoint */ + bool is_checkpoint; + /* Whether is restore */ + bool is_restore; + size_t call_chain_size; + struct AOTFrame **restore_call_chain; /* The WASM stack of current thread */ union { diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index ee39242b1e..656399ec39 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -280,6 +280,14 @@ wasm_runtime_get_mem_alloc_info(mem_alloc_info_t *mem_alloc_info) return false; } +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define SHARED_MEMORY_LOCK(memory) shared_memory_lock(memory) +#define SHARED_MEMORY_UNLOCK(memory) shared_memory_unlock(memory) +#else +#define SHARED_MEMORY_LOCK(memory) (void)0 +#define SHARED_MEMORY_UNLOCK(memory) (void)0 +#endif + bool wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, uint32 app_offset, uint32 size) @@ -553,6 +561,8 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, return false; } + SHARED_MEMORY_LOCK(memory_inst); + native_addr = memory_inst->memory_data + app_buf_addr; bounds_checks = is_bounds_checks_enabled((wasm_module_inst_t)module_inst); @@ -566,32 +576,34 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, /* No need to check the app_offset and buf_size if memory access boundary check with hardware trap is enabled */ -#ifndef OS_ENABLE_HW_BOUND_CHECK - SHARED_MEMORY_LOCK(memory_inst); - - if (app_buf_addr >= memory_inst->memory_data_size) { - goto fail; - } - - if (!is_str) { - if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) { - goto fail; - } - } - else { - const char *str, *str_end; - - /* The whole string must be in the linear memory */ - str = (const char *)native_addr; - str_end = (const char *)memory_inst->memory_data_end; - while (str < str_end && *str != '\0') - str++; - if (str == str_end) - goto fail; - } +// #ifndef OS_ENABLE_HW_BOUND_CHECK +// SHARED_MEMORY_LOCK(memory_inst); + +// if (app_buf_addr >= memory_inst->memory_data_size) { +// goto fail; +// } + +// if (!is_str) { +// if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) { +// goto fail; +// } +// } +// else { +// const char *str, *str_end; + +// /* The whole string must be in the linear memory */ +// str = (const char *)native_addr; +// str_end = (const char *)memory_inst->memory_data_end; +// while (str < str_end && *str != '\0') +// str++; +// if (str == str_end) +// goto fail; +// } + +// SHARED_MEMORY_UNLOCK(memory_inst); +// #endif SHARED_MEMORY_UNLOCK(memory_inst); -#endif success: *p_native_addr = (void *)native_addr; diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 86bf14ffe5..a105ae91b8 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -39,7 +39,9 @@ #endif #include "../common/wasm_c_api_internal.h" #include "../../version.h" - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#include "../libraries/ckpt-restore/ckpt_restore.h" +#endif /** * For runtime build, BH_MALLOC/BH_FREE should be defined as * wasm_runtime_malloc/wasm_runtime_free. @@ -62,6 +64,10 @@ #undef CHECK #undef CHECK1 +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#undef wasm_runtime_invoke_native +#endif + #if WASM_ENABLE_MULTI_MODULE != 0 /** * A safety insurance to prevent @@ -3350,7 +3356,8 @@ wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, uint32 i; bool ret = false; - ctx = runtime_malloc(sizeof(*ctx), module_inst, error_buf, error_buf_size); + ctx = runtime_malloc(sizeof(WASIContext), module_inst, error_buf, + error_buf_size); if (!ctx) return false; uvwasi = &ctx->uvwasi; @@ -4993,6 +5000,9 @@ bool wasm_runtime_call_indirect(WASMExecEnv *exec_env, uint32 element_index, uint32 argc, uint32 argv[]) { +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + // LOG_DEBUG("wasm_runtime_call_indirect from %d\n", gettid()); +#endif bool ret = false; if (!wasm_runtime_exec_env_check(exec_env)) { @@ -6046,7 +6056,7 @@ wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, wasm_runtime_set_exception( module_inst, "native function throw unknown exception"); } - wasm_trap_delete(trap); + // wasm_trap_delete(trap); goto fail; } @@ -6539,3 +6549,25 @@ wasm_runtime_set_linux_perf(bool flag) enable_linux_perf = flag; } #endif + +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +bool +wasm_runtime_invoke_native_shim(WASMExecEnv *exec_env, void *func_ptr, + const WASMType *func_type, + const char *signature, void *attachment, + uint32 *argv, uint32 argc, uint32 *argv_ret) +{ +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + // Commented because assuming native funcs are not blocking + lightweight_checkpoint(exec_env); +#endif + bool ret = + wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, + attachment, argv, argc, argv_ret); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + // Commented because assuming native funcs are not blocking + lightweight_uncheckpoint(exec_env); +#endif + return ret; +} +#endif diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 0d449c3283..b95556fb53 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -1082,6 +1082,14 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, const char *signature, void *attachment, uint32 *argv, uint32 argc, uint32 *ret); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#define wasm_runtime_invoke_native wasm_runtime_invoke_native_shim +bool +wasm_runtime_invoke_native_shim(WASMExecEnv *exec_env, void *func_ptr, + const WASMType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *argv_ret); +#endif void wasm_runtime_read_v128(const uint8 *bytes, uint64 *ret1, uint64 *ret2); diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index 3856b7d19c..65d56f7cc9 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -3,12 +3,17 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +#include "aot_runtime.h" #include "bh_log.h" +#include "wasm_exec_env.h" +#include "wasm_runtime_common.h" #include "wasm_shared_memory.h" #if WASM_ENABLE_THREAD_MGR != 0 #include "../libraries/thread-mgr/thread_manager.h" #endif - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#include "../libraries/ckpt-restore/ckpt_restore.h" +#endif /* * Note: this lock can be per memory. * @@ -243,6 +248,11 @@ map_try_release_wait_info(HashMap *wait_hash_map, AtomicWaitInfo *wait_info, destroy_wait_info(wait_info); } +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_CHECKPOINT_RESTORE != 0 +extern korp_mutex syncop_mutex; +extern korp_cond syncop_cv; +#endif + uint32 wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, uint64 expect, int64 timeout, bool wait64) @@ -291,6 +301,12 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, /* Lock the shared_mem_lock for the whole atomic wait process, and use it to os_cond_reltimedwait */ os_mutex_lock(lock); +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (exec_env->is_restore) { + os_cond_signal(&syncop_cv); + os_mutex_unlock(&syncop_mutex); + } +#endif no_wait = (!wait64 && *(uint32 *)address != (uint32)expect) || (wait64 && *(uint64 *)address != expect); @@ -330,13 +346,34 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, /* unit of timeout is nsec, convert it to usec */ timeout_left = (uint64)timeout / 1000; timeout_1sec = (uint64)1e6; - +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (!exec_env->is_restore) { + printf("wait %p %ld %ld %ld %d\n", address, + ((uint8 *)address) + - ((WASMModuleInstance *)exec_env->module_inst) + ->memories[0] + ->memory_data, + expect, timeout, wait64); + insert_sync_op_atomic_wait(exec_env, address, expect, wait64); + } +#endif while (1) { if (timeout < 0) { /* wait forever until it is notified or terminatied here we keep waiting and checking every second */ +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_checkpoint(exec_env); +#endif os_cond_reltimedwait(&wait_node->wait_cond, lock, (uint64)timeout_1sec); +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_uncheckpoint(exec_env); + if (exec_env->is_restore) { + // os_mutex_unlock(&syncop_mutex); + printf("timed out so signaling again\n"); + os_cond_signal(&syncop_cv); + } +#endif if (wait_node->status == S_NOTIFIED /* notified by atomic.notify */ #if WASM_ENABLE_THREAD_MGR != 0 /* terminated by other thread */ @@ -378,7 +415,17 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, map_try_release_wait_info(wait_map, wait_info, address); os_mutex_unlock(lock); - +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (!exec_env->is_restore) { + printf("wake %p %ld %ld %ld %d\n", address, + ((uint8 *)address) + - ((WASMModuleInstance *)exec_env->module_inst) + ->memories[0] + ->memory_data, + expect, timeout, wait64); + insert_sync_op_atomic_wake(exec_env, address); + } +#endif return is_timeout ? 2 : 0; } @@ -426,10 +473,22 @@ wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, os_mutex_unlock(lock); return 0; } - +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_CHECKPOINT_RESTORE != 0 + WASMExecEnv *exec_env = + wasm_clusters_search_exec_env((WASMModuleInstanceCommon *)module_inst); + bh_assert(exec_env); + if (!exec_env->is_restore) { + printf("notify %p %ld %d\n", address, + ((uint8 *)address) + - ((WASMModuleInstance *)exec_env->module_inst) + ->memories[0] + ->memory_data, + count); + insert_sync_op_atomic_notify(exec_env, address, count); + } +#endif /* Notify each wait node in the wait list */ notify_result = notify_wait_list(wait_info->wait_list, count); - os_mutex_unlock(lock); return notify_result; diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 81ad9b7a38..ee067264d7 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -920,6 +920,671 @@ init_comp_frame(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +static bool +store_value(AOTCompContext *comp_ctx, LLVMValueRef value, uint8 value_type, + LLVMValueRef cur_frame, uint32 offset) +{ + LLVMValueRef value_offset, value_addr, value_ptr = NULL, res; + LLVMTypeRef value_ptr_type; + + if (!(value_offset = I32_CONST(offset))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "value_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + switch (value_type) { + case VALUE_TYPE_I32: + value_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + value_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F32: + value_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_F64: + value_ptr_type = F64_PTR_TYPE; + break; + case VALUE_TYPE_V128: + value_ptr_type = V128_PTR_TYPE; + break; + default: + bh_assert(0); + break; + } + + if (!(value_ptr = LLVMBuildBitCast(comp_ctx->builder, value_addr, + value_ptr_type, "value_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(res = LLVMBuildStore(comp_ctx->builder, value, value_ptr))) { + aot_set_last_error("llvm build store failed"); + return false; + } + + LLVMSetAlignment(res, 1); + + return true; +} + +bool +aot_gen_commit_value(AOTCompFrame *frame, bool reset_dirty_bit, AOTValueSlot **p, uint32 local_idx) { + AOTCompContext *comp_ctx = frame->comp_ctx; + AOTFuncContext *func_ctx = frame->func_ctx; + LLVMValueRef value; + LLVMTypeRef llvm_value_type; + uint32 n; + + if (!(*p)->dirty) { + switch ((*p)->type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + case VALUE_TYPE_F32: + case VALUE_TYPE_I1: + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + (*p)++; + break; + case VALUE_TYPE_V128: + (*p) += 3; + break; + default: + bh_assert(0); + break; + } + return true; + } + + if (reset_dirty_bit) + (*p)->dirty = 0; + n = (*p) - frame->lp; + + llvm_value_type = TO_LLVM_TYPE((*p)->type); + if (llvm_value_type == NULL) { + fprintf(stderr, "gen value %d type error\n", (*p)->type); + fprintf(stderr, "local_idx %d n %d\n", local_idx, n); + return false; + } + if (local_idx < func_ctx->aot_func->func_type->param_count + + func_ctx->aot_func->local_count) { + // fprintf(stderr, "LLVMBuildLoad2 %d < %d\n", local_idx, + // func_ctx->aot_func->local_count); + value = LLVMBuildLoad2(comp_ctx->builder, llvm_value_type, + func_ctx->locals[local_idx], + "commit_stack_load"); + // fprintf(stderr, "DONE LLVMBuildLoad2 %d\n", local_idx); + } + else { + if (!(*p)->value) { + fprintf(stderr, "value is null, %d %d\n", local_idx, n); + exit(-1); + } + value = LLVMBuildLoad2(comp_ctx->builder, llvm_value_type, (*p)->value, + "commit_stack_load"); + } + if (value == NULL) { + fprintf(stderr, "gen value load error\n"); + return false; + } + + switch ((*p)->type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + if (!store_value(comp_ctx, value, VALUE_TYPE_I32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_I64: + if (reset_dirty_bit) + ((*p)+1)->dirty = 0; + (*p) += 1; + if (!store_value(comp_ctx, value, VALUE_TYPE_I64, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_F32: + if (!store_value(comp_ctx, value, VALUE_TYPE_F32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_F64: + if (reset_dirty_bit) + ((*p)+1)->dirty = 0; + (*p) += 1; + if (!store_value(comp_ctx, value, VALUE_TYPE_F64, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_V128: + if (reset_dirty_bit) { + ((*p)+1)->dirty = 0; + ((*p)+2)->dirty = 0; + ((*p)+3)->dirty = 0; + } + (*p) += 3; + if (!store_value(comp_ctx, value, VALUE_TYPE_V128, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + case VALUE_TYPE_I1: + if (!(value = LLVMBuildZExt(comp_ctx->builder, value, I32_TYPE, + "i32_val"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (!store_value(comp_ctx, value, VALUE_TYPE_I32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n))) + return false; + break; + default: + bh_assert(0); + break; + } + return true; +} + +bool +aot_gen_commit_values(AOTCompFrame *frame, bool reset_dirty_bit) +{ + AOTCompContext *comp_ctx = frame->comp_ctx; + AOTValueSlot *p; + uint32 local_idx; + + reset_dirty_bit |= comp_ctx->enable_aux_stack_dirty_bit; + + for (p = frame->lp, local_idx = 0; p < frame->sp; p++, local_idx++) { + if (!aot_gen_commit_value(frame, reset_dirty_bit, &p, local_idx)) + return false; + } + + return true; +} + +static LLVMValueRef +load_value(AOTCompContext *comp_ctx, uint8 value_type, LLVMValueRef cur_frame, + uint32 offset) +{ + LLVMValueRef value_offset, value_addr, value_ptr = NULL, res; + LLVMTypeRef value_ptr_type, llvm_value_type; + + if (!(value_offset = I32_CONST(offset))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "value_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + switch (value_type) { + case VALUE_TYPE_I32: + llvm_value_type = I32_TYPE; + value_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + llvm_value_type = I64_TYPE; + value_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F32: + llvm_value_type = F32_TYPE; + value_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_F64: + llvm_value_type = F64_TYPE; + value_ptr_type = F64_PTR_TYPE; + break; + case VALUE_TYPE_V128: + llvm_value_type = V128_TYPE; + value_ptr_type = V128_PTR_TYPE; + break; + default: + bh_assert(0); + break; + } + + if (!(value_ptr = LLVMBuildBitCast(comp_ctx->builder, value_addr, + value_ptr_type, "value_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(res = LLVMBuildLoad2(comp_ctx->builder, llvm_value_type, value_ptr, + "restore_val"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + return res; +} + +bool +aot_gen_restore_values(AOTCompFrame *frame) +{ + AOTCompContext *comp_ctx = frame->comp_ctx; + AOTFuncContext *func_ctx = frame->func_ctx; + AOTValueSlot *p; + LLVMValueRef restore_value, store, value_ptr; + uint32 n, local_idx = 0; + + for (p = frame->lp; p < frame->sp; p++, local_idx++) { + n = p - frame->lp; + + if (local_idx < func_ctx->aot_func->func_type->param_count + + func_ctx->aot_func->local_count) { + value_ptr = func_ctx->locals[local_idx]; + } + else { + value_ptr = p->value; + } + if (!value_ptr) { + fprintf(stderr, "restore value is null, %d %d\n", local_idx, n); + exit(-1); + } + + switch (p->type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + if (!(restore_value = load_value(comp_ctx, VALUE_TYPE_I32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n)))) + return false; + break; + case VALUE_TYPE_I64: + ++p; + if (!(restore_value = load_value(comp_ctx, VALUE_TYPE_I64, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n)))) + return false; + break; + case VALUE_TYPE_F32: + if (!(restore_value = load_value(comp_ctx, VALUE_TYPE_F32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n)))) + return false; + break; + case VALUE_TYPE_F64: + ++p; + if (!(restore_value = load_value(comp_ctx, VALUE_TYPE_F64, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n)))) + return false; + break; + case VALUE_TYPE_V128: + ++p; + ++p; + ++p; + if (!(restore_value = load_value(comp_ctx, VALUE_TYPE_V128, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n)))) + return false; + break; + case VALUE_TYPE_I1: + if (!(restore_value = load_value(comp_ctx, VALUE_TYPE_I32, + func_ctx->cur_frame, + offset_of_local(comp_ctx, n)))) + return false; + if (!(restore_value = + LLVMBuildTrunc(comp_ctx->builder, restore_value, + INT1_TYPE, "restore_i1_val"))) { + aot_set_last_error("llvm build bit cast failed"); + } + break; + default: + bh_assert(0); + break; + } + + store = LLVMBuildStore(comp_ctx->builder, restore_value, value_ptr); + if (!store) { + aot_set_last_error("llvm build store failed"); + return false; + } + } + + return true; + +fail: + return false; +} + +bool +aot_gen_commit_sp_ip(AOTCompFrame *frame, const AOTValueSlot *sp, + const uint8 *ip) +{ + AOTCompContext *comp_ctx = frame->comp_ctx; + AOTFuncContext *func_ctx = frame->func_ctx; + LLVMValueRef cur_frame = func_ctx->cur_frame; + LLVMValueRef value_offset, value_addr, value_ptr, value; + LLVMTypeRef int8_ptr_ptr_type; + uint32 offset_ip, offset_sp, n; + bool is_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; + + if (!comp_ctx->is_jit_mode) { + offset_ip = offsetof(AOTFrame, ip_offset); + offset_sp = offsetof(AOTFrame, sp); + } + else { + offset_ip = offsetof(WASMInterpFrame, ip); + offset_sp = offsetof(WASMInterpFrame, sp); + } + + /* commit ip */ + + if (!(value_offset = I32_CONST(offset_ip))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "ip_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(value_ptr = LLVMBuildBitCast( + comp_ctx->builder, value_addr, + is_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE, "ip_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!comp_ctx->is_jit_mode) { + if (is_64bit) + value = + I64_CONST((uint64)(uintptr_t)(ip - func_ctx->aot_func->code)); + else + value = + I32_CONST((uint32)(uintptr_t)(ip - func_ctx->aot_func->code)); + } + else { + if (is_64bit) + value = I64_CONST((uint64)(uintptr_t)ip); + else + value = I32_CONST((uint32)(uintptr_t)ip); + } + + if (!value) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!LLVMBuildStore(comp_ctx->builder, value, value_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + + /* commit sp */ + + n = sp - frame->lp; + value = I32_CONST(offset_of_local(comp_ctx, n)); + if (!value) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value, 1, "sp"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(value_offset = I32_CONST(offset_sp))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "sp_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(int8_ptr_ptr_type = LLVMPointerType(INT8_PTR_TYPE, 0))) { + aot_set_last_error("llvm build pointer type failed"); + return false; + } + + if (!(value_ptr = LLVMBuildBitCast(comp_ctx->builder, value_addr, + int8_ptr_ptr_type, "sp_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!LLVMBuildStore(comp_ctx->builder, value, value_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + + /* commit sp */ + return true; +} + +bool +aot_gen_commit_ref_flags(AOTCompFrame *frame) +{ + /* TODO */ + return true; +} + +static bool +init_comp_frame(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 func_idx) +{ + AOTCompFrame *aot_frame; + WASMModule *wasm_module = comp_ctx->comp_data->wasm_module; + AOTFunc *aot_func = func_ctx->aot_func; + AOTFuncType *func_type = aot_func->func_type; + AOTBlock *block = func_ctx->block_stack.block_list_end; + LLVMValueRef local_value; + uint32 max_local_cell_num = + aot_func->param_cell_num + aot_func->local_cell_num; + uint32 max_stack_cell_num = aot_func->max_stack_cell_num; + uint32 all_cell_num = max_local_cell_num + max_stack_cell_num; + uint32 i, n; + uint64 total_size; + uint8 local_type; + + /* Free aot_frame if it was allocated previously for + compiling other functions */ + if (comp_ctx->aot_frame) { + wasm_runtime_free(comp_ctx->aot_frame); + comp_ctx->aot_frame = NULL; + } + + /* Allocate extra 2 cells since some operations may push more + operands than the number calculated in wasm loader, such as + PUSH_F64(F64_CONST(1.0)) in aot_compile_op_f64_promote_f32 */ + all_cell_num += 2; + total_size = offsetof(AOTCompFrame, lp) + + (uint64)sizeof(AOTValueSlot) * all_cell_num; + + if (total_size > UINT32_MAX + || !(comp_ctx->aot_frame = aot_frame = + wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + return false; + } + memset(aot_frame, 0, (uint32)total_size); + + aot_frame->cur_wasm_module = wasm_module; + aot_frame->cur_wasm_func = wasm_module->functions[func_idx]; + aot_frame->cur_wasm_func_idx = + func_idx + wasm_module->import_function_count; + aot_frame->comp_ctx = comp_ctx; + aot_frame->func_ctx = func_ctx; + + aot_frame->max_local_cell_num = max_local_cell_num; + aot_frame->max_stack_cell_num = max_stack_cell_num; + + aot_frame->sp = aot_frame->lp + max_local_cell_num; + + /* Init the frame_sp_begin of the function block */ + block->frame_sp_begin = aot_frame->sp; + + n = 0; + + /* Set all params dirty since they were set to llvm value but + haven't been committed to the AOT/JIT stack frame */ + for (i = 0; i < func_type->param_count; i++) { + local_type = func_type->types[i]; + local_value = LLVMGetParam(func_ctx->func, i + 1); + + switch (local_type) { + case VALUE_TYPE_I32: + set_local_i32(comp_ctx->aot_frame, n, local_value); + n++; + break; + case VALUE_TYPE_I64: + set_local_i64(comp_ctx->aot_frame, n, local_value); + n += 2; + break; + case VALUE_TYPE_F32: + set_local_f32(comp_ctx->aot_frame, n, local_value); + n++; + break; + case VALUE_TYPE_F64: + set_local_f64(comp_ctx->aot_frame, n, local_value); + n += 2; + break; + case VALUE_TYPE_V128: + set_local_v128(comp_ctx->aot_frame, n, local_value); + n += 4; + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + set_local_ref(comp_ctx->aot_frame, n, local_value, local_type); + n++; + break; + default: + bh_assert(0); + break; + } + } + + /* Set all locals dirty since they were set to llvm value but + haven't been committed to the AOT/JIT stack frame */ + for (i = 0; i < aot_func->local_count; i++) { + local_type = aot_func->local_types[i]; + + switch (local_type) { + case VALUE_TYPE_I32: + set_local_i32(comp_ctx->aot_frame, n, I32_ZERO); + n++; + break; + case VALUE_TYPE_I64: + set_local_i64(comp_ctx->aot_frame, n, I64_ZERO); + n += 2; + break; + case VALUE_TYPE_F32: + set_local_f32(comp_ctx->aot_frame, n, F32_ZERO); + n++; + break; + case VALUE_TYPE_F64: + set_local_f64(comp_ctx->aot_frame, n, F64_ZERO); + n += 2; + break; + case VALUE_TYPE_V128: + set_local_v128(comp_ctx->aot_frame, n, V128_f64x2_ZERO); + n += 4; + break; + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: + set_local_ref(comp_ctx->aot_frame, n, I32_ZERO, local_type); + n++; + break; + default: + bh_assert(0); + break; + } + } + + return true; +} + +bool +aot_gen_checkpoint(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + const uint8 *frame_ip) +{ + comp_ctx->inst_checkpointed = true; + if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, comp_ctx->aot_frame->sp, + frame_ip)) + return false; + if (!aot_gen_commit_values(comp_ctx->aot_frame, false)) + return false; + + char name[32]; + LLVMBasicBlockRef block_restore_jump, block_restore_value; + + snprintf(name, sizeof(name), "restore-%zu", + (uint64)(uintptr_t)(frame_ip - func_ctx->aot_func->code)); + if (!(block_restore_value = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, name))) { + aot_set_last_error("add LLVM basic block failed."); + return false; + } + + snprintf(name, sizeof(name), "restore-jump-%zu", + (uint64)(uintptr_t)(frame_ip - func_ctx->aot_func->code)); + if (!(block_restore_jump = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, name))) { + aot_set_last_error("add LLVM basic block failed."); + return false; + } + + LLVMMoveBasicBlockAfter(block_restore_value, + LLVMGetInsertBlock(comp_ctx->builder)); + LLVMMoveBasicBlockAfter(block_restore_jump, block_restore_value); + + if (!LLVMBuildBr(comp_ctx->builder, block_restore_jump)) { + aot_set_last_error("llvm build br failed."); + return false; + } + + LLVMValueRef ip_offset; + if (comp_ctx->pointer_size == sizeof(uint64)) + ip_offset = + I64_CONST((uint64)(uintptr_t)(frame_ip - func_ctx->aot_func->code)); + else + ip_offset = + I32_CONST((uint32)(uintptr_t)(frame_ip - func_ctx->aot_func->code)); + LLVMAddCase(func_ctx->restore_switch, ip_offset, block_restore_value); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_restore_value); + if (!aot_gen_restore_values(comp_ctx->aot_frame)) + return false; + LLVMBuildBr(comp_ctx->builder, block_restore_jump); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_restore_jump); + if (!aot_compile_emit_fence_nop(comp_ctx, func_ctx)) + return false; + + return true; +} + static bool aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) { @@ -941,6 +1606,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) float32 f32_const; float64 f64_const; AOTFuncType *func_type = NULL; + bool last_op_is_loop = false; #if WASM_ENABLE_DEBUG_AOT != 0 LLVMMetadataRef location; #endif @@ -955,13 +1621,72 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) LLVMPositionBuilderAtEnd( comp_ctx->builder, func_ctx->block_stack.block_list_head->llvm_entry_block); + + { + if (comp_ctx->aot_frame) { + uint32 offset_ip = offsetof(AOTFrame, ip_offset); + LLVMValueRef cur_frame = func_ctx->cur_frame; + LLVMValueRef value_offset, value_addr, value_ptr, value; + bool is_64bit = + (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; + if (!(value_offset = I32_CONST(offset_ip))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, cur_frame, &value_offset, 1, + "ip_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(value_ptr = LLVMBuildBitCast( + comp_ctx->builder, value_addr, + is_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE, "ip_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(value = LLVMBuildLoad2(comp_ctx->builder, + is_64bit ? I64_TYPE : I32_TYPE, + value_ptr, "init_ip"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + LLVMBasicBlockRef normal_block; + char name[32]; + snprintf(name, sizeof(name), "restore-no_restore"); + if (!(normal_block = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, name))) { + aot_set_last_error("add LLVM basic block failed."); + goto fail; + } + LLVMMoveBasicBlockAfter(normal_block, + LLVMGetInsertBlock(comp_ctx->builder)); + func_ctx->restore_switch = + LLVMBuildSwitch(comp_ctx->builder, value, normal_block, 0); + LLVMPositionBuilderAtEnd(comp_ctx->builder, normal_block); + + LLVMPositionBuilderBefore(comp_ctx->aot_frame_alloca_builder, + func_ctx->restore_switch); + } + } + while (frame_ip < frame_ip_end) { + // fprintf(stderr, "%p\n", (void*)comp_ctx->aot_frame); + comp_ctx->inst_checkpointed = false; opcode = *frame_ip++; + if (comp_ctx->enable_every_checkpoint || (comp_ctx->enable_loop_checkpoint && last_op_is_loop)) { + bh_assert(comp_ctx->aot_frame); + aot_gen_checkpoint(comp_ctx, func_ctx, frame_ip); + last_op_is_loop = false; + } if (comp_ctx->aot_frame) { comp_ctx->aot_frame->frame_ip = frame_ip - 1; } - #if WASM_ENABLE_DEBUG_AOT != 0 location = dwarf_gen_location( comp_ctx, func_ctx, @@ -980,8 +1705,13 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) case WASM_OP_BLOCK: case WASM_OP_LOOP: - case WASM_OP_IF: { + if (opcode == WASM_OP_LOOP) { + last_op_is_loop = true; + } + if (comp_ctx->enable_loop_checkpoint) { + aot_gen_commit_values(comp_ctx->aot_frame, true); + } value_type = *frame_ip++; if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_I64 || value_type == VALUE_TYPE_F32 @@ -1017,6 +1747,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) result_count = func_type->result_count; result_types = func_type->types + param_count; } + // generate the stable mapping to the register if (!aot_compile_op_block( comp_ctx, func_ctx, &frame_ip, frame_ip_end, (uint32)(LABEL_TYPE_BLOCK + opcode - WASM_OP_BLOCK), @@ -1025,6 +1756,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) break; } + case WASM_OP_IF: case EXT_OP_BLOCK: case EXT_OP_LOOP: case EXT_OP_IF: @@ -1045,8 +1777,8 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) } case WASM_OP_ELSE: - if (!aot_compile_op_else(comp_ctx, func_ctx, &frame_ip)) - return false; + aot_set_last_error("encounter opcode without aot csp support."); + goto fail; break; case WASM_OP_END: @@ -1054,16 +1786,34 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) return false; break; - case WASM_OP_BR: - { + case WASM_OP_BR: { + if (comp_ctx->enable_checkpoint && !comp_ctx->inst_checkpointed) { + if (comp_ctx->enable_br_checkpoint) { + aot_gen_checkpoint(comp_ctx, func_ctx, frame_ip); + } else { + if (comp_ctx->enable_aux_stack_dirty_bit) { + aot_gen_commit_values(comp_ctx->aot_frame, false); + } + } + } + read_leb_uint32(frame_ip, frame_ip_end, br_depth); if (!aot_compile_op_br(comp_ctx, func_ctx, br_depth, &frame_ip)) return false; break; } - case WASM_OP_BR_IF: - { + case WASM_OP_BR_IF: { + if (comp_ctx->enable_checkpoint && !comp_ctx->inst_checkpointed) { + if (comp_ctx->enable_br_checkpoint) { + aot_gen_checkpoint(comp_ctx, func_ctx, frame_ip); + } else { + if (comp_ctx->enable_aux_stack_dirty_bit) { + aot_gen_commit_values(comp_ctx->aot_frame, false); + } + } + } + read_leb_uint32(frame_ip, frame_ip_end, br_depth); if (!aot_compile_op_br_if(comp_ctx, func_ctx, br_depth, &frame_ip)) @@ -1071,8 +1821,17 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) break; } - case WASM_OP_BR_TABLE: - { + case WASM_OP_BR_TABLE: { + if (comp_ctx->enable_checkpoint && !comp_ctx->inst_checkpointed) { + if (comp_ctx->enable_br_checkpoint) { + aot_gen_checkpoint(comp_ctx, func_ctx, frame_ip); + } else { + if (comp_ctx->enable_aux_stack_dirty_bit) { + aot_gen_commit_values(comp_ctx->aot_frame, false); + } + } + } + read_leb_uint32(frame_ip, frame_ip_end, br_count); if (!(br_depths = wasm_runtime_malloc((uint32)sizeof(uint32) * (br_count + 1)))) { @@ -1100,6 +1859,16 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) #if WASM_ENABLE_FAST_INTERP == 0 case EXT_OP_BR_TABLE_CACHE: { + if (comp_ctx->enable_checkpoint && !comp_ctx->inst_checkpointed) { + if (comp_ctx->enable_br_checkpoint) { + aot_gen_checkpoint(comp_ctx, func_ctx, frame_ip); + } else { + if (comp_ctx->enable_aux_stack_dirty_bit) { + aot_gen_commit_values(comp_ctx->aot_frame, false); + } + } + } + BrTableCache *node = bh_list_first_elem( comp_ctx->comp_data->wasm_module->br_table_cache_list); BrTableCache *node_next; @@ -1133,16 +1902,29 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) case WASM_OP_CALL: { + uint8 *frame_ip_org = frame_ip; + if (comp_ctx->enable_checkpoint && !comp_ctx->inst_checkpointed) { + bh_assert(comp_ctx->aot_frame); + aot_gen_checkpoint(comp_ctx, func_ctx, frame_ip_org); + } + read_leb_uint32(frame_ip, frame_ip_end, func_idx); - if (!aot_compile_op_call(comp_ctx, func_ctx, func_idx, false)) + if (!aot_compile_op_call(comp_ctx, func_ctx, func_idx, false, + frame_ip_org)) return false; break; } case WASM_OP_CALL_INDIRECT: { + uint8 *frame_ip_org = frame_ip; uint32 tbl_idx; + if (comp_ctx->enable_checkpoint && !comp_ctx->inst_checkpointed) { + bh_assert(comp_ctx->aot_frame); + aot_gen_checkpoint(comp_ctx, func_ctx, frame_ip_org); + } + read_leb_uint32(frame_ip, frame_ip_end, type_idx); if (comp_ctx->enable_gc || comp_ctx->enable_ref_types) { @@ -1154,7 +1936,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) } if (!aot_compile_op_call_indirect(comp_ctx, func_ctx, type_idx, - tbl_idx)) + tbl_idx, frame_ip_org)) return false; break; } @@ -1168,7 +1950,8 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) } read_leb_uint32(frame_ip, frame_ip_end, func_idx); - if (!aot_compile_op_call(comp_ctx, func_ctx, func_idx, true)) + if (!aot_compile_op_call(comp_ctx, func_ctx, func_idx, true, + frame_ip_org)) return false; if (!aot_compile_op_return(comp_ctx, func_ctx, &frame_ip)) return false; @@ -1177,6 +1960,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) case WASM_OP_RETURN_CALL_INDIRECT: { + uint8 *frame_ip_org = frame_ip; uint32 tbl_idx; if (!comp_ctx->enable_tail_call) { @@ -1194,7 +1978,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) } if (!aot_compile_op_call_indirect(comp_ctx, func_ctx, type_idx, - tbl_idx)) + tbl_idx, frame_ip_org)) return false; if (!aot_compile_op_return(comp_ctx, func_ctx, &frame_ip)) return false; @@ -1850,12 +2634,22 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) read_leb_uint32(frame_ip, frame_ip_end, local_idx); if (!aot_compile_op_set_local(comp_ctx, func_ctx, local_idx)) return false; + if (comp_ctx->enable_loop_checkpoint && !comp_ctx->enable_aux_stack_dirty_bit) { + AOTValueSlot* p = comp_ctx->aot_frame->lp + comp_ctx->aot_frame->cur_wasm_func->local_offsets[local_idx]; + if (!aot_gen_commit_value(comp_ctx->aot_frame, true, &p, local_idx)) + return false; + } break; case WASM_OP_TEE_LOCAL: read_leb_uint32(frame_ip, frame_ip_end, local_idx); if (!aot_compile_op_tee_local(comp_ctx, func_ctx, local_idx)) return false; + if (comp_ctx->enable_loop_checkpoint && !comp_ctx->enable_aux_stack_dirty_bit) { + AOTValueSlot* p = comp_ctx->aot_frame->lp + comp_ctx->aot_frame->cur_wasm_func->local_offsets[local_idx]; + if (!aot_gen_commit_value(comp_ctx->aot_frame, true, &p, local_idx)) + return false; + } break; case WASM_OP_GET_GLOBAL: @@ -3989,6 +4783,8 @@ aot_compile_wasm(AOTCompContext *comp_ctx) } } + // aot_apply_mvvm_pass(comp_ctx, comp_ctx->module); + /* Run IR optimization before feeding in ORCJIT and AOT codegen */ if (comp_ctx->optimize) { /* Run passes for AOT/JIT mode. @@ -4312,4 +5108,4 @@ aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name) } return true; -} +} \ No newline at end of file diff --git a/core/iwasm/compilation/aot_compiler.h b/core/iwasm/compilation/aot_compiler.h index 5038a4130a..6ec4301b4b 100644 --- a/core/iwasm/compilation/aot_compiler.h +++ b/core/iwasm/compilation/aot_compiler.h @@ -187,6 +187,16 @@ offset_of_ref(AOTCompContext *comp_ctx, unsigned n) bool aot_gen_commit_values(AOTCompFrame *frame); +/** + * Generate instructions to restore computation result to the frame. + * The general principle is to only restore values that will be used + * through the frame. + * + * @param frame the frame information + */ +bool +aot_gen_restore_values(AOTCompFrame *frame); + /** * Generate instructions to commit SP and IP pointers to the frame. * @@ -198,6 +208,8 @@ aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip); bool aot_frame_store_value(AOTCompContext *comp_ctx, LLVMValueRef value, uint8 value_type, LLVMValueRef cur_frame, uint32 offset); +aot_gen_commit_sp_ip(AOTCompFrame *frame, const AOTValueSlot *sp, + const uint8 *ip); static inline void push_32bit(AOTCompFrame *frame, AOTValue *aot_value) diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 64947281a5..89bb9f1bd4 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -877,16 +877,6 @@ get_func_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data, else size = (uint32)sizeof(uint64) * comp_data->func_count; - /* function type indexes */ - size += (uint32)sizeof(uint32) * comp_data->func_count; - - /* aot_func#xxx + aot_func_internal#xxx in XIP mode for xtensa */ - if (need_call_wrapped_indirect(obj_data)) - size *= 2; - - /* max_local_cell_nums */ - size += (uint32)sizeof(uint32) * comp_data->func_count; - /* max_stack_cell_nums */ size += (uint32)sizeof(uint32) * comp_data->func_count; @@ -1570,11 +1560,6 @@ get_name_section_size(AOTCompData *comp_data) uint32 offset = 0; uint32 max_aot_buf_size = 0; - if (p >= p_end) { - aot_set_last_error("unexpected end"); - return 0; - } - max_aot_buf_size = 4 * (uint32)(p_end - p); if (!(buf = comp_data->aot_name_section_buf = wasm_runtime_malloc(max_aot_buf_size))) { @@ -1587,11 +1572,6 @@ get_name_section_size(AOTCompData *comp_data) offset = align_uint(offset, 4); EMIT_U32(name_len); - if (name_len == 0 || p + name_len > p_end) { - aot_set_last_error("unexpected end"); - return 0; - } - if (!check_utf8_str(p, name_len)) { aot_set_last_error("invalid UTF-8 encoding"); return 0; @@ -1642,6 +1622,7 @@ get_name_section_size(AOTCompData *comp_data) aot_set_last_error("out-of-order function index "); return 0; } + // printf("%d:%s\n",func_index,p); previous_func_index = func_index; read_leb_uint32(p, p_end, func_name_len); offset = align_uint(offset, 2); @@ -4467,9 +4448,20 @@ aot_obj_data_create(AOTCompContext *comp_ctx) if (comp_ctx->enable_gc) { obj_data->target_info.feature_flags |= WASM_FEATURE_GARBAGE_COLLECTION; } - +#if WASM_ENABLE_SHARED_MEMORY != 0 bh_print_time("Begin to resolve object file info"); + char *object_file_name = + malloc(strlen(comp_ctx->aot_file_name) + strlen(".o") + 1); + strcpy(object_file_name, comp_ctx->aot_file_name); + strcat(object_file_name, ".o"); + FILE *object_file = fopen(object_file_name, "w"); + fwrite(LLVMGetBufferStart(obj_data->mem_buf), 1, + LLVMGetBufferSize(obj_data->mem_buf), object_file); + fclose(object_file); + printf("Write object file to %s\n", object_file_name); + free(object_file_name); +#endif /* resolve target info/text/relocations/functions */ if (!aot_resolve_target_info(comp_ctx, obj_data) || !aot_resolve_text(obj_data) || !aot_resolve_literal(obj_data) @@ -4561,7 +4553,9 @@ aot_emit_aot_file(AOTCompContext *comp_ctx, AOTCompData *comp_data, uint32 aot_file_size; bool ret = false; FILE *file; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + comp_ctx->aot_file_name = file_name; +#endif bh_print_time("Begin to emit AOT file"); if (!(aot_file_buf = @@ -4586,6 +4580,8 @@ aot_emit_aot_file(AOTCompContext *comp_ctx, AOTCompData *comp_data, fail1: wasm_runtime_free(aot_file_buf); - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + comp_ctx->aot_file_name = NULL; +#endif return ret; -} +} \ No newline at end of file diff --git a/core/iwasm/compilation/aot_emit_control.c b/core/iwasm/compilation/aot_emit_control.c index 8b24bcab81..ec7b309fc0 100644 --- a/core/iwasm/compilation/aot_emit_control.c +++ b/core/iwasm/compilation/aot_emit_control.c @@ -11,6 +11,7 @@ #endif #include "../aot/aot_runtime.h" #include "../interpreter/wasm_loader.h" +#include "aot_emit_function.h" #if WASM_ENABLE_DEBUG_AOT != 0 #include "debug/dwarf_extractor.h" @@ -277,6 +278,11 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, block_prev = block->prev; block = aot_block_stack_pop(&func_ctx->block_stack); + if (aot_frame) { + /* Restore the frame sp */ + restore_frame_sp(block, aot_frame); + } + if (block->label_type == LABEL_TYPE_IF) { if (block->llvm_else_block && !block->skip_wasm_code_else && *p_frame_ip <= block->wasm_code_else) { @@ -366,6 +372,8 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } if (block->label_type == LABEL_TYPE_FUNCTION) { + if (comp_ctx->aot_frame) + call_aot_free_frame_func(comp_ctx, func_ctx); if (block->result_count) { /* Return the first return value */ if (!(ret = @@ -528,7 +536,7 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint8 *else_addr, *end_addr; LLVMValueRef value; char name[32]; - + // write to file about the mapping of the block that exists in every arch /* Check block stack */ if (!func_ctx->block_stack.block_list_end) { aot_set_last_error("WASM block stack underflow."); @@ -578,14 +586,14 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, block->wasm_code_end = end_addr; block->block_index = func_ctx->block_stack.block_index[label_type]; func_ctx->block_stack.block_index[label_type]++; - +#if WASM_ENABLE_CHECKPOINT_RESTORE == 0 if (comp_ctx->aot_frame) { if (label_type != LABEL_TYPE_BLOCK && comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame)) { goto fail; } } - +#endif if (label_type == LABEL_TYPE_BLOCK || label_type == LABEL_TYPE_LOOP) { /* Create block */ format_block_name(name, sizeof(name), block->block_index, label_type, @@ -751,13 +759,14 @@ aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, POP(value, block->result_types[result_index]); ADD_TO_RESULT_PHIS(block, value, result_index); } - +#if WASM_ENABLE_CHECKPOINT_RESTORE == 0 if (aot_frame) { bh_assert(block->frame_sp_begin == aot_frame->sp); if (comp_ctx->enable_gc && !aot_gen_commit_values(aot_frame)) { goto fail; } } +#endif /* Jump to end block */ BUILD_BR(block->llvm_end_block); @@ -810,14 +819,14 @@ aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if ((next_llvm_end_block = find_next_llvm_end_block(block))) MOVE_BLOCK_BEFORE(block->llvm_end_block, next_llvm_end_block); } - +#if WASM_ENABLE_CHECKPOINT_RESTORE == 0 if (comp_ctx->aot_frame) { if (block->label_type != LABEL_TYPE_FUNCTION && comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame)) { return false; } } - +#endif /* Handle block result values */ CREATE_RESULT_VALUE_PHIS(block); for (i = 0; i < block->result_count; i++) { @@ -830,6 +839,9 @@ aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (comp_ctx->aot_frame) { bh_assert(comp_ctx->aot_frame->sp == block->frame_sp_begin); +#if WASM_ENABLE_CHECKPOINT_RESTORE == 0 + restore_frame_sp(block, comp_ctx->aot_frame); +#endif } /* Jump to the end block */ @@ -1006,7 +1018,7 @@ aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!(block_dst = get_target_block(func_ctx, br_depth))) { return false; } - +#if WASM_ENABLE_CHECKPOINT_RESTORE == 0 if (comp_ctx->aot_frame) { if (comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame)) return false; @@ -1032,6 +1044,11 @@ aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!check_suspend_flags(comp_ctx, func_ctx, true)) return false; } + } + +#else + POP_COND(value_cmp); +#endif if (LLVMIsUndef(value_cmp) #if LLVM_VERSION_NUMBER >= 12 @@ -1048,6 +1065,16 @@ aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!LLVMIsEfficientConstInt(value_cmp)) { /* Compare value is not constant, create condition br IR */ +#if WASM_ENABLE_THREAD_MGR != 0 + /* Terminate or suspend current thread only when this is + a backward jump */ + if (comp_ctx->enable_thread_mgr + && block_dst->label_type == LABEL_TYPE_LOOP) { + if (!check_suspend_flags(comp_ctx, func_ctx, true)) + return false; + } +#endif + /* Create llvm else block */ CREATE_BLOCK(llvm_else_block, "br_if_else"); MOVE_BLOCK_AFTER_CURR(llvm_else_block); @@ -1355,6 +1382,8 @@ aot_compile_op_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, (*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code); #endif + if (comp_ctx->aot_frame) + call_aot_free_frame_func(comp_ctx, func_ctx); if (block_func->result_count) { /* Store extra result values to function parameters */ for (i = 0; i < block_func->result_count - 1; i++) { diff --git a/core/iwasm/compilation/aot_emit_exception.c b/core/iwasm/compilation/aot_emit_exception.c index 1e4cccbe68..d5ea8b869c 100644 --- a/core/iwasm/compilation/aot_emit_exception.c +++ b/core/iwasm/compilation/aot_emit_exception.c @@ -225,3 +225,56 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, fail: return false; } + +bool +aot_compile_emit_fence_nop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef value, inline_asm; +#if defined(_WIN32) + LLVMTypeRef param_types[] = { comp_ctx->exec_env_type, LLVMInt32Type() }; + LLVMTypeRef ret_type = LLVMInt32Type(); + LLVMValueRef param_values[2], func; + + LLVMTypeRef func_type,func_ptr_type; + GET_AOT_FUNCTION(aot_raise, 2); + LLVMValueRef arg = LLVMConstInt(LLVMInt32Type(), 4, false); + LLVMValueRef args[] = { func_ctx->exec_env, arg }; + if (!(value = LLVMBuildCall2(comp_ctx->builder, func_type, + func, args, 2, "call_aot_raise"))) { + aot_set_last_error("llvm build call nop failed."); + return false; + } +#else + if (!(value = LLVMBuildFence(comp_ctx->builder, + LLVMAtomicOrderingSequentiallyConsistent, + false, ""))) { + aot_set_last_error("llvm build fence failed."); + return false; + } + LLVMTypeRef ty = LLVMFunctionType(LLVMVoidType(), NULL, 0, false); + char *asm_string; + if (!strcmp(comp_ctx->target_arch, "x86_64") + || !strcmp(comp_ctx->target_arch, "i386")) { + asm_string = strdup("int $$3"); + } + else { + asm_string = strdup("svc 0"); + } + if (!(inline_asm = + LLVMGetInlineAsm(ty, asm_string, strlen(asm_string), + "~{dirflag},~{fpsr},~{flags}", 27, true, + false, LLVMInlineAsmDialectATT, false))) { + aot_set_last_error("llvm build nop asm failed."); + return false; + } + free(asm_string); + if (!(value = LLVMBuildCall2(comp_ctx->builder, ty, inline_asm, NULL, 0, + ""))) { + aot_set_last_error("llvm build call nop failed."); + return false; + } +#endif + return true; +fail: + return false; +} \ No newline at end of file diff --git a/core/iwasm/compilation/aot_emit_exception.h b/core/iwasm/compilation/aot_emit_exception.h index 91c8bd3cf5..f25f6b965d 100644 --- a/core/iwasm/compilation/aot_emit_exception.h +++ b/core/iwasm/compilation/aot_emit_exception.h @@ -16,6 +16,8 @@ bool aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, int32 exception_id, bool is_cond_br, LLVMValueRef cond_br_if, LLVMBasicBlockRef cond_br_else_block); +bool +aot_compile_emit_fence_nop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); #ifdef __cplusplus } /* end of extern "C" */ diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index 36bbc2222f..a291d97a20 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -4,6 +4,7 @@ */ #include "aot_emit_function.h" +#include "aot_compiler.h" #include "aot_emit_exception.h" #include "aot_emit_control.h" #include "aot_emit_table.h" @@ -192,6 +193,7 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return false; } } + // indirect mode here else if (comp_ctx->is_indirect_mode) { int32 func_index; if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { @@ -506,9 +508,6 @@ call_aot_alloc_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { LLVMValueRef param_values[2], ret_value, value, func; LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type; - LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef frame_alloc_fail, frame_alloc_success; - AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; param_types[0] = comp_ctx->exec_env_type; param_types[1] = I32_TYPE; @@ -531,33 +530,6 @@ call_aot_alloc_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return false; } - if (!(ret_value = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGT, ret_value, - I8_ZERO, "frame_alloc_ret"))) { - aot_set_last_error("llvm build icmp failed."); - return false; - } - - ADD_BASIC_BLOCK(frame_alloc_fail, "frame_alloc_fail"); - ADD_BASIC_BLOCK(frame_alloc_success, "frame_alloc_success"); - - LLVMMoveBasicBlockAfter(frame_alloc_fail, block_curr); - LLVMMoveBasicBlockAfter(frame_alloc_success, block_curr); - - if (!LLVMBuildCondBr(comp_ctx->builder, ret_value, frame_alloc_success, - frame_alloc_fail)) { - aot_set_last_error("llvm build cond br failed."); - return false; - } - - /* If frame alloc failed, return this function - so the runtime can catch the exception */ - LLVMPositionBuilderAtEnd(comp_ctx->builder, frame_alloc_fail); - if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) { - return false; - } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, frame_alloc_success); - return true; fail: @@ -1385,7 +1357,7 @@ commit_params_to_frame_of_import_func(AOTCompContext *comp_ctx, bool aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, - uint32 func_idx, bool tail_call) + uint32 func_idx, bool tail_call, const uint8 *frame_ip) { uint32 import_func_count = comp_ctx->comp_data->import_func_count; AOTImportFunc *import_funcs = comp_ctx->comp_data->import_funcs; @@ -1452,6 +1424,19 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Get param cell number */ param_cell_num = func_type->param_cell_num; +// #if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0) +// if (comp_ctx->enable_aux_stack_frame) { +// LLVMValueRef func_idx_const; + +// if (!(func_idx_const = I32_CONST(func_idx))) { +// aot_set_last_error("llvm build const failed."); +// return false; +// } +// if (!call_aot_alloc_frame_func(comp_ctx, func_ctx, func_idx_const)) +// return false; +// } +// #endif + /* Allocate memory for parameters. * Parameters layout: * - exec env @@ -1624,6 +1609,8 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } else { /* call native func directly */ + // calculate all + // know which function's id LLVMTypeRef native_func_type, func_ptr_type; LLVMValueRef func_ptr; @@ -1972,7 +1959,8 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, bool aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, - uint32 type_idx, uint32 tbl_idx) + uint32 type_idx, uint32 tbl_idx, + const uint8 *frame_ip) { AOTFuncType *func_type; LLVMValueRef tbl_idx_value, elem_idx, func_idx; @@ -2039,6 +2027,15 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, func_param_count = func_type->param_count; func_result_count = func_type->result_count; + // if (comp_ctx->aot_frame) { + // if (!aot_gen_commit_values(comp_ctx->aot_frame)) + // return false; + // if (!aot_gen_commit_sp_ip( + // comp_ctx->aot_frame, + // comp_ctx->aot_frame->sp - func_type->param_cell_num, frame_ip)) + // return false; + // } + POP_I32(elem_idx); /* get the cur size of the table instance */ diff --git a/core/iwasm/compilation/aot_emit_function.h b/core/iwasm/compilation/aot_emit_function.h index 45f7bbe596..697dc39515 100644 --- a/core/iwasm/compilation/aot_emit_function.h +++ b/core/iwasm/compilation/aot_emit_function.h @@ -14,11 +14,12 @@ extern "C" { bool aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, - uint32 func_idx, bool tail_call); + uint32 func_idx, bool tail_call, const uint8 *frame_ip); bool aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, - uint32 type_idx, uint32 tbl_idx); + uint32 type_idx, uint32 tbl_idx, + const uint8 *frame_ip); bool aot_compile_op_ref_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index c8417e6d6d..fbfda964ab 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -4,6 +4,7 @@ */ #include "aot_llvm.h" +#include "aot_emit_function.h" #include "aot_llvm_extra2.h" #include "aot_compiler.h" #include "aot_emit_exception.h" @@ -32,6 +33,8 @@ wasm_type_to_llvm_type(const AOTCompContext *comp_ctx, const AOTLLVMTypes *llvm_types, uint8 wasm_type) { switch (wasm_type) { + case VALUE_TYPE_I1: + return llvm_types->int1_type; case VALUE_TYPE_I32: return llvm_types->int32_type; case VALUE_TYPE_FUNCREF: @@ -1786,9 +1789,15 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx, goto fail; } - if (comp_ctx->enable_aux_stack_frame - && !create_aux_stack_frame(comp_ctx, func_ctx)) { - goto fail; + if (comp_ctx->enable_aux_stack_frame) { + LLVMValueRef func_idx_const; + if (!(func_idx_const = I32_CONST(func_index + module->import_function_count))) + goto fail; + if (!call_aot_alloc_frame_func(comp_ctx, func_ctx, func_idx_const)) + goto fail; + + if (!create_aux_stack_frame(comp_ctx, func_ctx)) + goto fail; } /* Create local variables */ @@ -2464,12 +2473,15 @@ aot_compiler_init(void) LLVMInitializeAllTargetInfos(); LLVMInitializeAllTargets(); LLVMInitializeAllTargetMCs(); + LLVMInitializeAllAsmParsers(); LLVMInitializeAllAsmPrinters(); #else /* Init environment of native for JIT compiler */ - LLVMInitializeNativeTarget(); - LLVMInitializeNativeTarget(); - LLVMInitializeNativeAsmPrinter(); + LLVMInitializeAllTargetInfos(); + LLVMInitializeAllTargets(); + LLVMInitializeAllTargetMCs(); + LLVMInitializeAllAsmParsers(); + LLVMInitializeAllAsmPrinters(); #endif return true; @@ -2526,6 +2538,12 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) goto fail; } + if (!(comp_ctx->aot_frame_alloca_builder = + LLVMCreateBuilderInContext(comp_ctx->context))) { + aot_set_last_error("create LLVM builder failed."); + goto fail; + } + /* Create LLVM module for each jit function, note: different from non ORC JIT mode, no need to dispose it, it will be disposed when the thread safe context is disposed */ @@ -2552,17 +2570,28 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) goto fail; } + LLVMAddModuleFlag( + comp_ctx->module, LLVMModuleFlagBehaviorWarning, "Debug Info Version", + strlen("Debug Info Version"), + LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 3, false))); + + LLVMAddModuleFlag( + comp_ctx->module, LLVMModuleFlagBehaviorWarning, "CodeView", + strlen("CodeView"), + LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 1, false))); + + LLVMAddModuleFlag( + comp_ctx->module, LLVMModuleFlagBehaviorWarning, "Dwarf Version", + strlen("Dwarf Version"), + LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 5, false))); + + #if WASM_ENABLE_DEBUG_AOT != 0 if (!(comp_ctx->debug_builder = LLVMCreateDIBuilder(comp_ctx->module))) { aot_set_last_error("create LLVM Debug Infor builder failed."); goto fail; } - LLVMAddModuleFlag( - comp_ctx->module, LLVMModuleFlagBehaviorWarning, "Debug Info Version", - strlen("Debug Info Version"), - LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 3, false))); - comp_ctx->debug_file = dwarf_gen_file_info(comp_ctx); if (!comp_ctx->debug_file) { aot_set_last_error("dwarf generate file info failed"); @@ -2595,6 +2624,20 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_memory_profiling) comp_ctx->enable_memory_profiling = true; + if (option->enable_checkpoint) + comp_ctx->enable_checkpoint = true; + + if (option->enable_loop_checkpoint) + comp_ctx->enable_loop_checkpoint = true; + + if (option->enable_br_checkpoint) + comp_ctx->enable_br_checkpoint = true; + + if (option->enable_every_checkpoint) + comp_ctx->enable_every_checkpoint = true; + + if (option->enable_aux_stack_dirty_bit) + comp_ctx->enable_aux_stack_dirty_bit = true; if (option->enable_aux_stack_check) comp_ctx->enable_aux_stack_check = true; @@ -3369,6 +3412,33 @@ void aot_value_stack_push(const AOTCompContext *comp_ctx, AOTValueStack *stack, AOTValue *value) { + LLVMValueRef alloca, store; + LLVMTypeRef llvm_value_type; + + llvm_value_type = TO_LLVM_TYPE(value->type); + if (llvm_value_type == NULL) { + fprintf(stderr, "gen value %d type error\n", value->type); + exit(-1); + } + if (comp_ctx->aot_frame) { + alloca = LLVMBuildAlloca(comp_ctx->aot_frame_alloca_builder, + llvm_value_type, "wasm_stack_alloca"); + } + else { + alloca = LLVMBuildAlloca(comp_ctx->builder, llvm_value_type, + "wasm_stack_alloca"); + } + if (alloca == NULL) { + fprintf(stderr, "gen value alloca error\n"); + exit(-1); + } + store = LLVMBuildStore(comp_ctx->builder, value->value, alloca); + if (store == NULL) { + fprintf(stderr, "gen value store error\n"); + exit(-1); + } + value->value = alloca; + if (!stack->value_list_head) stack->value_list_head = stack->value_list_end = value; else { @@ -3419,6 +3489,21 @@ aot_value_stack_pop(const AOTCompContext *comp_ctx, AOTValueStack *stack) bh_assert(stack->value_list_end); + LLVMValueRef llvm_value; + LLVMTypeRef llvm_value_type; + + llvm_value_type = TO_LLVM_TYPE(value->type); + if (llvm_value_type == NULL) { + fprintf(stderr, "gen value %d type error\n", value->type); + exit(-1); + } + llvm_value = LLVMBuildLoad2(comp_ctx->builder, llvm_value_type, + value->value, "wasm_stack_load"); + if (llvm_value == NULL) { + fprintf(stderr, "gen value load error\n"); + exit(-1); + } + if (stack->value_list_head == stack->value_list_end) stack->value_list_head = stack->value_list_end = NULL; else { diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index a47c054c92..eb0ed249cd 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -113,6 +113,12 @@ typedef struct AOTValueSlot { /* Frame information for translation */ typedef struct AOTCompFrame { + /* The current wasm module */ + WASMModule *cur_wasm_module; + /* The current wasm function */ + WASMFunction *cur_wasm_func; + /* The current wasm function index */ + uint32 cur_wasm_func_idx; /* The current compilation context */ struct AOTCompContext *comp_ctx; /* The current function context */ @@ -250,6 +256,8 @@ typedef struct AOTFuncContext { LLVMMetadataRef debug_func; #endif + LLVMValueRef restore_switch; + unsigned int stack_consumption_for_func_call; LLVMValueRef locals[1]; @@ -414,6 +422,12 @@ typedef struct AOTCompContext { /* Memory usage profiling */ bool enable_memory_profiling; + bool enable_checkpoint; + bool enable_loop_checkpoint; + bool enable_br_checkpoint; + bool enable_every_checkpoint; + bool inst_checkpointed; + bool enable_aux_stack_dirty_bit; /* Thread Manager */ bool enable_thread_mgr; @@ -510,6 +524,10 @@ typedef struct AOTCompContext { /* Current frame information for translation */ AOTCompFrame *aot_frame; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + /* AOT output file name, to generate {aot_file_name}.o file for analysis */ + const char *aot_file_name; +#endif } AOTCompContext; enum { @@ -619,6 +637,9 @@ aot_add_simple_loop_unswitch_pass(LLVMPassManagerRef pass); void aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module); +void +aot_apply_mvvm_pass(AOTCompContext *comp_ctx, LLVMModuleRef module); + void aot_handle_llvm_errmsg(const char *string, LLVMErrorRef err); diff --git a/core/iwasm/compilation/aot_llvm_extra.cpp b/core/iwasm/compilation/aot_llvm_extra.cpp index ed9447afc4..ec1140411b 100644 --- a/core/iwasm/compilation/aot_llvm_extra.cpp +++ b/core/iwasm/compilation/aot_llvm_extra.cpp @@ -3,6 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +#include +#include +#include +#include +#include "llvm/IR/InlineAsm.h" #include #include #if LLVM_VERSION_MAJOR < 17 @@ -32,6 +37,7 @@ #include #include #include +#include #include #include #if LLVM_VERSION_MAJOR >= 17 @@ -85,6 +91,9 @@ aot_add_simple_loop_unswitch_pass(LLVMPassManagerRef pass); void aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module); +void +aot_apply_mvvm_pass(AOTCompContext *comp_ctx, LLVMModuleRef module); + LLVM_C_EXTERN_C_END ExitOnError ExitOnErr; @@ -189,6 +198,82 @@ aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str) #endif /* WASM_ENABLE_SIMD */ } +struct AddNopPass : public PassInfoMixin { + public: + AddNopPass() {} + + int line_no = 1; + int count = 0; + + int getLineNo() { return line_no++; } + + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) + { + LLVMContext &context = F.getContext(); + + // To get a valid scope + DIBuilder di_builder(*F.getParent()); + auto f = di_builder.createFile("nop-pass", "."); + auto cu = di_builder.createCompileUnit(dwarf::DW_LANG_C, f, "Nop Pass", + false, "", 0); + SmallVector tys; + auto fty = di_builder.createSubroutineType( + di_builder.getOrCreateTypeArray(tys)); + + auto sp = di_builder.createFunction(f, F.getName(), StringRef(), f, 1, + fty, 1, DINode::FlagPrototyped, + DISubprogram::SPFlagDefinition); + F.setSubprogram(sp); + + IRBuilder<> builder(context); + Function *donothing_func = Function::Create( + FunctionType::get(builder.getVoidTy(), false), + Function::ExternalLinkage, "llvm.donothing", F.getParent()); + + for (Function::iterator bb = F.begin(), bbe = F.end(); bb != bbe; + ++bb) { + BasicBlock &b = *bb; + bool first_inst = true; + for (auto it = bb->begin(); it != bb->end(); it++) { + Instruction *instruction = &*it; + + it->setDebugLoc(DILocation::get(context, getLineNo(), 1, sp)); + + if (llvm::isa(instruction)) + continue; + + if (first_inst) { + builder.SetInsertPoint(instruction); + + auto functy = + llvm::FunctionType::get(builder.getVoidTy(), false); + auto inline_asm = llvm::InlineAsm::get( + functy, "nop", "~{dirflag},~{fpsr},~{flags}", true); + + auto dummy_inst = builder.CreateCall(donothing_func); + dummy_inst->setDebugLoc( + DILocation::get(context, getLineNo(), 2, sp)); + + auto fence_inst = builder.CreateFence(AtomicOrdering::SequentiallyConsistent); + fence_inst->setDebugLoc( + DILocation::get(context, getLineNo(), 514, sp)); + + auto asm_inst = builder.CreateCall(inline_asm); + asm_inst->setDebugLoc( + DILocation::get(context, getLineNo(), 514, sp)); + + count++; + + first_inst = false; + } + } + } + + di_builder.finalize(); + return PreservedAnalyses::all(); + } +}; + void aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module) { @@ -196,7 +281,7 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module) reinterpret_cast(comp_ctx->target_machine); PipelineTuningOptions PTO; PTO.LoopVectorization = true; - PTO.SLPVectorization = true; + PTO.SLPVectorization = false; PTO.LoopUnrolling = true; #if LLVM_VERSION_MAJOR >= 16 @@ -329,7 +414,7 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module) /* Apply Vectorize related passes for AOT mode */ FPM.addPass(LoopVectorizePass()); - FPM.addPass(SLPVectorizerPass()); + // FPM.addPass(SLPVectorizerPass()); FPM.addPass(LoadStoreVectorizerPass()); FPM.addPass(VectorCombinePass()); @@ -391,6 +476,33 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module) MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM1))); } } + MPM.run(*M, MAM); +} + +void +aot_apply_mvvm_pass(AOTCompContext *comp_ctx, LLVMModuleRef module) +{ + PassBuilder PB; + + /* Register all the basic analyses with the managers */ + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + Module *M = reinterpret_cast(module); + + ModulePassManager MPM; + + FunctionPassManager mvvm_fpm; + mvvm_fpm.addPass(AddNopPass()); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(mvvm_fpm))); MPM.run(*M, MAM); } diff --git a/core/iwasm/interpreter/wasm_interp.h b/core/iwasm/interpreter/wasm_interp.h index 1416405460..067580c1cd 100644 --- a/core/iwasm/interpreter/wasm_interp.h +++ b/core/iwasm/interpreter/wasm_interp.h @@ -115,6 +115,11 @@ uint8 * wasm_interp_get_frame_ref(WASMInterpFrame *frame); #endif +void +wasm_interp_call_func_bytecode(struct WASMModuleInstance *module, + struct WASMExecEnv *exec_env, + struct WASMFunctionInstance *cur_func, + struct WASMInterpFrame *prev_frame); #ifdef __cplusplus } #endif diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 16291b3f51..b1543efd29 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -27,7 +27,14 @@ #if WASM_ENABLE_FAST_JIT != 0 #include "../fast-jit/jit_compiler.h" #endif +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#include "../libraries/ckpt-restore/ckpt_restore.h" +const char *func_to_stop = NULL; +int func_to_stop_count = 0; +int func_count_ = 0; +#endif +uint64_t counter_ = 0; typedef int32 CellType_I32; typedef int64 CellType_I64; typedef float32 CellType_F32; @@ -645,11 +652,13 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame) #define GET_OPCODE() (void)0 #endif -#define DEF_OP_I_CONST(ctype, src_op_type) \ - do { \ - ctype cval; \ - read_leb_##ctype(frame_ip, frame_ip_end, cval); \ - PUSH_##src_op_type(cval); \ +#define DEF_OP_I_CONST(ctype, src_op_type) \ + do { \ + ctype cval; \ + read_leb_##ctype(frame_ip, frame_ip_end, cval); \ + PUSH_##src_op_type(cval); \ + LOG_DEBUG("i32.const %d %d %ld", cval, *frame_sp, \ + ((uint8 *)frame_sp) - exec_env->wasm_stack.s.bottom); \ } while (0) #define DEF_OP_EQZ(src_op_type) \ @@ -1329,7 +1338,33 @@ wasm_interp_call_func_import(WASMModuleInstance *module_inst, goto *handle_table[*frame_ip++]; \ } while (0) #else -#define HANDLE_OP_END() FETCH_OPCODE_AND_DISPATCH() +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#define HANDLE_OP_END() \ + do { \ + SYNC_ALL_TO_FRAME(); \ + if (!exec_env->is_restore) { \ + if (!exec_env->is_checkpoint && counter_ < SNAPSHOT_STEP) { \ + counter_++; \ + FETCH_OPCODE_AND_DISPATCH(); \ + } \ + else { \ + LOG_DEBUG("[%s:%d]\n", __FILE__, __LINE__); \ + counter_++; \ + if (counter_ > SNAPSHOT_STEP + SNAPSHOT_DEBUG_STEP) { \ + serialize_to_file(exec_env); \ + } \ + FETCH_OPCODE_AND_DISPATCH(); \ + } \ + } \ + else { \ + LOG_DEBUG("[%s:%d]\n", __FILE__, __LINE__); \ + counter_++; \ + FETCH_OPCODE_AND_DISPATCH(); \ + } \ + } while (0) +#else +#define HANDLE_OP_END() FETCH_OPCODE_AND_DISPATCH(); +#endif #endif #else /* else of WASM_ENABLE_LABELS_AS_VALUES */ @@ -1346,9 +1381,32 @@ wasm_interp_call_func_import(WASMModuleInstance *module_inst, os_mutex_unlock(&exec_env->wait_lock); \ continue #else +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#define HANDLE_OP_END() \ + SYNC_ALL_TO_FRAME(); \ + if (!exec_env->is_restore) { \ + if (!exec_env->is_checkpoint && counter_ < SNAPSHOT_STEP) { \ + counter_++; \ + continue; \ + } \ + else { \ + LOG_DEBUG("[%s:%d]\n", __FILE__, __LINE__); \ + counter_++; \ + if (counter_ > SNAPSHOT_STEP + SNAPSHOT_DEBUG_STEP) { \ + serialize_to_file(exec_env); \ + } \ + } \ + } \ + else { \ + LOG_DEBUG("[%s:%d]\n", __FILE__, __LINE__); \ + counter_++; \ + continue; \ + } +#else #define HANDLE_OP_END() continue #endif +#endif #endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ static inline uint8 * @@ -1364,7 +1422,7 @@ get_global_addr(uint8 *global_data, WASMGlobalInstance *global) #endif } -static void +void wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMExecEnv *exec_env, WASMFunctionInstance *cur_func, @@ -1411,6 +1469,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, int32_t exception_tag_index; #endif uint8 value_type; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (exec_env->is_restore) { + // WASMFunction *cur_wasm_func = cur_func->u.func; + frame = exec_env->cur_frame; + UPDATE_ALL_FROM_FRAME(); + frame_ip_end = wasm_get_func_code_end(cur_func); + frame_lp = frame->lp; + } +#endif #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 #if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 @@ -1465,7 +1532,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } + HANDLE_OP(WASM_OP_NOP) + { + HANDLE_OP_END(); + } #if WASM_ENABLE_EXCE_HANDLING != 0 HANDLE_OP(WASM_OP_RETHROW) @@ -1976,6 +2046,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_END) { if (frame_csp > frame->csp_bottom + 1) { + LOG_DEBUG("csp %p csp_bottom %p", frame_csp, + frame->csp_bottom); POP_CSP(); } else { /* end of function, treat as WASM_OP_RETURN */ @@ -3932,6 +4004,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, GET_I64_FROM_ADDR(frame_lp + (local_offset & 0x7F))); else PUSH_I32(*(int32 *)(frame_lp + local_offset)); + LOG_DEBUG("local.get %d %d %ld %d %ld", local_offset, *frame_lp, + ((uint8 *)frame_lp) - exec_env->wasm_stack.s.bottom, + *frame_sp, + ((uint8 *)frame_sp) - exec_env->wasm_stack.s.bottom); + HANDLE_OP_END(); } @@ -3978,6 +4055,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, POP_I64()); else *(int32 *)(frame_lp + local_offset) = POP_I32(); + LOG_DEBUG("local.set %d %d %ld", local_offset, *frame_sp, + ((uint8 *)frame_sp) - exec_env->wasm_stack.s.bottom); + HANDLE_OP_END(); } @@ -4121,6 +4201,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); + LOG_DEBUG("set.global %d %d %ld", global_idx, *frame_sp, + ((uint8 *)frame_sp) - exec_env->wasm_stack.s.bottom); PUT_I64_TO_ADDR((uint32 *)global_addr, POP_I64()); HANDLE_OP_END(); } @@ -4134,6 +4216,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); addr = POP_I32(); + LOG_DEBUG("load.i32 %d %d %d %ld", offset, flags, *frame_sp, + ((uint8 *)frame_sp) - exec_env->wasm_stack.s.bottom); CHECK_MEMORY_OVERFLOW(4); PUSH_I32(LOAD_I32(maddr)); CHECK_READ_WATCHPOINT(addr, offset); @@ -4680,6 +4764,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_ADD) { DEF_OP_NUMERIC(uint32, uint32, I32, +); + LOG_DEBUG("i32.add %d %ld", *frame_sp, + ((uint8 *)frame_sp) - exec_env->wasm_stack.s.bottom); HANDLE_OP_END(); } @@ -5329,7 +5415,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_REINTERPRET_F32) HANDLE_OP(WASM_OP_I64_REINTERPRET_F64) HANDLE_OP(WASM_OP_F32_REINTERPRET_I32) - HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) { HANDLE_OP_END(); } + HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) + { + HANDLE_OP_END(); + } HANDLE_OP(WASM_OP_I32_EXTEND8_S) { @@ -6389,7 +6478,20 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* Called from native. */ return; } - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("return_func: %s %p", + prev_frame->function->u.func->field_name, prev_frame); + if (func_to_stop + && !strcmp(prev_frame->function->u.func->field_name, + func_to_stop)) { + if (func_to_stop_count > func_count_) { + func_count_++; + } + else { + counter_ = INT_MAX; + } + } +#endif RECOVER_CONTEXT(prev_frame); #if WASM_ENABLE_EXCE_HANDLING != 0 if (wasm_get_exception(module)) { diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index a75a204bbe..3e2e73eb3d 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2279,6 +2279,9 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, uint8 *memory_data = NULL; uint64 memory_size = 0; WASMDataSeg *data_seg = module->data_segments[i]; +#if WASM_ENABLE_SHARED_MEMORY != 0 + bool is_shared_memory; +#endif #if WASM_ENABLE_BULK_MEMORY != 0 if (data_seg->is_passive) @@ -2289,6 +2292,23 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, initialized */ continue; +#if WASM_ENABLE_SHARED_MEMORY != 0 + /* Currently we have only one memory instance */ + if (module->import_memory_count > 0) { + is_shared_memory = (module->import_memories[0].u.memory.flags & 0x2) + ? true + : false; + } + else { + is_shared_memory = (module->memories[0].flags & 0x2) ? true : false; + } + if (is_shared_memory && parent != NULL) { + /* Ignore setting memory init data if the memory has been + * initialized */ + continue; + } +#endif + /* has check it in loader */ memory = module_inst->memories[data_seg->memory_index]; bh_assert(memory); @@ -3092,14 +3112,27 @@ call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst, wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node); if (os_setjmp(jmpbuf_node.jmpbuf) == 0) { -#ifndef BH_PLATFORM_WINDOWS +#if defined(__MSVC__) + __try { wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); + } __except (wasm_copy_exception(module_inst, NULL) + ? EXCEPTION_EXECUTE_HANDLER + : EXCEPTION_CONTINUE_SEARCH) { + /* exception was thrown in wasm_exception_handler */ + ret = false; + } + has_exception = wasm_copy_exception(module_inst, exception); + if (has_exception && strstr(exception, "native stack overflow")) { + /* After a stack overflow, the stack was left + in a damaged state, let the CRT repair it */ + result = _resetstkoflw(); + bh_assert(result != 0); + } #else __try { wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); } __except (wasm_copy_exception(module_inst, NULL) ? EXCEPTION_EXECUTE_HANDLER - : EXCEPTION_CONTINUE_SEARCH) { /* Exception was thrown in wasm_exception_handler */ ret = false; } @@ -3112,9 +3145,6 @@ call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst, } #endif } - else { - /* Exception has been set in signal handler before calling longjmp */ - ret = false; } /* Note: can't check wasm_get_exception(module_inst) here, there may be @@ -3125,7 +3155,7 @@ call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst, if (wasm_interp_create_call_stack(exec_env)) { wasm_interp_dump_call_stack(exec_env, true, NULL, 0); } -#endif +#endif /* Restore operand frames */ wasm_exec_env_set_cur_frame(exec_env, prev_frame); exec_env->wasm_stack.top = prev_top; @@ -3281,10 +3311,7 @@ wasm_module_malloc_internal(WASMModuleInstance *module_inst, return 0; } - if (memory->heap_handle) { - addr = mem_allocator_malloc(memory->heap_handle, size); - } - else if (module_inst->e->malloc_function && module_inst->e->free_function) { + if (module_inst->e->malloc_function && module_inst->e->free_function) { if (!execute_malloc_function( module_inst, exec_env, module_inst->e->malloc_function, module_inst->e->retain_function, size, &offset)) { diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 1007dc27cb..95d29a728f 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -95,7 +95,14 @@ typedef union { uint64 u64; uint32 u32[2]; } MemBound; - +#if WASM_ENABLE_CHECKPOINT_RESTORE == 0 +struct SocketAddrPool { + uint32 ip4[4]; + uint16 ip6[8]; + bool is_4; + uint16 port; +}; +#endif struct WASMMemoryInstance { /* Module type */ uint32 module_type; diff --git a/core/iwasm/libraries/ckpt-restore/ckpt-restore.cmake b/core/iwasm/libraries/ckpt-restore/ckpt-restore.cmake new file mode 100644 index 0000000000..4f5905f2ce --- /dev/null +++ b/core/iwasm/libraries/ckpt-restore/ckpt-restore.cmake @@ -0,0 +1,12 @@ +# Copyright (C) 2021 Ant Group. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (CKPT_RESTORE_DIR ${CMAKE_CURRENT_LIST_DIR}) + +add_definitions (-DWASM_ENABLE_CHECKPOINT_RESTORE=1) + +include_directories(${CKPT_RESTORE_DIR}) + +file (GLOB source_all ${CKPT_RESTORE_DIR}/*.c) + +set (CKPT_RESTORE_SOURCE ${source_all}) diff --git a/core/iwasm/libraries/ckpt-restore/ckpt_restore.h b/core/iwasm/libraries/ckpt-restore/ckpt_restore.h new file mode 100755 index 0000000000..8d15f875e1 --- /dev/null +++ b/core/iwasm/libraries/ckpt-restore/ckpt_restore.h @@ -0,0 +1,93 @@ +/* + * The WebAssembly Live Migration Project + * + * By: Aibo Hu + * Yiwei Yang + * Brian Zhao + * Andrew Quinn + * + * Copyright 2024 Regents of the Univeristy of California + * UC Santa Cruz Sluglab. + */ + +#include "platform_common.h" +#include "wasm_runtime.h" + +#define SNAPSHOT_DEBUG_STEP 0 +#define SNAPSHOT_STEP 1e8 +enum fd_op { MVVM_FOPEN = 0, MVVM_FWRITE = 1, MVVM_FREAD = 2, MVVM_FSEEK = 3 }; + +enum sync_op { + SYNC_OP_MUTEX_LOCK = 0, + SYNC_OP_MUTEX_UNLOCK, + SYNC_OP_COND_WAIT, + SYNC_OP_COND_SIGNAL, + SYNC_OP_COND_BROADCAST, + SYNC_OP_ATOMIC_WAIT, + SYNC_OP_ATOMIC_NOTIFY, +}; + +struct sync_op_t { + uint64 tid; + uint32 ref; + enum sync_op sync_op; + uint64 expected; + bool wait64; +}; + +#if WASM_ENABLE_LIBC_WASI != 0 +void insert_sock_send_to_data(uint32_t, uint8 *, uint32, uint16_t, __wasi_addr_t *); +void insert_sock_open_data(uint32_t, int, int, uint32_t); +void insert_sock_recv_from_data(uint32_t, uint8 *, uint32, uint16_t, __wasi_addr_t *); +void replay_sock_recv_from_data(uint32_t, uint8 **, unsigned long *, __wasi_addr_t *); +void insert_socket(int, int, int, int); +void update_socket_fd_address(int, struct SocketAddrPool *); +void init_gateway(struct SocketAddrPool *address); +void set_tcp(); +int get_sock_fd(int); +#endif +#if WASM_ENABLE_LIB_PTHREAD != 0 +void insert_sync_op(wasm_exec_env_t exec_env, const uint32 *mutex, enum sync_op locking); +void insert_sync_op_atomic_notify(wasm_exec_env_t exec_env, const uint32 *mutex, uint32); +void insert_sync_op_atomic_wait(wasm_exec_env_t exec_env, const uint32 *mutex, uint64, bool); +void insert_sync_op_atomic_wake(wasm_exec_env_t exec_env, const uint32 *mutex); +void restart_execution(uint32 targs); +void insert_tid_start_arg(uint64_t, size_t, size_t); +void insert_parent_child(uint64_t, uint64_t); +extern int pthread_create_wrapper(wasm_exec_env_t exec_env, uint32 *thread, const void *attr, uint32 elem_index, + uint32 arg); +extern int32 pthread_mutex_lock_wrapper(wasm_exec_env_t, uint32 *); +extern int32 pthread_mutex_unlock_wrapper(wasm_exec_env_t, uint32 *); +extern int32 pthread_mutex_init_wrapper(wasm_exec_env_t, uint32 *, void *); +extern int32 thread_spawn_wrapper(wasm_exec_env_t exec_env, uint32 start_arg); +extern int32 pthread_cond_wait_wrapper(wasm_exec_env_t, uint32 *, uint32 *); +extern int32 pthread_cond_signal_wrapper(wasm_exec_env_t, uint32 *); +extern int32 pthread_cond_broadcast_wrapper(wasm_exec_env_t, uint32 *); +extern uint32 wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, uint64 expect, int64 timeout, + bool wait64); +extern uint32 wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, uint32 count); +#endif +void serialize_to_file(struct WASMExecEnv *); + +void insert_fd(int, const char *, int, int, enum fd_op op); +void remove_fd(int); +void rename_fd(int, char const *, int, char const *); +bool is_atomic_checkpointable(); +void wamr_handle_map(uint64_t old_tid, uint64_t new_tid); +korp_tid wamr_get_new_korp_tid(korp_tid new_tid); +korp_tid wamr_get_korp_tid(korp_tid new_tid); +void lightweight_checkpoint(WASMExecEnv *); +void lightweight_uncheckpoint(WASMExecEnv *); +void wamr_wait(wasm_exec_env_t); +void sigint_handler(int sig); +void register_sigtrap(); +void register_sigint(); +void sigtrap_handler(int sig); +extern size_t snapshot_threshold; +extern int stop_func_threshold; +extern bool checkpoint; +extern bool is_debug; +extern int stop_func_index; +extern int cur_func_count; +void print_memory(WASMExecEnv *); +void print_exec_env_debug_info(WASMExecEnv *); diff --git a/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c b/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c index 3092f5d031..e1e0cd12e9 100644 --- a/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c +++ b/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c @@ -18,6 +18,10 @@ #include "aot_runtime.h" #endif +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#include "../ckpt-restore/ckpt_restore.h" +#endif + #define WAMR_PTHREAD_KEYS_MAX 32 /* clang-format off */ @@ -511,12 +515,16 @@ pthread_start_routine(void *arg) wasm_exec_env_set_thread_info(exec_env); argv[0] = routine_args->arg; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + register_sigtrap(); + if (exec_env->is_restore) { + wamr_wait(exec_env); + } +#endif if (!wasm_runtime_call_indirect(exec_env, routine_args->elem_index, 1, argv)) { /* Exception has already been spread during throwing */ } - /* destroy pthread key values */ call_key_destructor(exec_env); @@ -544,7 +552,7 @@ pthread_start_routine(void *arg) return (void *)(uintptr_t)argv[0]; } -static int +int pthread_create_wrapper(wasm_exec_env_t exec_env, uint32 *thread, /* thread_handle */ const void *attr, /* not supported */ @@ -578,18 +586,19 @@ pthread_create_wrapper(wasm_exec_env_t exec_env, } #endif - if (!(new_module_inst = wasm_runtime_instantiate_internal( - module, module_inst, exec_env, stack_size, 0, NULL, 0))) - return -1; + new_module_inst = wasm_exec_env_get_module_inst(exec_env); + // if (!(new_module_inst = wasm_runtime_instantiate_internal( + // module, module_inst, exec_env, stack_size, 0, NULL, 0))) + // return -1; - /* Set custom_data to new module instance */ - wasm_runtime_set_custom_data_internal( - new_module_inst, wasm_runtime_get_custom_data(module_inst)); + // /* Set custom_data to new module instance */ + // wasm_runtime_set_custom_data_internal( + // new_module_inst, wasm_runtime_get_custom_data(module_inst)); - wasm_native_inherit_contexts(new_module_inst, module_inst); + // wasm_native_inherit_contexts(new_module_inst, module_inst); - if (!(wasm_cluster_dup_c_api_imports(new_module_inst, module_inst))) - goto fail; + // if (!(wasm_cluster_dup_c_api_imports(new_module_inst, module_inst))) + // goto fail; if (!(info_node = wasm_runtime_malloc(sizeof(ThreadInfoNode)))) goto fail; @@ -622,6 +631,9 @@ pthread_create_wrapper(wasm_exec_env_t exec_env, goto fail; } +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + insert_parent_child(((uint64_t)exec_env->handle), routine_args->arg); +#endif os_mutex_lock(&exec_env->wait_lock); ret = wasm_cluster_create_thread( exec_env, new_module_inst, true, aux_stack_start, aux_stack_size, @@ -688,8 +700,16 @@ pthread_join_wrapper(wasm_exec_env_t exec_env, uint32 thread, bh_assert(target_exec_env); if (node->status != THREAD_EXIT) { +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_checkpoint(exec_env); + LOG_DEBUG("start join thread \n"); +#endif /* if the thread is still running, call the platforms join API */ join_ret = wasm_cluster_join_thread(target_exec_env, (void **)&ret); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + printf("finish join thread \n"); + lightweight_uncheckpoint(exec_env); +#endif } else { /* if the thread has exited, return stored results */ @@ -807,7 +827,7 @@ pthread_exit_wrapper(wasm_exec_env_t exec_env, int32 retval_offset) wasm_cluster_exit_thread(exec_env, (void *)(uintptr_t)retval_offset); } -static int32 +int32 pthread_mutex_init_wrapper(wasm_exec_env_t exec_env, uint32 *mutex, void *attr) { korp_mutex *pmutex; @@ -850,23 +870,33 @@ pthread_mutex_init_wrapper(wasm_exec_env_t exec_env, uint32 *mutex, void *attr) return -1; } -static int32 +int32 pthread_mutex_lock_wrapper(wasm_exec_env_t exec_env, uint32 *mutex) { ThreadInfoNode *info_node = get_thread_info(exec_env, *mutex); if (!info_node || info_node->type != T_MUTEX) return -1; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_checkpoint(exec_env); +#endif - return os_mutex_lock(info_node->u.mutex); + int32 rc = os_mutex_lock(info_node->u.mutex); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_uncheckpoint(exec_env); + insert_sync_op(exec_env, mutex, SYNC_OP_MUTEX_LOCK); +#endif + return rc; } -static int32 +int32 pthread_mutex_unlock_wrapper(wasm_exec_env_t exec_env, uint32 *mutex) { ThreadInfoNode *info_node = get_thread_info(exec_env, *mutex); if (!info_node || info_node->type != T_MUTEX) return -1; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + insert_sync_op(exec_env, mutex, SYNC_OP_MUTEX_UNLOCK); +#endif return os_mutex_unlock(info_node->u.mutex); } @@ -929,7 +959,7 @@ pthread_cond_init_wrapper(wasm_exec_env_t exec_env, uint32 *cond, void *attr) return -1; } -static int32 +int32 pthread_cond_wait_wrapper(wasm_exec_env_t exec_env, uint32 *cond, uint32 *mutex) { ThreadInfoNode *cond_info_node, *mutex_info_node; @@ -942,7 +972,14 @@ pthread_cond_wait_wrapper(wasm_exec_env_t exec_env, uint32 *cond, uint32 *mutex) if (!mutex_info_node || mutex_info_node->type != T_MUTEX) return -1; - return os_cond_wait(cond_info_node->u.cond, mutex_info_node->u.mutex); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_checkpoint(exec_env); +#endif + int32 rc = os_cond_wait(cond_info_node->u.cond, mutex_info_node->u.mutex); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_uncheckpoint(exec_env); +#endif + return rc; } /** @@ -967,7 +1004,7 @@ pthread_cond_timedwait_wrapper(wasm_exec_env_t exec_env, uint32 *cond, mutex_info_node->u.mutex, useconds); } -static int32 +int32 pthread_cond_signal_wrapper(wasm_exec_env_t exec_env, uint32 *cond) { ThreadInfoNode *info_node = get_thread_info(exec_env, *cond); @@ -977,7 +1014,7 @@ pthread_cond_signal_wrapper(wasm_exec_env_t exec_env, uint32 *cond) return os_cond_signal(info_node->u.cond); } -static int32 +int32 pthread_cond_broadcast_wrapper(wasm_exec_env_t exec_env, uint32 *cond) { ThreadInfoNode *info_node = get_thread_info(exec_env, *cond); diff --git a/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h b/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h index 98429bbb0a..fb58aa514b 100644 --- a/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h +++ b/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h @@ -993,7 +993,11 @@ __wasi_sock_get_ipv6_only(__wasi_fd_t fd, bool *option) * since don't want to re-compile the wasi-libc, * we tend to keep original implentations of recv() and send(). */ - +int s_(int domain, int socktype, int protocol, uint32_t sockfd); +int o_(char *path, int fd, int offset); +void ci_(int socket_fd); +void si_(int socket_fd); +uint32_t inet_addr(const char *ip); #ifdef __cplusplus } #endif diff --git a/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c b/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c index 1172d0a773..b89fc04867 100644 --- a/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c +++ b/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #define HANDLE_ERROR(error) \ @@ -17,7 +19,129 @@ errno = error; \ return -1; \ } +uint32_t +inet_addr(const char *ip) +{ + uint32_t result = 0; + unsigned int part; + const char *start; + + start = ip; + for (int i = 0; i < 4; i++) { + char c; + part = 0; + while ((c = *start++) != '\0') { + if (c == '.') { + break; + } + if (c < '0' || c > '9') { + return -1; // Invalid character encountered + } + part = part * 10 + (c - '0'); + } + if (part > 255) { + return -1; // Single part is larger than 255 + } + result = result | (part << (i * 8)); + } + + return result; +} +static void +init_sockaddr_sever_inet(struct sockaddr_in *addr) +{ + /* 0.0.0.0:1234 */ + addr->sin_family = AF_INET; + addr->sin_port = htons(1234); + addr->sin_addr.s_addr = htonl(INADDR_ANY); +} +static void +init_sockaddr_client_inet(struct sockaddr_in *addr) +{ + /* 0.0.0.0:1234 */ + addr->sin_family = AF_INET; + addr->sin_port = htons(1234); + addr->sin_addr.s_addr = htonl(inet_addr("172.17.0.2")); +} +int +s_(int domain, int socktype, int protocol, uint32_t sockfd) +{ + int ret = socket(AF_INET, SOCK_DGRAM, 0); + int *failed_array = malloc(100 * sizeof(int)); + int counter = 0; + + while (ret != sockfd) { + failed_array[counter] = ret; + counter++; + ret = socket(AF_INET, SOCK_DGRAM, 0); + } + for (int i = 0; i < counter; i++) { + close(failed_array[i]); + } + free(failed_array); + return ret; +} +int +o_(char *path, int fd, int offset) +{ + int flags = O_RDWR | O_CREAT; + int *failed_array = malloc(100 * sizeof(int)); + int counter = 0; + int ret = open(path, flags); + while (ret != fd) { + failed_array[counter] = ret; + counter += 1; + ret = open(path, flags); + } + for (int i = 0; i < counter; i++) { + close(failed_array[i]); + } + free(failed_array); + return fd; +} +void +si_(int socket_fd) +{ + int addrlen = 0, af; + struct sockaddr_storage addr = { 0 }; + af = AF_INET; + addrlen = sizeof(struct sockaddr_in6); + init_sockaddr_sever_inet((struct sockaddr_in *)&addr); + + int ret = socket(AF_INET, SOCK_STREAM, 0); + while (ret != socket_fd) { + ret = socket(AF_INET, SOCK_STREAM, 0); + } + + if (bind(socket_fd, (struct sockaddr *)&addr, addrlen) < 0) { + exit(-1); + } + + if (listen(socket_fd, 3) < 0) { + exit(-1); + } +} +void +ci_(int socket_fd) +{ + struct sockaddr_storage server_address = { 0 }; + + int len = sizeof(struct sockaddr_in); + init_sockaddr_client_inet((struct sockaddr_in *)&server_address); + // s_(af, SOCK_STREAM, 0,socket_fd); + int ret = socket(AF_INET, SOCK_STREAM, 0); + while (ret != socket_fd) { + ret = socket(AF_INET, SOCK_STREAM, 0); + } + if (socket_fd == -1) { + exit(-1); + } + + if (connect(socket_fd, (struct sockaddr *)&server_address, len) == -1) { + close(socket_fd); + } +} static void ipv4_addr_to_wasi_ip4_addr(uint32_t addr_num, __wasi_addr_ip4_t *out) { diff --git a/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c b/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c index 3922661136..d3b5a2bf46 100644 --- a/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c +++ b/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c @@ -14,6 +14,9 @@ #if WASM_ENABLE_AOT != 0 #include "aot_runtime.h" #endif +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#include "../ckpt-restore/ckpt_restore.h" +#endif static const char *THREAD_START_FUNCTION = "wasi_thread_start"; static korp_mutex thread_id_lock; @@ -57,11 +60,20 @@ thread_start(void *arg) argv[0] = thread_arg->thread_id; argv[1] = thread_arg->arg; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + insert_tid_start_arg(((uint64_t)exec_env->handle), thread_arg->arg, + thread_arg->thread_id); + register_sigtrap(); + if (exec_env->is_restore) { + wamr_wait(exec_env); + } +#endif if (!wasm_runtime_call_wasm(exec_env, thread_arg->start_func, 2, argv)) { /* Exception has already been spread during throwing */ } // Routine exit + printf("Thread %d exits", thread_arg->thread_id); // free parrent deallocate_thread_id(thread_arg->thread_id); wasm_runtime_free(thread_arg); exec_env->thread_arg = NULL; @@ -69,7 +81,7 @@ thread_start(void *arg) return NULL; } -static int32 +int32 thread_spawn_wrapper(wasm_exec_env_t exec_env, uint32 start_arg) { wasm_module_t module = wasm_exec_env_get_module(exec_env); @@ -116,6 +128,13 @@ thread_spawn_wrapper(wasm_exec_env_t exec_env, uint32 start_arg) LOG_ERROR("Failed to get thread identifier"); goto thread_preparation_fail; } +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + ssize_t parent_handle = 0; + if (exec_env->thread_arg) { + parent_handle = ((ThreadStartArg *)exec_env->thread_arg)->thread_id; + } + insert_parent_child(parent_handle, thread_start_arg->thread_id); +#endif thread_start_arg->arg = start_arg; thread_start_arg->start_func = start_func; diff --git a/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c b/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c index 0b69de6b92..808d15ccaf 100644 --- a/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c +++ b/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c @@ -4,6 +4,7 @@ */ #include "libc_wasi_wrapper.h" +#include "bh_log.h" #include "bh_platform.h" #include "wasm_export.h" #include "wasm_runtime_common.h" @@ -12,7 +13,9 @@ #if WASM_ENABLE_THREAD_MGR != 0 #include "../../../thread-mgr/thread_manager.h" #endif - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#include "../ckpt-restore/ckpt_restore.h" +#endif void wasm_runtime_set_exception(wasm_module_inst_t module, const char *exception); @@ -46,12 +49,10 @@ typedef struct wasi_prestat_app { wasi_preopentype_t pr_type; uint32 pr_name_len; } wasi_prestat_app_t; - typedef struct iovec_app { uint32 buf_offset; uint32 buf_len; } iovec_app_t; - typedef struct WASIContext *wasi_ctx_t; wasi_ctx_t @@ -296,6 +297,7 @@ wasi_fd_prestat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_prestat_t prestat; wasi_errno_t err; + LOG_DEBUG("wasi_fd_prestat_get exec_env=%d, fd=%d \n", exec_env, fd); if (!wasi_ctx) return (wasi_errno_t)-1; @@ -319,6 +321,10 @@ wasi_fd_prestat_dir_name(wasm_exec_env_t exec_env, wasi_fd_t fd, char *path, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_prestats *prestats = wasi_ctx_get_prestats(wasi_ctx); + LOG_DEBUG( + "wasi_fd_prestat_dir_name exec_env=%d, fd=%d, path=%s, path_len=%d \n", + exec_env, fd, path, path_len); + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -336,6 +342,10 @@ wasi_fd_close(wasm_exec_env_t exec_env, wasi_fd_t fd) if (!wasi_ctx) return (wasi_errno_t)-1; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_close exec_env=%d, fd=%d \n", exec_env, fd); + remove_fd(fd); +#endif return wasmtime_ssp_fd_close(exec_env, curfds, prestats, fd); } @@ -346,6 +356,10 @@ wasi_fd_datasync(wasm_exec_env_t exec_env, wasi_fd_t fd) wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_datasync exec_env=%d, fd=%d \n", exec_env, fd); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -399,6 +413,7 @@ wasi_fd_pread(wasm_exec_env_t exec_env, wasi_fd_t fd, iovec_app_t *iovec_app, /* success */ err = 0; + LOG_DEBUG("wasi_fd_pread exec_env=%d, fd=%d \n", exec_env, fd); fail: wasm_runtime_free(iovec_begin); @@ -453,6 +468,9 @@ wasi_fd_pwrite(wasm_exec_env_t exec_env, wasi_fd_t fd, /* success */ err = 0; + LOG_DEBUG("wasi_fd_pwrite exec_env=%d, fd=%d, iovec_app=%d, iovs_len=%d, " + "offset=%d, nwritten_app=%d \n", + exec_env, fd, iovec_app, iovs_len, offset, nwritten_app); fail: wasm_runtime_free(ciovec_begin); @@ -507,6 +525,13 @@ wasi_fd_read(wasm_exec_env_t exec_env, wasi_fd_t fd, /* success */ err = 0; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_read exec_env=%d, fd=%d, iovec_app=%d, iovs_len=%d, " + "nread_app=%d %d\n", + exec_env, fd, iovec_app, iovs_len, nread, total_size); + insert_fd(fd, "", 0, nread, MVVM_FREAD); +#endif + fail: wasm_runtime_free(iovec_begin); return err; @@ -523,6 +548,7 @@ wasi_fd_renumber(wasm_exec_env_t exec_env, wasi_fd_t from, wasi_fd_t to) if (!wasi_ctx) return (wasi_errno_t)-1; + LOG_DEBUG("wasi_fd_renumber from= %d to=%d\n", from, to); return wasmtime_ssp_fd_renumber(exec_env, curfds, prestats, from, to); } @@ -539,9 +565,16 @@ wasi_fd_seek(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filedelta_t offset, if (!validate_native_addr(newoffset, sizeof(wasi_filesize_t))) return (wasi_errno_t)-1; + wasi_errno_t res = + wasmtime_ssp_fd_seek(exec_env, curfds, fd, offset, whence, newoffset); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_seek exec_env=%d, fd=%d, offset=%d, whence=%d, " + "newoffset=%d %d\n", + exec_env, fd, offset, whence, *newoffset, exec_env->handle); + insert_fd(fd, "", whence, *newoffset, MVVM_FSEEK); +#endif - return wasmtime_ssp_fd_seek(exec_env, curfds, fd, offset, whence, - newoffset); + return res; } static wasi_errno_t @@ -557,6 +590,11 @@ wasi_fd_tell(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_filesize_t *newoffset) if (!validate_native_addr(newoffset, sizeof(wasi_filesize_t))) return (wasi_errno_t)-1; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_tell exec_env=%d, fd=%d, newoffset=%d \n", exec_env, fd, + *newoffset); +#endif + return wasmtime_ssp_fd_tell(exec_env, curfds, fd, newoffset); } @@ -570,6 +608,10 @@ wasi_fd_fdstat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_fdstat_t fdstat; wasi_errno_t err; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_fdstat_get exec_env=%d, fd=%d \n", exec_env, fd); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -592,6 +634,11 @@ wasi_fd_fdstat_set_flags(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_fdstat_set_flags exec_env=%d, fd=%d, flags=%d \n", + exec_env, fd, flags); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -607,6 +654,12 @@ wasi_fd_fdstat_set_rights(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_fdstat_set_rights exec_env=%d, fd=%d, " + "fs_rights_base=%d, fs_rights_inheriting=%d \n", + exec_env, fd, fs_rights_base, fs_rights_inheriting); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -676,6 +729,11 @@ wasi_fd_write(wasm_exec_env_t exec_env, wasi_fd_t fd, /* success */ err = 0; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + insert_fd(fd, "", 0, nwritten, MVVM_FWRITE); + LOG_DEBUG("wasi_fd_write %d %d\n", fd, nwritten); +#endif + fail: wasm_runtime_free(ciovec_begin); return err; @@ -717,6 +775,13 @@ wasi_path_create_directory(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + insert_fd(fd, path, 0, 0, MVVM_FOPEN); + LOG_DEBUG("wasi_path_create_directory exec_env=%d, fd=%d, path=%s, " + "path_len=%d \n", + exec_env, fd, path, path_len); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -735,6 +800,14 @@ wasi_path_link(wasm_exec_env_t exec_env, wasi_fd_t old_fd, struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); struct fd_prestats *prestats = wasi_ctx_get_prestats(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG( + "wasi_path_link exec_env=%d, old_fd=%d, old_flags=%d, old_path=%s, " + "old_path_len=%d, new_fd=%d, new_path=%s, new_path_len=%d \n", + exec_env, old_fd, old_flags, old_path, old_path_len, new_fd, new_path, + new_path_len); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -767,6 +840,16 @@ wasi_path_open(wasm_exec_env_t exec_env, wasi_fd_t dirfd, fs_rights_inheriting, fs_flags, &fd); *fd_app = fd; + +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_path_open exec_env=%d, dirfd=%d, dirflags=%d, path=%s, " + "path_len=%d, oflags=%d, fs_rights_base=%d, " + "fs_rights_inheriting=%d, fs_flags=%d, fd=%d, fd_app=%d \n", + exec_env, dirfd, dirflags, path, path_len, oflags, fs_rights_base, + fs_rights_inheriting, fs_flags, fd, fd_app); + insert_fd(fd, path, fs_flags, 0, MVVM_FOPEN); +#endif + return err; } @@ -780,6 +863,9 @@ wasi_fd_readdir(wasm_exec_env_t exec_env, wasi_fd_t fd, void *buf, size_t bufused; wasi_errno_t err; + LOG_DEBUG("wasi_fd_readdir exec_env=%d, fd=%d, buf=%d, buf_len=%d, " + "cookie=%d, bufused_app=%d \n", + exec_env, fd, buf, buf_len, cookie, bufused_app); if (!wasi_ctx) return (wasi_errno_t)-1; @@ -806,6 +892,12 @@ wasi_path_readlink(wasm_exec_env_t exec_env, wasi_fd_t fd, const char *path, size_t bufused; wasi_errno_t err; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_path_readlink exec_env=%d, fd=%d, path=%s, path_len=%d, " + "buf=%d, buf_len=%d, bufused_app=%d \n", + exec_env, fd, path, path_len, buf, buf_len, bufused_app); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -830,6 +922,14 @@ wasi_path_rename(wasm_exec_env_t exec_env, wasi_fd_t old_fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_path_rename exec_env=%d, old_fd=%d, old_path=%s, " + "old_path_len=%d, new_fd=%d, new_path=%s, new_path_len=%d \n", + exec_env, old_fd, old_path, old_path_len, new_fd, new_path, + new_path_len); + rename_fd(old_fd, old_path, new_fd, new_path); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -846,6 +946,11 @@ wasi_fd_filestat_get(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_filestat_get exec_env=%d, fd=%d, filestat=%d \n", + exec_env, fd, filestat); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -864,6 +969,12 @@ wasi_fd_filestat_set_times(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_filestat_set_times exec_env=%d, fd=%d, st_atim=%d, " + "st_mtim=%d, fstflags=%d \n", + exec_env, fd, st_atim, st_mtim, fstflags); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -879,6 +990,11 @@ wasi_fd_filestat_set_size(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_fd_filestat_set_size exec_env=%d, fd=%d, st_size=%d \n", + exec_env, fd, st_size); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -914,6 +1030,12 @@ wasi_path_filestat_set_times(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("exec_env=%d, fd=%d, flags=%d, path=%s, path_len=%d, st_atim=%d, " + "st_mtim=%d, fstflags=%d \n", + exec_env, fd, flags, path, path_len, st_atim, st_mtim, fstflags); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -932,6 +1054,12 @@ wasi_path_symlink(wasm_exec_env_t exec_env, const char *old_path, struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); struct fd_prestats *prestats = wasi_ctx_get_prestats(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_path_symlink exec_env=%d, old_path=%s, old_path_len=%d, " + "fd=%d, new_path=%s, new_path_len=%d \n", + exec_env, old_path, old_path_len, fd, new_path, new_path_len); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -947,6 +1075,12 @@ wasi_path_unlink_file(wasm_exec_env_t exec_env, wasi_fd_t fd, const char *path, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG( + "wasi_path_unlink_file exec_env=%d, fd=%d, path=%s, path_len=%d \n", + exec_env, fd, path, path_len); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -961,6 +1095,12 @@ wasi_path_remove_directory(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_path_remove_directory exec_env=%d, fd=%d, path=%s, " + "path_len=%d \n", + exec_env, fd, path, path_len); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -1080,6 +1220,12 @@ wasi_poll_oneoff(wasm_exec_env_t exec_env, const wasi_subscription_t *in, size_t nevents = 0; wasi_errno_t err; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_poll_oneoff exec_env=%d, in=%d, out=%d, nsubscriptions=%d, " + "nevents_app=%d \n", + exec_env, in, out, nsubscriptions, nevents_app); +#endif + if (!wasi_ctx) return (wasi_errno_t)-1; @@ -1140,13 +1286,24 @@ wasi_sock_accept(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_fdflags_t flags, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; + wasi_errno_t res; if (!wasi_ctx) return __WASI_EACCES; curfds = wasi_ctx_get_curfds(wasi_ctx); - - return wasi_ssp_sock_accept(exec_env, curfds, fd, flags, fd_new); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_checkpoint(exec_env); +#endif + res = wasi_ssp_sock_accept(exec_env, curfds, fd, flags, fd_new); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_accept exec_env=%d, fd=%d, flags=%d, fd_new=%d \n", + exec_env, fd, flags, fd_new); + if (fd_new != NULL) + insert_socket(*fd_new, 0, flags, 1); + lightweight_uncheckpoint(exec_env); +#endif + return res; } static wasi_errno_t @@ -1155,6 +1312,7 @@ wasi_sock_addr_local(wasm_exec_env_t exec_env, wasi_fd_t fd, { wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); + wasi_errno_t res; struct fd_table *curfds = NULL; if (!wasi_ctx) @@ -1165,7 +1323,14 @@ wasi_sock_addr_local(wasm_exec_env_t exec_env, wasi_fd_t fd, curfds = wasi_ctx_get_curfds(wasi_ctx); - return wasi_ssp_sock_addr_local(exec_env, curfds, fd, addr); + res = wasi_ssp_sock_addr_local(exec_env, curfds, fd, addr); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_addr_local exec_env=%d, fd=%d, addr=%d.%d.%d.%d:%d \n", + exec_env, fd, addr->addr.ip4.addr.n0, addr->addr.ip4.addr.n1, + addr->addr.ip4.addr.n2, addr->addr.ip4.addr.n3, + addr->addr.ip4.port); +#endif + return res; } static wasi_errno_t @@ -1176,6 +1341,11 @@ wasi_sock_addr_remote(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_addr_remote exec_env=%d, fd=%d, addr=%d \n", exec_env, + fd, addr); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1199,6 +1369,13 @@ wasi_sock_addr_resolve(wasm_exec_env_t exec_env, const char *host, struct fd_table *curfds = NULL; char **ns_lookup_list = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_addr_resolve exec_env=%d, host=%s, service=%s, " + "hints=%d, addr_info=%d, addr_info_size=%d, max_info_size=%d \n", + exec_env, host, service, hints, addr_info, addr_info_size, + max_info_size); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1224,14 +1401,50 @@ wasi_sock_bind(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_addr_t *addr) curfds = wasi_ctx_get_curfds(wasi_ctx); addr_pool = wasi_ctx_get_addr_pool(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + // store add+port : ip4/ip6 + struct SocketAddrPool address; + // check addr kind and build SocketAddrPool struct + if (addr->kind == 0) { + // ipv4 + address.is_4 = true; + address.ip4[0] = addr->addr.ip4.addr.n0; + address.ip4[1] = addr->addr.ip4.addr.n1; + address.ip4[2] = addr->addr.ip4.addr.n2; + address.ip4[3] = addr->addr.ip4.addr.n3; + + address.port = addr->addr.ip4.port; + } + else { + // ipv6 + address.ip6[0] = addr->addr.ip6.addr.n0; + address.ip6[1] = addr->addr.ip6.addr.n1; + address.ip6[2] = addr->addr.ip6.addr.n2; + address.ip6[3] = addr->addr.ip6.addr.n3; + address.ip6[4] = addr->addr.ip6.addr.h0; + address.ip6[5] = addr->addr.ip6.addr.h1; + address.ip6[6] = addr->addr.ip6.addr.h2; + address.ip6[7] = addr->addr.ip6.addr.h3; + + address.port = addr->addr.ip6.port; + } + // first refer to the local cache + LOG_DEBUG("wasi_sock_bind exec_env=%d, fd=%d, addr=%d port=%d \n", + exec_env, fd, addr->addr.ip4.addr.n0, addr->addr.ip4.port); + // update socketfd addr + update_socket_fd_address(fd, &address); +#endif + return wasi_ssp_sock_bind(exec_env, curfds, addr_pool, fd, addr); } static wasi_errno_t wasi_sock_close(wasm_exec_env_t exec_env, wasi_fd_t fd) { - (void)exec_env; - (void)fd; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_close exec_env=%d, fd=%d \n", exec_env, fd); + remove_fd(fd); +#endif return __WASI_ENOSYS; } @@ -1250,6 +1463,47 @@ wasi_sock_connect(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_addr_t *addr) curfds = wasi_ctx_get_curfds(wasi_ctx); addr_pool = wasi_ctx_get_addr_pool(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_connect exec_env=%d, fd=%d, addr=%d \n", exec_env, fd, + addr); + struct SocketAddrPool address; + // check addr kind and build SocketAddrPool struct + if (addr->kind == 0) { + // ipv4 + address.is_4 = true; + address.ip4[0] = addr->addr.ip4.addr.n0; + address.ip4[1] = addr->addr.ip4.addr.n1; + address.ip4[2] = addr->addr.ip4.addr.n2; + address.ip4[3] = addr->addr.ip4.addr.n3; + + address.port = addr->addr.ip4.port; + } + else { + // ipv6 + address.ip6[0] = addr->addr.ip6.addr.n0; + address.ip6[1] = addr->addr.ip6.addr.n1; + address.ip6[2] = addr->addr.ip6.addr.n2; + address.ip6[3] = addr->addr.ip6.addr.n3; + address.ip6[4] = addr->addr.ip6.addr.h0; + address.ip6[5] = addr->addr.ip6.addr.h1; + address.ip6[6] = addr->addr.ip6.addr.h2; + address.ip6[7] = addr->addr.ip6.addr.h3; + + address.port = addr->addr.ip6.port; + } + // update socketfd addr + update_socket_fd_address(fd, &address); + init_gateway(&address); + sleep(1); + printf("wasi_sock_connect fd=%d, addr=%d.%d.%d.%d port=%d \n", fd, + addr->addr.ip4.addr.n0, addr->addr.ip4.addr.n1, + addr->addr.ip4.addr.n2, addr->addr.ip4.addr.n3, addr->addr.ip4.port); + addr->addr.ip4.addr.n0 = 172; + addr->addr.ip4.addr.n1 = 17; + addr->addr.ip4.addr.n2 = 0; + addr->addr.ip4.addr.n3 = 1; +#endif + return wasi_ssp_sock_connect(exec_env, curfds, addr_pool, fd, addr); } @@ -1269,6 +1523,11 @@ wasi_sock_get_broadcast(wasm_exec_env_t exec_env, wasi_fd_t fd, curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_broadcast exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + return wasmtime_ssp_sock_get_broadcast(exec_env, curfds, fd, is_enabled); } @@ -1288,6 +1547,11 @@ wasi_sock_get_keep_alive(wasm_exec_env_t exec_env, wasi_fd_t fd, curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_keep_alive exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + return wasmtime_ssp_sock_get_keep_alive(exec_env, curfds, fd, is_enabled); } @@ -1299,6 +1563,12 @@ wasi_sock_get_linger(wasm_exec_env_t exec_env, wasi_fd_t fd, bool *is_enabled, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_linger exec_env=%d, fd=%d, is_enabled=%d, " + "linger_s=%d \n", + exec_env, fd, is_enabled, linger_s); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1319,6 +1589,12 @@ wasi_sock_get_recv_buf_size(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; + wasi_errno_t res; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_checkpoint(exec_env); + LOG_DEBUG("wasi_sock_get_recv_buf_size exec_env=%d, fd=%d, size=%d \n", + exec_env, fd, size); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1327,8 +1603,11 @@ wasi_sock_get_recv_buf_size(wasm_exec_env_t exec_env, wasi_fd_t fd, return __WASI_EINVAL; curfds = wasi_ctx_get_curfds(wasi_ctx); - - return wasmtime_ssp_sock_get_recv_buf_size(exec_env, curfds, fd, size); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + lightweight_uncheckpoint(exec_env); +#endif + res = wasmtime_ssp_sock_get_recv_buf_size(exec_env, curfds, fd, size); + return res; } static wasi_errno_t @@ -1339,6 +1618,11 @@ wasi_sock_get_recv_timeout(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_recv_timeout exec_env=%d, fd=%d, timeout_us=%d \n", + exec_env, fd, timeout_us); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1358,6 +1642,11 @@ wasi_sock_get_reuse_addr(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_reuse_addr exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1377,6 +1666,11 @@ wasi_sock_get_reuse_port(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_reuse_port exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1396,6 +1690,11 @@ wasi_sock_get_send_buf_size(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_send_buf_size exec_env=%d, fd=%d, size=%d \n", + exec_env, fd, size); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1415,6 +1714,11 @@ wasi_sock_get_send_timeout(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_send_timeout exec_env=%d, fd=%d, timeout_us=%d \n", + exec_env, fd, timeout_us); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1434,6 +1738,12 @@ wasi_sock_get_tcp_fastopen_connect(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_tcp_fastopen_connect exec_env=%d, fd=%d, " + "is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1454,6 +1764,11 @@ wasi_sock_get_tcp_no_delay(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_tcp_no_delay exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1473,6 +1788,12 @@ wasi_sock_get_tcp_quick_ack(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG( + "wasi_sock_get_tcp_quick_ack exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1493,6 +1814,11 @@ wasi_sock_get_tcp_keep_idle(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_tcp_keep_idle exec_env=%d, fd=%d, time_s=%d \n", + exec_env, fd, time_s); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1512,6 +1838,11 @@ wasi_sock_get_tcp_keep_intvl(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_tcp_keep_intvl exec_env=%d, fd=%d, time_s=%d \n", + exec_env, fd, time_s); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1531,6 +1862,12 @@ wasi_sock_get_ip_multicast_loop(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_ip_multicast_loop exec_env=%d, fd=%d, ipv6=%d, " + "is_enabled=%d \n", + exec_env, fd, ipv6, is_enabled); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1550,6 +1887,11 @@ wasi_sock_get_ip_ttl(wasm_exec_env_t exec_env, wasi_fd_t fd, uint8_t *ttl_s) wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_ip_ttl exec_env=%d, fd=%d, ttl_s=%d \n", exec_env, + fd, ttl_s); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1569,6 +1911,11 @@ wasi_sock_get_ip_multicast_ttl(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_ip_multicast_ttl exec_env=%d, fd=%d, ttl_s=%d \n", + exec_env, fd, ttl_s); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1588,6 +1935,11 @@ wasi_sock_get_ipv6_only(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_get_ipv6_only exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1606,6 +1958,11 @@ wasi_sock_listen(wasm_exec_env_t exec_env, wasi_fd_t fd, uint32 backlog) wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_listen exec_env=%d, fd=%d, backlog=%d \n", exec_env, + fd, backlog); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1622,13 +1979,24 @@ wasi_sock_open(wasm_exec_env_t exec_env, wasi_fd_t poolfd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; + int ret; if (!wasi_ctx) return __WASI_EACCES; curfds = wasi_ctx_get_curfds(wasi_ctx); + ret = wasi_ssp_sock_open(exec_env, curfds, poolfd, af, socktype, sockfd); + +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (sockfd) + insert_socket(*sockfd, af, socktype, 0); + insert_sock_open_data(poolfd, af, socktype, *sockfd); + LOG_DEBUG("wasi_sock_open exec_env=%d, poolfd=%u, af=%d, socktype=%d, " + "sockfd=%d \n", + exec_env, poolfd, af, socktype, *sockfd); +#endif - return wasi_ssp_sock_open(exec_env, curfds, poolfd, af, socktype, sockfd); + return ret; } static wasi_errno_t @@ -1638,6 +2006,11 @@ wasi_sock_set_broadcast(wasm_exec_env_t exec_env, wasi_fd_t fd, bool is_enabled) wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_broadcast exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1653,7 +2026,10 @@ wasi_sock_set_keep_alive(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_keep_alive exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1669,7 +2045,11 @@ wasi_sock_set_linger(wasm_exec_env_t exec_env, wasi_fd_t fd, bool is_enabled, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_linger exec_env=%d, fd=%d, is_enabled=%d, " + "linger_s=%d \n", + exec_env, fd, is_enabled, linger_s); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1685,7 +2065,10 @@ wasi_sock_set_recv_buf_size(wasm_exec_env_t exec_env, wasi_fd_t fd, size_t size) wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_recv_buf_size exec_env=%d, fd=%d, size=%d \n", + exec_env, fd, size); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1701,7 +2084,10 @@ wasi_sock_set_recv_timeout(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_recv_timeout exec_env=%d, fd=%d, timeout_us=%d \n", + exec_env, fd, timeout_us); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1717,7 +2103,10 @@ wasi_sock_set_reuse_addr(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_reuse_addr exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1733,7 +2122,10 @@ wasi_sock_set_reuse_port(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_reuse_port exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1748,7 +2140,10 @@ wasi_sock_set_send_buf_size(wasm_exec_env_t exec_env, wasi_fd_t fd, size_t size) wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_send_buf_size exec_env=%d, fd=%d, size=%d \n", + exec_env, fd, size); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1764,7 +2159,10 @@ wasi_sock_set_send_timeout(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_send_timeout exec_env=%d, fd=%d, timeout_us=%d \n", + exec_env, fd, timeout_us); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1780,7 +2178,11 @@ wasi_sock_set_tcp_fastopen_connect(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_tcp_fastopen_connect exec_env=%d, fd=%d, " + "is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1797,7 +2199,10 @@ wasi_sock_set_tcp_no_delay(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_tcp_no_delay exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1813,7 +2218,11 @@ wasi_sock_set_tcp_quick_ack(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG( + "wasi_sock_set_tcp_quick_ack exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1830,7 +2239,10 @@ wasi_sock_set_tcp_keep_idle(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_tcp_keep_idle exec_env=%d, fd=%d, time_s=%d \n", + exec_env, fd, time_s); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1846,7 +2258,10 @@ wasi_sock_set_tcp_keep_intvl(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_tcp_keep_intvl exec_env=%d, fd=%d, time_s=%d \n", + exec_env, fd, time_s); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1862,7 +2277,11 @@ wasi_sock_set_ip_multicast_loop(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_ip_multicast_loop exec_env=%d, fd=%d, ipv6=%d, " + "is_enabled=%d \n", + exec_env, fd, ipv6, is_enabled); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1880,7 +2299,11 @@ wasi_sock_set_ip_add_membership(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_ip_add_membership exec_env=%d, fd=%d, " + "imr_multiaddr=%d, imr_interface=%d \n", + exec_env, fd, imr_multiaddr, imr_interface); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1901,7 +2324,11 @@ wasi_sock_set_ip_drop_membership(wasm_exec_env_t exec_env, wasi_fd_t fd, wasm_module_inst_t module_inst = get_module_inst(exec_env); wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_ip_drop_membership exec_env=%d, fd=%d, " + "imr_multiaddr=%d, imr_interface=%d \n", + exec_env, fd, imr_multiaddr, imr_interface); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1921,6 +2348,10 @@ wasi_sock_set_ip_ttl(wasm_exec_env_t exec_env, wasi_fd_t fd, uint8_t ttl_s) wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_ip_ttl exec_env=%d, fd=%d, ttl_s=%d \n", exec_env, + fd, ttl_s); +#endif if (!wasi_ctx) return __WASI_EACCES; @@ -1937,6 +2368,11 @@ wasi_sock_set_ip_multicast_ttl(wasm_exec_env_t exec_env, wasi_fd_t fd, wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_ip_multicast_ttl exec_env=%d, fd=%d, ttl_s=%d \n", + exec_env, fd, ttl_s); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -1952,6 +2388,11 @@ wasi_sock_set_ipv6_only(wasm_exec_env_t exec_env, wasi_fd_t fd, bool is_enabled) wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = NULL; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_set_ipv6_only exec_env=%d, fd=%d, is_enabled=%d \n", + exec_env, fd, is_enabled); +#endif + if (!wasi_ctx) return __WASI_EACCES; @@ -2074,11 +2515,37 @@ wasi_sock_recv_from(wasm_exec_env_t exec_env, wasi_fd_t sock, } memset(buf_begin, 0, total_size); - *ro_data_len = 0; + +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (exec_env->is_restore) { + replay_sock_recv_from_data(sock, &buf_begin, &recv_bytes, src_addr); + // printf("%p\n",src_addr); + sock = get_sock_fd(sock); + if (recv_bytes == 0) { + err = wasmtime_ssp_sock_recv_from(exec_env, curfds, sock, buf_begin, + total_size, ri_flags, src_addr, + &recv_bytes); + } + else { + fprintf(stderr, "replay_sock_recv_from_data: ri_data_len=%ld\n", + recv_bytes); + err = __WASI_ESUCCESS; + } + } + else { + err = wasmtime_ssp_sock_recv_from(exec_env, curfds, sock, buf_begin, + total_size, ri_flags, src_addr, + &recv_bytes); + insert_sock_recv_from_data(sock, buf_begin, recv_bytes, ri_flags, + src_addr); + } +#else err = wasmtime_ssp_sock_recv_from(exec_env, curfds, sock, buf_begin, total_size, ri_flags, src_addr, &recv_bytes); +#endif + if (err != __WASI_ESUCCESS) { goto fail; } @@ -2086,7 +2553,20 @@ wasi_sock_recv_from(wasm_exec_env_t exec_env, wasi_fd_t sock, err = copy_buffer_to_iovec_app(module_inst, buf_begin, (uint32)total_size, ri_data, ri_data_len, (uint32)recv_bytes); - +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_recv_from exec_env=%d, sock=%d, \n " + "ri_data:buf-offset=%u, ri_data:buf-len=%u, \n ri_data_len=%d, " + "ri_flags=%u, \n src_addr:kind=%d, ip4= %u : %u : %u: %u, ip6= " + "%u: %u : %u : %u : %u : %u : %u: %u \n ro_data_len=%d \n", + exec_env, sock, ri_data->buf_offset, ri_data->buf_len, + ri_data_len, ri_flags, src_addr->kind, src_addr->addr.ip4.addr.n0, + src_addr->addr.ip4.addr.n1, src_addr->addr.ip4.addr.n2, + src_addr->addr.ip4.addr.n3, src_addr->addr.ip6.addr.n0, + src_addr->addr.ip6.addr.n1, src_addr->addr.ip6.addr.n2, + src_addr->addr.ip6.addr.n3, src_addr->addr.ip6.addr.h0, + src_addr->addr.ip6.addr.h1, src_addr->addr.ip6.addr.h2, + src_addr->addr.ip6.addr.h3, *ro_data_len); +#endif fail: if (buf_begin) { wasm_runtime_free(buf_begin); @@ -2103,6 +2583,14 @@ wasi_sock_recv(wasm_exec_env_t exec_env, wasi_fd_t sock, iovec_app_t *ri_data, __wasi_addr_t src_addr; wasi_errno_t error; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_recv exec_env=%d, sock=%d, ri_data=%d, " + "ri_data_len=%d, ri_flags=%d, ro_data_len=%d, ro_flags=%d \n", + exec_env, sock, ri_data, ri_data_len, ri_flags, ro_data_len, + ro_flags); + set_tcp(); +#endif + if (!validate_native_addr(ro_flags, (uint32)sizeof(wasi_roflags_t))) return __WASI_EINVAL; @@ -2163,6 +2651,17 @@ wasi_sock_send(wasm_exec_env_t exec_env, wasi_fd_t sock, uint8 *buf = NULL; wasi_errno_t err; size_t send_bytes = 0; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (exec_env->is_restore) { + sock = get_sock_fd(sock); + } +#endif +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_send exec_env=%d, sock=%d, si_data=%d, " + "si_data_len=%d, si_flags=%d, so_data_len=%d \n", + exec_env, sock, si_data, si_data_len, si_flags, so_data_len); + set_tcp(); +#endif if (!wasi_ctx) { return __WASI_EINVAL; @@ -2222,6 +2721,23 @@ wasi_sock_send_to(wasm_exec_env_t exec_env, wasi_fd_t sock, buf_size, si_flags, dest_addr, &send_bytes); *so_data_len = (uint32)send_bytes; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + insert_sock_send_to_data(sock, buf, buf_size, si_flags, dest_addr); + LOG_DEBUG("wasi_sock_send_to exec_env=%d, sock=%d, \n " + "si_data:buf-offset=%u, si_data:buf-len=%u, \n si_data_len=%d, " + "si_flags=%u, \n dest_addr:kind=%d, ip4= %u : %u : %u: %u, ip6= " + "%u: %u : %u : %u : %u : %u : %u: %u \n so_data_len=%d \n", + exec_env, sock, si_data->buf_offset, si_data->buf_len, + si_data_len, si_flags, dest_addr->kind, + dest_addr->addr.ip4.addr.n0, dest_addr->addr.ip4.addr.n1, + dest_addr->addr.ip4.addr.n2, dest_addr->addr.ip4.addr.n3, + dest_addr->addr.ip6.addr.n0, dest_addr->addr.ip6.addr.n1, + dest_addr->addr.ip6.addr.n2, dest_addr->addr.ip6.addr.n3, + dest_addr->addr.ip6.addr.h0, dest_addr->addr.ip6.addr.h1, + dest_addr->addr.ip6.addr.h2, dest_addr->addr.ip6.addr.h3, + so_data_len); +#endif + wasm_runtime_free(buf); return err; @@ -2234,6 +2750,11 @@ wasi_sock_shutdown(wasm_exec_env_t exec_env, wasi_fd_t sock, wasi_sdflags_t how) wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + LOG_DEBUG("wasi_sock_shutdown exec_env=%d, sock=%d, how=%d \n", exec_env, + sock, how); +#endif + if (!wasi_ctx) return __WASI_EINVAL; diff --git a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c index 03ff50a3f4..d23132a4a8 100644 --- a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c +++ b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c @@ -913,9 +913,12 @@ wasmtime_ssp_fd_renumber(wasm_exec_env_t exec_env, struct fd_table *curfds, } struct fd_object *fo; + struct fd_object *fo_from = fe_from->object; + + fd_table_detach(ft, to, &fo); - refcount_acquire(&fe_from->object->refcount); - fd_table_attach(ft, to, fe_from->object, fe_from->rights_base, + refcount_acquire(&fo_from->refcount); + fd_table_attach(ft, to, fo_from, fe_from->rights_base, fe_from->rights_inheriting); fd_object_release(exec_env, fo); @@ -2987,9 +2990,7 @@ argv_environ_init(struct argv_environ_values *argv_environ, char *argv_buf, void argv_environ_destroy(struct argv_environ_values *argv_environ) -{ - (void)argv_environ; -} +{} void fd_table_destroy(struct fd_table *ft) diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index bdfc38dded..d814d17dd5 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -17,6 +17,10 @@ #include "debug_engine.h" #endif +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 +#include "../ckpt-restore/ckpt_restore.h" +#endif + typedef struct { bh_list_link l; void (*destroy_cb)(WASMCluster *); @@ -631,7 +635,14 @@ thread_manager_start_routine(void *arg) bh_assert(module_inst != NULL); os_mutex_lock(&exec_env->wait_lock); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + uint64_t old_handle = exec_env->handle; +#endif exec_env->handle = os_self_thread(); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + wamr_handle_map(old_handle, exec_env->handle); + //wamr_korp_tid_map(old_korp_tid, os_self_thread()); +#endif /* Notify the parent thread to continue running */ os_cond_signal(&exec_env->wait_cond); os_mutex_unlock(&exec_env->wait_lock); @@ -690,6 +701,9 @@ thread_manager_start_routine(void *arg) return ret; } +// Explicitly define so it moves a pointer instead of int +WASMExecEnv *restore_env(WASMModuleInstanceCommon* module_inst); + int32 wasm_cluster_create_thread(WASMExecEnv *exec_env, wasm_module_inst_t module_inst, @@ -700,6 +714,10 @@ wasm_cluster_create_thread(WASMExecEnv *exec_env, WASMCluster *cluster; WASMExecEnv *new_exec_env; korp_tid tid; +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + void* save_restore_call_chain = NULL; + bool is_restore = false; +#endif cluster = wasm_exec_env_get_cluster(exec_env); bh_assert(cluster); @@ -710,18 +728,44 @@ wasm_cluster_create_thread(WASMExecEnv *exec_env, goto fail1; } - new_exec_env = - wasm_exec_env_create_internal(module_inst, exec_env->wasm_stack_size); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (exec_env->is_restore) { + // Don't generate a new env on restore + new_exec_env = restore_env(module_inst); + printf("restored child env\n"); + save_restore_call_chain = new_exec_env->restore_call_chain; + is_restore = true; + exec_env->restore_call_chain = NULL; + exec_env->is_restore = false; + fprintf(stderr, "thread mgr: save_restore_call_chain %p\n", save_restore_call_chain); + } + else { +#endif + new_exec_env = wasm_exec_env_create_internal(module_inst, + exec_env->wasm_stack_size); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + } +#endif + LOG_DEBUG("new exec env is at %p\n", new_exec_env); + if (!new_exec_env) goto fail1; - if (is_aux_stack_allocated) { - /* Set aux stack for current thread */ - if (!wasm_exec_env_set_aux_stack(new_exec_env, aux_stack_start, - aux_stack_size)) { - goto fail2; + +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + // only need to transfer if not restoring + if (!exec_env->is_restore) { +#endif + if (is_aux_stack_allocated) { + /* Set aux stack for current thread */ + if (!wasm_exec_env_set_aux_stack(new_exec_env, aux_stack_start, + aux_stack_size)) { + goto fail3; + } } +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 } +#endif else { /* Disable aux stack */ new_exec_env->aux_stack_boundary.boundary = 0; @@ -740,6 +784,13 @@ wasm_cluster_create_thread(WASMExecEnv *exec_env, os_mutex_lock(&new_exec_env->wait_lock); +#if WASM_ENABLE_CHECKPOINT_RESTORE != 0 + if (is_restore) { + exec_env->restore_call_chain = save_restore_call_chain; + exec_env->is_restore = true; + } +#endif + if (0 != os_thread_create(&tid, thread_manager_start_routine, (void *)new_exec_env, diff --git a/doc/build_wamr.md b/doc/build_wamr.md index 1331f96019..1271aab8e9 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -134,6 +134,10 @@ cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM - **WAMR_BUILD_AOT_STACK_FRAME**=1/0, default to disable if not set > Note: if it is enabled, the AOT or JIT stack frames (like stack frame of classic interpreter but only necessary data is committed) will be created for AOT or JIT mode in function calls. And please add `--enable-dump-call-stack` option to wamrc during compiling AOT module. +#### **Enable Checkpoint Restore feature** +- **WAMR_BUILD_CHECKPOINT_RESTORE_FRAME**=1/0, default to disable if not set +> Note: if it is enabled, the checkpoint restore will be enabled, this feature requires `--enable-dump-call-stack` and `--enable-checkpoint` or `--enable-*-checkpoint` variant options to wamrc during compiling AOT module and `-g` for enableing the custum name section for wasm module. If the WASI or THREAD_MGR is switched on, we can support record and replay for WASI or PTHREAD. + #### **Enable dump call stack feature** - **WAMR_BUILD_DUMP_CALL_STACK**=1/0, default to disable if not set @@ -276,6 +280,21 @@ Currently we only profile the memory consumption of module, module_instance and wasm_runtime_get_context ``` +#### **Configurale memory access boundary check** +- **WAMR_CONFIGUABLE_BOUNDS_CHECKS**=1/0, default to disable if not set +> Note: If it is enabled, allow to run `iwasm --disable-bounds-checks` to disable the memory access boundary checks for interpreter mode. + +#### **Module instance context APIs** +- **WAMR_BUILD_MODULE_INST_CONTEXT**=1/0, default to disable if not set +> Note: If it is enabled, allow to set one or more contexts created by embedder for a module instance, the below APIs are provided: +```C + wasm_runtime_create_context_key + wasm_runtime_destroy_context_key + wasm_runtime_set_context + wasm_runtime_set_context_spread + wasm_runtime_get_context +``` + **Combination of configurations:** We can combine the configurations. For example, if we want to disable interpreter, enable AOT and WASI, we can run command: diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 9097525231..4dcd89a768 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -463,6 +463,27 @@ main(int argc, char *argv[]) else if (!strcmp(argv[0], "--enable-dump-call-stack")) { option.enable_aux_stack_frame = true; } + else if (!strcmp(argv[0], "--enable-checkpoint")) { + option.enable_aux_stack_frame = true; + option.enable_checkpoint = true; + } else if (!strcmp(argv[0], "--enable-loop-checkpoint")) { + option.enable_aux_stack_frame = true; + option.enable_checkpoint = true; + option.enable_loop_checkpoint = true; + } + else if (!strcmp(argv[0], "--enable-br-checkpoint")) { + option.enable_aux_stack_frame = true; + option.enable_checkpoint = true; + option.enable_br_checkpoint = true; + } + else if (!strcmp(argv[0], "--enable-every-checkpoint")) { + option.enable_aux_stack_frame = true; + option.enable_checkpoint = true; + option.enable_every_checkpoint = true; + } + else if (!strcmp(argv[0], "--enable-aux-stack-dirty-bit")) { + option.enable_aux_stack_dirty_bit = true; + } else if (!strcmp(argv[0], "--enable-perf-profiling")) { option.enable_aux_stack_frame = true; option.enable_perf_profiling = true;