Merge branch 'Multi-V-VM-main'

build-scripts/config_common.cmake

@@ -304,6 +304,10 @@ if (WAMR_BUILD_AOT_STACK_FRAME EQUAL 1)
   add_definitions (-DWASM_ENABLE_AOT_STACK_FRAME=1)
   message ("     AOT stack frame enabled")
 endif ()
+if (WAMR_BUILD_CHECKPOINT_RESTORE EQUAL 1)
+  add_definitions (-DWASM_ENABLE_CHECKPOINT_RESTORE=1)
+  message ("     Checkpoint Restore enabled")
+endif ()
 if (WAMR_BUILD_MEMORY_PROFILING EQUAL 1)
   add_definitions (-DWASM_ENABLE_MEMORY_PROFILING=1)
   message ("     Memory profiling enabled")

core/config.h

@@ -335,6 +335,11 @@
 #define WASM_ENABLE_AOT_STACK_FRAME 0
 #endif
 
+/* Checkpoint Restore */
+#ifndef WASM_ENABLE_CHECKPOINT_RESTORE
+#define WASM_ENABLE_CHECKPOINT_RESTORE 0
+#endif
+
 /* Heap verification */
 #ifndef BH_ENABLE_GC_VERIFY
 #define BH_ENABLE_GC_VERIFY 0

core/iwasm/aot/aot_reloc.h

@@ -55,6 +55,7 @@ typedef struct {
 
 #if WASM_ENABLE_AOT_STACK_FRAME != 0
 #define REG_AOT_TRACE_SYM()               \
+    REG_SYM(aot_raise),                   \
     REG_SYM(aot_alloc_frame),             \
     REG_SYM(aot_free_frame),              \
     REG_SYM(aot_frame_update_profile_info),

core/iwasm/aot/aot_runtime.c

@@ -16,6 +16,12 @@
 #include "../libraries/thread-mgr/thread_manager.h"
 #endif
 
+#include "wasm_interp.h"
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+#include "wamr_export.h"
+extern int counter_;
+#endif
+
 /*
  * Note: These offsets need to match the values hardcoded in
  * AoT compilation code: aot_create_func_context, check_suspend_flags.
@@ -72,6 +78,11 @@ bh_static_assert(offsetof(AOTFrame, sp) == sizeof(uintptr_t) * 5);
 bh_static_assert(offsetof(AOTFrame, frame_ref) == sizeof(uintptr_t) * 6);
 bh_static_assert(offsetof(AOTFrame, lp) == sizeof(uintptr_t) * 7);
 
+bh_static_assert(offsetof(AOTFrame, ip_offset) == sizeof(uintptr_t) * 4);
+bh_static_assert(offsetof(AOTFrame, sp) == sizeof(uintptr_t) * 5);
+bh_static_assert(offsetof(AOTFrame, frame_ref) == sizeof(uintptr_t) * 6);
+bh_static_assert(offsetof(AOTFrame, lp) == sizeof(uintptr_t) * 7);
+
 static void
 set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
 {
@@ -804,6 +815,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent,
         bh_assert(memory_idx == 0);
         bh_assert(parent->memory_count > memory_idx);
         shared_memory_instance = parent->memories[memory_idx];
+        shared_memory_instance->ref_count++;
         shared_memory_inc_reference(shared_memory_instance);
         return shared_memory_instance;
     }
@@ -992,6 +1004,9 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent,
     AOTMemoryInstance *memories, *memory_inst;
     AOTMemInitData *data_seg;
     uint64 total_size;
+#if WASM_ENABLE_SHARED_MEMORY != 0
+    bool is_shared_memory;
+#endif
 
     module_inst->memory_count = memory_count;
     total_size = sizeof(AOTMemoryInstance *) * (uint64)memory_count;
@@ -1019,6 +1034,15 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent,
         return true;
     }
 
+#if WASM_ENABLE_SHARED_MEMORY != 0
+    /* Currently we have only one memory instance */
+    is_shared_memory = module->memories[0].memory_flags & 0x02 ? true : false;
+    if (is_shared_memory && parent != NULL) {
+        /* Ignore setting memory init data if the memory has been initialized */
+        return true;
+    }
+#endif
+
     for (i = 0; i < module->mem_init_data_count; i++) {
         data_seg = module->mem_init_data_list[i];
 #if WASM_ENABLE_BULK_MEMORY != 0
@@ -1860,6 +1884,7 @@ destroy_c_api_frames(Vector *frames)
 void
 aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst)
 {
+#if WASM_ENABLE_CHECKPOINT_RESTORE == 0
     WASMModuleInstanceExtraCommon *common =
         &((AOTModuleInstanceExtra *)module_inst->e)->common;
     if (module_inst->exec_env_singleton) {
@@ -1934,6 +1959,7 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst)
 #endif
 
     wasm_runtime_free(module_inst);
+#endif
 }
 
 AOTFunctionInstance *
@@ -2225,6 +2251,7 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function,
         while (exec_env->cur_frame != prev_frame)
             aot_free_frame(exec_env);
 #endif
+        // checkpoint
         if (!ret) {
             if (argv1 != argv1_buf)
                 wasm_runtime_free(argv1);
@@ -2789,6 +2816,17 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
     }
 
     tbl_elem_val = ((table_elem_type_t *)tbl_inst->elems)[table_elem_idx];
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+    if (exec_env->is_restore && exec_env->restore_call_chain) {
+        struct AOTFrame *rcc = *(exec_env->restore_call_chain);
+        while (rcc->prev_frame) {
+            rcc = rcc->prev_frame;
+        }
+        LOG_DEBUG("func_idx: %d instead of %d of thread %ld\n", rcc->func_index,
+                  func_idx, exec_env->handle);
+        func_idx = rcc->func_index;
+    }
+#endif
     if (tbl_elem_val == NULL_REF) {
         aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT);
         goto fail;
@@ -3525,9 +3563,17 @@ aot_free_frame(WASMExecEnv *exec_env)
 
 #else /* else of WASM_ENABLE_GC == 0 */
 
+int
+aot_raise(WASMExecEnv *exec_env, int sig)
+{
+    raise(sig);
+}
 bool
 aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index)
 {
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+    LOG_DEBUG("aot_alloc_frame %u thread %d\n", func_index, exec_env->handle);
+#endif
     AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
     AOTModule *module = (AOTModule *)module_inst->module;
 #if WASM_ENABLE_PERF_PROFILING != 0
@@ -3537,6 +3583,27 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index)
     AOTFrame *frame;
     uint32 max_local_cell_num, max_stack_cell_num, all_cell_num;
     uint32 aot_func_idx, frame_size;
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+    if (exec_env->restore_call_chain) {
+        frame = exec_env->restore_call_chain[exec_env->call_chain_size - 1];
+        LOG_DEBUG("frame restored, func idx %zu\n", frame->func_index);
+        exec_env->call_chain_size--;
+        frame->prev_frame = (AOTFrame *)exec_env->cur_frame;
+        exec_env->cur_frame = (struct WASMInterpFrame *)frame;
+        if (exec_env->call_chain_size == 0) {
+            // TODO: fix memory leak
+            exec_env->restore_call_chain = NULL;
+                    }
+        LOG_DEBUG("restore call chain %zu==%u, %p, %p, %d\n",
+                ((AOTFrame *)exec_env->cur_frame)->func_index, func_index,
+                exec_env, exec_env->restore_call_chain, exec_env->handle);
+        if (((AOTFrame *)exec_env->cur_frame)->func_index != func_index) {
+            LOG_DEBUG("NOT MATCH!!!\n");
+            exit(1);
+        }
+        return true;
+    }
+#endif
 
     if (func_index >= module->import_func_count) {
         aot_func_idx = func_index - module->import_func_count;
@@ -3568,6 +3635,11 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index)
     frame->time_started = (uintptr_t)os_time_thread_cputime_us();
     frame->func_perf_prof_info = func_perf_prof;
 #endif
+    frame->ip_offset = 0;
+    frame->sp = frame->lp + max_local_cell_num;
+#if WASM_ENABLE_GC != 0
+    frame->frame_ref = frame->sp + max_stack_cell_num;
+#endif
 
 #if WASM_ENABLE_GC != 0
     frame->sp = frame->lp + max_local_cell_num;
@@ -3584,6 +3656,10 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index)
 static inline void
 aot_free_frame_internal(WASMExecEnv *exec_env)
 {
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+    int func_index = ((AOTFrame *)exec_env->cur_frame)->func_index;
+    LOG_DEBUG("aot_free_frame %zu %d\n", func_index, exec_env->handle);
+#endif
     AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame;
     AOTFrame *prev_frame = cur_frame->prev_frame;
 
@@ -3598,9 +3674,18 @@ aot_free_frame_internal(WASMExecEnv *exec_env)
     if (prev_frame)
         prev_frame->func_perf_prof_info->children_exec_time += time_elapsed;
 #endif
-
     wasm_exec_env_free_wasm_frame(exec_env, cur_frame);
     exec_env->cur_frame = (struct WASMInterpFrame *)prev_frame;
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+    if (func_index == stop_func_index && is_debug) {
+        LOG_DEBUG("cur_func_count %d %d\n", cur_func_count,
+                  stop_func_threshold);
+        if (cur_func_count == stop_func_threshold) {
+            serialize_to_file(exec_env);
+        }
+        cur_func_count++;
+    }
+#endif
 }
 
 void

core/iwasm/aot/aot_runtime.h

@@ -673,6 +673,9 @@ aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx,
 bool
 aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index);
 
+int
+aot_raise(WASMExecEnv *exec_env, int exception);
+
 void
 aot_free_frame(WASMExecEnv *exec_env);
 

core/iwasm/common/wasm_c_api.c

@@ -26,7 +26,9 @@
 #if WASM_ENABLE_THREAD_MGR != 0
 #include "thread_manager.h"
 #endif
-
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+#include "ckpt_restore.h"
+#endif
 /*
  * Thread Model:
  * - Only one wasm_engine_t in one process
@@ -518,6 +520,12 @@ static bool
 search_thread_local_store_num(Vector *stores_by_tid, korp_tid tid,
                               thread_local_stores *out_ts, unsigned *out_i)
 {
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+    korp_tid temp;
+    if((temp = wamr_get_new_korp_tid(tid))){
+        tid = temp;
+    }
+#endif
     unsigned i;
 
     for (i = 0; i < stores_by_tid->num_elems; i++) {
@@ -538,6 +546,12 @@ search_thread_local_store_num(Vector *stores_by_tid, korp_tid tid,
 static unsigned
 retrive_thread_local_store_num(Vector *stores_by_tid, korp_tid tid)
 {
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+    korp_tid temp;
+    if((temp = wamr_get_new_korp_tid(tid))){
+        tid = temp;
+    }
+#endif
 #ifndef os_thread_local_attribute
     unsigned i = 0;
     thread_local_stores ts = { 0 };
@@ -568,6 +582,12 @@ retrive_thread_local_store_num(Vector *stores_by_tid, korp_tid tid)
 static bool
 increase_thread_local_store_num(Vector *stores_by_tid, korp_tid tid)
 {
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+    korp_tid temp;
+    if((temp = wamr_get_new_korp_tid(tid))){
+        tid = temp;
+    }
+#endif
 #ifndef os_thread_local_attribute
     unsigned i = 0;
     thread_local_stores ts = { 0 };
@@ -614,6 +634,12 @@ increase_thread_local_store_num(Vector *stores_by_tid, korp_tid tid)
 static bool
 decrease_thread_local_store_num(Vector *stores_by_tid, korp_tid tid)
 {
+#if WASM_ENABLE_CHECKPOINT_RESTORE != 0
+    korp_tid temp;
+    if((temp = wamr_get_new_korp_tid(tid))){
+        tid = temp;
+    }
+#endif
 #ifndef os_thread_local_attribute
     unsigned i = 0;
     thread_local_stores ts = { 0 };

core/iwasm/common/wasm_exec_env.c

@@ -72,6 +72,7 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst,
     exec_env->wasm_stack.top_boundary =
         exec_env->wasm_stack.bottom + stack_size;
     exec_env->wasm_stack.top = exec_env->wasm_stack.bottom;
+    exec_env->is_checkpoint = false;
 
 #if WASM_ENABLE_AOT != 0
     if (module_inst->module_type == Wasm_Module_AoT) {
@@ -85,6 +86,10 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst,
     wasm_runtime_dump_exec_env_mem_consumption(exec_env);
 #endif
 
+    exec_env->is_checkpoint = false;
+    exec_env->is_restore = false;
+    exec_env->call_chain_size = 0;
+    exec_env->restore_call_chain = NULL;
     return exec_env;
 
 #ifdef OS_ENABLE_HW_BOUND_CHECK
@@ -174,6 +179,7 @@ wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst,
     }
 #endif
 
+
 #if WASM_ENABLE_THREAD_MGR != 0
     /* Create a new cluster for this exec_env */
     if (!(cluster = wasm_cluster_create(exec_env))) {

core/iwasm/common/wasm_exec_env.h

@@ -158,6 +158,12 @@ typedef struct WASMExecEnv {
 
     /* The WASM stack size */
     uint32 wasm_stack_size;
+    /* Whether is checkpoint */
+    bool is_checkpoint;
+    /* Whether is restore */
+    bool is_restore;
+    size_t call_chain_size;
+    struct AOTFrame **restore_call_chain;
 
     /* The WASM stack of current thread */
     union {