[PAL/Linux-SGX] Add AEX-Notify enabling code

Documentation/manifest-syntax.rst

@@ -1392,6 +1392,30 @@ In addition, the application manifest must also contain ``sgx.debug = true``.
 
 See :ref:`vtune-sgx-profiling` for more information.
 
+Enabling AEX-Notify
+^^^^^^^^^^^^^^^^^^^
+
+::
+
+ sgx.experimental_enable_aex_notify = [true|false]
+ (Default: false)
+
+When enabled, this option instructs Gramine to use the AEX-Notify hardware feature.
+AEX-Notify is a flexible hardware extension that makes SGX enclaves interrupt
+aware: enclaves can register a trusted handler to be run after an interrupt or
+exception. AEX-Notify can be used as a building block for implementing
+countermeasures against different types of interrupt-based attacks in software.
+
+For more information on AEX-Notify, refer to the `academic paper
+<https://www.usenix.org/conference/usenixsecurity23/presentation/constable>`__
+and to the `official whitepaper
+<https://cdrdv2-public.intel.com/736463/aex-notify-white-paper-public.pdf>`__.
+
+.. warning::
+ Support for AEX-Notify in Gramine is not yet thoroughly tested and may
+ contain security vulnerabilities. This is temporary; the prefix
+ ``experimental_`` will be removed in the future after thorough validation.
+
 Deprecated options
 ------------------
 

pal/src/host/linux-sgx/generated_offsets.c

@@ -15,6 +15,7 @@ const struct generated_offset generated_offsets[] = {
  /* defines from sgx_arch.h */
  DEFINE(SGX_FLAGS_DEBUG, SGX_FLAGS_DEBUG),
  DEFINE(SGX_FLAGS_MODE64BIT, SGX_FLAGS_MODE64BIT),
+ DEFINE(SGX_FLAGS_AEXNOTIFY, SGX_FLAGS_AEXNOTIFY),
  DEFINE(SGX_XFRM_LEGACY, SGX_XFRM_LEGACY),
  DEFINE(SGX_XFRM_AVX, SGX_XFRM_AVX),
  DEFINE(SGX_XFRM_MPX, SGX_XFRM_MPX),
@@ -54,6 +55,7 @@ const struct generated_offset generated_offsets[] = {
  OFFSET_T(SGX_GPR_RFLAGS, sgx_pal_gpr_t, rflags),
  OFFSET_T(SGX_GPR_RIP, sgx_pal_gpr_t, rip),
  OFFSET_T(SGX_GPR_EXITINFO, sgx_pal_gpr_t, exitinfo),
+ OFFSET_T(SGX_GPR_AEXNOTIFY, sgx_pal_gpr_t, aexnotify),
  DEFINE(SGX_GPR_SIZE, sizeof(sgx_pal_gpr_t)),
 
  /* sgx_cpu_context_t */
@@ -103,6 +105,7 @@ const struct generated_offset generated_offsets[] = {
  OFFSET(SGX_HEAP_MIN, pal_enclave_tcb, heap_min),
  OFFSET(SGX_HEAP_MAX, pal_enclave_tcb, heap_max),
  OFFSET(SGX_CLEAR_CHILD_TID, pal_enclave_tcb, clear_child_tid),
+ OFFSET(SGX_READY_FOR_AEX_NOTIFY, pal_enclave_tcb, ready_for_aex_notify),
 
  /* struct pal_host_tcb aka PAL_HOST_TCB */
  OFFSET(PAL_HOST_TCB_TCS, pal_host_tcb, tcs),
@@ -123,6 +126,7 @@ const struct generated_offset generated_offsets[] = {
  OFFSET_T(TCS_OFS_LIMIT, sgx_arch_tcs_t, ofs_limit),
  OFFSET_T(TCS_OGS_LIMIT, sgx_arch_tcs_t, ogs_limit),
  DEFINE(TCS_SIZE, sizeof(sgx_arch_tcs_t)),
+ DEFINE(TCS_FLAGS_AEXNOTIFY, TCS_FLAGS_AEXNOTIFY),
 
  /* sgx_attributes_t */
  OFFSET_T(SGX_ATTRIBUTES_XFRM, sgx_attributes_t, xfrm),

pal/src/host/linux-sgx/host_framework.c

@@ -175,6 +175,24 @@ bool is_wrfsbase_supported(void) {
  return true;
 }
 
+bool is_aexnotify_supported(void) {
+ uint32_t cpuinfo[4];
+
+ cpuid(INTEL_SGX_LEAF, 1, cpuinfo);
+ if (!((cpuinfo[CPUID_WORD_EAX] >> 10) & 0x1)) {
+ log_error("AEX-Notify hardware feature is not supported.");
+ return false;
+ }
+
+ cpuid(INTEL_SGX_LEAF, 0, cpuinfo);
+ if (!((cpuinfo[CPUID_WORD_EAX] >> 11) & 0x1)) {
+ log_error("ENCLU[EDECCSSA] leaf instruction is not supported.");
+ return false;
+ }
+
+ return true;
+}
+
 int create_enclave(sgx_arch_secs_t* secs, sgx_arch_token_t* token) {
  assert(secs->size && IS_POWER_OF_2(secs->size));
  assert(IS_ALIGNED(secs->base, secs->size));

pal/src/host/linux-sgx/host_internal.h

@@ -66,6 +66,7 @@ void* realloc(void* ptr, size_t new_size);
 
 int open_sgx_driver(void);
 bool is_wrfsbase_supported(void);
+bool is_aexnotify_supported(void);
 
 int read_enclave_token(int token_file, sgx_arch_token_t* out_token);
 int create_dummy_enclave_token(sgx_sigstruct_t* sig, sgx_arch_token_t* out_token);

pal/src/host/linux-sgx/host_main.c

@@ -552,7 +552,10 @@ static int initialize_enclave(struct pal_enclave* enclave, const char* manifest_
  tcs->ogs_base = tls_area->addr - enclave->baseaddr + t * g_page_size;
  tcs->ofs_limit = 0xfff;
  tcs->ogs_limit = 0xfff;
+ tcs->flags |= (enclave_token.body.attributes.flags & SGX_FLAGS_AEXNOTIFY)
+ ? TCS_FLAGS_AEXNOTIFY : 0;
  tcs_addrs[t] = (void*)tcs_area->addr + g_page_size * t;
+
  }
  } else if (areas[i].data_src == BUF) {
  memcpy(data, areas[i].buf, areas[i].buf_size);
@@ -690,6 +693,21 @@ static int parse_loader_config(char* manifest, struct pal_enclave* enclave_info,
  goto out;
  }
 
+ bool aex_notify_enabled;
+ ret = toml_bool_in(manifest_root, "sgx.experimental_enable_aex_notify",
+ /*defaultval=*/false, &aex_notify_enabled);
+ if (ret < 0) {
+ log_error("Cannot parse 'sgx.experimental_enable_aex_notify'");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (aex_notify_enabled && !is_aexnotify_supported()) {
+ log_error("Cannot enable AEX-Notify on this platform (hardware doesn't support it)");
+ ret = -EPERM;
+ goto out;
+ }
+
  int64_t thread_num_int64;
  ret = toml_int_in(manifest_root, "sgx.max_threads", /*defaultval=*/-1, &thread_num_int64);
  if (ret < 0) {

pal/src/host/linux-sgx/pal_exception.c

@@ -22,6 +22,35 @@
 
 #define ADDR_IN_PAL(addr) ((void*)(addr) > TEXT_START && (void*)(addr) < TEXT_END)
 
+bool g_aex_notify_enabled = false;
+
+void init_aex_notify_for_thread(void) {
+ if (!g_aex_notify_enabled)
+ return;
+
+ SET_ENCLAVE_TCB(ready_for_aex_notify, 1UL);
+ MB();
+#if 0
+ /*
+ * FIXME: Re-enable in the following commit, when all AEX-Notify flows are added.
+ * Currently this would fail, as the untrusted runtime expects AEX-Notify flows but
+ * in-enclave runtime doesn't yet implement AEX-Notify flows.
+ */
+ GET_ENCLAVE_TCB(gpr)->aexnotify = 1U;
+ MB();
+#endif
+}
+
+void fini_aex_notify_for_thread(void) {
+ if (!g_aex_notify_enabled)
+ return;
+
+ SET_ENCLAVE_TCB(ready_for_aex_notify, 0UL);
+ MB();
+ GET_ENCLAVE_TCB(gpr)->aexnotify = 0U;
+ MB();
+}
+
 /* Restore an sgx_cpu_context_t as generated by .Lhandle_exception. Execution will
  * continue as specified by the rip in the context. */
 __attribute_no_sanitize_address

pal/src/host/linux-sgx/pal_linux.h

@@ -95,6 +95,10 @@ void save_xregs(PAL_XREGS_STATE* xsave_area);
 void restore_xregs(const PAL_XREGS_STATE* xsave_area);
 noreturn void _restore_sgx_context(sgx_cpu_context_t* uc, PAL_XREGS_STATE* xsave_area);
 
+extern bool g_aex_notify_enabled;
+void init_aex_notify_for_thread(void);
+void fini_aex_notify_for_thread(void);
+
 void _PalExceptionHandler(uint32_t trusted_exit_info_,
  uint32_t untrusted_external_event, sgx_cpu_context_t* uc,
  PAL_XREGS_STATE* xregs_state, sgx_arch_exinfo_t* exinfo);

pal/src/host/linux-sgx/pal_main.c

@@ -722,6 +722,13 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void*
  ocall_exit(1, /*is_exitgroup=*/true);
  }
 
+ ret = toml_bool_in(g_pal_public_state.manifest_root, "sgx.experimental_enable_aex_notify",
+ /*defaultval=*/false, &g_aex_notify_enabled);
+ if (ret < 0) {
+ log_error("Cannot parse 'sgx.experimental_enable_aex_notify'");
+ ocall_exit(1, /*is_exitgroup=*/true);
+ }
+
  /* Get host information about domain name configuration only for the first process.
  * This information will be checkpointed and restored during forking of the child
  * process(es). */
@@ -764,6 +771,8 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void*
  assert(!g_pal_linuxsgx_state.enclave_initialized);
  g_pal_linuxsgx_state.enclave_initialized = true;
 
+ init_aex_notify_for_thread();
+
  /* call main function */
  pal_main(instance_id, parent, first_thread, arguments, environments, post_callback);
 }

pal/src/host/linux-sgx/pal_tcb.h

@@ -42,6 +42,7 @@ struct pal_enclave_tcb {
  uint64_t ocall_exit_called;
  uint64_t thread_started;
  uint64_t ready_for_exceptions;
+ uint64_t ready_for_aex_notify;
  uint64_t manifest_size;
  void* heap_min;
  void* heap_max;

pal/src/host/linux-sgx/pal_threading.c

@@ -90,6 +90,7 @@ static void init_dynamic_thread(void* addr) {
  tcs->ogs_base = (uint64_t)tcb - g_enclave_base;
  tcs->ofs_limit = 0xfff;
  tcs->ogs_limit = 0xfff;
+ tcs->flags |= g_aex_notify_enabled ? TCS_FLAGS_AEXNOTIFY : 0;
 }
 
 static int create_dynamic_tcs_if_none_available(void** out_tcs) {
@@ -160,6 +161,8 @@ void pal_start_thread(void) {
  pal_set_tcb_stack_canary(stack_protector_canary);
  PAL_TCB* pal_tcb = pal_get_tcb();
  memset(&pal_tcb->libos_tcb, 0, sizeof(pal_tcb->libos_tcb));
+ init_aex_notify_for_thread();
+
  callback((void*)param);
  _PalThreadExit(/*clear_child_tid=*/NULL);
  /* UNREACHABLE */
@@ -227,6 +230,8 @@ void _PalThreadYieldExecution(void) {
 noreturn void _PalThreadExit(int* clear_child_tid) {
  struct pal_handle_thread* exiting_thread = GET_ENCLAVE_TCB(thread);
 
+ fini_aex_notify_for_thread();
+
  /* thread is ready to exit, must inform LibOS by erasing clear_child_tid;
  * note that we don't do it now (because this thread still occupies SGX
  * TCS slot) but during handle_thread_reset in assembly code */

pal/src/host/linux-sgx/sgx_arch.h

@@ -66,6 +66,7 @@ typedef uint8_t sgx_isvfamily_id_t[SGX_ISV_FAMILY_ID_SIZE];
 #define SGX_FLAGS_MODE64BIT 0x04ULL
 #define SGX_FLAGS_PROVISION_KEY 0x10ULL
 #define SGX_FLAGS_LICENSE_KEY 0x20ULL
+#define SGX_FLAGS_AEXNOTIFY 0x400ULL
 
 /* EINIT must verify *all* SECS.ATTRIBUTES[63..0] bits (FLAGS bits) against
  * SIGSTRUCT.ATTRIBUTES[63..0].
@@ -168,7 +169,8 @@ typedef struct {
 } sgx_arch_tcs_t;
 static_assert(sizeof(sgx_arch_tcs_t) == 4096, "incorrect struct size");
 
-#define TCS_FLAGS_DBGOPTIN (01ULL)
+#define TCS_FLAGS_DBGOPTIN (01ULL)
+#define TCS_FLAGS_AEXNOTIFY (02ULL)
 
 typedef struct {
  uint64_t rax;
@@ -192,10 +194,12 @@ typedef struct {
  uint64_t ursp;
  uint64_t urbp;
  uint32_t exitinfo;
- uint32_t reserved;
+ uint8_t reserved[3];
+ uint8_t aexnotify;
  uint64_t fsbase;
  uint64_t gsbase;
 } sgx_pal_gpr_t;
+static_assert(offsetof(sgx_pal_gpr_t, aexnotify) == 167, "Wrong offset of AEX-Notify bit in SSA");
 
 typedef struct {
  uint64_t rax;
@@ -443,6 +447,7 @@ static inline int enclu(uint32_t eax, uint64_t rbx, uint64_t rcx, uint64_t rdx)
 #define EACCEPT 5
 #define EMODPE 6
 #define EACCEPTCOPY 7
+#define EDECCSSA 9
 
 #define SGX_LAUNCH_KEY 0
 #define SGX_PROVISION_KEY 1

python/graminelibos/manifest.py

@@ -309,6 +309,7 @@ def __init__(self, manifest_str):
  sgx.setdefault('debug', False)
  sgx.setdefault('enable_stats', False)
  sgx.setdefault('edmm_enable', False)
+ sgx.setdefault('experimental_enable_aex_notify', False)
 
  if sgx['edmm_enable']:
  sgx.setdefault('enclave_size', DEFAULT_ENCLAVE_SIZE_WITH_EDMM)

python/graminelibos/manifest_check.py

@@ -81,6 +81,7 @@
  },
  'debug': bool,
  'edmm_enable': bool,
+ 'experimental_enable_aex_notify': bool,
  'enable_stats': bool,
  'enclave_size': _size,
  'file_check_policy': Any('strict', 'allow_all_but_log'),

python/graminelibos/sgx_sign.py

@@ -100,6 +100,7 @@ def collect_cpu_feature_bits(manifest_cpu_features, options_dict, val, mask, sec
 def get_enclave_attributes(manifest_sgx):
  flags_dict = {
  'debug': offs.SGX_FLAGS_DEBUG,
+ 'experimental_enable_aex_notify': offs.SGX_FLAGS_AEXNOTIFY,
  }
  flags = collect_bits(manifest_sgx, flags_dict)
  if ARCHITECTURE == 'amd64':
@@ -271,6 +272,9 @@ def set_tls_field(t, offset, value):
  set_tcs_field(t, offs.TCS_OFS_LIMIT, '<L', 0xfff)
  set_tcs_field(t, offs.TCS_OGS_LIMIT, '<L', 0xfff)
 
+ if attr['enable_aex_notify']:
+ set_tcs_field(t, offs.TCS_FLAGS, '<Q', offs.TCS_FLAGS_AEXNOTIFY)
+
  set_tls_field(t, offs.SGX_COMMON_SELF, tls_area.addr + offs.PAGESIZE * t)
  set_tls_field(t, offs.SGX_COMMON_STACK_PROTECTOR_CANARY,
  offs.STACK_PROTECTOR_CANARY_DEFAULT)
@@ -473,13 +477,15 @@ def get_mrenclave_and_manifest(manifest_path, libpal, verbose=False):
  attr = {
  'enclave_size': parse_size(manifest_sgx['enclave_size']),
  'edmm_enable': manifest_sgx.get('edmm_enable', False),
+ 'enable_aex_notify': manifest_sgx.get('experimental_enable_aex_notify', False),
  'max_threads': manifest_sgx['max_threads'],
  }
 
  if verbose:
  print('Attributes (required for enclave measurement):')
  print(f' size: {attr["enclave_size"]:#x}')
  print(f' edmm: {attr["edmm_enable"]}')
+ print(f' aex-notify: {attr["enable_aex_notify"]}')
  print(f' max_threads: {attr["max_threads"]}')
 
  print('SGX remote attestation:')