Move the executable heap flag into the PEB and make it a runtime setting

CHANGELOG.md

@@ -4,6 +4,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Prerelease] - Unreleased
 
+### Added
+* Add `SandboxConfiguration::set_executable_heap` and `SandboxBuilder::executable_heap` to choose per sandbox whether guest heap pages are executable. The `executable_heap` cargo feature sets the default.
+
+### Changed
+* **Breaking:** Snapshots use ABI 10. Existing snapshots must be regenerated.
+
 ## [v0.19.0] - 2026-11-04
 
 ### Added

Justfile

@@ -297,7 +297,7 @@ test-isolated target=default-target features="" :
 
 # runs integration tests
 test-integration target=default-target features="":
-    @# run execute_on_heap test with feature "executable_heap" on (runs with off during normal tests)
+    @# run execute_on_heap with feature "executable_heap" on, which sets the default heap permission
     {{ cargo-cmd }} test {{ if features =="" {"--features executable_heap"} else if features=="no-default-features" {"--no-default-features --features executable_heap"} else {"--no-default-features -F executable_heap," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} --test integration_test execute_on_heap
 
     @# run component-util unit tests and the integration tests that exercise WIT input codegen

docs/snapshot-versioning.md

@@ -52,6 +52,9 @@ regenerated.
 ABI 9 removes the heap region from snapshot memory. Earlier snapshots must
 be regenerated.
 
+ABI 10 moves the executable heap flag from the scratch-top metadata into the
+PEB. Earlier snapshots must be regenerated.
+
 The `OCI_LAYOUT_VERSION` constant is pinned by the OCI image-layout
 spec at `1.0.0`.
 

src/hyperlight_common/src/layout.rs

@@ -25,8 +25,8 @@ pub const SCRATCH_TOP_RESERVED_PAGES: usize = 12;
 // down from the top of scratch memory.
 #[repr(C)]
 struct ScratchTopMetadata {
-    /// Nonzero if guest heap pages are mapped executable.
-    heap_executable: u64,
+    /// Padding that keeps the exception stack 16-byte aligned.
+    _alignment_padding: [u8; 8],
     /// Host-published capacity of each H2G buffer.
     h2g_buffer_size: u64,
     /// Number of pages reserved for the H2G pool.
@@ -85,8 +85,6 @@ pub const SCRATCH_TOP_LIBC_RNG_SEED_OFFSET: u64 =
     scratch_top_offset(offset_of!(ScratchTopMetadata, libc_rng_seed));
 pub const SCRATCH_TOP_GUEST_LOG_LEVEL_OFFSET: u64 =
     scratch_top_offset(offset_of!(ScratchTopMetadata, guest_log_level));
-pub const SCRATCH_TOP_HEAP_EXECUTABLE_OFFSET: u64 =
-    scratch_top_offset(offset_of!(ScratchTopMetadata, heap_executable));
 pub const SCRATCH_TOP_EXN_STACK_OFFSET: u64 = size_of::<ScratchTopMetadata>() as u64;
 
 const _: () = {
@@ -107,7 +105,6 @@ const _: () = {
     assert!(SCRATCH_TOP_H2G_QUEUE_SIZE_OFFSET == 0x58);
     assert!(SCRATCH_TOP_H2G_POOL_PAGES_OFFSET == 0x60);
     assert!(SCRATCH_TOP_H2G_BUFFER_SIZE_OFFSET == 0x68);
-    assert!(SCRATCH_TOP_HEAP_EXECUTABLE_OFFSET == 0x70);
     assert!(SCRATCH_TOP_EXN_STACK_OFFSET == 0x70);
 
     assert!(VIRTQ_BUFFER_GVA_START < VIRTQ_BUFFER_GVA_END);

src/hyperlight_common/src/mem.rs

@@ -15,6 +15,8 @@ pub struct GuestMemoryRegion {
 #[repr(C)]
 pub struct HyperlightPEB {
     pub init_data: GuestMemoryRegion,
+    /// Nonzero if guest heap pages are mapped executable.
+    pub heap_executable: u64,
 }
 
 #[cfg(test)]
@@ -28,6 +30,7 @@ mod tests {
                 size: 0x1111,
                 ptr: 0x2222,
             },
+            heap_executable: 1,
         };
         let bytes = bytemuck::bytes_of(&peb);
         let peb2 = *bytemuck::from_bytes::<HyperlightPEB>(bytes);

src/hyperlight_guest/src/layout.rs

@@ -58,9 +58,6 @@ pub fn guest_log_level_gva() -> *mut u64 {
     use hyperlight_common::layout::{SCRATCH_TOP_GUEST_LOG_LEVEL_OFFSET, SCRATCH_TOP_GVA};
     (SCRATCH_TOP_GVA as u64 - SCRATCH_TOP_GUEST_LOG_LEVEL_OFFSET + 1) as *mut u64
 }
-pub fn heap_executable_gva() -> *mut u64 {
-    scratch_top_gva(hyperlight_common::layout::SCRATCH_TOP_HEAP_EXECUTABLE_OFFSET)
-}
 pub use arch::{scratch_base_gpa, scratch_base_gva};
 
 /// Exclusive upper GPA for the scratch page allocator: the transport arena,

src/hyperlight_guest_bin/src/arch/aarch64/exception/handle.rs

@@ -106,9 +106,7 @@ fn handle_internal_fault(exn: Exception, far: u64, orig_mapping: Option<Mapping>
                 unsafe { handle_stack_or_heap_fault(far, false) };
                 true
             } else if (HEAP_GVA_START..HEAP_GVA_END).contains(&far) {
-                // SAFETY: the host writes the scratch-top metadata before entry.
-                let executable =
-                    unsafe { hyperlight_guest::layout::heap_executable_gva().read_volatile() } != 0;
+                let executable = crate::HEAP_EXECUTABLE.load(Ordering::Relaxed);
                 // SAFETY: a translation fault in the heap region means
                 // the page is unmapped, and exception handling is the
                 // only page table writer.

src/hyperlight_guest_bin/src/arch/amd64/exception/handle.rs

@@ -133,9 +133,7 @@ fn try_handle_internal_pagefault(
             return true;
         }
         if (HEAP_GVA_START..HEAP_GVA_END).contains(&gva) {
-            // SAFETY: the host writes the scratch-top metadata before entry.
-            let executable =
-                unsafe { hyperlight_guest::layout::heap_executable_gva().read_volatile() } != 0;
+            let executable = crate::HEAP_EXECUTABLE.load(Ordering::Relaxed);
             // SAFETY: the fault is not-present in the heap region, and
             // exception handling is the only page table writer.
             unsafe { handle_stack_or_heap_pagefault(gva, executable) };

src/hyperlight_guest_bin/src/lib.rs

@@ -6,6 +6,7 @@
 extern crate alloc;
 
 use core::fmt::Write;
+use core::sync::atomic::{AtomicBool, Ordering, compiler_fence};
 
 use arch::dispatch::dispatch_function;
 use guest_function::register::GuestFunctionRegister;
@@ -116,6 +117,9 @@ pub(crate) static HEAP_ALLOCATOR: GuestHeap = GuestHeap::new(Manual);
 #[global_allocator]
 pub(crate) static HEAP_ALLOCATOR: ProfiledLockedHeap = ProfiledLockedHeap(GuestHeap::new(Manual));
 
+/// Whether heap pages are mapped executable on first touch.
+pub(crate) static HEAP_EXECUTABLE: AtomicBool = AtomicBool::new(false);
+
 pub static mut GUEST_HANDLE: GuestHandle = GuestHandle::new();
 pub(crate) static mut REGISTERED_GUEST_FUNCTIONS: GuestFunctionRegister<GuestFunc> =
     GuestFunctionRegister::new();
@@ -265,6 +269,11 @@ pub(crate) extern "C" fn generic_init(
         #[allow(static_mut_refs)]
         let peb_ptr = GUEST_HANDLE.peb().unwrap();
 
+        // SAFETY: the host writes the PEB at `peb_address` before entry.
+        HEAP_EXECUTABLE.store((*peb_ptr).heap_executable != 0, Ordering::Relaxed);
+        // The heap accesses below fault into the handler that reads this flag.
+        compiler_fence(Ordering::SeqCst);
+
         #[cfg(not(all(feature = "mem_profile", target_arch = "x86_64")))]
         let heap_allocator = &HEAP_ALLOCATOR;
         #[cfg(all(feature = "mem_profile", target_arch = "x86_64"))]

src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs

@@ -1467,7 +1467,9 @@ mod tests {
         let mut snapshot_contents = vec![0u8; mem_size];
         snapshot_contents[layout.guest_code_offset()..layout.guest_code_offset() + code.len()]
             .copy_from_slice(code);
-        layout.write_peb(&mut snapshot_contents).unwrap();
+        layout
+            .write_peb(&mut snapshot_contents, config.get_executable_heap())
+            .unwrap();
         let snapshot_memory = crate::sandbox::snapshot::memory::tests::flat_memory(
             &snapshot_contents,
             &pt_bytes,

src/hyperlight_host/src/mem/layout.rs

@@ -451,7 +451,7 @@ impl SandboxMemoryLayout {
     /// Note: `mem` may have been modified, even if `Err` was returned
     /// from this function.
     #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
-    pub(crate) fn write_peb(&self, mem: &mut [u8]) -> Result<()> {
+    pub(crate) fn write_peb(&self, mem: &mut [u8], heap_executable: bool) -> Result<()> {
         use hyperlight_common::mem::GuestMemoryRegion;
 
         let guest_base = Self::BASE_ADDRESS as u64;
@@ -461,6 +461,7 @@ impl SandboxMemoryLayout {
                 size: (self.get_unaligned_memory_size() - self.init_data_offset()) as u64,
                 ptr: guest_base + self.init_data_offset() as u64,
             },
+            heap_executable: heap_executable as u64,
         };
 
         let offset = self.peb_offset();

src/hyperlight_host/src/mem/mgr.rs

@@ -1160,10 +1160,6 @@ impl SandboxMemoryManager<HostSharedMemory> {
             SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET,
             self.snapshot_count,
         )?;
-        self.update_scratch_bookkeeping_item(
-            SCRATCH_TOP_HEAP_EXECUTABLE_OFFSET,
-            cfg!(feature = "executable_heap") as u64,
-        )?;
         // Record the G2H and H2G queue sizes, pool page counts, and buffer sizes.
         self.update_scratch_bookkeeping_item(
             SCRATCH_TOP_G2H_QUEUE_SIZE_OFFSET,

src/hyperlight_host/src/sandbox/builder.rs

@@ -120,7 +120,8 @@ impl SandboxBuilder {
 
     /// Build a sandbox restoring the guest from `snapshot`.
     ///
-    /// The snapshot's layout overrides scratch and transport settings.
+    /// The snapshot's layout overrides scratch and transport settings. The
+    /// snapshot keeps its executable heap setting.
     pub fn from_snapshot(snapshot: Arc<Snapshot>) -> Self {
         Self::with_source(Source::Snapshot(snapshot))
     }
@@ -316,6 +317,14 @@ impl SandboxBuilder {
         self.cfg.get_scratch_size()
     }
 
+    /// Set whether guest heap pages are mapped executable.
+    ///
+    /// See [`SandboxConfiguration::set_executable_heap`].
+    pub fn executable_heap(mut self, executable: bool) -> Self {
+        self.cfg.set_executable_heap(executable);
+        self
+    }
+
     /// Set the G2H virtqueue descriptor count.
     ///
     /// See [`SandboxConfiguration::set_g2h_queue_size`] for normalization.

src/hyperlight_host/src/sandbox/config.rs

@@ -83,6 +83,8 @@ pub struct SandboxConfiguration {
     g2h_pool_pages: usize,
     /// Number of pages in the H2G buffer pool.
     h2g_pool_pages: usize,
+    /// Whether guest heap pages are mapped executable.
+    executable_heap: bool,
     /// Declared guest MSRs, stored inline to keep this type `Copy`.
     #[cfg(target_arch = "x86_64")]
     guest_msrs: [u32; Self::MAX_GUEST_MSRS],
@@ -113,6 +115,9 @@ impl SandboxConfiguration {
     pub const DEFAULT_G2H_POOL_PAGES: usize = 12;
     /// The default total number of H2G pool pages.
     pub const DEFAULT_H2G_POOL_PAGES: usize = 8;
+    /// Whether guest heap pages are executable by default. The
+    /// `executable_heap` feature sets this.
+    pub const DEFAULT_EXECUTABLE_HEAP: bool = cfg!(feature = "executable_heap");
     /// The minimum G2H virtqueue descriptor count.
     const MIN_QUEUE_SIZE: usize = 2;
     /// The maximum G2H virtqueue descriptor count.
@@ -146,6 +151,7 @@ impl SandboxConfiguration {
             h2g_buffer_size: Self::DEFAULT_H2G_BUFFER_SIZE,
             g2h_pool_pages: Self::DEFAULT_G2H_POOL_PAGES,
             h2g_pool_pages: Self::DEFAULT_H2G_POOL_PAGES,
+            executable_heap: Self::DEFAULT_EXECUTABLE_HEAP,
             interrupt_retry_delay,
             interrupt_vcpu_sigrtmin_offset,
             #[cfg(gdb)]
@@ -281,6 +287,22 @@ impl SandboxConfiguration {
         self.scratch_size = scratch_size;
     }
 
+    /// Set whether guest heap pages are mapped executable.
+    ///
+    /// The guest sets this permission in its own page tables, so it protects
+    /// a well-behaved guest from running injected heap data but cannot stop a
+    /// malicious guest. Sandboxes built from a snapshot keep the value of the
+    /// sandbox built from the guest binary.
+    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
+    pub fn set_executable_heap(&mut self, executable: bool) {
+        self.executable_heap = executable;
+    }
+
+    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
+    pub(crate) fn get_executable_heap(&self) -> bool {
+        self.executable_heap
+    }
+
     /// Sets the maximum log level for guest code execution.
     ///
     /// If not set, the level is determined from the `RUST_LOG` environment

src/hyperlight_host/src/sandbox/snapshot/file/config.rs

@@ -926,7 +926,7 @@ mod schema_pin {
     const PINNED_CALL: &str = r#"{
   "hyperlight_version": "x.y.z",
   "arch": "x86_64",
-  "abi_version": 9,
+  "abi_version": 10,
   "hypervisor": "mshv",
   "cpu_vendor": "intel",
   "layer_alignment": 4096,
@@ -1122,7 +1122,7 @@ mod schema_pin {
     const PINNED_CALL: &str = r#"{
   "hyperlight_version": "x.y.z",
   "arch": "aarch64",
-  "abi_version": 9,
+  "abi_version": 10,
   "hypervisor": "mshv",
   "cpu_vendor": "intel",
   "layer_alignment": 16384,

src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs

@@ -25,7 +25,7 @@ pub(in crate::sandbox::snapshot) const MT_TRANSPORT_CURRENT: &str = MT_TRANSPORT
 /// ABI version for the snapshot memory blob. Bumped when the
 /// host-guest contract for the snapshot bytes changes. See
 /// docs/snapshot-versioning.md.
-pub(in crate::sandbox::snapshot) const SNAPSHOT_ABI_VERSION: u32 = 9;
+pub(in crate::sandbox::snapshot) const SNAPSHOT_ABI_VERSION: u32 = 10;
 
 /// OCI standard annotation key for a manifest's tag inside an image
 /// index. Set on the manifest descriptor in `index.json`, not on the

src/hyperlight_host/src/sandbox/snapshot/file_tests.rs

@@ -291,7 +291,7 @@ fn layered_writer_preserves_blobs_and_transport_geometry() {
         manifest["artifactType"],
         "application/vnd.hyperlight.snapshot.config.v5+json"
     );
-    assert_eq!(config["abi_version"], 9);
+    assert_eq!(config["abi_version"], 10);
     let blobs = snapshot.snapshot_memory().blobs().collect::<Vec<_>>();
     assert_eq!(layers.len(), blobs.len() + 1);
     for (index, blob) in blobs.iter().enumerate() {
@@ -2398,7 +2398,7 @@ fn legacy_config_versions_rejected() {
             &path,
             OciTag::new("latest").unwrap(),
         ));
-        assert_err_contains(err, "incompatible with snapshot ABI 9");
+        assert_err_contains(err, "incompatible with snapshot ABI 10");
     }
 }
 

src/hyperlight_host/src/sandbox/snapshot/mod.rs

@@ -675,7 +675,7 @@ impl Snapshot {
             &mut data[layout.guest_code_offset()..],
         )?;
 
-        layout.write_peb(data)?;
+        layout.write_peb(data, cfg.get_executable_heap())?;
 
         blob.map(|x| layout.write_init_data(data, x.data))
             .transpose()?;

src/hyperlight_host/src/sandbox/snapshot/tripwires.rs

@@ -16,7 +16,7 @@ use super::file::{
     SNAPSHOT_ABI_VERSION,
 };
 
-const EXPECTED_ABI_VERSION: u32 = 9;
+const EXPECTED_ABI_VERSION: u32 = 10;
 const EXPECTED_MT_CONFIG: &str = "application/vnd.hyperlight.snapshot.config.v5+json";
 const EXPECTED_MT_SNAPSHOT: &str = "application/vnd.hyperlight.snapshot.memory.v2";
 const EXPECTED_MT_TRANSPORT: &str = "application/vnd.hyperlight.snapshot.transport.v1";

src/hyperlight_host/tests/integration_test.rs

@@ -633,27 +633,44 @@ fn guard_page_check_2() {
 
 #[test]
 fn execute_on_heap() {
-    with_rust_sandbox(|mut sbox1| {
-        let result = sbox1.call::<String>("ExecuteOnHeap", ());
-
-        #[cfg(feature = "executable_heap")]
-        assert_eq!(
-            result.unwrap(),
-            "Executed on heap successfully",
-            "should execute successfully"
+    for executable in [true, false] {
+        with_rust_sandbox_from(
+            |builder| builder.executable_heap(executable),
+            |mut sbox| {
+                let result = sbox.call::<String>("ExecuteOnHeap", ());
+                if executable {
+                    assert_eq!(result.unwrap(), "Executed on heap successfully");
+                } else {
+                    let err = result.unwrap_err().to_string();
+                    assert!(
+                        err.contains(
+                            #[cfg(target_arch = "x86_64")]
+                            "PageFault",
+                            #[cfg(target_arch = "aarch64")]
+                            "Exception Syndrome: 0x86",
+                        ),
+                        "should get page fault: {err}"
+                    );
+                }
+            },
         );
+    }
+}
 
-        #[cfg(not(feature = "executable_heap"))]
-        assert!(
-            result.unwrap_err().to_string().contains(
-                #[cfg(target_arch = "x86_64")]
-                "PageFault",
-                #[cfg(target_arch = "aarch64")]
-                "Exception Syndrome: 0x86",
-            ),
-            "should get page fault"
-        );
-    });
+#[test]
+fn snapshot_keeps_executable_heap_setting() {
+    let mut sbox = SandboxBuilder::from_file(hyperlight_testing::simple_guest_as_pathbuf())
+        .executable_heap(true)
+        .build()
+        .unwrap();
+    let snapshot = sbox.snapshot().unwrap();
+
+    let mut restored = SandboxBuilder::from_snapshot(snapshot)
+        .executable_heap(false)
+        .build()
+        .unwrap();
+    let result: String = restored.call("ExecuteOnHeap", ()).unwrap();
+    assert_eq!(result, "Executed on heap successfully");
 }
 
 // checks that a recursive function with stack allocation eventually fails with stackoverflow

src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs

@@ -8,7 +8,7 @@
 //! publish. See `docs/snapshot-versioning.md`.
 
 /// Goldens version, a `vMAJOR.MINOR` string.
-pub(crate) const GOLDENS_VERSION: &str = "v9.0";
+pub(crate) const GOLDENS_VERSION: &str = "v10.0";
 
 /// Old majors kept loadable through a compatibility path, verified
 /// alongside `GOLDENS_VERSION`. A backwards-compatible break (Option 2)