wasm: make JS WebAssembly.instantiate operand stack/heap tunable

lib/quickbeam.ex

@@ -41,7 +41,11 @@ defmodule QuickBEAM do
       automatically bundled — imports are resolved from the filesystem and
       `node_modules/`, then compiled into a single script via OXC.
     * `:memory_limit` — maximum JS heap in bytes (default: 256 MB)
-    * `:max_stack_size` — maximum JS call stack in bytes (default: 4 MB)
+    * `:max_stack_size` — maximum JS call stack in bytes (default: 8 MB)
+    * `:wasm_stack_size` — WASM operand stack in bytes for guests started via the JS
+      `WebAssembly.instantiate` path (default: 65536). Distinct from `:max_stack_size`
+      (the JS call stack); raise it for guests whose deep init overflows the 64 KB default.
+    * `:wasm_heap_size` — WASM auxiliary heap in bytes for the same path (default: 65536)
     * `:max_convert_depth` — maximum nesting depth for JS→BEAM value conversion (default: 32)
     * `:max_convert_nodes` — maximum total nodes for JS→BEAM value conversion (default: 10,000)
 
@@ -83,6 +87,10 @@ defmodule QuickBEAM do
 
     * `:memory_limit` — maximum JS heap in bytes (default: 256 MB)
     * `:max_stack_size` — maximum JS call stack in bytes (default: 8 MB)
+    * `:wasm_stack_size` — WASM operand stack in bytes for guests started via the JS
+      `WebAssembly.instantiate` path (default: 65536). Distinct from `:max_stack_size`
+      (the JS call stack); raise it for guests whose deep init overflows the 64 KB default.
+    * `:wasm_heap_size` — WASM auxiliary heap in bytes for the same path (default: 65536)
     * `:max_convert_depth` — maximum nesting depth for JS→BEAM value conversion (default: 32)
     * `:max_convert_nodes` — maximum total nodes for JS→BEAM value conversion (default: 10,000)
   """

lib/quickbeam/context_pool.ex

@@ -22,6 +22,10 @@ defmodule QuickBEAM.ContextPool do
     * `:size` — number of runtime threads (default: `System.schedulers_online()`)
     * `:memory_limit` — maximum JS heap per thread in bytes (default: 256 MB)
     * `:max_stack_size` — maximum JS call stack in bytes (default: 8 MB)
+    * `:wasm_stack_size` — WASM operand stack in bytes for guests started via the JS
+      `WebAssembly.instantiate` path (default: 65536). Distinct from `:max_stack_size`
+      (the JS call stack); raise it for guests whose deep init overflows the 64 KB default.
+    * `:wasm_heap_size` — WASM auxiliary heap in bytes for the same path (default: 65536)
     * `:max_convert_depth` — maximum nesting depth for JS→BEAM value conversion (default: 32)
     * `:max_convert_nodes` — maximum total nodes for JS→BEAM value conversion (default: 10,000)
   """
@@ -46,7 +50,14 @@ defmodule QuickBEAM.ContextPool do
 
     nif_opts =
       opts
-      |> Keyword.take([:memory_limit, :max_stack_size, :max_convert_depth, :max_convert_nodes])
+      |> Keyword.take([
+        :memory_limit,
+        :max_stack_size,
+        :wasm_stack_size,
+        :wasm_heap_size,
+        :max_convert_depth,
+        :max_convert_nodes
+      ])
       |> Map.new()
 
     threads =

lib/quickbeam/context_types.zig

@@ -141,6 +141,12 @@ pub const PoolData = struct {
     thread: ?std.Thread,
     memory_limit: usize = 256 * 1024 * 1024,
     max_stack_size: usize = 8 * 1024 * 1024,
+    // WASM operand stack / heap for the JS `WebAssembly.instantiate` path
+    // (distinct from `max_stack_size`, the JS call stack). Default mirrors the
+    // WASM NIF path; raised via the pool `:wasm_stack_size` opt. Copied into
+    // each context's RuntimeData at create time.
+    wasm_stack_size: u32 = 65_536,
+    wasm_heap_size: u32 = 65_536,
     max_convert_depth: u32 = 32,
     max_convert_nodes: u32 = 10_000,
     shutting_down: std.atomic.Value(bool) = std.atomic.Value(bool).init(false),

lib/quickbeam/context_worker.zig

@@ -209,6 +209,8 @@ fn handle_create_context(
         .thread = null,
         .max_convert_depth = pd.max_convert_depth,
         .max_convert_nodes = pd.max_convert_nodes,
+        .wasm_stack_size = pd.wasm_stack_size,
+        .wasm_heap_size = pd.wasm_heap_size,
     };
     entry.owner_pid = p.owner_pid;
     entry.id = p.context_id;

lib/quickbeam/quickbeam.zig

@@ -85,6 +85,18 @@ pub fn start_runtime(owner_pid: beam.pid, opts: beam.term) !RuntimeResource {
     if (get_map_uint(env, opts.v, "max_stack_size")) |v| {
         data.max_stack_size = v;
     }
+    if (get_map_uint(env, opts.v, "wasm_stack_size")) |v| {
+        data.wasm_stack_size = std.math.cast(u32, v) orelse {
+            gpa.destroy(data);
+            return error.WasmStackSizeTooLarge;
+        };
+    }
+    if (get_map_uint(env, opts.v, "wasm_heap_size")) |v| {
+        data.wasm_heap_size = std.math.cast(u32, v) orelse {
+            gpa.destroy(data);
+            return error.WasmHeapSizeTooLarge;
+        };
+    }
     if (get_map_uint(env, opts.v, "max_convert_depth")) |v| {
         data.max_convert_depth = @intCast(v);
     }
@@ -573,6 +585,18 @@ pub fn pool_start(opts: beam.term) !PoolResource {
     if (get_map_uint(env, opts.v, "max_stack_size")) |v| {
         data.max_stack_size = v;
     }
+    if (get_map_uint(env, opts.v, "wasm_stack_size")) |v| {
+        data.wasm_stack_size = std.math.cast(u32, v) orelse {
+            gpa.destroy(data);
+            return error.WasmStackSizeTooLarge;
+        };
+    }
+    if (get_map_uint(env, opts.v, "wasm_heap_size")) |v| {
+        data.wasm_heap_size = std.math.cast(u32, v) orelse {
+            gpa.destroy(data);
+            return error.WasmHeapSizeTooLarge;
+        };
+    }
     if (get_map_uint(env, opts.v, "max_convert_depth")) |v| {
         data.max_convert_depth = @intCast(v);
     }

lib/quickbeam/runtime.ex

@@ -299,7 +299,14 @@ defmodule QuickBEAM.Runtime do
 
     nif_opts =
       opts
-      |> Keyword.take([:memory_limit, :max_stack_size, :max_convert_depth, :max_convert_nodes])
+      |> Keyword.take([
+        :memory_limit,
+        :max_stack_size,
+        :wasm_stack_size,
+        :wasm_heap_size,
+        :max_convert_depth,
+        :max_convert_nodes
+      ])
       |> Map.new()
 
     resource = QuickBEAM.Native.start_runtime(self(), nif_opts)

lib/quickbeam/types.zig

@@ -35,6 +35,11 @@ pub const RuntimeData = struct {
     thread: ?std.Thread,
     memory_limit: usize = 256 * 1024 * 1024,
     max_stack_size: usize = 8 * 1024 * 1024,
+    // WASM operand stack / heap for the JS `WebAssembly.instantiate` path
+    // (distinct from `max_stack_size`, the JS call stack). Default mirrors the
+    // WASM NIF path; raised via the runtime `:wasm_stack_size` opt.
+    wasm_stack_size: u32 = 65_536,
+    wasm_heap_size: u32 = 65_536,
     max_convert_depth: u32 = 32,
     max_convert_nodes: u32 = 10_000,
     sync_slots_mutex: std.Thread.Mutex = .{},

lib/quickbeam/wasm.ex

@@ -33,6 +33,15 @@ defmodule QuickBEAM.WASM do
     * `:name` — GenServer name registration
     * `:stack_size` — execution stack in bytes (default: 65536)
     * `:heap_size` — auxiliary heap in bytes (default: 65536)
+
+  > #### JS `WebAssembly` path {: .info}
+  >
+  > These `:stack_size`/`:heap_size` options apply to this native NIF path. Guests
+  > started from JavaScript via `WebAssembly.instantiate` instead take their WASM
+  > operand stack / heap from the owning runtime's `:wasm_stack_size` /
+  > `:wasm_heap_size` options (see `QuickBEAM.Runtime` / `QuickBEAM.ContextPool`),
+  > which also default to 65536. Raise those for guests (e.g. Go `GOOS=js`) whose
+  > deep initialization overflows the 64 KB default.
   """
 
   @type instance :: GenServer.server()

lib/quickbeam/wasm_js.zig

@@ -17,6 +17,13 @@ const InstanceEntry = struct {
 const ContextState = struct {
     next_instance_id: u64 = 1,
     max_reductions: i64 = 0,
+    // WASM operand stack / auxiliary heap for instances started via the JS
+    // `WebAssembly.instantiate` path. Distinct from the JS call stack
+    // (`max_stack_size`). Default mirrors the WASM NIF path; a consumer raises
+    // it (via the runtime/pool `:wasm_stack_size` opt) for guests whose deep
+    // init would otherwise overflow the 64 KB default.
+    wasm_stack_size: u32 = 65_536,
+    wasm_heap_size: u32 = 65_536,
     instances: std.AutoHashMapUnmanaged(u64, InstanceEntry) = .{},
 
     fn deinit(self: *ContextState) void {
@@ -46,18 +53,20 @@ fn context_key(ctx: *qjs.JSContext) usize {
     return @intFromPtr(ctx);
 }
 
-fn ensure_context_state(ctx: *qjs.JSContext, max_reductions: i64) ?*ContextState {
+fn ensure_context_state(ctx: *qjs.JSContext, max_reductions: i64, wasm_stack_size: u32, wasm_heap_size: u32) ?*ContextState {
     states_mutex.lock();
     defer states_mutex.unlock();
 
     const key = context_key(ctx);
     if (states.get(key)) |state| {
         state.max_reductions = max_reductions;
+        state.wasm_stack_size = wasm_stack_size;
+        state.wasm_heap_size = wasm_heap_size;
         return state;
     }
 
     const state = gpa.create(ContextState) catch return null;
-    state.* = .{ .max_reductions = max_reductions };
+    state.* = .{ .max_reductions = max_reductions, .wasm_stack_size = wasm_stack_size, .wasm_heap_size = wasm_heap_size };
     states.put(gpa, key, state) catch {
         gpa.destroy(state);
         return null;
@@ -82,8 +91,8 @@ pub fn destroy_context(ctx: *qjs.JSContext) void {
     }
 }
 
-pub fn install(ctx: *qjs.JSContext, global: qjs.JSValue, max_reductions: i64) void {
-    _ = ensure_context_state(ctx, max_reductions) orelse return;
+pub fn install(ctx: *qjs.JSContext, global: qjs.JSValue, max_reductions: i64, wasm_stack_size: u32, wasm_heap_size: u32) void {
+    _ = ensure_context_state(ctx, max_reductions, wasm_stack_size, wasm_heap_size) orelse return;
 
     _ = qjs.JS_SetPropertyStr(ctx, global, "__qb_wasm_start", qjs.JS_NewCFunction(ctx, &wasm_start_impl, "__qb_wasm_start", 3));
     _ = qjs.JS_SetPropertyStr(ctx, global, "__qb_wasm_call", qjs.JS_NewCFunction(ctx, &wasm_call_impl, "__qb_wasm_call", 3));
@@ -327,7 +336,7 @@ fn wasm_start_impl(
         wasm_host_imports.PreparedImports.empty();
     errdefer prepared_imports.deinit();
 
-    const managed = wasm_common.start_managed_instance(mod orelse return throw_error(ctx, "null module"), 65_536, 65_536, if (prepared_imports.registrations.len > 0) &prepared_imports else null, &err_buf) orelse return throw_error(ctx, std.mem.sliceTo(&err_buf, 0));
+    const managed = wasm_common.start_managed_instance(mod orelse return throw_error(ctx, "null module"), state.wasm_stack_size, state.wasm_heap_size, if (prepared_imports.registrations.len > 0) &prepared_imports else null, &err_buf) orelse return throw_error(ctx, std.mem.sliceTo(&err_buf, 0));
     const mod_nn = mod orelse return throw_error(ctx, "null module");
     errdefer managed.destroy();
 

lib/quickbeam/worker.zig

@@ -899,7 +899,7 @@ pub const WorkerState = struct {
 
         const global = qjs.JS_GetGlobalObject(self.ctx);
         defer qjs.JS_FreeValue(self.ctx, global);
-        wasm_js.install(self.ctx, global, self.max_reductions);
+        wasm_js.install(self.ctx, global, self.max_reductions, self.rd.wasm_stack_size, self.rd.wasm_heap_size);
     }
 };
 

test/wasm_test.exs

@@ -696,6 +696,100 @@ defmodule QuickBEAM.WASMTest do
         """)
     end
 
+    # The tests below prove the JS `WebAssembly.instantiate` path honors the
+    # runtime/pool `:wasm_stack_size` by reusing the `add` guest above
+    # (`@wasm_js_bytes`): a tiny operand stack makes its very first call overflow,
+    # a generous one lets it run. The probe is the *contrast* between stack sizes,
+    # not a deep guest.
+    #
+    # Why a too-small stack rather than a deep-recursion guest: under the Debug
+    # build (MIX_ENV=test => Zig UBSan on the vendored WAMR C), executing a guest
+    # whose call frame lands a `WASMBranchBlock` on WAMR's 4-byte-aligned
+    # `csp_bottom` trips a UBSan alignment trap and aborts the BEAM instead of
+    # raising a catchable error — a pre-existing WAMR/UBSan interaction unrelated
+    # to this feature. A too-small stack instead overflows at frame *allocation*
+    # (`wasm_exec_env_alloc_wasm_frame` returns NULL => "wasm operand stack
+    # overflow") before any branch block is touched, so it fails safely, and the
+    # generous-stack path reuses a guest the existing suite already runs cleanly.
+    @tiny_wasm_stack 32
+
+    test "JS instantiate path overflows a too-small :wasm_stack_size" do
+      {:ok, rt} = QuickBEAM.start(wasm_stack_size: @tiny_wasm_stack)
+
+      {:error, err} =
+        QuickBEAM.eval(rt, """
+          const bytes = #{@wasm_js_bytes};
+          const {instance} = await WebAssembly.instantiate(bytes);
+          instance.exports.add(40, 2);
+        """)
+
+      assert err.message =~ "stack"
+
+      QuickBEAM.stop(rt)
+    end
+
+    test "JS instantiate path honors a raised :wasm_stack_size" do
+      {:ok, rt} = QuickBEAM.start(wasm_stack_size: 8 * 1024 * 1024)
+
+      assert {:ok, 42} =
+               QuickBEAM.eval(rt, """
+                 const bytes = #{@wasm_js_bytes};
+                 const {instance} = await WebAssembly.instantiate(bytes);
+                 instance.exports.add(40, 2);
+               """)
+
+      QuickBEAM.stop(rt)
+    end
+
+    test "ContextPool propagates :wasm_stack_size to the pooled JS instantiate path" do
+      # Exercises the pool threading path (PoolData -> RuntimeData copy in
+      # context_worker), which the standalone QuickBEAM.start/1 test above does
+      # not cover. The tiny-stack pool below proves the pooled context applies the
+      # value rather than silently keeping the 64 KB default.
+      {:ok, big_pool} = QuickBEAM.ContextPool.start_link(size: 1, wasm_stack_size: 8 * 1024 * 1024)
+      {:ok, big_ctx} = QuickBEAM.Context.start_link(pool: big_pool)
+
+      assert {:ok, 42} =
+               QuickBEAM.Context.eval(big_ctx, """
+                 const bytes = #{@wasm_js_bytes};
+                 const {instance} = await WebAssembly.instantiate(bytes);
+                 instance.exports.add(40, 2);
+               """)
+
+      QuickBEAM.Context.stop(big_ctx)
+
+      {:ok, tiny_pool} = QuickBEAM.ContextPool.start_link(size: 1, wasm_stack_size: @tiny_wasm_stack)
+      {:ok, tiny_ctx} = QuickBEAM.Context.start_link(pool: tiny_pool)
+
+      {:error, err} =
+        QuickBEAM.Context.eval(tiny_ctx, """
+          const bytes = #{@wasm_js_bytes};
+          const {instance} = await WebAssembly.instantiate(bytes);
+          instance.exports.add(40, 2);
+        """)
+
+      assert err.message =~ "stack"
+
+      QuickBEAM.Context.stop(tiny_ctx)
+    end
+
+    # No behavioral test for the sibling `:wasm_heap_size` option (it is plumbed
+    # through the same start/pool paths exercised above): it has no effect that the
+    # JS `WebAssembly.instantiate` path can observe, so any test would pass whether
+    # or not the value is honored (a false green). The value sizes WAMR's host
+    # *app heap*, and on this path nothing reaches it:
+    #   * The app heap only backs `wasm_runtime_module_malloc` / a guest-exported
+    #     malloc. A plain instantiated module called from JS never allocates from it.
+    #   * It cannot fail instantiation: WAMR clamps it to APP_HEAP_SIZE_MAX (1 GiB,
+    #     wasm_runtime.c ~2451) and the insertion guards only trip near UINT32_MAX
+    #     or >DEFAULT_MAX_PAGES, unreachable after the clamp.
+    #   * It is not visible as memory size: `memory.buffer.byteLength` reports the
+    #     app-visible page count (`cur_page_count * 65536`), which excludes the
+    #     appended app heap — see "WebAssembly exposes exported memory" below, where
+    #     the default 64 KiB heap still yields byteLength 65536, not 131072.
+    # `:wasm_stack_size` is observable (tiny stack => first call overflows), hence
+    # tested above; `:wasm_heap_size` is covered only at the plumbing level.
+
     test "WebAssembly.compile + instantiate", %{rt: rt} do
       {:ok, 300} =
         QuickBEAM.eval(rt, """