callouts: build + bake m-stdlib YDB call-out .so's into the engine image

.gitignore

@@ -0,0 +1,4 @@
+# Callout sources staged from m-stdlib at build time (single-sourced there,
+# never vendored here). Populated by `make callouts-stage`, consumed by the
+# Dockerfile's callout-builder stage.
+docker/_callouts/

Makefile

@@ -17,15 +17,47 @@
 # But run-from-here works too: cd into m-test-engine and `make smoke`
 # for a quick check.
 
-.PHONY: up down logs shell smoke clean manifest check-manifest check-docs-prose skill-install
+.PHONY: up down logs shell smoke clean manifest check-manifest check-docs-prose skill-install callouts-stage test-optional
 
 COMPOSE := docker compose -f docker/compose.yml
 
-up:
+# m-stdlib checkout that owns the callout C sources + .xc descriptors. They are
+# single-sourced there and staged into the build context at build time (never
+# vendored into this repo — see docker/_callouts/ in .gitignore). Override if
+# your m-stdlib lives elsewhere.
+M_STDLIB    ?= $(HOME)/vista-cloud-dev/m-stdlib
+CALLOUT_DIR := docker/_callouts
+
+# callouts-stage — copy m-stdlib's callout build inputs into the Docker build
+# context so the callout-builder image stage can compile them. Idempotent;
+# fails loudly if m-stdlib (or its callout sources) can't be found.
+callouts-stage:
+	@if [ ! -d "$(M_STDLIB)/src/callouts" ]; then \
+	  echo "ERROR: m-stdlib callout sources not found at $(M_STDLIB)/src/callouts" >&2; \
+	  echo "       set M_STDLIB=/path/to/m-stdlib" >&2; exit 1; \
+	fi
+	@rm -rf $(CALLOUT_DIR)
+	@mkdir -p $(CALLOUT_DIR)/src/callouts $(CALLOUT_DIR)/tools $(CALLOUT_DIR)/xc
+	@cp $(M_STDLIB)/src/callouts/*.c            $(CALLOUT_DIR)/src/callouts/
+	@cp $(M_STDLIB)/tools/build-callouts.sh     $(CALLOUT_DIR)/tools/
+	@cp $(M_STDLIB)/tools/std_compress.xc $(M_STDLIB)/tools/std_crypto.xc $(M_STDLIB)/tools/std_http.xc $(CALLOUT_DIR)/xc/
+	@echo "callouts-stage: staged $$(ls $(CALLOUT_DIR)/src/callouts/*.c | wc -l) C source(s) + 3 .xc from $(M_STDLIB)"
+
+up: callouts-stage
 	$(COMPOSE) up -d --build
 	@echo
 	@echo "m-test-engine is up. Verify with: make smoke"
 
+# test-optional — run m-stdlib's callout-backed suites against this engine in
+# byte (M) mode (the contract for the byte-oriented modules). Proves the baked
+# callouts resolve with no hang. Drives the Go `m` binary directly because the
+# byte suites need --chset m, which m-stdlib's own `make test-optional` does
+# not yet pass.
+M       ?= $(HOME)/vista-cloud-dev/m-cli/dist/m
+test-optional:
+	cd $(M_STDLIB) && $(M) test tests/STDCRYPTOTST.m tests/STDCRYPTODOCTST.m tests/STDCOMPRESSTST.m tests/STDHTTPTST.m \
+	    --engine ydb --docker m-test-engine --routines=src --chset m
+
 down:
 	$(COMPOSE) down
 

dist/lifecycle.json

@@ -16,14 +16,26 @@
     "logs":  { "make": "make logs",  "compose": "docker compose -f docker/compose.yml logs -f" },
     "shell": { "make": "make shell", "compose": "docker exec -it m-test-engine bash" },
     "smoke": { "make": "make smoke", "compose": "docker exec m-test-engine bash -lc '$ydb_dist/mumps -run %XCMD '\\''write \"smoke ok\",!'\\'''" },
-    "clean": { "make": "make clean", "compose": "docker compose -f docker/compose.yml down -v" }
+    "clean": { "make": "make clean", "compose": "docker compose -f docker/compose.yml down -v" },
+    "test-optional": { "make": "make test-optional", "compose": "n/a — drives the Go `m` binary against this container in byte mode" }
   },
   "exec_convention": {
     "shape": "docker exec m-test-engine bash -lc '<command>'",
-    "ydb_env_loaded_via": "/etc/profile.d/ydb-env.sh (sources /opt/yottadb/current/ydb_env_set)",
-    "available_env_vars": ["ydb_dist", "ydb_routines"],
+    "ydb_env_loaded_via": "/etc/profile.d/ydb-env.sh (sources /opt/yottadb/current/ydb_env_set); /etc/profile.d/stdlib-callouts.sh (m-stdlib callout vars)",
+    "available_env_vars": ["ydb_dist", "ydb_routines", "STDLIB_LIB", "ydb_xc_stdcompress", "ydb_xc_stdcrypto", "ydb_xc_stdhttp"],
     "notes": "Use bash -lc so the YDB env loads. Single-quote M commands so the outer bash does not expand $ZVERSION etc. (those are YDB special variables, not shell vars)."
   },
+  "stdlib_callouts": {
+    "provides": "m-stdlib optional-module YDB call-out libraries, baked at image build from m-stdlib's src/callouts/*.c via tools/build-callouts.sh (single-sourced in m-stdlib; staged into the build context by `make callouts-stage`, never vendored here).",
+    "lib_dir": "/opt/stdlib/lib",
+    "xc_dir": "/opt/stdlib/xc",
+    "libraries": {
+      "stdcompress.so": { "package": "stdcompress", "xc": "std_compress.xc", "links": ["libz.so.1 (1.3)", "libzstd.so.1 (1.5.5)"] },
+      "std_crypto.so":  { "package": "stdcrypto",   "xc": "std_crypto.xc",   "links": ["libcrypto.so.3 (OpenSSL 3)"] },
+      "http.so":        { "package": "stdhttp",     "xc": "std_http.xc",     "links": ["libcurl.so.4 (4.8.0)"] }
+    },
+    "verify": "make test-optional — runs m-stdlib's 4 callout-backed suites (STDCRYPTOTST/STDCRYPTODOCTST/STDCOMPRESSTST/STDHTTPTST) in byte mode; last run 4 suites / 151 assertions / 0 failed, no hang."
+  },
   "consumers": [
     { "repo": "m-cli",    "transport": "DockerEngine (src/m_cli/engine.py)" },
     { "repo": "m-stdlib", "transport": "test runner; reuses m-cli's DockerEngine when m-cli is installed" }

dist/m-test-engine.json

@@ -44,5 +44,5 @@
     "watch":        { "destructive": false, "read_only": true  },
     "capabilities": { "destructive": false, "read_only": true  }
   },
-  "verified_on": "2026-05-11"
+  "verified_on": "2026-05-30"
 }

dist/repo.meta.json

@@ -6,7 +6,7 @@
   "language": ["dockerfile"],
   "license": "AGPL-3.0",
   "agent_instructions": "AGENTS.md",
-  "verified_on": "2026-05-11",
+  "verified_on": "2026-05-30",
   "exposes": {
     "lifecycle":       "dist/lifecycle.json",
     "engine_contract": "dist/m-test-engine.json",

docker/Dockerfile

@@ -14,6 +14,24 @@
 #
 # Pinned to yottadb-base:latest-master to match what m-stdlib's CI uses.
 
+# ── callout-builder stage ─────────────────────────────────────────────
+# Compiles m-stdlib's optional-module YDB call-out .so's (libz/libzstd ->
+# stdcompress.so, libcrypto -> std_crypto.so, libcurl -> http.so) from the C
+# sources staged into the build context by `make callouts-stage`. The build
+# toolchain (gcc + -dev headers) lives ONLY in this stage so the final image
+# stays minimal — it carries the compiled .so's and the already-present runtime
+# libs, not the compilers. The .c sources are single-sourced in m-stdlib and
+# never committed here (docker/_callouts/ is gitignored).
+FROM yottadb/yottadb-base:latest-master AS callout-builder
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        gcc libc6-dev zlib1g-dev libzstd-dev libssl-dev libcurl4-openssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+COPY _callouts/ /build/
+# build-callouts.sh reads $ydb_dist for libyottadb.h and each .c's `// link:`
+# directive for its -l flags; outputs so/<platform>/*.so under /build.
+RUN cd /build && ydb_dist=/opt/yottadb/current bash tools/build-callouts.sh
+
+# ── final image ───────────────────────────────────────────────────────
 FROM yottadb/yottadb-base:latest-master
 
 # Build-time identity. Defaults are sentinels so a local `docker build`
@@ -52,6 +70,30 @@ RUN echo '. /opt/yottadb/current/ydb_env_set 2>/dev/null || true' \
     > /etc/profile.d/ydb-env.sh \
     && chmod +x /etc/profile.d/ydb-env.sh
 
+# m-stdlib optional-module call-outs. The compiled .so's (from the
+# callout-builder stage) land in $STDLIB_LIB; the .xc descriptors (staged from
+# m-stdlib) land beside them. The descriptors' first line is "$STDLIB_LIB/<lib>.so",
+# resolved at load time from the process env. Wiring ydb_xc_<pkg> lets YottaDB
+# resolve $&stdcompress.* / $&stdcrypto.* / $&stdhttp.* — so m-stdlib's optional
+# suites run on this engine. Exported via /etc/profile.d so `bash -lc` (m-cli's
+# DockerEngine transport, the smoke target, healthcheck) picks them up.
+#
+# HANG-GUARD invariant: a .xc descriptor whose .so is missing makes YottaDB hang
+# on first $& resolution. The .so's are COPYed first and below the build asserts
+# all three exist, so a descriptor is never wired without its library present.
+COPY --from=callout-builder /build/so/linux-x86_64/*.so /opt/stdlib/lib/
+COPY _callouts/xc/*.xc /opt/stdlib/xc/
+RUN set -e; \
+    for so in stdcompress std_crypto http; do \
+      test -f /opt/stdlib/lib/$so.so || { echo "MISSING callout .so: $so.so" >&2; exit 1; }; \
+    done; \
+    { echo 'export STDLIB_LIB=/opt/stdlib/lib'; \
+      echo 'export ydb_xc_stdcompress=/opt/stdlib/xc/std_compress.xc'; \
+      echo 'export ydb_xc_stdcrypto=/opt/stdlib/xc/std_crypto.xc'; \
+      echo 'export ydb_xc_stdhttp=/opt/stdlib/xc/std_http.xc'; \
+    } > /etc/profile.d/stdlib-callouts.sh; \
+    chmod +x /etc/profile.d/stdlib-callouts.sh
+
 # Container-side introspection script (Phase 4 of the m-engine plan).
 # `mte status --json` returns a structured snapshot consumed by
 # m-cli's `m engine status --verbose`. The script uses a login-shell