From ed36ac46fc7e3ca15bd9ce4205f9dd7a0985f1d0 Mon Sep 17 00:00:00 2001 From: Rafael Richards Date: Sat, 30 May 2026 21:19:09 -0400 Subject: [PATCH] callouts: build + bake m-stdlib YDB call-out .so's into the engine image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit m-stdlib's three optional modules (STDCOMPRESS / STDCRYPTO / STDHTTP) reach native code on YottaDB via $&pkg.fn -> a .so registered through a .xc descriptor. The running container had no such libraries, so the optional suites couldn't run — and with a .xc descriptor present but its .so missing, STDCOMPRESSTST hung the engine. This bakes the libraries in and wires them. How: - New multi-stage Dockerfile `callout-builder` stage: installs gcc + the -dev headers (zlib1g-dev, libzstd-dev, libssl-dev, libcurl4-openssl-dev), compiles m-stdlib's src/callouts/*.c via tools/build-callouts.sh. The toolchain lives ONLY in this stage; the final image carries just the .so's and the already-present runtime libs (libz 1.3 / libzstd 1.5.5 / libcrypto 3 / libcurl 4.8.0) — staying minimal. - Final stage: COPY the compiled .so's -> /opt/stdlib/lib, the .xc descriptors -> /opt/stdlib/xc, and export STDLIB_LIB + ydb_xc_stdcompress / _stdcrypto / _stdhttp via /etc/profile.d/stdlib-callouts.sh (sourced by `bash -lc`, the m-cli DockerEngine transport). A build-time assert fails the image if any of the three .so's is absent — enforcing the hang-guard invariant (never a descriptor without its library). - Makefile: `callouts-stage` copies the C sources + .xc from m-stdlib into the build context (gitignored docker/_callouts/ — single-sourced in m-stdlib, never vendored here); `up` depends on it. New `test-optional` runs the four callout-backed suites in byte mode. - dist/: lifecycle.json gains the callout env vars + a stdlib_callouts block + the test-optional target; verified_on bumped to 2026-05-30 (re-verified with the new image). Verification (via `m test`, no hand docker exec): - make smoke: healthy (mumps ok, OCI labels, healthcheck, mte status). - make test-optional (YDB byte mode, --chset m): 4 suites, 151 assertions, 0 failed, no hang — STDCRYPTOTST 23, STDCRYPTODOCTST 1, STDCOMPRESSTST 59, STDHTTPTST 68. The deployed .so's resolve the Stage-A STDCOMPRESS hang (which was the missing-.so case). Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 4 ++++ Makefile | 36 +++++++++++++++++++++++++++++++++-- dist/lifecycle.json | 18 +++++++++++++++--- dist/m-test-engine.json | 2 +- dist/repo.meta.json | 2 +- docker/Dockerfile | 42 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3cc24d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# Callout sources staged from m-stdlib at build time (single-sourced there, +# never vendored here). Populated by `make callouts-stage`, consumed by the +# Dockerfile's callout-builder stage. +docker/_callouts/ diff --git a/Makefile b/Makefile index 09dfb73..4aba009 100644 --- a/Makefile +++ b/Makefile @@ -17,15 +17,47 @@ # But run-from-here works too: cd into m-test-engine and `make smoke` # for a quick check. -.PHONY: up down logs shell smoke clean manifest check-manifest check-docs-prose skill-install +.PHONY: up down logs shell smoke clean manifest check-manifest check-docs-prose skill-install callouts-stage test-optional COMPOSE := docker compose -f docker/compose.yml -up: +# m-stdlib checkout that owns the callout C sources + .xc descriptors. They are +# single-sourced there and staged into the build context at build time (never +# vendored into this repo — see docker/_callouts/ in .gitignore). Override if +# your m-stdlib lives elsewhere. +M_STDLIB ?= $(HOME)/vista-cloud-dev/m-stdlib +CALLOUT_DIR := docker/_callouts + +# callouts-stage — copy m-stdlib's callout build inputs into the Docker build +# context so the callout-builder image stage can compile them. Idempotent; +# fails loudly if m-stdlib (or its callout sources) can't be found. +callouts-stage: + @if [ ! -d "$(M_STDLIB)/src/callouts" ]; then \ + echo "ERROR: m-stdlib callout sources not found at $(M_STDLIB)/src/callouts" >&2; \ + echo " set M_STDLIB=/path/to/m-stdlib" >&2; exit 1; \ + fi + @rm -rf $(CALLOUT_DIR) + @mkdir -p $(CALLOUT_DIR)/src/callouts $(CALLOUT_DIR)/tools $(CALLOUT_DIR)/xc + @cp $(M_STDLIB)/src/callouts/*.c $(CALLOUT_DIR)/src/callouts/ + @cp $(M_STDLIB)/tools/build-callouts.sh $(CALLOUT_DIR)/tools/ + @cp $(M_STDLIB)/tools/std_compress.xc $(M_STDLIB)/tools/std_crypto.xc $(M_STDLIB)/tools/std_http.xc $(CALLOUT_DIR)/xc/ + @echo "callouts-stage: staged $$(ls $(CALLOUT_DIR)/src/callouts/*.c | wc -l) C source(s) + 3 .xc from $(M_STDLIB)" + +up: callouts-stage $(COMPOSE) up -d --build @echo @echo "m-test-engine is up. Verify with: make smoke" +# test-optional — run m-stdlib's callout-backed suites against this engine in +# byte (M) mode (the contract for the byte-oriented modules). Proves the baked +# callouts resolve with no hang. Drives the Go `m` binary directly because the +# byte suites need --chset m, which m-stdlib's own `make test-optional` does +# not yet pass. +M ?= $(HOME)/vista-cloud-dev/m-cli/dist/m +test-optional: + cd $(M_STDLIB) && $(M) test tests/STDCRYPTOTST.m tests/STDCRYPTODOCTST.m tests/STDCOMPRESSTST.m tests/STDHTTPTST.m \ + --engine ydb --docker m-test-engine --routines=src --chset m + down: $(COMPOSE) down diff --git a/dist/lifecycle.json b/dist/lifecycle.json index c5e0a55..2afccd5 100644 --- a/dist/lifecycle.json +++ b/dist/lifecycle.json @@ -16,14 +16,26 @@ "logs": { "make": "make logs", "compose": "docker compose -f docker/compose.yml logs -f" }, "shell": { "make": "make shell", "compose": "docker exec -it m-test-engine bash" }, "smoke": { "make": "make smoke", "compose": "docker exec m-test-engine bash -lc '$ydb_dist/mumps -run %XCMD '\\''write \"smoke ok\",!'\\'''" }, - "clean": { "make": "make clean", "compose": "docker compose -f docker/compose.yml down -v" } + "clean": { "make": "make clean", "compose": "docker compose -f docker/compose.yml down -v" }, + "test-optional": { "make": "make test-optional", "compose": "n/a — drives the Go `m` binary against this container in byte mode" } }, "exec_convention": { "shape": "docker exec m-test-engine bash -lc ''", - "ydb_env_loaded_via": "/etc/profile.d/ydb-env.sh (sources /opt/yottadb/current/ydb_env_set)", - "available_env_vars": ["ydb_dist", "ydb_routines"], + "ydb_env_loaded_via": "/etc/profile.d/ydb-env.sh (sources /opt/yottadb/current/ydb_env_set); /etc/profile.d/stdlib-callouts.sh (m-stdlib callout vars)", + "available_env_vars": ["ydb_dist", "ydb_routines", "STDLIB_LIB", "ydb_xc_stdcompress", "ydb_xc_stdcrypto", "ydb_xc_stdhttp"], "notes": "Use bash -lc so the YDB env loads. Single-quote M commands so the outer bash does not expand $ZVERSION etc. (those are YDB special variables, not shell vars)." }, + "stdlib_callouts": { + "provides": "m-stdlib optional-module YDB call-out libraries, baked at image build from m-stdlib's src/callouts/*.c via tools/build-callouts.sh (single-sourced in m-stdlib; staged into the build context by `make callouts-stage`, never vendored here).", + "lib_dir": "/opt/stdlib/lib", + "xc_dir": "/opt/stdlib/xc", + "libraries": { + "stdcompress.so": { "package": "stdcompress", "xc": "std_compress.xc", "links": ["libz.so.1 (1.3)", "libzstd.so.1 (1.5.5)"] }, + "std_crypto.so": { "package": "stdcrypto", "xc": "std_crypto.xc", "links": ["libcrypto.so.3 (OpenSSL 3)"] }, + "http.so": { "package": "stdhttp", "xc": "std_http.xc", "links": ["libcurl.so.4 (4.8.0)"] } + }, + "verify": "make test-optional — runs m-stdlib's 4 callout-backed suites (STDCRYPTOTST/STDCRYPTODOCTST/STDCOMPRESSTST/STDHTTPTST) in byte mode; last run 4 suites / 151 assertions / 0 failed, no hang." + }, "consumers": [ { "repo": "m-cli", "transport": "DockerEngine (src/m_cli/engine.py)" }, { "repo": "m-stdlib", "transport": "test runner; reuses m-cli's DockerEngine when m-cli is installed" } diff --git a/dist/m-test-engine.json b/dist/m-test-engine.json index af426c0..2a9ffd8 100644 --- a/dist/m-test-engine.json +++ b/dist/m-test-engine.json @@ -44,5 +44,5 @@ "watch": { "destructive": false, "read_only": true }, "capabilities": { "destructive": false, "read_only": true } }, - "verified_on": "2026-05-11" + "verified_on": "2026-05-30" } \ No newline at end of file diff --git a/dist/repo.meta.json b/dist/repo.meta.json index 6c9488d..02c685e 100644 --- a/dist/repo.meta.json +++ b/dist/repo.meta.json @@ -6,7 +6,7 @@ "language": ["dockerfile"], "license": "AGPL-3.0", "agent_instructions": "AGENTS.md", - "verified_on": "2026-05-11", + "verified_on": "2026-05-30", "exposes": { "lifecycle": "dist/lifecycle.json", "engine_contract": "dist/m-test-engine.json", diff --git a/docker/Dockerfile b/docker/Dockerfile index e75f784..64399b3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,6 +14,24 @@ # # Pinned to yottadb-base:latest-master to match what m-stdlib's CI uses. +# ── callout-builder stage ───────────────────────────────────────────── +# Compiles m-stdlib's optional-module YDB call-out .so's (libz/libzstd -> +# stdcompress.so, libcrypto -> std_crypto.so, libcurl -> http.so) from the C +# sources staged into the build context by `make callouts-stage`. The build +# toolchain (gcc + -dev headers) lives ONLY in this stage so the final image +# stays minimal — it carries the compiled .so's and the already-present runtime +# libs, not the compilers. The .c sources are single-sourced in m-stdlib and +# never committed here (docker/_callouts/ is gitignored). +FROM yottadb/yottadb-base:latest-master AS callout-builder +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc libc6-dev zlib1g-dev libzstd-dev libssl-dev libcurl4-openssl-dev \ + && rm -rf /var/lib/apt/lists/* +COPY _callouts/ /build/ +# build-callouts.sh reads $ydb_dist for libyottadb.h and each .c's `// link:` +# directive for its -l flags; outputs so//*.so under /build. +RUN cd /build && ydb_dist=/opt/yottadb/current bash tools/build-callouts.sh + +# ── final image ─────────────────────────────────────────────────────── FROM yottadb/yottadb-base:latest-master # Build-time identity. Defaults are sentinels so a local `docker build` @@ -52,6 +70,30 @@ RUN echo '. /opt/yottadb/current/ydb_env_set 2>/dev/null || true' \ > /etc/profile.d/ydb-env.sh \ && chmod +x /etc/profile.d/ydb-env.sh +# m-stdlib optional-module call-outs. The compiled .so's (from the +# callout-builder stage) land in $STDLIB_LIB; the .xc descriptors (staged from +# m-stdlib) land beside them. The descriptors' first line is "$STDLIB_LIB/.so", +# resolved at load time from the process env. Wiring ydb_xc_ lets YottaDB +# resolve $&stdcompress.* / $&stdcrypto.* / $&stdhttp.* — so m-stdlib's optional +# suites run on this engine. Exported via /etc/profile.d so `bash -lc` (m-cli's +# DockerEngine transport, the smoke target, healthcheck) picks them up. +# +# HANG-GUARD invariant: a .xc descriptor whose .so is missing makes YottaDB hang +# on first $& resolution. The .so's are COPYed first and below the build asserts +# all three exist, so a descriptor is never wired without its library present. +COPY --from=callout-builder /build/so/linux-x86_64/*.so /opt/stdlib/lib/ +COPY _callouts/xc/*.xc /opt/stdlib/xc/ +RUN set -e; \ + for so in stdcompress std_crypto http; do \ + test -f /opt/stdlib/lib/$so.so || { echo "MISSING callout .so: $so.so" >&2; exit 1; }; \ + done; \ + { echo 'export STDLIB_LIB=/opt/stdlib/lib'; \ + echo 'export ydb_xc_stdcompress=/opt/stdlib/xc/std_compress.xc'; \ + echo 'export ydb_xc_stdcrypto=/opt/stdlib/xc/std_crypto.xc'; \ + echo 'export ydb_xc_stdhttp=/opt/stdlib/xc/std_http.xc'; \ + } > /etc/profile.d/stdlib-callouts.sh; \ + chmod +x /etc/profile.d/stdlib-callouts.sh + # Container-side introspection script (Phase 4 of the m-engine plan). # `mte status --json` returns a structured snapshot consumed by # m-cli's `m engine status --verbose`. The script uses a login-shell