From 865be6915c1c477526d444b2d3c6bb9bc71150ec Mon Sep 17 00:00:00 2001 From: swapnil Date: Thu, 17 Sep 2026 21:38:33 -0700 Subject: [PATCH] build: make the published package installable and runnable `npm install` of the packed tarball produced an install that could not run. Four causes, each fatal on its own, none visible from a source checkout. - files: absent, so npm fell back to the ignore rules and honoured a workspace's own .gitignore. parser/.gitignore ignores dist/, so the built parser was dropped while parser/src/test-data shipped. Installed clean, failed on first use for every language. - bin: absent, so nothing put axiomcode on PATH. - bin/axiomcode: the root walk started at dirname "$0", which under a node_modules/.bin symlink is .bin/. It climbed the user's project, found a package.json with no graph/ beside it and returned "/", reporting `parser not built at //parser/dist/index.js` and advising a build that an installed copy cannot run. Resolve the link first, and fail clearly when the marker is never found. - dependencies: absent. The parser ships here as parser/dist but is a workspace and is never published, so nothing installed what it requires and first use died on `Cannot find module 'typescript'`. Adding files also stops the corpora shipping: 81 MB to 3.1 MB packed, 428 MB to 13.6 MB unpacked. All engine rules still ship. graph/test/tools/package-contents-test.sh reads the pack manifest and asserts the fields and the contents; it fails on each of the four defects above. Wired into the java preflight. Verified on a clean VM with no souffle, gcc, make or cmake, asserted absent before and after the node install: installing only the tarballs and running four public OSS projects, one per language, all four complete and write graph.sqlite (java 11s, typescript 7s, python 25s, javascript 7s). The native parser dependencies installed from prebuilt binaries with no node-gyp runs. Fixes #886 --- bin/axiomcode | 25 ++++++++- graph/test/java/run-tests.sh | 8 +++ graph/test/tools/package-contents-test.sh | 66 +++++++++++++++++++++++ package.json | 23 ++++++++ 4 files changed, 121 insertions(+), 1 deletion(-) create mode 100755 graph/test/tools/package-contents-test.sh diff --git a/bin/axiomcode b/bin/axiomcode index 6de412ec..529c92e7 100755 --- a/bin/axiomcode +++ b/bin/axiomcode @@ -30,7 +30,30 @@ # or is compiled locally when souffle is present). # ───────────────────────────────────────────────────────────────────────────── set -eu -ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" +# RESOLVE $0 THROUGH SYMLINKS BEFORE THE WALK. npm installs a `bin` entry as a link in +# node_modules/.bin, so under `npx axiomcode` or a global install $0 is that link and +# dirname "$0" is .bin/, a directory this package does not live under. The walk below then +# climbs the USER'S project, finds a package.json with no graph/ beside it, runs out of +# parents and returns "/" -- and the run fails with `parser not built at +# //parser/dist/index.js`, a path with a doubled slash that names nothing, plus advice to run +# `npm install && npm run build`, which on an installed copy is neither possible nor correct. +# readlink -f where it exists (GNU, macOS 12.3+), else walk the links by hand: the same +# fallback souffle-include.sh uses, and for the same reason, no interpreter and no coreutils. +self="$0" +if [ -L "$self" ]; then + r="$(readlink -f "$self" 2>/dev/null)" || r="" + if [ -z "$r" ]; then + r="$self"; while [ -L "$r" ]; do + t="$(readlink "$r")" + case "$t" in /*) r="$t";; *) r="$(dirname "$r")/$t";; esac + done + fi + self="$r" +fi +ROOT="$(d="$(cd "$(dirname "$self")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" +# No marker found means every path built from ROOT is nonsense; say so here rather than fail +# later on a path the user cannot interpret. +[ -d "$ROOT/graph" ] || { echo "axiomcode: cannot locate the package root from $0 (resolved to $self)" >&2; exit 2; } PARSER="${AXIOM_PARSER:-$ROOT/parser/dist/index.js}" usage(){ sed -n '3,30p' "$0" | sed 's/^# \{0,1\}//'; } die(){ echo "axiomcode: $*" >&2; exit 2; } diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index a5e3745e..a5a14149 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -107,6 +107,14 @@ if ! bash "$ROOT/graph/test/tools/engine-id-locale-test.sh"; then echo "aborting: the program text depends on the shell locale" exit 1 fi +# ── The published package must be installable ────────────────────────────── +# Reads the pack manifest, so it costs a second. It guards defects that a source checkout +# cannot show: here the parser is built, the CLI is run by path and the dependencies are +# present, none of which is true of the tarball a user installs. +if ! bash "$ROOT/graph/test/tools/package-contents-test.sh"; then + echo "aborting: the published package would not be installable" + exit 1 +fi # ── The bundle stage must build the language-neutral output ───────────────── # Every solve below ends by joining the raw relations to the IR and writing graph.sqlite # (graph/bundle/SCHEMA.md); graph/*.csv is the same core tables and is written only under diff --git a/graph/test/tools/package-contents-test.sh b/graph/test/tools/package-contents-test.sh new file mode 100755 index 00000000..82b603fe --- /dev/null +++ b/graph/test/tools/package-contents-test.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# What `npm pack` would actually put in the tarball, and whether an install of it could run. +# +# Every defect this guards against was invisible from a source checkout, which is why none of +# them was caught before: here the parser is built, the CLI is invoked by path, and the +# dependencies are present because a workspace install put them there. The published tarball +# has none of those advantages, and it failed on first use, for every language, on a machine +# where the advice it printed could not be followed. +# +# It reads the pack MANIFEST rather than installing anything, so it costs a second and needs +# no network. --ignore-scripts because `prepare` would otherwise run a full build and print +# over the JSON. The full acceptance test remains an install on a machine with no toolchain. +# ───────────────────────────────────────────────────────────────────────────── +set -uo pipefail +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" +cd "$ROOT" +command -v node >/dev/null 2>&1 || { echo "package-contents: SKIP (no node)"; exit 0; } + +node - "$ROOT" <<'JS' +const {execFileSync}=require("child_process"), root=process.argv[2], fs=require("fs"); +let fail=0, checks=0; +const ok =(m)=>{checks++; if(process.env.PACKAGE_CONTENTS_VERBOSE) console.log(" ok "+m);}; +const bad=(m)=>{checks++; fail=1; console.log(" FAIL "+m);}; + +// ---- the manifest fields an installable package needs ---- +const j=JSON.parse(fs.readFileSync(root+"/package.json","utf8")); +// Without bin there is no command after install; the CLI exists only as a path in node_modules. +(j.bin&&j.bin.axiomcode) ? ok("bin.axiomcode is declared") : bad("package.json has no bin.axiomcode entry"); +// Without files npm falls back to the ignore rules and honours a WORKSPACE .gitignore, which +// is how parser/dist stopped shipping while parser/src/test-data did. +(Array.isArray(j.files)&&j.files.length) ? ok("files is declared") : bad("package.json has no files array"); +// The parser ships inside this tarball but is a workspace and is never published, so nothing +// installs what it requires unless this package declares it. +const deps=j.dependencies||{}, pdeps=JSON.parse(fs.readFileSync(root+"/parser/package.json","utf8")).dependencies||{}; +Object.keys(deps).length ? ok("runtime dependencies are declared") : bad("package.json declares no runtime dependencies"); +for(const k of Object.keys(pdeps)) deps[k] ? ok("dependency hoisted: "+k) : bad("runtime dependency not hoisted from the parser workspace: "+k); + +// ---- what the tarball would contain ---- +let files; +try{ + files=JSON.parse(execFileSync("npm",["pack","--dry-run","--json","--ignore-scripts"], + {cwd:root, encoding:"utf8", stdio:["ignore","pipe","ignore"], maxBuffer:64*1024*1024}))[0].files.map(f=>f.path); +}catch(e){ console.log(" FAIL npm pack --dry-run failed: "+e.message.split("\n")[0]); process.exit(1); } +const has=(p)=>files.includes(p); +const count=(re)=>files.filter(f=>re.test(f)).length; + +has("bin/axiomcode") ? ok("ships bin/axiomcode") : bad("does NOT ship bin/axiomcode"); +// The corpora are the bulk of the repository and run nothing for a consumer. +count(/^graph\/test\//)===0 ? ok("does not ship graph/test") : bad("ships "+count(/^graph\/test\//)+" files under graph/test"); +count(/^parser\/src\/test-data\//)===0 ? ok("does not ship parser test-data") : bad("ships "+count(/^parser\/src\/test-data\//)+" files under parser/src/test-data"); +// The rules ARE the engine; the CLI without them installs a tool that cannot solve. +for(const lang of ["java","typescript","python","javascript"]){ + const n=count(new RegExp("^graph/"+lang+"/.*\\.dl$")); + n>0 ? ok("ships "+n+" "+lang+" rule files") : bad("ships no "+lang+" .dl rules"); +} +// Build outputs can only be asserted when they exist. A source checkout that has not been +// built is not a failing package, so this SKIPS rather than failing -- but it must never skip +// silently, because that is exactly how the missing parser went unnoticed. +for(const [dir,probe] of [["dist/", "dist/reason.js"],["parser/dist/","parser/dist/index.js"]]){ + if(!fs.existsSync(root+"/"+probe)){ console.log(" skip "+dir+" not built in this checkout (run npm run build to assert it ships)"); continue; } + count(new RegExp("^"+dir.replace("/","\\/")))>0 ? ok("ships "+dir) : bad("does NOT ship "+dir+" (it is built but excluded from the pack)"); +} +console.log("package-contents: "+checks+" checks, "+(fail?"FAIL":"PASS")); +process.exit(fail); +JS diff --git a/package.json b/package.json index a6d9cdeb..500d357b 100644 --- a/package.json +++ b/package.json @@ -31,5 +31,28 @@ "@axiomcode/engine-linux-x64": "0.1.0", "@axiomcode/engine-linux-arm64": "0.1.0", "@axiomcode/engine-win32-x64": "0.1.0" + }, + "bin": { + "axiomcode": "bin/axiomcode" + }, + "files": [ + "bin/", + "dist/", + "graph/", + "!graph/test/", + "parser/dist/", + "parser/package.json", + "README.md", + "LICENSE.md" + ], + "dependencies": { + "sax": "^1.4.4", + "tree-sitter": "^0.21.1", + "tree-sitter-c-sharp": "0.23.1", + "tree-sitter-groovy": "^0.1.2", + "tree-sitter-java": "^0.21.0", + "tree-sitter-python": "^0.21.0", + "typescript": "^6.0.0", + "yaml": "^2.8.2" } }