From 9740a98c5797625fa8d9f05fe37b03efbdfbef12 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Mon, 9 Nov 2026 17:53:07 +0100 Subject: [PATCH 1/3] npm packages are made from each published release and staged for approval - .github/workflows/npm.yml runs when a release is published. It downloads the five command line archives, holds each to the release's checksums and to the attestation of the files people download, and lays out six packages: testing-files-generator and one per system with its tfg binary. Each system's package is installed from its file and run on that system, and only then are all six staged on npm. Nothing is published until the owner approves each one with two-factor authentication. A pull request that touches the packages tries them against the latest release and stages nothing. - On macOS the package carries the whole tfg.app, because the binary only runs inside the app it was signed and notarised in, and the package looks for it there. - npm/testing-files-generator/release/assemble.js lays the packages out from the list of systems in lib/binary.js, at the version of the release, with every binary at exactly that version. It refuses a directory that is not empty and a symbolic link, which npm would leave out of a package. - A guard holds the systems the workflow tries to the ones the package installs a binary for, and staging to waiting for that try. - The package README no longer says it is not on npm. The main README and the changelog say how to install it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/npm.yml | 393 ++++++++++++++++++ CHANGELOG.md | 9 + README.md | 13 + internal/guard/mutation/entries.py | 27 ++ internal/guard/npmpackage_test.go | 53 ++- npm/testing-files-generator/README.md | 4 +- npm/testing-files-generator/lib/binary.js | 21 +- .../release/assemble.js | 240 +++++++++++ .../test/assemble.test.js | 187 +++++++++ .../test/binary.test.js | 3 +- 10 files changed, 935 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/npm.yml create mode 100644 npm/testing-files-generator/release/assemble.js create mode 100644 npm/testing-files-generator/test/assemble.test.js diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml new file mode 100644 index 00000000..b718595f --- /dev/null +++ b/.github/workflows/npm.yml @@ -0,0 +1,393 @@ +name: npm packages + +# The packages a release puts on npm: testing-files-generator, and one package +# per system carrying the tfg binary of that release. They are made of the +# PUBLISHED release rather than built again, as the container image is +# (docker.yml), and this workflow never publishes them. It stages them, and the +# owner approves each one on npmjs.com with two-factor authentication - the +# systems first, testing-files-generator last. docs/NPM-PUBLISH-2026-11-09.md. +# +# Every archive is held to the release's checksums and to the attestation +# attest-release.yml made of the files people download. That is the only one +# that fits all of them: the Windows archives are signed and the macOS one is +# notarised after release.yml built them, so their bytes are not the ones it +# attested. The packages are then installed from their files on the system +# each is for, and only after every system has run its own does anything +# reach npm. +# +# It starts when the owner publishes a release, and can be run by hand for the +# latest one - a run that stopped half way stages only what is not staged or +# published yet. A pull request that touches the packages tries them against +# the latest release and stages nothing. +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Published tag to stage the npm packages of, for example v0.6.0" + required: true + pull_request: + paths: + - "npm/**" + - ".github/workflows/npm.yml" + +permissions: + contents: read + +concurrency: + group: npm-${{ github.event.release.tag_name || inputs.tag || github.ref }} + cancel-in-progress: false + +jobs: + which: + name: which release, and whether it goes to npm + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + tag: ${{ steps.which.outputs.tag }} + version: ${{ steps.which.outputs.version }} + stage: ${{ steps.which.outputs.stage }} + steps: + - name: which release + id: which + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + INPUT_TAG: ${{ inputs.tag }} + # Asked of the API rather than taken from the event, because a run by + # hand has no event payload and a draft must never reach npm. + run: | + set -euo pipefail + latest="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name)" + case "$EVENT" in + release) tag="$RELEASE_TAG"; stage=true ;; + workflow_dispatch) tag="$INPUT_TAG"; stage=true ;; + *) tag="$latest"; stage=false ;; + esac + if ! [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "::error::\"$tag\" is not a release tag, for example v0.6.0" + exit 1 + fi + draft="$(gh release view "$tag" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)" + if [ "$draft" != "false" ]; then + echo "::error::$tag is not published yet, and npm only ever gets what people can download" + exit 1 + fi + # npm files a release under latest whatever its age, so staging an + # older one would take latest away from the newest. A prerelease goes + # under next and takes nothing away. + if [ "$stage" = true ] && [[ "$tag" != *-* ]] && [ "$tag" != "$latest" ]; then + echo "::error::$tag is not the latest release ($latest), and staging it would make npm install give people an older tfg" + exit 1 + fi + { + echo "tag=$tag" + echo "version=${tag#v}" + echo "stage=$stage" + } >> "$GITHUB_OUTPUT" + echo "$tag, stage: $stage" + + assemble: + name: the packages, made of the release's own archives + needs: which + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + TAG: ${{ needs.which.outputs.tag }} + VERSION: ${{ needs.which.outputs.version }} + steps: + # The code of the package from the tagged commit, the one its binaries + # were built from. A pull request tries its own code against the latest + # release. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ needs.which.outputs.stage == 'true' && needs.which.outputs.tag || '' }} + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: the published archives, each checked before it is opened + env: + GH_TOKEN: ${{ github.token }} + # The checksum says the download is the file the release lists. The + # attestation says attest-release.yml made a statement about these very + # bytes, and --signer-workflow holds it to that workflow. + run: | + set -euo pipefail + assemble=npm/testing-files-generator/release/assemble.js + if [ ! -f "$assemble" ]; then + echo "::error::the tree of $TAG has no $assemble, so it has no npm packages to make" + exit 1 + fi + mkdir -p download binaries + gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir download --pattern verify-SHA256SUMS.txt + node "$assemble" archives "$VERSION" > archives.txt + # Read into a list first, because gh reads standard input and a + # download inside the read loop eats the rest of the list. + mapfile -t lines < archives.txt + for line in "${lines[@]}"; do + read -r dir archive binary <<< "$line" + gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir download --pattern "$archive" + awk -v name="$archive" '$2 == name' download/verify-SHA256SUMS.txt > "download/$archive.sum" + if [ "$(wc -l < "download/$archive.sum")" != "1" ]; then + echo "::error::verify-SHA256SUMS.txt of $TAG does not name $archive exactly once" + exit 1 + fi + (cd download && sha256sum -c "$archive.sum") + gh attestation verify "download/$archive" --repo "$GITHUB_REPOSITORY" \ + --predicate-type https://spdx.dev/Document/v2.3 \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/attest-release.yml" + mkdir -p "binaries/$dir" + case "$archive" in + *.zip) unzip -q "download/$archive" -d "binaries/$dir" ;; + *) tar -xzf "download/$archive" -C "binaries/$dir" ;; + esac + echo "$dir $binary $(sha256sum "binaries/$dir/$binary" | cut -d' ' -f1)" >> binaries.txt + done + + - name: lay out and pack them + # Packed here, on Linux, because npm packs a file without its + # executable bit on Windows and the binary for Linux would not run. + # Each package goes by its absolute path: npm reads packages/win32-x64 + # as a repository on GitHub. + run: | + set -euo pipefail + node npm/testing-files-generator/release/assemble.js build "$VERSION" binaries packages > order.txt + mkdir tarballs + mapfile -t dirs < order.txt + for dir in "${dirs[@]}"; do + name="$(node -p "require('$dir/package.json').name")" + tarball="$(npm pack "$dir" --pack-destination tarballs --ignore-scripts | tail -n 1)" + echo "$name $tarball" >> tarballs/order.txt + done + node npm/testing-files-generator/release/assemble.js tag "$VERSION" > tarballs/dist-tag.txt + mv binaries.txt tarballs/binaries.txt + cat tarballs/order.txt + ls -l tarballs + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: npm-packages + path: tarballs/ + retention-days: 7 + + try: + name: installed and run on ${{ matrix.package }} + needs: [which, assemble] + strategy: + fail-fast: false + # One runner per package, each on the system the package is for. + # internal/guard/npmpackage_test.go holds this list to lib/binary.js. + matrix: + include: + - package: win32-x64 + runner: windows-latest + - package: win32-arm64 + runner: windows-11-arm + - package: linux-x64 + runner: ubuntu-latest + - package: linux-arm64 + runner: ubuntu-24.04-arm + - package: darwin-arm64 + runner: macos-latest + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + env: + VERSION: ${{ needs.which.outputs.version }} + PACKAGE: ${{ matrix.package }} + steps: + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: npm-packages + path: tarballs + + - name: install from the files, the way npm installs from the registry + shell: bash + # The runner is asked what it is rather than trusted to be it, because + # a runner label that moved to another architecture would otherwise try + # one package and report on another. + run: | + set -euo pipefail + here="$(node -p "process.platform + '-' + process.arch")" + if [ "$here" != "$PACKAGE" ]; then + echo "::error::this runner is $here, and it was to try $PACKAGE" + exit 1 + fi + main="$(awk '$1 == "testing-files-generator" { print $2 }' tarballs/order.txt)" + platform="$(awk -v name="@testing-files-generator/$PACKAGE" '$1 == name { print $2 }' tarballs/order.txt)" + if [ -z "$main" ] || [ -z "$platform" ]; then + echo "::error::the packages made for $VERSION hold no testing-files-generator or no $PACKAGE" + exit 1 + fi + mkdir try + cd try + npm init -y > /dev/null + npm install --no-audit --no-fund "../tarballs/$platform" "../tarballs/$main" + + - name: the binary of the release answers, and a set is written through it + shell: bash + working-directory: try + run: | + set -euo pipefail + said="$(npx --no-install tfg version)" + if [ "$said" != "$VERSION" ]; then + echo "::error::npx tfg version says $said, and the release is $VERSION" + exit 1 + fi + read -r _ binary sum <<< "$(awk -v dir="$PACKAGE" '$1 == dir' ../tarballs/binaries.txt)" + node -e ' + const crypto = require("node:crypto"); + const fs = require("node:fs"); + const [pkg, binary, want] = process.argv.slice(1); + const where = require.resolve(pkg + "/" + binary); + const got = crypto.createHash("sha256").update(fs.readFileSync(where)).digest("hex"); + if (got !== want) { + console.error(where + " hashes to " + got + ", and the binary in the release archive to " + want); + process.exit(1); + } + console.log(where + " is the binary of the release"); + ' "@testing-files-generator/$PACKAGE" "$binary" "$sum" + if [ "$RUNNER_OS" = macOS ]; then + # The binary runs only inside the app it was signed in, so the app + # has to come out of npm whole. + codesign --verify --deep --strict "node_modules/@testing-files-generator/$PACKAGE/tfg.app" + fi + cat > try.cjs <<'JS' + const assert = require('node:assert/strict'); + const fs = require('node:fs'); + const { fileSet } = require('testing-files-generator'); + + (async () => { + const set = fileSet({ preset: 'upload-validation', options: { limit: '64kb', bulk: 0 } }); + assert.ok(set.files.length > 0, 'the set has no files'); + for (const file of set.files) { + const where = await file.path(); + assert.equal(fs.statSync(where).size, file.bytes, `${file.title} is not the size the manifest gives`); + if (!file.unspecified) { + const { outcome, reason } = file.expected; + file.check(outcome === 'reject' ? { outcome, reason } : outcome); + } + } + console.log(`${set.files.length} files written, each the size it should be`); + })().catch((err) => { + console.error(err); + process.exit(1); + }); + JS + node try.cjs + + rehearse: + name: what staging would send, without sending it + needs: [which, try] + if: needs.which.outputs.stage != 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "26" + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: npm-packages + path: tarballs + + - name: stage with --dry-run + # Every tarball as ./tarballs/..., because npm reads tarballs/x.tgz as a + # repository on GitHub and asks git for it (measured 2026-11-09). + run: | + set -euo pipefail + tag="$(cat tarballs/dist-tag.txt)" + mapfile -t lines < tarballs/order.txt + for line in "${lines[@]}"; do + read -r _ tarball <<< "$line" + npm stage publish "./tarballs/$tarball" --dry-run --access public --tag "$tag" + done + + stage: + name: staged on npm for the owner to approve + needs: [which, try] + if: needs.which.outputs.stage == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + # The OIDC token npm trusts once a package has a trusted publisher, and + # the provenance statement npm publishes beside each package. + id-token: write + env: + VERSION: ${{ needs.which.outputs.version }} + steps: + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "26" + registry-url: "https://registry.npmjs.org" + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: npm-packages + path: tarballs + + - name: stage every package, the systems first + env: + # Only for the first release on npm, when no package exists yet and so + # none can trust this workflow. Remove the secret once the packages + # trust it - docs/NPM-PUBLISH-2026-11-09.md. + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + # With no secret, the placeholder setup-node leaves stays, as in npm's + # own example of trusted publishing, and npm signs in with OIDC. + run: | + set -euo pipefail + if [ -n "$NPM_TOKEN" ]; then + export NODE_AUTH_TOKEN="$NPM_TOKEN" + fi + npm_version="$(npm --version)" + if ! node -e 'const [a, b] = process.argv[1].split(".").map(Number); process.exit(a > 11 || (a === 11 && b >= 15) ? 0 : 1)' "$npm_version"; then + echo "::error::npm $npm_version cannot stage a package, and staging needs npm 11.15.0 or later" + exit 1 + fi + tag="$(cat tarballs/dist-tag.txt)" + staged=() + # Read into a list first, so that nothing npm runs can read the rest + # of it from standard input. + mapfile -t lines < tarballs/order.txt + for line in "${lines[@]}"; do + read -r name tarball <<< "$line" + if [ "$(npm view "$name@$VERSION" version 2>/dev/null || true)" = "$VERSION" ]; then + echo "$name@$VERSION is already published" + continue + fi + # Kept whole before it is searched: grep -q stops at the first + # match, and npm still writing would then end the pipe with a + # failure that reads as "not staged". + waiting="$(npm stage list "$name" --json 2>/dev/null || true)" + if grep -q -F "\"$VERSION\"" <<< "$waiting"; then + echo "$name@$VERSION is already staged" + staged+=("$name") + continue + fi + if [ -z "$NPM_TOKEN" ] && ! npm view "$name" name > /dev/null 2>&1; then + echo "::error::$name is not on npm yet, so it cannot trust this workflow, and there is no NPM_TOKEN secret to stage it with. The first release on npm needs one - docs/NPM-PUBLISH-2026-11-09.md" + exit 1 + fi + npm stage publish "./tarballs/$tarball" --access public --tag "$tag" --provenance + staged+=("$name") + done + { + echo "## $VERSION staged on npm" + echo + echo "Approve each one on npmjs.com with two-factor authentication, in this order." + echo "The systems come first, so that nobody installs testing-files-generator before its binary is there." + echo + for name in "${staged[@]}"; do + echo "1. \`$name@$VERSION\`" + done + } >> "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index 8339ea66..33eb33cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,15 @@ because it turns other people's test suites red. ### Added +- **tfg is on npm, for tests in Node.** + `npm install --save-dev testing-files-generator` gives Playwright, Vitest + and `node:test` a generated set of files, and says what the system under + test should do with each one. A test written for a file fails with that + file's own sentence on what the difference means. The package installs the + `tfg` of the same release for Windows and Linux on x64 and Arm and for macOS + on Apple silicon, and `npx tfg` runs it. [The example](examples/playwright) + is an upload form with four mistakes, and the generated set finds all four. + - **How much memory a run may use is yours to set.** `--memory` on `generate`, `validate` and `preset show`, and Memory on the window's Preferences screen: `auto` (half of this machine or container), a size diff --git a/README.md b/README.md index ed457076..08e20b04 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,19 @@ The files land in the current directory. On Linux add belong to you. With no directory mounted the run stops with exit code 5 instead of writing files that disappear with the container. +**With npm, for tests in Node.** The +[testing-files-generator](npm/testing-files-generator/README.md) package +gives Playwright, Vitest and `node:test` a generated set and what the system +under test should do with each file, and installs the `tfg` of the same +release for your system: + +``` +npm install --save-dev testing-files-generator +npx tfg version +``` + +It comes for Windows and Linux on x64 and Arm, and for macOS on Apple silicon. + **With Go installed:** ``` diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 2fd25b5f..39bc8550 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -16888,6 +16888,33 @@ "; unset it to use", "TestTheTextOfThePackageIsAsciiWithPlainPunctuation", ), + ( + "a system's npm package goes to npm without being installed on that system", + ".github/workflows/npm.yml", + " - package: linux-arm64 + runner: ubuntu-24.04-arm +", + "", + "TestEverySystemThePackageKnowsIsTriedBeforeAnythingIsStaged", + ), + ( + "npm.yml tries a package the npm package installs nothing from", + ".github/workflows/npm.yml", + " - package: win32-arm64 +", + " - package: windows-arm64 +", + "TestEverySystemThePackageKnowsIsTriedBeforeAnythingIsStaged", + ), + ( + "the npm packages are staged before they were tried", + ".github/workflows/npm.yml", + " needs: [which, try] + if: needs.which.outputs.stage == 'true'", + " needs: [which, assemble] + if: needs.which.outputs.stage == 'true'", + "TestEverySystemThePackageKnowsIsTriedBeforeAnythingIsStaged", + ), ( "the linked npm package changes its licence while the dependency gate still lets it through as MPL-2.0", "npm/testing-files-generator/package.json", diff --git a/internal/guard/npmpackage_test.go b/internal/guard/npmpackage_test.go index 82797f91..17458050 100644 --- a/internal/guard/npmpackage_test.go +++ b/internal/guard/npmpackage_test.go @@ -78,6 +78,42 @@ func TestThePackageKnowsEveryPlatformTheReleaseBuilds(t *testing.T) { } } +// Every package of a system is installed and run on that system before +// anything goes to npm (.github/workflows/npm.yml), and nothing is staged until +// every one of them has. The systems tried are written out in the workflow, +// one runner each, so they are held to lib/binary.js here: a system missing +// there goes to npm untried, and a system the package does not know is a run +// that tries nothing. docs/NPM-PUBLISH-2026-11-09.md. +func TestEverySystemThePackageKnowsIsTriedBeforeAnythingIsStaged(t *testing.T) { + workflow := npmFile(t, ".github/workflows/npm.yml") + var tried []string + for _, m := range regexp.MustCompile(`(?m)^\s+- package: ([a-z0-9-]+)\s*$`).FindAllStringSubmatch(workflow, -1) { + tried = append(tried, m[1]) + } + var known []string + binary := npmFile(t, npmPackage+"/lib/binary.js") + for _, m := range regexp.MustCompile(`'[a-z0-9]+ [a-z0-9]+': '@testing-files-generator/([a-z0-9-]+)'`).FindAllStringSubmatch(binary, -1) { + known = append(known, m[1]) + } + if len(tried) == 0 || len(known) == 0 { + t.Fatalf("read %d systems tried in npm.yml and %d known to lib/binary.js - this guard no longer reads them", len(tried), len(known)) + } + sort.Strings(tried) + sort.Strings(known) + if !reflect.DeepEqual(tried, known) { + t.Errorf("npm.yml tries the packages of %v, and the package installs a binary for %v", tried, known) + } + + stage := regexp.MustCompile(`(?ms)^ stage:\n(.*?)(?:^ [a-z][a-z-]*:\n|\z)`).FindStringSubmatch(workflow) + if stage == nil { + t.Fatal("npm.yml has no stage job - this guard no longer reads it") + } + needs := regexp.MustCompile(`(?m)^ needs: \[([^\]]*)\]`).FindStringSubmatch(stage[1]) + if needs == nil || !strings.Contains(", "+needs[1]+",", " try,") { + t.Errorf("the stage job of npm.yml does not wait for the try job, so a package can reach npm untried: needs %v", needs) + } +} + // The dependency gate lets the package through by name, because the graph has // no licence for a directory linked with file: (#220). That is only true while // the package is what the gate's exception says it is. @@ -218,14 +254,15 @@ var npmTextRoots = []string{npmPackage + "/", "examples/playwright/"} // header value, and judging it as prose would be crying wolf - the same line // goFileFaults draws for Go. var npmLiterals = map[string]bool{ - npmPackage + "/index.js": true, - npmPackage + "/bin/tfg.js": true, - npmPackage + "/lib/binary.js": true, - npmPackage + "/lib/fileset.js": true, - npmPackage + "/lib/run.js": true, - npmPackage + "/lib/testfile.js": true, - "examples/playwright/app/page.js": true, - "examples/playwright/check.mjs": true, + npmPackage + "/index.js": true, + npmPackage + "/bin/tfg.js": true, + npmPackage + "/lib/binary.js": true, + npmPackage + "/lib/fileset.js": true, + npmPackage + "/lib/run.js": true, + npmPackage + "/lib/testfile.js": true, + npmPackage + "/release/assemble.js": true, + "examples/playwright/app/page.js": true, + "examples/playwright/check.mjs": true, } func npmTextFiles(t *testing.T) []string { diff --git a/npm/testing-files-generator/README.md b/npm/testing-files-generator/README.md index d8623afc..7e2135bf 100644 --- a/npm/testing-files-generator/README.md +++ b/npm/testing-files-generator/README.md @@ -4,8 +4,6 @@ Real files of exact sizes and formats for your tests, and what the system under A PDF of exactly 1 MiB and one byte is a real PDF, so a server that looks inside it can only turn it away for its size. A file made with `Buffer.alloc` is zeros, so the same server turns it away for what it holds, and a test meant to prove a size limit passes on a server that has none. [The example](https://github.com/donislawdev/TestingFilesGenerator/tree/main/examples/playwright) shows exactly that. -> **Not on npm yet.** It will be published with a release of tfg. Until then, [the example](https://github.com/donislawdev/TestingFilesGenerator/tree/main/examples/playwright) runs it from the repository. - ## Contents - [What it is](#what-it-is) @@ -43,7 +41,7 @@ npm install --save-dev testing-files-generator ``` - **Node 22 or later.** Types for TypeScript come with the package, and both `import` and `require` work. -- **The tfg program comes in a second package** that npm picks for your system: Windows on x64 and Arm, Linux on x64 and Arm, macOS on Apple silicon. Nothing runs when the package is installed, and nothing is downloaded when the tests run. +- **The tfg program comes in a second package** that npm picks for your system: Windows on x64 and Arm, Linux on x64 and Arm, macOS on Apple silicon. Nothing runs when the package is installed, and nothing is downloaded when the tests run. The program is the one from the tfg release of the same version: signed on Windows, and on macOS signed and notarised inside the app it was signed in. - **Do not install with `--omit=optional`.** npm leaves that second package out with it, and the tests then stop with a message saying so. On a system with no package of its own, such as an Intel Mac, see [The tfg program](#the-tfg-program). ## Your first test diff --git a/npm/testing-files-generator/lib/binary.js b/npm/testing-files-generator/lib/binary.js index d3248281..418d426c 100644 --- a/npm/testing-files-generator/lib/binary.js +++ b/npm/testing-files-generator/lib/binary.js @@ -20,6 +20,22 @@ const platforms = { 'darwin arm64': '@testing-files-generator/darwin-arm64', }; +// binaryIn is where the binary sits inside the package of a system, as the +// release archive for that system lays it out. On macOS that is inside the app +// the release signed and notarised: its signature covers the app's Info.plist, +// and macOS kills the binary taken out on its own (exit 137, measured +// 2026-11-09 on macOS 27.2 - docs/NPM-PUBLISH-2026-11-09.md). +function binaryIn(platform) { + switch (platform) { + case 'win32': + return 'tfg.exe'; + case 'darwin': + return 'tfg.app/Contents/MacOS/tfg'; + default: + return 'tfg'; + } +} + // binaryPath finds the tfg binary this package runs. // // TFG_BINARY comes first and is taken as given - a tfg installed another way, @@ -43,9 +59,8 @@ function binaryPath({ env = process.env, platform = process.platform, arch = pro 'On another system, run the tests in a Linux container, where npm installs the package for Linux, ' + 'or install tfg another way and set TFG_BINARY to its path.'); } - const exe = platform === 'win32' ? 'tfg.exe' : 'tfg'; try { - return resolve(`${pkg}/${exe}`); + return resolve(`${pkg}/${binaryIn(platform)}`); } catch (err) { // Only a package that is not there is a matter of --omit. Anything else, // such as a package that does not export the binary, is told as it is, @@ -60,4 +75,4 @@ function binaryPath({ env = process.env, platform = process.platform, arch = pro } } -module.exports = { binaryPath, platforms }; +module.exports = { binaryIn, binaryPath, platforms }; diff --git a/npm/testing-files-generator/release/assemble.js b/npm/testing-files-generator/release/assemble.js new file mode 100644 index 00000000..373d940c --- /dev/null +++ b/npm/testing-files-generator/release/assemble.js @@ -0,0 +1,240 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +// assemble lays out the packages a release puts on npm: this one, and one per +// system carrying the tfg binary of that release. .github/workflows/npm.yml +// runs it on the archives it has downloaded and checked, and nothing here +// fetches or publishes anything. It is not part of the package - "files" in +// package.json leaves it out. +// +// The systems come from lib/binary.js and from nowhere else. A guard holds that +// list to the targets of the release, so a second list here could only ever +// disagree with it. docs/NPM-PUBLISH-2026-11-09.md. + +const fs = require('node:fs'); +const path = require('node:path'); +const { binaryIn, platforms } = require('../lib/binary.js'); + +const packageDir = path.resolve(__dirname, '..'); + +// How the release names a system and an architecture in its archives, against +// how Node names them in process.platform and process.arch. +const releaseSystem = { win32: 'windows', darwin: 'macos', linux: 'linux' }; +const releaseArch = { x64: 'amd64', arm64: 'arm64' }; +const systemLabel = { win32: 'Windows', darwin: 'macOS', linux: 'Linux' }; + +// A version npm takes and the release tags carry: three numbers, and a +// prerelease after a hyphen. Build metadata (+...) is left out, because npm +// drops it and two releases would then claim one version. +const versionShape = /^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$/; + +function checkedVersion(version) { + if (typeof version !== 'string' || !versionShape.test(version)) { + throw new Error(`${JSON.stringify(version)} is not a release version. ` + + 'Give the version of a release tag without its v, for example 0.6.0 or 0.6.0-rc1.'); + } + return version; +} + +// targets lists every system this package installs a binary for, in the order +// lib/binary.js gives them, with what each is called on both sides. +function targets() { + return Object.entries(platforms).map(([key, name]) => { + const [platform, arch] = key.split(' '); + if (!releaseSystem[platform] || !releaseArch[arch]) { + throw new Error(`lib/binary.js names ${key}, and assemble does not know what the release calls it. ` + + 'Add the release name of the system and of the architecture beside releaseSystem and releaseArch.'); + } + // binary is where lib/binary.js looks for it, and entry is what holds it at + // the top of the package - the binary itself, or the app on macOS. + const binary = binaryIn(platform); + return { + platform, + arch, + name, + dir: name.split('/')[1], + binary, + entry: binary.split('/')[0], + label: `${systemLabel[platform]} on ${arch}`, + }; + }); +} + +// archiveOf is the release asset that carries the binary for a target. +function archiveOf(version, target) { + const ext = target.platform === 'win32' ? 'zip' : 'tar.gz'; + return `tfg_${checkedVersion(version)}_${releaseSystem[target.platform]}_${releaseArch[target.arch]}.${ext}`; +} + +// distTag is the tag npm files a version under. A prerelease must not become +// latest, so that npm install keeps giving people the last release. +function distTag(version) { + return checkedVersion(version).includes('-') ? 'next' : 'latest'; +} + +function mainManifest() { + return JSON.parse(fs.readFileSync(path.join(packageDir, 'package.json'), 'utf8')); +} + +// platformManifest is the package.json of the package that carries one +// binary. os and cpu are what npm reads to install it on that system alone, +// and repository has to match this package's, because npm checks a +// provenance statement against it. +function platformManifest(version, target, main) { + return { + name: target.name, + version: checkedVersion(version), + description: `The tfg command line for ${target.label}, installed by testing-files-generator. ` + + 'Install testing-files-generator rather than this package.', + license: 'GPL-3.0-only', + author: main.author, + homepage: main.homepage, + repository: main.repository, + bugs: main.bugs, + os: [target.platform], + cpu: [target.arch], + files: [target.entry, 'LICENSE', 'THIRD-PARTY-NOTICES.md'], + publishConfig: { access: 'public' }, + }; +} + +function platformReadme(target) { + return `# ${target.name}\n\n` + + `The tfg command line for ${target.label}. ` + + '[testing-files-generator](https://www.npmjs.com/package/testing-files-generator) installs it ' + + 'on the system it belongs to, so install that package rather than this one:\n\n' + + '```sh\nnpm install --save-dev testing-files-generator\n```\n\n' + + 'tfg is licensed under GPL-3.0-only, in LICENSE. THIRD-PARTY-NOTICES.md names the code built into it ' + + 'and the licence of each part.\n'; +} + +// releasedManifest is this package's package.json as the release publishes it: +// the version of the release, and the package of every system at exactly that +// version, so a binary of another release is never installed beside it. +function releasedManifest(version, main) { + const optionalDependencies = {}; + for (const target of targets()) { + optionalDependencies[target.name] = checkedVersion(version); + } + return { ...main, version: checkedVersion(version), optionalDependencies }; +} + +function emptyDirectory(dir) { + if (fs.existsSync(dir) && fs.readdirSync(dir).length > 0) { + throw new Error(`${dir} is not empty. assemble writes into an empty directory, so that nothing left ` + + 'from an earlier run is published with this one.'); + } + fs.mkdirSync(dir, { recursive: true }); +} + +// copyFrom copies one file or directory, and says whose it was when it is not +// there - the archive of a release, or this package. +function copyFrom(from, to, name, whose) { + const source = path.join(from, name); + if (!fs.existsSync(source)) { + throw new Error(`${source} is not there, and ${whose} has to carry ${name}.`); + } + refuseLinks(source); + fs.cpSync(source, path.join(to, name), { recursive: true }); +} + +// refuseLinks stops at a symbolic link anywhere in what is to be packed. npm +// leaves links out of a package without a word - the tfg beside the macOS app +// is one, and the package made with it carried no binary at all (measured +// 2026-11-09). +function refuseLinks(source) { + const stat = fs.lstatSync(source); + if (stat.isSymbolicLink()) { + throw new Error(`${source} is a symbolic link, and npm leaves links out of a package. ` + + 'Copy what it points at instead, or change where lib/binary.js looks for the binary.'); + } + if (stat.isDirectory()) { + for (const name of fs.readdirSync(source)) { + refuseLinks(path.join(source, name)); + } + } +} + +function writeJson(file, value) { + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`); +} + +// build writes one directory per package under out and gives them in the +// order they are to be published: every system first, this package last, so +// that the package which asks for a binary never comes out before the binary. +// binaries holds what each archive unpacked to, one directory per target +// named as archives() names it. +// +// The directories are given as absolute paths, because npm reads a relative +// one such as packages/win32-x64 as a GitHub repository and asks git for it +// (measured 2026-11-09, npm 10.9.9). +function build(version, binaries, out) { + checkedVersion(version); + out = path.resolve(out); + emptyDirectory(out); + const main = mainManifest(); + const order = []; + for (const target of targets()) { + const from = path.join(binaries, target.dir); + const to = path.join(out, target.dir); + fs.mkdirSync(to); + const archive = `the archive for ${target.label}`; + copyFrom(from, to, target.entry, archive); + if (!fs.existsSync(path.join(to, target.binary))) { + throw new Error(`${path.join(from, target.binary)} is not there, and ${archive} has to carry ${target.binary}.`); + } + if (target.platform !== 'win32') { + // Set rather than trusted to the archive, because a binary npm unpacks + // without it is a package that cannot run on the one system it is for. + fs.chmodSync(path.join(to, target.binary), 0o755); + } + copyFrom(from, to, 'LICENSE', archive); + copyFrom(from, to, 'THIRD-PARTY-NOTICES.md', archive); + fs.writeFileSync(path.join(to, 'README.md'), platformReadme(target)); + writeJson(path.join(to, 'package.json'), platformManifest(version, target, main)); + order.push(to); + } + const to = path.join(out, main.name); + fs.mkdirSync(to); + for (const name of main.files) { + copyFrom(packageDir, to, name.replace(/\/$/, ''), 'this package'); + } + writeJson(path.join(to, 'package.json'), releasedManifest(version, main)); + order.push(to); + return order; +} + +// archives gives one line per system: the directory its archive is to be +// unpacked into, the archive, and where the binary sits in it. +const usage = 'Usage: node assemble.js archives | tag | build '; + +function main(argv) { + const [command, version, ...rest] = argv; + switch (command) { + case 'archives': + return targets().map((target) => `${target.dir} ${archiveOf(version, target)} ${target.binary}`); + case 'tag': + return [distTag(version)]; + case 'build': + if (rest.length !== 2) { + throw new Error(usage); + } + return build(version, rest[0], rest[1]); + default: + throw new Error(usage); + } +} + +if (require.main === module) { + try { + process.stdout.write(`${main(process.argv.slice(2)).join('\n')}\n`); + } catch (err) { + process.stderr.write(`assemble: ${err.message}\n`); + process.exitCode = 1; + } +} + +module.exports = { archiveOf, build, distTag, main, platformManifest, releasedManifest, targets }; diff --git a/npm/testing-files-generator/test/assemble.test.js b/npm/testing-files-generator/test/assemble.test.js new file mode 100644 index 00000000..4c22e759 --- /dev/null +++ b/npm/testing-files-generator/test/assemble.test.js @@ -0,0 +1,187 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { execSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { archiveOf, build, distTag, main, targets } = require('../release/assemble.js'); +const { platforms } = require('../lib/binary.js'); +const ours = require('../package.json'); + +function tempDir(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tfg-assemble-test-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +// binaries lays out what the release archives unpack to, one directory per +// system, each binary holding the name of its system so a mix-up shows. The +// macOS one sits in an app beside the Info.plist its signature covers, as in +// the release. +function binaries(t) { + const dir = tempDir(t); + for (const target of targets()) { + const into = path.join(dir, target.dir); + fs.mkdirSync(path.dirname(path.join(into, target.binary)), { recursive: true }); + fs.writeFileSync(path.join(into, target.binary), `binary for ${target.dir}`); + if (target.entry !== target.binary) { + fs.writeFileSync(path.join(into, target.entry, 'Contents', 'Info.plist'), 'plist'); + } + fs.writeFileSync(path.join(into, 'LICENSE'), 'GNU GENERAL PUBLIC LICENSE'); + fs.writeFileSync(path.join(into, 'THIRD-PARTY-NOTICES.md'), 'notices'); + fs.writeFileSync(path.join(into, 'README.md'), 'the README of the program, not of a package'); + } + return dir; +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +// What npm itself would put in the tarball, asked of npm rather than of the +// files list, because the list is what this test would otherwise be trusting. +// npm 10 and 11 answer with a list of packages, npm 12 with an object keyed by +// name (measured 2026-11-09), and CI runs the first while a person may run the +// last. +function packed(dir) { + const out = JSON.parse(execSync('npm pack --dry-run --json --ignore-scripts', { cwd: dir, encoding: 'utf8' })); + const [only] = Array.isArray(out) ? out : Object.values(out); + return only.files.map((f) => f.path).sort(); +} + +test('the archive of each system is named as the release names it', () => { + // The names of the assets of v0.5.0, as the release page lists them. + const released = [ + 'tfg_0.5.0_linux_amd64.tar.gz', + 'tfg_0.5.0_linux_arm64.tar.gz', + 'tfg_0.5.0_macos_arm64.tar.gz', + 'tfg_0.5.0_windows_amd64.zip', + 'tfg_0.5.0_windows_arm64.zip', + ]; + assert.deepEqual(targets().map((target) => archiveOf('0.5.0', target)).sort(), released); + assert.deepEqual(main(['archives', '0.5.0']).map((line) => line.split(' ')[1]).sort(), released); + assert.ok(main(['archives', '0.5.0']).includes('darwin-arm64 tfg_0.5.0_macos_arm64.tar.gz tfg.app/Contents/MacOS/tfg')); +}); + +test('every system lib/binary.js knows is a target, under the package it names', () => { + assert.deepEqual(targets().map((target) => `${target.platform} ${target.arch}`), Object.keys(platforms)); + assert.deepEqual(targets().map((target) => target.name), Object.values(platforms)); +}); + +test('a prerelease is filed under next, a release under latest', () => { + assert.equal(distTag('0.6.0'), 'latest'); + assert.equal(distTag('1.0.0-rc1'), 'next'); + assert.deepEqual(main(['tag', '0.6.0-rc.2']), ['next']); +}); + +test('a version that is not a release version is refused before anything is written', (t) => { + const out = path.join(tempDir(t), 'out'); + for (const version of ['v0.6.0', '0.6', '0.6.0+build.1', '', undefined, '0.6.0\n']) { + assert.throws(() => build(version, binaries(t), out), (err) => err.message.includes('is not a release version')); + assert.throws(() => distTag(version), /is not a release version/); + } + assert.equal(fs.existsSync(out), false); +}); + +test('each system gets a package with its own binary, installed on that system alone', (t) => { + const from = binaries(t); + const out = path.join(tempDir(t), 'out'); + const order = build('0.6.0', from, out); + assert.equal(order.length, targets().length + 1); + for (const [i, target] of targets().entries()) { + const dir = order[i]; + assert.equal(dir, path.join(out, target.dir)); + const manifest = readJson(path.join(dir, 'package.json')); + assert.equal(manifest.name, target.name); + assert.equal(manifest.version, '0.6.0'); + assert.deepEqual(manifest.os, [target.platform]); + assert.deepEqual(manifest.cpu, [target.arch]); + assert.equal(manifest.license, 'GPL-3.0-only'); + assert.deepEqual(manifest.repository, ours.repository); + assert.deepEqual(manifest.publishConfig, { access: 'public' }); + assert.equal(fs.readFileSync(path.join(dir, target.binary), 'utf8'), `binary for ${target.dir}`); + assert.equal(fs.readFileSync(path.join(dir, 'LICENSE'), 'utf8'), 'GNU GENERAL PUBLIC LICENSE'); + const readme = fs.readFileSync(path.join(dir, 'README.md'), 'utf8'); + assert.ok(readme.includes('npm install --save-dev testing-files-generator'), readme); + if (process.platform !== 'win32' && target.platform !== 'win32') { + assert.equal(fs.statSync(path.join(dir, target.binary)).mode & 0o777, 0o755); + } + } +}); + +test('this package comes last, at the version of the release and asking for every binary at exactly it', (t) => { + const order = build('0.6.0', binaries(t), path.join(tempDir(t), 'out')); + const dir = order[order.length - 1]; + const manifest = readJson(path.join(dir, 'package.json')); + assert.equal(manifest.version, '0.6.0'); + const wanted = Object.fromEntries(Object.values(platforms).map((name) => [name, '0.6.0'])); + assert.deepEqual(manifest.optionalDependencies, wanted); + const { version, optionalDependencies, ...rest } = manifest; + const { version: _v, optionalDependencies: _o, ...kept } = ours; + assert.deepEqual(rest, kept); +}); + +test('npm packs each package with what it needs and nothing else', (t) => { + const order = build('0.6.0', binaries(t), path.join(tempDir(t), 'out')); + for (const [i, target] of targets().entries()) { + const app = target.entry === target.binary ? [] : [`${target.entry}/Contents/Info.plist`]; + assert.deepEqual(packed(order[i]), [target.binary, ...app, 'LICENSE', 'README.md', 'THIRD-PARTY-NOTICES.md', 'package.json'].sort()); + } + const files = packed(order[order.length - 1]); + assert.ok(files.includes('lib/binary.js') && files.includes('bin/tfg.js') && files.includes('index.d.ts'), files.join(' ')); + assert.deepEqual(files.filter((f) => f.startsWith('test/') || f.startsWith('release/')), []); +}); + +test('a directory that is not empty is refused, so nothing from an earlier run goes out', (t) => { + const out = tempDir(t); + fs.writeFileSync(path.join(out, 'left-over.tgz'), ''); + assert.throws(() => build('0.6.0', binaries(t), out), (err) => err.message.includes('is not empty')); +}); + +test('an archive without its licence is refused, naming the system', (t) => { + const from = binaries(t); + const target = targets()[0]; + fs.rmSync(path.join(from, target.dir, 'LICENSE')); + assert.throws(() => build('0.6.0', from, path.join(tempDir(t), 'out')), + (err) => err.message.includes(`the archive for ${target.label} has to carry LICENSE`)); +}); + +test('an archive without its binary is refused, naming the binary', (t) => { + const from = binaries(t); + const target = targets()[targets().length - 1]; + fs.rmSync(path.join(from, target.dir, target.binary)); + assert.throws(() => build('0.6.0', from, path.join(tempDir(t), 'out')), + (err) => err.message.includes(`has to carry ${target.binary}`)); +}); + +test('the command line refuses what it does not know and says how it is used', () => { + assert.throws(() => main(['publish', '0.6.0']), /Usage: node assemble.js/); + assert.throws(() => main(['build', '0.6.0']), /Usage: node assemble.js/); +}); + +test('the macOS package carries the whole app, where lib/binary.js looks for the binary', (t) => { + const order = build('0.6.0', binaries(t), path.join(tempDir(t), 'out')); + const mac = targets().findIndex((target) => target.platform === 'darwin'); + assert.ok(mac >= 0, 'no macOS target, so this test checked nothing'); + const dir = order[mac]; + assert.equal(fs.readFileSync(path.join(dir, 'tfg.app', 'Contents', 'MacOS', 'tfg'), 'utf8'), 'binary for darwin-arm64'); + assert.equal(fs.readFileSync(path.join(dir, 'tfg.app', 'Contents', 'Info.plist'), 'utf8'), 'plist'); + assert.deepEqual(readJson(path.join(dir, 'package.json')).files, ['tfg.app', 'LICENSE', 'THIRD-PARTY-NOTICES.md']); +}); + +test('a symbolic link in what is packed is refused, because npm would leave it out', (t) => { + const from = binaries(t); + const target = targets().find((x) => x.entry !== x.binary); + const link = path.join(from, target.dir, target.entry, 'Contents', 'Link'); + try { + fs.symlinkSync('Info.plist', link); + } catch (err) { + t.skip(`this system does not let the test make a link: ${err.code}`); + return; + } + assert.throws(() => build('0.6.0', from, path.join(tempDir(t), 'out')), + (err) => err.message.includes('is a symbolic link') && err.message.includes(link)); +}); diff --git a/npm/testing-files-generator/test/binary.test.js b/npm/testing-files-generator/test/binary.test.js index d0d8b993..b8ab137c 100644 --- a/npm/testing-files-generator/test/binary.test.js +++ b/npm/testing-files-generator/test/binary.test.js @@ -22,7 +22,8 @@ test('TFG_BINARY naming nothing is refused with the path it named', () => { test('each platform a release builds resolves to its own package', () => { for (const [key, pkg] of Object.entries(platforms)) { const [platform, arch] = key.split(' '); - const exe = platform === 'win32' ? 'tfg.exe' : 'tfg'; + // Written out rather than asked of binaryIn, so a wrong place there shows. + const exe = { win32: 'tfg.exe', darwin: 'tfg.app/Contents/MacOS/tfg', linux: 'tfg' }[platform]; const asked = []; const found = binaryPath({ env: {}, platform, arch, resolve: (id) => { asked.push(id); return `/resolved/${id}`; } }); assert.deepEqual(asked, [`${pkg}/${exe}`]); From 73e13bfe9b49db899c5bf268b405b13398d6375f Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Mon, 9 Nov 2026 17:56:18 +0100 Subject: [PATCH 2/3] The mutation entries for npm.yml name their patterns on one line Co-Authored-By: Claude Opus 5.5 --- internal/guard/mutation/entries.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 39bc8550..71e7dbce 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -16891,28 +16891,22 @@ ( "a system's npm package goes to npm without being installed on that system", ".github/workflows/npm.yml", - " - package: linux-arm64 - runner: ubuntu-24.04-arm -", + " - package: linux-arm64\n runner: ubuntu-24.04-arm\n", "", "TestEverySystemThePackageKnowsIsTriedBeforeAnythingIsStaged", ), ( "npm.yml tries a package the npm package installs nothing from", ".github/workflows/npm.yml", - " - package: win32-arm64 -", - " - package: windows-arm64 -", + " - package: win32-arm64\n", + " - package: windows-arm64\n", "TestEverySystemThePackageKnowsIsTriedBeforeAnythingIsStaged", ), ( "the npm packages are staged before they were tried", ".github/workflows/npm.yml", - " needs: [which, try] - if: needs.which.outputs.stage == 'true'", - " needs: [which, assemble] - if: needs.which.outputs.stage == 'true'", + " needs: [which, try]\n if: needs.which.outputs.stage == 'true'", + " needs: [which, assemble]\n if: needs.which.outputs.stage == 'true'", "TestEverySystemThePackageKnowsIsTriedBeforeAnythingIsStaged", ), ( From 451dc025b32d53676b8b90fdecf32290c9c2604f Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Mon, 9 Nov 2026 18:16:39 +0100 Subject: [PATCH 3/3] Sets kept in the system's temporary directory are in a directory only their user can change - When neither TFG_CACHE_DIR nor node_modules gives the npm package a place for its sets, it used testing-files-generator in the system's temporary directory. On Linux every user of the machine shares that directory, so someone else could make it first and change the files the tests run on. The package now uses testing-files-generator- there, made open to its owner alone, and refuses one that belongs to another user, that others can write to, or that is a link or a file, saying to set TFG_CACHE_DIR or cacheDir instead. - The plan tfg writes for the package, and its error stream, go to files made new and readable by their owner alone, so nothing left under the same name is written through. - Every error of the npm workflow and of assemble.js now says what to do. The one about a missing NPM_TOKEN says which token to create and where to put it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/npm.yml | 20 ++++---- npm/testing-files-generator/README.md | 3 +- npm/testing-files-generator/index.d.ts | 2 +- npm/testing-files-generator/lib/fileset.js | 47 ++++++++++++++++++- npm/testing-files-generator/lib/run.js | 9 +++- .../release/assemble.js | 16 +++++-- .../test/fileset.test.js | 45 +++++++++++++++++- npm/testing-files-generator/test/run.test.js | 15 ++++++ 8 files changed, 136 insertions(+), 21 deletions(-) diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml index b718595f..aee8febc 100644 --- a/.github/workflows/npm.yml +++ b/.github/workflows/npm.yml @@ -67,19 +67,19 @@ jobs: *) tag="$latest"; stage=false ;; esac if ! [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then - echo "::error::\"$tag\" is not a release tag, for example v0.6.0" + echo "::error::\"$tag\" is not a release tag. Run this workflow with the tag of a published release, such as v0.6.0." exit 1 fi draft="$(gh release view "$tag" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)" if [ "$draft" != "false" ]; then - echo "::error::$tag is not published yet, and npm only ever gets what people can download" + echo "::error::$tag is not published yet, and npm only ever gets what people can download. Publish the release, and this workflow starts by itself." exit 1 fi # npm files a release under latest whatever its age, so staging an # older one would take latest away from the newest. A prerelease goes # under next and takes nothing away. if [ "$stage" = true ] && [[ "$tag" != *-* ]] && [ "$tag" != "$latest" ]; then - echo "::error::$tag is not the latest release ($latest), and staging it would make npm install give people an older tfg" + echo "::error::$tag is not the latest release ($latest), and staging it would make npm install give people an older tfg. Run this workflow for $latest instead." exit 1 fi { @@ -120,7 +120,7 @@ jobs: set -euo pipefail assemble=npm/testing-files-generator/release/assemble.js if [ ! -f "$assemble" ]; then - echo "::error::the tree of $TAG has no $assemble, so it has no npm packages to make" + echo "::error::the tree of $TAG has no $assemble, so it has no npm packages to make. npm gets the releases tagged after the packages came into the repository, so run this workflow for one of those." exit 1 fi mkdir -p download binaries @@ -134,7 +134,7 @@ jobs: gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir download --pattern "$archive" awk -v name="$archive" '$2 == name' download/verify-SHA256SUMS.txt > "download/$archive.sum" if [ "$(wc -l < "download/$archive.sum")" != "1" ]; then - echo "::error::verify-SHA256SUMS.txt of $TAG does not name $archive exactly once" + echo "::error::verify-SHA256SUMS.txt of $TAG does not name $archive exactly once, so the archive cannot be checked. Compare the release's assets with what release.yml attaches before running this again." exit 1 fi (cd download && sha256sum -c "$archive.sum") @@ -218,13 +218,13 @@ jobs: set -euo pipefail here="$(node -p "process.platform + '-' + process.arch")" if [ "$here" != "$PACKAGE" ]; then - echo "::error::this runner is $here, and it was to try $PACKAGE" + echo "::error::this runner is $here, and it was to try $PACKAGE. Give $PACKAGE a runner of its own system in the matrix of this job." exit 1 fi main="$(awk '$1 == "testing-files-generator" { print $2 }' tarballs/order.txt)" platform="$(awk -v name="@testing-files-generator/$PACKAGE" '$1 == name { print $2 }' tarballs/order.txt)" if [ -z "$main" ] || [ -z "$platform" ]; then - echo "::error::the packages made for $VERSION hold no testing-files-generator or no $PACKAGE" + echo "::error::the packages made for $VERSION hold no testing-files-generator or no $PACKAGE. lib/binary.js and the matrix of this job have to name the same systems." exit 1 fi mkdir try @@ -239,7 +239,7 @@ jobs: set -euo pipefail said="$(npx --no-install tfg version)" if [ "$said" != "$VERSION" ]; then - echo "::error::npx tfg version says $said, and the release is $VERSION" + echo "::error::npx tfg version says $said, and the release is $VERSION, so the package carries another release's binary. Nothing is staged. Look at the archives of $VERSION before running this again." exit 1 fi read -r _ binary sum <<< "$(awk -v dir="$PACKAGE" '$1 == dir' ../tarballs/binaries.txt)" @@ -351,7 +351,7 @@ jobs: fi npm_version="$(npm --version)" if ! node -e 'const [a, b] = process.argv[1].split(".").map(Number); process.exit(a > 11 || (a === 11 && b >= 15) ? 0 : 1)' "$npm_version"; then - echo "::error::npm $npm_version cannot stage a package, and staging needs npm 11.15.0 or later" + echo "::error::npm $npm_version cannot stage a package, and staging needs npm 11.15.0 or later. Raise node-version of this job to a Node that comes with it." exit 1 fi tag="$(cat tarballs/dist-tag.txt)" @@ -375,7 +375,7 @@ jobs: continue fi if [ -z "$NPM_TOKEN" ] && ! npm view "$name" name > /dev/null 2>&1; then - echo "::error::$name is not on npm yet, so it cannot trust this workflow, and there is no NPM_TOKEN secret to stage it with. The first release on npm needs one - docs/NPM-PUBLISH-2026-11-09.md" + echo "::error::$name is not on npm yet, so it cannot trust this workflow, and there is no NPM_TOKEN secret to stage it with. Create a granular access token on npmjs.com that can stage (Read and write, stage only), set it as the NPM_TOKEN secret of this repository, and run this workflow again." exit 1 fi npm stage publish "./tarballs/$tarball" --access public --tag "$tag" --provenance diff --git a/npm/testing-files-generator/README.md b/npm/testing-files-generator/README.md index 7e2135bf..a75525f4 100644 --- a/npm/testing-files-generator/README.md +++ b/npm/testing-files-generator/README.md @@ -380,7 +380,7 @@ Plans a set and gives it back. Nothing is written until a test asks for a file. | `preset` | a preset by name, as `npx tfg preset list` lists them | | `recipe` | the path of a recipe file. A relative path is taken from the directory the tests were started in | | `options` | the parameters of the preset and the flags of `tfg generate`, without the dashes | -| `cacheDir` | where sets are kept. By default `TFG_CACHE_DIR`, then `node_modules/.cache/testing-files-generator`, then the system's temporary directory | +| `cacheDir` | where sets are kept. By default `TFG_CACHE_DIR`, then `node_modules/.cache/testing-files-generator`, then a directory of your own in the system's temporary directory, which nobody else can write to | | `binary` | the tfg to run. By default `TFG_BINARY`, then the one this package installed | `fileSet` throws at once, while the spec file loads, when the binary or the recipe is not there or tfg refuses the plan. Two calls with the same spec in one process share one plan and one write. @@ -438,6 +438,7 @@ Every message names what it is about and what to do. The ones you are most likel | `The binary for ... comes in the package ..., and it is not installed` | npm installed with `--omit=optional`. Install again without it | | `testing-files-generator has no binary for ...` | This system has no package of tfg. See [The tfg program](#the-tfg-program) | | `tfg wrote a manifest of version ...` | `TFG_BINARY` points at a tfg whose manifest this package cannot read. Point it at a tfg of the same version as this package, or leave it unset | +| `... where testing-files-generator keeps sets when it is given no other place, ...` | The directory in the system's temporary directory belongs to someone else, or others can write to it, so they could change your files. Set `TFG_CACHE_DIR` or `cacheDir` to a directory of your own | | `testing-files-generator: 1 unfinished set was left in ...` | A warning. A run stopped part way, and what it left can be deleted | ## The tfg program diff --git a/npm/testing-files-generator/index.d.ts b/npm/testing-files-generator/index.d.ts index a994fe3d..22f1a24b 100644 --- a/npm/testing-files-generator/index.d.ts +++ b/npm/testing-files-generator/index.d.ts @@ -96,7 +96,7 @@ export interface FileSetSpec { * { limit: '2mb', bulk: 0, seed: 7 }. npx tfg preset show lists what a preset takes. */ options?: Record; - /** Where sets are kept. TFG_CACHE_DIR, or node_modules/.cache/testing-files-generator, or the system's temporary directory. */ + /** Where sets are kept. TFG_CACHE_DIR, or node_modules/.cache/testing-files-generator, or a directory of your own in the system's temporary directory. */ cacheDir?: string; /** The tfg binary to run. TFG_BINARY, or the one this package installed. */ binary?: string; diff --git a/npm/testing-files-generator/lib/fileset.js b/npm/testing-files-generator/lib/fileset.js index 663c106c..09ff987d 100644 --- a/npm/testing-files-generator/lib/fileset.js +++ b/npm/testing-files-generator/lib/fileset.js @@ -126,7 +126,50 @@ function defaultCacheDir() { if (fs.existsSync(modules)) { return path.join(modules, '.cache', 'testing-files-generator'); } - return path.join(os.tmpdir(), 'testing-files-generator'); + return ownTempDir(); +} + +// ownTempDir is the last place sets go: a directory of this user's own in the +// system's temporary directory. On Linux that directory is shared by every +// user of the machine, and a directory of a fixed name there could be made +// first by someone else, who could then change the files the tests run on. So +// the name carries the user's id, a new one is open to its owner alone, and +// one already there is taken only when it is a directory of this user that +// nobody else can write to. Windows and macOS give each user a temporary +// directory of their own, and Windows has no id to ask for - a uid of null. +// docs/NPM-PUBLISH-2026-11-09.md, code scanning alerts 23 and 24. +function ownTempDir({ tmp = os.tmpdir(), uid = process.getuid ? process.getuid() : null } = {}) { + if (uid === null) { + return path.join(tmp, 'testing-files-generator'); + } + const dir = path.join(tmp, `testing-files-generator-${uid}`); + let stat; + try { + stat = fs.lstatSync(dir); + } catch (err) { + if (err.code !== 'ENOENT') { + throw err; + } + // Asked again after it is made, because someone else can make it first. + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + stat = fs.lstatSync(dir); + } + let fault = ''; + if (stat.isSymbolicLink()) { + fault = 'is a link to another place'; + } else if (!stat.isDirectory()) { + fault = 'is not a directory'; + } else if (stat.uid !== uid) { + fault = 'belongs to another user'; + } else if ((stat.mode & 0o022) !== 0) { + fault = 'can be written to by other users'; + } + if (fault) { + throw new Error(`${dir}, where testing-files-generator keeps sets when it is given no other place, ${fault}, ` + + 'so someone else could change the files your tests run on. ' + + 'Set TFG_CACHE_DIR or the cacheDir option to a directory of your own, or remove that one.'); + } + return dir; } function labelOf(spec) { @@ -457,4 +500,4 @@ function matches(file, filter = {}) { return has(filter.group, file.group) && has(filter.target, file.target) && has(filter.outcome, file.expected.outcome); } -module.exports = { events, fileSet, readable }; +module.exports = { events, fileSet, ownTempDir, readable }; diff --git a/npm/testing-files-generator/lib/run.js b/npm/testing-files-generator/lib/run.js index dbea3fdf..3f3db81a 100644 --- a/npm/testing-files-generator/lib/run.js +++ b/npm/testing-files-generator/lib/run.js @@ -43,12 +43,17 @@ function notStarted(binary, err) { // gives a child process by default. The error stream goes to a file of its own // for the same reason - a buffer that fills stops tfg part way - and only its // end is kept. +// +// Both files are made new and readable by their owner alone, so neither a file +// nor a link someone else left under the same name is written through. The +// names are drawn at random by the caller, and the directory may be the +// system's temporary one. function runToFile(binary, args, outFile) { const errFile = `${outFile}.err`; - const out = fs.openSync(outFile, 'w'); + const out = fs.openSync(outFile, 'wx', 0o600); let err; try { - err = fs.openSync(errFile, 'w+'); + err = fs.openSync(errFile, 'wx+', 0o600); const done = spawnSync(binary, args, { stdio: ['ignore', out, err], windowsHide: true }); if (done.error) { throw notStarted(binary, done.error); diff --git a/npm/testing-files-generator/release/assemble.js b/npm/testing-files-generator/release/assemble.js index 373d940c..be5bf445 100644 --- a/npm/testing-files-generator/release/assemble.js +++ b/npm/testing-files-generator/release/assemble.js @@ -125,17 +125,25 @@ function releasedManifest(version, main) { function emptyDirectory(dir) { if (fs.existsSync(dir) && fs.readdirSync(dir).length > 0) { throw new Error(`${dir} is not empty. assemble writes into an empty directory, so that nothing left ` + - 'from an earlier run is published with this one.'); + 'from an earlier run is published with this one. Give it a new directory, or empty this one.'); } fs.mkdirSync(dir, { recursive: true }); } +// missing says that a file a package needs is not where it should be, whose it +// was - the archive of a release, or this package - and where to look. +function missing(source, whose, name) { + return new Error(`${source} is not there, and ${whose} has to carry ${name}. ` + + 'Check that the directory given for it holds what the release archive unpacked to, ' + + 'or that this package still has the file it lists.'); +} + // copyFrom copies one file or directory, and says whose it was when it is not -// there - the archive of a release, or this package. +// there. function copyFrom(from, to, name, whose) { const source = path.join(from, name); if (!fs.existsSync(source)) { - throw new Error(`${source} is not there, and ${whose} has to carry ${name}.`); + throw missing(source, whose, name); } refuseLinks(source); fs.cpSync(source, path.join(to, name), { recursive: true }); @@ -184,7 +192,7 @@ function build(version, binaries, out) { const archive = `the archive for ${target.label}`; copyFrom(from, to, target.entry, archive); if (!fs.existsSync(path.join(to, target.binary))) { - throw new Error(`${path.join(from, target.binary)} is not there, and ${archive} has to carry ${target.binary}.`); + throw missing(path.join(from, target.binary), archive, target.binary); } if (target.platform !== 'win32') { // Set rather than trusted to the archive, because a binary npm unpacks diff --git a/npm/testing-files-generator/test/fileset.test.js b/npm/testing-files-generator/test/fileset.test.js index 70b88712..5b93e172 100644 --- a/npm/testing-files-generator/test/fileset.test.js +++ b/npm/testing-files-generator/test/fileset.test.js @@ -5,7 +5,7 @@ const fs = require('node:fs'); const path = require('node:path'); const { test } = require('node:test'); -const { fileSet, readable } = require('../lib/fileset.js'); +const { fileSet, ownTempDir, readable } = require('../lib/fileset.js'); const { TfgError } = require('../lib/run.js'); const { node, pkg, small, tempDir, tfg } = require('./helpers.js'); @@ -242,3 +242,46 @@ test('npx tfg is tfg - the same output and the same exit code', async () => { assert.equal(viaPackage.status, direct.status); assert.equal(viaPackage.stderr, direct.stderr); }); + +// The last place sets go is the system's temporary directory, which on Linux +// every user of the machine shares. Asked of a directory made here, with the +// user's id passed in, so another user can be played without one. +test('in the temporary directory a user gets a directory of their own, open to them alone', (t) => { + const tmp = tempDir(t); + assert.equal(ownTempDir({ tmp, uid: null }), path.join(tmp, 'testing-files-generator'), + 'a system with no user id keeps the plain name, because its temporary directory is the user\'s own'); + if (!process.getuid) { + return; + } + const uid = process.getuid(); + const dir = ownTempDir({ tmp, uid }); + assert.equal(dir, path.join(tmp, `testing-files-generator-${uid}`)); + assert.ok(fs.statSync(dir).isDirectory()); + assert.equal(fs.statSync(dir).mode & 0o777, 0o700); + assert.equal(ownTempDir({ tmp, uid }), dir, 'a directory of this user is taken again'); +}); + +test('a directory in the temporary directory that someone else could change is refused, with the way out', { + skip: process.getuid ? false : 'Windows has no user id, and gives each user a temporary directory of their own', +}, (t) => { + const uid = process.getuid(); + const refused = (tmp, as, why) => assert.throws(() => ownTempDir({ tmp, uid: as }), + (err) => err.message.includes(why) && err.message.includes('TFG_CACHE_DIR') && err.message.includes('cacheDir')); + + const shared = tempDir(t); + fs.mkdirSync(path.join(shared, `testing-files-generator-${uid}`)); + fs.chmodSync(path.join(shared, `testing-files-generator-${uid}`), 0o777); + refused(shared, uid, 'can be written to by other users'); + + const someoneElses = tempDir(t); + fs.mkdirSync(path.join(someoneElses, `testing-files-generator-${uid + 1}`), { mode: 0o700 }); + refused(someoneElses, uid + 1, 'belongs to another user'); + + const linked = tempDir(t); + fs.symlinkSync(tempDir(t), path.join(linked, `testing-files-generator-${uid}`)); + refused(linked, uid, 'is a link to another place'); + + const file = tempDir(t); + fs.writeFileSync(path.join(file, `testing-files-generator-${uid}`), ''); + refused(file, uid, 'is not a directory'); +}); diff --git a/npm/testing-files-generator/test/run.test.js b/npm/testing-files-generator/test/run.test.js index ace16476..a7633ec9 100644 --- a/npm/testing-files-generator/test/run.test.js +++ b/npm/testing-files-generator/test/run.test.js @@ -1,6 +1,7 @@ 'use strict'; const assert = require('node:assert/strict'); +const fs = require('node:fs'); const path = require('node:path'); const { test } = require('node:test'); @@ -28,3 +29,17 @@ test('a child that says a great deal on its error stream runs to its own end, an (err) => err instanceof TfgError && err.status === 3 && err.stderr.endsWith('the last words') && err.stderr.length <= 64 * 1024); }); + +test('the output file is made new and private, and a file already under its name is never written through', (t) => { + const [binary, args] = child('process.stdout.write("planned")'); + const out = path.join(tempDir(t), 'out.json'); + runToFile(binary, args, out); + assert.equal(fs.readFileSync(out, 'utf8'), 'planned'); + if (process.platform !== 'win32') { + assert.equal(fs.statSync(out).mode & 0o777, 0o600); + } + const planted = path.join(tempDir(t), 'planted.json'); + fs.writeFileSync(planted, 'left by someone else'); + assert.throws(() => runToFile(binary, args, planted), (err) => err.code === 'EEXIST'); + assert.equal(fs.readFileSync(planted, 'utf8'), 'left by someone else'); +});