diff --git a/.github/actions/release-archives/action.yml b/.github/actions/release-archives/action.yml new file mode 100644 index 00000000..c6d714f4 --- /dev/null +++ b/.github/actions/release-archives/action.yml @@ -0,0 +1,70 @@ +name: The archives of a published release, each checked before it is opened +description: >- + Downloads the archives a published release attaches, holds each to the + release's checksums and to the attestation attest-release.yml made of the + files people download, and extracts it. npm.yml and pypi.yml both make their + packages of what this leaves, so the check lives in one place. + +# Every archive is held to the release's checksums and to the attestation +# attest-release.yml made of the files people download. That is the only one +# that fits all of them: the Windows archives are signed and the macOS one is +# notarised after release.yml built them, so their bytes are not the ones it +# attested. The checksum says the download is the file the release lists. The +# attestation says attest-release.yml made a statement about these very bytes, +# and --signer-workflow holds it to that workflow. +# +# It leaves each archive extracted in binaries/, and binaries.txt with a +# line for each: the directory, the binary inside it, and the binary's sha256. + +inputs: + tag: + description: The tag of a published release, such as v0.6.0. + required: true + archives: + description: >- + A file with a line for each archive: the directory to extract it into, + the name of the archive, and where the binary is inside it. + required: true + token: + description: A token that can read the release, github.token. + required: true + +runs: + using: composite + steps: + - name: download, check and extract every archive + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + TAG: ${{ inputs.tag }} + ARCHIVES: ${{ inputs.archives }} + run: | + set -euo pipefail + mkdir -p download binaries + gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir download --pattern verify-SHA256SUMS.txt + # Read into a list first, because gh reads standard input and a + # download inside the read loop eats the rest of the list. + mapfile -t lines < "$ARCHIVES" + if [ "${#lines[@]}" -eq 0 ]; then + echo "::error::$ARCHIVES names no archive, so there is nothing to check. The step before this one lists them." + exit 1 + fi + for line in "${lines[@]}"; do + read -r dir archive binary <<< "$line" + gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir download --pattern "$archive" + awk -v name="$archive" '$2 == name' download/verify-SHA256SUMS.txt > "download/$archive.sum" + if [ "$(wc -l < "download/$archive.sum")" != "1" ]; then + echo "::error::verify-SHA256SUMS.txt of $TAG does not name $archive exactly once, so the archive cannot be checked. Compare the release's assets with what release.yml attaches before running this again." + exit 1 + fi + (cd download && sha256sum -c "$archive.sum") + gh attestation verify "download/$archive" --repo "$GITHUB_REPOSITORY" \ + --predicate-type https://spdx.dev/Document/v2.3 \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/attest-release.yml" + mkdir -p "binaries/$dir" + case "$archive" in + *.zip) unzip -q "download/$archive" -d "binaries/$dir" ;; + *) tar -xzf "download/$archive" -C "binaries/$dir" ;; + esac + echo "$dir $binary $(sha256sum "binaries/$dir/$binary" | cut -d' ' -f1)" >> binaries.txt + done diff --git a/.github/requirements-pypi.in b/.github/requirements-pypi.in new file mode 100644 index 00000000..fb9854d4 --- /dev/null +++ b/.github/requirements-pypi.in @@ -0,0 +1,30 @@ +# What the PyPI packages need in CI beyond Python itself, pinned, in a file a +# bot can edit. +# +# python/testing-files-generator is tested with pytest in ci.yml, and +# .github/workflows/pypi.yml installs the wheels of a release with uv as well +# as pip, because uv keeps a binary's execute bit only when the wheel says it +# was made on Unix and pip keeps it either way (docs/PYTEST-2026-11-09.md). +# tomli reads pyproject.toml for release/assemble.py on Python 3.10, which has +# no tomllib - it is the library tomllib was taken from. None of them is +# linked into either binary or shipped in a wheel, and go.mod never sees them. +# +# The metadata of the wheels is checked by pypa/gh-action-pypi-publish, which +# runs twine check before it uploads anything. twine is not here because its +# own tree did not install on Python 3.10 and brought a licence the dependency +# gate cannot read (docs/REVIEW-226-2026-11-09.md). +# +# Licences read from the pinned versions rather than recalled, on 2026-11-09, +# from the metadata on PyPI: pytest MIT, uv MIT OR Apache-2.0, tomli MIT (and +# the LICENSE file of its tag 2.5.0). +# +# requirements-pypi.txt next to this file is compiled from it by pip-compile, +# the same way requirements-semgrep.txt is, so Dependabot proposes a set pip +# can install. It is compiled on Python 3.10, the oldest the package declares, +# because a set resolved on a newer Python can pin a version 3.10 cannot +# install. To compile it by hand, from the root of the repository: +# +# docker run --rm -v "$PWD/.github:/w" -w /w python:3.10-slim sh -c "pip install -q pip==26.1.1 pip-tools==7.5.3 && pip-compile --strip-extras --no-emit-index-url --output-file=requirements-pypi.txt requirements-pypi.in" +pytest==9.2.0 +tomli==2.5.0 ; python_version < "3.11" +uv==0.13.7 diff --git a/.github/requirements-pypi.txt b/.github/requirements-pypi.txt new file mode 100644 index 00000000..b03c2193 --- /dev/null +++ b/.github/requirements-pypi.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --no-emit-index-url --no-index --output-file=requirements-pypi.txt --strip-extras requirements-pypi.in +# +exceptiongroup==1.3.1 + # via pytest +iniconfig==2.3.1 + # via pytest +packaging==26.3 + # via pytest +pluggy==1.7.0 + # via pytest +pygments==2.21.0 + # via pytest +pytest==9.2.0 + # via -r requirements-pypi.in +tomli==2.5.0 ; python_version < "3.11" + # via + # -r requirements-pypi.in + # pytest +typing-extensions==4.16.0 + # via exceptiongroup +uv==0.13.7 + # via -r requirements-pypi.in diff --git a/.github/scripts/dependency_gate.py b/.github/scripts/dependency_gate.py index 987f1381..e35a6461 100644 --- a/.github/scripts/dependency_gate.py +++ b/.github/scripts/dependency_gate.py @@ -126,7 +126,13 @@ # names in front of this gate - four of them with a licence GitHub leaves empty # or describes with an identifier of its own. Vulnerabilities in this file are # still checked, by the dependency review action and by OSV-Scanner. -SKIPPED_MANIFESTS = frozenset({".github/requirements-semgrep.txt"}) +# +# requirements-pypi.txt joined it on 2026-11-09 (#226): pytest, uv and what they +# bring, which ci.yml and pypi.yml install on a runner to test the Python +# package and try its wheels. On Python 3.10 pytest brings typing-extensions, +# whose PSF-2.0 is not in ALLOWED. What the Python packages themselves depend +# on is in python/*/pyproject.toml, and those are still asked. +SKIPPED_MANIFESTS = frozenset({".github/requirements-semgrep.txt", ".github/requirements-pypi.txt"}) def added(review): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dae8548b..6f63c353 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1542,6 +1542,59 @@ jobs: node check.mjs shell: bash + python: + name: Python package on ${{ matrix.os }} with Python ${{ matrix.python }} + # The Python package (python/testing-files-generator) runs tfg and reads + # what it writes, so nothing in the Go suite runs it. Its own tests run + # here against the binary this commit builds, on the three systems a + # release has binaries for - Windows above all, because the rename that + # settles a race between test workers is refused there in ways it is not + # elsewhere. Python 3.10 is the oldest the package declares, and the + # newest is tried beside it on Linux. The tests of the release tools are + # among them: the wheels are laid out from archives the tests make. + # docs/PYTEST-2026-11-09.md. + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python: "3.10" + - os: ubuntu-latest + python: "3.14" + - os: windows-latest + python: "3.10" + - os: macos-latest + python: "3.10" + runs-on: ${{ matrix.os }} + env: + CGO_ENABLED: "0" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: false + cache: true + + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ matrix.python }} + + - name: build tfg where the package's tests look for it + run: go build -tags "$(cat .github/build-tags)" -o "tfg$(go env GOEXE)" ./cmd/tfg + shell: bash + + - name: the package's own tests + working-directory: python/testing-files-generator + run: | + python -m pip install --disable-pip-version-check -r ../../.github/requirements-pypi.txt + python -m pytest -p no:cacheprovider + shell: bash + fuzz: name: fuzzing with a time budget # Weekly and by hand, never on a push. Fuzzing searches without end, so a diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml index aee8febc..d3761116 100644 --- a/.github/workflows/npm.yml +++ b/.github/workflows/npm.yml @@ -31,6 +31,7 @@ on: paths: - "npm/**" - ".github/workflows/npm.yml" + - ".github/actions/release-archives/**" permissions: contents: read @@ -110,12 +111,7 @@ jobs: with: node-version: "22" - - name: the published archives, each checked before it is opened - env: - GH_TOKEN: ${{ github.token }} - # The checksum says the download is the file the release lists. The - # attestation says attest-release.yml made a statement about these very - # bytes, and --signer-workflow holds it to that workflow. + - name: which archives the packages are made of run: | set -euo pipefail assemble=npm/testing-files-generator/release/assemble.js @@ -123,31 +119,17 @@ jobs: echo "::error::the tree of $TAG has no $assemble, so it has no npm packages to make. npm gets the releases tagged after the packages came into the repository, so run this workflow for one of those." exit 1 fi - mkdir -p download binaries - gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir download --pattern verify-SHA256SUMS.txt node "$assemble" archives "$VERSION" > archives.txt - # Read into a list first, because gh reads standard input and a - # download inside the read loop eats the rest of the list. - mapfile -t lines < archives.txt - for line in "${lines[@]}"; do - read -r dir archive binary <<< "$line" - gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir download --pattern "$archive" - awk -v name="$archive" '$2 == name' download/verify-SHA256SUMS.txt > "download/$archive.sum" - if [ "$(wc -l < "download/$archive.sum")" != "1" ]; then - echo "::error::verify-SHA256SUMS.txt of $TAG does not name $archive exactly once, so the archive cannot be checked. Compare the release's assets with what release.yml attaches before running this again." - exit 1 - fi - (cd download && sha256sum -c "$archive.sum") - gh attestation verify "download/$archive" --repo "$GITHUB_REPOSITORY" \ - --predicate-type https://spdx.dev/Document/v2.3 \ - --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/attest-release.yml" - mkdir -p "binaries/$dir" - case "$archive" in - *.zip) unzip -q "download/$archive" -d "binaries/$dir" ;; - *) tar -xzf "download/$archive" -C "binaries/$dir" ;; - esac - echo "$dir $binary $(sha256sum "binaries/$dir/$binary" | cut -d' ' -f1)" >> binaries.txt - done + + # The same check pypi.yml makes, in one place: the checksum says the + # download is the file the release lists, and the attestation that + # attest-release.yml made a statement about these very bytes. + - name: the published archives, each checked before it is opened + uses: ./.github/actions/release-archives + with: + tag: ${{ needs.which.outputs.tag }} + archives: archives.txt + token: ${{ github.token }} - name: lay out and pack them # Packed here, on Linux, because npm packs a file without its diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 00000000..df89531b --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,292 @@ +name: PyPI packages + +# The packages a release puts on PyPI: testing-files-generator, as one wheel +# for each system with that system's tfg binary of the release inside and one +# wheel with none, and pytest-testing-files-generator, which installs it. They +# are made of the PUBLISHED release, as npm.yml makes the npm packages, and +# reach PyPI only after every system has installed and used them and the owner +# has approved the run in the pypi environment. docs/PYTEST-2026-11-09.md. +# +# PyPI takes them through trusted publishing, so no token is kept anywhere, +# and the publishing action puts an attestation of each file beside it. A run +# by hand can send them to TestPyPI instead, to see that PyPI takes them +# before a release depends on it. A pull request that touches the packages +# makes and tries them from the latest release and publishes nothing. + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Published tag to publish the PyPI packages of, for example v0.6.0" + required: true + index: + description: "testpypi to try PyPI's test index, pypi to publish" + required: true + default: testpypi + type: choice + options: [testpypi, pypi] + pull_request: + paths: + - "python/**" + - "testdata/wrappers/**" + - ".github/requirements-pypi.txt" + - ".github/workflows/pypi.yml" + - ".github/actions/release-archives/**" + +permissions: + contents: read + +concurrency: + group: pypi-${{ github.event.release.tag_name || inputs.tag || github.ref }} + cancel-in-progress: false + +jobs: + which: + name: which release, and where it goes + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + tag: ${{ steps.which.outputs.tag }} + version: ${{ steps.which.outputs.version }} + index: ${{ steps.which.outputs.index }} + steps: + - name: which release + id: which + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + INPUT_TAG: ${{ inputs.tag }} + INPUT_INDEX: ${{ inputs.index }} + # Asked of the API rather than taken from the event, because a run by + # hand has no event payload and a draft must never reach PyPI. An older + # release may go to PyPI after a newer one: pip installs the highest + # version, whatever order they were published in. + run: | + set -euo pipefail + case "$EVENT" in + release) tag="$RELEASE_TAG"; index=pypi ;; + workflow_dispatch) tag="$INPUT_TAG"; index="$INPUT_INDEX" ;; + *) tag="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name)"; index=none ;; + esac + if ! [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "::error::\"$tag\" is not a release tag. Run this workflow with the tag of a published release, such as v0.6.0." + exit 1 + fi + draft="$(gh release view "$tag" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)" + if [ "$draft" != "false" ]; then + echo "::error::$tag is not published yet, and PyPI only ever gets what people can download. Publish the release, and this workflow starts by itself." + exit 1 + fi + { + echo "tag=$tag" + echo "version=${tag#v}" + echo "index=$index" + } >> "$GITHUB_OUTPUT" + echo "$tag, index: $index" + + assemble: + name: the wheels, made of the release's own archives + needs: which + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + TAG: ${{ needs.which.outputs.tag }} + VERSION: ${{ needs.which.outputs.version }} + steps: + # The code of the package from the tagged commit, the one its binaries + # were built from. A pull request tries its own code against the latest + # release, and so does a run to TestPyPI: that is a rehearsal of the + # code the next release will carry, with binaries that are already out. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ needs.which.outputs.index == 'pypi' && needs.which.outputs.tag || '' }} + + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.13" + + - name: which archives the wheels are made of + run: | + set -euo pipefail + assemble=python/testing-files-generator/release/assemble.py + if [ ! -f "$assemble" ]; then + echo "::error::the tree of $TAG has no $assemble, so it has no PyPI packages to make. PyPI gets the releases tagged after the packages came into the repository, so run this workflow for one of those." + exit 1 + fi + python "$assemble" archives "$VERSION" > archives.txt + + - name: the published archives, each checked before it is opened + uses: ./.github/actions/release-archives + with: + tag: ${{ needs.which.outputs.tag }} + archives: archives.txt + token: ${{ github.token }} + + - name: lay out the wheels and check them + # Laid out here, on Linux, where the archives keep the execute bit of + # the binaries for Linux and macOS. + run: | + set -euo pipefail + python -m pip install --disable-pip-version-check -r .github/requirements-pypi.txt + python python/testing-files-generator/release/assemble.py build "$VERSION" binaries dist + python python/testing-files-generator/release/try_wheels.py --without-binary dist "$VERSION" + mv binaries.txt dist/binaries.txt + ls -l dist/main dist/plugin + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: pypi-wheels + path: dist/ + retention-days: 7 + + try: + name: installed and run on ${{ matrix.name }} + needs: [which, assemble] + strategy: + fail-fast: false + # One runner per system, each the system its wheel is for, and Alpine + # in a container for the wheels for musl. Each installs with pip and + # with uv. internal/guard/pypipackage_test.go holds this list to + # src/testing_files_generator/_binary.py. + matrix: + include: + - name: win32-x64 + system: win32-x64 + runner: windows-latest + python: "3.13" + - name: win32-arm64 + system: win32-arm64 + runner: windows-11-arm + python: "3.13" + - name: linux-x64 + system: linux-x64 + runner: ubuntu-latest + python: "3.10" + - name: linux-arm64 + system: linux-arm64 + runner: ubuntu-24.04-arm + python: "3.13" + - name: darwin-arm64 + system: darwin-arm64 + runner: macos-latest + python: "3.13" + - name: linux-x64 on Alpine + system: linux-x64 + runner: ubuntu-latest + alpine: python:3.13-alpine + - name: linux-arm64 on Alpine + system: linux-arm64 + runner: ubuntu-24.04-arm + alpine: python:3.13-alpine + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + env: + VERSION: ${{ needs.which.outputs.version }} + SYSTEM: ${{ matrix.system }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ needs.which.outputs.index == 'pypi' && needs.which.outputs.tag || '' }} + + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + if: matrix.alpine == '' + with: + python-version: ${{ matrix.python }} + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: pypi-wheels + path: dist + + - name: installed with pip and with uv, and used + if: matrix.alpine == '' + shell: bash + run: | + set -euo pipefail + python -m pip install --disable-pip-version-check -r .github/requirements-pypi.txt + for installer in pip uv; do + python python/testing-files-generator/release/try_wheels.py dist "$VERSION" dist/binaries.txt "$SYSTEM" "$installer" + done + + - name: installed with pip and with uv on Alpine, and used + if: matrix.alpine != '' + env: + IMAGE: ${{ matrix.alpine }} + run: | + set -euo pipefail + docker run --rm -v "$PWD:/w" -w /w -e VERSION -e SYSTEM "$IMAGE" sh -c ' + set -eu + python -m pip install --root-user-action=ignore --disable-pip-version-check -r .github/requirements-pypi.txt + for installer in pip uv; do + python python/testing-files-generator/release/try_wheels.py dist "$VERSION" dist/binaries.txt "$SYSTEM" "$installer" + done' + + publish: + name: published on ${{ needs.which.outputs.index }} + needs: [which, try] + if: needs.which.outputs.index != 'none' + runs-on: ubuntu-latest + timeout-minutes: 15 + # The run waits here until the owner approves it in the environment, and + # PyPI takes the files only from this workflow in this environment - the + # trusted publisher of each project names both. + environment: + name: ${{ needs.which.outputs.index }} + url: ${{ needs.which.outputs.index == 'testpypi' && 'https://test.pypi.org/project/testing-files-generator/' || 'https://pypi.org/project/testing-files-generator/' }} + permissions: + contents: read + # The OIDC token PyPI trusts, and the attestations published beside each file. + id-token: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: pypi-wheels + path: dist + + # testing-files-generator first, so pytest-testing-files-generator never + # names a version PyPI does not have. A run that stopped half way skips + # what is there already. verify-metadata runs twine check on the files + # before anything is sent, and it is written out because nothing else in + # this workflow checks the metadata the way PyPI reads it. + - name: testing-files-generator on TestPyPI + if: needs.which.outputs.index == 'testpypi' + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: dist/main/ + repository-url: https://test.pypi.org/legacy/ + verify-metadata: true + skip-existing: true + print-hash: true + + - name: pytest-testing-files-generator on TestPyPI + if: needs.which.outputs.index == 'testpypi' + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: dist/plugin/ + repository-url: https://test.pypi.org/legacy/ + verify-metadata: true + skip-existing: true + print-hash: true + + - name: testing-files-generator on PyPI + if: needs.which.outputs.index == 'pypi' + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: dist/main/ + verify-metadata: true + skip-existing: true + print-hash: true + + - name: pytest-testing-files-generator on PyPI + if: needs.which.outputs.index == 'pypi' + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: dist/plugin/ + verify-metadata: true + skip-existing: true + print-hash: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 33eb33cc..63ccc83e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,14 @@ because it turns other people's test suites red. on Apple silicon, and `npx tfg` runs it. [The example](examples/playwright) is an upload form with four mistakes, and the generated set finds all four. +- **tfg is on PyPI, for tests in Python.** `pip install + testing-files-generator` gives pytest a test for each file of a generated + set, named by what the file should meet, and a file with no right answer is + skipped with the reason. A test fails with the file's own sentence on what + the difference means. The package installs the `tfg` of the same release + for Windows and Linux on x64 and Arm, Alpine among them, and for macOS on + Apple silicon, and pip and uv both install it. + - **How much memory a run may use is yours to set.** `--memory` on `generate`, `validate` and `preset show`, and Memory on the window's Preferences screen: `auto` (half of this machine or container), a size diff --git a/README.md b/README.md index 08e20b04..9ca979da 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,23 @@ npx tfg version It comes for Windows and Linux on x64 and Arm, and for macOS on Apple silicon. +**With pip, for tests in Python.** The +[testing-files-generator](python/testing-files-generator/README.md) package +gives pytest and unittest a generated set and what the system under test +should do with each file, and installs the `tfg` of the same release for your +system. pytest finds its plugin on its own: + +``` +pip install testing-files-generator +tfg version +``` + +If `tfg` is not found, the folder pip puts programs in is not on your +`PATH`, and `python -m testing_files_generator version` runs the same tfg. + +It comes for Windows and Linux on x64 and Arm, Alpine among them, and for +macOS on Apple silicon. uv installs it as well. + **With Go installed:** ``` diff --git a/internal/guard/actionpinning_test.go b/internal/guard/actionpinning_test.go index e1428d3e..96a89fd8 100644 --- a/internal/guard/actionpinning_test.go +++ b/internal/guard/actionpinning_test.go @@ -1,7 +1,11 @@ package guard import ( + "errors" + "fmt" + "io/fs" "os" + "path" "path/filepath" "strings" "testing" @@ -28,8 +32,18 @@ import ( // This walks every file in the directory rather than a list of names, so a // workflow added next month is covered without anybody remembering to come // back here. +// +// An action of this repository - `uses: ./.github/actions/...` - has no tag +// and no commit, and needs neither: it is code in this tree, and it changes +// only through a commit that is reviewed here. It is not a door, though. A +// composite action runs steps with `uses:` of their own, so every action.yml +// under .github/actions is read by the same rule, and a local reference is let +// through only when this guard read the file it points at. Measured on #226, +// 2026-11-09: the first local action turned this guard red for both workflows +// that run it, while its own steps were read by nothing. func TestEveryActionIsPinnedToACommitRatherThanATag(t *testing.T) { - dir := filepath.Join(repoRoot(t), ".github", "workflows") + root := repoRoot(t) + dir := filepath.Join(root, ".github", "workflows") entries, err := os.ReadDir(dir) if err != nil { t.Skipf("the workflows are not here: %v", err) @@ -58,17 +72,41 @@ func TestEveryActionIsPinnedToACommitRatherThanATag(t *testing.T) { t.Errorf("pinnedToACommit(%q) should be %v, because it is %s", c.uses, c.pinned, c.why) } } + for _, c := range []struct { + uses string + local bool + why string + }{ + {"./.github/actions/release-archives", true, "an action kept in this repository"}, + {"./.github/workflows/build.yml", true, "a workflow of this repository called by another"}, + {".github/actions/release-archives", false, "a path without ./, which GitHub reads as OWNER/NAME"}, + {"actions/checkout@v7", false, "an action of another account"}, + } { + if inThisRepository(c.uses) != c.local { + t.Errorf("inThisRepository(%q) should be %v, because it is %s", c.uses, c.local, c.why) + } + } + + files := yamlFilesIn(entries, dir) + files = append(files, actionFiles(t, filepath.Join(root, ".github", "actions"))...) + // The files read, by their path from the root with forward slashes - the + // way a local `uses:` names them. + read := map[string]bool{} + type reference struct{ where, uses string } + var local []reference seen := 0 - for _, e := range entries { - name := e.Name() - if e.IsDir() || (!strings.HasSuffix(name, ".yml") && !strings.HasSuffix(name, ".yaml")) { - continue + for _, file := range files { + body, err := os.ReadFile(file) + if err != nil { + t.Fatalf("reading %s: %v", file, err) } - body, err := os.ReadFile(filepath.Join(dir, name)) + rel, err := filepath.Rel(root, file) if err != nil { - t.Fatalf("reading %s: %v", name, err) + t.Fatalf("%s is not under %s: %v", file, root, err) } + rel = filepath.ToSlash(rel) + read[rel] = true // Comments are removed first, because the version now lives in one and // "# v7.0.1" would otherwise read as a reference. for i, line := range strings.Split(withoutYamlComments(string(body)), "\n") { @@ -81,6 +119,10 @@ func TestEveryActionIsPinnedToACommitRatherThanATag(t *testing.T) { continue } seen++ + if inThisRepository(uses) { + local = append(local, reference{fmt.Sprintf("%s:%d", rel, i+1), uses}) + continue + } if pinnedToACommit(uses) { continue } @@ -89,8 +131,21 @@ func TestEveryActionIsPinnedToACommitRatherThanATag(t *testing.T) { "its releases here. Resolve the tag to its commit and keep the version in a "+ "comment beside it:\n"+ " uses: OWNER/NAME@<40 hex characters> # v1.2.3", - name, i+1, uses) + rel, i+1, uses) + } + } + + // A local reference is let through because what it points at is read + // here too. One this walk did not read would be let through unread. + for _, r := range local { + target := path.Clean(strings.TrimPrefix(r.uses, "./")) + if read[target] || read[path.Join(target, "action.yml")] || read[path.Join(target, "action.yaml")] { + continue } + t.Errorf("%s runs %s, an action of this repository, and this guard did not read it.\n"+ + "A local action is let through only because the steps it runs are held to the same rule. "+ + "Keep it as .github/actions//action.yml, where this guard reads it, or teach the "+ + "guard where it lives.", r.where, r.uses) } // A walk that matched nothing would report a clean tree, which is the way @@ -102,6 +157,49 @@ func TestEveryActionIsPinnedToACommitRatherThanATag(t *testing.T) { } } +// yamlFilesIn are the .yml and .yaml files directly in a directory, of the +// entries read from it. workflowFiles in buildfloor_test.go takes .yml alone. +func yamlFilesIn(entries []os.DirEntry, dir string) []string { + var files []string + for _, e := range entries { + name := e.Name() + if e.IsDir() || (!strings.HasSuffix(name, ".yml") && !strings.HasSuffix(name, ".yaml")) { + continue + } + files = append(files, filepath.Join(dir, name)) + } + return files +} + +// actionFiles are the action.yml files of the actions this repository keeps +// for its workflows, or none when it keeps none. +func actionFiles(t *testing.T, dir string) []string { + t.Helper() + var files []string + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + if p == dir && errors.Is(err, fs.ErrNotExist) { + return filepath.SkipDir + } + return err + } + if !d.IsDir() && (d.Name() == "action.yml" || d.Name() == "action.yaml") { + files = append(files, p) + } + return nil + }) + if err != nil { + t.Fatalf("reading the actions in %s: %v", dir, err) + } + return files +} + +// inThisRepository reports whether a `uses:` value names an action or a +// workflow of this repository, which GitHub knows by the leading "./". +func inThisRepository(uses string) bool { + return strings.HasPrefix(uses, "./") +} + // pinnedToACommit reports whether a `uses:` value names a commit. // // Forty hexadecimal characters and nothing else. An abbreviated hash is refused diff --git a/internal/guard/ascii_test.go b/internal/guard/ascii_test.go index 1de7d63d..6366c535 100644 --- a/internal/guard/ascii_test.go +++ b/internal/guard/ascii_test.go @@ -70,6 +70,10 @@ var englishFiles = []string{ // the second what they read before running it - the same criterion again. "npm/testing-files-generator/README.md", "examples/playwright/README.md", + // The pages PyPI shows for the Python package and its pytest plugin, + // added 2026-11-09 with them - the same criterion as the npm package's. + "python/testing-files-generator/README.md", + "python/pytest-testing-files-generator/README.md", } // symbolsAllowed names the files that may hold a symbol such as an emoji. diff --git a/internal/guard/dependencygate_test.go b/internal/guard/dependencygate_test.go index d0b56ef2..f39eb2ce 100644 --- a/internal/guard/dependencygate_test.go +++ b/internal/guard/dependencygate_test.go @@ -146,6 +146,23 @@ func TestTheDependencyGateRefusesALicenceThisProjectCannotDistribute(t *testing. "name": "typing_extensions", "version": "4.16.0", "license": nil}}, 0, }, + { + // Measured on #226, 2026-11-09: compiled on Python 3.10, pytest + // brings typing-extensions, PSF-2.0. The file lists what ci.yml and + // pypi.yml install on a runner to test the Python package. + "a package the PyPI jobs install, under a licence the list does not name", + []dep{{"change_type": "added", "ecosystem": "pip", "manifest": ".github/requirements-pypi.txt", + "name": "typing-extensions", "version": "4.16.0", "license": "PSF-2.0"}}, + 0, + }, + { + // What the Python package itself depends on is still asked: its + // pyproject.toml is a manifest of something people install. + "the same licence in the manifest of the Python package", + []dep{{"change_type": "added", "ecosystem": "pip", "manifest": "python/testing-files-generator/pyproject.toml", + "name": "typing-extensions", "version": "4.16.0", "license": "PSF-2.0"}}, + 1, + }, { // And the skip is that file, not the ecosystem: the same package // listed anywhere else is asked like any other. diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 71e7dbce..fa8189f9 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -16923,4 +16923,96 @@ "'Uploaded; done'", "TestTheTextOfThePackageIsAsciiWithPlainPunctuation", ), + ( + "the Python package forgets Windows on Arm, which the release builds", + "python/testing-files-generator/src/testing_files_generator/_binary.py", + '"win32 x64", "win32 arm64", "linux x64"', + '"win32 x64", "linux x64"', + "TestThePythonPackageKnowsEveryPlatformTheReleaseBuilds", + ), + ( + "pypi.yml stops trying the wheel for musl on Arm", + ".github/workflows/pypi.yml", + " - name: linux-arm64 on Alpine\n system: linux-arm64\n runner: ubuntu-24.04-arm\n alpine: python:3.13-alpine\n", + "", + "TestEverySystemThePythonPackageKnowsIsTriedBeforeAnythingIsPublished", + ), + ( + "pypi.yml tries a system the Python package installs nothing for", + ".github/workflows/pypi.yml", + " - name: win32-arm64\n system: win32-arm64\n", + " - name: win32-arm64\n system: windows-arm64\n", + "TestEverySystemThePythonPackageKnowsIsTriedBeforeAnythingIsPublished", + ), + ( + "the wheels are published before they were tried", + ".github/workflows/pypi.yml", + " needs: [which, try]\n if: needs.which.outputs.index != 'none'", + " needs: [which, assemble]\n if: needs.which.outputs.index != 'none'", + "TestEverySystemThePythonPackageKnowsIsTriedBeforeAnythingIsPublished", + ), + ( + "the wheels are published without waiting for the owner's approval", + ".github/workflows/pypi.yml", + " environment:\n name: ${{ needs.which.outputs.index }}\n", + " environment:\n name: release\n", + "TestEverySystemThePythonPackageKnowsIsTriedBeforeAnythingIsPublished", + ), + ( + "the Python package forgets an outcome the program declares", + "python/testing-files-generator/src/testing_files_generator/_testfile.py", + 'OUTCOMES = ("accept", "reject", "sanitize", "unspecified")', + 'OUTCOMES = ("accept", "reject", "unspecified")', + "TestThePythonPackageHoldsTheManifestsClosedLists", + ), + ( + "the pytest plugin registers no marker for one outcome", + "python/testing-files-generator/src/testing_files_generator/pytest_plugin.py", + ' "sanitize": "files the system under test should take and change",\n', + "", + "TestThePythonPackageHoldsTheManifestsClosedLists", + ), + ( + "the Python package's README leaves a reason out", + "python/testing-files-generator/README.md", + "`dimensions_limit`, `duplicate`, `encoding_invalid`", + "`dimensions_limit`, `encoding_invalid`", + "TestThePythonPackageHoldsTheManifestsClosedLists", + ), + ( + "a message of the Python package says something with a semicolon", + "python/testing-files-generator/src/testing_files_generator/_fileset.py", + 'Nothing uses them and they can be deleted."), stacklevel=4)', + 'Nothing uses them; they can be deleted."), stacklevel=4)', + "TestTheTextOfThePythonPackageIsAsciiWithPlainPunctuation", + ), + ( + "an action of this repository runs another account's action by a tag", + ".github/actions/release-archives/action.yml", + " - name: download, check and extract every archive\n", + " - uses: actions/checkout@v7\n - name: download, check and extract every archive\n", + "TestEveryActionIsPinnedToACommitRatherThanATag", + ), + ( + "the pinning guard stops reading the actions this repository keeps, " + "and lets their references through unread", + "internal/guard/actionpinning_test.go", + 'if !d.IsDir() && (d.Name() == "action.yml" || d.Name() == "action.yaml") {', + 'if false && !d.IsDir() && (d.Name() == "action.yml" || d.Name() == "action.yaml") {', + "TestEveryActionIsPinnedToACommitRatherThanATag", + ), + ( + "a path without ./ passes for an action of this repository", + "internal/guard/actionpinning_test.go", + 'return strings.HasPrefix(uses, "./")', + 'return strings.HasPrefix(uses, ".")', + "TestEveryActionIsPinnedToACommitRatherThanATag", + ), + ( + "the tools the PyPI jobs install are asked for a licence like a shipped dependency", + ".github/scripts/dependency_gate.py", + 'SKIPPED_MANIFESTS = frozenset({".github/requirements-semgrep.txt", ".github/requirements-pypi.txt"})', + 'SKIPPED_MANIFESTS = frozenset({".github/requirements-semgrep.txt"})', + "TestTheDependencyGateRefusesALicenceThisProjectCannotDistribute", + ), ] diff --git a/internal/guard/pypipackage_test.go b/internal/guard/pypipackage_test.go new file mode 100644 index 00000000..91d295c3 --- /dev/null +++ b/internal/guard/pypipackage_test.go @@ -0,0 +1,306 @@ +package guard + +import ( + "path/filepath" + "reflect" + "regexp" + "sort" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" +) + +// The Python package (python/testing-files-generator) runs this program and +// reads what it writes, from Python, as the npm package does from +// JavaScript. Nothing in Go calls it, so nothing in Go notices when the two +// drift apart - a system the release builds and the package cannot find, a +// system the package knows that reaches PyPI untried, an outcome the program +// declares and the package refuses. These guards read the package's files +// and hold them to the release, the workflow and the manifest. +// docs/PYTEST-2026-11-09.md. + +const pyPackage = "python/testing-files-generator" + +// pyPlatforms reads PLATFORMS from _binary.py, such as "linux x64". +func pyPlatforms(t *testing.T) []string { + t.Helper() + decl := regexp.MustCompile(`(?m)^PLATFORMS = \(([^)]*)\)`).FindStringSubmatch(npmFile(t, pyPackage+"/src/testing_files_generator/_binary.py")) + if decl == nil { + t.Fatal("_binary.py has no PLATFORMS tuple on one line - this guard no longer reads it") + } + var out []string + for _, m := range regexp.MustCompile(`"([a-z0-9]+ [a-z0-9]+)"`).FindAllStringSubmatch(decl[1], -1) { + out = append(out, m[1]) + } + if len(out) == 0 { + t.Fatal("PLATFORMS in _binary.py names no system this guard can read") + } + sort.Strings(out) + return out +} + +func TestThePythonPackageKnowsEveryPlatformTheReleaseBuilds(t *testing.T) { + name := map[string]string{"windows": "win32", "linux": "linux", "darwin": "darwin", "amd64": "x64", "arm64": "arm64"} + var want []string + for _, target := range releaseTargets(t) { + goos, goarch, _ := strings.Cut(target, "/") + if name[goos] == "" || name[goarch] == "" { + t.Fatalf("the release builds %s, and this guard does not know what the package calls it", target) + } + want = append(want, name[goos]+" "+name[goarch]) + } + sort.Strings(want) + if got := pyPlatforms(t); !reflect.DeepEqual(got, want) { + t.Errorf("the Python package looks for a binary on %v, and the release builds one for %v", got, want) + } +} + +// Every wheel with a binary is installed and used on its own system before +// anything goes to PyPI (.github/workflows/pypi.yml), the wheels for musl on +// Alpine beside those for glibc, and nothing is published until every one of +// them has. The systems tried are written out in the workflow, so they are +// held to _binary.py here: a system missing there goes to PyPI untried, and +// one the package does not know is a run that tries nothing. +func TestEverySystemThePythonPackageKnowsIsTriedBeforeAnythingIsPublished(t *testing.T) { + workflow := npmFile(t, ".github/workflows/pypi.yml") + try := regexp.MustCompile(`(?ms)^ try:\n(.*?)(?:^ [a-z][a-z-]*:\n|\z)`).FindStringSubmatch(workflow) + if try == nil { + t.Fatal("pypi.yml has no try job - this guard no longer reads it") + } + entries := regexp.MustCompile(`(?m)^ - name: .*\n((?: .*\n)+)`).FindAllStringSubmatch(try[1], -1) + glibc, musl := map[string]bool{}, map[string]bool{} + for _, e := range entries { + system := regexp.MustCompile(`(?m)^ system: ([a-z0-9-]+)$`).FindStringSubmatch(e[1]) + if system == nil { + t.Errorf("an entry of the try matrix in pypi.yml names no system:\n%s", e[1]) + continue + } + if strings.Contains(e[1], " alpine: ") { + musl[system[1]] = true + } else { + glibc[system[1]] = true + } + } + var tried, onAlpine, known, linux []string + for s := range glibc { + tried = append(tried, s) + } + for s := range musl { + onAlpine = append(onAlpine, s) + } + for _, p := range pyPlatforms(t) { + known = append(known, strings.ReplaceAll(p, " ", "-")) + if strings.HasPrefix(p, "linux ") { + linux = append(linux, strings.ReplaceAll(p, " ", "-")) + } + } + sort.Strings(tried) + sort.Strings(onAlpine) + sort.Strings(known) + sort.Strings(linux) + if len(tried) == 0 { + t.Fatal("read no system from the try matrix of pypi.yml - this guard no longer reads it") + } + if !reflect.DeepEqual(tried, known) { + t.Errorf("pypi.yml tries the wheels of %v, and the package installs a binary for %v", tried, known) + } + if !reflect.DeepEqual(onAlpine, linux) { + t.Errorf("pypi.yml tries the wheels for musl on %v, and the package has a binary for %v", onAlpine, linux) + } + + publish := regexp.MustCompile(`(?ms)^ publish:\n(.*?)(?:^ [a-z][a-z-]*:\n|\z)`).FindStringSubmatch(workflow) + if publish == nil { + t.Fatal("pypi.yml has no publish job - this guard no longer reads it") + } + needs := regexp.MustCompile(`(?m)^ needs: \[([^\]]*)\]`).FindStringSubmatch(publish[1]) + if needs == nil || !strings.Contains(", "+needs[1]+",", " try,") { + t.Errorf("the publish job of pypi.yml does not wait for the try job, so a wheel can reach PyPI untried: needs %v", needs) + } + if !strings.Contains(publish[1], " environment:\n name: ${{ needs.which.outputs.index }}\n") { + t.Error("the publish job of pypi.yml does not run in the environment the owner approves, so nothing waits for the approval") + } +} + +// pyTuple reads a tuple of string literals assigned to name on one line. +func pyTuple(t *testing.T, body, file, name string) []string { + t.Helper() + decl := regexp.MustCompile(`(?m)^` + name + ` = \(([^)]*)\)`).FindStringSubmatch(body) + if decl == nil { + t.Fatalf("%s has no %s tuple on one line", file, name) + } + var out []string + for _, m := range regexp.MustCompile(`"([a-z_]+)"`).FindAllStringSubmatch(decl[1], -1) { + out = append(out, m[1]) + } + sort.Strings(out) + return out +} + +func TestThePythonPackageHoldsTheManifestsClosedLists(t *testing.T) { + if got, want := pyTuple(t, npmFile(t, pyPackage+"/src/testing_files_generator/_testfile.py"), "_testfile.py", "OUTCOMES"), recipe.Outcomes(); !reflect.DeepEqual(got, want) { + t.Errorf("the Python package accepts the outcomes %v, and the program declares %v", got, want) + } + + plugin := npmFile(t, pyPackage+"/src/testing_files_generator/pytest_plugin.py") + var marked []string + for _, m := range regexp.MustCompile(`(?m)^ "([a-z]+)": "files `).FindAllStringSubmatch(plugin, -1) { + marked = append(marked, m[1]) + } + sort.Strings(marked) + if want := recipe.Outcomes(); !reflect.DeepEqual(marked, want) { + t.Errorf("the pytest plugin registers a marker for the outcomes %v, and the program declares %v", marked, want) + } + + readme := npmFile(t, pyPackage+"/README.md") + sentence := regexp.MustCompile(`The reasons a file can name are ([^.]*)\.`).FindStringSubmatch(readme) + if sentence == nil { + t.Fatal("the Python package's README has no sentence listing the reasons - this guard no longer reads it") + } + var reasons []string + for _, m := range regexp.MustCompile("`([a-z_]+)`").FindAllStringSubmatch(sentence[1], -1) { + reasons = append(reasons, m[1]) + } + sort.Strings(reasons) + if want := recipe.Reasons(); !reflect.DeepEqual(reasons, want) { + t.Errorf("the Python package's README lists the reasons %v, and the program declares %v", reasons, want) + } +} + +// pyLiterals are the Python files whose strings are sentences somebody reads. +// In the rest a string is a test's data or a script for a child process. +var pyLiterals = map[string]bool{ + pyPackage + "/src/testing_files_generator/__init__.py": true, + pyPackage + "/src/testing_files_generator/__main__.py": true, + pyPackage + "/src/testing_files_generator/_binary.py": true, + pyPackage + "/src/testing_files_generator/_fileset.py": true, + pyPackage + "/src/testing_files_generator/_run.py": true, + pyPackage + "/src/testing_files_generator/_testfile.py": true, + pyPackage + "/src/testing_files_generator/pytest_plugin.py": true, + pyPackage + "/release/assemble.py": true, +} + +func TestTheTextOfThePythonPackageIsAsciiWithPlainPunctuation(t *testing.T) { + sources, literals := 0, 0 + for _, name := range strings.Fields(gitOutput(t, "ls-files", "--cached", "--others", "--exclude-standard")) { + if !strings.HasPrefix(name, "python/") && !strings.HasPrefix(name, "testdata/wrappers/") { + continue + } + body := npmFile(t, name) + for n, line := range strings.Split(body, "\n") { + if i := strings.IndexFunc(line, func(r rune) bool { return r > 127 }); i >= 0 { + t.Errorf("%s:%d holds %q - text in the repository is English", name, n+1, []rune(line[i:])[0]) + } + } + if filepath.Ext(name) != ".py" { + continue + } + sources++ + prose, err := pyProse(body, pyLiterals[name]) + if err != nil { + t.Errorf("%s cannot be read for its prose: %v", name, err) + continue + } + if pyLiterals[name] { + literals++ + } + for _, p := range prose { + for _, fault := range proseFaults(p.text, false) { + t.Errorf("%s:%d holds %s", name, p.line, fault) + } + } + } + if sources == 0 || literals < len(pyLiterals) { + t.Fatalf("read %d sources and the strings of %d of %d - the package is not where this guard looks", sources, literals, len(pyLiterals)) + } +} + +// pyProse returns what a person reads in a Python file: its comments, and +// with literals the words of its strings, the code inside an f-string's +// braces left out. It reads only as much of the language as that takes, and +// says so rather than guess when a file ends inside a string. +func pyProse(src string, literals bool) ([]jsText, error) { + var out []jsText + line := 1 + add := func(at int, text string) { + if strings.TrimSpace(text) != "" { + out = append(out, jsText{at, text}) + } + } + isLetter := func(c byte) bool { return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' } + for i := 0; i < len(src); i++ { + c := src[i] + switch { + case c == '\n': + line++ + case c == '#': + end := strings.IndexByte(src[i:], '\n') + if end < 0 { + end = len(src) - i + } + add(line, src[i+1:i+end]) + i += end - 1 + case c == '\'' || c == '"': + p := i + for p > 0 && isLetter(src[p-1]) { + p-- + } + prefix := strings.ToLower(src[p:i]) + raw, format := strings.Contains(prefix, "r"), strings.Contains(prefix, "f") + quote := string(c) + if strings.HasPrefix(src[i:], strings.Repeat(quote, 3)) { + quote = strings.Repeat(quote, 3) + } + start, j := line, i+len(quote) + var text strings.Builder + for ; j < len(src) && !strings.HasPrefix(src[j:], quote); j++ { + switch ch := src[j]; { + case ch == '\\' && j+1 < len(src): + if src[j+1] == '\n' { + line++ + } + if raw { + text.WriteByte(ch) + text.WriteByte(src[j+1]) + } else { + text.WriteByte(' ') + } + j++ + case format && ch == '{' && j+1 < len(src) && src[j+1] == '{': + text.WriteByte('{') + j++ + case format && ch == '{': + // The code of a replacement field is skipped to its + // closing brace. A brace inside a string in there would + // end it early, and the package writes none. + for open := 1; open > 0 && j+1 < len(src); { + j++ + switch src[j] { + case '{': + open++ + case '}': + open-- + } + } + text.WriteByte(' ') + case ch == '\n': + if len(quote) == 1 { + return nil, errorAt(line, "a string that never ends on its line") + } + line++ + text.WriteByte('\n') + default: + text.WriteByte(ch) + } + } + if j >= len(src) { + return nil, errorAt(start, "a string that never ends") + } + if literals { + add(start, text.String()) + } + i = j + len(quote) - 1 + } + } + return out, nil +} diff --git a/npm/testing-files-generator/README.md b/npm/testing-files-generator/README.md index a75525f4..45d860f4 100644 --- a/npm/testing-files-generator/README.md +++ b/npm/testing-files-generator/README.md @@ -383,7 +383,7 @@ Plans a set and gives it back. Nothing is written until a test asks for a file. | `cacheDir` | where sets are kept. By default `TFG_CACHE_DIR`, then `node_modules/.cache/testing-files-generator`, then a directory of your own in the system's temporary directory, which nobody else can write to | | `binary` | the tfg to run. By default `TFG_BINARY`, then the one this package installed | -`fileSet` throws at once, while the spec file loads, when the binary or the recipe is not there or tfg refuses the plan. Two calls with the same spec in one process share one plan and one write. +`fileSet` throws at once, while the spec file loads, when the binary or the recipe is not there, the recipe is not a file, the directory for sets cannot be used, or tfg refuses the plan. Two calls with the same spec in one process share one plan and one write. ### A set @@ -432,12 +432,17 @@ Every message names what it is about and what to do. The ones you are most likel | `fileSet({ ... }) gives the option ...` | An option tfg does not have, often a typo. The message lists the flags tfg generate has | | `fileSet was not told which files to write` | The spec has no `preset`, `recipe` or `options` | | `The recipe option names ..., and there is no file there` | A relative recipe path is taken from the directory the tests were started in. Use `path.join(__dirname, ...)`, or `import.meta.dirname` in an ES module | +| `The recipe option names ..., which is a directory, a pipe or a device` | The path names something other than a file. Give the path of the recipe's YAML file | | `... is where this set belongs, and it does not hold it` | A kept set was changed, usually by a test writing into one of its files. Delete the directory the message names and run again | | `... has no expected outcome, so there is nothing to check it against` | `check` was called on a file with no right answer. Skip it, or decide what your system should do and write it in a recipe | | `Test timeout of 30000ms exceeded` in the first test | The set was being written inside the first test. Write it in `beforeAll` with a longer timeout | | `The binary for ... comes in the package ..., and it is not installed` | npm installed with `--omit=optional`. Install again without it | | `testing-files-generator has no binary for ...` | This system has no package of tfg. See [The tfg program](#the-tfg-program) | | `tfg wrote a manifest of version ...` | `TFG_BINARY` points at a tfg whose manifest this package cannot read. Point it at a tfg of the same version as this package, or leave it unset | +| `tfg could not be started from ...` | The system would not run that file, for the reason after the colon - a tfg for another system, or one without its execute bit. Install the package again, or point `TFG_BINARY` at a tfg this user can run | +| `Sets are kept in ..., and ...` | The directory for sets is a file, cannot be made where it would go, or is there and this user cannot read or write it, as when a run under another user made it. Set `TFG_CACHE_DIR` or `cacheDir` to a directory you can read and write | +| `... printed a plan that is not a manifest of tfg` | `TFG_BINARY`, or the `binary` option, names a program that is not tfg. Point it at a tfg of the same version as this package, or leave it unset | +| `The set was written, and it could not be moved to ...` | Another program held the new set's files open, often a virus scanner on Windows. Run again. If it repeats, set `TFG_CACHE_DIR` or `cacheDir` to a directory that program leaves alone | | `... where testing-files-generator keeps sets when it is given no other place, ...` | The directory in the system's temporary directory belongs to someone else, or others can write to it, so they could change your files. Set `TFG_CACHE_DIR` or `cacheDir` to a directory of your own | | `testing-files-generator: 1 unfinished set was left in ...` | A warning. A run stopped part way, and what it left can be deleted | diff --git a/npm/testing-files-generator/bin/tfg.js b/npm/testing-files-generator/bin/tfg.js index 09847900..209e90ca 100644 --- a/npm/testing-files-generator/bin/tfg.js +++ b/npm/testing-files-generator/bin/tfg.js @@ -11,6 +11,7 @@ const { spawn } = require('node:child_process'); const { binaryPath } = require('../lib/binary.js'); +const { notStarted } = require('../lib/run.js'); // RUNTIME in docs/CLI.md section 3 - the tool could not do its work, and the // arguments were never looked at. @@ -38,7 +39,7 @@ for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { } child.on('error', (err) => { - process.stderr.write(`tfg: tfg could not be started from ${binary}: ${err.message}\n`); + process.stderr.write(`tfg: ${notStarted(binary, err).message}\n`); process.exit(cannotRun); }); diff --git a/npm/testing-files-generator/lib/fileset.js b/npm/testing-files-generator/lib/fileset.js index 09ff987d..aa2f217b 100644 --- a/npm/testing-files-generator/lib/fileset.js +++ b/npm/testing-files-generator/lib/fileset.js @@ -65,13 +65,21 @@ function fileSet(spec = {}) { } const binary = spec.binary === undefined ? binaryPath() : named(spec.binary, 'binary', 'Point it at a tfg binary, or leave it out to use the one this package installs.'); + let recipe = ''; if (spec.recipe) { - named(path.resolve(spec.recipe), 'recipe', + const where = named(path.resolve(spec.recipe), 'recipe', `A relative path is taken from the directory the tests started in, here ${process.cwd()}, ` + 'so path.join(__dirname, ...) names a recipe beside the spec file, or path.join(import.meta.dirname, ...) in an ES module.'); + // A pipe or a device would be read here without end, and what is read + // here is gone before tfg reads it. + if (!fs.statSync(where).isFile()) { + throw new Error(`The recipe option names ${where}, which is a directory, a pipe or a device rather than a file. ` + + 'A recipe is read twice, once here to tell whether it changed and once by tfg, so it has to be a YAML file ' + + 'on the disk. Give the path of that file.'); + } + recipe = contentOf(where); } const cacheDir = path.resolve(spec.cacheDir ?? defaultCacheDir()); - const recipe = spec.recipe ? fs.readFileSync(path.resolve(spec.recipe), 'utf8') : ''; const key = digest([identity(binary), args, recipe, cacheDir]); let source = sources.get(key); if (!source) { @@ -118,6 +126,23 @@ function named(file, option, wayOut) { return file; } +// contentOf is the sha256 of a recipe's bytes, so an edited recipe is planned +// again. Read in pieces and never decoded: tfg reads the recipe itself and says +// what is wrong with one too large or not in UTF-8, in its own words. +function contentOf(file) { + const hash = crypto.createHash('sha256'); + const piece = Buffer.alloc(1 << 16); + const fd = fs.openSync(file, 'r'); + try { + for (let n; (n = fs.readSync(fd, piece, 0, piece.length, null)) > 0;) { + hash.update(piece.subarray(0, n)); + } + } finally { + fs.closeSync(fd); + } + return hash.digest('hex'); +} + function defaultCacheDir() { if (process.env.TFG_CACHE_DIR) { return process.env.TFG_CACHE_DIR; @@ -172,6 +197,23 @@ function ownTempDir({ tmp = os.tmpdir(), uid = process.getuid ? process.getuid() return dir; } +// makeCache makes the directory sets are kept in, or says where it comes from +// and how to give another. +function makeCache(cacheDir) { + try { + fs.mkdirSync(cacheDir, { recursive: true }); + } catch (err) { + throw cacheRefused(cacheDir, 'it could not be made a directory', err); + } +} + +// cacheRefused is the directory of sets this user cannot use, with the +// system's reason and how to give another. +function cacheRefused(cacheDir, what, err) { + return new Error(`Sets are kept in ${cacheDir}, and ${what} - ${err.message}. Set TFG_CACHE_DIR, or the cacheDir ` + + 'option of fileSet, to a directory this user can read and write.', { cause: err }); +} + function labelOf(spec) { const name = spec.preset ?? (spec.recipe ? path.basename(spec.recipe).replace(/\.[^.]*$/, '') : 'flags'); return name.replace(/[^A-Za-z0-9_-]+/g, '-').slice(0, 40); @@ -213,7 +255,7 @@ class Source { this.cacheDir = cacheDir; this.preset = spec.preset; this.shown = described(spec); - fs.mkdirSync(cacheDir, { recursive: true }); + makeCache(cacheDir); noteUnfinished(cacheDir); this.plan = this._planned(); // The recipe hash leaves out the seed and --clean, measured 2026-11-04, so @@ -233,12 +275,24 @@ class Source { try { runToFile(this.binary, [...this.args, `--out=${scratch(this.cacheDir)}`, '--dry-run', '--json'], out); plan = JSON.parse(fs.readFileSync(out, 'utf8')); + readable(plan); } catch (err) { - throw err instanceof TfgError ? refusedPlan(this.shown, this.preset, err) : err; + if (err instanceof TfgError) { + throw refusedPlan(this.shown, this.preset, err); + } + // The plan is the first file written in the directory of sets. + if (err.code && err.path && path.dirname(err.path) === this.cacheDir) { + throw cacheRefused(this.cacheDir, 'nothing could be written in it', err); + } + if (err instanceof SyntaxError || err instanceof NotAManifest) { + throw new Error(`${this.binary} printed a plan that is not a manifest of tfg - ${err.message}. It has to be ` + + 'tfg of the same version as this package. Install the package again, or set TFG_BINARY to the path of ' + + 'such a tfg.', { cause: err }); + } + throw err; } finally { fs.rmSync(out, { force: true }); } - readable(plan); return plan; } @@ -291,7 +345,10 @@ class Source { } if (!busy.has(err.code) || attempt === renameTries) { discard(this.binary, tmp); - throw new Error(`The set written in ${tmp} could not be moved to ${this.dir} - ${err.message}`); + throw new Error(`The set was written, and it could not be moved to ${this.dir} - ${err.message}. ` + + 'Run the tests again. If this repeats, another program may be holding its files open, such as a virus ' + + 'scanner. Set TFG_CACHE_DIR, or the cacheDir option of fileSet, to a directory that program leaves alone.', + { cause: err }); } await new Promise((resolve) => setTimeout(resolve, attempt * 100)); } @@ -341,6 +398,10 @@ function refusedPlan(shown, preset, err) { return new TfgError([], err.status, err.stderr, `${shown} could not be planned - tfg ${ended(err.status)}.\n${said(err.stderr)}`); } +// NotAManifest is a plan of this manifest version that has no run, or no list +// of files described by objects. +class NotAManifest extends Error {} + // readable refuses a manifest this package cannot read for what it says. function readable(plan) { const version = String(plan?.manifest_version ?? ''); @@ -348,6 +409,9 @@ function readable(plan) { throw new Error(`tfg wrote a manifest of version ${version || 'none'}, and this package reads version ${manifestMajor}. ` + 'Install the version of testing-files-generator that comes with that tfg, or point TFG_BINARY at a tfg of the same version as this package.'); } + if (!isObject(plan.run) || !Array.isArray(plan.files) || !plan.files.every(isObject)) { + throw new NotAManifest('it has no run, or no list of files each described by an object'); + } for (const entry of plan.files) { if (!outcomes.includes(entry.expected?.outcome)) { throw new Error(`${entry.path} expects ${JSON.stringify(entry.expected?.outcome)}, which this package does not know. ` + @@ -373,6 +437,9 @@ function mismatch(dir, plan) { } catch (err) { return `its manifest cannot be read (${err.message})`; } + if (!isObject(written)) { + return 'its manifest cannot be read (it is not an object)'; + } if (written.run?.recipe_hash !== plan.run.recipe_hash || written.run?.seed !== plan.run.seed) { return 'it was written from another recipe or seed'; } @@ -385,6 +452,11 @@ function mismatch(dir, plan) { for (let i = 0; i < plan.files.length; i++) { const want = plan.files[i]; const got = written.files[i]; + // The manifest of a kept set is on the disk, where a person or a test can + // change it, so an entry is not taken to be what tfg writes. + if (!isObject(got)) { + return `its file ${i + 1} is not described by an object where the plan has ${want.path}`; + } if (got.path !== want.path || got.bytes !== want.bytes) { return `its file ${i + 1} is ${got.path} of ${got.bytes} B where the plan has ${want.path} of ${want.bytes} B`; } @@ -404,6 +476,10 @@ function mismatch(dir, plan) { return ''; } +function isObject(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + function stale(dir, problem, details = '') { return new Error(`${dir} is where this set belongs, and it does not hold it - ${problem}. ` + 'Nothing here removes that directory, because a person may have changed it on purpose. Delete it and run again.' + @@ -454,7 +530,13 @@ function noteUnfinished(cacheDir) { return; } noted = true; - const left = fs.readdirSync(cacheDir).filter((name) => { + let names; + try { + names = fs.readdirSync(cacheDir); + } catch (err) { + throw cacheRefused(cacheDir, 'it could not be read', err); + } + const left = names.filter((name) => { const pid = Number(name.slice(unfinished.length).split('-')[0]); return name.startsWith(unfinished) && Number.isInteger(pid) && pid > 0 && !alive(pid); }); diff --git a/npm/testing-files-generator/lib/run.js b/npm/testing-files-generator/lib/run.js index 3f3db81a..b8b49594 100644 --- a/npm/testing-files-generator/lib/run.js +++ b/npm/testing-files-generator/lib/run.js @@ -34,8 +34,12 @@ function said(stderr) { return stderr.trim() || 'tfg said nothing about why.'; } +// notStarted is a binary the system would not start, with the system's reason +// and the way out. The reason stays: EACCES and ENOEXEC are what tell a binary +// without its execute bit from one for another system. function notStarted(binary, err) { - return new Error(`tfg could not be started from ${binary}: ${err.message}`); + return new Error(`tfg could not be started from ${binary}: ${err.message}. It has to be a tfg for this system ` + + 'that this user can run. Install the package again, or set TFG_BINARY to the path of such a tfg.', { cause: err }); } // runToFile runs tfg to the end, writing its standard output to a file. A file @@ -99,4 +103,4 @@ function run(binary, args) { }); } -module.exports = { TfgError, ended, run, runToFile, said }; +module.exports = { TfgError, ended, notStarted, run, runToFile, said }; diff --git a/npm/testing-files-generator/test/fileset.test.js b/npm/testing-files-generator/test/fileset.test.js index 5b93e172..931e071f 100644 --- a/npm/testing-files-generator/test/fileset.test.js +++ b/npm/testing-files-generator/test/fileset.test.js @@ -121,6 +121,107 @@ test('a set changed by hand is refused, and left where it is', async (t) => { assert.ok(fs.existsSync(path.join(dir, 'manifest.json')), 'the changed set was removed'); }); +test('a kept manifest, or a file in it, that is not an object is refused like any changed set', async (t) => { + const cacheDir = tempDir(t); + const dir = (await node(writer(cacheDir))).stdout.trim(); + const manifest = path.join(dir, 'manifest.json'); + const written = JSON.parse(fs.readFileSync(manifest, 'utf8')); + const first = written.files[0].path; + fs.writeFileSync(manifest, JSON.stringify({ ...written, files: [null, ...written.files.slice(1)] })); + const entry = await node(writer(cacheDir)); + assert.equal(entry.status, 3); + assert.ok(entry.stderr.includes(`does not hold it - its file 1 is not described by an object where the plan has ${first}. ` + + 'Nothing here removes that directory'), entry.stderr); + fs.writeFileSync(manifest, 'null'); + const whole = await node(writer(cacheDir)); + assert.equal(whole.status, 3); + assert.ok(whole.stderr.includes('does not hold it - its manifest cannot be read (it is not an object).'), whole.stderr); +}); + +test('a cache directory that cannot be made says how to give another', (t) => { + const taken = path.join(tempDir(t), 'a-file'); + fs.writeFileSync(taken, ''); + assert.throws(() => fileSet({ ...small, binary: tfg(), cacheDir: taken }), + (err) => err.message.startsWith(`Sets are kept in ${taken}, and it could not be made a directory - `) && + err.message.includes('TFG_CACHE_DIR') && err.message.includes('cacheDir')); +}); + +const posixUser = process.getuid && process.getuid() !== 0 + ? false : 'the modes of a directory hold on Linux and macOS, for a user that is not root'; + +for (const [mode, what] of [[0o333, 'it could not be read'], [0o555, 'nothing could be written in it']]) { + test(`a cache directory this user cannot use says how to give another - ${what}`, { skip: posixUser }, async (t) => { + // A directory that is there already passes the making of it, as one left + // by a run under another user does. + const there = path.join(tempDir(t), 'sets'); + fs.mkdirSync(there); + const plan = `require(${pkg}).fileSet({ ...${JSON.stringify(small)}, cacheDir: ${JSON.stringify(there)} })`; + fs.chmodSync(there, mode); + let said; + try { + said = await node(plan, { env: { TEST_WORKER_INDEX: undefined } }); + } finally { + fs.chmodSync(there, 0o755); + } + assert.notEqual(said.status, 0); + assert.ok(said.stderr.includes(`Sets are kept in ${there}, and ${what} - EACCES`), said.stderr); + assert.ok(said.stderr.includes('TFG_CACHE_DIR') && said.stderr.includes('cacheDir'), said.stderr); + }); +} + +for (const printed of ['', 'not json', '{"manifest_version": "1.0"}', '{"manifest_version": "1.0", "run": {}, "files": [null]}']) { + test(`a binary that prints no manifest for the plan is named, with what to do - ${JSON.stringify(printed)}`, { + skip: process.platform === 'win32' ? 'a script runs as a binary on Linux and macOS' : false, + }, (t) => { + // TFG_BINARY can point at any program, and one that ends well and prints + // anything but a manifest is named rather than a parser's error. + const dir = tempDir(t); + const out = path.join(dir, 'printed'); + fs.writeFileSync(out, printed); + const binary = path.join(dir, 'not-tfg'); + fs.writeFileSync(binary, `#!/bin/sh\ncat '${out}'\n`, { mode: 0o755 }); + const cacheDir = path.join(dir, 'sets'); + assert.throws(() => fileSet({ ...small, binary, cacheDir }), + (err) => err.message.startsWith(`${binary} printed a plan that is not a manifest of tfg - `) && + err.message.includes('TFG_BINARY') && err.message.includes('Install the package again')); + assert.deepEqual(entries(cacheDir), [], 'the plan was left in the cache'); + }); +} + +test('a set that cannot be moved to its name says what to do, and leaves nothing', async (t) => { + const cacheDir = tempDir(t); + const set = plannedIn(cacheDir); + t.mock.method(fs, 'renameSync', () => { + throw Object.assign(new Error('ENAMETOOLONG: name too long'), { code: 'ENAMETOOLONG' }); + }); + await assert.rejects(set.write(), (err) => { + assert.match(err.message, /^The set was written, and it could not be moved to .+ - ENAMETOOLONG: name too long\. Run the tests again\./); + assert.ok(err.message.includes('TFG_CACHE_DIR') && err.message.includes('cacheDir'), err.message); + assert.equal(err.cause.code, 'ENAMETOOLONG'); + return true; + }); + assert.deepEqual(entries(cacheDir), [], 'the set that was written is left in the cache'); +}); + +test('a recipe that is a directory is refused before anything reads it', (t) => { + const dir = tempDir(t); + assert.throws(() => fileSet({ recipe: dir, binary: tfg(), cacheDir: path.join(dir, 'sets') }), + (err) => err.message.startsWith(`The recipe option names ${dir}, which is a directory, a pipe or a device`) && + err.message.includes('Give the path of that file')); +}); + +test('an edited recipe is planned again in the same process', (t) => { + const dir = tempDir(t); + const recipe = path.join(dir, 'set.yaml'); + const cacheDir = path.join(dir, 'sets'); + fs.writeFileSync(recipe, 'version: 1\ntargets:\n - id: one\n format: txt\n size: 1kb\n'); + const first = fileSet({ recipe, binary: tfg(), cacheDir }); + fs.writeFileSync(recipe, 'version: 1\ntargets:\n - id: one\n format: txt\n size: 2kb\n'); + const second = fileSet({ recipe, binary: tfg(), cacheDir }); + assert.deepEqual(first.files.map((f) => f.bytes), [1024]); + assert.deepEqual(second.files.map((f) => f.bytes), [2048]); +}); + test('a file of the set cut short by hand is refused', async (t) => { const cacheDir = tempDir(t); const dir = (await node(writer(cacheDir))).stdout.trim(); @@ -223,7 +324,7 @@ test('the flags this package sets itself cannot be set again', (t) => { }); test('a manifest of another major, or an outcome this package does not know, is refused', () => { - const plan = { manifest_version: '1.0', files: [{ path: 'a', expected: { outcome: 'accept' } }] }; + const plan = { manifest_version: '1.0', run: {}, files: [{ path: 'a', expected: { outcome: 'accept' } }] }; readable(plan); assert.throws(() => readable({ ...plan, manifest_version: '2.0' }), /manifest of version 2\.0, and this package reads version 1/); assert.throws(() => readable({ ...plan, files: [{ path: 'a', expected: { outcome: 'quarantine' } }] }), /"quarantine", which this package does not know/); @@ -243,6 +344,16 @@ test('npx tfg is tfg - the same output and the same exit code', async () => { assert.equal(viaPackage.stderr, direct.stderr); }); +test('npx tfg with a binary the system will not start names it, with what to do and the code of a tool that could not run', async (t) => { + const bin = path.join(__dirname, '..', 'bin', 'tfg.js'); + const binary = path.join(tempDir(t), 'not-tfg'); + fs.writeFileSync(binary, 'not a program\n', { mode: 0o644 }); + const refused = await node('', { file: bin, args: ['version'], env: { TFG_BINARY: binary } }); + assert.equal(refused.status, 1); + assert.ok(refused.stderr.startsWith(`tfg: tfg could not be started from ${binary}: `), refused.stderr); + assert.ok(refused.stderr.includes('TFG_BINARY'), refused.stderr); +}); + // The last place sets go is the system's temporary directory, which on Linux // every user of the machine shares. Asked of a directory made here, with the // user's id passed in, so another user can be played without one. diff --git a/npm/testing-files-generator/test/run.test.js b/npm/testing-files-generator/test/run.test.js index a7633ec9..b666aee3 100644 --- a/npm/testing-files-generator/test/run.test.js +++ b/npm/testing-files-generator/test/run.test.js @@ -30,6 +30,15 @@ test('a child that says a great deal on its error stream runs to its own end, an err.stderr.length <= 64 * 1024); }); +test('a binary the system will not start is named with the reason and what to do, from both ways of running it', async (t) => { + const binary = path.join(tempDir(t), 'not-tfg'); + fs.writeFileSync(binary, 'not a program\n', { mode: 0o644 }); + const named = (err) => err.message.startsWith(`tfg could not be started from ${binary}: `) && + err.message.includes('Install the package again') && err.message.includes('TFG_BINARY'); + assert.throws(() => runToFile(binary, ['version'], path.join(tempDir(t), 'out.json')), named); + await assert.rejects(run(binary, ['version']), named); +}); + test('the output file is made new and private, and a file already under its name is never written through', (t) => { const [binary, args] = child('process.stdout.write("planned")'); const out = path.join(tempDir(t), 'out.json'); diff --git a/npm/testing-files-generator/test/vectors.test.js b/npm/testing-files-generator/test/vectors.test.js new file mode 100644 index 00000000..9e2e1e19 --- /dev/null +++ b/npm/testing-files-generator/test/vectors.test.js @@ -0,0 +1,66 @@ +'use strict'; + +// What a file says of itself and what check makes of what a system did, read +// from the file the Python package's tests read as well, so a sentence changed +// in one package and not in the other fails here or there. + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { TestFile } = require('../lib/testfile.js'); + +const vectors = JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', '..', 'testdata', 'wrappers', 'vectors.json'), 'utf8')); + +function file(name) { + const entry = vectors.entries[name]; + assert.ok(entry, `vectors.json has no entry ${name}`); + return new TestFile({ write: async () => '/set' }, entry); +} + +test('every file says what the vectors say it says', () => { + assert.ok(vectors.files.length >= 4); + for (const want of vectors.files) { + const f = file(want.entry); + assert.equal(f.title, want.title, want.entry); + assert.deepEqual(f.details.tag, want.marks.map((m) => `@${m}`), want.entry); + assert.equal(f.whyUnspecified, want.why_unspecified, want.entry); + } +}); + +test('check makes of every observation what the vectors say it makes', () => { + const kinds = new Set(); + for (const c of vectors.checks) { + const label = `${c.entry} ${JSON.stringify(c.observed)}`; + kinds.add(c.result); + const f = file(c.entry); + let thrown; + try { + f.check(c.observed); + } catch (err) { + thrown = err; + } + switch (c.result) { + case 'pass': + assert.equal(thrown, undefined, label); + break; + case 'mismatch': + assert.equal(thrown?.name, 'ExpectationError', label); + assert.equal(thrown.message, c.message, label); + break; + case 'unspecified': + assert.ok(thrown && thrown.name === 'Error', label); + assert.match(thrown.message, /has no expected outcome, so there is nothing to check it against/, label); + break; + case 'bad-input': + assert.ok(thrown instanceof TypeError, label); + break; + default: + assert.fail(`vectors.json has a result ${c.result} this test does not know`); + } + } + // Every kind is there, so a vectors file that lost one is not green by + // having nothing left to compare. + assert.deepEqual([...kinds].sort(), ['bad-input', 'mismatch', 'pass', 'unspecified']); +}); diff --git a/python/pytest-testing-files-generator/LICENSE b/python/pytest-testing-files-generator/LICENSE new file mode 100644 index 00000000..d0a1fa14 --- /dev/null +++ b/python/pytest-testing-files-generator/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/python/pytest-testing-files-generator/README.md b/python/pytest-testing-files-generator/README.md new file mode 100644 index 00000000..bb02622e --- /dev/null +++ b/python/pytest-testing-files-generator/README.md @@ -0,0 +1,33 @@ +# pytest-testing-files-generator + +The pytest plugin of [testing-files-generator](https://pypi.org/project/testing-files-generator/): real files of exact sizes and formats for your tests, and what the system under test should do with each one. + +This package has no code of its own. It installs testing-files-generator of the same version, and pytest loads its plugin on its own. Installing either package gives the same thing: + +```bash +pip install pytest-testing-files-generator +``` + +```python +import re + +import pytest +from playwright.sync_api import expect +from testing_files_generator import file_set + +uploads = file_set(preset="upload-validation", options={"limit": "2mb", "bulk": 0}) + + +@pytest.mark.parametrize("file", uploads.params()) +def test_upload(file, page): + page.goto("/upload") + page.get_by_label("File").set_input_files(file.path()) + page.get_by_role("button", name="Upload").click() + status = page.get_by_role("status") + expect(status).to_have_text(re.compile("Uploaded|Refused")) + file.check("accept" if "Uploaded" in status.text_content() else "reject") +``` + +Everything else is in [the README of testing-files-generator](https://github.com/donislawdev/TestingFilesGenerator/tree/main/python/testing-files-generator#readme). + +This package is under the Mozilla Public License 2.0, in LICENSE. diff --git a/python/pytest-testing-files-generator/pyproject.toml b/python/pytest-testing-files-generator/pyproject.toml new file mode 100644 index 00000000..c4e6d27c --- /dev/null +++ b/python/pytest-testing-files-generator/pyproject.toml @@ -0,0 +1,30 @@ +# A package with no code of its own. It installs testing-files-generator of +# the same version, whose pytest plugin pytest loads on its own, and is here so +# that pytest's list of plugins and a search for a pytest plugin find it. +# release/assemble.py of testing-files-generator writes its wheel, and pins +# testing-files-generator to the version of the release. The version here is +# never raised by hand. + +[project] +name = "pytest-testing-files-generator" +version = "0.0.0.dev0" +description = "pytest plugin for testing-files-generator: real files of exact sizes and formats for tests, and what the system under test should do with each one." +readme = "README.md" +requires-python = ">=3.10" +license = "MPL-2.0" +license-files = ["LICENSE"] +authors = [{ name = "DonislawDev" }] +keywords = ["pytest", "pytest-plugin", "test files", "fixtures", "file upload", "upload testing", "qa"] +classifiers = [ + "Development Status :: 4 - Beta", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Testing", +] +dependencies = ["testing-files-generator", "pytest>=8"] + +[project.urls] +Homepage = "https://github.com/donislawdev/TestingFilesGenerator/tree/main/python/testing-files-generator#readme" +Source = "https://github.com/donislawdev/TestingFilesGenerator" +Issues = "https://github.com/donislawdev/TestingFilesGenerator/issues" diff --git a/python/testing-files-generator/LICENSE b/python/testing-files-generator/LICENSE new file mode 100644 index 00000000..d0a1fa14 --- /dev/null +++ b/python/testing-files-generator/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/python/testing-files-generator/README.md b/python/testing-files-generator/README.md new file mode 100644 index 00000000..3dcca595 --- /dev/null +++ b/python/testing-files-generator/README.md @@ -0,0 +1,443 @@ +# testing-files-generator + +Real files of exact sizes and formats for your tests, and what the system under test should do with each one. + +A PDF of exactly 1 MiB and one byte is a real PDF, so a server that looks inside it can only turn it away for its size. A file made with `b"\0" * size` is zeros, so the same server turns it away for what it holds, and a test meant to prove a size limit passes on a server that has none. + +## Contents + +- [What it is](#what-it-is) +- [Install](#install) +- [Your first test](#your-first-test) +- [The words this package uses](#the-words-this-package-uses) +- [Telling the set what your system did](#telling-the-set-what-your-system-did) +- [Files with no right answer](#files-with-no-right-answer) +- [Choosing the files](#choosing-the-files) +- [When the files are written, and where they are kept](#when-the-files-are-written-and-where-they-are-kept) +- [In CI](#in-ci) +- [Without pytest](#without-pytest) +- [Reference](#reference) +- [When something goes wrong](#when-something-goes-wrong) +- [The tfg program](#the-tfg-program) +- [Licences](#licences) + +## What it is + +tfg, the Testing Files Generator, is a free and open source program that writes real files for tests: a JPEG that opens in a viewer, a PDF a reader can read, a CSV a spreadsheet imports, each at exactly the size you ask for. Beside the files it writes a manifest, a list that says for every file what it is for and what a system should do with it. + +This package runs tfg from your tests: + +1. You choose a set of files. Usually that is a preset, a set tfg knows by name that answers one question, such as "does my upload form take what it should and turn the rest away?". +2. The package gives you one test for each file, with an id that says what the file should meet. +3. Each test hands its file to your system the way your system takes files, and ends with one line that tells the set what your system did. +4. When that is not what the file expects, the test fails with a sentence that says what the difference means. + +tfg comes with this package, so there is nothing else to install. It works offline and never connects to the network. + +## Install + +```bash +pip install testing-files-generator +``` + +- **Python 3.10 or later.** pytest finds the package's plugin on its own. `pip install pytest-testing-files-generator` installs the same thing, under the name pytest's list of plugins knows. +- **The tfg program is inside the wheel pip picks for your system**: Windows on x64 and Arm, Linux on x64 and Arm (glibc and Alpine), macOS on Apple silicon. Nothing runs when the package is installed, and nothing is downloaded when the tests run. The program is the one from the tfg release of the same version: signed on Windows, and on macOS signed and notarised inside the app it was signed in. +- **pip and uv both install it.** +- On any other system pip installs the package without the program, and the first call says so and what to do. See [The tfg program](#the-tfg-program). + +## Your first test + +Say your app has an upload form at `/upload` that takes one file at a time, has a size limit of `2mb`, and says `Uploaded` when it takes a file. With [pytest-playwright](https://pypi.org/project/pytest-playwright/), this module gives it one test for each file of the `upload-validation` preset: + +```python +# tests/test_upload.py +import re + +import pytest +from playwright.sync_api import expect +from testing_files_generator import file_set + +# 1. The set, planned around the limit your form declares. Nothing is written yet. +# bulk 0 leaves out the mass upload, which a form taking one file at a time has no use for. +uploads = file_set(preset="upload-validation", options={"limit": "2mb", "bulk": 0}) + + +# 2. One test for each file of the set. A file whose right answer is your own policy +# is skipped, and the report says why. +@pytest.mark.parametrize("file", uploads.params()) +def test_upload(file, page): + # 3. Upload the file the way a person does. The first call to file.path() writes the set. + page.goto("/upload") + page.get_by_label("File").set_input_files(file.path()) + page.get_by_role("button", name="Upload").click() + + # 4. Tell the set what your app did. check fails the test when the file expected something else. + status = page.get_by_role("status") + expect(status).to_have_text(re.compile("Uploaded|Refused")) + file.check("accept" if "Uploaded" in status.text_content() else "reject") +``` + +Steps 3 and 4 are the only part about your app. Change them to however your app takes a file and shows what it did with it. Keep step 4 waiting for an answer your app gives on purpose, `Uploaded` or `Refused` here, so a page that shows an error fails the test instead of passing as a refusal. + +**See the tests before you run them.** Listing them writes none of the files: + +```bash +pytest --collect-only -q +``` + +Each test is named by its file and what the file expects: + +``` +tests/test_upload.py::test_upload[2mb_under_1b.jpg - expected accept] +tests/test_upload.py::test_upload[2mb_at_limit.jpg - expected accept] +tests/test_upload.py::test_upload[2mb_over_1b.jpg - expected reject (size_limit)] +tests/test_upload.py::test_upload[empty.jpg - expected unspecified (size_zero)] +tests/test_upload.py::test_upload[denied.exe - expected reject (extension_rule)] +tests/test_upload.py::test_upload[pdf_as_jpg.jpg - expected reject (mime_mismatch)] +... +``` + +**Run them** with `pytest`. The first test that asks for a file writes the whole set. A test fails when your app does something other than what its file expects, and the failure says what that means: + +``` +ExpectationError: 2mb_at_limit.jpg was turned away, and it should have been taken (size_limit). +A JPEG file of exactly 2mb, the limit itself. Your system should take it. A refusal here usually +means the comparison is off by one, or your system counts a kilobyte as 1000 bytes where this +set counts 1024. +``` + +A file with no right answer is skipped, never passed, and `pytest -rs` gives the reason, such as `empty.jpg has no expected outcome - what your system does with it is its own policy`. [Files with no right answer](#files-with-no-right-answer) says what to do once you have decided. + +**Run part of the set.** Every test carries its expected outcome and its file's group as markers: + +```bash +pytest -m reject +pytest -m size-limit +pytest -k "pdf" +``` + +pytest knows a marker by its name alone. A group named after one of pytest's own markers, such as `skip` or `xfail`, would change how its tests run, so it is refused with a message that names the group. A group named after a marker of a plugin you use, such as `timeout` of pytest-timeout, is that plugin's marker too. Give such a group another name in the recipe. + +## The words this package uses + +| word | what it means | +|---|---| +| set | the files one `file_set` call describes, each with what it should meet | +| preset | a set tfg knows by name, built around one question. `upload-validation` is one | +| recipe | a YAML file that describes a set file by file. A preset can be turned into one and edited | +| plan | the files of a set and what each expects, known before any of them is written | +| manifest | `manifest.json`, which tfg writes beside a set: every file with its size, its hash and what it expects | +| outcome | what your system should do with a file: `accept`, `reject`, `sanitize` or `unspecified` | +| reason | the rule a file is about, such as `size_limit` or `mime_mismatch`, from a closed list | +| group | the part of a set a file belongs to, such as `size-limit` or `hostile-names` | +| target | one entry of a recipe. A target can make several files | +| purpose | one sentence on what a file is for and what it means when a system gets it wrong. A failure shows it | + +The four outcomes: + +- `accept` - your system should take the file. +- `reject` - your system should turn it away. A file to be turned away usually names the rule as its reason. +- `sanitize` - your system should take the file and change it, for example store it under a safe name. No preset expects this today, and a recipe can. +- `unspecified` - there is no right answer until you decide one, as for an empty file. These files are skipped. + +## Telling the set what your system did + +`file.check` compares what your system did with what the file expects, and fails the test when the two differ. It takes one of three words: + +```python +file.check("accept") # your system took the file +file.check("reject") # it turned the file away +file.check("sanitize") # it took the file and changed it +``` + +When your system says why it turned a file away, pass the reason too, and a file turned away for the wrong rule fails as well. Here the same files go straight to the server with [requests](https://pypi.org/project/requests/), and the server answers with a JSON body holding a `reason`: + +```python +import pytest +import requests + + +@pytest.mark.parametrize("file", uploads.params()) +def test_upload_api(file): + with open(file.path(), "rb") as body: + response = requests.post("http://localhost:3000/upload", files={"file": (file.name, body)}) + assert response.status_code < 500, "a server error is not a refusal" + file.check("accept" if response.ok else {"outcome": "reject", "reason": response.json().get("reason")}) +``` + +- **A server error is not a refusal.** An answer of 500 or more fails the test before `check`, so a file that breaks your server is found rather than counted as turned away. +- **A reason is compared only for a file that should be turned away, and only when you give one.** A reason is a name from the list in [Reference](#reference), such as `size_limit`. If your system has codes of its own, turn them into those names, or pass the outcome alone. +- **`check` never passes a file with no right answer.** On an `unspecified` file it raises, so such a file cannot pass by accident. +- **What your system did can come from anywhere a test can read**: the text on the page, an HTTP status, the body of a response, a row in a database. +- **The same files can go through the page and straight to the server.** The checks of a page run in the browser and anything other than that page skips them, so a limit only the page enforces shows up only in the tests that go to the server. + +## Files with no right answer + +Some files have no answer the set could know: whether an empty file should go through, whether `PHOTO.JPG` is the same file as `photo.jpg`, whether a name in Japanese is stored as it is. Their outcome is `unspecified`, and `params()` skips them with the reason in the report. In `filename-handling` nearly every file is like this, because what a system does with a strange name is its own rule. + +Once you have decided what your system should do with such a file, write that down, and the file is checked like any other: + +1. **Turn the preset into a recipe**, a file of your own to edit and commit: + + ```bash + tfg preset eject upload-validation --limit 2mb --bulk 0 -o tests/uploads.yaml + ``` + +2. **Find the file in the recipe** and change `unspecified` to what your system does. Change its `purpose` too, because that is the sentence a failure shows: + + ```yaml + - id: empty + format: txt + count: 1 + size: 0 + name: empty.jpg + group: degenerate + purpose: "An empty file. Our form turns it away, because an empty upload is always a mistake." + expected: + outcome: reject + reason: size_zero + ``` + +3. **Use the recipe instead of the preset:** + + ```python + from pathlib import Path + + uploads = file_set(recipe=Path(__file__).parent / "uploads.yaml") + ``` + +`empty.jpg` is then a test like any other, named `empty.jpg - expected reject (size_zero)`, and it fails when your form takes an empty file. + +## Choosing the files + +### Presets + +`file_set(preset=...)` takes one of these: + +| preset | the question it answers | +|---|---| +| `upload-validation` | Does my upload form take what it should and turn the rest away? | +| `size-boundaries` | Is a size limit enforced exactly where it is declared? | +| `filename-handling` | Will my system store, show and give back a file name it did not expect? | +| `tabular-import` | Does my table import survive what real tools export? | +| `text-encoding` | Does my reader know which encoding a file is in, or is it guessing? | +| `empty-and-minimal` | Does a file that is valid and as small as the format allows get through? | + +`tfg preset show ` lists what a preset takes, what it would write and what it typically catches: + +``` +$ tfg preset show upload-validation +... +parameters: + --limit a size such as 2mb, or a plain byte count, default 10mb + --allow format ids separated by commas, default jpg,png,pdf + --deny extensions separated by commas, default svg,html,exe,sh + --far-over one of: 10x, 2x, off, default 2x + --bulk whole number of files from 0 to 10000, default 50 +... +``` + +### Options + +Every parameter there is an option of `file_set`, written without the dashes: + +```python +file_set(preset="upload-validation", options={"limit": "5mb", "allow": "jpg,png", "far_over": "off", "bulk": 0}) +``` + +- **`far_over` and `far-over` are the same option.** A name with underscores is the flag with hyphens, so `options=dict(far_over="off")` works too. +- **A list is one string with commas**, as `allow` above. +- **Sizes count in 1024s**, so `2mb` is 2 097 152 bytes. If your system means 2 000 000 bytes by 2 MB, give the number of bytes, `"limit": 2000000`, and the files are named after it, such as `2000000_over_1b.jpg`. +- **The same options give the same files**, byte for byte, on every run and every machine, for as long as the major version of tfg stays the same. `"seed": 7` gives other bytes, and those are again the same on every run. +- **Options also take the flags of `tfg generate` itself**, such as `seed` or `clean`. `True` stands for a flag with no value, `"clean": True`, and a list gives a flag that repeats, `"set": ["width=640", "height=480"]`. + +### Part of a set + +`only` and `except_` take part of a set by `group`, `target` or `outcome`, one name or a list of them. `except_` has its underscore because `except` is a word of Python: + +```python +refusals = uploads.only(outcome="reject") +single_files = uploads.except_(group=["bulk-upload", "hostile-names"]) +``` + +Every part shares the set's one write. `paths()` gives the files of a part together, for a form that takes several at once: + +```python +uploads = file_set(preset="upload-validation", options={"limit": "2mb"}) + + +def test_the_mass_upload_is_taken_whole(page): + page.goto("/upload") + page.get_by_label("Files").set_input_files(uploads.only(group="bulk-upload").paths()) + ... +``` + +### Your own files + +A recipe describes a set of your own, file by file, and the [recipe section of the tfg README](https://github.com/donislawdev/TestingFilesGenerator#-recipes) says how to write one. A relative recipe path is taken from the directory the tests were started in, so `Path(__file__).parent / "uploads.yaml"` names one beside the test module wherever the tests are started from. + +A few files of one kind need no recipe: + +```python +file_set(options={"format": "png", "size": "5mb", "count": 3, "expected": "accept"}) +``` + +Files made this way expect nothing unless `expected` says what, so without it every one of them is skipped. `tfg formats` lists the formats. + +## When the files are written, and where they are kept + +``` +pytest collects the tests file_set plans the set: tfg lists the files and writes none of them +the first file.path() tfg writes the whole set, and the first copy finished is the one kept +every later run the set is taken from where it was kept, once every byte of it is checked +``` + +1. **While pytest collects the tests, none of the set is written.** `file_set` asks tfg for the plan, and the tests are made from it. That is why `--collect-only` writes none of the files. The plan comes back through a short-lived file in the directory sets are kept in, so that directory has to be one the tests can write to, even for `--collect-only`. +2. **The first test that asks for a path writes the whole set.** Workers of [pytest-xdist](https://pypi.org/project/pytest-xdist/) that ask at the same moment each write a copy of their own. The first one finished is kept and the others are removed, so for a moment the disk holds a copy for each of those workers. A worker that asks later takes the kept set. The `upload-validation` set at its defaults, 71 files and 115 MB, was written in under a second on the machine it was measured on. +3. **Every later run takes the same files.** Under pytest a set is kept in the run's `.pytest_cache`, which pytest keeps out of git, and `pytest --cache-clear` empties. `TFG_CACHE_DIR`, or the `cache_dir` argument, names another directory. Before a kept set is used, every byte of it is checked against its manifest, which took under a tenth of a second for the same set. A set a test wrote into, or a file someone changed, is refused with the names of the files that differ. + +**Do not write into a file the set gave you.** Copy it first. A file changed in place is changed for every later run, and the next one refuses the set. + +**To look at the files yourself**, `uploads.write()` gives the directory of the set. It holds the files, `manifest.json` and `manifest.instructions.md`, which says in plain words what each file is for. + +**A large set is better written before the tests start**, so the time it takes does not count against the first test: + +```python +@pytest.fixture(scope="session", autouse=True) +def written_sets(): + uploads.write() +``` + +**Nothing is ever removed on its own.** Delete the directory whenever you like, and the next run writes the sets again. A run stopped part way leaves an unfinished set, which nothing uses, and the next run names it in a warning so it does not sit there unseen. + +## In CI + +- **Nothing to set up.** `pip install` puts tfg in with the package, and the tests write their sets when they run. +- **A fresh checkout has no `.pytest_cache`**, so every CI run writes its sets again. For the `upload-validation` set at its defaults that is under a second on the machine it was measured on. To keep sets from one run to the next, point `TFG_CACHE_DIR` at a directory your CI keeps. +- **Runners on Windows, Linux and macOS on Apple silicon all get a wheel with tfg.** An Intel macOS runner gets none, see [The tfg program](#the-tfg-program). + +## Without pytest + +`file_set` gives plain data and plain paths, and imports nothing from pytest until `params()` is called. Any runner that can make tests in a loop works the same way. With unittest: + +```python +import unittest + +import requests +from testing_files_generator import file_set + +uploads = file_set(preset="upload-validation", options={"limit": "2mb", "bulk": 0}) + + +class Uploads(unittest.TestCase): + def test_every_file(self): + for file in uploads.files: + with self.subTest(file.title): + if file.unspecified: + self.skipTest(file.why_unspecified) + with open(file.path(), "rb") as body: + response = requests.post("http://localhost:3000/upload", files={"file": (file.name, body)}) + self.assertLess(response.status_code, 500, "a server error is not a refusal") + file.check("accept" if response.ok else "reject") +``` + +Outside pytest, sets are kept in a directory of your own: `%LOCALAPPDATA%\testing-files-generator` on Windows, `~/Library/Caches/testing-files-generator` on macOS, and `$XDG_CACHE_HOME/testing-files-generator` or `~/.cache/testing-files-generator` on Linux. + +## Reference + +### `file_set(preset=None, recipe=None, options=None, cache_dir=None, binary=None)` + +Plans a set and gives it back. Nothing is written until a test asks for a file. + +| argument | what it is | +|---|---| +| `preset` | a preset by name, as `tfg preset list` lists them | +| `recipe` | the path of a recipe file. A relative path is taken from the directory the tests were started in | +| `options` | a dict of the parameters of the preset and the flags of `tfg generate`, without the dashes | +| `cache_dir` | where sets are kept. By default `TFG_CACHE_DIR`, then the `.pytest_cache` of the run, then a directory of your own as [Without pytest](#without-pytest) says | +| `binary` | the tfg to run. By default `TFG_BINARY`, then the one this package installed | + +`file_set` raises at once, while the tests are collected, when the binary or the recipe is not there, the recipe is not a file, the directory for sets cannot be used, or tfg refuses the plan. Two calls with the same arguments in one process share one plan and one write. + +### A set + +| member | what it is | +|---|---| +| `files` | every file of the set, in order. A set also iterates over its files and has a `len` | +| `plan` | the manifest tfg gave for the plan: every file, nothing written yet | +| `params()` | one `pytest.param` for each file, for `@pytest.mark.parametrize`: the title as the id, the expected outcome and the group as markers, and a skip for a file with no right answer | +| `only(...)`, `except_(...)` | part of the set, by `group`, `target` or `outcome` | +| `write()` | writes the whole set, once for the process, and gives its directory | +| `paths()` | writes the set if it is not written yet, and gives the paths of this part's files | + +### A file + +| member | what it is | +|---|---| +| `title` | the file's path and what it expects, such as `2mb_over_1b.jpg - expected reject (size_limit)`. Unique within a set | +| `marks` | the names of its markers: the expected outcome, and the group with anything other than a letter, digit, underscore or hyphen made a hyphen | +| `unspecified` | `True` for a file with no right answer | +| `why_unspecified` | the sentence a skip shows | +| `path()` | writes the set the first time it is called, and gives this file's absolute path | +| `check(observed)` | `"accept"`, `"reject"`, `"sanitize"` or `{"outcome": ..., "reason": ...}`. Raises `ExpectationError` when it is not what the file expects | +| `name`, `relative_path` | the file's name, and its path within the set | +| `format`, `bytes` | the format, such as `jpg`, and the exact size in bytes | +| `group`, `target`, `purpose` | the file's group, the recipe target that made it, and its purpose | +| `expected` | `{"outcome", "reason", "detail", "confidence"}` as the manifest has it | +| `entry` | the file's whole entry in the manifest | + +The reasons a file can name are `content_malformed`, `count_limit`, `dimensions_limit`, `duplicate`, `encoding_invalid`, `extension_rule`, `filename_invalid`, `filename_too_long`, `filename_traversal`, `malware_signature`, `mime_mismatch`, `nesting_depth`, `none`, `size_limit` and `size_zero`. + +### Errors + +- **`ExpectationError`**, a kind of `AssertionError`, is what `check` raises when your system did something other than what the file expects. `expected` and `observed` hold the two. +- **`TfgError`** is what `file_set`, `write`, `path` and `paths` raise when tfg itself refuses or fails. `status` is the code tfg ended with, or the name of the signal that stopped it, and `stderr` is the end of everything tfg said. The codes are listed in [the tfg README](https://github.com/donislawdev/TestingFilesGenerator), under Using it in CI. +- **`UnfinishedSetWarning`** is the warning about a set a stopped run left behind. + +### `binary_path()` + +The path of the tfg binary this package runs. + +## When something goes wrong + +Every message names what it is about and what to do. The ones you are most likely to meet: + +| the message starts with | what it means and what to do | +|---|---| +| `... was taken, and it should have been turned away` | Your system did something other than what the file expects. That is the test doing its job, and the next line says what the difference means | +| `file_set(...) could not be planned` | tfg refused what the call asks for. The line under it is tfg's own sentence, which names the value and the way out | +| `file_set(...) gives the option ...` | An option tfg does not have, often a typo. The message lists the flags tfg generate has | +| `file_set was not told which files to write` | The call has no `preset`, `recipe` or `options` | +| `The recipe option names ..., and there is no file there` | A relative recipe path is taken from the directory the tests were started in. Use `Path(__file__).parent / ...` | +| `The recipe option names ..., which is a directory, a pipe or a device` | The path names something other than a file. Give the path of the recipe's YAML file | +| `The group ... becomes the pytest marker ...` | A group in the recipe is named after one of pytest's own markers, such as `skip`, which would change how its tests run. Give the group another name | +| `... is where this set belongs, and it does not hold it` | A kept set was changed, usually by a test writing into one of its files. Delete the directory the message names and run again | +| `... has no expected outcome, so there is nothing to check it against` | `check` was called on a file with no right answer. Skip it, or decide what your system should do and write it in a recipe | +| `This install of testing-files-generator has no tfg binary` | pip installed the wheel without the program, and the message says why, such as a 32-bit Python. Install with a 64-bit Python made for your system | +| `testing-files-generator has no binary for ...` | This system has no wheel with tfg. See [The tfg program](#the-tfg-program) | +| `tfg wrote a manifest of version ...` | `TFG_BINARY` points at a tfg whose manifest this package cannot read. Point it at a tfg of the same version as this package, or leave it unset | +| `tfg could not be started from ...` | The system would not run that file, for the reason after the colon - a tfg for another system, or one without its execute bit. Reinstall the package, or point `TFG_BINARY` at a tfg this user can run | +| `Sets are kept in ..., and ...` | The directory for sets is a file, cannot be made where it would go, or is there and this user cannot read or write it, as when a run under another user made it. Set `TFG_CACHE_DIR` or `cache_dir` to a directory you can read and write | +| `... printed a plan that is not a manifest of tfg` | `TFG_BINARY`, or the `binary` argument, names a program that is not tfg. Point it at a tfg of the same version as this package, or leave it unset | +| `The set was written, and it could not be moved to ...` | Another program held the new set's files open, often a virus scanner on Windows. Run again. If it repeats, set `TFG_CACHE_DIR` or `cache_dir` to a directory that program leaves alone | +| `testing-files-generator: 1 unfinished set was left in ...` | A warning. A run stopped part way, and what it left can be deleted | + +## The tfg program + +`tfg`, and `python -m testing_files_generator`, run the tfg this package installed, with the same arguments, output and exit codes as tfg run directly: + +```bash +tfg preset list +tfg formats +tfg generate --help +``` + +To use a tfg installed another way, set `TFG_BINARY` to its path, or pass `binary` to `file_set`. What decides is the manifest it writes. A tfg whose manifest is of another major version than this package reads is refused, because that manifest can mean something else, and a tfg of the same version as this package never is. + +On a system with no wheel of its own, such as an Intel Mac, run the tests in a Linux container, where pip installs the wheel for Linux. A Python made for Intel running under Rosetta on a Mac with Apple silicon is told to use a Python made for Apple silicon. + +tfg also writes sets from the command line on its own, has a desktop window, and installs in other ways. All of that is in [the tfg README](https://github.com/donislawdev/TestingFilesGenerator). + +## Licences + +The code of this package is under the [Mozilla Public License 2.0](LICENSE). The `tfg` program inside its wheels is under the GNU General Public License 3.0 and runs as a separate program. Its licence and the notices of the code built into it are in the wheel's licence files. The files it generates are yours: they are output of the program and not derived works of it, so you can commit them and ship them inside a closed source product with no obligation of any kind. diff --git a/python/testing-files-generator/pyproject.toml b/python/testing-files-generator/pyproject.toml new file mode 100644 index 00000000..72644c57 --- /dev/null +++ b/python/testing-files-generator/pyproject.toml @@ -0,0 +1,46 @@ +# The package's metadata. The wheels are not built from this file by a build +# backend: release/assemble.py reads it and writes one wheel for each system +# the release builds tfg for, with that system's binary inside, plus one with +# no binary for every other system. The version here is never raised by hand: +# the release writes the version of its tag into the wheels. + +[project] +name = "testing-files-generator" +version = "0.0.0.dev0" +description = "Real files of exact sizes and formats for tests, and what the system under test should do with each one. For pytest and any other test runner." +readme = "README.md" +requires-python = ">=3.10" +license = "MPL-2.0" +license-files = ["LICENSE"] +authors = [{ name = "DonislawDev" }] +keywords = ["pytest", "playwright", "test files", "fixtures", "file upload", "upload testing", "test data", "boundary testing", "qa"] +classifiers = [ + "Development Status :: 4 - Beta", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Testing", +] + +[project.urls] +Homepage = "https://github.com/donislawdev/TestingFilesGenerator/tree/main/python/testing-files-generator#readme" +Source = "https://github.com/donislawdev/TestingFilesGenerator" +Issues = "https://github.com/donislawdev/TestingFilesGenerator/issues" +Changelog = "https://github.com/donislawdev/TestingFilesGenerator/blob/main/CHANGELOG.md" + +[project.scripts] +tfg = "testing_files_generator.__main__:main" + +[project.entry-points.pytest11] +testing_files_generator = "testing_files_generator.pytest_plugin" + +[tool.pytest.ini_options] +# The tests run the package from src, not installed, so the plugin is named here. +# Where the package is installed as well, its entry point would load the plugin +# a second time and pytest would refuse to start, so the entry point is blocked. +pythonpath = ["src"] +testpaths = ["tests"] +addopts = ["-p", "no:testing_files_generator", "-p", "testing_files_generator.pytest_plugin", "-p", "pytester"] diff --git a/python/testing-files-generator/release/assemble.py b/python/testing-files-generator/release/assemble.py new file mode 100644 index 00000000..888137c3 --- /dev/null +++ b/python/testing-files-generator/release/assemble.py @@ -0,0 +1,299 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Lays out the wheels a release puts on PyPI. + +testing-files-generator gets one wheel for each system the release builds tfg +for, with that system's binary inside, and one wheel with no binary that pip +takes on every other system. pytest-testing-files-generator gets one wheel +with no code, which installs testing-files-generator of the same version. +.github/workflows/pypi.yml runs this on the archives it has downloaded and +checked, and nothing here fetches or publishes anything. + +Written with zipfile rather than a build backend: the binary has to keep its +execute bit and the macOS app has to stay whole, which a backend for Python +code does not do, and nothing else is needed. The systems come from +src/testing_files_generator/_binary.py, which a guard holds to the release's +targets. docs/PYTEST-2026-11-09.md. + + python assemble.py archives the archive of each system: dir archive binary + python assemble.py version the version as PyPI writes it + python assemble.py build the wheels, into /main and /plugin +""" + +from __future__ import annotations + +import base64 +import hashlib +import re +import stat +import struct +import sys +import zipfile +from pathlib import Path + +if sys.version_info >= (3, 11): + import tomllib +else: + # Python 3.10, the oldest the package declares, has no tomllib. tomli is + # the library it was taken from, and .github/requirements-pypi.in pins it + # for 3.10 alone. try_wheels.py runs there and imports this module. + import tomli as tomllib + +HERE = Path(__file__).resolve().parent +PACKAGE = HERE.parent +PLUGIN = PACKAGE.parent / "pytest-testing-files-generator" +sys.path.insert(0, str(PACKAGE / "src")) +from testing_files_generator._binary import PLATFORMS, binary_in # noqa: E402 + +# How the release names a system and an architecture in its archives. +RELEASE_SYSTEM = {"win32": "windows", "darwin": "macos", "linux": "linux"} +RELEASE_ARCH = {"x64": "amd64", "arm64": "arm64"} + +# The platform tags of the wheels each system's binary goes into - pip takes a +# wheel by its tag alone. Linux gets one for glibc and one for musl (Alpine): +# the binary is static and runs on both (measured 2026-11-09, Alpine 3.22). +# macOS gets its tag from the binary, below. +WHEEL_TAGS = { + "win32 x64": ["win_amd64"], + "win32 arm64": ["win_arm64"], + "linux x64": ["manylinux_2_17_x86_64.manylinux2014_x86_64", "musllinux_1_2_x86_64"], + "linux arm64": ["manylinux_2_17_aarch64.manylinux2014_aarch64", "musllinux_1_2_aarch64"], + "darwin arm64": None, +} + +VERSION_SHAPE = re.compile(r"^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$") +PRERELEASE = {"alpha": "a", "a": "a", "beta": "b", "b": "b", "rc": "rc"} + +# Every entry of every wheel carries this date, so the same release gives the +# same bytes. The zip format starts in 1980. +STAMP = (1980, 1, 1, 0, 0, 0) + + +def fail(message: str) -> None: + raise SystemExit(f"assemble: {message}") + + +def checked_version(version: str) -> str: + if not VERSION_SHAPE.match(version or ""): + fail(f"{version!r} is not a release version. Give the version of a release tag without its v, " + "for example 0.6.0 or 0.6.0-rc.1.") + return version + + +def pep440(version: str) -> str: + """The version as PyPI takes it. 0.6.0-rc.1 is 0.6.0rc1, and pip installs it only when asked with --pre.""" + core, _, pre = checked_version(version).partition("-") + if not pre: + return core + found = re.fullmatch(r"([a-z]+)\.?(\d+)", pre) + if not found or found.group(1) not in PRERELEASE: + fail(f"the prerelease {pre} of {version} has no form PyPI takes. Tag it -alpha.N, -beta.N or -rc.N, " + "or leave PyPI out of that release.") + return f"{core}{PRERELEASE[found.group(1)]}{int(found.group(2))}" + + +def targets() -> list[dict]: + out = [] + for key in PLATFORMS: + system, arch = key.split(" ") + if system not in RELEASE_SYSTEM or arch not in RELEASE_ARCH or key not in WHEEL_TAGS: + fail(f"_binary.py names {key}, and assemble does not know what the release calls it or which " + "wheel it goes in. Add it to RELEASE_SYSTEM, RELEASE_ARCH and WHEEL_TAGS.") + binary = binary_in(system) + out.append({"key": key, "system": system, "arch": arch, "dir": f"{system}-{arch}", "binary": binary, + "entry": binary.split("/")[0]}) + return out + + +def archive_of(version: str, target: dict) -> str: + ext = "zip" if target["system"] == "win32" else "tar.gz" + return f"tfg_{checked_version(version)}_{RELEASE_SYSTEM[target['system']]}_{RELEASE_ARCH[target['arch']]}.{ext}" + + +def macos_minimum(binary: bytes) -> tuple[int, int]: + """The oldest macOS a 64-bit Mach-O binary runs on, from its LC_BUILD_VERSION load command.""" + if binary[:4] != b"\xcf\xfa\xed\xfe": + fail("the macOS binary is not a 64-bit Mach-O file, so the oldest macOS it runs on cannot be read") + ncmds = struct.unpack_from("> 16, (minos >> 8) & 0xFF + at += size + fail("the macOS binary says nothing of the oldest macOS it runs on (no LC_BUILD_VERSION)") + raise AssertionError + + +def _project(directory: Path) -> dict: + with open(directory / "pyproject.toml", "rb") as f: + return tomllib.load(f)["project"] + + +def _metadata(project: dict, version: str, license_expression: str, license_files: list[str], + requires: list[str], readme: str) -> bytes: + lines = [ + "Metadata-Version: 2.4", + f"Name: {project['name']}", + f"Version: {version}", + f"Summary: {project['description']}", + f"Author: {', '.join(a['name'] for a in project.get('authors', []))}", + f"License-Expression: {license_expression}", + *(f"License-File: {name}" for name in license_files), + f"Keywords: {','.join(project.get('keywords', []))}", + *(f"Classifier: {c}" for c in project.get("classifiers", [])), + *(f"Project-URL: {label}, {url}" for label, url in project.get("urls", {}).items()), + f"Requires-Python: {project['requires-python']}", + *(f"Requires-Dist: {r}" for r in requires), + "Description-Content-Type: text/markdown", + ] + for line in lines: + if "\n" in line or "\r" in line: + fail(f"a metadata field holds a line break: {line!r}") + return ("\n".join(lines) + "\n\n" + readme).encode("utf-8") + + +def _entry_points(project: dict) -> bytes: + groups = {} + if project.get("scripts"): + groups["console_scripts"] = project["scripts"] + groups.update(project.get("entry-points", {})) + text = "".join(f"[{group}]\n" + "".join(f"{k} = {v}\n" for k, v in entries.items()) + "\n" + for group, entries in groups.items()) + return text.encode("utf-8") + + +def _record_hash(data: bytes) -> str: + return "sha256=" + base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + + +def write_wheel(out: Path, dist: str, version: str, tags: str, files: list[tuple[str, bytes, bool]], + info: list[tuple[str, bytes]]) -> Path: + """Writes one wheel. files are the installed files, info the files of its .dist-info besides WHEEL and RECORD.""" + stem = f"{dist.replace('-', '_')}-{version}" + dist_info = f"{stem}.dist-info" + pure = tags == "any" + platforms = tags.split(".") + wheel_text = ("Wheel-Version: 1.0\nGenerator: testing-files-generator release/assemble.py\n" + f"Root-Is-Purelib: {'true' if pure else 'false'}\n" + + "".join(f"Tag: py3-none-{p}\n" for p in platforms)) + entries = sorted(files) + [(f"{dist_info}/{name}", data, False) for name, data in info] + entries.append((f"{dist_info}/WHEEL", wheel_text.encode(), False)) + names = [name for name, _, _ in entries] + if len(set(names)) != len(names): + fail(f"the wheel of {dist} for {tags} would hold one name twice") + record = "".join(f"{name},{_record_hash(data)},{len(data)}\n" for name, data, _ in entries) + record += f"{dist_info}/RECORD,,\n" + entries.append((f"{dist_info}/RECORD", record.encode(), False)) + path = out / f"{stem}-py3-none-{tags}.whl" + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as z: + for name, data, executable in entries: + entry = zipfile.ZipInfo(name, date_time=STAMP) + entry.compress_type = zipfile.ZIP_DEFLATED + # Made on Unix whatever system writes it: an entry marked as made + # on MS-DOS is installed by uv without its execute bit, where pip + # keeps it (measured 2026-11-09, uv 0.9.30). + entry.create_system = 3 + entry.external_attr = ((0o755 if executable else 0o644) | stat.S_IFREG) << 16 + z.writestr(entry, data) + return path + + +def _code() -> list[tuple[str, bytes, bool]]: + src = PACKAGE / "src" / "testing_files_generator" + files = [] + for path in sorted(src.iterdir()): + if path.is_file() and (path.suffix == ".py" or path.name == "py.typed"): + files.append((f"testing_files_generator/{path.name}", path.read_bytes(), False)) + if not any(name.endswith("/__init__.py") for name, _, _ in files): + fail(f"{src} holds no __init__.py") + return files + + +def _binary_files(binaries: Path, target: dict) -> list[tuple[str, bytes, bool]]: + root = binaries / target["dir"] + entry = root / target["entry"] + if not entry.exists(): + fail(f"{root} holds no {target['entry']}. Extract the archive of {target['key']} there.") + files = [] + paths = [entry] if entry.is_file() else sorted(entry.rglob("*")) + for path in paths: + if path.is_symlink(): + fail(f"{path} is a link, and a wheel cannot hold one. The release archive changed its layout.") + if path.is_file(): + rel = path.relative_to(root).as_posix() + executable = rel == target["binary"] or bool(path.stat().st_mode & 0o111) + files.append((f"testing_files_generator/bin/{rel}", path.read_bytes(), executable)) + if not any(rel == f"testing_files_generator/bin/{target['binary']}" for rel, _, _ in files): + fail(f"{root} holds no {target['binary']}") + return files + + +def _release_licences(binaries: Path, target: dict) -> list[tuple[str, bytes]]: + out = [] + for name in ("LICENSE", "THIRD-PARTY-NOTICES.md"): + path = binaries / target["dir"] / name + if not path.is_file(): + fail(f"the archive of {target['key']} has no {name}, which has to go with the binary") + out.append((f"licenses/tfg/{name}", path.read_bytes())) + return out + + +def build(version: str, binaries: Path, out: Path) -> list[Path]: + """Writes every wheel of a release and gives their paths: testing-files-generator first, the plugin last.""" + wheel_version = pep440(version) + main_out, plugin_out = out / "main", out / "plugin" + for d in (main_out, plugin_out): + d.mkdir(parents=True, exist_ok=False) + project = _project(PACKAGE) + readme = (PACKAGE / "README.md").read_text(encoding="utf-8") + wrapper_licence = ("licenses/LICENSE", (PACKAGE / "LICENSE").read_bytes()) + entry_points = ("entry_points.txt", _entry_points(project)) + code = _code() + written = [] + for target in targets(): + files = code + _binary_files(binaries, target) + licences = [wrapper_licence, *_release_licences(binaries, target)] + tag_sets = WHEEL_TAGS[target["key"]] + if tag_sets is None: + major, minor = macos_minimum(dict((n, d) for n, d, _ in files)[f"testing_files_generator/bin/{target['binary']}"]) + tag_sets = [f"macosx_{major}_{minor}_arm64"] + # The package code is MPL-2.0 and the binary beside it GPL-3.0-only, + # each with its own licence file - the same split as the npm packages. + meta = _metadata(project, wheel_version, "MPL-2.0 AND GPL-3.0-only", + ["LICENSE", "tfg/LICENSE", "tfg/THIRD-PARTY-NOTICES.md"], [], readme) + for tags in tag_sets: + written.append(write_wheel(main_out, project["name"], wheel_version, tags, files, + [("METADATA", meta), entry_points, *licences])) + meta = _metadata(project, wheel_version, "MPL-2.0", ["LICENSE"], [], readme) + written.append(write_wheel(main_out, project["name"], wheel_version, "any", code, + [("METADATA", meta), entry_points, wrapper_licence])) + plugin = _project(PLUGIN) + requires = [f"{r}=={wheel_version}" if r == project["name"] else r for r in plugin.get("dependencies", [])] + if f"{project['name']}=={wheel_version}" not in requires: + fail(f"{PLUGIN / 'pyproject.toml'} does not depend on {project['name']}") + meta = _metadata(plugin, wheel_version, plugin["license"], ["LICENSE"], requires, + (PLUGIN / "README.md").read_text(encoding="utf-8")) + written.append(write_wheel(plugin_out, plugin["name"], wheel_version, "any", [], + [("METADATA", meta), ("licenses/LICENSE", (PLUGIN / "LICENSE").read_bytes())])) + return written + + +def main(argv: list[str]) -> None: + if argv[:1] == ["archives"] and len(argv) == 2: + for target in targets(): + print(target["dir"], archive_of(argv[1], target), target["binary"]) + elif argv[:1] == ["version"] and len(argv) == 2: + print(pep440(argv[1])) + elif argv[:1] == ["build"] and len(argv) == 4: + for path in build(argv[1], Path(argv[2]).resolve(), Path(argv[3]).resolve()): + print(path) + else: + fail("usage: assemble.py archives | version | build ") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/python/testing-files-generator/release/try_wheels.py b/python/testing-files-generator/release/try_wheels.py new file mode 100644 index 00000000..e2e2d8d4 --- /dev/null +++ b/python/testing-files-generator/release/try_wheels.py @@ -0,0 +1,170 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Installs the wheels of a release on the system this runs on, the way people install them, and uses them. + +.github/workflows/pypi.yml runs it on every system before anything reaches +PyPI. It makes a fresh environment with pip or with uv, installs +testing-files-generator from the wheels alone, and then checks that: + +- the system is the one it was told to try, so a runner that moved to + another architecture does not try one wheel and report on another, +- tfg answers with the version of the release, and the installed binary is + byte for byte the one in the release's archive, +- on macOS the app the binary runs in is still signed, +- a whole set is written through file_set, every file the size it should be, +- pytest-testing-files-generator installs testing-files-generator of its own + version, pytest loads the plugin, and a set's params pass. + + python try_wheels.py pip|uv +""" + +from __future__ import annotations + +import hashlib +import os +import re +import subprocess +import sys +import tempfile +import textwrap +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from assemble import pep440 # noqa: E402 + +SET = """ +from testing_files_generator import file_set +s = file_set(preset="upload-validation", options={"limit": "64kb", "bulk": 0}) +assert len(s) > 0, "the set has no files" +import os +for f in s.files: + assert os.path.getsize(f.path()) == f.bytes, f"{f.title} is not the size the manifest gives" + if not f.unspecified: + o = f.expected["outcome"] + f.check({"outcome": o, "reason": f.expected.get("reason")} if o == "reject" else o) +print(f"{len(s)} files written, each the size it should be") +""" + +TEST_MODULE = """ +import pytest +from testing_files_generator import file_set + +uploads = file_set(preset="upload-validation", options={"limit": "64kb", "bulk": 0}) + + +@pytest.mark.parametrize("file", uploads.params()) +def test_upload(file): + file.check(file.expected["outcome"]) +""" + + +def fail(message: str) -> None: + print(f"::error::{message}", flush=True) + raise SystemExit(1) + + +def run(*args: str, **kwargs) -> str: + print("+", " ".join(args), flush=True) + done = subprocess.run(args, capture_output=True, text=True, encoding="utf-8", check=False, **kwargs) + if done.returncode != 0: + print(done.stdout, done.stderr, sep="\n") + fail(f"{' '.join(args[:3])} ended with code {done.returncode}") + return done.stdout + + +def main(dist: str, version: str, binaries: str, system: str, installer: str) -> None: + dist_dir = Path(dist).resolve() + links = ["--find-links", str(dist_dir / "main"), "--find-links", str(dist_dir / "plugin")] + wanted = pep440(version) + work = Path(tempfile.mkdtemp(prefix="try-wheels-")) + venv = work / "venv" + if installer == "uv": + # uv picks an interpreter of its own unless told, and this pass tries + # the Python the pip pass tries, the oldest the package declares on + # one runner. + run("uv", "venv", "--quiet", "--python", sys.executable, str(venv)) + else: + run(sys.executable, "-m", "venv", str(venv)) + bin_dir = venv / ("Scripts" if os.name == "nt" else "bin") + python = str(bin_dir / ("python.exe" if os.name == "nt" else "python")) + + def install(*what: str, index: bool = False) -> None: + if installer == "uv": + run("uv", "pip", "install", "--quiet", "--python", python, *([] if index else ["--no-index"]), *links, *what) + else: + run(python, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", + *([] if index else ["--no-index"]), *links, *what) + + install(f"testing-files-generator=={wanted}") + + here = run(python, "-c", "from testing_files_generator._binary import system_name; " + "print(system_name().replace(' ', '-'))").strip() + if here != system: + fail(f"this runner is {here}, and it was to try {system}. Give {system} a runner of its own system.") + + tfg = str(bin_dir / ("tfg.exe" if os.name == "nt" else "tfg")) + said = run(tfg, "version").strip() + if said != version: + fail(f"tfg version says {said}, and the release is {version}, so the wheel carries another release's " + "binary. Nothing is published. Look at the archives of the release before running this again.") + + installed = run(python, "-c", "import testing_files_generator as t; print(t.binary_path())").strip() + sums = dict((line.split()[0], line.split()[2]) for line in Path(binaries).read_text().splitlines() if line) + got = hashlib.sha256(Path(installed).read_bytes()).hexdigest() + if got != sums.get(system): + fail(f"{installed} hashes to {got}, and the binary in the release archive to {sums.get(system)}") + print(f"{installed} is the binary of the release") + + if sys.platform == "darwin": + # The binary runs only inside the app it was signed in, so the app has + # to come out of the wheel whole. + run("codesign", "--verify", "--deep", "--strict", str(Path(installed).parents[2])) + + print(run(python, "-c", SET, cwd=work).strip()) + + install("pytest", index=True) + install(f"pytest-testing-files-generator=={wanted}") + (work / "test_uploads.py").write_text(textwrap.dedent(TEST_MODULE), encoding="utf-8") + out = run(python, "-m", "pytest", "-q", "-p", "no:cacheprovider", "--strict-markers", str(work / "test_uploads.py"), + cwd=work, env={**os.environ, "TFG_CACHE_DIR": str(work / "sets")}) + summary = out.strip().splitlines()[-1] + # pytest ends with code nought when every test is skipped, so a params() + # that skips all of them would pass here without a file checked. + passed = re.search(r"(\d+) passed", summary) + if not passed or int(passed.group(1)) == 0: + fail(f"pytest passed no test of the set. It said: {summary}") + print(summary) + header = run(python, "-m", "pytest", "--collect-only", "-q", "-p", "no:cacheprovider", "-v", + str(work / "test_uploads.py"), cwd=work) + if f"testing-files-generator: tfg at {installed}" not in header: + fail("pytest did not load the plugin of testing-files-generator, so its report header names no tfg") + print(f"{system} with {installer}: installed, tfg {said}, a set written, the plugin loaded") + + +def without_binary(dist: str, version: str) -> None: + """The wheel pip takes on a system with no binary installs, and its first call says why there is no tfg.""" + work = Path(tempfile.mkdtemp(prefix="try-any-")) + run(sys.executable, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", "--no-index", + "--find-links", str(Path(dist).resolve() / "main"), "--only-binary=:all:", "--platform", "freebsd_14_0_amd64", + "--target", str(work), f"testing-files-generator=={pep440(version)}") + if (work / "testing_files_generator" / "bin").exists(): + fail("pip installed a wheel with a binary for a system no binary is made for") + done = subprocess.run([sys.executable, "-c", "from testing_files_generator import binary_path; binary_path()"], + capture_output=True, text=True, encoding="utf-8", check=False, + env={**os.environ, "PYTHONPATH": str(work), "TFG_BINARY": ""}) + if done.returncode == 0 or "TFG_BINARY to its path" not in done.stderr: + print(done.stderr) + fail("the wheel with no binary does not say what to do when it is asked for tfg") + print("the wheel with no binary installs where no other does, and says what to do") + + +if __name__ == "__main__": + if sys.argv[1:2] == ["--without-binary"] and len(sys.argv) == 4: + without_binary(sys.argv[2], sys.argv[3]) + elif len(sys.argv) == 6 and sys.argv[5] in ("pip", "uv"): + main(*sys.argv[1:]) + else: + raise SystemExit("usage: try_wheels.py pip|uv\n" + " try_wheels.py --without-binary ") diff --git a/python/testing-files-generator/src/testing_files_generator/__init__.py b/python/testing-files-generator/src/testing_files_generator/__init__.py new file mode 100644 index 00000000..84e6949f --- /dev/null +++ b/python/testing-files-generator/src/testing_files_generator/__init__.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Real files of exact sizes and formats for your tests, and what the system under test should do with each one. + + from testing_files_generator import file_set + + uploads = file_set(preset="upload-validation", options={"limit": "2mb", "bulk": 0}) + + @pytest.mark.parametrize("file", uploads.params()) + def test_upload(file): + ... # hand file.path() to your system + file.check("accept" if taken else "reject") +""" + +from ._binary import binary_path +from ._fileset import FileSet, UnfinishedSetWarning, file_set +from ._run import TfgError +from ._testfile import ExpectationError, TestFile + +try: + from importlib.metadata import PackageNotFoundError, version + + __version__ = version("testing-files-generator") +except PackageNotFoundError: # run from the repository, not installed + __version__ = "0.0.0.dev0" + +__all__ = [ + "ExpectationError", + "FileSet", + "TestFile", + "TfgError", + "UnfinishedSetWarning", + "binary_path", + "file_set", +] diff --git a/python/testing-files-generator/src/testing_files_generator/__main__.py b/python/testing-files-generator/src/testing_files_generator/__main__.py new file mode 100644 index 00000000..11736642 --- /dev/null +++ b/python/testing-files-generator/src/testing_files_generator/__main__.py @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""tfg, and python -m testing_files_generator, run the binary this package installed. + +The same arguments, the same streams and the same exit code, so a script +cannot tell it from tfg run directly. +""" + +from __future__ import annotations + +import os +import signal +import subprocess +import sys + +from ._binary import binary_path +from ._run import not_started + +# RUNTIME in docs/CLI.md section 3 - the tool could not do its work, and the +# arguments were never looked at. +_CANNOT_RUN = 1 + + +def main() -> int: + try: + binary = binary_path() + except RuntimeError as err: + print(f"tfg: {err}", file=sys.stderr) + return _CANNOT_RUN + args = [binary, *sys.argv[1:]] + try: + if sys.platform != "win32": + # tfg takes this process's place, so signals and the exit code are + # its own with nothing in between. + os.execv(binary, args) + # Windows has no such thing. A Ctrl+C in a console reaches tfg as well + # as this process, so this one waits for tfg to stop the way it stops + # on its own, and its exit code is the one a script sees. + signal.signal(signal.SIGINT, signal.SIG_IGN) + return subprocess.call(args) + except OSError as err: + print(f"tfg: {not_started(binary, err)}", file=sys.stderr) + return _CANNOT_RUN + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/testing-files-generator/src/testing_files_generator/_binary.py b/python/testing-files-generator/src/testing_files_generator/_binary.py new file mode 100644 index 00000000..b94e4e41 --- /dev/null +++ b/python/testing-files-generator/src/testing_files_generator/_binary.py @@ -0,0 +1,111 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Where the tfg binary this package runs is.""" + +from __future__ import annotations + +import os +import platform as _platform +import sys +from collections.abc import Mapping + +# The systems a release builds a command line for, named as Node names them so +# the list reads the same here and in the npm package. A guard holds it to the +# list of targets in .github/workflows/release.yml, and the release build in +# release/assemble.py makes one wheel with the binary for each of them. Intel +# Macs are on neither list, by the owner's decision of 2026-08-20. +PLATFORMS = ("win32 x64", "win32 arm64", "linux x64", "linux arm64", "darwin arm64") + +# How a machine name reads as an architecture. Windows says AMD64 and ARM64, +# Linux x86_64 and aarch64, macOS x86_64 and arm64. +_ARCH = {"amd64": "x64", "x86_64": "x64", "arm64": "arm64", "aarch64": "arm64"} + +_HERE = os.path.dirname(os.path.abspath(__file__)) + + +def binary_in(system: str) -> str: + """Where the binary sits inside the package's bin directory, as the release archive lays it out. + + On macOS that is inside the app the release signed and notarised: its + signature covers the app's Info.plist, and macOS kills the binary taken out + on its own (exit 137, measured 2026-11-09 on macOS 27.2 - + docs/NPM-PUBLISH-2026-11-09.md). + """ + if system == "win32": + return "tfg.exe" + if system == "darwin": + return "tfg.app/Contents/MacOS/tfg" + return "tfg" + + +def system_name(system: str | None = None, machine: str | None = None) -> str: + """This system as PLATFORMS names one, such as linux x64.""" + system = sys.platform if system is None else system + if system.startswith("linux"): + system = "linux" + machine = (_platform.machine() if machine is None else machine).lower() + return f"{system} {_ARCH.get(machine, machine)}" + + +def _why_none(system: str, bits: int) -> str: + # Why pip took the wheel without a binary on a system that has one. The + # wheel pip takes is decided by the Python that installs it, and a 32-bit + # Python on 64-bit Windows is matched to wheels for 32-bit Python. On + # macOS the wheel with the binary also names the oldest macOS the binary + # runs on, and pip passes it over on an older one. + if bits == 32: + return "this Python is 32-bit, and the binary comes in wheels for 64-bit Python" + if system == "darwin": + return ("pip found no wheel with a binary for this version of macOS. The name of the wheel for macOS " + "says the oldest version the binary runs on, such as macosx_13_0 for macOS 13") + return "pip found no wheel with a binary for this system" + + +# A Python made for Intel runs on a Mac with Apple silicon under Rosetta, and +# says it is on an Intel Mac. A Python made for Apple silicon gets the wheel +# with the binary. +_ROSETTA = (" If this Mac has Apple silicon, this Python runs under Rosetta - install the package with a " + "Python made for Apple silicon.") + + +def binary_path(*, env: Mapping[str, str] | None = None, system: str | None = None, + machine: str | None = None, bits: int | None = None, here: str | None = None) -> str: + """The tfg binary this package runs. + + TFG_BINARY comes first and is taken as given - a tfg installed another way, + or one built from the repository. PATH is never searched: a different tfg + found there would answer with its own version and say nothing about it. + Everything is passed in so the tests can ask about a system they are not on. + """ + env = os.environ if env is None else env + given = env.get("TFG_BINARY") + if given: + if not os.path.isfile(given): + raise RuntimeError(f"TFG_BINARY is set to {given} and there is no file there. " + "Point it at a tfg binary, or unset it to use the one this package installs.") + return given + raw = sys.platform if system is None else system + name = system_name(raw, machine) + bits = (64 if sys.maxsize > 2**32 else 32) if bits is None else bits + here = _HERE if here is None else here + bin_dir = os.path.join(here, "bin") + found = os.path.join(bin_dir, *binary_in(name.split(" ")[0]).split("/")) + if os.path.isfile(found): + return found + known = ", ".join(PLATFORMS) + if name not in PLATFORMS: + raise RuntimeError(f"testing-files-generator has no binary for {name}. It comes for {known}. " + "On another system, run the tests in a Linux container, where pip installs the " + "package for Linux, or install tfg another way and set TFG_BINARY to its path." + + (_ROSETTA if name == "darwin x64" else "")) + if not os.path.isdir(bin_dir): + raise RuntimeError(f"This install of testing-files-generator has no tfg binary: {_why_none(name.split(' ')[0], bits)}. " + f"The binary comes for {known}, each in a wheel of its own. Install the package with " + "a 64-bit Python made for this system, or install tfg another way and set " + "TFG_BINARY to its path.") + raise RuntimeError(f"The tfg binary of this install is not at {found}, where the package puts it. " + "Something removed it after the install. Install the package again with " + "pip install --force-reinstall testing-files-generator, or set TFG_BINARY to the path " + "of a tfg binary.") diff --git a/python/testing-files-generator/src/testing_files_generator/_fileset.py b/python/testing-files-generator/src/testing_files_generator/_fileset.py new file mode 100644 index 00000000..64eb6d14 --- /dev/null +++ b/python/testing-files-generator/src/testing_files_generator/_fileset.py @@ -0,0 +1,678 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Sets of files: planned when they are declared, written once, kept and checked. + +The same logic as lib/fileset.js in the npm package, in the same order, so +the two can be read side by side. What a file says of itself and the sentences +check raises are held to the npm package's by testdata/wrappers/vectors.json, +which the tests of both packages read. +""" + +from __future__ import annotations + +import errno +import hashlib +import json +import ntpath +import os +import posixpath +import re +import secrets +import subprocess +import sys +import threading +import time +import warnings +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from typing import Any + +from ._binary import binary_path +from ._run import _FLAGS, TfgError, ended, run, run_to_file, said +from ._testfile import OUTCOMES, TestFile + +# The manifest major this package reads. Another major is refused rather than +# read as if it were this one - docs/MANIFEST.md, the version of the contract. +MANIFEST_MAJOR = "1" + +# The code tfg verify ends with when a directory does not match its manifest, +# from the exit codes in docs/CLI.md. Any other code is verify failing to look. +_EXIT_VERIFY = 7 + +# Flags this package sets itself. An option of the same name would fight it. +RESERVED = { + "out": "this package writes every set into a directory of its own", + "json": "this package reads the manifest from the directory it writes", + "dry-run": "this package plans each set with a dry run and then writes it", + "preset": "a preset is named by the preset argument beside the options", +} + +# Everything this package leaves while it works starts with this, followed by +# the process id, so a later run can tell a set being written now from one a +# stopped run left behind. +UNFINISHED = ".tmp-" + +# The markers pytest registers itself, as pytest --markers lists them with no +# plugin and no configuration, which a test compares. A group by one of these +# names would not mark its files but change how their tests run, measured +# 2026-11-09 on pytest 9.2.0: skip and skipif skip them, xfail hides a failure, +# and pytest.param refuses usefixtures. A plugin's markers are not known here, +# and the README says so. +PYTEST_OWN_MARKS = frozenset({"filterwarnings", "skip", "skipif", "xfail", "parametrize", "usefixtures", + "tryfirst", "trylast"}) + +# A rename the system refuses for a moment. On Windows a scanner holding a new +# file open is enough, so these are tried again before anything is said. +_BUSY_ERRNO = {errno.EACCES, errno.EPERM, errno.EBUSY} +_BUSY_WINERROR = {5, 32} # access denied, sharing violation +_RENAME_TRIES = 10 + +# One plan per set and process. A test module is imported once per process, +# and two modules can ask for one set. +_sources: dict[str, _Source] = {} +_sources_lock = threading.Lock() +_noted = False + +# What a write did - wrote a copy, moved it to its name, or took one another +# process moved first. Not part of the package's interface: the tests listen, +# because a race that never happened passes as well as one won. +_listeners: list[Callable[[str, str], None]] = [] + +# The pytest run this process is part of, given by the plugin, for the +# directory sets are kept in and the markers a set's params use. +_pytest_config: Any = None +_registered: set[str] = set() + + +class UnfinishedSetWarning(UserWarning): + """A run that stopped part way left an unfinished set, which nothing uses.""" + + +def _emit(event: str, path: str) -> None: + for listener in list(_listeners): + listener(event, path) + + +def use_pytest(config: Any) -> None: + """Called by the pytest plugin when a run starts, and with None when it ends.""" + global _pytest_config + _pytest_config = config + _registered.clear() + + +def file_set(preset: str | None = None, recipe: str | os.PathLike[str] | None = None, + options: Mapping[str, Any] | None = None, cache_dir: str | os.PathLike[str] | None = None, + binary: str | os.PathLike[str] | None = None) -> FileSet: + """Plans a set of files, writing nothing, so tests can be declared from it while their module loads. + + The files are written the first time one of them is asked for. + """ + if options is not None and not isinstance(options, Mapping): + raise TypeError("options takes a dict, for example {'limit': '2mb', 'bulk': 0}.") + shown = _described(preset, recipe, options) + # Told from the flags and not from the arguments: an option set to False + # or to an empty list adds no flag, and tfg asked for nothing answers in flags. + args = _args_of(preset, recipe, options) + if len(args) == 1: + raise ValueError("file_set was not told which files to write. Name a preset, a recipe or the options " + "of a set, for example file_set(preset='upload-validation'). tfg preset list lists " + "the presets.") + tfg = binary_path() if binary is None else _named(os.fspath(binary), "binary", + "Point it at a tfg binary, or leave it out to use the one " + "this package installs.") + content = "" + if recipe: + where = os.path.abspath(os.fspath(recipe)) + _named(where, "recipe", f"A relative path is taken from the directory the tests started in, here " + f"{os.getcwd()}, so Path(__file__).parent / 'uploads.yaml' names a recipe beside " + "the test file.") + # A pipe or a device would be read here without end, and what is read + # here is gone before tfg reads it. + if not os.path.isfile(where): + raise ValueError(f"The recipe option names {where}, which is a directory, a pipe or a device rather " + "than a file. A recipe is read twice, once here to tell whether it changed and once " + "by tfg, so it has to be a YAML file on the disk. Give the path of that file.") + content = _content_of(where) + cache = os.path.abspath(os.fspath(cache_dir) if cache_dir is not None else default_cache_dir()) + key = _digest([_identity(tfg), args, content, cache]) + with _sources_lock: + source = _sources.get(key) + if source is None: + source = _Source(tfg, args, cache, preset, recipe, shown) + _sources[key] = source + return FileSet(source, source.files) + + +def _content_of(recipe: str) -> str: + """The sha256 of a recipe's bytes, so an edited recipe is planned again. + + Read in pieces and never decoded: tfg reads the recipe itself and says + what is wrong with one too large or not in UTF-8, in its own words. + """ + digest = hashlib.sha256() + with open(recipe, "rb") as f: + for piece in iter(lambda: f.read(1 << 16), b""): + digest.update(piece) + return digest.hexdigest() + + +def _text(value: Any) -> str: + # A whole number given as a float is passed as a whole number, 2 and not + # 2.0, as the npm package passes it. + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return str(value) + + +def _args_of(preset: str | None, recipe: Any, options: Mapping[str, Any] | None) -> list[str]: + args = ["generate"] + if recipe: + args.append(os.path.abspath(os.fspath(recipe))) + if preset: + args.append(f"--preset={preset}") + for raw, value in (options or {}).items(): + # far_over is far-over: a Python name cannot hold a hyphen, and no flag + # of tfg holds an underscore. + key = raw.replace("_", "-") if isinstance(raw, str) else raw + if key in RESERVED: + raise ValueError(f"The option {raw} cannot be set here - {RESERVED[key]}.") + if not isinstance(key, str) or not re.fullmatch(r"[a-z][a-z0-9-]*", key): + raise ValueError(f"{raw!r} is not a flag tfg has. Options are flag names without the dashes, " + "such as limit or far-over.") + for one in value if isinstance(value, (list, tuple)) else [value]: + if one is True: + args.append(f"--{key}") + elif one is False or one is None: + continue + elif isinstance(one, (str, int, float)): + args.append(f"--{key}={_text(one)}") + else: + raise TypeError(f"The option {raw} takes a string, a number, True or a list of them. " + f"It was given a {type(one).__name__}.") + return args + + +def _named(file: str, option: str, way_out: str) -> str: + """A file an argument names, or a sentence saying there is none, before anything reads it.""" + if not os.path.exists(file): + raise FileNotFoundError(f"The {option} option names {file}, and there is no file there. {way_out}") + return file + + +def default_cache_dir() -> str: + """Where sets are kept: TFG_CACHE_DIR, the cache of the pytest run, or the user's own cache directory.""" + given = os.environ.get("TFG_CACHE_DIR") + if given: + return given + cache = getattr(_pytest_config, "cache", None) + if cache is not None: + return str(cache.mkdir("testing-files-generator")) + return _user_cache_dir() + + +def _user_cache_dir(env: Mapping[str, str] | None = None, system: str | None = None, + home: str | None = None) -> str: + # A directory of the user's own, never the system's temporary directory, + # which every user of a Linux machine shares - someone else could make the + # directory first and change the files the tests run on (code scanning + # alerts 23 and 24, docs/NPM-PUBLISH-2026-11-09.md). + env = os.environ if env is None else env + system = sys.platform if system is None else system + home = os.path.expanduser("~") if home is None else home + # The paths of the system asked about, which are this system's paths + # except in the tests. + paths = ntpath if system == "win32" else posixpath + if system == "win32": + base = env.get("LOCALAPPDATA") or paths.join(home, "AppData", "Local") + elif system == "darwin": + base = paths.join(home, "Library", "Caches") + else: + xdg = env.get("XDG_CACHE_HOME", "") + base = xdg if paths.isabs(xdg) else paths.join(home, ".cache") + if not paths.isabs(base): + raise RuntimeError("testing-files-generator found no home directory to keep sets in. Set TFG_CACHE_DIR, " + "or the cache_dir argument of file_set, to a directory of your own.") + return paths.join(base, "testing-files-generator") + + +def _make_cache(cache: str) -> None: + """Makes the directory sets are kept in, or says where it comes from and how to give another.""" + try: + os.makedirs(cache, exist_ok=True) + except OSError as err: + raise _cache_refused(cache, "it could not be made a directory", err) from err + + +def _cache_refused(cache: str, what: str, err: OSError) -> RuntimeError: + """The directory of sets this user cannot use, with the system's reason and how to give another.""" + return RuntimeError(f"Sets are kept in {cache}, and {what} - {err.strerror or err}. Set TFG_CACHE_DIR, or the " + "cache_dir argument of file_set, to a directory this user can read and write.") + + +def _label_of(preset: str | None, recipe: Any) -> str: + if preset: + name = preset + elif recipe: + name = re.sub(r"\.[^.]*$", "", os.path.basename(os.fspath(recipe))) + else: + name = "flags" + return re.sub(r"[^A-Za-z0-9_-]+", "-", name)[:40] + + +def _described(preset: Any, recipe: Any, options: Any) -> str: + """The call as the test wrote it, for a message - what makes the set, without where it is kept.""" + shown = [f"{key}={value!r}" for key, value in (("preset", preset), ("recipe", recipe), ("options", options)) + if value is not None] + return f"file_set({', '.join(shown)})" + + +def _identity(binary: str) -> list[Any]: + # What tells two binaries apart without running them, so a new build or + # another version plans again rather than reusing an old plan. + stat = os.stat(binary) + return [os.path.realpath(binary), stat.st_size, stat.st_mtime_ns] + + +def _digest(value: Any) -> str: + return hashlib.sha256(json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest() + + +def _scratch(cache: str) -> str: + return os.path.join(cache, f"{UNFINISHED}{os.getpid()}-{secrets.token_hex(4)}") + + +def set_name(label: str, args: Sequence[str], plan: Mapping[str, Any]) -> str: + """The name of a set's directory. + + The recipe hash leaves out the seed and --clean, measured 2026-11-04, so + the arguments are part of the name and two seeds never share a set. + """ + run_ = plan["run"] + return f"{label}-{_digest([list(args), run_.get('recipe_hash'), run_['seed'], plan['tool']])[:20]}" + + +class _Source: + """One set: its plan, where it is written, and the one write a process makes of it however many files ask.""" + + def __init__(self, binary: str, args: list[str], cache: str, preset: str | None, recipe: Any, shown: str): + self.binary = binary + self.args = args + self.cache = cache + self.preset = preset + self.shown = shown + _make_cache(cache) + _note_unfinished(cache) + self.plan = self._planned() + self.dir = os.path.join(cache, set_name(_label_of(preset, recipe), args, self.plan)) + self.files = tuple(TestFile(self, entry) for entry in self.plan["files"]) + self._lock = threading.Lock() + self._written: str | None = None + + def _planned(self) -> dict[str, Any]: + # The dry run names a directory that is never made, because a dry run + # into a directory holding a manifest is refused like a real one. + out = f"{_scratch(self.cache)}.json" + try: + run_to_file(self.binary, [*self.args, f"--out={_scratch(self.cache)}", "--dry-run", "--json"], out) + with open(out, encoding="utf-8") as f: + plan = json.load(f) + readable(plan) + except TfgError as err: + raise _refused_plan(self.shown, self.preset, err) from err + except OSError as err: + # The plan is the first file written in the directory of sets. + if err.filename != out: + raise + raise _cache_refused(self.cache, "nothing could be written in it", err) from err + except (ValueError, RecursionError) as err: + raise RuntimeError(f"{self.binary} printed a plan that is not a manifest of tfg - {err}. It has to be " + "tfg of the same version as this package. Reinstall the package with pip install " + "--force-reinstall testing-files-generator, or set TFG_BINARY to the path of such " + "a tfg.") from err + finally: + try: + os.remove(out) + except FileNotFoundError: + pass + return plan + + def write(self) -> str: + with self._lock: + if self._written is None: + self._written = self._write() + return self._written + + def _write(self) -> str: + if os.path.exists(self.dir): + self._taken() + return self.dir + tmp = _scratch(self.cache) + os.mkdir(tmp) + try: + run(self.binary, [*self.args, f"--out={tmp}"]) + problem = mismatch(tmp, self.plan) + if problem: + raise RuntimeError(f"tfg wrote a set that does not match its own plan - {problem}. " + "This is a fault in tfg or in this package. Please report it.") + except BaseException: + _discard(self.binary, tmp) + raise + _emit("wrote", tmp) + return self._settle(tmp) + + def _settle(self, tmp: str) -> str: + # Several workers can write the same set at once, and the first to move + # it wins - the others find the winner whole, take it, and remove their + # own copy through its manifest. + for attempt in range(1, _RENAME_TRIES + 1): + try: + os.rename(tmp, self.dir) + _emit("moved", self.dir) + return self.dir + except OSError as err: + if os.path.exists(self.dir): + _discard(self.binary, tmp) + self._taken() + _emit("took", self.dir) + return self.dir + busy = err.errno in _BUSY_ERRNO or getattr(err, "winerror", None) in _BUSY_WINERROR + if not busy or attempt == _RENAME_TRIES: + _discard(self.binary, tmp) + raise RuntimeError(f"The set was written, and it could not be moved to {self.dir} - " + f"{err.strerror or err}. Run the tests again. If this repeats, another " + "program may be holding its files open, such as a virus scanner. Set " + "TFG_CACHE_DIR, or the cache_dir argument of file_set, to a directory that " + "program leaves alone.") from err + time.sleep(attempt * 0.1) + raise AssertionError("unreachable") + + def _taken(self) -> None: + # A set this process did not write is checked before it is used: + # against the plan, and then every byte against the manifest through + # tfg verify. A test that writes into a file it was given changes the + # set for every later run, and the size alone does not show it. + problem = mismatch(self.dir, self.plan) + if problem: + raise _stale(self.dir, problem) + try: + run(self.binary, ["verify", os.path.join(self.dir, "manifest.json")]) + except TfgError as err: + if err.status == _EXIT_VERIFY: + raise _stale(self.dir, "its files are not the ones it was written with", err.stderr.strip()) from err + raise + + +def _refused_plan(shown: str, preset: str | None, err: TfgError) -> TfgError: + """A plan tfg refused, told in the words the test was written in. + + An option tfg does not know brings the whole help of tfg generate, every + flag behind a dash, where one sentence names the mistake. Everything tfg + said stays on the error, in stderr. + """ + unknown = re.search(r"^flag provided but not defined: -+(\S+)", err.stderr, re.M) + known = [name for name in re.findall(r"^\s+-([a-z][a-z0-9-]*)\b", err.stderr, re.M) if name not in RESERVED] + if unknown and known: + help_ = (f"tfg preset show {preset} says what the preset's own options do, and tfg generate --help " + "the rest." if preset else "tfg generate --help says what each one does.") + return TfgError([], err.status, err.stderr, + f"{shown} gives the option {unknown.group(1)}, and tfg generate has no flag of that name. " + f"Its flags are {', '.join(known)}. Not every one goes with every set, and tfg says why " + f"when one does not. {help_}") + return TfgError([], err.status, err.stderr, f"{shown} could not be planned - tfg {ended(err.status)}.\n" + f"{said(err.stderr)}") + + +def readable(plan: Any) -> None: + """Refuses a manifest this package cannot read for what it says. + + A manifest of another version is a RuntimeError, and one of this version + that has no run or no list of files described by objects a ValueError. + """ + version = str((plan or {}).get("manifest_version") or "") if isinstance(plan, dict) else "" + if version.split(".")[0] != MANIFEST_MAJOR: + raise RuntimeError(f"tfg wrote a manifest of version {version or 'none'}, and this package reads version " + f"{MANIFEST_MAJOR}. Install the version of testing-files-generator that comes with that " + "tfg, or point TFG_BINARY at a tfg of the same version as this package.") + files = plan.get("files") + if not isinstance(plan.get("run"), dict) or not isinstance(files, list) \ + or not all(isinstance(entry, dict) for entry in files): + raise ValueError("it has no run, or no list of files each described by an object") + for entry in files: + outcome = (entry.get("expected") or {}).get("outcome") + if outcome not in OUTCOMES: + raise RuntimeError(f"{entry.get('path')} expects {json.dumps(outcome)}, which this package does not " + f"know. It knows {', '.join(OUTCOMES)}. Use the version of this package that " + "comes with the tfg it runs.") + + +def mismatch(directory: str, plan: Mapping[str, Any]) -> str: + """How a written set differs from its plan, or an empty string when it does not. + + It compares names and sizes, which is enough for a set tfg has just + written. The bytes of a set written earlier are checked through tfg verify. + Every file has to be on the disk: a run that ends with code nought writes + every file it plans, until a run can describe a file without writing it on + purpose - the archive and manifest-only output of D10. + """ + try: + with open(os.path.join(directory, "manifest.json"), encoding="utf-8") as f: + written = json.load(f) + if not isinstance(written, dict): + raise ValueError("it is not an object") + except (OSError, ValueError, RecursionError) as err: + # RecursionError is JSON nested deeper than the parser goes, which is + # not a ValueError. + return f"its manifest cannot be read ({err})" + run_ = written.get("run") if isinstance(written.get("run"), dict) else {} + if run_.get("recipe_hash") != plan["run"].get("recipe_hash") or run_.get("seed") != plan["run"]["seed"]: + return "it was written from another recipe or seed" + if run_.get("complete") is not True: + return "the run that wrote it did not finish" + files = written.get("files") + if not isinstance(files, list) or len(files) != len(plan["files"]): + count = len(files) if isinstance(files, list) else "no" + return f"it holds {count} files where the plan has {len(plan['files'])}" + for i, (want, got) in enumerate(zip(plan["files"], files), start=1): + # The manifest of a kept set is on the disk, where a person or a test + # can change it, so an entry is not taken to be what tfg writes. + if not isinstance(got, dict): + return f"its file {i} is not described by an object where the plan has {want['path']}" + if got.get("path") != want["path"] or got.get("bytes") != want["bytes"]: + return f"its file {i} is {got.get('path')} of {got.get('bytes')} B where the plan has " \ + f"{want['path']} of {want['bytes']} B" + if not got.get("materialized"): + return f"{got['path']} was not written" + try: + size = os.stat(os.path.join(directory, *got["path"].split("/"))).st_size + except OSError: + return f"{got['path']} is missing" + if size != got["bytes"]: + return f"{got['path']} is {size} B and its manifest says {got['bytes']} B" + return "" + + +def _stale(directory: str, problem: str, details: str = "") -> RuntimeError: + return RuntimeError(f"{directory} is where this set belongs, and it does not hold it - {problem}. Nothing here " + "removes that directory, because a person may have changed it on purpose. Delete it and " + "run again." + (f"\n{details}" if details else "")) + + +def _discard(binary: str, directory: str) -> None: + # Removes a set this process wrote and will not use. The files go through + # tfg cleanup, which removes only what the manifest lists, and then the + # manifest and the directories it leaves empty. Whatever is left stays, + # and a later run says so. + manifest = os.path.join(directory, "manifest.json") + if os.path.exists(manifest): + try: + code = subprocess.run([binary, "cleanup", manifest, "--yes"], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, creationflags=_FLAGS, + check=False).returncode + except OSError: + return + if code != 0: + return + for name in ("manifest.json", "manifest.instructions.md"): + try: + os.remove(os.path.join(directory, name)) + except FileNotFoundError: + pass + _remove_empty(directory) + + +def _remove_empty(directory: str) -> None: + try: + entries = list(os.scandir(directory)) + except OSError: + return + for entry in entries: + if entry.is_dir(follow_symlinks=False): + _remove_empty(entry.path) + try: + os.rmdir(directory) + except OSError: + pass # not empty: left where it is + + +def _note_unfinished(cache: str) -> None: + """Says, once per process and only outside a pytest-xdist worker, which sets a stopped run left behind. + + Nothing reads them and nothing removes them on its own, so without this + they would fill the disk in silence. + """ + global _noted + if _noted or os.environ.get("PYTEST_XDIST_WORKER"): + return + _noted = True + try: + names = sorted(os.listdir(cache)) + except OSError as err: + raise _cache_refused(cache, "it could not be read", err) from err + left = [] + for name in names: + if not name.startswith(UNFINISHED): + continue + pid = name[len(UNFINISHED):].split("-")[0] + if pid.isdigit() and int(pid) > 0 and not _alive(int(pid)): + left.append(name) + if left: + what = "set was" if len(left) == 1 else "sets were" + warnings.warn(UnfinishedSetWarning( + f"testing-files-generator: {len(left)} unfinished {what} left in {cache} by a run that stopped part " + f"way - {', '.join(left)}. Nothing uses them and they can be deleted."), stacklevel=4) + + +def _alive(pid: int) -> bool: + if sys.platform == "win32": + # Never os.kill here: on Windows it ends the process whatever the signal. + import ctypes + from ctypes import wintypes + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.OpenProcess.restype = wintypes.HANDLE + kernel32.OpenProcess.argtypes = (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD) + kernel32.GetExitCodeProcess.argtypes = (wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)) + kernel32.CloseHandle.argtypes = (wintypes.HANDLE,) + handle = kernel32.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION + if not handle: + return ctypes.get_last_error() == 5 # there, and not this user's to ask about + try: + code = wintypes.DWORD() + if not kernel32.GetExitCodeProcess(handle, ctypes.byref(code)): + return True + return code.value == 259 # STILL_ACTIVE + finally: + kernel32.CloseHandle(handle) + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + +def _register(names: Iterable[str]) -> None: + # A marker pytest does not know is a warning, and an error under + # --strict-markers. A group is known only once a set is planned, so each + # one is registered as the params that carry it are made. + if _pytest_config is None: + return + for name in names: + if name not in _registered: + _pytest_config.addinivalue_line("markers", f"{name}: files of a testing-files-generator set") + _registered.add(name) + + +class FileSet: + """A set of files, or a part of one. Every part shares the set's one write.""" + + def __init__(self, source: _Source, files: Iterable[TestFile]): + self._source = source + self.files: tuple[TestFile, ...] = tuple(files) + + @property + def plan(self) -> Mapping[str, Any]: + """The manifest tfg gave for the plan of the whole set. Nothing in it is written yet.""" + return self._source.plan + + def only(self, *, group: str | Iterable[str] | None = None, target: str | Iterable[str] | None = None, + outcome: str | Iterable[str] | None = None) -> FileSet: + """The files that match every argument given. Each takes one name or a list of them.""" + return FileSet(self._source, [f for f in self.files if _matches(f, group, target, outcome)]) + + def except_(self, *, group: str | Iterable[str] | None = None, target: str | Iterable[str] | None = None, + outcome: str | Iterable[str] | None = None) -> FileSet: + """The files that do not match. The underscore is there because except is a word of Python.""" + return FileSet(self._source, [f for f in self.files if not _matches(f, group, target, outcome)]) + + def write(self) -> str: + """Writes the whole set once per process, or takes a set kept from an earlier run, and gives its directory.""" + return self._source.write() + + def paths(self) -> list[str]: + """Writes the set if it is not written yet, and gives the absolute paths of this part's files.""" + directory = self._source.write() + return [os.path.join(directory, *f.relative_path.split("/")) for f in self.files] + + def params(self) -> list[Any]: + """One pytest.param for each file, for @pytest.mark.parametrize. + + The id is the file's title, the markers are its expected outcome and its + group, and a file with no right answer is skipped with the reason. + """ + import pytest + + params = [] + for f in self.files: + names = f.marks + own = PYTEST_OWN_MARKS.intersection(names) + if own: + raise ValueError(f"The group {f.group} of {f.relative_path} becomes the pytest marker " + f"{sorted(own)[0]}, which is one of pytest's own and changes how the test runs - " + "skip and skipif skip it, xfail hides its failure. Give the group another name " + "in the recipe.") + _register(names) + marks = [getattr(pytest.mark, name) for name in names] + if f.unspecified: + marks.append(pytest.mark.skip(reason=f.why_unspecified)) + params.append(pytest.param(f, id=f.title, marks=marks)) + return params + + def __iter__(self) -> Iterator[TestFile]: + return iter(self.files) + + def __len__(self) -> int: + return len(self.files) + + +def _matches(file: TestFile, group: Any, target: Any, outcome: Any) -> bool: + def has(wanted: Any, value: str) -> bool: + if wanted is None: + return True + return value == wanted if isinstance(wanted, str) else value in wanted + + return has(group, file.group) and has(target, file.target) and has(outcome, file.expected["outcome"]) diff --git a/python/testing-files-generator/src/testing_files_generator/_run.py b/python/testing-files-generator/src/testing_files_generator/_run.py new file mode 100644 index 00000000..c4772259 --- /dev/null +++ b/python/testing-files-generator/src/testing_files_generator/_run.py @@ -0,0 +1,105 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Running tfg and telling how it ended.""" + +from __future__ import annotations + +import os +import signal +import subprocess +import sys +import tempfile +from collections.abc import Sequence + +# How much of what tfg says on its error stream is kept for a message. Its +# refusals are a few lines, and a run of many files can print a note for each. +KEPT_ERROR = 64 * 1024 + +# A console window is not opened for tfg when the tests run without one. +_FLAGS = subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0 + + +def ended(status: int | str) -> str: + """How tfg stopped - with a code, or by a signal named as it is.""" + return f"was stopped by {status}" if isinstance(status, str) else f"ended with code {status}" + + +def said(stderr: str) -> str: + return stderr.strip() or "tfg said nothing about why." + + +def not_started(binary: str, cause: OSError) -> str: + """A binary the system would not start, with the system's reason and the way out. + + The reason stays: "Exec format error" and "Permission denied" are what + tell a binary for another system from one without its execute bit. + """ + return (f"tfg could not be started from {binary}: {cause}. It has to be a tfg for this system that this " + "user can run. Reinstall the package with pip install --force-reinstall testing-files-generator, " + "or set TFG_BINARY to the path of such a tfg.") + + +class TfgError(Exception): + """tfg ending with a code other than nought, or stopped by a signal, whose name is then the status. + + The message carries what tfg said, because its refusals already name the + value and the way out. A caller that can say what went wrong in the words + the test was written in passes its own message, and the status and + everything tfg said stay on the error. + """ + + def __init__(self, args: Sequence[str], status: int | str, stderr: str, message: str | None = None): + if message is None: + message = f"tfg {' '.join(args)} {ended(status)}.\n{said(stderr)}" + super().__init__(message) + self.status = status + self.stderr = stderr + + +def _status(code: int) -> int | str: + # A negative code on Linux and macOS is the signal that stopped the process. + if code < 0: + try: + return signal.Signals(-code).name + except ValueError: + return f"signal {-code}" + return code + + +def _tail(handle) -> str: + handle.seek(0, os.SEEK_END) + size = handle.tell() + handle.seek(max(0, size - KEPT_ERROR)) + return handle.read().decode("utf-8", errors="replace") + + +def _run(binary: str, args: Sequence[str], stdout) -> None: + # The error stream goes to a file with no name, because a pipe that fills + # stops tfg part way, and only its end is kept. + with tempfile.TemporaryFile() as err: + try: + code = subprocess.run([binary, *args], stdin=subprocess.DEVNULL, stdout=stdout, stderr=err, + creationflags=_FLAGS, check=False).returncode + except OSError as cause: + raise RuntimeError(not_started(binary, cause)) from cause + if code != 0: + raise TfgError(args, _status(code), _tail(err)) + + +def run_to_file(binary: str, args: Sequence[str], out_file: str) -> None: + """Runs tfg to the end, writing its standard output to a file. + + A file and not memory, because the manifest of a large set is large. The + file is made new and readable by its owner alone, so neither a file nor a + link someone else left under the same name is written through. + """ + fd = os.open(out_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0), 0o600) + with os.fdopen(fd, "wb") as out: + _run(binary, args, out) + + +def run(binary: str, args: Sequence[str]) -> None: + """Runs tfg to the end, keeping the end of what it says on its error stream.""" + _run(binary, args, subprocess.DEVNULL) diff --git a/python/testing-files-generator/src/testing_files_generator/_testfile.py b/python/testing-files-generator/src/testing_files_generator/_testfile.py new file mode 100644 index 00000000..ed636d71 --- /dev/null +++ b/python/testing-files-generator/src/testing_files_generator/_testfile.py @@ -0,0 +1,146 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""One file of a set, with what the system under test should do with it.""" + +from __future__ import annotations + +import os +import re +from collections.abc import Mapping +from typing import Any + +# The closed list of outcomes a manifest declares, from docs/MANIFEST.md. A +# guard holds this copy to recipe.Outcomes in the program, because two copies +# of a closed list is how they drift. +OUTCOMES = ("accept", "reject", "sanitize", "unspecified") + +# How each outcome reads in a sentence about what a system did with a file. +_DONE = {"accept": "taken", "reject": "turned away", "sanitize": "taken and cleaned"} + + +class ExpectationError(AssertionError): + """What check raises when a system did something other than what the file expects.""" + + def __init__(self, message: str, expected: Mapping[str, Any], observed: Mapping[str, Any]): + super().__init__(message) + self.expected = expected + self.observed = observed + + +def mark_name(word: str) -> str: + """A word as a pytest marker - anything other than a letter, digit, underscore or hyphen becomes a hyphen.""" + return re.sub(r"[^A-Za-z0-9_-]+", "-", str(word)) + + +def expected_text(expected: Mapping[str, Any]) -> str: + reason = expected.get("reason") + return f"{expected['outcome']} ({reason})" if reason else expected["outcome"] + + +class TestFile: + """One file of a set: a title, its marks, its path once the set is written, and a check against the manifest.""" + + # Not a test class, whatever its name, when a test module imports it. + __test__ = False + + def __init__(self, source, entry: Mapping[str, Any]): + self._source = source + self.entry = entry + + @property + def relative_path(self) -> str: + return self.entry["path"] + + @property + def name(self) -> str: + return self.entry["name"] + + @property + def format(self) -> str: + return self.entry["format"] + + @property + def bytes(self) -> int: + return self.entry["bytes"] + + @property + def group(self) -> str: + return self.entry.get("group") or "" + + @property + def target(self) -> str: + return self.entry["target_id"] + + @property + def purpose(self) -> str: + return self.entry.get("purpose") or "" + + @property + def expected(self) -> Mapping[str, Any]: + return self.entry["expected"] + + @property + def unspecified(self) -> bool: + return self.entry["expected"]["outcome"] == "unspecified" + + @property + def title(self) -> str: + """The path and what the file expects. The path is unique in a set, which a test id has to be.""" + return f"{self.entry['path']} - expected {expected_text(self.entry['expected'])}" + + @property + def marks(self) -> list[str]: + """The pytest markers of the file: its expected outcome and its group.""" + names = [mark_name(self.entry["expected"]["outcome"])] + if self.group: + names.append(mark_name(self.group)) + return names + + @property + def why_unspecified(self) -> str: + why = f"{self.entry['path']} has no expected outcome - what your system does with it is its own policy." + return f"{why} {self.purpose}" if self.purpose else why + + def path(self) -> str: + """Writes the set the first time any of its files is asked for, and gives this file's absolute path.""" + return os.path.join(self._source.write(), *self.entry["path"].split("/")) + + def check(self, observed: str | Mapping[str, Any]) -> None: + """Compares what the system did with what the manifest expects, and raises when they differ. + + The error carries the file's purpose, the sentence that says what the + difference means. A reason is compared only for a refusal: a reason + names the rule a file is about, so a file a byte under a limit is taken + with size_limit and there is nothing to compare it with. + """ + __tracebackhide__ = True # pytest shows the line of the test, not this one + want = self.entry["expected"] + if want["outcome"] == "unspecified": + # Not an ExpectationError: the system did nothing wrong, the test + # asked about a file with no right answer. + raise RuntimeError( + f"{self.entry['path']} has no expected outcome, so there is nothing to check it against. " + "Skip it - the params of a set skip it on their own - or decide what your system should do " + "with it and check that.") + got = {"outcome": observed} if isinstance(observed, str) else observed + if not isinstance(got, Mapping) or got.get("outcome") not in _DONE: + raise TypeError("check takes accept, reject or sanitize, or a dict with an outcome and a reason. " + f"It was given {observed!r}.") + if got["outcome"] != want["outcome"]: + reason = want.get("reason") + self._differs(f"{self.entry['path']} was {_DONE[got['outcome']]}, and it should have been " + f"{_DONE[want['outcome']]}" + (f" ({reason})." if reason else "."), got) + if want["outcome"] == "reject" and want.get("reason") and got.get("reason") \ + and got["reason"] != want["reason"]: + self._differs(f"{self.entry['path']} was turned away for {got['reason']}, " + f"and the rule it is about is {want['reason']}.", got) + + def _differs(self, sentence: str, observed: Mapping[str, Any]) -> None: + __tracebackhide__ = True + message = f"{sentence}\n{self.purpose}" if self.purpose else sentence + raise ExpectationError(message, self.entry["expected"], dict(observed)) + + def __repr__(self) -> str: + return f"" diff --git a/python/testing-files-generator/src/testing_files_generator/py.typed b/python/testing-files-generator/src/testing_files_generator/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/python/testing-files-generator/src/testing_files_generator/pytest_plugin.py b/python/testing-files-generator/src/testing_files_generator/pytest_plugin.py new file mode 100644 index 00000000..6c01c468 --- /dev/null +++ b/python/testing-files-generator/src/testing_files_generator/pytest_plugin.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""The pytest plugin, which pytest loads on its own once the package is installed. + +It registers the markers a set's params carry, keeps sets in the run's +.pytest_cache, and names the tfg the run uses in the report header. Turn it +off with -p no:testing_files_generator. +""" + +from __future__ import annotations + +from typing import Any + +from . import _fileset +from ._binary import binary_path + +_MEANING = { + "accept": "files the system under test should take", + "reject": "files the system under test should turn away", + "sanitize": "files the system under test should take and change", + "unspecified": "files with no expected outcome, which are skipped", +} + + +def pytest_configure(config: Any) -> None: + for outcome, meaning in _MEANING.items(): + config.addinivalue_line("markers", f"{outcome}: {meaning} (testing-files-generator)") + _fileset.use_pytest(config) + + +def pytest_unconfigure(config: Any) -> None: + _fileset.use_pytest(None) + + +def pytest_report_header(config: Any) -> str: + try: + return f"testing-files-generator: tfg at {binary_path()}" + except RuntimeError as err: + return f"testing-files-generator: {err}" diff --git a/python/testing-files-generator/tests/conftest.py b/python/testing-files-generator/tests/conftest.py new file mode 100644 index 00000000..4395a049 --- /dev/null +++ b/python/testing-files-generator/tests/conftest.py @@ -0,0 +1,63 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +PACKAGE = Path(__file__).resolve().parents[1] +SRC = str(PACKAGE / "src") +ROOT = PACKAGE.parents[1] +BUILT = ROOT / ("tfg.exe" if sys.platform == "win32" else "tfg") +VECTORS = json.loads((ROOT / "testdata" / "wrappers" / "vectors.json").read_text(encoding="utf-8")) + +# A small set that still holds every kind of case: one either side of the +# limit, unspecified files and a group of several files. +SMALL = {"preset": "upload-validation", "options": {"limit": "1mb", "bulk": 2, "far-over": "off"}} + + +def tfg_binary() -> str: + """TFG_BINARY, or the one the repository's build command leaves at its root. + + Neither is a failure and not a skip - a suite that skips without a binary + is green while testing nothing. + """ + binary = os.environ.get("TFG_BINARY") or str(BUILT) + if not os.path.isfile(binary): + raise RuntimeError(f"No tfg binary at {binary}. Build one from the repository root with " + f'go build -tags "$(cat .github/build-tags)" -o {BUILT.name} ./cmd/tfg, or set TFG_BINARY.') + return binary + + +def python(script: str, *args: str, env: dict | None = None) -> subprocess.CompletedProcess: + """Runs a script in a fresh Python with this package and a binary. A variable given as None is taken out.""" + child = {**os.environ, "TFG_BINARY": tfg_binary(), "PYTHONPATH": SRC} + for key, value in (env or {}).items(): + if value is None: + child.pop(key, None) + else: + child[key] = value + return subprocess.run([sys.executable, "-c", script, *args], env=child, capture_output=True, text=True, + encoding="utf-8", check=False) + + +# The script a fresh process runs to write a set, printing its directory. +WRITER = """ +import json, sys +from testing_files_generator import file_set +try: + print(file_set(**json.loads(sys.argv[1])).write()) +except Exception as err: + print(err, file=sys.stderr) + sys.exit(3) +""" + + +def writer(cache, spec=None) -> subprocess.CompletedProcess: + return python(WRITER, json.dumps({**(spec or SMALL), "cache_dir": str(cache)})) + + +@pytest.fixture +def cache(tmp_path): + return tmp_path / "cache" diff --git a/python/testing-files-generator/tests/test_assemble.py b/python/testing-files-generator/tests/test_assemble.py new file mode 100644 index 00000000..f43ca57c --- /dev/null +++ b/python/testing-files-generator/tests/test_assemble.py @@ -0,0 +1,223 @@ +"""The wheels a release puts on PyPI, laid out from archives made here.""" + +import base64 +import hashlib +import importlib.util +import os +import stat +import struct +import sys +import zipfile +from email.parser import Parser +from pathlib import Path + +import pytest +from conftest import PACKAGE + +_spec = importlib.util.spec_from_file_location("assemble", PACKAGE / "release" / "assemble.py") +assemble = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(assemble) + + +def macho(minos=0x000D0000): + """A 64-bit Mach-O header with one other load command and LC_BUILD_VERSION, as Go writes them.""" + other = struct.pack(" Path: + """Extracted archives as pypi.yml lays them out, with a binary of each system made up here.""" + binaries = root / "binaries" + for t in assemble.targets(): + d = binaries / t["dir"] + if t["system"] == "darwin": + app = d / "tfg.app" / "Contents" + (app / "MacOS").mkdir(parents=True) + (app / "MacOS" / "tfg").write_bytes(macho()) + (app / "Info.plist").write_text("", encoding="utf-8") + (app / "_CodeSignature").mkdir() + (app / "_CodeSignature" / "CodeResources").write_text("", encoding="utf-8") + else: + d.mkdir(parents=True) + (d / t["binary"]).write_bytes(f"tfg for {t['key']}".encode()) + (d / "LICENSE").write_text("GNU GENERAL PUBLIC LICENSE", encoding="utf-8") + (d / "THIRD-PARTY-NOTICES.md").write_text("notices", encoding="utf-8") + (d / "README.md").write_text("not in the wheel", encoding="utf-8") + return binaries + + +def built(tmp_path, version="0.6.0-rc.1", out="out"): + binaries = tmp_path / "binaries" if (tmp_path / "binaries").exists() else release_archives(tmp_path) + return assemble.build(version, binaries, tmp_path / out) + + +def test_every_system_gets_its_wheels_and_every_other_one_the_wheel_with_no_binary(tmp_path): + names = [p.name for p in built(tmp_path)] + assert names == [ + "testing_files_generator-0.6.0rc1-py3-none-win_amd64.whl", + "testing_files_generator-0.6.0rc1-py3-none-win_arm64.whl", + "testing_files_generator-0.6.0rc1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "testing_files_generator-0.6.0rc1-py3-none-musllinux_1_2_x86_64.whl", + "testing_files_generator-0.6.0rc1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "testing_files_generator-0.6.0rc1-py3-none-musllinux_1_2_aarch64.whl", + "testing_files_generator-0.6.0rc1-py3-none-macosx_13_0_arm64.whl", + "testing_files_generator-0.6.0rc1-py3-none-any.whl", + "pytest_testing_files_generator-0.6.0rc1-py3-none-any.whl", + ] + + +def _read(wheel: Path): + with zipfile.ZipFile(wheel) as z: + return {i.filename: (i, z.read(i)) for i in z.infolist()} + + +def test_a_wheel_holds_the_code_the_binary_with_its_execute_bit_and_both_licences(tmp_path): + wheels = {p.name: p for p in built(tmp_path)} + linux = _read(wheels["testing_files_generator-0.6.0rc1-py3-none-musllinux_1_2_x86_64.whl"]) + info = "testing_files_generator-0.6.0rc1.dist-info" + for name, (entry, _) in linux.items(): + assert entry.create_system == 3, f"{name} is not marked as made on Unix, and uv drops its execute bit" + assert entry.date_time == assemble.STAMP + mode = entry.external_attr >> 16 + want = 0o755 if name == "testing_files_generator/bin/tfg" else 0o644 + assert stat.S_IMODE(mode) == want, name + assert linux["testing_files_generator/bin/tfg"][1] == b"tfg for linux x64" + assert "testing_files_generator/__init__.py" in linux + assert "testing_files_generator/py.typed" in linux + assert "testing_files_generator/bin/README.md" not in linux + assert linux[f"{info}/licenses/tfg/LICENSE"][1] == b"GNU GENERAL PUBLIC LICENSE" + assert linux[f"{info}/licenses/LICENSE"][1].startswith(b"Mozilla Public License Version 2.0") + meta = Parser().parsestr(linux[f"{info}/METADATA"][1].decode()) + assert meta["Version"] == "0.6.0rc1" + assert meta["License-Expression"] == "MPL-2.0 AND GPL-3.0-only" + assert meta.get_all("License-File") == ["LICENSE", "tfg/LICENSE", "tfg/THIRD-PARTY-NOTICES.md"] + assert meta["Requires-Python"] == ">=3.10" + assert meta.get_payload().startswith("# testing-files-generator") + wheel = linux[f"{info}/WHEEL"][1].decode() + assert "Root-Is-Purelib: false" in wheel and "Tag: py3-none-musllinux_1_2_x86_64" in wheel + points = linux[f"{info}/entry_points.txt"][1].decode() + assert "tfg = testing_files_generator.__main__:main" in points + assert "[pytest11]\ntesting_files_generator = testing_files_generator.pytest_plugin" in points + + +def test_a_tag_set_is_written_out_one_tag_a_line(tmp_path): + wheels = {p.name: p for p in built(tmp_path)} + many = _read(wheels["testing_files_generator-0.6.0rc1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"]) + wheel = many["testing_files_generator-0.6.0rc1.dist-info/WHEEL"][1].decode() + assert "Tag: py3-none-manylinux_2_17_x86_64\nTag: py3-none-manylinux2014_x86_64\n" in wheel + + +def test_the_macos_wheel_holds_the_whole_app_and_the_windows_one_the_exe(tmp_path): + wheels = {p.name: p for p in built(tmp_path)} + mac = _read(wheels["testing_files_generator-0.6.0rc1-py3-none-macosx_13_0_arm64.whl"]) + for rel in ("Contents/MacOS/tfg", "Contents/Info.plist", "Contents/_CodeSignature/CodeResources"): + assert f"testing_files_generator/bin/tfg.app/{rel}" in mac, rel + assert stat.S_IMODE(mac["testing_files_generator/bin/tfg.app/Contents/MacOS/tfg"][0].external_attr >> 16) == 0o755 + win = _read(wheels["testing_files_generator-0.6.0rc1-py3-none-win_arm64.whl"]) + assert win["testing_files_generator/bin/tfg.exe"][1] == b"tfg for win32 arm64" + + +def test_the_wheel_with_no_binary_is_the_code_alone_under_the_licence_of_the_code(tmp_path): + wheels = {p.name: p for p in built(tmp_path)} + pure = _read(wheels["testing_files_generator-0.6.0rc1-py3-none-any.whl"]) + assert not [n for n in pure if "/bin/" in n] + meta = Parser().parsestr(pure["testing_files_generator-0.6.0rc1.dist-info/METADATA"][1].decode()) + assert meta["License-Expression"] == "MPL-2.0" + assert "Root-Is-Purelib: true" in pure["testing_files_generator-0.6.0rc1.dist-info/WHEEL"][1].decode() + + +def test_the_plugin_package_installs_testing_files_generator_of_exactly_its_version(tmp_path): + wheels = {p.name: p for p in built(tmp_path)} + plugin = _read(wheels["pytest_testing_files_generator-0.6.0rc1-py3-none-any.whl"]) + assert all(n.startswith("pytest_testing_files_generator-0.6.0rc1.dist-info/") for n in plugin) + meta = Parser().parsestr(plugin["pytest_testing_files_generator-0.6.0rc1.dist-info/METADATA"][1].decode()) + assert meta["Name"] == "pytest-testing-files-generator" + assert meta.get_all("Requires-Dist") == ["testing-files-generator==0.6.0rc1", "pytest>=8"] + assert "Framework :: Pytest" in meta.get_all("Classifier") + + +def test_every_file_of_every_wheel_is_in_its_record_with_its_hash(tmp_path): + for wheel in built(tmp_path): + files = _read(wheel) + record = next(data for name, (_, data) in files.items() if name.endswith(".dist-info/RECORD")) + rows = [line.split(",") for line in record.decode().splitlines()] + assert sorted(r[0] for r in rows) == sorted(files), wheel.name + for name, digest, size in rows: + if name.endswith("/RECORD"): + assert (digest, size) == ("", "") + continue + data = files[name][1] + want = "sha256=" + base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + assert (digest, int(size)) == (want, len(data)), name + + +def test_the_same_release_gives_the_same_bytes(tmp_path): + first = built(tmp_path, out="one") + second = built(tmp_path, out="two") + for a, b in zip(first, second): + assert a.read_bytes() == b.read_bytes(), a.name + + +def test_an_archive_that_lost_its_binary_or_its_licence_is_refused(tmp_path): + binaries = release_archives(tmp_path) + os.remove(binaries / "linux-arm64" / "THIRD-PARTY-NOTICES.md") + with pytest.raises(SystemExit, match="has no THIRD-PARTY-NOTICES.md, which has to go with the binary"): + assemble.build("0.6.0", binaries, tmp_path / "out") + os.remove(binaries / "win32-x64" / "tfg.exe") + with pytest.raises(SystemExit, match="holds no tfg.exe"): + assemble.build("0.6.0", binaries, tmp_path / "again") + + +def test_a_link_in_the_macos_app_is_refused(tmp_path): + binaries = release_archives(tmp_path) + target = binaries / "darwin-arm64" / "tfg.app" / "Contents" / "Info.plist" + link = binaries / "darwin-arm64" / "tfg.app" / "Contents" / "Resources" + try: + os.symlink(target, link) + except OSError as err: + if sys.platform == "win32": + pytest.skip(f"this Windows user cannot make a link ({err}). The release builds the wheels on Linux.") + raise + with pytest.raises(SystemExit, match="is a link, and a wheel cannot hold one"): + assemble.build("0.6.0", binaries, tmp_path / "out") + + +def test_an_out_directory_already_there_is_not_written_into(tmp_path): + built(tmp_path) + with pytest.raises(FileExistsError): + built(tmp_path) diff --git a/python/testing-files-generator/tests/test_binary.py b/python/testing-files-generator/tests/test_binary.py new file mode 100644 index 00000000..e0ebb34d --- /dev/null +++ b/python/testing-files-generator/tests/test_binary.py @@ -0,0 +1,74 @@ +import os + +import pytest + +from testing_files_generator._binary import PLATFORMS, binary_in, binary_path, system_name + + +def test_tfg_binary_is_taken_as_given_and_named_when_it_is_not_there(tmp_path): + given = tmp_path / "my-tfg" + given.write_bytes(b"") + assert binary_path(env={"TFG_BINARY": str(given)}) == str(given) + missing = str(tmp_path / "gone") + with pytest.raises(RuntimeError, match="TFG_BINARY is set to .* and there is no file there") as caught: + binary_path(env={"TFG_BINARY": missing}) + assert "unset it" in str(caught.value) + + +def test_each_system_has_its_binary_where_its_release_archive_puts_it(): + assert binary_in("win32") == "tfg.exe" + assert binary_in("darwin") == "tfg.app/Contents/MacOS/tfg" + assert binary_in("linux") == "tfg" + + +@pytest.mark.parametrize(("system", "machine", "name"), [ + ("win32", "AMD64", "win32 x64"), + ("win32", "ARM64", "win32 arm64"), + ("linux", "x86_64", "linux x64"), + ("linux", "aarch64", "linux arm64"), + ("darwin", "arm64", "darwin arm64"), + ("darwin", "x86_64", "darwin x64"), + ("linux", "i686", "linux i686"), +]) +def test_a_system_is_named_as_the_list_names_it(system, machine, name): + assert system_name(system, machine) == name + + +def _installed(tmp_path, system): + here = tmp_path / "testing_files_generator" + binary = here / "bin" / binary_in(system) + binary.parent.mkdir(parents=True) + binary.write_bytes(b"") + return here, binary + + +@pytest.mark.parametrize("platform", PLATFORMS) +def test_the_binary_of_an_installed_wheel_is_found_on_every_system_of_the_list(tmp_path, platform): + system, arch = platform.split(" ") + machine = {"x64": "x86_64", "arm64": "arm64"}[arch] + here, binary = _installed(tmp_path, system) + assert binary_path(env={}, system=system, machine=machine, bits=64, here=str(here)) == str(binary) + + +def test_a_system_with_no_binary_is_told_where_the_package_runs(tmp_path): + with pytest.raises(RuntimeError) as caught: + binary_path(env={}, system="linux", machine="i686", bits=32, here=str(tmp_path)) + assert "has no binary for linux i686" in str(caught.value) + assert "Linux container" in str(caught.value) and "TFG_BINARY" in str(caught.value) + with pytest.raises(RuntimeError, match="Rosetta"): + binary_path(env={}, system="darwin", machine="x86_64", bits=64, here=str(tmp_path)) + + +def test_the_wheel_with_no_binary_says_why_pip_took_it(tmp_path): + with pytest.raises(RuntimeError) as caught: + binary_path(env={}, system="win32", machine="AMD64", bits=32, here=str(tmp_path)) + assert "has no tfg binary: this Python is 32-bit" in str(caught.value) + assert "TFG_BINARY" in str(caught.value) + + +def test_a_binary_removed_after_the_install_is_told_as_that(tmp_path): + here, binary = _installed(tmp_path, "linux") + os.remove(binary) + with pytest.raises(RuntimeError, match="Something removed it after the install") as caught: + binary_path(env={}, system="linux", machine="x86_64", bits=64, here=str(here)) + assert "--force-reinstall" in str(caught.value) diff --git a/python/testing-files-generator/tests/test_fileset.py b/python/testing-files-generator/tests/test_fileset.py new file mode 100644 index 00000000..5a037049 --- /dev/null +++ b/python/testing-files-generator/tests/test_fileset.py @@ -0,0 +1,420 @@ +import errno +import json +import os +import subprocess +import sys +import time +import warnings + +import pytest +from conftest import SMALL, SRC, python, tfg_binary, writer + +from testing_files_generator import TfgError, UnfinishedSetWarning, file_set +from testing_files_generator import _fileset + + +def planned(cache, spec=None): + return file_set(**(spec or SMALL), binary=tfg_binary(), cache_dir=cache) + + +def entries(cache): + return sorted(os.listdir(cache)) + + +def test_a_plan_writes_nothing_and_says_every_file_with_what_is_expected_of_it(cache): + s = planned(cache) + assert entries(cache) == [], "a plan left something in the cache" + assert len(s.files) == s.plan["summary"]["file_count"] + assert len(s.files) >= 20, f"the set this test asks for holds {len(s.files)} files" + assert len({f.title for f in s.files}) == len(s.files), "two files share a title" + assert len(s.only(outcome="unspecified")) == s.plan["summary"]["by_expected"]["unspecified"] + assert len(s.only(group="bulk-upload")) == 2 + assert len(s.except_(group="bulk-upload")) == len(s) - 2 + assert len(s.only(group=["bulk-upload", "size-limit"], outcome="accept")) > 0 + + +def test_a_target_or_a_list_of_them_picks_the_files_made_for_it(cache): + s = planned(cache) + targets = sorted({f.target for f in s.files}) + assert len(targets) >= 3, f"the set this test asks for has the targets {targets}" + one, two = targets[0], targets[1] + made_for_one = [f for f in s.files if f.target == one] + assert list(s.only(target=one)) == made_for_one + assert list(s.except_(target=one)) == [f for f in s.files if f.target != one] + assert {f.target for f in s.only(target=[one, two])} == {one, two} + assert list(s.only(target=one, outcome=made_for_one[0].expected["outcome"])) == \ + [f for f in made_for_one if f.expected["outcome"] == made_for_one[0].expected["outcome"]] + + +def test_a_set_is_written_once_whole_and_its_files_are_where_their_paths_say(cache): + s = planned(cache) + directory = s.write() + assert s.write() == directory + assert entries(cache) == [os.path.basename(directory)] + for f in s.files: + assert os.path.getsize(f.path()) == f.bytes, f.relative_path + assert s.only(group="bulk-upload").paths() == [os.path.join(directory, f.relative_path) + for f in s.only(group="bulk-upload")] + + +# A racer plans a set, waits until every racer has planned, and only then +# writes, so that all of them find the set unwritten and write it at once. A +# racer that fails before it reaches the barrier would hold the others there +# for ever, so the wait has an end. +RACER = """ +import json, os, sys, time +from testing_files_generator import _fileset, file_set +said = [] +_fileset._listeners.append(lambda event, path: said.append(event)) +spec, barrier, racers = json.loads(sys.argv[1]), sys.argv[2], int(sys.argv[3]) +s = file_set(**spec) +open(os.path.join(barrier, str(os.getpid())), "w").close() +until = time.monotonic() + 30 +while len(os.listdir(barrier)) < racers: + if time.monotonic() > until: + sys.exit("not every racer planned within 30 s") + time.sleep(0.005) +print(json.dumps({"dir": s.write(), "said": said})) +""" + + +def test_workers_writing_one_set_at_once_end_with_one_copy_and_nothing_left_over(cache, tmp_path): + barrier = tmp_path / "barrier" + barrier.mkdir() + racers = 4 + env = {**os.environ, "TFG_BINARY": tfg_binary(), "PYTHONPATH": SRC} + spec = json.dumps({**SMALL, "cache_dir": str(cache)}) + children = [subprocess.Popen([sys.executable, "-c", RACER, spec, str(barrier), str(racers)], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") + for _ in range(racers)] + # communicate and not wait: a racer whose traceback outgrows the pipe + # would wait on it for ever. A racer still running when one times out is + # stopped, so none holds the cache open after the test. + runs = [] + try: + for c in children: + out, err = c.communicate(timeout=120) + runs.append((c.returncode, out, err)) + finally: + for c in children: + if c.poll() is None: + c.kill() + c.communicate() + # Every failed racer is shown, because the first to fail may only be one + # that waited for another. + assert [err for code, _, err in runs if code != 0] == [], "a racer failed" + results = [json.loads(out) for _, out, _ in runs] + said = [e for r in results for e in r["said"]] + assert said.count("wrote") == racers, f"not every worker wrote, so there was no race: {said}" + assert said.count("moved") == 1, said + assert said.count("took") == racers - 1, said + dirs = {r["dir"] for r in results} + assert len(dirs) == 1, f"the workers were given {dirs}" + assert entries(cache) == [os.path.basename(next(iter(dirs)))] + assert planned(cache).write() == next(iter(dirs)) + + +def test_a_later_run_takes_the_set_an_earlier_one_wrote(cache): + first = writer(cache) + assert first.returncode == 0, first.stderr + directory = first.stdout.strip() + before = os.stat(os.path.join(directory, "manifest.json")).st_mtime_ns + second = writer(cache) + assert second.stdout.strip() == directory + assert os.stat(os.path.join(directory, "manifest.json")).st_mtime_ns == before, "the set was written again" + + +def test_another_seed_or_clean_is_another_set_though_the_recipe_hash_is_the_same(cache): + plain = planned(cache) + seeded = planned(cache, {**SMALL, "options": {**SMALL["options"], "seed": 7}}) + clean = planned(cache, {**SMALL, "options": {**SMALL["options"], "clean": True}}) + assert seeded.plan["run"]["recipe_hash"] == plain.plan["run"]["recipe_hash"], \ + "the hash now covers the seed - this test is no longer about anything" + assert len({plain.write(), seeded.write(), clean.write()}) == 3 + + +def test_an_option_written_with_underscores_is_the_flag_with_hyphens(cache): + hyphens = planned(cache) + underscores = planned(cache, {**SMALL, "options": {"limit": "1mb", "bulk": 2, "far_over": "off"}}) + assert underscores.write() == hyphens.write() + + +def test_a_set_changed_by_hand_is_refused_and_left_where_it_is(cache): + directory = writer(cache).stdout.strip() + os.remove(os.path.join(directory, "1mb_at_limit.jpg")) + second = writer(cache) + assert second.returncode == 3 + assert "does not hold it - 1mb_at_limit.jpg is missing. Nothing here removes that directory" in second.stderr + assert os.path.exists(os.path.join(directory, "manifest.json")), "the changed set was removed" + + +def test_a_kept_manifest_whose_file_is_not_an_object_is_refused_like_any_changed_set(cache): + directory = writer(cache).stdout.strip() + manifest = os.path.join(directory, "manifest.json") + with open(manifest, encoding="utf-8") as f: + written = json.load(f) + first = written["files"][0]["path"] + written["files"][0] = None + with open(manifest, "w", encoding="utf-8") as f: + json.dump(written, f) + again = writer(cache) + assert again.returncode == 3 + assert (f"does not hold it - its file 1 is not described by an object where the plan has {first}. " + "Nothing here removes that directory") in again.stderr, again.stderr + + +def test_a_file_of_the_set_cut_short_by_hand_is_refused(cache): + directory = writer(cache).stdout.strip() + os.truncate(os.path.join(directory, "1mb_at_limit.jpg"), 10) + again = writer(cache) + assert again.returncode == 3 + assert "1mb_at_limit.jpg is 10 B and its manifest says 1048576 B" in again.stderr + + +def test_a_file_written_over_by_hand_its_size_the_same_is_refused(cache): + directory = writer(cache).stdout.strip() + changed = os.path.join(directory, "1mb_at_limit.jpg") + with open(changed, "r+b") as f: + f.write(b"changed") + again = writer(cache) + assert again.returncode == 3, again.stderr + assert "its files are not the ones it was written with. Nothing here removes that directory" in again.stderr + assert "wrong-hash" in again.stderr and "1mb_at_limit.jpg" in again.stderr + assert os.path.exists(changed), "the changed set was removed" + + +def test_a_binary_or_a_recipe_that_is_not_there_is_named_with_what_to_do(cache): + missing = os.path.join(str(cache), "no-such-tfg") + with pytest.raises(FileNotFoundError) as caught: + file_set(**SMALL, binary=missing, cache_dir=cache) + assert f"The binary option names {missing}, and there is no file there." in str(caught.value) + assert "leave it out" in str(caught.value) + with pytest.raises(FileNotFoundError) as caught: + file_set(recipe="no-such-recipe.yaml", binary=tfg_binary(), cache_dir=cache) + assert os.path.abspath("no-such-recipe.yaml") in str(caught.value) + assert "Path(__file__).parent" in str(caught.value) + + +def test_a_recipe_not_in_utf8_is_refused_by_tfg_in_its_own_words(cache, tmp_path): + # Read here only for its hash, so what is wrong with it is said by tfg, + # which reads it, and not by a decoder on the way. + recipe = tmp_path / "latin.yaml" + recipe.write_bytes(b"version: 1\n# caf\xe9\n") + with pytest.raises(TfgError) as caught: + file_set(recipe=recipe, binary=tfg_binary(), cache_dir=cache) + assert "is not valid UTF-8" in str(caught.value) and "Save the file as UTF-8" in str(caught.value) + + +def test_an_edited_recipe_is_planned_again_in_the_same_process(cache, tmp_path): + recipe = tmp_path / "set.yaml" + recipe.write_text("version: 1\ntargets:\n - id: one\n format: txt\n size: 1kb\n", encoding="utf-8") + first = file_set(recipe=recipe, binary=tfg_binary(), cache_dir=cache) + recipe.write_text("version: 1\ntargets:\n - id: one\n format: txt\n size: 2kb\n", encoding="utf-8") + second = file_set(recipe=recipe, binary=tfg_binary(), cache_dir=cache) + assert [f.bytes for f in first] == [1024] + assert [f.bytes for f in second] == [2048] + + +def test_a_cache_directory_that_cannot_be_made_says_how_to_give_another(tmp_path): + taken = tmp_path / "a-file" + taken.write_text("", encoding="utf-8") + with pytest.raises(RuntimeError) as caught: + file_set(**SMALL, binary=tfg_binary(), cache_dir=taken) + said = str(caught.value) + assert said.startswith(f"Sets are kept in {taken}, and it could not be made a directory - ") + assert "TFG_CACHE_DIR" in said and "cache_dir" in said + + +@pytest.mark.skipif(sys.platform == "win32" or os.geteuid() == 0, + reason="the modes of a directory hold on Linux and macOS, for a user that is not root") +@pytest.mark.parametrize(("mode", "what"), [(0o333, "it could not be read"), + (0o555, "nothing could be written in it")]) +def test_a_cache_directory_this_user_cannot_use_says_how_to_give_another(tmp_path, mode, what): + # A directory that is there already passes the making of it, as one left + # by a run under another user does. + there = tmp_path / "sets" + there.mkdir() + there.chmod(mode) + try: + said = python(f"from testing_files_generator import file_set; " + f"file_set(**{SMALL!r}, cache_dir={str(there)!r})", env={"PYTEST_XDIST_WORKER": None}) + finally: + there.chmod(0o755) + assert said.returncode != 0 + assert f"RuntimeError: Sets are kept in {there}, and {what} - Permission denied." in said.stderr, said.stderr + assert "TFG_CACHE_DIR" in said.stderr and "cache_dir" in said.stderr + + +@pytest.mark.parametrize("printed", ["", "not json", "[" * 200_000, '{"manifest_version": "1.0"}', + '{"manifest_version": "1.0", "run": {}, "files": [null]}']) +def test_a_binary_that_prints_no_manifest_for_the_plan_is_named_with_what_to_do(cache, monkeypatch, printed): + # TFG_BINARY can point at any program, and one that ends well and prints + # anything but a manifest is named rather than a parser's error. + def prints(binary, args, out_file): + with open(out_file, "w", encoding="utf-8") as f: + f.write(printed) + + monkeypatch.setattr(_fileset, "run_to_file", prints) + with pytest.raises(RuntimeError) as caught: + file_set(**SMALL, binary=tfg_binary(), cache_dir=cache) + said = str(caught.value) + assert said.startswith(f"{tfg_binary()} printed a plan that is not a manifest of tfg - "), said + assert "TFG_BINARY" in said and "pip install --force-reinstall" in said + assert entries(cache) == [], "the plan was left in the cache" + + +def test_a_set_that_cannot_be_moved_to_its_name_says_what_to_do_and_leaves_nothing(cache, monkeypatch): + s = planned(cache) + + def refuses(src, dst): + raise OSError(errno.ENAMETOOLONG, "File name too long", dst) + + monkeypatch.setattr(_fileset.os, "rename", refuses) + with pytest.raises(RuntimeError) as caught: + s.write() + said = str(caught.value) + assert said.startswith(f"The set was written, and it could not be moved to {s._source.dir} - " + "File name too long. Run the tests again."), said + assert "TFG_CACHE_DIR" in said and "cache_dir" in said + assert entries(cache) == [], "the set that was written is left in the cache" + + +def test_a_recipe_that_is_a_directory_is_refused_before_anything_reads_it(cache, tmp_path): + with pytest.raises(ValueError) as caught: + file_set(recipe=tmp_path, binary=tfg_binary(), cache_dir=cache) + assert f"The recipe option names {tmp_path}, which is a directory, a pipe or a device" in str(caught.value) + assert "Give the path of that file" in str(caught.value) + + +def test_a_kept_manifest_nested_past_the_parser_is_refused_like_any_changed_set(cache): + # Python's parser raises RecursionError there, which is not a ValueError. + directory = writer(cache).stdout.strip() + with open(os.path.join(directory, "manifest.json"), "w", encoding="utf-8") as f: + f.write("[" * 200_000) + again = writer(cache) + assert again.returncode == 3 + assert "does not hold it - its manifest cannot be read (" in again.stderr, again.stderr + + +def test_a_set_a_stopped_run_left_behind_is_named_once_outside_the_workers_and_never_used(cache): + gone = subprocess.Popen([sys.executable, "-c", ""]) + gone.wait() + cache.mkdir() + left = cache / f".tmp-{gone.pid}-0bad0bad" + left.mkdir() + plan = (f"import warnings; warnings.simplefilter('always'); from testing_files_generator import file_set; " + f"file_set(**{SMALL!r}, cache_dir={str(cache)!r})") + runner = python(plan, env={"PYTEST_XDIST_WORKER": None}) + assert runner.returncode == 0, runner.stderr + worker = python(plan, env={"PYTEST_XDIST_WORKER": "gw0"}) + assert worker.returncode == 0, worker.stderr + assert worker.stderr == "", "a worker spoke about the cache" + assert f"1 unfinished set was left in {cache}" in runner.stderr + assert f".tmp-{gone.pid}-0bad0bad. Nothing uses them" in runner.stderr + assert "UnfinishedSetWarning" in runner.stderr + assert left.exists(), "the unfinished set was removed" + + +def test_a_process_that_ended_is_not_alive_and_this_one_is(): + gone = subprocess.Popen([sys.executable, "-c", ""]) + gone.wait() + time.sleep(0.1) + assert _fileset._alive(os.getpid()) + assert not _fileset._alive(gone.pid) + + +def test_what_tfg_refuses_reaches_the_test_with_its_own_words_and_code(cache): + with pytest.raises(TfgError) as caught: + planned(cache, {"preset": "upload-validation", "options": {"limit": "1kb"}}) + assert caught.value.status == 4 + assert "smallest PDF" in str(caught.value) + + +def test_a_refused_plan_is_told_in_the_words_of_the_call_without_the_flags_this_package_adds(cache): + with pytest.raises(TfgError) as caught: + planned(cache, {"preset": "no-such-preset"}) + message = str(caught.value) + assert caught.value.status == 2 + assert message.startswith("file_set(preset='no-such-preset') could not be planned - tfg ended with code 2.\n") + assert 'unknown preset "no-such-preset"' in message + for added in ("dry-run", "--json", ".tmp-"): + assert added not in message + + +def test_an_option_tfg_does_not_know_is_named_with_the_flags_tfg_has(cache): + with pytest.raises(TfgError) as caught: + planned(cache, {**SMALL, "options": {**SMALL["options"], "limt": "2mb"}}) + err = caught.value + assert err.status == 2 + assert "gives the option limt, and tfg generate has no flag of that name." in str(err) + assert "flag provided" not in str(err), "the help of tfg is in the message" + assert "flag provided but not defined: -limt" in err.stderr + taken = str(err).split("Its flags are ", 1)[1].split(". Not every one goes with every set", 1)[0].split(", ") + for name in ("limit", "far-over", "bulk", "seed"): + assert name in taken + for name in ("out", "json", "dry-run", "preset"): + assert name not in taken, f"{name} is set by this package and is offered as an option" + assert "tfg preset show upload-validation" in str(err) + + +def test_a_call_that_names_no_files_to_write_is_told_so_before_tfg_runs(cache): + for spec in ({}, {"options": {}}, {"options": {"clean": False}}, {"options": {"set": []}}): + with pytest.raises(ValueError, match="file_set was not told which files to write"): + file_set(**spec, binary=tfg_binary(), cache_dir=cache) + assert not cache.exists(), "something ran" + + +def test_the_flags_this_package_sets_itself_cannot_be_set_again(cache): + for flag in ("out", "json", "dry-run", "dry_run", "preset"): + with pytest.raises(ValueError, match=f"The option {flag} cannot be set here"): + planned(cache, {"preset": "upload-validation", "options": {flag: "x"}}) + with pytest.raises(ValueError, match="not a flag tfg has"): + planned(cache, {"preset": "upload-validation", "options": {"--limit": "1mb"}}) + with pytest.raises(TypeError): + planned(cache, {"preset": "upload-validation", "options": {"limit": {"mb": 1}}}) + with pytest.raises(TypeError, match="options takes a dict"): + planned(cache, {"preset": "upload-validation", "options": ["limit=1mb"]}) + + +def test_a_manifest_of_another_major_or_an_outcome_this_package_does_not_know_is_refused(): + plan = {"manifest_version": "1.0", "run": {}, "files": [{"path": "a", "expected": {"outcome": "accept"}}]} + _fileset.readable(plan) + with pytest.raises(RuntimeError, match="manifest of version 2.0, and this package reads version 1"): + _fileset.readable({**plan, "manifest_version": "2.0"}) + with pytest.raises(RuntimeError, match='"quarantine", which this package does not know'): + _fileset.readable({**plan, "files": [{"path": "a", "expected": {"outcome": "quarantine"}}]}) + + +def test_sets_are_kept_in_a_directory_of_the_users_own_never_the_shared_temporary_one(): + user = _fileset._user_cache_dir + # Homes made up for the test, in no real profile. + assert user(env={"LOCALAPPDATA": r"D:\Local"}, system="win32", home=r"D:\profile") == \ + r"D:\Local\testing-files-generator" + assert user(env={}, system="win32", home=r"D:\profile") == r"D:\profile\AppData\Local\testing-files-generator" + assert user(env={}, system="darwin", home="/srv/profile") == "/srv/profile/Library/Caches/testing-files-generator" + assert user(env={"XDG_CACHE_HOME": "/xdg"}, system="linux", home="/srv/profile") == "/xdg/testing-files-generator" + # A relative XDG_CACHE_HOME is to be ignored, by the XDG specification. + assert user(env={"XDG_CACHE_HOME": "rel"}, system="linux", home="/srv/profile") == \ + "/srv/profile/.cache/testing-files-generator" + with pytest.raises(RuntimeError, match="found no home directory") as caught: + user(env={}, system="linux", home="~") + assert "TFG_CACHE_DIR" in str(caught.value) and "cache_dir" in str(caught.value) + + +def test_tfg_cache_dir_comes_first_and_then_the_cache_of_the_pytest_run(monkeypatch, tmp_path, pytestconfig): + monkeypatch.setenv("TFG_CACHE_DIR", str(tmp_path)) + assert _fileset.default_cache_dir() == str(tmp_path) + monkeypatch.delenv("TFG_CACHE_DIR") + assert _fileset._pytest_config is pytestconfig, "the plugin did not tell the package about this run" + if getattr(pytestconfig, "cache", None) is None: + # Run with -p no:cacheprovider: there is no cache of the run to keep sets in. + assert _fileset.default_cache_dir() == _fileset._user_cache_dir() + else: + assert _fileset.default_cache_dir() == str(pytestconfig.cache.mkdir("testing-files-generator")) + + +def test_no_warning_is_given_for_a_cache_with_nothing_left_behind(cache, monkeypatch): + monkeypatch.setattr(_fileset, "_noted", False) + with warnings.catch_warnings(): + warnings.simplefilter("error", UnfinishedSetWarning) + planned(cache) diff --git a/python/testing-files-generator/tests/test_main.py b/python/testing-files-generator/tests/test_main.py new file mode 100644 index 00000000..b5b6c300 --- /dev/null +++ b/python/testing-files-generator/tests/test_main.py @@ -0,0 +1,37 @@ +import subprocess + +from conftest import python, tfg_binary + +TFG = "import sys; from testing_files_generator.__main__ import main; sys.argv[0] = 'tfg'; sys.exit(main())" + + +def test_tfg_from_the_package_is_tfg_with_the_same_output_and_exit_code(): + args = ["generate", "--format", "nope", "--dry-run"] + via_package = python(TFG, *args) + direct = subprocess.run([tfg_binary(), *args], capture_output=True, text=True, encoding="utf-8", check=False) + assert direct.returncode != 0 + assert via_package.returncode == direct.returncode + assert via_package.stderr == direct.stderr + version = python(TFG, "version") + assert version.returncode == 0 + assert version.stdout == subprocess.run([tfg_binary(), "version"], capture_output=True, text=True, + encoding="utf-8", check=False).stdout + + +def test_a_package_with_no_binary_says_so_with_the_code_of_a_tool_that_could_not_run(tmp_path): + gone = python(TFG, "version", env={"TFG_BINARY": str(tmp_path / "gone")}) + assert gone.returncode == 1 + assert gone.stderr.startswith("tfg: TFG_BINARY is set to") + + +def test_a_binary_the_system_will_not_start_is_named_with_what_to_do(tmp_path): + # A file that is there and is not a program: Windows refuses it as not an + # application, Linux and macOS for the missing execute bit. + binary = tmp_path / "not-tfg" + binary.write_bytes(b"not a program\n") + binary.chmod(0o644) + refused = python(TFG, "version", env={"TFG_BINARY": str(binary)}) + assert refused.returncode == 1 + assert refused.stderr.startswith(f"tfg: tfg could not be started from {binary}: ") + assert "pip install --force-reinstall testing-files-generator" in refused.stderr + assert "TFG_BINARY" in refused.stderr diff --git a/python/testing-files-generator/tests/test_pytest.py b/python/testing-files-generator/tests/test_pytest.py new file mode 100644 index 00000000..84b3ec08 --- /dev/null +++ b/python/testing-files-generator/tests/test_pytest.py @@ -0,0 +1,112 @@ +"""The set as pytest sees it: one test for each file, by its title, its markers and its skip.""" + +import os +import re +import subprocess +import sys + +import pytest +from conftest import SMALL, SRC, tfg_binary + +from testing_files_generator import _fileset, file_set + + +def _module(cache): + return f""" +import pytest +from testing_files_generator import file_set + +uploads = file_set(**{SMALL!r}, cache_dir={str(cache)!r}, binary={tfg_binary()!r}) + + +@pytest.mark.parametrize("file", uploads.params()) +def test_upload(file): + file.check(file.expected["outcome"]) +""" + + +@pytest.fixture(autouse=True) +def _package_on_the_path(monkeypatch): + # The pytest a test here starts imports the package from src, as this one does. + monkeypatch.setenv("PYTHONPATH", SRC) + + +def _run(pytester, cache, *args, module=None): + pytester.makepyfile(test_uploads=module or _module(cache)) + pytester.makeini("[pytest]\n") + # The plugin is named here because the package runs from src. Where it is + # installed too, pytest would load it a second time through its entry + # point and refuse to start, so the entry point is blocked. + return pytester.runpytest_subprocess("-p", "no:testing_files_generator", + "-p", "testing_files_generator.pytest_plugin", "--strict-markers", + "-W", "error::pytest.PytestUnknownMarkWarning", "-p", "no:cacheprovider", + *args) + + +def test_every_file_is_a_test_and_a_file_with_no_right_answer_is_skipped_with_the_reason(pytester, tmp_path): + cache = tmp_path / "cache" + s = file_set(**SMALL, cache_dir=cache, binary=tfg_binary()) + unspecified = len(s.only(outcome="unspecified")) + result = _run(pytester, cache, "-rs") + result.assert_outcomes(passed=len(s) - unspecified, skipped=unspecified) + result.stdout.fnmatch_lines(["*has no expected outcome - what your system does with it is its own policy*"]) + + +def test_the_test_ids_are_the_titles_of_the_files(pytester, tmp_path): + cache = tmp_path / "cache" + s = file_set(**SMALL, cache_dir=cache, binary=tfg_binary()) + result = _run(pytester, cache, "--collect-only", "-q") + for f in s.files: + if f.title.isascii(): + result.stdout.fnmatch_lines([f"*test_upload[[]{f.title}[]]*"]) + + +def test_a_run_is_narrowed_by_the_expected_outcome_and_by_the_group(pytester, tmp_path): + cache = tmp_path / "cache" + s = file_set(**SMALL, cache_dir=cache, binary=tfg_binary()) + result = _run(pytester, cache, "-m", "reject") + result.assert_outcomes(passed=len(s.only(outcome="reject"))) + result = _run(pytester, cache, "-m", "bulk-upload") + result.assert_outcomes(passed=len(s.only(group="bulk-upload"))) + + +def test_the_report_header_names_the_tfg_the_run_uses(pytester, tmp_path, monkeypatch): + monkeypatch.setenv("TFG_BINARY", tfg_binary()) + result = _run(pytester, tmp_path / "cache", "--collect-only", "-q", "-v") + result.stdout.fnmatch_lines([f"testing-files-generator: tfg at {tfg_binary()}"]) + + +def test_the_markers_a_group_cannot_become_are_the_ones_pytest_registers_itself(tmp_path): + # Read from pytest and not written out a second time, so a marker a later + # pytest adds is refused too. + ini = tmp_path / "pytest.ini" + ini.write_text("[pytest]\n", encoding="utf-8") + listed = subprocess.run([sys.executable, "-m", "pytest", "--markers", "-p", "no:cacheprovider", "-c", str(ini)], + cwd=tmp_path, env={**os.environ, "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, + capture_output=True, text=True, encoding="utf-8", check=False) + assert listed.returncode == 0, listed.stderr + own = set(re.findall(r"^@pytest\.mark\.([A-Za-z0-9_-]+)", listed.stdout, re.M)) + assert "skip" in own, f"pytest --markers lists no skip, so this reads nothing:\n{listed.stdout}" + assert own == _fileset.PYTEST_OWN_MARKS + + +def test_a_group_that_would_become_one_of_pytests_own_markers_is_refused_before_a_test_runs(pytester, tmp_path): + cache = tmp_path / "cache" + recipe = tmp_path / "set.yaml" + recipe.write_text("version: 1\ntargets:\n - id: one\n format: txt\n size: 1kb\n group: skip\n", + encoding="utf-8") + module = f""" +import pytest +from testing_files_generator import file_set + +files = file_set(recipe={str(recipe)!r}, cache_dir={str(cache)!r}, binary={tfg_binary()!r}) + + +@pytest.mark.parametrize("file", files.params()) +def test_file(file): + assert False, "a file of a group named skip ran" +""" + result = _run(pytester, cache, module=module) + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines(["*The group skip of one_0001.txt becomes the pytest marker skip, which is one of " + "pytest's own*Give the group another name in the recipe.*"]) diff --git a/python/testing-files-generator/tests/test_run.py b/python/testing-files-generator/tests/test_run.py new file mode 100644 index 00000000..2dd34e97 --- /dev/null +++ b/python/testing-files-generator/tests/test_run.py @@ -0,0 +1,82 @@ +import os +import sys + +import pytest +from conftest import tfg_binary + +from testing_files_generator import TfgError +from testing_files_generator._run import KEPT_ERROR, _status, ended, run, run_to_file + + +def test_the_output_goes_to_a_new_file_only_its_owner_can_read(tmp_path): + out = tmp_path / "plan.json" + run_to_file(tfg_binary(), ["version"], str(out)) + assert out.read_text(encoding="utf-8").strip() + if sys.platform != "win32": + assert os.stat(out).st_mode & 0o777 == 0o600 + + +def test_a_file_already_under_that_name_is_not_written_through(tmp_path): + planted = tmp_path / "plan.json" + planted.write_text("someone else's", encoding="utf-8") + with pytest.raises(FileExistsError): + run_to_file(tfg_binary(), ["version"], str(planted)) + assert planted.read_text(encoding="utf-8") == "someone else's" + + +def test_tfg_ending_with_a_code_is_an_error_with_the_code_and_what_tfg_said(): + with pytest.raises(TfgError) as caught: + run(tfg_binary(), ["generate", "--no-such-flag"]) + assert caught.value.status == 2 + assert "flag provided but not defined: -no-such-flag" in caught.value.stderr + assert str(caught.value).startswith("tfg generate --no-such-flag ended with code 2.\n") + + +def test_a_binary_that_cannot_start_is_named_with_what_to_do(tmp_path): + with pytest.raises(RuntimeError) as caught: + run(str(tmp_path / "no-such-tfg"), ["version"]) + said = str(caught.value) + assert said.startswith(f"tfg could not be started from {tmp_path / 'no-such-tfg'}: ") + assert "pip install --force-reinstall testing-files-generator" in said and "TFG_BINARY" in said + + +@pytest.mark.skipif(sys.platform == "win32", reason="the execute bit is a thing of Linux and macOS") +def test_a_binary_without_its_execute_bit_keeps_the_reason_the_system_gave(tmp_path): + binary = tmp_path / "tfg" + binary.write_bytes(b"#!/bin/sh\n") + binary.chmod(0o644) + with pytest.raises(RuntimeError) as caught: + run(str(binary), ["version"]) + assert "Permission denied" in str(caught.value) + assert "TFG_BINARY" in str(caught.value) + + +def test_a_signal_is_the_status_by_its_name_and_a_number_with_no_name_is_still_a_signal(): + assert _status(-15) == "SIGTERM" + assert _status(-200) == "signal 200" + assert _status(3) == 3 + assert ended("SIGTERM") == "was stopped by SIGTERM" + assert ended(3) == "ended with code 3" + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows ends a process with a code, never a signal") +def test_tfg_stopped_by_a_signal_is_an_error_naming_the_signal(): + # Python stands in for tfg, so the run stops by a signal for certain. + script = "import os, signal; os.kill(os.getpid(), signal.SIGTERM)" + with pytest.raises(TfgError) as caught: + run(sys.executable, ["-c", script]) + assert caught.value.status == "SIGTERM" + assert str(caught.value).startswith(f"tfg -c {script} was stopped by SIGTERM.\n") + + +def test_only_the_end_of_a_long_error_stream_is_kept(): + # More than is kept, with the start and the end told apart, and a long + # line that does not fit in a pipe's buffer. + script = (f"import sys; sys.stderr.write('S' + 'x' * {KEPT_ERROR * 2} + 'the end'); sys.stderr.flush(); " + "sys.exit(5)") + with pytest.raises(TfgError) as caught: + run(sys.executable, ["-c", script]) + assert caught.value.status == 5 + assert len(caught.value.stderr) == KEPT_ERROR + assert caught.value.stderr.endswith("the end") + assert "S" not in caught.value.stderr diff --git a/python/testing-files-generator/tests/test_vectors.py b/python/testing-files-generator/tests/test_vectors.py new file mode 100644 index 00000000..00d59c12 --- /dev/null +++ b/python/testing-files-generator/tests/test_vectors.py @@ -0,0 +1,73 @@ +"""What a file says of itself and what check makes of what a system did. + +Read from the file the npm package's tests read as well +(testdata/wrappers/vectors.json), so a sentence changed in one package and not +in the other fails here or there. +""" + +import os + +import pytest +from conftest import VECTORS + +from testing_files_generator import ExpectationError, TestFile + + +class _Written: + def write(self): + return os.path.join(os.sep, "set") + + +def file(name): + assert name in VECTORS["entries"], f"vectors.json has no entry {name}" + return TestFile(_Written(), VECTORS["entries"][name]) + + +def test_every_file_says_what_the_vectors_say_it_says(): + assert len(VECTORS["files"]) >= 4 + for want in VECTORS["files"]: + f = file(want["entry"]) + assert f.title == want["title"], want["entry"] + assert f.marks == want["marks"], want["entry"] + assert f.why_unspecified == want["why_unspecified"], want["entry"] + + +def test_check_makes_of_every_observation_what_the_vectors_say_it_makes(): + kinds = set() + for c in VECTORS["checks"]: + label = f"{c['entry']} {c['observed']!r}" + kinds.add(c["result"]) + f = file(c["entry"]) + if c["result"] == "pass": + f.check(c["observed"]) + elif c["result"] == "mismatch": + with pytest.raises(ExpectationError) as caught: + f.check(c["observed"]) + assert str(caught.value) == c["message"], label + elif c["result"] == "unspecified": + with pytest.raises(RuntimeError, match="has no expected outcome, so there is nothing to check it against"): + f.check(c["observed"]) + elif c["result"] == "bad-input": + with pytest.raises(TypeError): + f.check(c["observed"]) + else: + pytest.fail(f"vectors.json has a result {c['result']} this test does not know") + # Every kind is there, so a vectors file that lost one is not green by + # having nothing left to compare. + assert sorted(kinds) == ["bad-input", "mismatch", "pass", "unspecified"] + + +def test_a_difference_carries_what_was_expected_and_what_was_seen(): + with pytest.raises(ExpectationError) as caught: + file("big").check({"outcome": "reject", "reason": "mime_mismatch"}) + assert caught.value.expected == {"outcome": "reject", "reason": "size_limit"} + assert caught.value.observed == {"outcome": "reject", "reason": "mime_mismatch"} + assert isinstance(caught.value, AssertionError), "pytest is to show a difference as a failed test" + + +def test_the_path_is_the_set_directory_joined_with_the_file(): + assert file("ok").path() == os.path.join(os.sep, "set", "ok.png") + + +def test_a_file_is_not_taken_for_a_test_class(): + assert TestFile.__test__ is False diff --git a/testdata/wrappers/vectors.json b/testdata/wrappers/vectors.json new file mode 100644 index 00000000..2886bd59 --- /dev/null +++ b/testdata/wrappers/vectors.json @@ -0,0 +1,228 @@ +{ + "about": "What a file of a set says of itself, and what check makes of what a system did, for the npm package (npm/testing-files-generator/test/vectors.test.js) and the Python package (python/testing-files-generator/tests/test_vectors.py). Both read this file, so a sentence changed in one package and not the other fails the tests of the other. result is pass, mismatch (with the whole message), unspecified or bad-input - the last two say what to do in the words of each language, so only the kind is shared. The groups are spelled differently on purpose - with a space, with a hyphen and with a punctuation mark (size limit, size-limit, hostile-names!) - because a group becomes a Playwright tag and a pytest marker, and these are the ways a group name can be written.", + "entries": { + "big": { + "path": "big.jpg", + "name": "big.jpg", + "target_id": "over", + "bytes": 9, + "format": "jpg", + "group": "size limit", + "purpose": "Taking it means the limit is not enforced.", + "expected": { + "outcome": "reject", + "reason": "size_limit" + } + }, + "ok": { + "path": "ok.png", + "name": "ok.png", + "target_id": "allowed", + "bytes": 67, + "format": "png", + "expected": { + "outcome": "accept" + } + }, + "empty": { + "path": "empty.jpg", + "name": "empty.jpg", + "target_id": "empty", + "bytes": 0, + "format": "txt", + "group": "degenerate", + "purpose": "An empty file.", + "expected": { + "outcome": "unspecified", + "reason": "size_zero" + } + }, + "under": { + "path": "under.jpg", + "name": "under.jpg", + "target_id": "under", + "bytes": 8, + "format": "jpg", + "group": "size-limit", + "purpose": "Turning it away means the limit is off by one.", + "expected": { + "outcome": "accept", + "reason": "size_limit" + } + }, + "spaced": { + "path": "a b.txt", + "name": "a b.txt", + "target_id": "names", + "bytes": 3, + "format": "txt", + "group": "hostile-names!", + "expected": { + "outcome": "sanitize" + } + } + }, + "files": [ + { + "entry": "big", + "title": "big.jpg - expected reject (size_limit)", + "marks": [ + "reject", + "size-limit" + ], + "why_unspecified": "big.jpg has no expected outcome - what your system does with it is its own policy. Taking it means the limit is not enforced." + }, + { + "entry": "ok", + "title": "ok.png - expected accept", + "marks": [ + "accept" + ], + "why_unspecified": "ok.png has no expected outcome - what your system does with it is its own policy." + }, + { + "entry": "empty", + "title": "empty.jpg - expected unspecified (size_zero)", + "marks": [ + "unspecified", + "degenerate" + ], + "why_unspecified": "empty.jpg has no expected outcome - what your system does with it is its own policy. An empty file." + }, + { + "entry": "spaced", + "title": "a b.txt - expected sanitize", + "marks": [ + "sanitize", + "hostile-names-" + ], + "why_unspecified": "a b.txt has no expected outcome - what your system does with it is its own policy." + } + ], + "checks": [ + { + "entry": "big", + "observed": "accept", + "result": "mismatch", + "message": "big.jpg was taken, and it should have been turned away (size_limit).\nTaking it means the limit is not enforced." + }, + { + "entry": "big", + "observed": "sanitize", + "result": "mismatch", + "message": "big.jpg was taken and cleaned, and it should have been turned away (size_limit).\nTaking it means the limit is not enforced." + }, + { + "entry": "big", + "observed": "reject", + "result": "pass" + }, + { + "entry": "big", + "observed": { + "outcome": "reject", + "reason": "size_limit" + }, + "result": "pass" + }, + { + "entry": "big", + "observed": { + "outcome": "reject" + }, + "result": "pass" + }, + { + "entry": "big", + "observed": { + "outcome": "reject", + "reason": "mime_mismatch" + }, + "result": "mismatch", + "message": "big.jpg was turned away for mime_mismatch, and the rule it is about is size_limit.\nTaking it means the limit is not enforced." + }, + { + "entry": "ok", + "observed": "reject", + "result": "mismatch", + "message": "ok.png was turned away, and it should have been taken." + }, + { + "entry": "ok", + "observed": "accept", + "result": "pass" + }, + { + "entry": "under", + "observed": { + "outcome": "accept", + "reason": "anything" + }, + "result": "pass" + }, + { + "entry": "under", + "observed": "reject", + "result": "mismatch", + "message": "under.jpg was turned away, and it should have been taken (size_limit).\nTurning it away means the limit is off by one." + }, + { + "entry": "spaced", + "observed": "accept", + "result": "mismatch", + "message": "a b.txt was taken, and it should have been taken and cleaned." + }, + { + "entry": "spaced", + "observed": "sanitize", + "result": "pass" + }, + { + "entry": "empty", + "observed": "accept", + "result": "unspecified" + }, + { + "entry": "empty", + "observed": "reject", + "result": "unspecified" + }, + { + "entry": "ok", + "observed": "toString", + "result": "bad-input" + }, + { + "entry": "ok", + "observed": "unspecified", + "result": "bad-input" + }, + { + "entry": "ok", + "observed": "ACCEPT", + "result": "bad-input" + }, + { + "entry": "ok", + "observed": null, + "result": "bad-input" + }, + { + "entry": "ok", + "observed": 3, + "result": "bad-input" + }, + { + "entry": "ok", + "observed": {}, + "result": "bad-input" + }, + { + "entry": "ok", + "observed": { + "outcome": "toString" + }, + "result": "bad-input" + } + ] +}