diff --git a/.github/workflows/repository-checks.yml b/.github/workflows/repository-checks.yml index f2f49d7..4407e23 100644 --- a/.github/workflows/repository-checks.yml +++ b/.github/workflows/repository-checks.yml @@ -30,6 +30,10 @@ jobs: run: python3 scripts/check-agent-data.py --self-test - name: Validate static release artifacts and rejection cases run: python3 scripts/test-release-artifact.py + - name: Validate Commons code artifacts and existing schema profiles + run: | + python3 scripts/test-commons-artifact.py + python3 -O scripts/test-commons-artifact.py - name: Validate release rehearsal identity and transport fixtures run: python3 scripts/test-release-rehearsal.py - name: Validate completed candidate identity and archive boundaries diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 71740f4..641154c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,6 +35,7 @@ More bounded ideas and acceptance criteria are on [Help request to agents](https | Fixed-command remote static observation | [Observer guide](docs/release-static-observer.md), [`scripts/static-remote-observe.py`](scripts/static-remote-observe.py), [`scripts/test-static-remote-observe.py`](scripts/test-static-remote-observe.py) | | Restricted static writes, retention and remote recovery | [Remote writer guide](docs/release-static-remote.md), [`scripts/static-remote-release.py`](scripts/static-remote-release.py), [`scripts/static_remote.py`](scripts/static_remote.py), [`scripts/static_retention.py`](scripts/static_retention.py), [`scripts/test-static-remote.py`](scripts/test-static-remote.py) | | End-to-end static publication | [Publication guide](docs/release-publication.md), [`scripts/static_publication.py`](scripts/static_publication.py), [`scripts/test-static-publication.py`](scripts/test-static-publication.py), [publication workflow](.github/workflows/static-publication.yml) | +| Commons code packaging and schema fingerprints | [Commons artifact guide](docs/release-commons-artifacts.md), [`scripts/commons_artifact.py`](scripts/commons_artifact.py), [`scripts/test-commons-artifact.py`](scripts/test-commons-artifact.py) | Generated `dist/` is intentionally ignored. Edit the authored source, then rebuild; changing a generated page will be lost. Keep the editable brand/social SVGs and their committed deliverables together, and respect [BRANDING.md](BRANDING.md). @@ -76,6 +77,8 @@ node --test services/commons/test/*.test.mjs node --test scripts/test-mission-handoff.mjs scripts/test-theme.mjs scripts/test-work-items-ui.mjs python3 scripts/check-agent-data.py --self-test python3 scripts/test-release-artifact.py +python3 scripts/test-commons-artifact.py +python3 -O scripts/test-commons-artifact.py python3 scripts/test-release-rehearsal.py python3 scripts/test-release-candidate.py python3 scripts/test-release-checks.py @@ -100,4 +103,4 @@ Voluntary testing contributions are welcome. The [local security-testing guide]( ## Production boundary -Merging source does not authorize a production deployment. Deployment credentials stay outside the repository, and production changes follow the separately documented review, backup, allowlist, and rollback gates. Website and Worker/database releases are separate operations; this contributor workflow changes neither hosting nor production data. +Eligible static changes merged into protected `main` are published automatically after the exact commit passes its required checks and release gates. Review the [publication outcome](docs/release-publication.md), including live verification and retained rollback. Deployment credentials stay outside the repository. Commons Worker and schema promotion remain separate operations; the offline [Commons artifact tools](docs/release-commons-artifacts.md) prepare and verify a code package without contacting production. diff --git a/docs/README.md b/docs/README.md index 36ed495..28940b0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -117,6 +117,7 @@ publication and recovery: | [Remote observer](release-static-observer.md) | Reading one independently bound installation through a restricted SSH command. | | [Remote writer](release-static-remote.md) | Applying and recovering static file changes under an independently installed policy. | | [Static publication](release-publication.md) | Joining the candidate, checks, restricted endpoint and live verification; configuration, pilot and recovery requirements. | +| [Commons artifacts](release-commons-artifacts.md) | Packaging the six production modules, pinning the existing schema profile and independently checking the captured bytes. This is the offline foundation for separate Worker promotion. | Automatic static publication is enabled for eligible changes merged into protected `main`. The deployed Commons source and static server policy must remain compatible; diff --git a/docs/release-automation.md b/docs/release-automation.md index 5c61bf8..203ac9d 100644 --- a/docs/release-automation.md +++ b/docs/release-automation.md @@ -30,7 +30,7 @@ rollback and republication pilot are verified; see the | Constrained static adapter | Installed and verified with a separate restricted identity. | | Trusted candidate and required checks | Canonical artifact transport, independent rebuild and exact protected-commit checks are active. | | Static promotion | Enabled after publication, retained rollback and republication verification. | -| Worker and database promotion | Planned separately; static jobs cannot update either. | +| Worker and database promotion | Offline [code packaging and schema checks](release-commons-artifacts.md) are implemented; constrained remote promotion and schema migration remain separate work. Static jobs cannot update either. | The target is exclusively OSS Singularity: its static destination, Commons Worker, dedicated D1 database and necessary cache invalidation. The existing diff --git a/docs/release-commons-artifacts.md b/docs/release-commons-artifacts.md new file mode 100644 index 0000000..9221939 --- /dev/null +++ b/docs/release-commons-artifacts.md @@ -0,0 +1,114 @@ +# Commons code artifacts + +The offline Commons artifact tools create a reproducible code package containing +the six production ES modules and a descriptor bound to a caller-supplied commit +and existing schema profile. This is the first component of the separate Worker +release stage in [Release automation](release-automation.md). It does not publish +a Worker, apply SQL to D1 or replace the current +[deployment handoff](../services/commons/README.md#deployment-handoff). + +## Create and check a candidate + +Run from a reviewed checkout with Python 3.12 or later on Linux: + +```sh +candidate_dir=$(mktemp -d) +candidate_commit=$(git rev-parse HEAD) +python3 scripts/commons-artifact.py create \ + --source-dir services/commons \ + --commit "$candidate_commit" \ + --out "$candidate_dir/commons.json" +python3 scripts/commons-artifact.py verify \ + --candidate "$candidate_dir/commons.json" \ + --expected-commit "$candidate_commit" \ + --expected-schema-sha256 1d1800a100d598b2076c4932ad866e4e254595326eb7fb387522b51775f01278 \ + --rebuild-source services/commons +``` + +The schema fingerprint above belongs to profile 1: the existing three pinned +migrations. Verification receives the expected commit and schema independently; +copying those values from an untrusted packet would only check self-consistency. +A supplied commit is a claim until the separate canonical-source, successful-CI +and trusted-artifact-transport gates establish its provenance. + +`create` writes a new owner-readable file outside the service source directory. +It never overwrites an earlier candidate. `verify` opens the captured packet +again and can compare it with a separate source capture. The JSON result contains +module sizes and SHA-256 digests, schema requirements, `artifact_verified`, +`rebuild_matched` and the remaining gates. `deployment_authorized` is always +`false`. Keep or remove the temporary candidate directory as appropriate. + +## Package and source boundaries + +The packet contains exactly two top-level fields: `descriptor` and `files`. +Each production module is transported as canonical Base64 with its byte count +and SHA-256 digest in the descriptor. JSON key ordering and serialization are +deterministic. Reordered input keys are accepted only when all validated values +and decoded bytes still match; duplicate keys are rejected. + +The only packaged files are `worker.mjs`, `security.mjs`, `identity.mjs`, +`participations.mjs`, `activity.mjs` and `work-items.mjs`. Local servers, database +adapters, tests, configuration and migration SQL are excluded. The source capture +rejects an unknown root `.mjs` file or any unexpected migration file, so an +extension cannot silently omit a newly introduced module or migration. + +Profile 1 fixes the module entry point, compatibility date `2026-09-04`, no +compatibility flags and the exact hashes of the three existing migration files. +It does not accept arbitrary bindings, credentials, provider resource IDs, +upload destinations or extra descriptor fields. A future module or migration +requires a deliberate profile update and the corresponding release evidence. + +The limits are 512 KiB per module, 2 MiB of decoded code and a 3 MiB packet. +Modules must be nonempty UTF-8 without NUL bytes. Reads use the existing bounded, +no-follow directory-descriptor implementation and reject symlinks, hardlinks, +special files and source changes during capture. Errors crossing the CLI boundary +contain fixed codes, without payload contents or supplied paths. + +## Schema identity and its limits + +Only the three hash-pinned initialization files are evaluated, in a fresh +in-memory SQLite database. Digest validation finishes before SQL evaluation. +The packet carries migration hashes and a schema fingerprint; it carries no SQL +to execute. The tools never receive a production database path or credential. + +`schema_hash` accepts bounded `sqlite_master` metadata: object type, name, table +name and SQL. It includes tables, indexes, triggers and views. It removes SQL +formatting and comments outside quoted values while preserving token boundaries, +quoted identifiers and literal contents. For example, changing a default from +`'a b'` to `'a b'` changes the fingerprint. It never executes observed SQL or +reads stored application rows. + +The fixed metadata query excludes SQLite's internal objects and D1's `_cf_KV` +object. On 2026-09-08, a separate read-only comparison matched all 35 existing +schema objects and the six deployed modules; no application rows were read and +no production writes were performed. That observation does not replace a fresh +check immediately before a future release. + +A matching schema fingerprint establishes structural identity. It does not prove +that changed application queries or behavior are compatible with existing data. +The service tests, exact source checks and later live acceptance remain required. +Changed migrations fail this code-only profile; they need their own rehearsal, +private backup and preservation evidence. A Worker rollback must retain newer +community records and additive tables. + +## Validation and remaining integration + +Run the artifact tests in both normal and optimized Python: + +```sh +python3 scripts/test-commons-artifact.py +python3 -O scripts/test-commons-artifact.py +``` + +They exercise real source capture and a CLI round trip, independent rebuild +mismatch, changed initialization before SQL execution, schema literal changes, +malformed or oversized packets, wrong commits, extra modules, altered runtime +fields, unsafe filesystem references and private error-output boundaries. +The [complete contributor checks](../CONTRIBUTING.md#before-opening-a-pull-request) +also exercise Commons behavior with real SQLite transactions. + +The remaining Worker stage needs authenticated candidate provenance, an +independently bound destination and schema, constrained provider access, +serialization, a durable deployment journal, uncertain-outcome reconciliation +and live acceptance with conditional recovery. Neither the existing static +publication workflow nor this packet grants Worker or database write access. diff --git a/scripts/commons-artifact.py b/scripts/commons-artifact.py new file mode 100644 index 0000000..840b153 --- /dev/null +++ b/scripts/commons-artifact.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Create or verify a Commons code candidate offline, without deployment authority.""" +import argparse +import json +from pathlib import Path +import sys + +import commons_artifact as artifact +from site_artifact import ArtifactError + + +class Parser(argparse.ArgumentParser): + def error(self, _message): + raise ArtifactError('invalid_arguments') + + +def main(argv=None): + parser = Parser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True, parser_class=Parser) + create = commands.add_parser('create') + create.add_argument('--source-dir', type=Path, required=True) + create.add_argument('--commit', required=True) + create.add_argument('--out', type=Path, required=True) + verify = commands.add_parser('verify') + verify.add_argument('--candidate', type=Path, required=True) + verify.add_argument('--expected-commit', required=True) + verify.add_argument('--expected-schema-sha256', required=True) + verify.add_argument('--rebuild-source', type=Path) + args = parser.parse_args(argv) + if args.command == 'create': + result = artifact.create(args.source_dir, args.commit, args.out) + else: + result = artifact.verify(args.candidate, args.expected_commit, args.expected_schema_sha256, args.rebuild_source) + print(json.dumps(result, sort_keys=True, indent=2)) + return 0 + + +if __name__ == '__main__': + try: + raise SystemExit(main()) + except ArtifactError as error: + print(json.dumps({'error': error.code}), file=sys.stderr) + raise SystemExit(1) from None + except OSError: + print(json.dumps({'error': 'unsafe_or_unavailable_path'}), file=sys.stderr) + raise SystemExit(1) from None + except Exception: + print(json.dumps({'error': 'verification_failed'}), file=sys.stderr) + raise SystemExit(1) from None diff --git a/scripts/commons_artifact.py b/scripts/commons_artifact.py new file mode 100644 index 0000000..c52ccae --- /dev/null +++ b/scripts/commons_artifact.py @@ -0,0 +1,252 @@ +"""Offline, bounded transport for Commons code and its existing schema profile. + +Nothing here executes Worker code, contacts a provider or applies a migration. +Only hash-pinned initialization SQL is evaluated in a fresh in-memory database. +""" +import base64 +import binascii +import hashlib +import json +import os +from pathlib import Path +import re +import sqlite3 + +from site_artifact import ArtifactError, TreeReader, fingerprint, open_directory, read_external, require + +REPOSITORY = 'oss-singularity/website' +MODULES = {'worker.mjs', 'security.mjs', 'identity.mjs', 'participations.mjs', 'activity.mjs', 'work-items.mjs'} +LOCAL_MODULES = {'local-d1.mjs', 'dev-server.mjs'} +MIGRATIONS = { + '0001_commons.sql': '3832201d0a7d80c9b33e8908fc2b91354619d8326f69e8c4750363b38a98345e', + '0002_participations.sql': '5a8f020d34c307b4fe87c66907299639c3eb6efb6b535cc0949bd96c72289ad3', + '0003_work_items.sql': '45960c242ae7d41b5ed960724e3bdef8ef887670c32b5773c7d2e34137017fa0', +} +RUNTIME = {'entrypoint': 'worker.mjs', 'compatibility_date': '2026-09-04', 'compatibility_flags': []} +MAX_MODULE = 512 * 1024 +MAX_CODE = 2 * 1024 * 1024 +MAX_PACKET = 3 * 1024 * 1024 +MAX_SCHEMA = 256 * 1024 +SCHEMA_QUERY = ("SELECT type,name,tbl_name,sql FROM sqlite_master WHERE sql IS NOT NULL " + "AND name NOT GLOB 'sqlite_*' AND name != '_cf_KV' ORDER BY type,name") +PENDING = ['canonical-source-and-checks', 'trusted-artifact-transport', 'fresh-installed-schema', + 'scoped-provider-access', 'serialized-promotion', 'durable-recovery', 'live-verification'] + + +def encode(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode('utf-8') + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def commit(value): + require(type(value) is str and re.fullmatch(r'[a-f0-9]{40}', value) is not None, 'invalid_commit') + return value + + +def hash_value(value): + require(type(value) is str and re.fullmatch(r'[a-f0-9]{64}', value) is not None, 'invalid_digest') + return value + + +def sql_tokens(sql): + """Ignore formatting outside SQL quotes, preserving literal and token boundaries.""" + require(type(sql) is str and 0 < len(sql.encode('utf-8')) <= MAX_SCHEMA + and all(ord(c) >= 32 or c in '\t\r\n' for c in sql), 'invalid_schema') + tokens, position = [], 0 + while position < len(sql): + c = sql[position] + if c in ' \t\r\n': + position += 1 + continue + if sql.startswith('--', position): + end = sql.find('\n', position + 2) + position = len(sql) if end < 0 else end + 1 + continue + if sql.startswith('/*', position): + end = sql.find('*/', position + 2) + require(end >= 0, 'invalid_schema') + position = end + 2 + continue + start = position + if c in "'\"`[": + terminator = ']' if c == '[' else c + position += 1 + while position < len(sql): + if sql[position] == terminator: + position += 1 + if c != '[' and position < len(sql) and sql[position] == terminator: + position += 1 + continue + break + position += 1 + else: + raise ArtifactError('invalid_schema') + elif c.isalnum() or c in '_$': + position += 1 + while position < len(sql) and (sql[position].isalnum() or sql[position] in '_$'): + position += 1 + else: + position += 1 + tokens.append(sql[start:position]) + require(bool(tokens), 'invalid_schema') + return tokens + + +def schema_hash(rows): + """Fingerprint schema metadata only; never execute observed SQL or read data rows.""" + require(type(rows) is list and 0 < len(rows) <= 256 and len(encode(rows)) <= MAX_SCHEMA, 'invalid_schema') + seen, normalized = set(), [] + for row in rows: + require(type(row) is dict and set(row) == {'type', 'name', 'tbl_name', 'sql'}, 'invalid_schema') + require(type(row['type']) is str and row['type'] in {'table', 'index', 'trigger', 'view'}, 'invalid_schema') + for key in ['name', 'tbl_name']: + require(type(row[key]) is str and re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]{0,127}', row[key]), 'invalid_schema') + key = (row['type'], row['name']) + require(key not in seen, 'invalid_schema') + seen.add(key) + normalized.append({**row, 'sql': sql_tokens(row['sql'])}) + return digest(encode(sorted(normalized, key=lambda row: (row['type'], row['name'])))) + + +def expected_schema(migrations): + require(type(migrations) is dict and set(migrations) == set(MIGRATIONS), 'schema_profile_changed') + for name, raw in migrations.items(): + require(type(raw) is bytes and digest(raw) == MIGRATIONS[name], 'schema_profile_changed') + # Digest validation precedes every SQL evaluation. Candidate packets never + # supply SQL to this function, and no caller-supplied database path exists. + with sqlite3.connect(':memory:') as database: + database.row_factory = sqlite3.Row + for name in sorted(migrations): + database.executescript(migrations[name].decode('utf-8')) + rows = [dict(row) for row in database.execute(SCHEMA_QUERY)] + return schema_hash(rows) + + +def validate_code(files): + require(type(files) is dict and set(files) == MODULES, 'module_allowlist_mismatch') + total = 0 + for raw in files.values(): + require(type(raw) is bytes and 0 < len(raw) <= MAX_MODULE and b'\0' not in raw, 'invalid_module') + try: + raw.decode('utf-8') + except UnicodeError: + raise ArtifactError('invalid_module') from None + total += len(raw) + require(total <= MAX_CODE, 'code_size_limit') + return {name: {'sha256': digest(files[name]), 'size': len(files[name])} for name in sorted(files)} + + +def source_files(root): + """Capture only the declared production files; detect unknown root modules/migrations.""" + reader = TreeReader(root) + try: + reader.files, reader.directories = reader.inventory() + require({name for name in reader.files if '/' not in name and name.endswith('.mjs')} + == MODULES | LOCAL_MODULES, 'source_module_allowlist_mismatch') + require({name for name in reader.files if name.startswith('migrations/')} + == {'migrations/' + name for name in MIGRATIONS}, 'schema_profile_changed') + require(all(0 < reader.files[name][4] <= MAX_MODULE for name in MODULES), 'invalid_module') + require(sum(reader.files[name][4] for name in MODULES) <= MAX_CODE, 'code_size_limit') + files = {name: reader.read(name) for name in sorted(MODULES)} + migrations = {name: reader.read('migrations/' + name) for name in sorted(MIGRATIONS)} + require(reader.inventory() == (reader.files, reader.directories), 'tree_changed') + current = open_directory(reader.root) + try: + require(fingerprint(os.fstat(current)) == reader.root_identity, 'tree_changed') + finally: + os.close(current) + return files, expected_schema(migrations) + finally: + reader.close() + + +def metadata(files, sha, schema): + return {'schema_version': 1, 'kind': 'commons-code', 'repository': REPOSITORY, 'commit': commit(sha), + 'runtime': {**RUNTIME, 'compatibility_flags': []}, + 'schema': {'profile': 1, 'migrations': dict(MIGRATIONS), 'sha256': hash_value(schema)}, + 'modules': validate_code(files)} + + +def packet(files, sha, schema): + return encode({'descriptor': metadata(files, sha, schema), + 'files': {name: base64.b64encode(raw).decode('ascii') for name, raw in sorted(files.items())}}) + + +def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'duplicate_packet_key') + result[key] = value + return result + + +def invalid_constant(_value): + raise ArtifactError('invalid_packet') + + +def unpack(raw, sha, expected_schema_sha): + commit(sha) + hash_value(expected_schema_sha) + require(type(raw) is bytes and 0 < len(raw) <= MAX_PACKET, 'packet_size_limit') + try: + value = json.loads(raw.decode('utf-8'), object_pairs_hook=unique, parse_constant=invalid_constant) + require(type(value) is dict and set(value) == {'descriptor', 'files'}, 'invalid_packet') + encoded = value['files'] + require(type(encoded) is dict and set(encoded) == MODULES, 'module_allowlist_mismatch') + files = {} + for name, content in encoded.items(): + require(type(content) is str and len(content) <= 4 * ((MAX_MODULE + 2) // 3), 'invalid_module') + data = base64.b64decode(content, validate=True) + require(base64.b64encode(data).decode('ascii') == content, 'invalid_module') + files[name] = data + descriptor = value['descriptor'] + require(type(descriptor) is dict and type(descriptor.get('schema_version')) is int, 'invalid_descriptor') + schema = descriptor.get('schema') + require(type(schema) is dict and type(schema.get('profile')) is int, 'invalid_descriptor') + sizes = descriptor.get('modules') + require(type(sizes) is dict and set(sizes) == MODULES, 'invalid_descriptor') + require(all(type(row) is dict and type(row.get('size')) is int for row in sizes.values()), 'invalid_descriptor') + require(descriptor == metadata(files, sha, expected_schema_sha), 'descriptor_mismatch') + return files, descriptor + except (UnicodeError, json.JSONDecodeError, RecursionError, binascii.Error, ValueError) as error: + if isinstance(error, ArtifactError): + raise + raise ArtifactError('invalid_packet') from None + + +def report(descriptor, rebuilt=False): + return {**descriptor, 'artifact_verified': True, 'rebuild_matched': rebuilt, + 'deployment_authorized': False, 'pending_gates': list(PENDING)} + + +def create(source, sha, destination): + source, destination = Path(os.path.abspath(source)), Path(os.path.abspath(destination)) + require(source not in destination.parents, 'output_inside_source') + files, schema = source_files(source) + raw = packet(files, sha, schema) + # Reopen the exact output independently in verify; never overwrite an older + # artifact or follow an output symlink. Parent directories must already exist. + parent = TreeReader(destination.parent) + try: + fd = os.open(destination.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=parent.fd) + try: + with os.fdopen(fd, 'wb', closefd=False) as stream: + stream.write(raw) + stream.flush() + os.fsync(fd) + finally: + os.close(fd) + finally: + parent.close() + return report(metadata(files, sha, schema)) + + +def verify(path, sha, schema, source=None): + files, descriptor = unpack(read_external(path, MAX_PACKET), sha, schema) + if source is not None: + rebuilt, rebuilt_schema = source_files(source) + require(files == rebuilt and schema == rebuilt_schema, 'rebuild_mismatch') + return report(descriptor, source is not None) diff --git a/scripts/test-commons-artifact.py b/scripts/test-commons-artifact.py new file mode 100644 index 0000000..542da3d --- /dev/null +++ b/scripts/test-commons-artifact.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Real source capture, transport rejection and literal-preserving schema fixtures.""" +import base64 +import json +import os +from pathlib import Path +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import commons_artifact as a +from site_artifact import ArtifactError + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / 'services' / 'commons' +SHA = 'a' * 40 + + +class CommonsArtifactTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.files, cls.schema = a.source_files(SOURCE) + cls.raw = a.packet(cls.files, SHA, cls.schema) + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix='oss-commons-artifact-test-') + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + + def source(self): + path = self.root / 'source' + shutil.copytree(SOURCE, path) + return path + + def rejection(self, value, code=None): + with self.assertRaises(ArtifactError) as caught: + a.unpack(a.encode(value) if type(value) is dict else value, SHA, self.schema) + if code: + self.assertEqual(caught.exception.code, code) + + def test_real_source_roundtrip_and_cli_rebuild(self): + path = self.root / 'candidate.json' + result = a.create(SOURCE, SHA, path) + self.assertEqual(path.read_bytes(), self.raw) + self.assertFalse(result['deployment_authorized']) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + command = [sys.executable, *(['-O'] if sys.flags.optimize else []), + str(ROOT / 'scripts/commons-artifact.py'), 'verify', '--candidate', str(path), + '--expected-commit', SHA, '--expected-schema-sha256', self.schema, + '--rebuild-source', str(SOURCE)] + process = subprocess.run(command, capture_output=True, timeout=20, check=False) + self.assertEqual(process.returncode, 0, process.stderr) + result = json.loads(process.stdout) + self.assertTrue(result['artifact_verified'] and result['rebuild_matched']) + self.assertFalse(result['deployment_authorized']) + self.assertEqual(set(result['modules']), a.MODULES) + self.assertIn('fresh-installed-schema', result['pending_gates']) + + def test_no_local_helpers_tests_or_config_in_payload(self): + value = json.loads(self.raw) + self.assertEqual(set(value['files']), a.MODULES) + for excluded in ['local-d1.mjs', 'dev-server.mjs', 'wrangler.example.toml', 'ADMIN_TOKEN', 'database_id']: + self.assertNotIn(excluded, value['files']) + self.assertEqual(set(value['descriptor']), {'schema_version', 'kind', 'repository', 'commit', 'runtime', 'schema', 'modules'}) + + def test_source_changes_are_detected_by_independent_rebuild(self): + source = self.source() + path = self.root / 'candidate.json' + a.create(source, SHA, path) + with (source / 'worker.mjs').open('ab') as stream: + stream.write(b'\n// another source revision\n') + with self.assertRaisesRegex(ArtifactError, 'rebuild_mismatch'): + a.verify(path, SHA, self.schema, source) + + def test_worker_code_is_captured_without_execution(self): + source = self.source() + (source / 'worker.mjs').write_text("throw new Error('do not execute candidate code');\n") + files, schema = a.source_files(source) + self.assertEqual(files['worker.mjs'], (source / 'worker.mjs').read_bytes()) + self.assertEqual(schema, self.schema) + + def test_unknown_root_module_is_not_silently_dropped(self): + source = self.source() + (source / 'unexpected.mjs').write_text('export default {};\n') + with self.assertRaisesRegex(ArtifactError, 'source_module_allowlist_mismatch'): + a.source_files(source) + + def test_missing_production_module_rejected(self): + source = self.source() + (source / 'security.mjs').unlink() + with self.assertRaisesRegex(ArtifactError, 'source_module_allowlist_mismatch'): + a.source_files(source) + + def test_changed_or_extra_migrations_are_rejected_before_sql_evaluation(self): + source = self.source() + first = source / 'migrations/0001_commons.sql' + first.write_text("ATTACH DATABASE '/not-an-allowed-target' AS stolen;") + with patch.object(a.sqlite3, 'connect', side_effect=AssertionError('SQL must not run')): + with self.assertRaisesRegex(ArtifactError, 'schema_profile_changed'): + a.source_files(source) + first.write_bytes((SOURCE / 'migrations/0001_commons.sql').read_bytes()) + (source / 'migrations/0004_future.sql').write_text('CREATE TABLE future (id TEXT);') + with self.assertRaisesRegex(ArtifactError, 'schema_profile_changed'): + a.source_files(source) + + def test_modified_initialization_cannot_be_rebound_to_a_packet(self): + value = json.loads(self.raw) + value['descriptor']['schema']['migrations']['0001_commons.sql'] = 'b' * 64 + self.rejection(value, 'descriptor_mismatch') + value = json.loads(self.raw) + value['descriptor']['schema']['sha256'] = 'b' * 64 + self.rejection(value, 'descriptor_mismatch') + + def test_schema_tokens_preserve_literal_and_identifier_boundaries(self): + prefix = 'CREATE TABLE example (text TEXT DEFAULT ' + self.assertEqual(a.sql_tokens(prefix + "'a b' )"), a.sql_tokens(prefix + "'a b'\n)")) + self.assertNotEqual(a.sql_tokens(prefix + "'a b')"), a.sql_tokens(prefix + "'a b')")) + self.assertNotEqual(a.sql_tokens('SELECT a b'), a.sql_tokens('SELECT ab')) + self.assertNotEqual(a.sql_tokens('SELECT "a b"'), a.sql_tokens('SELECT "ab"')) + self.assertEqual(a.sql_tokens("SELECT 'it''s /* literal */' -- comment\n"), + ['SELECT', "'it''s /* literal */'"]) + self.assertEqual(a.sql_tokens('SELECT /* formatting */ [a b]'), ['SELECT', '[a b]']) + for invalid in ["SELECT 'broken", '/* broken', 'SELECT "broken', 'SELECT [broken', '\0', '-- only comment']: + with self.subTest(invalid=invalid), self.assertRaises(ArtifactError): + a.sql_tokens(invalid) + + def test_schema_hash_uses_structure_and_preserves_literal_values(self): + def schema(sql): + with sqlite3.connect(':memory:') as database: + database.row_factory = sqlite3.Row + database.execute(sql) + return [dict(row) for row in database.execute(a.SCHEMA_QUERY)] + first = schema("CREATE TABLE example (text TEXT DEFAULT 'a b')") + formatted = schema("CREATE TABLE example\n( text TEXT DEFAULT 'a b' )") + changed = schema("CREATE TABLE example (text TEXT DEFAULT 'a b')") + self.assertEqual(a.schema_hash(first), a.schema_hash(formatted)) + self.assertNotEqual(a.schema_hash(first), a.schema_hash(changed)) + for bad in [[], first + first, [{**first[0], 'type': []}], [{**first[0], 'sql': '-- absent'}], + [{**first[0], 'data_rows': ['never accepted']}]]: + with self.subTest(bad=bad), self.assertRaises(ArtifactError): + a.schema_hash(bad) + + def test_internal_name_filter_does_not_hide_application_objects(self): + with sqlite3.connect(':memory:') as database: + database.row_factory = sqlite3.Row + database.execute('CREATE TABLE sqliteXapplication (id TEXT)') + database.execute('CREATE TABLE _cf_KV (key TEXT)') + rows = [dict(row) for row in database.execute(a.SCHEMA_QUERY)] + self.assertEqual([row['name'] for row in rows], ['sqliteXapplication']) + + def test_tampered_bytes_and_self_consistent_wrong_commit_rejected(self): + value = json.loads(self.raw) + value['files']['worker.mjs'] = base64.b64encode(b'export default {};').decode() + self.rejection(value, 'descriptor_mismatch') + self.rejection(a.packet(self.files, 'b' * 40, self.schema), 'descriptor_mismatch') + + def test_foreign_runtime_and_extra_authority_fields_rejected(self): + variants = [] + for key, replacement in [('repository', 'other/project'), ('kind', 'static-site'), ('schema_version', True)]: + value = json.loads(self.raw) + value['descriptor'][key] = replacement + variants.append(value) + for key, replacement in [('entrypoint', '../worker.mjs'), ('compatibility_date', '2020-01-01'), + ('compatibility_flags', ['nodejs_compat'])]: + value = json.loads(self.raw) + value['descriptor']['runtime'][key] = replacement + variants.append(value) + value = json.loads(self.raw) + value['descriptor']['provider_token'] = 'not-an-accepted-field' + variants.append(value) + value = json.loads(self.raw) + value['descriptor']['schema']['profile'] = True + variants.append(value) + value = json.loads(self.raw) + value['descriptor']['modules']['worker.mjs']['size'] = float(len(self.files['worker.mjs'])) + variants.append(value) + for value in variants: + with self.subTest(descriptor=value['descriptor']): + self.rejection(value) + + def test_extra_missing_and_unsafe_payload_members_rejected(self): + for name in ['dev-server.mjs', '../worker.mjs', '/worker.mjs', 'worker.mjs/extra']: + value = json.loads(self.raw) + value['files'][name] = 'YQ==' + self.rejection(value, 'module_allowlist_mismatch') + value = json.loads(self.raw) + del value['files']['security.mjs'] + self.rejection(value, 'module_allowlist_mismatch') + + def test_invalid_json_and_base64_are_bounded(self): + for raw in [b'', b'[]', b'{}', b'\xff', self.raw + b'{}', b'{"files":{},"files":{}}', + b'{"files": NaN}', b'[' * 1500 + b']' * 1500, b' ' * (a.MAX_PACKET + 1)]: + with self.subTest(size=len(raw)): + self.rejection(raw) + for encoded in ['!', 'YQ', 'YR==', 'YQ==\n', '', base64.b64encode(b'\xff').decode(), + base64.b64encode(b'\0').decode(), 'A' * (4 * ((a.MAX_MODULE + 2) // 3) + 1)]: + value = json.loads(self.raw) + value['files']['worker.mjs'] = encoded + self.rejection(value) + + def test_no_follow_for_source_payload_output_and_parent_aliases(self): + source = self.source() + target = source / 'worker.mjs' + target.unlink() + target.symlink_to(SOURCE / 'worker.mjs') + with self.assertRaises((ArtifactError, OSError)): + a.source_files(source) + path = self.root / 'candidate.json' + path.write_bytes(self.raw) + alias = self.root / 'candidate-alias.json' + alias.symlink_to(path) + with self.assertRaises((ArtifactError, OSError)): + a.verify(alias, SHA, self.schema) + with self.assertRaises((ArtifactError, OSError)): + a.create(SOURCE, SHA, alias) + directory_alias = self.root / 'parent-alias' + directory_alias.symlink_to(SOURCE, target_is_directory=True) + with self.assertRaises((ArtifactError, OSError)): + a.source_files(directory_alias) + self.assertEqual(path.read_bytes(), self.raw) + + def test_hardlinks_existing_outputs_and_outputs_in_source_rejected(self): + source = self.source() + os.link(source / 'worker.mjs', self.root / 'linked.mjs') + with self.assertRaisesRegex(ArtifactError, 'unsafe_file'): + a.source_files(source) + path = self.root / 'existing.json' + path.write_text('preserve this file') + with self.assertRaises(FileExistsError): + a.create(SOURCE, SHA, path) + self.assertEqual(path.read_text(), 'preserve this file') + with self.assertRaisesRegex(ArtifactError, 'output_inside_source'): + a.create(SOURCE, SHA, SOURCE / 'never-created.json') + + def test_source_size_limits_are_checked_before_reading_code(self): + source = self.source() + (source / 'worker.mjs').write_bytes(b'x' * (a.MAX_MODULE + 1)) + with patch.object(a.TreeReader, 'read', side_effect=AssertionError('oversized input must not be read')): + with self.assertRaisesRegex(ArtifactError, 'invalid_module'): + a.source_files(source) + for name in a.MODULES: + (source / name).write_bytes(b'x' * 360000) + with self.assertRaisesRegex(ArtifactError, 'code_size_limit'): + a.source_files(source) + with self.assertRaisesRegex(ArtifactError, 'code_size_limit'): + a.packet({name: b'x' * 360000 for name in a.MODULES}, SHA, self.schema) + + def test_replacing_source_directory_during_capture_is_rejected(self): + source = self.source() + original = a.TreeReader.read + replaced = False + + def replace(reader, name): + nonlocal replaced + raw = original(reader, name) + if name == 'worker.mjs' and not replaced: + replaced = True + moved = self.root / 'moved-source' + source.rename(moved) + shutil.copytree(moved, source) + return raw + + with patch.object(a.TreeReader, 'read', new=replace): + with self.assertRaisesRegex(ArtifactError, 'tree_changed'): + a.source_files(source) + + def test_cli_errors_do_not_echo_payload_or_path(self): + path = self.root / 'private-name.json' + path.write_text('{"private-value": "never log me"}') + command = [sys.executable, *(['-O'] if sys.flags.optimize else []), str(ROOT / 'scripts/commons-artifact.py'), + 'verify', '--candidate', str(path), '--expected-commit', SHA, '--expected-schema-sha256', self.schema] + result = subprocess.run(command, capture_output=True, timeout=20, check=False) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, b'') + self.assertEqual(json.loads(result.stderr), {'error': 'invalid_packet'}) + self.assertNotIn(b'private', result.stderr) + + def test_descriptor_cannot_mutate_shared_profile(self): + value = a.metadata(self.files, SHA, self.schema) + value['schema']['migrations'].clear() + value['runtime']['compatibility_flags'].append('unexpected') + self.assertEqual(len(a.MIGRATIONS), 3) + self.assertEqual(a.RUNTIME['compatibility_flags'], []) + + +if __name__ == '__main__': + unittest.main()