diff --git a/.github/workflows/environments-live.yml b/.github/workflows/environments-live.yml index 753060b..2a63e2e 100644 --- a/.github/workflows/environments-live.yml +++ b/.github/workflows/environments-live.yml @@ -4,18 +4,19 @@ name: Environments Live Matrix # providers, nightly (PLAN_ENV.md E2-11): a real, hash-verified lock, a real # build, a real launch, and the formal core tier — tests/test_environment_live_matrix.py. # -# E2B and Modal carry their own already-documented, unticked gaps there -# (E2-03's uid/gid mismatch; modal_sandbox.py's missing setpriv) and are -# `xfail(strict=False)` in that file, so an ordinary failure there is -# expected and does not fail this workflow. Daytona carries no such marker: -# a failure there is a real regression, and this workflow opens (or -# comments on) an issue naming it. +# E2B carries its own documented, unticked gap there (E2-03: its services run +# as root) and is `xfail(strict=False)` in that file, so an ordinary failure +# there is expected and does not fail this workflow. Daytona and Modal carry +# no such marker: a failure there is a real regression, and this workflow +# opens (or comments on) an issue naming it. # -# Needs, as repository secrets: E2B_API_KEY, DAYTONA_API_KEY, MODAL_TOKEN_ID, -# MODAL_TOKEN_SECRET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY — the last two -# scoped to read the approved base from ECR only (D-18). A secret that is -# not set is not a workflow failure: the test each secret belongs to skips, -# by name, and says so in the job summary. +# Needs one repository secret, DATALAYER_API_KEY. The providers' keys are +# Datalayer secrets of the account it belongs to, read at run time: E2B_API_KEY, +# DAYTONA_API_KEY, MODAL_TOKEN_ID, MODAL_TOKEN_SECRET, and the AWS pair under +# CODE_SANDBOXES_CI_AWS_ACCESS_KEY_ID and CODE_SANDBOXES_CI_AWS_SECRET_ACCESS_KEY, +# a key of the base reader that can only pull the approved base from ECR (D-18). +# A key that is not there skips the test it belongs to, by name, and the job +# summary lists each leg's outcome; a Daytona leg that did not run fails the job. on: schedule: @@ -55,6 +56,48 @@ jobs: uv pip install ".[e2b,daytona,modal,test]" uv pip install boto3 + # Each value is masked before it reaches the job's environment, and none + # is set on a later step's own `env:`, which would override it with an + # empty string. + - name: Read the providers' keys from Datalayer + env: + DATALAYER_API_KEY: ${{ secrets.DATALAYER_API_KEY }} + DATALAYER_IAM_URL: https://prod1.datalayer.run + run: | + python - <<'PY' + import json + import os + import urllib.request + + key = os.environ.get("DATALAYER_API_KEY", "") + if not key: + print("DATALAYER_API_KEY is not a secret of this repository: every leg will skip") + raise SystemExit(0) + request = urllib.request.Request( + os.environ["DATALAYER_IAM_URL"] + "/api/iam/v1/secrets/values", + headers={"Authorization": f"Bearer {key}"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + secrets = json.load(response).get("secrets") or {} + wanted = { + "DAYTONA_API_KEY": "DAYTONA_API_KEY", + "E2B_API_KEY": "E2B_API_KEY", + "MODAL_TOKEN_ID": "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET": "MODAL_TOKEN_SECRET", + "AWS_ACCESS_KEY_ID": "CODE_SANDBOXES_CI_AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY": "CODE_SANDBOXES_CI_AWS_SECRET_ACCESS_KEY", + } + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as environment: + for variable, name in wanted.items(): + value = str(secrets.get(name) or "") + if not value or "\n" in value: + print(f"{variable}: no Datalayer secret {name} for this account") + continue + print(f"::add-mask::{value}") + environment.write(f"{variable}={value}\n") + print(f"{variable}: read from the Datalayer secret {name}") + PY + - name: Configure AWS credentials run: | mkdir -p ~/.aws @@ -63,9 +106,6 @@ jobs: echo "aws_access_key_id=${AWS_ACCESS_KEY_ID}" echo "aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}" } > ~/.aws/credentials - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: Run the live matrix id: live @@ -75,13 +115,51 @@ jobs: tests/test_environment_live_matrix.py \ --junitxml=live-matrix-results.xml env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: us-east-1 - E2B_API_KEY: ${{ secrets.E2B_API_KEY }} - DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} - MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} - MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + + # A leg whose secret is not set skips, and a run where every leg skipped + # reported success in under a second: the nightly was green every night + # while it tested nothing (found on 2026-09-18, "3 skipped in 0.61s"). + # Each leg's outcome goes in the job summary, and a Daytona leg that did + # not run fails the job: it is the one this matrix is required to prove + # (PLAN_ENV.md, the Datalayer and Daytona target, E2-11). + - name: Say which legs ran, and refuse a Daytona leg that did not + if: always() + run: | + python - <<'PY' + import os + import sys + import xml.etree.ElementTree as ET + + try: + cases = list(ET.parse("live-matrix-results.xml").getroot().iter("testcase")) + except (OSError, ET.ParseError) as error: + sys.exit(f"no JUnit report to read: {error}") + rows, daytona = [], None + for case in cases: + leg = (case.get("classname") or "").rsplit(".", 1)[-1] + skipped = case.find("skipped") + if skipped is not None: + outcome = "skipped: " + (skipped.get("message") or "").strip() + elif case.find("failure") is not None or case.find("error") is not None: + outcome = "failed" + else: + outcome = "passed" + rows.append(f"| {leg} | {outcome} |") + if leg == "TestDaytona": + daytona = outcome + summary = "| Leg | Outcome |\n|---|---|\n" + "\n".join(rows) + "\n" + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as handle: + handle.write(summary) + print(summary) + if daytona is None or daytona.startswith("skipped"): + sys.exit( + f"The Daytona leg did not run ({daytona or 'absent'}): it is the one this " + "matrix must prove, so a run without it is not a green run. Set the " + "DATALAYER_API_KEY repository secret, whose account holds the " + "DAYTONA_API_KEY and CODE_SANDBOXES_CI_AWS_* Datalayer secrets." + ) + PY - name: Upload the JUnit report if: always() diff --git a/CHANGELOG.md b/CHANGELOG.md index c79771c..5a417bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,87 @@ ## Unreleased +## 1.9.18 + +- **A lock no longer carries a wall clock, so the build cache can hit** + (`environments/resolve.py`, `resolve_conda.py`; PLAN_ENVS.md E1-26, D-12). + The lock's header carried a `# resolved-at:` line, and its digest is over + the whole text — so two resolves of the same spec, in the same base, + pinning the same 320 packages, produced two different digests. Found by + resolving one environment twice on r1 on 2026-09-16: the texts differed in + exactly that one line out of 5,388. Section 5's cache key is over the lock + digest, so D-12's build cache could never hit, and it never had: + `environments.cache.lookups` read `hit=false` twelve times out of twelve. + When a lock was resolved is on the lock document Runtimes stores, in its + `created_at`, which is where it belongs. `resolved_at` is gone from + `lock_document`, `conda_lock_document`, `resolve_environment` and + `resolve_conda_environment`; the two tests that asserted determinism by + freezing the clock now assert it without one. + +## 1.9.17 + +- **An artifact's size is read from the registry** (`environments/attest.py`; + PLAN_ENVS.md E1-25). `attest_artifact` took `size_bytes` from its caller and + nobody ever passed one — the builder answers a reference, not a weight — so + every artefact was recorded with `sizeBytes: null` and + `environments.artifact.bytes`, the series section 14 tracks the artifact + size in, had no point in it although artifacts had been recorded (seen on r1, + 2026-09-16, through the OTEL query API). `Attestor.size_of()` asks the + registry, with the client the scan is already read from, and a size that + cannot be read is logged rather than raised: a missing number on a dashboard + is not a reason to refuse an artifact that is otherwise signed. 3 new tests. + +## 1.9.15 + +- **A restart restarts the kernel, not just this client's socket** + (`jupyter_server_sandbox`, `client`; PLAN_ENV.md E0-09, Appendix B check + 7). `CodeSandboxClient.restart()` was `stop()` then `start()`, which is + right for a sandbox this process owns — it is destroyed and recreated, and + nothing survives — and wrong for one *attached* to a Jupyter server + somebody else runs, which is every Datalayer runtime pod: stopping drops + the websocket while the kernel process keeps running, so the reconnect + lands in the same interpreter with every global still set. Check 7 is + "nothing is assumed to persist across restarts", and it read `state survived the restart ('True')` for exactly this reason — found live on r1, + 2026-09-16, the first drill whose smoke test reached the check. + `JupyterServerSandbox.restart_kernel()` now asks the server's own + `POST /api/kernels/{id}/restart` (the way `_do_interrupt` already uses the + API rather than the client's lifecycle) and reconnects onto the new + kernel; `restart()` prefers it and falls back to the lifecycle for every + variant that draws no such distinction. 7 new tests. +- **`datalayer/python-cpu:2026.09` repinned** to + `sha256:122d3e31f5e2507251457cbf47871c39ac1753adb1d83777ab0743fa11cd6148`: + the contract layer now sets `MappingKernelManager.root_dir`, so kernels + start in `/home/datalayer/content`. The image already declared `WORKDIR` + there and `sandbox-contract/v1`'s User row already required it, but a + kernel's cwd is the Jupyter server's to choose and jupyter-python's config + roots it at `$HOME` — so every environment's kernel ran in + `/home/datalayer` and Appendix B check 2 read `cwd is '/home/datalayer', not '/home/datalayer/content'`. The file browser stays rooted at `$HOME`, + where a person expects to see everything they have; only the kernel moves. +- Two assertions that had rotted through three base releases are pinned in + one place again: the channel's digest and its apt snapshot were duplicated + across `test_environment_bases.py` and `test_environment_resolve.py`, and + 2026-09-15's and 2026-09-16's releases left both red rather than catching + anything. + +## 1.9.14 + +- **`datalayer/python-cpu:2026.09` base channel repinned** to the rebuilt + `jupyter-python:0.2.2` (now carrying `jupyter-kernels==1.2.23`) plus the + contract layer, digest + `sha256:aa5413000bb5b6ecd0a0cf03959b107f0d572f65bf230c08bbdf9a4569775545`, + released 2026-09-16 to `environments/base/python-cpu`. Every variant pins the + same digest. + +## 1.9.13 + +- **`jupyter-kernels==1.2.23` forced into `sandbox-contract/v1`**: it carries + the pooled kernel manager the runtime's Jupyter config selects + (`kernel_manager_class = jupyter_kernels.pool.mapping.PooledMappingKernelManager`), + replacing the deprecated private `datalayer-kernels`. PyPI serves it, so a + resolve satisfies it from the index and the wheelhouse carries no wheel for + it; the pin keeps `uv pip sync --require-hashes` from stripping it out of a + user environment's image. + ## 1.9.12 - **`owner_repository`, `owner_cache_repository` and `ECR_ENVIRONMENT_PREFIX` diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index 1e71576..9371279 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.12" +__version__ = "1.9.37" diff --git a/code_sandboxes/client.py b/code_sandboxes/client.py index 3e8105f..4ac2604 100644 --- a/code_sandboxes/client.py +++ b/code_sandboxes/client.py @@ -508,7 +508,21 @@ def is_alive(self) -> bool: return self.is_started def restart(self) -> None: - """Restart the wrapped sandbox through its public lifecycle.""" + """Restart the wrapped sandbox, clearing what it was holding. + + A sandbox this process owns is restarted by its own lifecycle: stop + and start destroy and recreate it, and nothing survives. One that is + merely *attached* to a server somebody else runs — a Jupyter server + in a Datalayer runtime pod — is not: stopping drops this client's + websocket while the kernel process goes on running, and starting + reconnects to the same interpreter with every global still set. A + sandbox that knows how to restart what it is attached to says so with + `restart_kernel`, and that is used in preference; the lifecycle is + the fallback for every variant that has no such distinction. + """ + restart_kernel = getattr(self._sandbox, "restart_kernel", None) + if callable(restart_kernel) and restart_kernel(): + return self._sandbox.stop() self._sandbox.start() diff --git a/code_sandboxes/daytona_sandbox.py b/code_sandboxes/daytona_sandbox.py index 2d1cf1c..78ec77a 100644 --- a/code_sandboxes/daytona_sandbox.py +++ b/code_sandboxes/daytona_sandbox.py @@ -510,6 +510,24 @@ def _client_config(self, daytona: Any) -> Any | None: given = {key: value for key, value in settings.items() if value} return daytona.DaytonaConfig(**given) if given else None + def _snapshot_carries_a_gpu(self) -> bool: + """Whether the snapshot this sandbox starts from was built with a GPU (E2-17). + + A GPU is baked into a snapshot with its other resources, so a sandbox + of one asks for a GPU without saying so, and Daytona refuses it unless + it is ephemeral, the same as one asked for with `gpu=`. The snapshot + record says: a GPU environment's artifact, launched by its id, reads + `gpu` there. One that cannot be read is taken to have none, and + Daytona's own refusal then says why. + """ + if self._snapshot is None or self._daytona is None: + return False + try: + snapshot = self._daytona.snapshot.get(self._snapshot) + except Exception: + return False + return bool(getattr(snapshot, "gpu", 0)) + def _create_params(self, daytona: Any) -> Any: # noqa: C901 """What to ask Daytona for, from the configuration of this sandbox. @@ -534,7 +552,7 @@ def _create_params(self, daytona: Any) -> Any: # noqa: C901 ] resources = self._resources(daytona) - if _asks_for_a_gpu(resources): + if _asks_for_a_gpu(resources) or self._snapshot_carries_a_gpu(): # Daytona will not create a GPU sandbox that outlives its stop: # "GPU sandboxes must be ephemeral; set autoDeleteInterval to 0". # It is a property of asking for a GPU at all, not of asking for diff --git a/code_sandboxes/environments/__init__.py b/code_sandboxes/environments/__init__.py index fa98607..f3cb16e 100644 --- a/code_sandboxes/environments/__init__.py +++ b/code_sandboxes/environments/__init__.py @@ -49,8 +49,12 @@ CONTRACT_V1, SANDBOX_CONTRACT_V1, SUPPORTED_CONTRACTS, + BuildContextEntry, + BuildContextFinding, SandboxContract, + check_build_context, check_dockerfile, + validate_build_context, validate_dockerfile, ) from .errors import ERROR_CODES, EnvironmentsError, ErrorCode, map_provider_error @@ -91,6 +95,8 @@ "ApprovedBase", "ArtifactReference", "Attestor", + "BuildContextEntry", + "BuildContextFinding", "BuildRequest", "BuildkitResolveRunner", "CapabilityReport", @@ -114,6 +120,7 @@ "can_transition", "canonical_digest", "canonical_json", + "check_build_context", "check_dockerfile", "decide", "fingerprint_matches", @@ -131,6 +138,7 @@ "spec_digest", "spec_findings", "transition", + "validate_build_context", "validate_dockerfile", "validate_environment", ] diff --git a/code_sandboxes/environments/adapters/datalayer.py b/code_sandboxes/environments/adapters/datalayer.py index f3d82d8..004c8b4 100644 --- a/code_sandboxes/environments/adapters/datalayer.py +++ b/code_sandboxes/environments/adapters/datalayer.py @@ -61,7 +61,7 @@ owner_cache_repository, owner_repository, ) -from ..contract import SANDBOX_CONTRACT_V1 +from ..contract import SANDBOX_CONTRACT_V1, pin_dockerfile_base from ..errors import ( ARTIFACT_MISSING, BUILD_FAILED, @@ -76,8 +76,20 @@ WHEELHOUSE_PATH, apt_pins_in, apt_snapshot_in, + buildkit_proxy, + buildkit_proxy_options, + egress_findings, + egress_hosts_text, + egress_refused_hosts, locked_versions, ) +from ..resolve_conda import ( + MICROMAMBA_BINARY, + conda_lock_pip_requirements, + is_conda_lock, + micromamba_bootstrap_dockerfile_line, + micromamba_install_command, +) from ..spec import BuildSecret, Environment, command_names_secret __all__ = [ @@ -106,6 +118,24 @@ # does; nothing about their behavior changed. +#: The Dockerfile frontend the generated build asks for. +#: +#: At least **1.10**, because `_secret_mount` renders `mountAs: env` as +#: `--mount=type=secret,env=NAME` and that key does not exist before it: 1.7, +#: which this pinned for its first year, answers `unexpected key 'env' in +#: 'env=NAME'` and fails the whole build. The mount and the pin were written +#: against each other's assumptions and never ran together, because no build +#: had ever carried a build secret — found on r1 on 2026-09-17 by the first +#: one that did, and checked against both frontends before changing it. +#: +#: Raising this is a change to every Datalayer build, so it is one constant +#: with one reason, not a literal inside the Dockerfile's own text. +DOCKERFILE_FRONTEND = "docker/dockerfile:1.10" + +#: The first frontend with `--mount=type=secret,env=` (see above). +SECRET_ENV_FRONTEND = (1, 10) + + def _secret_mount(secret: BuildSecret) -> str: """The ``--mount=type=secret`` flag one `BuildSecret` renders as (E3-05). @@ -120,6 +150,41 @@ def _secret_mount(secret: BuildSecret) -> str: return f"--mount=type=secret,id={secret.id},env={secret.name}" +def _head(request: BuildRequest, authored: Any) -> list[str]: + """The Dockerfile's first lines: the frontend, what built it, and its base. + + For a `dockerfile` source (E3-03) the base is the author's own + Dockerfile, its `FROM` pinned to the digest the resolver chose; the + contract's lines follow it, whatever came before them. + """ + lines = [ + f"# syntax={DOCKERFILE_FRONTEND}", + f"# Generated by Datalayer for {request.environment.metadata.name} " + f"v{request.version}, build {request.build_uid}.", + f"# Lock: {request.lock_digest}", + ] + if authored is not None: + lines.append(pin_dockerfile_base(authored.content, request.resolved_base).rstrip("\n")) + else: + lines.append(f"FROM {request.resolved_base}") + return lines + + +def _pip_lock_command(find_links: str, *, authored: bool) -> str: + """Install the pip lock: `sync` to it, or `install` it over an authored Dockerfile. + + `sync` removes whatever the lock does not name, and a `dockerfile` + source's lock does not name what the Dockerfile itself installed + (`DOCKERFILE_COVERAGE`). The lock's pins, the protected ones included, + win either way. + """ + verb, target = ("install", "-r ") if authored else ("sync", "") + return ( + f"uv pip {verb} --system --require-hashes --find-links {find_links} " + f"{target}/opt/datalayer/lock.txt" + ) + + class Builder: """The Datalayer variant: BuildKit into the owner's ECR repository. @@ -154,6 +219,11 @@ class Builder: importing this module needs no credentials. run How a subprocess is run, so a test can watch the argv without a daemon. + proxy + The build pool's egress proxy, as ``buildkitd`` reaches it (E1-06): + every step of the build goes through it. ``DATALAYER_BUILDKIT_PROXY`` + by default; empty is a ``buildkitd`` whose steps reach the network + directly. """ variant = "datalayer" @@ -173,6 +243,7 @@ def __init__( run: Callable[..., subprocess.CompletedProcess[str]] | None = None, max_build_seconds: int = DEFAULT_MAX_BUILD_SECONDS, resolve_secret: Callable[..., str] = resolve_build_secret, + proxy: str | None = None, ) -> None: self._log = log or (lambda _line: None) self._credential = credential @@ -181,6 +252,8 @@ def __init__( self._tlscert = tlscert or os.environ.get("DATALAYER_BUILDKIT_TLSCERT", "").strip() self._tlskey = tlskey or os.environ.get("DATALAYER_BUILDKIT_TLSKEY", "").strip() self._tlscacert = tlscacert or os.environ.get("DATALAYER_BUILDKIT_TLSCACERT", "").strip() + #: The build pool's egress proxy (E1-06), or `DATALAYER_BUILDKIT_PROXY`. + self._proxy = buildkit_proxy(proxy) self._region = region or os.environ.get("AWS_REGION", "us-east-1") self._ecr = ecr self._run = run or subprocess.run @@ -199,7 +272,7 @@ def capabilities(self) -> CapabilitySet: # `image` resolves from an imported reference instead of an # approved base (E3-04). Neither changes how this builder itself # builds, once the resolver has done its part. - build_sources=("packages", "dependencyFile", "image"), + build_sources=("packages", "dependencyFile", "image", "dockerfile"), package_managers=("uv", "pip"), # Nothing is forbidden by the builder itself: BuildKit is the # reference implementation of a Dockerfile, and what the contract @@ -217,7 +290,10 @@ def validate(self, environment: Environment, lock_text: str | None = None) -> Ca """Whether this variant can build this spec, before anything is queued.""" findings: list[CapabilityFinding] = [] source = environment.spec.build.source - if source not in ("packages", "dependencyFile", "image"): + # The capability set is the one list of what this builder builds: a + # second copy here refused `dockerfile` on r1 after the capability + # set had learned it (E3-03, 2026-09-18). + if source not in self.capabilities().build_sources: findings.append( CapabilityFinding( code=CAPABILITY_UNSUPPORTED.code, @@ -229,8 +305,8 @@ def validate(self, environment: Environment, lock_text: str | None = None) -> Ca findings.append( CapabilityFinding( code=CAPABILITY_UNSUPPORTED.code, - message="conda environments are resolved by their own solver, " - "which is not built yet", + message="a conda environment is brought as a `dependencyFile` " + "whose `sourceFormat` is `conda`, not through `packages`", field="spec.packages.python.manager", ) ) @@ -253,7 +329,8 @@ def validate(self, environment: Environment, lock_text: str | None = None) -> Ca field="spec.platform.architecture", ) ) - if lock_text is not None and not locked_versions(lock_text): + pins_or_lock = lock_text is not None and not is_conda_lock(lock_text) + if pins_or_lock and not locked_versions(lock_text): findings.append( CapabilityFinding( code=SPEC_INVALID.code, @@ -275,12 +352,9 @@ def dockerfile(self, request: BuildRequest) -> str: """ spec = request.environment.spec apt = apt_pins_in(request.lock_text) - lines = [ - "# syntax=docker/dockerfile:1.7", - f"# Generated by Datalayer for {request.environment.metadata.name} " - f"v{request.version}, build {request.build_uid}.", - f"# Lock: {request.lock_digest}", - f"FROM {request.resolved_base}", + authored = spec.build.dockerfile if spec.build.source == "dockerfile" else None + lines = _head(request, authored) + lines += [ f'LABEL io.datalayer.environment="{request.environment.metadata.name}" \\', f' io.datalayer.environment.uid="{request.environment_uid}" \\', f' io.datalayer.version="{request.version}" \\', @@ -323,18 +397,45 @@ def dockerfile(self, request: BuildRequest) -> str: f"COPY wheelhouse/ {imported_wheelhouse}/", 'RUN pip install --no-cache-dir "uv==0.12.11"', ] - lines.extend( - [ - "COPY lock.txt /opt/datalayer/lock.txt", - # `sync` and not `install`: the artifact holds the lock's set, - # and `--require-hashes` means every byte was the resolved one. - # `--find-links` for what no index has — a protected pin's - # own wheel, the fork's local version above all (E1-04). - "RUN --mount=type=cache,target=/root/.cache/uv " - f"uv pip sync --system --require-hashes --find-links {find_links} " - "/opt/datalayer/lock.txt", - ] - ) + if is_conda_lock(request.lock_text): + # A conda source (E3-02): the lock is an `@EXPLICIT` file + # `micromamba install --file` installs without re-solving, and the + # pip layer the solve resolved — the user's own pip requirements and + # the protected pins forced over them — is in the lock's own + # `# datalayer-pip:` header. The conda layer goes into the base's + # own environment; the pip layer follows, so the kernel stack + # (E1-04) and everything the solve installed is present the same as + # every source. micromamba is copied in from its pinned image + # first: the approved base bakes uv and the wheelhouse but not it. + pip_requirements = conda_lock_pip_requirements(request.lock_text) + lines.extend( + [ + micromamba_bootstrap_dockerfile_line(), + "COPY lock.txt /opt/datalayer/lock.txt", + "RUN --mount=type=cache,target=/opt/conda/pkgs " + + micromamba_install_command( + "/opt/datalayer/lock.txt", micromamba=MICROMAMBA_BINARY + ), + ] + ) + if pip_requirements: + requirements = " ".join(shlex.quote(req) for req in pip_requirements) + lines.append( + "RUN --mount=type=cache,target=/root/.cache/uv " + f"uv pip install --system --find-links {find_links} {requirements}" + ) + else: + lines.extend( + [ + "COPY lock.txt /opt/datalayer/lock.txt", + # `sync` and not `install`: the artifact holds the lock's set, + # and `--require-hashes` means every byte was the resolved one. + # `--find-links` for what no index has — a protected pin's + # own wheel, the fork's local version above all (E1-04). + "RUN --mount=type=cache,target=/root/.cache/uv " + + _pip_lock_command(find_links, authored=authored is not None), + ] + ) # A build secret is mounted on the postInstall commands that name it # and nowhere else (§4.1, D-11): never an `ARG` or `ENV`, which bakes a # value into the image's history, never the package-install or files @@ -346,12 +447,12 @@ def dockerfile(self, request: BuildRequest) -> str: baked = files_step(request.environment, variant=self.variant) if baked: lines.append("USER 1000:100") - lines.append("WORKDIR /home/datalayer/content") + lines.append("WORKDIR /home/datalayer") for command in baked: lines.append(f"RUN {command}") if spec.commands.post_install: lines.append("USER 1000:100") - lines.append("WORKDIR /home/datalayer/content") + lines.append("WORKDIR /home/datalayer") for command in spec.commands.post_install: # No network: a command that fetches something makes an # artifact whose contents depend on the day it was built. @@ -364,7 +465,7 @@ def dockerfile(self, request: BuildRequest) -> str: lines.extend( [ "USER 1000:100", - "WORKDIR /home/datalayer/content", + "WORKDIR /home/datalayer", # The contract's own check, in the image, at build time. "RUN /opt/datalayer/bin/datalayer-sandbox doctor --json > /tmp/doctor.json", ] @@ -442,11 +543,30 @@ def build(self, request: BuildRequest) -> ArtifactReference: "--metadata-file", str(metadata), *self._cache_options(request), + *buildkit_proxy_options(self._proxy), *secret_args, ] self._log(f"Building {reference} from {request.resolved_base}") finished = self._invoke(command, timeout=self._max_build_seconds) if finished.returncode != 0: + refused = egress_refused_hosts(finished.stderr or "", proxy=self._proxy) + if refused: + # The one failure a log line hid best: a host the build + # pool may not reach, named as a finding (E2-19). + raise EnvironmentsError( + BUILD_FAILED, + f"The build pool's egress proxy refused {egress_hosts_text(refused)}: " + "a build reaches only the package indexes, mirrors and registries " + "the build pool allows. Put what it fetches in the build, or ask " + "for the host to be allowed", + detail={ + "variant": self.variant, + "reference": reference, + "exit": finished.returncode, + "refused_hosts": refused, + "findings": egress_findings(refused), + }, + ) raise EnvironmentsError( BUILD_FAILED, "The build failed; its log says where", diff --git a/code_sandboxes/environments/adapters/daytona.py b/code_sandboxes/environments/adapters/daytona.py index 0a56c2c..3a8a640 100644 --- a/code_sandboxes/environments/adapters/daytona.py +++ b/code_sandboxes/environments/adapters/daytona.py @@ -69,27 +69,24 @@ what it does once actually launched. A real answer needs a launched sandbox, which is a smoke test's job (E1-14), not this builder's. -**GPU classes are not built here.** `gpu = True` on this builder is a true -capability (Daytona's own hardware runs one, D-20), and `validate` leaves a -GPU size class buildable rather than refusing it — the constraint table of -section 6 has nothing against it, and an existing test -(`test_a_gpu_spec_is_buildable_on_modal_and_daytona`) already pins that -answer. What actually stops a GPU build today is upstream: the CUDA base -channel is E2-17's to publish, and `bases.py` has no digest to resolve -`python-cuda` to yet, so a `BuildRequest` for one cannot be constructed in -practice. `build()` itself still guards it explicitly, refusing plainly -rather than baking an unsourced guess at a GPU type and count into a -snapshot, in case that ever changes before the real numbers do (section 11.3 -item 7). +**A GPU is baked into the snapshot** (E2-17, D-20), from the spec's own +`resources.accelerator`: its `type` is one of Daytona's GPUs, checked at +`validate` so a GPU Daytona does not offer is refused before any build, and +its `count` is the number of them. CPU and memory come from the spec's hints, +or Daytona's own default for that GPU; the disk from the hints, or enough for +the CUDA base. A sandbox of a GPU snapshot is ephemeral, which Daytona +requires of every GPU sandbox (`DaytonaSandbox` reads the snapshot's `gpu`). @module code_sandboxes.environments.adapters.daytona """ from __future__ import annotations +import contextlib +import shlex import tempfile import uuid -from collections.abc import Callable +from collections.abc import Callable, Sequence from pathlib import Path from typing import Any @@ -99,8 +96,9 @@ ArtifactReference, BuildRequest, CapabilityFinding, + ValidationResult, ) -from ..contract import SANDBOX_CONTRACT_V1 +from ..contract import SANDBOX_CONTRACT_V1, pin_dockerfile_base from ..errors import ( ARTIFACT_MISSING, BUILD_FAILED, @@ -110,16 +108,22 @@ ) from ..files import files_step from ..resolve import WHEELHOUSE_IMAGE_PATH, apt_pins_in -from ..spec import GPU_SIZE_CLASSES, Environment +from ..resolve_conda import ( + conda_lock_pip_requirements, + is_conda_lock, + micromamba_bootstrap_command, + micromamba_install_command, +) +from ..spec import GPU_SIZE_CLASSES, Environment, EnvironmentSpec from .managed import ManagedBuilder -__all__ = ["Builder"] +__all__ = ["DAYTONA_GPUS", "Builder", "daytona_gpu"] #: Tags Daytona refuses for a snapshot's source image: each moves. MOVING_TAGS = ("latest", "lts", "stable") _LOCK_PATH = "/opt/datalayer/lock.txt" -_CONTENT_DIR = "/home/datalayer/content" +_CONTENT_DIR = "/home/datalayer" #: A long-running PID 1 (§11.3 item 3, contract's "Entrypoint" and "Signals" #: rows): the Datalayer base bakes none of its own, and Daytona's own default @@ -140,6 +144,44 @@ } +#: The GPUs Daytona offers, as its SDK's `GpuType` names them (E2-17). Spelled +#: out rather than read from the SDK, because `validate` runs where the SDK may +#: not be installed; a test holds the two together. +DAYTONA_GPUS: tuple[str, ...] = ("H100", "H200", "RTX-PRO-6000", "RTX-4090", "RTX-5090") + +#: A GPU snapshot's disk when the spec gives no hint: the CUDA base alone is +#: about 15 GB, and Daytona's own default would not hold it with room to work. +_GPU_DISK_GI = 50 + + +def daytona_gpu(accelerator_type: str) -> str | None: + """Daytona's name for an accelerator type, or `None` when Daytona has no such GPU. + + `h100`, `H100` and `rtx_4090` name what Daytona calls `H100` and + `RTX-4090`; a `T4` or an `A100-80GB` is Modal's vocabulary, not Daytona's. + """ + name = accelerator_type.strip().upper().replace("_", "-") + return name if name in DAYTONA_GPUS else None + + +def _snapshot_name(request: BuildRequest) -> str: + """The name a build gives its snapshot, unique to the build (its uid is in it).""" + return f"dl-{request.environment.metadata.name}-v{request.version}-{request.build_uid}" + + +def _pip_lock_command(*, authored: bool) -> str: + """Install the pip lock: `sync` to it, or `install` it over an authored Dockerfile. + + `sync` would remove what the Dockerfile installed and the lock does not + name (E3-03, `DOCKERFILE_COVERAGE`). + """ + verb, target = ("install", "-r ") if authored else ("sync", "") + return ( + f"uv pip {verb} --system --require-hashes " + f"--find-links {WHEELHOUSE_IMAGE_PATH} {target}{_LOCK_PATH}" + ) + + def _daytona_sdk() -> Any: try: import daytona @@ -171,8 +213,20 @@ class Builder(ManagedBuilder): variant = "daytona" item = "E2-04" title = "Daytona" - #: Daytona runs GPUs, on its own hardware and the owner's account (E2-17). - #: This builder does not build one yet: see `_own_findings`. + #: A `packages` list, or a dependency file: a conda `environment.yml` + #: installed with `micromamba` (E3-02), and a `requirements.txt` or a + #: `pyproject.toml` with its lock (E3-01), both of which resolve to the + #: very pip lock a `packages` list does — `build` tells a conda lock from + #: a pip one and nothing finer, so nothing here is format-specific. + build_sources = ("packages", "dependencyFile", "dockerfile") + #: None beyond the contract's own (E3-03): `Image.from_dockerfile` keeps + #: the Dockerfile text as it is and Daytona builds it on a real Docker + #: builder, so the grammar it accepts is Docker's. Checked in the SDK on + #: 2026-09-17. + forbidden_instructions = () + dependency_formats = ("requirements", "pyproject", "conda") + #: Daytona runs GPUs, on its own hardware and the owner's account (E2-17), + #: baked into the snapshot with the rest of its resources. gpu = True #: E0-04's spike found only a registry login for the private base, never #: a per-step arbitrary named secret (E3-05): `buildSecrets` is refused. @@ -197,6 +251,9 @@ def __init__( self._daytona_sdk = daytona_sdk or _daytona_sdk self._registry_sdk = registry_sdk or _daytona_registry_sdk self._client_instance: Any = None + #: The builds `cancel` was asked to stop: one whose snapshot is only + #: finished after the cancel deletes it itself (E2-18). + self._cancelled: set[str] = set() def _provider_secrets(self) -> dict[str, str]: """The owner's Daytona secrets the build credential carries (D-8, E2-01). @@ -267,15 +324,7 @@ def _own_findings( field="spec.compatibility.regions", ) ) - # A GPU class is left buildable here on purpose (`gpu = True`, - # D-20): the CUDA base E2-17 has not published yet, so a GPU - # `BuildRequest` cannot reach `build()` in practice — `bases.py`'s - # own resolver has no digest to resolve `python-cuda` to, and - # refuses first. `build()` itself still guards it explicitly (see - # its own docstring), so a spec that somehow got a `resolved_base` - # anyway is refused plainly rather than baking an unsourced guess - # at a GPU type and count into a snapshot. - # + findings.extend(self._accelerator_findings(environment.spec)) # `spec.buildSecrets` needs no check of its own here: `supports_build_secrets # = False` above (E3-05, merged since this branch started) makes # `ManagedBuilder._own_findings` refuse it before this method is @@ -297,21 +346,9 @@ def build(self, request: BuildRequest) -> ArtifactReference: (E0-04). """ spec = request.environment.spec - if request.size_class in GPU_SIZE_CLASSES: - # `validate` leaves a GPU class buildable (`gpu = True`, D-20): - # `bases.py` has no CUDA digest to resolve yet, so this cannot - # be reached in practice — refused plainly here rather than - # baking an unsourced guess at a GPU type and count into a - # snapshot (E2-17 is what will give this real numbers). - raise EnvironmentsError( - CAPABILITY_UNSUPPORTED, - f"Daytona runs `{request.size_class}` on its own GPUs, but the CUDA base " - "and the GPU resource shape this needs are E2-17's, not built yet", - detail={"variant": self.variant, "missing": "E2-17"}, - ) sdk = self._daytona_sdk() client = self._client(sdk) - name = f"dl-{request.environment.metadata.name}-v{request.version}-{request.build_uid}" + name = _snapshot_name(request) registry_id = self._register_base_pull(client, request.resolved_base) try: @@ -319,7 +356,8 @@ def build(self, request: BuildRequest) -> ArtifactReference: lock_file = Path(scratch) / "lock.txt" lock_file.write_text(request.lock_text, encoding="utf-8") - image = sdk.Image.base(request.resolved_base) + authored = spec.build.dockerfile if spec.build.source == "dockerfile" else None + image = self._starting_image(sdk, request, authored, Path(scratch)) # `env` before anything installs, the same order the # Datalayer and E2B builders keep: a package that compiles # against a library found through an env var behaves @@ -343,28 +381,38 @@ def build(self, request: BuildRequest) -> ArtifactReference: # wheelhouse again would only duplicate what `uv pip sync` # can already reach at `WHEELHOUSE_IMAGE_PATH`. Only the # lock is genuinely per-build. - image = ( - image.add_local_file(str(lock_file), _LOCK_PATH) + image = image.add_local_file(str(lock_file), _LOCK_PATH) + if is_conda_lock(request.lock_text): + # A conda source (E3-02): `micromamba install --file` + # reads the `@EXPLICIT` lock without re-solving, and the + # pip layer the solve resolved — the user's pip + # requirements and the protected pins over them — comes + # from the lock's own `# datalayer-pip:` header, so the + # kernel stack (E1-04) and everything the solve installed is + # present the same as for a pip source. micromamba is + # installed first: the approved base bakes uv but not it. + image = image.run_commands(micromamba_bootstrap_command()) + image = image.run_commands(micromamba_install_command(_LOCK_PATH)) + pip_requirements = conda_lock_pip_requirements(request.lock_text) + if pip_requirements: + requirements = " ".join(shlex.quote(req) for req in pip_requirements) + image = image.run_commands( + "pip install --no-cache-dir " + f"--find-links {WHEELHOUSE_IMAGE_PATH} {requirements}" + ) + else: # `uv` is not installed here: the approved base already # bakes it (E1-05, `resolve.py`'s own `bootstrap_uv` # docstring — "an approved Datalayer base already has it - # baked in"), and this phase's `build_sources` is - # `("packages",)` only, so every build starts from that - # base. Reinstalling it added an extra un-hashed network - # fetch outside the resolved lock for no reason (found in - # review) — matching the Datalayer builder's own - # `dockerfile()`, which installs `uv` only for the - # `image` source, not implemented for this variant yet. + # baked in"). Reinstalling it added an extra un-hashed + # network fetch outside the resolved lock for no reason + # (found in review). # # Packages install as root, the same reason the # Datalayer and E2B builders give: a user install lands # under the content directory's own home, which the # runtime mounts over. - .run_commands( - "uv pip sync --system --require-hashes " - f"--find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}" - ) - ) + image = image.run_commands(_pip_lock_command(authored=authored is not None)) image = image.dockerfile_commands([f"USER 1000:100\nWORKDIR {_CONTENT_DIR}"]) for command in files_step(request.environment, variant=self.variant): image = image.run_commands(command) @@ -393,7 +441,12 @@ def on_logs(line: str) -> None: logged.append(line) self._log(line) - resources = self._resources(sdk, request.size_class) + resources = self._resources(sdk, request.size_class, spec) + # A snapshot is region-scoped, and the region that scopes it is + # Daytona's, not this platform's. `validate` already refuses + # more than one, so the first is the only one. + declared = list(request.environment.spec.compatibility.regions) + region = declared[0] if declared else None try: snapshot = client.snapshot.create( sdk.CreateSnapshotParams( @@ -401,12 +454,24 @@ def on_logs(line: str) -> None: image=image, resources=resources, entrypoint=_CONTRACT_ENTRYPOINT, - region_id=request.region, + # Only a region Daytona knows. `request.region` is + # *Datalayer's* — `r1` — and sending it answered + # "Region not found" on the first real Daytona + # build (2026-09-17). The owner names a Daytona + # region in `compatibility.regions`; with none, + # the field is left out and the account's own + # default decides, which is what every snapshot + # in the owner's account already has. + **({"region_id": region} if region else {}), ), on_logs=on_logs, timeout=self.max_build_seconds, ) except Exception as error: + # Daytona keeps a snapshot that failed, in `error`, under + # the build's own name: one over its size limit was left + # there (E2-17, 2026-09-18). Nothing records it, so it goes. + self._discard_named(sdk, client, name) raise EnvironmentsError( BUILD_FAILED, f"The Daytona build failed: {error}", @@ -417,6 +482,17 @@ def on_logs(line: str) -> None: # not control, whether the build above succeeded or not (D-18). self._unregister_base_pull(client, registry_id) + if request.build_uid in self._cancelled: + # Finished after the build was cancelled: nothing will record it, + # so it goes now rather than wait in the account for nobody. + self._log(f"The build was cancelled: deleting the snapshot {snapshot.id} it made") + with contextlib.suppress(EnvironmentsError): + self._delete_by_id(sdk, client, snapshot.id) + raise EnvironmentsError( + BUILD_FAILED, + "The build was cancelled, and the snapshot it made was deleted", + detail={"variant": self.variant, "name": name}, + ) return ArtifactReference( variant=self.variant, immutable_reference=snapshot.id, @@ -431,10 +507,57 @@ def on_logs(line: str) -> None: contract_version=spec.contract or SANDBOX_CONTRACT_V1.version, ) - def _resources(self, sdk: Any, size_class: str) -> Any: - """The CPU resources a size class bakes into the snapshot (§11.3 item 4).""" - shape = _CPU_RESOURCES.get(size_class, _CPU_RESOURCES["small"]) - return sdk.Resources(cpu=shape["cpu"], memory=shape["memory"], disk=shape["disk"]) + @staticmethod + def _starting_image(sdk: Any, request: BuildRequest, authored: Any, scratch: Path) -> Any: + """The image the chain starts from: the base, or the author's Dockerfile on it. + + A Dockerfile source (E3-03) starts from the author's own Dockerfile, + its base pinned to the digest the resolver chose — which is also what + the build's registry entry lets Daytona pull. The contract's steps are + chained after it either way. + """ + if authored is None: + return sdk.Image.base(request.resolved_base) + dockerfile = scratch / "Dockerfile" + dockerfile.write_text( + pin_dockerfile_base(authored.content, request.resolved_base), encoding="utf-8" + ) + return sdk.Image.from_dockerfile(str(dockerfile)) + + @staticmethod + def _accelerator_findings(spec: EnvironmentSpec) -> list[CapabilityFinding]: + """A GPU Daytona does not offer, refused before any build (E2-17).""" + accelerator = spec.resources.accelerator + if accelerator == "none" or daytona_gpu(accelerator.type): + return [] + return [ + CapabilityFinding( + code="DL_ENV_CAPABILITY_UNSUPPORTED", + message=( + f"Daytona has no GPU called `{accelerator.type}`; it offers " + + ", ".join(DAYTONA_GPUS) + ), + field="spec.resources.accelerator.type", + ) + ] + + def _resources(self, sdk: Any, size_class: str, spec: EnvironmentSpec) -> Any: + """What the snapshot bakes in: a class's CPU, or the spec's GPU (§11.3 item 4, E2-17).""" + accelerator = spec.resources.accelerator + if size_class not in GPU_SIZE_CLASSES or accelerator == "none": + shape = _CPU_RESOURCES.get(size_class, _CPU_RESOURCES["small"]) + return sdk.Resources(cpu=shape["cpu"], memory=shape["memory"], disk=shape["disk"]) + # D-4 gives the GPU classes no CPU or memory of their own (E4-11 has + # them, for Datalayer's nodes): the GPU is what the class is for, and + # Daytona sizes the rest of the machine for it unless the spec hints. + hints = spec.resources.hints + return sdk.Resources( + cpu=round(hints.cpu) if hints.cpu else None, + memory=round(hints.memory_gi) if hints.memory_gi else None, + disk=round(hints.disk_gi) if hints.disk_gi else _GPU_DISK_GI, + gpu=accelerator.count, + gpu_type=sdk.GpuType(daytona_gpu(accelerator.type)), + ) def _register_base_pull(self, client: Any, resolved_base: str) -> str | None: """A private registry entry for this build's base pull (D-17, D-18). @@ -524,6 +647,156 @@ def exists(self, artifact: ArtifactReference) -> bool: raise self._provider_error("ask whether the snapshot exists", error) from error return True + def delete(self, artifact: ArtifactReference) -> None: + """Remove the snapshot, by id; one already gone is removed (E2-18). + + Deleting what is gone is a success because the collector deletes + first and marks second (E1-17): a sweep that died between the two + deletes again tomorrow, and must not be refused for having worked. + + **By id, never by name.** The SDK takes either, and a name is reused + once its snapshot is deleted (E0-04) — deleting by name could remove + a later build's snapshot that inherited it. + """ + sdk = self._daytona_sdk() + self._delete_by_id( + sdk, self._client(sdk), artifact.provider_artifact_id or artifact.immutable_reference + ) + + def _delete_by_id(self, sdk: Any, client: Any, snapshot: str) -> None: + try: + client.snapshot.delete(snapshot) + except sdk.DaytonaNotFoundError: + self._log(f"The Daytona snapshot {snapshot} was already gone") + return + except Exception as error: + raise self._provider_error("delete the snapshot", error) from error + self._log(f"Deleted the Daytona snapshot {snapshot}") + + def cancel(self, request: BuildRequest) -> None: + """Stop a build: delete the snapshot it is making, now or once it is made (E2-18). + + The build step calls this when the build is cancelled, and then lets + the build's own thread finish unheard, so a snapshot Daytona goes on + building is recorded by nobody. It is found by the name this build + gave it, which is unique to the build (the build uid is in it), and + deleted by its id. One not made yet is deleted by `build` itself, the + moment Daytona hands it back. + """ + self._cancelled.add(request.build_uid) + sdk = self._daytona_sdk() + client = self._client(sdk) + self._discard_named(sdk, client, _snapshot_name(request)) + + def _discard_named(self, sdk: Any, client: Any, name: str) -> None: + """Delete, by its id, the snapshot this build named, when Daytona has one. + + A build's name is its own (the build uid is in it), so looking it up + by name cannot find another build's. Never a second reason to fail: + what could not be deleted is said in the log. + """ + try: + snapshot = client.snapshot.get(name) + except Exception: + return + self._log(f"Deleting the snapshot {snapshot.id} this build made, which nothing records") + try: + self._delete_by_id(sdk, client, snapshot.id) + except EnvironmentsError as error: + self._log(f"The snapshot {snapshot.id} could not be deleted: {error.message}") + + def smoke_test( + self, + artifact: ArtifactReference, + *, + environment: Any = None, + lock_text: str | None = None, + secret_values: Sequence[str] = (), + ) -> ValidationResult: + """Launch the snapshot and run Appendix B's core tier in it (E2-04). + + This box's own `Done when` asks for exactly this — "a sandbox launched + from its id passes the core tier" — and until 2026-09-17 it refused + through `ManagedBuilder`, so **no Daytona build could reach + `succeeded`**: the workflow calls this step, the snapshot was built and + live at the provider, and the build was recorded failed. + + **Launched by id, never by name** (E0-04): a Daytona sandbox record + keeps the snapshot's *name*, and a name is republished, so only the id + says which artifact actually ran. + + **Restarted by stopping and starting the sandbox**, not the kernel + (E0-04 again): Daytona's own daemon is PID 1 here, so there is no + kernel to restart, and check 8 means the thing that survives a real + restart. + + The sandbox is deleted whether the tier passed or not — a smoke test + that leaves a sandbox running bills the owner for a check. + """ + if environment is None: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "A Daytona smoke test needs the version's spec: the core tier " + "asks for the Python version it declared and the packages its " + "lock pinned, and an artifact carries neither", + detail={"variant": self.variant}, + ) + from ..conformance import expected_packages, run_accelerator_check, run_core_tier + + snapshot = artifact.provider_artifact_id or artifact.immutable_reference + sandbox = self._smoke_test_sandbox(snapshot) + self._log(f"Launching {snapshot} to smoke-test it") + try: + sandbox.start() + result = run_core_tier( + sandbox, + python_version=environment.spec.language.version, + expected_packages=expected_packages(environment, lock_text or ""), + secret_values=tuple(secret_values), + restart=lambda: self._restart(sandbox), + ) + accelerator = environment.spec.resources.accelerator + if accelerator != "none": + # A GPU version is only the version its spec describes when + # its GPUs are visible and its CUDA is the spec's (check 11, + # E2-17): the core tier alone passes on a machine with none. + result.checks.append( + run_accelerator_check(sandbox, cuda=accelerator.cuda, count=accelerator.count) + ) + return result + except EnvironmentsError: + raise + except Exception as error: + raise self._provider_error("smoke-test the snapshot", error) from error + finally: + try: + sandbox.stop() + except Exception as error: + self._log(f"The smoke-test sandbox could not be stopped: {error}") + + def _smoke_test_sandbox(self, snapshot: str) -> Any: + """A sandbox of this build's own snapshot, deleted when it stops.""" + from ...daytona_sandbox import DaytonaSandbox + + secrets = self._provider_secrets() + return DaytonaSandbox( + api_key=secrets.get("DAYTONA_API_KEY"), + organization_id=secrets.get("DAYTONA_ORGANIZATION_ID"), + snapshot=snapshot, + delete_on_stop=True, + ) + + @staticmethod + def _restart(sandbox: Any) -> None: + """Check 8's restart, as Daytona can do it. + + Its daemon is PID 1, so there is no kernel to restart: the sandbox + itself is stopped and started, which is the stronger version of the + same question. + """ + sandbox.stop() + sandbox.start() + def _provider_error(self, what: str, error: BaseException) -> EnvironmentsError: return EnvironmentsError( PROVIDER_ERROR, diff --git a/code_sandboxes/environments/adapters/e2b.py b/code_sandboxes/environments/adapters/e2b.py index ce67135..90057cb 100644 --- a/code_sandboxes/environments/adapters/e2b.py +++ b/code_sandboxes/environments/adapters/e2b.py @@ -119,6 +119,7 @@ from __future__ import annotations +import shlex import tempfile from collections.abc import Callable from pathlib import Path @@ -140,6 +141,11 @@ ) from ..files import files_step from ..resolve import WHEELHOUSE_PATH, apt_pins_in +from ..resolve_conda import ( + conda_lock_pip_requirements, + is_conda_lock, + micromamba_bootstrap_command, +) from ..spec import Environment from .managed import ManagedBuilder @@ -157,7 +163,7 @@ _DOCTOR_PATH = "/opt/datalayer/bin/datalayer-sandbox" _WHEELHOUSE_PATH = "/opt/datalayer/wheelhouse" _LOCK_PATH = "/opt/datalayer/lock.txt" -_CONTENT_DIR = "/home/datalayer/content" +_CONTENT_DIR = "/home/datalayer" #: The sandbox contract's own user (D-4, §3). `set_user` makes it the #: template's persistent default; the steps that must run as root instead @@ -191,6 +197,27 @@ class Builder(ManagedBuilder): variant = "e2b" item = "E2-03" title = "E2B" + #: A `packages` list and, for conda (E3-02), an `environment.yml` + #: dependency file installed with `micromamba`. + build_sources = ("packages", "dependencyFile", "dockerfile") + #: What E2B's own Dockerfile parser does not handle (E3-03). Read from the + #: SDK on 2026-09-17: `e2b.template.dockerfile_parser` branches on FROM, + #: RUN, COPY, ADD, WORKDIR, USER, ENV, ARG, CMD and ENTRYPOINT, and for + #: anything else **prints `Unsupported instruction` and carries on**. So a + #: template built from a Dockerfile naming one of these comes back without + #: it and reports success — which is exactly what a capability report + #: exists to prevent. + forbidden_instructions = ( + "VOLUME", + "EXPOSE", + "HEALTHCHECK", + "SHELL", + "ONBUILD", + "STOPSIGNAL", + "LABEL", + "MAINTAINER", + ) + dependency_formats = ("conda",) #: Firecracker microVMs: no GPU passthrough. gpu = False #: E0-04's spike found only a registry login for the private base, never @@ -390,15 +417,38 @@ def build(self, request: BuildRequest) -> ArtifactReference: chain.copy("datalayer-sandbox", _DOCTOR_PATH, mode=0o755, user="root") .copy("wheelhouse", _WHEELHOUSE_PATH, user="root") .copy("lock.txt", _LOCK_PATH, user="root") - .run_cmd(f'pip install --no-cache-dir "uv=={_UV_VERSION}"', user="root") - # Packages install as root (E0-04): a user install lands - # under /home/user, which the runtime mounts over. - .run_cmd( + ) + if is_conda_lock(request.lock_text): + # A conda source (E3-02): `micromamba install --file` reads the + # `@EXPLICIT` lock without re-solving, and the pip layer the + # solve resolved — the user's pip requirements and the protected + # pins over them — comes from the lock's own `# datalayer-pip:` + # header, so the kernel stack (E1-04) and everything the solve + # installed is present the same as for a pip source. micromamba + # is installed first: the approved base bakes uv but not it. + chain = chain.run_cmd(micromamba_bootstrap_command(), user="root") + chain = chain.run_cmd( + f"micromamba install --yes --name base --file {_LOCK_PATH}", + user="root", + ) + pip_requirements = conda_lock_pip_requirements(request.lock_text) + if pip_requirements: + requirements = " ".join(shlex.quote(req) for req in pip_requirements) + chain = chain.run_cmd( + f"pip install --no-cache-dir --find-links {_WHEELHOUSE_PATH} " + f"{requirements}", + user="root", + ) + else: + chain = chain.run_cmd( + f'pip install --no-cache-dir "uv=={_UV_VERSION}"', user="root" + ).run_cmd( + # Packages install as root (E0-04): a user install lands + # under /home/user, which the runtime mounts over. "uv pip sync --system --require-hashes " f"--find-links {_WHEELHOUSE_PATH} {_LOCK_PATH}", user="root", ) - ) for command in files_step(request.environment, variant=self.variant): chain = chain.run_cmd(command) for command in spec.commands.post_install: diff --git a/code_sandboxes/environments/adapters/managed.py b/code_sandboxes/environments/adapters/managed.py index a236e9c..97f3423 100644 --- a/code_sandboxes/environments/adapters/managed.py +++ b/code_sandboxes/environments/adapters/managed.py @@ -24,6 +24,7 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Any, Callable from ..builders import ( @@ -36,7 +37,7 @@ ValidationResult, ) from ..errors import CAPABILITY_UNSUPPORTED, SPEC_INVALID, EnvironmentsError -from ..spec import GPU_SIZE_CLASSES, Environment +from ..spec import GPU_SIZE_CLASSES, Environment, EnvironmentSpec __all__ = ["ManagedBuilder"] @@ -67,8 +68,17 @@ class ManagedBuilder: supports_build_secrets = True #: The build sources it will accept in this phase. build_sources: tuple[str, ...] = ("packages",) + #: When `dependencyFile` is among `build_sources`, the `sourceFormat`s the + #: variant's own build actually installs. A conda source (E3-02) installs + #: with `micromamba`; a `pyproject`/`requirements` dependencyFile is not + #: built for a managed variant yet (E3-01), so it is not listed here. + dependency_formats: tuple[str, ...] = () package_managers: tuple[str, ...] = ("uv", "pip") - #: Dockerfile instructions its own builder does not implement (§6). + #: Dockerfile instructions its own builder does not implement (§6, E3-03). + #: Checked against a `dockerfile`-sourced spec at `validate`, before + #: anything is queued — the point being to refuse an instruction a variant + #: would otherwise drop, rather than hand back an image quietly missing + #: whatever it asked for. forbidden_instructions: tuple[str, ...] = () #: Where its artifacts live; empty when the variant is regionless. regions: tuple[str, ...] = () @@ -121,6 +131,10 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: field="spec.build.source", ) ) + elif spec.build.source == "dependencyFile": + findings.extend(self._dependency_file_findings(spec)) + elif spec.build.source == "dockerfile": + findings.extend(self._dockerfile_findings(spec)) if spec.packages.python.manager not in self.package_managers: findings.append( CapabilityFinding( @@ -141,34 +155,7 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: ) ) if not self.gpu: - # A GPU is asked for two ways, and the spec's own validation - # couples them; a `validate` can be reached before that, so both - # are read here rather than trusting the coupling. - if spec.resources.size_class in GPU_SIZE_CLASSES: - findings.append( - CapabilityFinding( - code=CAPABILITY_UNSUPPORTED.code, - message=( - f"{self.title} has no GPU, so `{spec.resources.size_class}` cannot be " - f"built for it. Drop {self.variant} from the variants, or build " - "the GPU classes for modal or daytona, which run them on their " - "own hardware" - ), - field="spec.resources.sizeClass", - ) - ) - elif spec.resources.accelerator != "none": - findings.append( - CapabilityFinding( - code=CAPABILITY_UNSUPPORTED.code, - message=( - f"{self.title} has no GPU, so an accelerator cannot be built for it. " - f"Drop {self.variant} from the variants, or build the GPU classes for " - "modal or daytona, which run them on their own hardware" - ), - field="spec.resources.accelerator", - ) - ) + findings.extend(self._gpu_findings(spec)) if self.regions: asked = [ region @@ -201,6 +188,58 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: ) return findings + def _gpu_findings(self, spec: EnvironmentSpec) -> list[CapabilityFinding]: + """A variant with no GPU refuses either way a GPU is asked for. A GPU + is asked two ways, and the spec's own validation couples them; a + `validate` can be reached before that, so both are read here rather + than trusting the coupling.""" + if spec.resources.size_class in GPU_SIZE_CLASSES: + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"{self.title} has no GPU, so `{spec.resources.size_class}` cannot be " + f"built for it. Drop {self.variant} from the variants, or build " + "the GPU classes for modal or daytona, which run them on their " + "own hardware" + ), + field="spec.resources.sizeClass", + ) + ] + if spec.resources.accelerator != "none": + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"{self.title} has no GPU, so an accelerator cannot be built for it. " + f"Drop {self.variant} from the variants, or build the GPU classes for " + "modal or daytona, which run them on their own hardware" + ), + field="spec.resources.accelerator", + ) + ] + return [] + + def _dependency_file_findings(self, spec: EnvironmentSpec) -> list[CapabilityFinding]: + """A `dependencyFile` this variant accepts still only builds the + `sourceFormat`s it has an install step for (E3-01, E3-02): a conda + file installs with `micromamba`, but a `pyproject` or `requirements` + one is not built for a managed variant yet.""" + source_format = spec.build.dependency_file.source_format + if source_format in self.dependency_formats: + return [] + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"a `{source_format}` dependency file is not built for " + f"{self.title} yet; it builds " + f"{', '.join(self.dependency_formats) or 'no dependency file'}" + ), + field="spec.build.dependencyFile.sourceFormat", + ) + ] + def _own_findings( self, environment: Environment, lock_text: str | None ) -> list[CapabilityFinding]: @@ -223,7 +262,23 @@ def build(self, request: BuildRequest) -> ArtifactReference: def inspect(self, artifact: ArtifactReference) -> ArtifactMetadata: raise self._not_built("inspect an artifact") - def smoke_test(self, artifact: ArtifactReference) -> ValidationResult: + def smoke_test( + self, + artifact: ArtifactReference, + *, + environment: Any = None, + lock_text: str | None = None, + secret_values: Sequence[str] = (), + ) -> ValidationResult: + """Launch the artifact and run Appendix B's core tier in it (E2-03/04/05). + + `environment` and `lock_text` are what the core tier needs and an + artifact does not carry: the Python version the spec asks for and the + packages its lock pins. The caller has both — it read them to build — + and passes them rather than making every adapter fetch them again. + They are keyword-only and optional so a caller that has neither still + type-checks, and an adapter that needs them says so itself. + """ raise self._not_built("smoke-test an artifact") def resolve(self, version_ref: str) -> ArtifactReference: @@ -237,6 +292,36 @@ def delete(self, artifact: ArtifactReference) -> None: # -- helpers for the subclasses ------------------------------------------- + def _dockerfile_findings(self, spec: Any) -> list[CapabilityFinding]: + """Every instruction in the spec's Dockerfile this variant would not honour. + + The contract's own refusals are `spec.py`'s to make and apply to every + variant alike; this is the narrower question of what *this* builder + does with a Dockerfile it accepts. Each is named with its line, since + a Dockerfile is somebody's file. + """ + if not self.forbidden_instructions: + return [] + dockerfile = getattr(spec.build, "dockerfile", None) + content = getattr(dockerfile, "content", "") or "" + if not content.strip(): + return [] + from ..contract import parse_dockerfile + + refused = set(self.forbidden_instructions) + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"line {instruction.line}: {self.title} does not implement " + f"`{instruction.keyword}`" + ), + field="spec.build.dockerfile.content", + ) + for instruction in parse_dockerfile(content) + if instruction.keyword in refused + ] + @staticmethod def _spec_finding(message: str, field: str) -> CapabilityFinding: return CapabilityFinding(code=SPEC_INVALID.code, message=message, field=field) diff --git a/code_sandboxes/environments/adapters/modal.py b/code_sandboxes/environments/adapters/modal.py index 391e18f..1253269 100644 --- a/code_sandboxes/environments/adapters/modal.py +++ b/code_sandboxes/environments/adapters/modal.py @@ -96,13 +96,20 @@ The entrypoint now execs `sleep infinity` when it is given no arguments, and `"$@"` when it is — correct either way a caller invokes it. -**A GPU size class is left buildable at `validate()`,** matching an existing -test (`test_a_gpu_spec_is_buildable_on_modal_and_daytona`), and refused at -`build()` instead, naming E2-17 — the same reasoning as Daytona's: the CUDA -base is not published yet, so this cannot be reached in practice, and this -builder does not know the launch-time `gpu=` argument to hand a size class -either (D-20 leaves that to a later item; a GPU is Modal's own launch -option, not an image property, per section 11.4 item 10). +**And a sandbox with no command is not given no arguments** (found live on +2026-09-18): Modal keeps the base's own CMD, `start-jupyter.sh`, under the +new ENTRYPOINT where Docker would reset it, so the main process was a +Jupyter server, which exits within a minute and ends the sandbox. A smoke +test's restarted sandbox, started warm, died between checks 8 and 9. The +artifact now sets its CMD to `sleep infinity`, and `ModalSandbox` names the +same command when it launches an artifact, which also covers the ones built +before. + +**A GPU is a launch option on Modal, not part of the image** (section 11.4 +item 10, E2-17). A GPU version builds the same image any version does, on the +CUDA base; the spec's `accelerator` names one of Modal's GPUs, checked at +`validate`, and the smoke test launches the image on it (`gpu="T4"`, +`"H100:2"`) and adds check 11 to the core tier. **A build secret is attached to the `postInstall` steps that name it (E3-05).** `run_commands` takes a per-step `secrets=` collection, a mechanism E2B and @@ -124,7 +131,7 @@ import io import os import tempfile -from collections.abc import Callable +from collections.abc import Callable, Sequence from pathlib import Path from typing import Any @@ -135,6 +142,7 @@ ArtifactReference, BuildRequest, CapabilityFinding, + ValidationResult, ) from ..contract import SANDBOX_CONTRACT_V1 from ..errors import ( @@ -147,19 +155,48 @@ from ..files import files_step from ..redact import redact from ..resolve import WHEELHOUSE_IMAGE_PATH, apt_pins_in -from ..spec import GPU_SIZE_CLASSES, BuildSecret, Environment, command_names_secret +from ..resolve_conda import conda_lock_pip_requirements, is_conda_lock +from ..spec import BuildSecret, Environment, command_names_secret from .managed import ManagedBuilder -__all__ = ["UNIMPLEMENTED_INSTRUCTIONS", "Builder"] +__all__ = ["MODAL_GPUS", "UNIMPLEMENTED_INSTRUCTIONS", "Builder", "modal_gpu"] #: What Modal's own Dockerfile builder does not implement (§6). UNIMPLEMENTED_INSTRUCTIONS = ("ONBUILD", "STOPSIGNAL", "VOLUME") +#: The GPUs a Modal sandbox takes by name (E2-17), as its `gpu=` spells them. +MODAL_GPUS: tuple[str, ...] = ( + "T4", + "L4", + "A10G", + "L40S", + "A100", + "A100-40GB", + "A100-80GB", + "H100", + "H200", + "B200", +) + + +def modal_gpu(accelerator_type: str, count: int = 1) -> str | None: + """Modal's `gpu=` for an accelerator, or `None` when Modal has no such GPU. + + `t4` and `a100_80gb` name what Modal calls `T4` and `A100-80GB`, and a + count above one is Modal's `"H100:2"`. An `RTX-4090` is Daytona's + vocabulary, not Modal's. + """ + name = accelerator_type.strip().upper().replace("_", "-") + if name not in MODAL_GPUS: + return None + return f"{name}:{count}" if count > 1 else name + + #: `uv`, pinned the same way every other builder's bootstrap is (E1-04/E3-04). _UV_VERSION = "0.12.11" _LOCK_PATH = "/opt/datalayer/lock.txt" -_CONTENT_DIR = "/home/datalayer/content" +_CONTENT_DIR = "/home/datalayer" #: The 2023.12 default fails installing Modal's own runtime deps on Python #: 3.13 (E0-04, confirmed live 2026-09-13): see the module docstring. @@ -178,6 +215,8 @@ #: is given keeps it alive for that; `exec "$@"` still wins when something #: is, for a caller that does supply a command directly. _ENTRYPOINT_SCRIPT = '#!/bin/sh\nif [ "$#" -eq 0 ]; then exec sleep infinity; fi\nexec "$@"\n' +#: The artifact's CMD: what a sandbox started with no command of its own runs. +_KEEP_ALIVE_COMMAND = ("sleep", "infinity") def _modal_sdk() -> Any: @@ -211,6 +250,28 @@ def _scrubbed(text: str, values: dict[str, str]) -> str: return redact(text, values.values()) if values else text +def _install_packages(image: Any, lock_text: str) -> Any: + """The package layer for this lock: a conda source (E3-02) installs the + `@EXPLICIT` lock with Modal's own `micromamba_install` — which brings + micromamba itself, so no bootstrap is needed here — and layers the pip + layer the solve resolved (the user's pip requirements and the protected + pins over them, from the lock's own `# datalayer-pip:` header); a pip + source runs `uv pip sync`.""" + if is_conda_lock(lock_text): + image = image.micromamba_install(spec_file=_LOCK_PATH) + pip_requirements = conda_lock_pip_requirements(lock_text) + if pip_requirements: + image = image.pip_install(*pip_requirements, find_links=WHEELHOUSE_IMAGE_PATH) + return image + return image.run_commands( + f'pip install --no-cache-dir "uv=={_UV_VERSION}"', + # Packages install as root: every Modal build step already runs as + # root regardless of any `USER` line (see the module docstring), so + # this is stating what is already true rather than asking for it. + f"uv pip sync --system --require-hashes --find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}", + ) + + def _post_install( image: Any, commands: list[str], declared: list[BuildSecret], step_secrets: dict[str, Any] ) -> Any: @@ -221,12 +282,53 @@ def _post_install( return image +def _intermediates_of(built: Any) -> tuple[str, ...]: + """Every layer under a built image, by id, deepest first (E2-05, E2-09). + + Each chained builder call leaves an image of its own, and deleting the + artifact does not delete them. Modal offers no call that lists an + account's images, so an intermediate nobody wrote down at build time can + never be found again — which is why this is recorded rather than + discovered. Confirmed live on 2026-09-17: a three-call chain built through + `Image.build` hydrates an `object_id` on every image in `deps()`, not only + on the last. + + The built image itself is not an intermediate: it is the artifact. + """ + found: list[str] = [] + seen: set[int] = set() + + def walk(image: Any) -> None: + if id(image) in seen: + return + seen.add(id(image)) + for dependency in getattr(image, "deps", lambda: ())(): + if hasattr(dependency, "deps"): + walk(dependency) + object_id = getattr(image, "object_id", None) + # Only images: the build's ECR secret is in `deps()` too, and has an + # `st-` id `ImageDelete` refuses ("not a valid Image ID", found live + # on 2026-09-18). It is deleted by the build itself, as a secret. + if object_id and image is not built and str(object_id).startswith("im-"): + found.append(str(object_id)) + + try: + walk(built) + except Exception: + return () + return tuple(dict.fromkeys(found)) + + class Builder(ManagedBuilder): """Modal: the capability half (E2-06) and the build (E2-05).""" variant = "modal" item = "E2-05" title = "Modal" + #: A `packages` list and, for conda (E3-02), an `environment.yml` + #: dependency file installed with `micromamba_install`. + build_sources = ("packages", "dependencyFile", "dockerfile") + dependency_formats = ("conda",) #: Modal runs GPUs, in the owner's workspace (E2-17). This builder does #: not build one yet: see `build`'s own guard. gpu = True @@ -331,6 +433,18 @@ def _own_findings( field=f"spec.buildSecrets[{index}].mountAs", ) ) + accelerator = environment.spec.resources.accelerator + if accelerator != "none" and modal_gpu(accelerator.type) is None: + findings.append( + CapabilityFinding( + code="DL_ENV_CAPABILITY_UNSUPPORTED", + message=( + f"Modal has no GPU called `{accelerator.type}`; it offers " + + ", ".join(MODAL_GPUS) + ), + field="spec.resources.accelerator.type", + ) + ) return findings # -- Building ------------------------------------------------------------- @@ -344,18 +458,6 @@ def build(self, request: BuildRequest) -> ArtifactReference: launch, not here, for the same reason. """ spec = request.environment.spec - if request.size_class in GPU_SIZE_CLASSES: - # `validate` leaves a GPU class buildable (`gpu = True`, D-20): - # the CUDA base E2-17 has not published yet, so this cannot be - # reached in practice — refused plainly here rather than - # guessing at the launch-time `gpu=` argument this class needs - # (section 11.4 item 10 is a launch concern, not a build one). - raise EnvironmentsError( - CAPABILITY_UNSUPPORTED, - f"Modal runs `{request.size_class}` on its own GPUs, but the CUDA base this " - "needs is E2-17's, not built yet", - detail={"variant": self.variant, "missing": "E2-17"}, - ) # Resolved before Modal is touched: a secret IAM will not give stops # the build with nothing to clean up in the owner's workspace. declared, values = self._resolved_secrets(request) @@ -403,19 +505,23 @@ def build(self, request: BuildRequest) -> ArtifactReference: image = image.add_local_file( str(entrypoint_file), _ENTRYPOINT_PATH, copy=True ).run_commands(f"chmod +x {_ENTRYPOINT_PATH}") - image = image.run_commands( - f'pip install --no-cache-dir "uv=={_UV_VERSION}"', - # Packages install as root: every Modal build step - # already runs as root regardless of any `USER` line - # (see the module docstring), so this is stating what - # is already true rather than asking for it. - "uv pip sync --system --require-hashes " - f"--find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}", - ) + image = _install_packages(image, request.lock_text) for command in files_step(request.environment, variant=self.variant): image = image.run_commands(command) image = _post_install(image, spec.commands.post_install, declared, step_secrets) - image = image.workdir(_CONTENT_DIR).entrypoint([_ENTRYPOINT_PATH]) + # The base's own CMD is `start-jupyter.sh`, and Modal keeps + # it under a new ENTRYPOINT where Docker would reset it: a + # sandbox started with no command ran a Jupyter server as its + # main process, which exits within a minute and takes the + # sandbox with it (found live on 2026-09-18, the restarted + # sandbox of a smoke test dying mid-tier). What holds the + # container is `sleep`; Jupyter is started by exec, when + # asked (`ModalSandbox.prepare_jupyter_server`). + image = ( + image.workdir(_CONTENT_DIR) + .entrypoint([_ENTRYPOINT_PATH]) + .cmd(list(_KEEP_ALIVE_COMMAND)) + ) logged: list[str] = [] buffer = io.StringIO() @@ -459,6 +565,7 @@ def build(self, request: BuildRequest) -> ArtifactReference: mutable_alias=name, provider_account=provider_account(self.variant, self._provider_secrets()) or None, contract_version=spec.contract or SANDBOX_CONTRACT_V1.version, + intermediates=_intermediates_of(built), ) def _resolved_secrets(self, request: BuildRequest) -> tuple[list[BuildSecret], dict[str, str]]: @@ -481,23 +588,31 @@ def _resolved_secrets(self, request: BuildRequest) -> tuple[list[BuildSecret], d return declared, values def _ecr_secret(self, sdk: Any, client: Any) -> Any | None: - """A Modal Secret carrying this build's own base-reader credential (D-17, D-18). - - `None` when the credential carries no registry login: the base is - then whatever the ambient workspace can already reach, the same - fallback `_client` takes with no owner token. + """A Modal Secret carrying this build's base-reader session (D-17, D-18). + + `from_aws_ecr` wants the IAM session itself — keys, token and region — + not the docker-login `AWS`/token pair Daytona and E2B take, which is + what the credential's `username`/`password` hold. So it reads + `aws_session`. Reading the login pair as IAM keys is what this did + until 2026-09-18: the worker minted `AWS`/, this wrote it + as `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, and AWS answered "The + security token included in the request is invalid" on the base pull, + so no Modal build through the worker could ever succeed. + + `None` when the credential carries no session: the base is then + whatever the ambient workspace can already reach, the same fallback + `_client` takes with no owner token. """ - username = str(getattr(self._credential, "username", "") or "") - password = str(getattr(self._credential, "password", "") or "") - if not (username and password): + session = dict(getattr(self._credential, "aws_session", None) or {}) + if not (session.get("AWS_ACCESS_KEY_ID") and session.get("AWS_SECRET_ACCESS_KEY")): return None - # `from_aws_ecr` wants IAM-shaped credentials, not the docker-login - # `AWS`/token pair Daytona and E2B take (found live, 2026-09-13): - # the same D-17 session, read differently. secret = sdk.Secret.from_dict( { - "AWS_ACCESS_KEY_ID": username, - "AWS_SECRET_ACCESS_KEY": password, + **{ + name: str(session[name]) + for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN") + if session.get(name) + }, "AWS_REGION": self._region, } ) @@ -611,6 +726,144 @@ def exists(self, artifact: ArtifactReference) -> bool: raise self._provider_error("ask whether the image exists", error) from error return True + def smoke_test( + self, + artifact: ArtifactReference, + *, + environment: Any = None, + lock_text: str | None = None, + secret_values: Sequence[str] = (), + ) -> ValidationResult: + """Launch the image and run Appendix B's core tier in it (E2-05). + + This box's own `Done when` asks for exactly this — "the live test + launches by image id and passes the core tier" — and it refused + through `ManagedBuilder`, so **no Modal build could reach + `succeeded`**: the image was built in the owner's workspace and the + build recorded failed at this step, the same state Daytona was in + until 2026-09-17. + + **Launched by image id, never by name**: a published name is mutable + by design (§6), so only the id says which artifact ran. Launched + through `ModalSandbox` — the same class a person's launch uses — + rather than a hand-rolled `Sandbox.create`, because a hand-rolled + launch is exactly what hid, on 2026-09-13, that every image this + builder made died the instant it was launched for real. + + **As the owner** (D-8): the sandbox is created with the same client + the build used, so it runs in the workspace the image is in. + + **Restarted by replacing the sandbox.** Modal has no in-place + restart, so check 7's restart stops this sandbox and starts another + from the same image, which is the stronger form of the question. + + The sandbox is terminated whether the tier passed or not. + """ + if environment is None: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "A Modal smoke test needs the version's spec: the core tier " + "asks for the Python version it declared and the packages its " + "lock pinned, and an artifact carries neither", + detail={"variant": self.variant}, + ) + from ...modal_sandbox import ModalSandbox + from ...models import SandboxConfig + from ..conformance import expected_packages, run_accelerator_check, run_core_tier + + sdk = self._modal_sdk() + accelerator = environment.spec.resources.accelerator + gpu = None if accelerator == "none" else modal_gpu(accelerator.type, accelerator.count) + sandbox = ModalSandbox( + config=SandboxConfig(name=f"smoke-{artifact.provider_artifact_id}", gpu=gpu), + app_name=f"dl-{environment.metadata.name}", + image_id=artifact.provider_artifact_id, + client=self._client(sdk), + ) + self._log(f"Launching {artifact.provider_artifact_id} to smoke-test it") + try: + sandbox.start() + result = run_core_tier( + sandbox, + python_version=environment.spec.language.version, + expected_packages=expected_packages(environment, lock_text or ""), + secret_values=tuple(secret_values), + restart=lambda: self._restart(sandbox), + ) + if gpu is not None: + # The core tier passes on a machine with no GPU: a GPU version + # is the version its spec describes only when its GPUs are + # visible and its CUDA is the spec's (check 11, E2-17). + result.checks.append( + run_accelerator_check(sandbox, cuda=accelerator.cuda, count=accelerator.count) + ) + return result + except EnvironmentsError: + raise + except Exception as error: + raise self._provider_error("smoke-test the image", error) from error + finally: + try: + sandbox.stop() + except Exception as error: + self._log(f"The smoke-test sandbox could not be stopped: {error}") + + @staticmethod + def _restart(sandbox: Any) -> None: + """Check 7's restart, as Modal can do it: a new sandbox from the same image.""" + sandbox.stop() + sandbox.start() + + def delete(self, artifact: ArtifactReference) -> None: + """Delete the image, and every intermediate layer this build recorded (E2-05, E2-09). + + Deleting the artifact does not delete the layers under it, and Modal + offers no call that lists an account's images, so what is removed is + what `build` wrote down — an intermediate nobody recorded can never be + found again. + + Two answers are outcomes rather than failures, both found live on + 2026-09-17: + + * **Already gone** is success. A replay of a collection must delete the + same set again with no harm, the same way the Datalayer collector + treats an artifact that is not there. + * **Not ours to delete.** The bottom of a chain can be an image the + workspace does not own — Modal's own `debian_slim` answers + `PermissionDenied` — and an image somebody else owns was never this + artifact's to collect. It is logged and stepped over, not raised. + """ + sdk = self._modal_sdk() + client = self._client(sdk) + synchronizer, api_pb2 = self._modal_internals() + + async def _delete(image_id: str) -> None: + await client.stub.ImageDelete(api_pb2.ImageDeleteRequest(image_id=image_id)) + + # The artifact last: an intermediate is only reachable while the + # record naming it survives, so a half-done collection that has + # dropped the image would strand them. + for image_id in (*artifact.intermediates, artifact.provider_artifact_id): + if not image_id: + continue + try: + # A bare coroutine on Modal's stub silently does nothing, so + # this runs on the SDK's own loop — see `_delete_secret`. + synchronizer.wrap(_delete)(image_id) + except sdk.exception.NotFoundError: + continue + except Exception as error: + if "permission" in str(error).lower(): + self._log( + f"The Modal image {image_id} is not this account's to delete: {error}" + ) + continue + if image_id == artifact.provider_artifact_id: + raise self._provider_error("delete the image", error) from error + # One layer's refusal does not strand the rest, nor the + # artifact this was called to collect. + self._log(f"The Modal intermediate {image_id} could not be deleted: {error}") + def _provider_error(self, what: str, error: BaseException) -> EnvironmentsError: return EnvironmentsError( PROVIDER_ERROR, diff --git a/code_sandboxes/environments/attest.py b/code_sandboxes/environments/attest.py index 94c750d..782036d 100644 --- a/code_sandboxes/environments/attest.py +++ b/code_sandboxes/environments/attest.py @@ -49,7 +49,16 @@ SCAN_BLOCKED, EnvironmentsError, ) -from .policy import DEFAULT_POLICY, PolicyDecision, ScanPolicy, decide, findings_of +from .policy import ( + DEFAULT_POLICY, + UNRESTRICTED_POLICY, + EnvironmentsPolicy, + PolicyDecision, + ScanPolicy, + decide, + findings_of, + refuse_unlicensed, +) logger = logging.getLogger(__name__) @@ -106,6 +115,101 @@ def signature_tag(digest: str) -> str: return text.replace(":", "-", 1) + ".sig" +#: Where a CycloneDX component keeps its licence, in the order they are read. +_CYCLONEDX_LICENSE_KEYS = ("id", "name") +#: Where an SPDX package keeps its licence. `licenseConcluded` is what the +#: tool decided; `licenseDeclared` is what the package claimed. BuildKit +#: writes SPDX, so this is the one that matters in practice. +_SPDX_LICENSE_KEYS = ("licenseConcluded", "licenseDeclared") +#: What SPDX writes when it could not tell, which is not a licence. +_SPDX_UNKNOWN = frozenset({"NOASSERTION", "NONE", ""}) + + +def _spdx_license_pairs(document: Mapping[str, Any]) -> list[tuple[str, str]]: + """Every SPDX package with a licence, as `(package, licence)`. + + `licenseConcluded` is what the tool decided and `licenseDeclared` what the + package claimed, so the concluded one is read first and the declared one + only when it said nothing. + """ + found: list[tuple[str, str]] = [] + for package in document.get("packages") or (): + if not isinstance(package, Mapping): + continue + name = str(package.get("name") or "").strip() + for key in _SPDX_LICENSE_KEYS: + value = str(package.get(key) or "").strip() + if value and value.upper() not in _SPDX_UNKNOWN: + # Kept even with no name: `licenses_of` reads only the licence + # half, and dropping an unnamed package's licence there would + # be exactly the silent behavior change this refactor must + # not make. `licenses_by_package`'s own callers decide what an + # empty package name means for them. + found.append((name, value)) + break + return found + + +def _cyclonedx_license_pairs(document: Mapping[str, Any]) -> list[tuple[str, str]]: + """Every CycloneDX component with a licence, as `(component, licence)`.""" + found: list[tuple[str, str]] = [] + for component in document.get("components") or (): + if not isinstance(component, Mapping): + continue + name = str(component.get("name") or "").strip() + for entry in component.get("licenses") or (): + if not isinstance(entry, Mapping): + continue + licence = entry.get("license") + value = "" + if isinstance(licence, Mapping): + for key in _CYCLONEDX_LICENSE_KEYS: + value = str(licence.get(key) or "").strip() + if value: + break + if not value: + value = str(entry.get("expression") or "").strip() + if value: + # Kept even with no component name; see the SPDX reader's own + # note above. + found.append((name, value)) + return found + + +def licenses_by_package(document: Any) -> list[tuple[str, str]]: + """Every `(package, licence)` an SBOM names, package first (E3-06). + + The same two shapes `licenses_of` reads, kept attributed rather than + flattened: a policy that denies a licence has to name the package that + carries it, which `licenses_of`'s own deduplicated set of licence strings + alone cannot answer. Insertion order, not sorted — the order the SBOM's + own packages/components came in, which is what a refusal naming "the + first one" should mean deterministically for the same document. + """ + if not isinstance(document, Mapping): + return [] + return _spdx_license_pairs(document) + _cyclonedx_license_pairs(document) + + +def licenses_of(document: Any) -> list[str]: + """Every licence an SBOM names, deduplicated and sorted. + + Reads both shapes the ecosystem writes: SPDX, which is what BuildKit's + `attest:sbom=` produces, and CycloneDX. A document in neither shape, or one + that names nothing, answers an empty list rather than raising: a + publication's licence list is worth having and never worth failing a build + over. + + This is what a published version's snapshot carries (D-12, E2-16). Until + it did, `licenses` came from the scan summary — and the registry's scanner + reports vulnerabilities, not licences, so every publication froze an empty + list beside an SBOM reference. + """ + if not isinstance(document, Mapping): + return [] + return sorted({licence for _package, licence in licenses_by_package(document)}) + + @dataclass(frozen=True) class AttestationResult: """What the workflow stores about an artifact once both gates have passed.""" @@ -117,6 +221,8 @@ class AttestationResult: size_bytes: int | None = None signed_now: bool = True """False when a replay found the signature that was already there.""" + licenses: tuple[str, ...] = () + """What the SBOM named, frozen onto the artifact for a publication to carry.""" def body(self) -> dict[str, Any]: """The mapping `activities_environments.attest` answers.""" @@ -127,6 +233,7 @@ def body(self) -> dict[str, Any]: "signature_ref": self.signature_ref, "size_bytes": self.size_bytes, "signed_now": self.signed_now, + "licenses": list(self.licenses), } @@ -515,8 +622,23 @@ def attest( size_bytes: int | None = None, sbom_ref: str = "", provenance_ref: str = "", + sbom: Any = None, + environments_policy: EnvironmentsPolicy = UNRESTRICTED_POLICY, ) -> AttestationResult: - """Scan, then sign: the order the Operator's check depends on (D-11).""" + """Scan, then sign: the order the Operator's check depends on (D-11). + + `sbom`, when the caller has the document, is read for the licences a + publication carries; the registry's scanner reports vulnerabilities + and never licences, so there is nowhere else they come from. + + **Refused before signing, never after (E3-06).** A licence an + organization's own policy denies is checked against the same `sbom`, + naming the package that carries it, and raised before `sign` runs — + an artifact whose licence policy denies it is never signed, the same + as one whose scan blocks it: a signature is Datalayer's word that an + artifact may run, and this is the second of the two words that go + into it. + """ self.can_sign() decision = self.scan(repository=repository, digest=digest) if not decision.passed: @@ -530,16 +652,44 @@ def attest( "blocking": [finding.id for finding in decision.blocking], }, ) + pairs = licenses_by_package(sbom) + refuse_unlicensed(pairs, environments_policy) signature, signed_now = self.sign(registry=registry, repository=repository, digest=digest) return AttestationResult( decision=decision, signature_ref=signature, sbom_ref=sbom_ref or f"{registry}/{repository}@{digest}.sbom", provenance_ref=provenance_ref or f"{registry}/{repository}@{digest}.att", - size_bytes=size_bytes, + size_bytes=size_bytes + if size_bytes is not None + else self.size_of(repository=repository, digest=digest), signed_now=signed_now, + licenses=tuple(sorted({licence for _package, licence in pairs})), ) + def size_of(self, *, repository: str, digest: str) -> int | None: + """What the registry says the artifact weighs, or None. + + Nobody hands the size down: the builder answers a reference, not a + weight, so until this asked the registry `size_bytes` was always None + — and with it `environments.artifact.bytes`, the series section 14 + tracks the artifact size in, which had no point in it on 2026-09-16 + although artifacts had been recorded. The registry has known all + along; the scan is read from the same client. + + Never a reason to fail an attestation: a size that could not be read + is a missing number on a dashboard, and the artifact is still signed. + """ + try: + images = self._client().describe_images( + repositoryName=repository, imageIds=[{"imageDigest": digest}] + )["imageDetails"] + except Exception as error: + self._log(f"The artifact's size could not be read: {error}") + return None + size = (images[0] or {}).get("imageSizeInBytes") if images else None + return int(size) if size else None + def _client(self) -> Any: if self._ecr is None: try: @@ -560,8 +710,10 @@ def attest_artifact( artifact: Any, credential: Any = None, policy: ScanPolicy = DEFAULT_POLICY, + environments_policy: EnvironmentsPolicy = UNRESTRICTED_POLICY, log: Callable[[str], None] | None = None, size_bytes: int | None = None, + sbom: Any = None, attestor: Attestor | None = None, ) -> dict[str, Any]: """The `attest` seam of `EnvironmentBuildWorkflow` (E1-08, E1-09). @@ -569,15 +721,51 @@ def attest_artifact( Takes the artifact the builder recorded and the build's credential, and answers the mapping the workflow stores: the scan's decision, the signature, the SBOM and provenance references, and the size. + + `environments_policy` is the caller's organization's own narrowing of + what licence a build may carry (E3-06); `sbom`, when the caller has the + document, is what it is checked against — nobody fetches one here. A + caller with neither passes nothing through, and nothing is refused: the + platform default is unrestricted on licences, the same as every other + dimension of this policy until an organization writes one. """ + variant = str(getattr(artifact, "variant", "") or "") reference = str(getattr(artifact, "immutable_reference", "") or "") + if variant and variant != "datalayer": + # D-11 is about the Datalayer artifact: it lives in this platform's + # registry, the scanner reads it there, cosign signs that digest, and + # the Operator refuses to start what is unsigned. A managed artifact + # is none of those things — it lives in the owner's own provider + # account (D-8) and is referenced the way that provider names it: a + # Daytona snapshot uuid, an E2B build id, a Modal `im-…`. There is no + # digest in a repository to scan or sign, and no Operator starting it. + # + # Attesting one anyway is what the first real Daytona build did, and + # it failed *after* the snapshot was built — the work done, the + # artifact live at the provider, and the build recorded as failed + # (2026-09-17). + # + # Publishing is not weakened by this: E2-15's gate requires the + # **Datalayer** artifact to have passed its scan, and that one is + # still attested here. + if log: + log(f"{variant} artifacts are not attested: {reference} is not a digest in a registry") + return { + "scan_summary": {}, + "sbom_ref": "", + "provenance_ref": "", + "signature_ref": "", + "size_bytes": size_bytes, + "signed_now": False, + "licenses": [], + } registry, _, rest = reference.partition("/") repository, _, digest = rest.partition("@") if not (registry and repository and _DIGEST.match(digest)): raise EnvironmentsError( PROVIDER_ERROR, f"`{reference}` is not a digest in a repository, so it cannot be attested", - detail={"reference": reference}, + detail={"reference": reference, "variant": variant}, ) use = attestor or Attestor( key=str(getattr(credential, "signing_key", "") or ""), @@ -590,6 +778,8 @@ def attest_artifact( repository=repository, digest=digest, size_bytes=size_bytes, + sbom=sbom, + environments_policy=environments_policy, ).body() diff --git a/code_sandboxes/environments/bases.py b/code_sandboxes/environments/bases.py index 252ab1a..1cc130a 100644 --- a/code_sandboxes/environments/bases.py +++ b/code_sandboxes/environments/bases.py @@ -59,6 +59,9 @@ class ApprovedBase(BaseModel): python_versions: tuple[str, ...] #: Whether the base carries CUDA, which a GPU size class requires. accelerator: bool = False + #: The CUDA toolkit the base pins, ``major.minor`` (E2-17): what a spec's + #: ``accelerator.cuda`` may ask for, and what check 11 reads back. + cuda: str | None = None #: Channel, then variant, to the digest resolution pins. A channel with no #: variant is approved and not yet published. channels: dict[str, dict[str, str]] = Field(default_factory=dict) @@ -109,8 +112,9 @@ def repository(self) -> str: base.ref: base for base in ( # E1-05: jupyter-python:0.2.2 (Ubuntu security packages and conda's own - # OpenSSL upgraded, JupyterLab's staging yarn.lock dropped) plus the - # contract layer, released 2026-09-14 to environments/base/python-cpu. + # OpenSSL upgraded, JupyterLab's staging yarn.lock dropped, jupyter-kernels + # installed) plus the contract layer, released 2026-09-16 to + # environments/base/python-cpu. # One image, so every variant pins the same digest until a variant # needs a base of its own. The prior digest, jupyter-python:0.2.1, # carried 31 fixable-critical findings the scan (E1-08) blocks every @@ -125,17 +129,40 @@ def repository(self) -> str: # `.spec.VARIANTS`, spelled out: `spec` imports from this # module, so importing it back here would be circular. ("datalayer", "e2b", "daytona", "modal"), - "sha256:334adf6c2714c8919ef60beeca1db12e3531a391c9dde41932c782f81c432b36", + # 2026-09-16: kernels now start in the contract's own + # working directory. The image already declared `WORKDIR + # /home/datalayer/content`, but a kernel's cwd is the + # Jupyter server's to choose, and jupyter-python's config + # roots it at `$HOME` — so every environment's kernel ran + # in `/home/datalayer` and Appendix B check 2 read "cwd is + # '/home/datalayer', not '/home/datalayer/content'". The + # contract layer now sets `MappingKernelManager.root_dir`, + # which moves the kernel without moving the file browser. + "sha256:122d3e31f5e2507251457cbf47871c39ac1753adb1d83777ab0743fa11cd6148", ) }, - snapshots={"2026.09": "20260914T150000Z"}, + snapshots={"2026.09": "20260916T120000Z"}, ), - # E2-17: jupyter-python-cuda plus the same layer. + # E2-17: jupyter-python-cuda:0.3.2 (jupyter-python 0.2.2 and CUDA 12.8's + # compiler and runtime headers, pinned) plus the same layer, released + # 2026-09-18 to environments/base/python-cuda as `2026.09-a3baa7b80931`. + # 6.9 GB: the first release carried the whole toolkit (15.7 GB), and a + # PyTorch environment on it made a snapshot Daytona refused, over its + # 20 GB limit. The doctor passes all fifteen checks and `nvcc` answers + # 12.8 without a GPU. ApprovedBase( ref="datalayer/python-cuda", python_versions=("3.13",), accelerator=True, - channels={"2026.09": {}}, + cuda="12.8", + channels={ + "2026.09": dict.fromkeys( + ("datalayer", "e2b", "daytona", "modal"), + "sha256:dc8f0015b4f7dbca92a88d2d6a4a96714b76f9a1af9dddd86835812493e40f08", + ) + }, + # After its CUDA layer's own `apt-get update`, which ran that day. + snapshots={"2026.09": "20260918T150000Z"}, ), ) } diff --git a/code_sandboxes/environments/build_secrets.py b/code_sandboxes/environments/build_secrets.py index de0abd3..2a3e9f3 100644 --- a/code_sandboxes/environments/build_secrets.py +++ b/code_sandboxes/environments/build_secrets.py @@ -28,7 +28,12 @@ import os from typing import Any -from .errors import BUILD_SECRET_UNAVAILABLE, EnvironmentsError +from .errors import ( + BUILD_SECRET_UNAVAILABLE, + CAPABILITY_UNSUPPORTED, + PROVIDER_ERROR, + EnvironmentsError, +) from .spec import BuildSecret __all__ = [ @@ -150,3 +155,141 @@ def resolve_build_secret( retryable=False, ) return value + + +#: The managed variants an owner keeps a credential for (D-8). `datalayer` is +#: not one: the platform's own builder uses the platform's own registry. +PROVIDER_VARIANTS = frozenset({"daytona", "e2b", "modal"}) + + +def resolve_provider_credential( + variant: str, + *, + owner_uid: str, + iam_url: str | None = None, + api_key: str | None = None, + timeout: float = 10.0, + transport: Any = None, +) -> dict[str, str]: + """The owner's own credential for a managed variant (PLAN_ENVS.md E2-01, D-8). + + A managed artifact exists only in the account the credential opens, so the + build runs with the owner's keys and records a non-secret fingerprint of + that account. The value IAM holds is a JSON object of the environment + names the provider's own SDK reads — ``DAYTONA_API_KEY``, + ``E2B_API_KEY``, ``MODAL_TOKEN_ID`` and ``MODAL_TOKEN_SECRET``, and + optionally the account names ``environments/accounts.py`` reads to + fingerprint it — because that is the shape ``BuildCredential`` carries and + every adapter already reads. + + Raises ``DL_ENV_CAPABILITY_UNSUPPORTED`` — not retryable — when the owner + has no credential for this variant. A build must be told it cannot run in + an account nobody configured, rather than falling back to whatever keys + the worker happens to hold: that would put one owner's artifact in + another's account, which is the one thing D-8 exists to prevent. + """ + variant = (variant or "").strip().lower() + if variant not in PROVIDER_VARIANTS: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"`{variant}` is not a managed variant a credential is kept for", + detail={"variant": variant}, + retryable=False, + ) + key = api_key if api_key is not None else os.environ.get(IAM_API_KEY_VARIABLE, "") + if not key: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"No {IAM_API_KEY_VARIABLE} to ask IAM for the {variant} credential with", + detail={"variant": variant, "missing": IAM_API_KEY_VARIABLE}, + retryable=False, + ) + base = (iam_url if iam_url is not None else os.environ.get(IAM_URL_VARIABLE, "")).rstrip("/") + if not base: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"No {IAM_URL_VARIABLE} to ask for the {variant} credential", + detail={"variant": variant, "missing": IAM_URL_VARIABLE}, + retryable=False, + ) + try: + import httpx + except ImportError as error: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "No HTTP client to resolve a provider credential with: install " + "`code-sandboxes[environments-builder]`", + detail={"variant": variant, "missing": "httpx"}, + retryable=False, + ) from error + url = f"{base}/api/iam/v1/secrets/provider/{variant}/value" + try: + with httpx.Client(transport=transport, timeout=timeout) as client: + response = client.get( + url, + params={"owner_uid": owner_uid}, + headers={"X-API-Key": key}, + ) + except httpx.HTTPError as error: + raise EnvironmentsError( + PROVIDER_ERROR, + f"IAM could not be reached for the {variant} credential: {error}", + detail={"variant": variant}, + retryable=True, + ) from error + if response.status_code == 404: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"This owner has no {variant} credential: a managed build runs in " + f"their own account, so one must be kept before {variant} can build", + detail={"variant": variant, "status": 404}, + retryable=False, + ) + if response.status_code != 200: + raise EnvironmentsError( + PROVIDER_ERROR, + f"IAM refused the {variant} credential: {response.status_code}", + detail={"variant": variant, "status": response.status_code}, + retryable=True, + ) + try: + body = response.json() + except ValueError as error: + raise EnvironmentsError( + PROVIDER_ERROR, + f"IAM answered the {variant} credential with a body that is not valid JSON", + detail={"variant": variant}, + retryable=True, + ) from error + return _provider_secrets_of(body.get("value") if isinstance(body, dict) else None, variant) + + +def _provider_secrets_of(value: Any, variant: str) -> dict[str, str]: + """The environment names a provider credential carries, as IAM answered. + + A mapping of names to values: IAM gathers them from the secrets the owner + already keeps, one per environment variable, and decodes them, so the + worker is not asked to know how a secret is stored. + """ + if not isinstance(value, dict) or not value: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"IAM answered the {variant} credential with no usable value; a " + "provider credential is the environment names that provider's own " + "SDK reads", + detail={"variant": variant}, + retryable=False, + ) + secrets = { + str(name): str(item) + for name, item in value.items() + if str(name).strip() and str(item).strip() + } + if not secrets: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"The {variant} credential names nothing", + detail={"variant": variant}, + retryable=False, + ) + return secrets diff --git a/code_sandboxes/environments/builders.py b/code_sandboxes/environments/builders.py index c0d4f34..9d786ee 100644 --- a/code_sandboxes/environments/builders.py +++ b/code_sandboxes/environments/builders.py @@ -141,6 +141,14 @@ class ArtifactReference(_Model): provider_account: str | None = None contract_version: str architecture: str = "linux/amd64" + #: The layers this build left behind that deleting the artifact does not + #: delete (E2-05, E2-09). Modal is the variant that has them: each chained + #: builder call leaves an image of its own with an id, and Modal offers no + #: call that lists an account's images — `ImageGetOrCreate`, `ImageFromId`, + #: `ImageGetByTag`, `ImageListTags`, `ImageTagRevisions`, `ImagePublish` + #: and `ImageDelete`, and nothing that enumerates — so an intermediate + #: nobody wrote down is an intermediate nobody can ever find again. + intermediates: tuple[str, ...] = () @model_validator(mode="after") def _immutable(self) -> ArtifactReference: diff --git a/code_sandboxes/environments/conformance.py b/code_sandboxes/environments/conformance.py index 200f6d6..d5a628b 100644 --- a/code_sandboxes/environments/conformance.py +++ b/code_sandboxes/environments/conformance.py @@ -37,6 +37,7 @@ "cross_variant_packages", "drifted_from", "package_versions_of", + "run_accelerator_check", "run_conformance", "run_core_tier", "run_extended_tier", @@ -457,6 +458,80 @@ def _secrets( ) +def expected_packages(environment: Any, lock_text: str) -> dict[str, str]: + """Each top-level distribution the spec names, pinned to what its lock resolved. + + What Appendix B check 5 is given. Never every package the lock pins — most + of a real lock is transitive, and check 5 tries to *import* each name it is + handed; a build tool or a C-library-only wheel that was never meant to be + imported directly would fail a check with nothing wrong to report. The + spec's own `packages.python.dependencies` is what a person declared + wanting, so it is what disagreeing across variants (E2-08) means something + about. + + Here rather than in the durable worker, which had the only copy, so that a + builder running its own smoke test asks the same question of its own + artifact (E2-03/04/05). + """ + from packaging.requirements import InvalidRequirement, Requirement + from packaging.utils import canonicalize_name + + from .resolve import locked_versions + + dependency_file = environment.spec.build.dependency_file + if dependency_file is not None and dependency_file.source_format == "conda": + # A conda source names its packages in the file, not in + # `packages.python`, and its lock is not a pip one (E3-02). Read the + # other way, check 5 was handed nothing and passed for it. + from .resolve_conda import conda_expected_packages + + return conda_expected_packages(dependency_file.content, lock_text or "") + pinned = locked_versions(lock_text) if lock_text else {} + names: list[str] = [] + for text in _declared_dependencies(environment): + try: + names.append(canonicalize_name(Requirement(text).name)) + except InvalidRequirement: + continue + return {name: pinned[name] for name in names if name in pinned} + + +def _declared_dependencies(environment: Any) -> list[str]: + """The requirements a person declared, wherever the source keeps them. + + A `dependencyFile` source names its packages in the file and leaves + `packages.python` empty, which is what the resolver reads too (E3-01). + Read from `packages.python` alone, check 5 was handed nothing for a + `requirements.txt` and passed for it, found live on 2026-09-18 (E3-08). + """ + from .spec import parse_requirements_txt + + build = environment.spec.build + dependency_file = build.dependency_file + if build.source == "dependencyFile" and dependency_file is not None: + if dependency_file.source_format == "requirements": + return parse_requirements_txt(dependency_file.content) + if dependency_file.source_format == "pyproject": + return _pyproject_dependencies(dependency_file.content) + return list(environment.spec.packages.python.dependencies) + + +def _pyproject_dependencies(text: str) -> list[str]: + """`[project].dependencies` of a `pyproject.toml`, or none when it cannot be read.""" + try: + import tomllib as toml + except ImportError: # Python 3.10 + try: + import tomli as toml # type: ignore[no-redef] + except ImportError: + return [] + try: + project = toml.loads(text).get("project") or {} + except toml.TOMLDecodeError: + return [] + return [str(item) for item in project.get("dependencies") or []] + + def run_core_tier( sandbox: Sandbox, *, @@ -522,33 +597,93 @@ def _egress( ) -def _gpu(sandbox: Sandbox, requested: bool, cuda: str | None, timeout: float | None) -> CheckResult: +def _gpu( + sandbox: Sandbox, + requested: bool, + cuda: str | None, + timeout: float | None, + count: int = 1, +) -> CheckResult: + """Check 11: the GPUs asked for are visible, and CUDA is the spec's. + + Two CUDA versions are read, and they answer different questions. The + image's own toolkit (`nvcc`, or the base's `CUDA_VERSION`) is what the + spec's `accelerator.cuda` names and the base channel pins. `nvidia-smi`'s + "CUDA Version" is the newest CUDA the host's *driver* runs, which the + image does not choose: comparing the spec with it failed a correct image + on any newer driver (found on 2026-09-18, E2-17). The driver only has to + be new enough for the toolkit. + """ if not requested: return _result(11, True, gating=False, detail="no accelerator was requested") body = ( - "import re as _dl_re, subprocess as _dl_sp\n" - "_dl_out = {'returncode': None, 'gpus': [], 'cuda': None}\n" + "import os as _dl_os, re as _dl_re, shutil as _dl_sh, subprocess as _dl_sp\n" + "_dl_out = {'returncode': None, 'gpus': [], 'driver': None, 'cuda': None}\n" "try:\n" " _dl_run = _dl_sp.run(['nvidia-smi'], capture_output=True, text=True, timeout=30)\n" " _dl_out['returncode'] = _dl_run.returncode\n" " _dl_match = _dl_re.search(r'CUDA Version:\\s*([0-9.]+)', _dl_run.stdout)\n" - " _dl_out['cuda'] = _dl_match.group(1) if _dl_match else None\n" + " _dl_out['driver'] = _dl_match.group(1) if _dl_match else None\n" " _dl_names = _dl_sp.run(['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'], " "capture_output=True, text=True, timeout=30)\n" " _dl_out['gpus'] = [_dl_l.strip() for _dl_l in _dl_names.stdout.splitlines() " "if _dl_l.strip()]\n" "except (OSError, _dl_sp.SubprocessError) as _dl_error:\n" " _dl_out['error'] = str(_dl_error)\n" + "try:\n" + " _dl_nvcc = _dl_sh.which('nvcc') or '/usr/local/cuda/bin/nvcc'\n" + " _dl_run = _dl_sp.run([_dl_nvcc, '--version'], capture_output=True, text=True, " + "timeout=30)\n" + " _dl_match = _dl_re.search(r'release ([0-9.]+),', _dl_run.stdout)\n" + " _dl_out['cuda'] = _dl_match.group(1) if _dl_match else None\n" + "except (OSError, _dl_sp.SubprocessError):\n" + " pass\n" + "_dl_out['cuda'] = _dl_out['cuda'] or _dl_os.environ.get('CUDA_VERSION') or None\n" ) answer = probe(sandbox, _code(11, body, "_dl_out"), timeout=timeout) problems = [] - if answer.get("returncode") != 0 or not answer.get("gpus"): + gpus = answer.get("gpus") or [] + if answer.get("returncode") != 0 or not gpus: problems.append("no GPU is visible") - if cuda and not str(answer.get("cuda") or "").startswith(cuda): - problems.append(f"CUDA is {answer.get('cuda')}, not {cuda}") - return _result( - 11, not problems, gating=False, detail="; ".join(problems) or None, actual=answer - ) + elif len(gpus) < count: + problems.append(f"{len(gpus)} GPU(s) visible, not the {count} asked for") + toolkit = str(answer.get("cuda") or "") + if cuda and not _version_is(toolkit, cuda): + problems.append(f"CUDA is {toolkit or None}, not {cuda}") + driver = str(answer.get("driver") or "") + if toolkit and driver and _version_tuple(driver) < _version_tuple(toolkit): + problems.append(f"the driver runs CUDA up to {driver}, older than the image's {toolkit}") + # A GPU version gates on this (E2-17): a version that asked for an + # accelerator and cannot see it, or sees the wrong CUDA, is not the + # version its spec describes. A version that asked for none never + # reaches here (the trivial pass above), so the extended tier still + # gates nothing for a CPU version. + return _result(11, not problems, gating=True, detail="; ".join(problems) or None, actual=answer) + + +def _version_tuple(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in re.findall(r"\d+", version)) + + +def _version_is(version: str, asked: str) -> bool: + """Whether `version` is what `asked` names: `12` and `12.8` both name 12.8.3.""" + wanted = _version_tuple(asked) + return bool(wanted) and _version_tuple(version)[: len(wanted)] == wanted + + +def run_accelerator_check( + sandbox: Sandbox, + *, + cuda: str | None = None, + count: int = 1, + timeout: float | None = 120.0, +) -> CheckResult: + """Appendix B check 11 alone, gating, for a version that asked for an accelerator (E2-17). + + What a builder's smoke test adds to the core tier for a GPU version: the + rest of the extended tier records, and this one decides. + """ + return _guard(11, True, lambda: _gpu(sandbox, True, cuda, timeout, count)) def _throughput( @@ -617,6 +752,7 @@ def run_extended_tier( contract: SandboxContract = SANDBOX_CONTRACT_V1, accelerator_requested: bool = False, cuda_version: str | None = None, + accelerator_count: int = 1, egress_allowed: Sequence[str] = (), egress_blocked: Sequence[str] = (), cold_start_seconds: float | None = None, @@ -625,10 +761,17 @@ def run_extended_tier( concurrent_kernels: int = 4, timeout: float | None = 120.0, ) -> ValidationResult: - """Appendix B checks 10-14: recorded per variant, gating nothing.""" + """Appendix B checks 10-14: recorded per variant. Only check 11 gates, and + only for a version that asked for an accelerator (E2-17) — a GPU version + that cannot see its GPU is not what its spec describes; every other + extended check records without gating.""" checks = [ _guard(10, False, lambda: _egress(sandbox, egress_allowed, egress_blocked, timeout)), - _guard(11, False, lambda: _gpu(sandbox, accelerator_requested, cuda_version, timeout)), + _guard( + 11, + accelerator_requested, + lambda: _gpu(sandbox, accelerator_requested, cuda_version, timeout, accelerator_count), + ), _guard(12, False, lambda: _throughput(sandbox, contract, minimum_mib_per_second, timeout)), _guard(13, False, lambda: _cold_start(cold_start_seconds, cold_start_budget)), _guard(14, False, lambda: _concurrent(sandbox, concurrent_kernels, timeout)), @@ -649,7 +792,10 @@ def run_conformance( extended: Mapping[str, Any] | None = None, timeout: float | None = 120.0, ) -> ValidationResult: - """Both tiers: the core tier decides, the extended tier is recorded beside it.""" + """Both tiers together. The core tier decides; the extended tier is recorded + beside it, save for check 11, which also decides for a version that asked for + an accelerator (E2-17) — a GPU version that cannot see its GPU has not built + what its spec described.""" core = run_core_tier( sandbox, python_version=python_version, diff --git a/code_sandboxes/environments/constraints/sandbox-contract-v1.txt b/code_sandboxes/environments/constraints/sandbox-contract-v1.txt index 3f27ac5..d5afa7c 100644 --- a/code_sandboxes/environments/constraints/sandbox-contract-v1.txt +++ b/code_sandboxes/environments/constraints/sandbox-contract-v1.txt @@ -20,8 +20,18 @@ # jupyterlab and jupyter-server-ydoc require jupyter-server>=2.19. The rebased fork # satisfies those ranges, so pip keeps it; its local label, which PyPI never serves, # tells it apart from the release. +# +# jupyter-kernels carries the pooled kernel manager the runtime's Jupyter config +# selects (kernel_manager_class = jupyter_kernels.pool.mapping.PooledMappingKernelManager, +# plane/etc/dockerfiles/jupyter-python/etc/jupyter/config/jupyter_config.py). Unlike the +# deprecated private datalayer-kernels it replaced, PyPI serves it, so a resolve satisfies +# it from the index and the wheelhouse carries no wheel for it. The pin is still forced in: +# without it `uv pip sync --require-hashes` strips it from a user environment's image and the +# runtime's Jupyter server crashloops on a kernel_manager_class it can no longer import +# (found live on 2026-09-15, the first user-environment build to reach the smoke test). ipykernel==7.3.0 jupyter-client==8.9.1 jupyter-server==2.21.0+datalayer.1 jupyter-server-nbmodel==0.2.8 +jupyter-kernels==1.2.23 datalayer==1.7.4 diff --git a/code_sandboxes/environments/constraints/wheelhouse/README.md b/code_sandboxes/environments/constraints/wheelhouse/README.md index bbf2e79..2ed7334 100644 --- a/code_sandboxes/environments/constraints/wheelhouse/README.md +++ b/code_sandboxes/environments/constraints/wheelhouse/README.md @@ -38,3 +38,10 @@ Pure Python (`py3-none-any`), so one wheel serves every base's Python version this channel carries. Rebuild it here, under this same file name, whenever `services/kernels/Dockerfile`'s pinned commit changes — the constraints file's own version pin and this wheel move together. + +The pooled kernel manager pin, `jupyter-kernels`, needs no wheel here: PyPI +serves it, so a resolve satisfies it from the index like any other pin. It +was previously the private `datalayer-kernels`, which no index carried and +so was baked here as a wheel; the migration to the public +[`jupyter-kernels`](https://pypi.org/project/jupyter-kernels/) package +dropped that wheel. diff --git a/code_sandboxes/environments/contract.py b/code_sandboxes/environments/contract.py index 592c3c7..5cacd7c 100644 --- a/code_sandboxes/environments/contract.py +++ b/code_sandboxes/environments/contract.py @@ -24,26 +24,38 @@ import re import shlex import sys +from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path from pydantic import BaseModel, ConfigDict -from .bases import is_approved_repository -from .errors import CAPABILITY_UNSUPPORTED, EnvironmentsError +from .bases import APPROVED_BASES, ApprovedBase, is_approved_repository +from .errors import CAPABILITY_UNSUPPORTED, SPEC_INVALID, EnvironmentsError __all__ = [ "CONTRACT_V1", + "MAX_CONTEXT_FILES", + "MAX_CONTEXT_FILE_BYTES", + "MAX_CONTEXT_TOTAL_BYTES", "SANDBOX_CONTRACT_V1", "SUPPORTED_CONTRACTS", + "BuildContextEntry", + "BuildContextFinding", "ContractRow", + "DockerfileBase", "DockerfileFinding", "DockerfileInstruction", "SandboxContract", + "check_build_context", "check_dockerfile", "contract_markdown", + "dockerfile_base", + "dockerfile_findings_for_build", "get_contract", "parse_dockerfile", + "pin_dockerfile_base", + "validate_build_context", "validate_dockerfile", ] @@ -97,7 +109,7 @@ class SandboxContract(BaseModel): uid=1000, gid=100, home="/home/datalayer", - workdir="/home/datalayer/content", + workdir="/home/datalayer", reserved_path="/opt/datalayer", doctor_path="/opt/datalayer/bin/datalayer-sandbox", python_executables=("python3", "pip"), @@ -129,7 +141,9 @@ class SandboxContract(BaseModel): area="User", requirement=( "Non-root user `datalayer`, uid 1000, gid 100, home `/home/datalayer`, " - "working directory `/home/datalayer/content`." + "working directory `/home/datalayer` — the home itself, which is " + "where a person's folders are mounted and what the file browser " + "shows, so `pwd` and the browser agree." ), reason=( "Runtime pods run as 1000:100 and home folders on the shared filesystem are owned " @@ -180,7 +194,7 @@ class SandboxContract(BaseModel): ContractRow( area="Filesystem", requirement=( - "`/opt/datalayer` is reserved and read-only to the user; `/home/datalayer/content` " + "`/opt/datalayer` is reserved and read-only to the user; `/home/datalayer` " "is writable; nothing is assumed to persist across restarts." ), reason="Datalayer's tools live under `/opt/datalayer`; " @@ -453,6 +467,272 @@ def check_dockerfile(text: str, *, contract: SandboxContract = SANDBOX_CONTRACT_ ) +# --- A Dockerfile source's base, and how it is built (E3-03) ----------------------------- + + +@dataclass(frozen=True) +class DockerfileBase: + """The approved base a Dockerfile builds on: its channel, and the `FROM` lines naming it.""" + + ref: str + channel: str + lines: tuple[int, ...] + + +@dataclass(frozen=True) +class _ApprovedFrom: + line: int + image: str + ref: str + tag: str + digest: str + + +def _from_image(instruction: DockerfileInstruction) -> tuple[str, str | None]: + """A `FROM`'s image and its stage name, flags skipped.""" + tokens = _tokens(instruction.arguments) + for index, argument in enumerate(tokens): + if argument.startswith("--"): + continue + rest = tokens[index + 1 :] + alias = rest[1].lower() if len(rest) >= 2 and rest[0].upper() == "AS" else None + return argument, alias + return "", None + + +def _approved_froms(text: str, bases: Mapping[str, ApprovedBase]) -> list[_ApprovedFrom]: + """Every `FROM` naming an approved base, in order; a stage name is not a base.""" + stages: set[str] = set() + found: list[_ApprovedFrom] = [] + for instruction in parse_dockerfile(text): + if instruction.keyword != "FROM": + continue + image, alias = _from_image(instruction) + if image and image.lower() not in stages and "$" not in image: + repository, _, digest = image.partition("@") + last = repository.rsplit("/", 1)[-1] + name, _, tag = last.partition(":") + repository = repository[: len(repository) - len(last)] + name + for base in bases.values(): + if any( + repository == known or repository.endswith("/" + known) + for known in (base.ref, base.repository) + ): + found.append(_ApprovedFrom(instruction.line, image, base.ref, tag, digest)) + break + if alias: + stages.add(alias) + return found + + +def dockerfile_findings_for_build( + text: str, bases: Mapping[str, ApprovedBase] = APPROVED_BASES +) -> list[DockerfileFinding]: + """What a Dockerfile source must also be to be built, beyond what the contract allows. + + - **One base, named by its channel.** Every `FROM` of an approved base + names the same base, and its channel as the tag + (`datalayer/python-cpu:2026.09`): the build pins that channel to its + digest, the way a `packages` source's base is pinned (D-9), and the + lock is solved in it. A digest is refused rather than trusted — which + channel it belongs to is what the build must know. + - **No build context yet.** A `COPY` or `ADD` from the context needs the + upload this item also describes, which is not taken yet; without it + the file is absent and the build fails halfway. From another stage + (`--from=`), from a heredoc, and `ADD` of a URL need none. + - **The default escape character.** The build appends its own lines, + continued with a backslash. + """ + findings: list[DockerfileFinding] = [] + for raw in text.splitlines(): + directive = _DIRECTIVE.match(raw.strip()) + if not directive: + break + if directive.group(1).lower() == "escape" and directive.group(2) != "\\": + findings.append( + DockerfileFinding( + 1, "escape", "keep the default escape character: the build appends lines to it" + ) + ) + named: list[_ApprovedFrom] = [] + for found in _approved_froms(text, bases): + channels = bases[found.ref].channels + if found.digest: + message = f"name `{found.ref}` by its channel, not a digest: the build pins the channel" + elif not found.tag: + example = next(iter(channels), "2026.09") + message = f"name `{found.ref}`'s channel as its tag, such as `{found.ref}:{example}`" + elif found.tag not in channels: + message = f"`{found.ref}` has no channel `{found.tag}`; channels: " + ( + ", ".join(channels) or "none" + ) + else: + named.append(found) + continue + findings.append(DockerfileFinding(found.line, "FROM", message)) + if len({(found.ref, found.tag) for found in named}) > 1: + findings.append( + DockerfileFinding( + named[1].line, + "FROM", + "every approved base is the same base and channel: the build pins one", + ) + ) + for instruction in parse_dockerfile(text): + if instruction.keyword not in ("COPY", "ADD"): + continue + tokens = _tokens(instruction.arguments) + if "<<" in instruction.arguments or any(t.startswith("--from=") for t in tokens): + continue + sources = [t for t in tokens if not t.startswith("--")][:-1] + local = [s for s in sources if not re.match(r"^(https?|git)://|^git@", s, re.IGNORECASE)] + if local: + findings.append( + DockerfileFinding( + instruction.line, + instruction.keyword, + f"copies `{local[0]}` from the build context, which this deployment does not " + "take yet: bake it with a `RUN` or a heredoc, or copy it from another stage", + ) + ) + return sorted(findings, key=lambda finding: finding.line) + + +def dockerfile_base( + text: str, bases: Mapping[str, ApprovedBase] = APPROVED_BASES +) -> DockerfileBase: + """The approved base a Dockerfile source is built on, or a refusal naming the line.""" + refused = [ + finding + for finding in dockerfile_findings_for_build(text, bases) + if finding.instruction in ("FROM", "escape") + ] + if refused: + raise EnvironmentsError( + SPEC_INVALID, + f"line {refused[0].line}: {refused[0].message}", + detail={"field": "spec.build.dockerfile", "line": refused[0].line}, + ) + found = _approved_froms(text, bases) + if not found: + raise EnvironmentsError( + SPEC_INVALID, + "the Dockerfile builds on no approved base", + detail={"field": "spec.build.dockerfile"}, + ) + return DockerfileBase( + ref=found[0].ref, + channel=found[0].tag, + lines=tuple(item.line for item in found), + ) + + +def pin_dockerfile_base( + text: str, reference: str, bases: Mapping[str, ApprovedBase] = APPROVED_BASES +) -> str: + """The Dockerfile as it is built: each `FROM` of its base pinned to `reference`. + + Only the image of those lines changes; comments, the author's own + continuations and the stage names stay as written. The parser directives + go, because the build states its own frontend first (`# syntax=`), and a + directive anywhere but the very top is only a comment. + """ + images = {found.line: found.image for found in _approved_froms(text, bases)} + rewritten: list[str] = [] + in_directives = True + for number, raw in enumerate(text.splitlines(), start=1): + if in_directives and _DIRECTIVE.match(raw.strip()): + continue + in_directives = False + image = images.get(number) + rewritten.append(raw.replace(image, reference, 1) if image else raw) + return "\n".join(rewritten) + "\n" + + +# --- The build context (E3-03) ------------------------------------------------------- + +#: A `dockerfile` source uploads a build context to object storage. These bound +#: what Runtimes accepts before it issues a presigned URL, so a context cannot +#: be a way to smuggle a host file in (a symlink or `..`), or to fill a bucket. +MAX_CONTEXT_FILES = 2000 +MAX_CONTEXT_FILE_BYTES = 50 * 1024 * 1024 +MAX_CONTEXT_TOTAL_BYTES = 100 * 1024 * 1024 + +_CONTEXT_SEPARATOR = re.compile(r"[\\/]") + + +@dataclass(frozen=True) +class BuildContextEntry: + """One member of an uploaded build context: its path, size, and whether it is a symlink.""" + + path: str + size_bytes: int = 0 + is_symlink: bool = False + + +@dataclass(frozen=True) +class BuildContextFinding: + """Something in a build context that must not be uploaded.""" + + path: str + message: str + + def to_dict(self) -> dict[str, object]: + return {"path": self.path, "message": self.message} + + +def validate_build_context( + entries: Sequence[BuildContextEntry], +) -> list[BuildContextFinding]: + """Everything in a build context the upload refuses, in the order given. + + Refused: an absolute path, a `..` that would escape the context, a symlink + (which could point at a host file the build then reads), a file over the + per-file limit, and — once — a context with too many files or too many + bytes in all. + """ + findings: list[BuildContextFinding] = [] + total = 0 + for entry in entries: + path = entry.path + components = _CONTEXT_SEPARATOR.split(path) + if not path or all(part in ("", ".") for part in components): + findings.append(BuildContextFinding(path, "is not a path inside the context")) + elif path.startswith("/") or path.startswith("\\"): + findings.append(BuildContextFinding(path, "is an absolute path, not a context path")) + elif ".." in components: + findings.append(BuildContextFinding(path, "escapes the context with `..`")) + if entry.is_symlink: + findings.append(BuildContextFinding(path, "is a symlink, which could read a host file")) + if entry.size_bytes > MAX_CONTEXT_FILE_BYTES: + findings.append( + BuildContextFinding( + path, f"is over the {MAX_CONTEXT_FILE_BYTES}-byte per-file limit" + ) + ) + total += entry.size_bytes + if len(entries) > MAX_CONTEXT_FILES: + findings.append(BuildContextFinding("", f"has more than {MAX_CONTEXT_FILES} files")) + if total > MAX_CONTEXT_TOTAL_BYTES: + findings.append( + BuildContextFinding("", f"is over the {MAX_CONTEXT_TOTAL_BYTES}-byte total limit") + ) + return findings + + +def check_build_context(entries: Sequence[BuildContextEntry]) -> None: + """Refuse a build context the upload does not allow, naming the first fault.""" + findings = validate_build_context(entries) + if findings: + first = findings[0] + where = f"`{first.path}`: " if first.path else "" + raise EnvironmentsError( + SPEC_INVALID, + f"{where}{first.message}", + detail={"findings": [finding.to_dict() for finding in findings]}, + ) + + # --- The documentation page ---------------------------------------------------------- diff --git a/code_sandboxes/environments/doctor/datalayer_sandbox.py b/code_sandboxes/environments/doctor/datalayer_sandbox.py index 243035c..51f6a6a 100644 --- a/code_sandboxes/environments/doctor/datalayer_sandbox.py +++ b/code_sandboxes/environments/doctor/datalayer_sandbox.py @@ -43,7 +43,7 @@ "uid": 1000, "gid": 100, "home": "/home/datalayer", - "workdir": "/home/datalayer/content", + "workdir": "/home/datalayer", "reserved_path": "/opt/datalayer", "python_executables": ["python3", "pip"], "kernel_packages": ["ipykernel", "jupyter_client"], diff --git a/code_sandboxes/environments/files.py b/code_sandboxes/environments/files.py index e31cac6..e26f516 100644 --- a/code_sandboxes/environments/files.py +++ b/code_sandboxes/environments/files.py @@ -25,12 +25,15 @@ def build_entries( environment: Environment, *, source_of: Callable[[FileEntry], str] | None = None ) -> list[BuildEntry]: - """The spec's files as build entries. + """The spec's baked files as build entries: ``files`` then ``contentsBuild``. - ``source_of`` turns a ``contentRef`` into a URL the build can fetch — - a presigned URL for a ``blob://`` reference — and defaults to the - reference itself. A file without its sha256 is refused: the build - verifies every byte it bakes. + ``files`` a user uploaded (referenced by ``contentRef``) and + ``contentsBuild`` fetched from an external ``source`` are both baked the + same way, so they are the same kind of entry here. ``source_of`` turns a + ``contentRef`` into a URL the build can fetch — a presigned URL for a + ``blob://`` reference — and defaults to the reference itself; a + ``contentsBuild`` source is already a URL and is used as is. A file without + its sha256 is refused: the build verifies every byte it bakes. """ entries: list[BuildEntry] = [] for index, entry in enumerate(environment.spec.files): @@ -48,6 +51,15 @@ def build_entries( size_bytes=entry.size_bytes, ) ) + for built in environment.spec.contents_build: + entries.append( + BuildEntry( + source_uri=built.source, + destination_path=built.path, + sha256=built.sha256, + size_bytes=built.size_bytes, + ) + ) return entries @@ -63,7 +75,7 @@ def files_step( """ if variant not in VARIANTS: raise ValueError(f"{variant!r} is not a variant") - if not environment.spec.files: + if not environment.spec.files and not environment.spec.contents_build: return [] build = EnvironmentBuild( environment=environment.metadata.name, diff --git a/code_sandboxes/environments/policy.py b/code_sandboxes/environments/policy.py index 9a238b2..b0848e7 100644 --- a/code_sandboxes/environments/policy.py +++ b/code_sandboxes/environments/policy.py @@ -32,17 +32,20 @@ from dataclasses import dataclass, field from typing import Any -from .errors import SCAN_BLOCKED, EnvironmentsError +from .errors import POLICY_DENIED, SCAN_BLOCKED, EnvironmentsError __all__ = [ "DEFAULT_POLICY", "SEVERITIES", + "EnvironmentsPolicy", "Finding", "PolicyDecision", "ScanPolicy", "decide", + "environments_policy_from_rules", "findings_of", "refuse_if_blocked", + "refuse_unlicensed", ] #: The severities a scanner reports, weakest first. Anything it reports that is @@ -300,3 +303,132 @@ def refuse_if_blocked(decision: PolicyDecision, *, reference: str = "") -> None: "blocking": [finding.id for finding in decision.blocking], }, ) + + +@dataclass(frozen=True) +class EnvironmentsPolicy: + """An organization's own narrowing of what a build may do (E3-06). + + Every allowlist is ``None`` until an organization writes one, meaning + that dimension is unrestricted beyond the platform's own defaults — the + approved bases, the public registries a spec's own validation already + checks, and nothing at all for indexes, packages or licences. Setting a + list, even an empty one, is a real policy: an organization that writes + ``allowed_licenses=()`` has denied every licence, and this does not + second-guess a policy that strict — the refusal it produces names + exactly why. + + This narrows; it never widens. A base, index, registry, package or + licence the platform already refuses stays refused whatever an + organization allows — the same rule `refuse_widening` already keeps for + the gateway's own policy layers (`iam/datalayer_iam/services/ + mcp_policies.py`), applied here to a different set of rules. + """ + + allowed_bases: tuple[str, ...] | None = None + allowed_indexes: tuple[str, ...] | None = None + allowed_registries: tuple[str, ...] | None = None + allowed_packages: tuple[str, ...] | None = None + allowed_licenses: tuple[str, ...] | None = None + scan: ScanPolicy = DEFAULT_POLICY + #: The policy document's own version, carried into E1-08's decision + #: record so a build made under one policy is not misread once an + #: organization changes it — the version answers "which policy" without + #: needing IAM asked again. + version: int | None = None + + def allows(self, dimension: str, value: str) -> bool: + """Whether `value` clears this policy's allowlist for `dimension`. + + `True` when the dimension is unrestricted (`None`) or the value is on + the list; the comparison is exact, not a prefix or a host match — an + index or a registry is compared the way `image_registry_allowed` + already compares one, and a base or a package by its own name. + """ + allowed = getattr(self, f"allowed_{dimension}") + return allowed is None or value in allowed + + +#: No organization has written a policy: every dimension unrestricted, the +#: platform's own default scan threshold. The reading a caller with no +#: policy document gets, and what every check below is a no-op against. +UNRESTRICTED_POLICY = EnvironmentsPolicy() + +#: What `environments_policy_from_rules` reads a list-shaped field as. Kept +#: beside `ScanPolicy`'s own fields so the two are validated the same way. +_LIST_FIELDS: tuple[str, ...] = ( + "allowedBases", + "allowedIndexes", + "allowedRegistries", + "allowedPackages", + "allowedLicenses", +) + + +def environments_policy_from_rules( + rules: Mapping[str, Any] | None, *, version: int | None = None +) -> EnvironmentsPolicy: + """The `environments` section of an organization's MCP policy document, + as the shape this module checks against (E3-06). + + `rules` is the raw object IAM stores under the policy's own + ``environments`` key — camelCase, the same convention every other rule in + `iam/datalayer_iam/services/mcp_policies.py` already uses. Absent, or not + an object, answers :data:`UNRESTRICTED_POLICY`: an organization that has + not written this section has narrowed nothing, the same reading every + other rule in that module gives an unset one. + + Never raises. A caller holding an organization's policy already trusts + IAM to have validated it at the write (`mcp_policies.validate_rules`); + asking twice, differently, would let the two readings disagree about + what a stored policy means. + """ + if not isinstance(rules, Mapping): + return UNRESTRICTED_POLICY + lists: dict[str, tuple[str, ...] | None] = {} + for camel in _LIST_FIELDS: + value = rules.get(camel) + snake = "".join( + f"_{c.lower()}" if c.isupper() else c for c in camel + ) # allowedBases -> allowed_bases + lists[snake] = ( + tuple(str(item) for item in value) if isinstance(value, (list, tuple)) else None + ) + scan_rules = rules.get("scan") if isinstance(rules.get("scan"), Mapping) else {} + blocks_at = str(scan_rules.get("blocksAt") or DEFAULT_POLICY.blocks_at).upper() + scan = ScanPolicy( + blocks_at=blocks_at if blocks_at in SEVERITIES else DEFAULT_POLICY.blocks_at, + only_fixable=bool(scan_rules.get("onlyFixable", DEFAULT_POLICY.only_fixable)), + allowed=tuple(str(item) for item in (scan_rules.get("allowed") or ())) + or DEFAULT_POLICY.allowed, + ) + return EnvironmentsPolicy(scan=scan, version=version, **lists) + + +def refuse_unlicensed( + pairs: Sequence[tuple[str, str]], policy: EnvironmentsPolicy = UNRESTRICTED_POLICY +) -> None: + """Raise `DL_ENV_POLICY_DENIED` on the first licence this policy does not + allow, naming the package that carries it (E3-06). + + `pairs` is `attest.licenses_by_package`'s own output: `(package, + licence)`, in the SBOM's own order, so the same artifact always names the + same first offender. A no-op when `policy.allowed_licenses` is unset — + every artifact today, until an organization writes one. + """ + if policy.allowed_licenses is None: + return + for package, licence in pairs: + if not policy.allows("licenses", licence): + raise EnvironmentsError( + POLICY_DENIED, + f"`{licence}` ({package or 'an unnamed package'}) is not an allowed " + f"licence (the allowed ones are {', '.join(policy.allowed_licenses) or '(none)'})", + detail={ + "field": "licenses", + "package": package, + "license": licence, + "allowed": list(policy.allowed_licenses), + **({"policyVersion": policy.version} if policy.version is not None else {}), + }, + ) diff --git a/code_sandboxes/environments/resolve.py b/code_sandboxes/environments/resolve.py index 811d65c..71002b7 100644 --- a/code_sandboxes/environments/resolve.py +++ b/code_sandboxes/environments/resolve.py @@ -39,6 +39,7 @@ from __future__ import annotations import hashlib +import os import re import shlex import shutil @@ -49,7 +50,7 @@ from datetime import datetime, timezone from functools import lru_cache from pathlib import Path -from typing import Any, Callable, Protocol +from typing import TYPE_CHECKING, Any, Callable, Protocol from .bases import APPROVED_BASES, ApprovedBase, channel_snapshot, resolve_base from .build_secrets import resolve_build_secret @@ -76,10 +77,16 @@ parse_requirements_txt, ) +if TYPE_CHECKING: + from .resolve_conda import CondaResolveRunner + __all__ = [ "APT_PIN_PREFIX", "APT_SNAPSHOT_PREFIX", + "BUILDKIT_PROXY_ENV", "CONSTRAINTS_PATH", + "COVERAGE_PREFIX", + "DOCKERFILE_COVERAGE", "LOCK_FORMAT", "PROTECTED_PIN_PREFIX", "BuildkitResolveRunner", @@ -92,6 +99,9 @@ "apt_pins", "apt_pins_in", "apt_snapshot_in", + "buildkit_proxy", + "buildkit_proxy_options", + "egress_refused_hosts", "lock_document", "locked_versions", "merge_requirements", @@ -126,12 +136,127 @@ #: How an apt pin is written in the lock. A comment, so every reader of a #: ``pip`` requirements file — the CLI's diff included — ignores it. APT_PIN_PREFIX = "# datalayer-apt: " +#: What a lock says it does not cover, as a header line. +COVERAGE_PREFIX = "# datalayer-coverage: " +#: A `dockerfile` source's lock (E3-03): what the resolver saw, and no more. +DOCKERFILE_COVERAGE = ( + "the declared packages and Datalayer's protected pins only; what the Dockerfile's own " + "instructions install is not locked, so a rebuild may install different versions of it" +) #: The Ubuntu snapshot the apt pins were taken from, as the lock records it. #: The builder installs the pins from the same snapshot, since a pinned #: version can leave the live mirror (D-9). APT_SNAPSHOT_PREFIX = "# datalayer-apt-snapshot: " _SNAPSHOT_ID = re.compile(r"^\d{8}T\d{6}Z$") +#: Where a build's own steps reach the network through, when the build pool +#: gives them nowhere else to go (E1-06): an HTTP proxy that allows only the +#: package indexes, the snapshot mirrors and the registries. The address is +#: the one `buildkitd` itself sees, since a step runs in its network. +BUILDKIT_PROXY_ENV = "DATALAYER_BUILDKIT_PROXY" +_PROXY_URL = re.compile(r"^http://[A-Za-z0-9.\-]+:\d{1,5}$") + + +def buildkit_proxy(proxy: str | None = None) -> str: + """The proxy a build's steps go through: ``proxy``, or the environment's. + + Empty is no proxy, which is what a `buildkitd` with open egress needs + (`plane local`'s own). Anything that is not ``http://host:port`` is + refused rather than handed to every package manager of every build. + """ + value = (os.environ.get(BUILDKIT_PROXY_ENV, "") if proxy is None else proxy).strip() + if value and not _PROXY_URL.match(value): + raise ValueError(f"{BUILDKIT_PROXY_ENV} must be http://host:port, not {value!r}") + return value + + +def buildkit_proxy_options(proxy: str) -> list[str]: + """The ``buildctl`` options that send a build's steps through ``proxy``. + + The Dockerfile frontend predefines these build args: a ``RUN`` step sees + them without an ``ARG``, they never reach the image's config or history, + and they do not change a cache key. Both spellings, since ``apt`` and + ``curl`` read only the lower-case one for plain HTTP and ``uv``, ``pip`` + and ``micromamba`` read either. + """ + if not proxy: + return [] + options: list[str] = [] + for name in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"): + options += ["--opt", f"build-arg:{name}={proxy}"] + for name in ("NO_PROXY", "no_proxy"): + options += ["--opt", f"build-arg:{name}=127.0.0.1,localhost"] + return options + + +#: What each tool writes when the build pool's egress proxy refuses it a +#: tunnel (E2-19). Read from the tools themselves, through the chart's own +#: Squid refusing every host, on 2026-09-19: pip, uv 0.12, micromamba 2.3, +#: git, curl 8 without `-f`, wget, and a Go client — `buildkitd`'s own pulls. +#: Each is a refused CONNECT and nothing else: a host that answered 403 over +#: its own TLS says so differently. `curl -f` ("The requested URL returned +#: error: 403"), `wget -q` (nothing) and `apt` over plain HTTP ("403 +#: Forbidden [IP: proxy]") cannot be told from the host's own 403, and are +#: left to the log. +_EGRESS_REFUSED = re.compile( + r"Tunnel connection failed: 403" # pip, requests, urllib3 + r"|tunnel error: unsuccessful" # uv + r"|CONNECT tunnel failed, response 403" # curl, git, micromamba + r"|Proxy tunneling failed: Forbidden" # wget + r'|"https?://[^"\s]+": Forbidden' # Go: buildkitd, containerd +) +#: A host a line names: urllib3's `host='…'`, or the host of a URL. +_NAMED_HOST = re.compile(r"host='(?P[A-Za-z0-9.\-]+)'|https?://(?P[A-Za-z0-9.\-]+)") +#: `buildctl`'s plain progress: `#12 0.874 `. Steps run +#: side by side, so a line is read against the lines of its own step. +_BUILDKIT_LINE = re.compile(r"^#(?P\d+) (?:\d+\.\d+ )?(?P.*)$") + + +def egress_refused_hosts(log: str, *, proxy: str = "", limit: int = 10) -> list[str]: + """The hosts the build pool's egress proxy refused, as a build's log shows them (E2-19). + + A refusal is found by what the tool writes (`_EGRESS_REFUSED`), and its + host is the one the same line names — pip's `host='files.pythonhosted.org'`, + git's URL — or else the last one its step named before it: uv and + micromamba write the URL a line or four above the refusal, wget and a + silent `curl` only in the `RUN` line. The proxy's own address is never a + host it refused. In the order they were first refused, at most ``limit``. + """ + own = {"127.0.0.1", "localhost"} + if proxy: + own.add(proxy.split("://", 1)[-1].rsplit(":", 1)[0]) + hosts: list[str] = [] + last: dict[str, str] = {} + for raw in log.splitlines(): + line = _BUILDKIT_LINE.match(raw) + step, text = (line.group("step"), line.group("text")) if line else ("", raw) + named = [ + host.lower().rstrip(".") + for match in _NAMED_HOST.finditer(text) + for host in (match.group("pool") or match.group("url"),) + if host and host.lower().rstrip(".") not in own + ] + if _EGRESS_REFUSED.search(text): + host = named[0] if named else last.get(step, "") + if host and host not in hosts: + hosts.append(host) + if len(hosts) >= limit: + break + if named: + last[step] = named[-1] + return hosts + + +def egress_findings(hosts: Sequence[str]) -> list[dict[str, str]]: + """The refused hosts as a failure's findings: what a page lists, not the log.""" + return [{"kind": "egress_refused", "subject": host} for host in hosts] + + +def egress_hosts_text(hosts: Sequence[str]) -> str: + quoted = [f"`{host}`" for host in hosts] + return quoted[0] if len(quoted) == 1 else ", ".join(quoted[:-1]) + " and " + quoted[-1] + + #: How a protected pin is recorded in the lock. PROTECTED_PIN_PREFIX = "# datalayer-protected: " @@ -465,6 +590,19 @@ def parse_resolver_failure( }, ) + # An index the build pool may not reach is not an outage: resolving again + # is refused again, so it is said with the host rather than retried. + refused = egress_refused_hosts(output) + if refused and "No solution found" not in text: + return EnvironmentsError( + PACKAGE_NOT_FOUND, + f"The build pool's egress proxy refused {egress_hosts_text(refused)}: a version " + "resolves only against the indexes the build pool allows. Use one of them, or " + "ask for the host to be allowed", + detail={**detail, "refused_hosts": refused, "findings": egress_findings(refused)}, + retryable=False, + ) + pair = _conflicting_pair(mentions) if "No solution found" in text else None if pair: return EnvironmentsError( @@ -610,8 +748,11 @@ def __init__( tlscacert: str | None = None, apt_snapshot: str = "", timeout: float = 900.0, + proxy: str | None = None, ) -> None: self._buildctl = (shutil.which("buildctl") or "") if buildctl is None else buildctl + #: The build pool's egress proxy (E1-06), or `DATALAYER_BUILDKIT_PROXY`. + self._proxy = buildkit_proxy(proxy) #: The build pool's `buildkitd` takes mTLS connections only #: (PLAN_ENV.md E1-06); a plain-socket one, such as `plane local`'s #: own ephemeral daemon, needs none of these three. All or nothing, @@ -748,6 +889,7 @@ def solve( f"dockerfile={root}", "--output", f"type=local,dest={out}", + *buildkit_proxy_options(self._proxy), ] say(f"Solving the lock in {request.base_reference}") try: @@ -848,9 +990,38 @@ def locked_versions(lock_text: str) -> dict[str, str]: ] if len(pinned) == 1: versions[canonicalize_name(requirement.name)] = pinned[0] + elif requirement.url: + wheel = _wheel_version(requirement.name, requirement.url) + if wheel: + versions[canonicalize_name(requirement.name)] = wheel return versions +def _wheel_version(name: str, url: str) -> str | None: + """The version of a direct reference to a wheel, read from its file name. + + How a `pyproject` lock brings Datalayer's `jupyter-server` fork, which no + index serves (E3-08): `uv` records a hash only for a wheel it fetched by + URL, so the export pins it as `jupyter-server @ https://…whl`, with no + `==` for the version. The wheel's name carries one, and nothing else here + does. A reference that is not a wheel of the same distribution pins none. + """ + from urllib.parse import unquote, urlsplit + + from packaging.utils import InvalidWheelFilename, canonicalize_name, parse_wheel_filename + + filename = unquote(urlsplit(url).path.rsplit("/", 1)[-1]) + if not filename.endswith(".whl"): + return None + try: + wheel_name, version, _build, _tags = parse_wheel_filename(filename) + except InvalidWheelFilename: + return None + if wheel_name != canonicalize_name(name): + return None + return str(version) + + def apt_pins_in(lock_text: str) -> dict[str, str]: """The apt versions a lock records, by package. @@ -886,7 +1057,7 @@ def lock_document( python_version: str, base_reference: str, merged: MergedRequirements, - resolved_at: datetime | None = None, + coverage: str | None = None, ) -> dict[str, Any]: """The stored lock: its text, its digest, and what a reader needs from it. @@ -894,14 +1065,26 @@ def lock_document( output, so the document is still a requirements file — ``pip install -r`` reads it, and so does every tool that only knows that format — while saying everything the build installs. + + **Nothing here is a wall clock.** The header carried a ``# resolved-at:`` + line until 2026-09-16, and since the digest is over the whole text, two + resolves of the same spec, in the same base, pinning the same 320 packages + produced two different digests — the texts differed in that one line out + of 5,388, found by resolving the same environment twice on r1. The cache + key of section 5 is over the lock digest, so D-12's build cache could + never hit: `environments.cache.lookups` read `hit=false` 12 times out of + 12. When the lock was resolved is on the lock document Runtimes stores, in + its ``created_at``, where it belongs. """ - when = (resolved_at or _utcnow()).replace(microsecond=0).isoformat() header = [ "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.", - f"# resolved-at: {when}", f"# python: {python_version}", f"# base: {base_reference}", ] + if coverage: + # What this lock does not cover, in the lock itself: it travels with + # the version, and a reader of the lock is who needs to know (E3-03). + header.append(f"{COVERAGE_PREFIX}{coverage}") for pin in outcome.apt_pins.items(): header.append(f"{APT_PIN_PREFIX}{pin[0]}={pin[1]}") if outcome.apt_pins and outcome.apt_source: @@ -928,6 +1111,9 @@ def resolve_bases( variants: Sequence[str], bases: dict[str, ApprovedBase] = APPROVED_BASES, registry: str | None = None, + *, + ref: str | None = None, + channel: str | None = None, ) -> dict[str, str]: """Each variant's base, pinned by digest (D-9, §4). @@ -937,15 +1123,15 @@ def resolve_bases( Hub. Bare ``@sha256:…`` otherwise, which is what every test and fixture that never passes a credential still gets. """ - base = bases.get(environment.spec.base.ref) - repository = base.repository if base is not None else environment.spec.base.ref + ref = ref or environment.spec.base.ref + channel = channel or environment.spec.base.channel + base = bases.get(ref) + repository = base.repository if base is not None else ref if registry: repository = f"{registry}/{repository}" resolved: dict[str, str] = {} for variant in variants: - digest = resolve_base( - environment.spec.base.ref, environment.spec.base.channel, variant, bases - ) + digest = resolve_base(ref, channel, variant, bases) resolved[variant] = f"{repository}@{digest}" return resolved @@ -1031,10 +1217,12 @@ def _verified_pyproject_lock( root = Path(directory) (root / "pyproject.toml").write_text(dependency_file.content, encoding="utf-8") (root / "uv.lock").write_text(dependency_file.lock_content, encoding="utf-8") + environment = _uv_environment(root) try: checked = invoke( [resolved_uv, "lock", "--dry-run"], cwd=root, + env=environment, capture_output=True, text=True, timeout=timeout, @@ -1082,6 +1270,7 @@ def _verified_pyproject_lock( exported = invoke( [resolved_uv, "export", "--locked", "--format", "requirements.txt"], cwd=root, + env=environment, capture_output=True, text=True, timeout=timeout, @@ -1111,6 +1300,23 @@ def _verified_pyproject_lock( } +def _uv_environment(root: Path) -> dict[str, str]: + """What `uv` checks a lock with: its cache in the scratch directory, and no downloads. + + The durable worker runs as a user with no home, so `uv`'s default cache + under `~/.cache` could not be made, and neither could the interpreter it + downloads when none satisfies `requires-python` (found on 2026-09-18, E3-08). + The interpreter comes from the worker's image instead, where the base + channel's own Python is installed: a check that fetched one per resolve would + be a check that depends on the network in a way nothing records. + """ + return { + **os.environ, + "UV_CACHE_DIR": str(root / ".uv-cache"), + "UV_PYTHON_DOWNLOADS": "never", + } + + def _refuse_unless_protected_pins_are_locked(packages: Mapping[str, str]) -> None: """A `pyproject` source is never merged with Datalayer's pins (E3-01): bringing your own lock is the point, so nothing is force-injected the @@ -1158,8 +1364,8 @@ def resolve_environment( credential: Any = None, log: Callable[[str], None] | None = None, runner: ResolveRunner | None = None, + conda_runner: CondaResolveRunner | None = None, bases: dict[str, ApprovedBase] = APPROVED_BASES, - resolved_at: datetime | None = None, uv: str | None = None, pyproject_run: Callable[..., subprocess.CompletedProcess[str]] | None = None, image_transport: Any = None, @@ -1185,11 +1391,15 @@ def resolve_environment( Where the solve's output goes, line by line: the build's log. runner Where the solve runs. D-9's BuildKit solve by default. + conda_runner + Where a conda ``dependencyFile``'s own ``micromamba`` solve runs + (E3-02): the conda seam's runner, injected by tests and by ``plane + local`` the same way ``runner`` is for a pip source, and D-9's BuildKit + conda solve by default. A pip source ignores it, and a conda source + ignores ``runner``, since the two solves are different tools. bases The approved bases, injected by tests and by a plane whose channel is published somewhere else. - resolved_at - The moment the lock records. Now by default. uv, pyproject_run A `pyproject` `dependencyFile` source's own verification (E3-01): the `uv` to check and export with, and how it is run — injected by @@ -1224,13 +1434,15 @@ def resolve_environment( environment = parse_environment(spec) python = environment.spec.packages.python source = environment.spec.build.source - if source not in ("packages", "dependencyFile", "image"): - raise EnvironmentsError( - CAPABILITY_UNSUPPORTED, - f"`{source}` is not resolved yet: only `packages`, `dependencyFile` and `image` " - "are, in this phase", - detail={"field": "spec.build.source", "source": source}, - ) + # A Dockerfile names its own base in `FROM` (E3-03): that is what is + # pinned and solved in, never `spec.base`, which the schema still asks for. + dockerfile = environment.spec.build.dockerfile if source == "dockerfile" else None + base_ref, base_channel = environment.spec.base.ref, environment.spec.base.channel + if dockerfile is not None: + from .contract import dockerfile_base + + named = dockerfile_base(dockerfile.content, bases) + base_ref, base_channel = named.ref, named.channel if python.manager == "conda": raise EnvironmentsError( CAPABILITY_UNSUPPORTED, @@ -1248,10 +1460,30 @@ def resolve_environment( resolve_secret=resolve_secret, ) if source == "image" - else resolve_bases(environment, wanted, bases, registry=_registry_of(credential)) + else resolve_bases( + environment, + wanted, + bases, + registry=_registry_of(credential), + ref=base_ref, + channel=base_channel, + ) ) dependency_file = environment.spec.build.dependency_file if source == "dependencyFile" and dependency_file is not None: + if dependency_file.source_format == "conda": + # A conda `environment.yml` resolves through its own micromamba + # solve into an explicit lock (E3-02), not uv's pip compile. + from .resolve_conda import resolve_conda_environment + + return resolve_conda_environment( + environment_yml=dependency_file.content, + python_version=environment.spec.language.version, + resolved_bases=resolved_bases, + credential=credential, + log=say, + runner=conda_runner, + ) if dependency_file.source_format == "pyproject": # Verified, not re-resolved (E3-01): the author's own uv.lock is # the answer, and this only proves it still matches pyproject.toml. @@ -1285,11 +1517,7 @@ def resolve_environment( registry_auth=_registry_auth(credential), bootstrap_uv=(source == "image"), # An imported image is not an approved base, and names no channel. - apt_snapshot=( - "" - if source == "image" - else channel_snapshot(environment.spec.base.ref, environment.spec.base.channel, bases) - ), + apt_snapshot=("" if source == "image" else channel_snapshot(base_ref, base_channel, bases)), ) outcome = (runner or BuildkitResolveRunner()).solve(request, say) document = lock_document( @@ -1297,7 +1525,7 @@ def resolve_environment( python_version=environment.spec.language.version, base_reference=solving_in, merged=merged, - resolved_at=resolved_at, + coverage=DOCKERFILE_COVERAGE if dockerfile is not None else None, ) say( f"Locked {document['package_count']} packages" diff --git a/code_sandboxes/environments/resolve_conda.py b/code_sandboxes/environments/resolve_conda.py new file mode 100644 index 0000000..a765431 --- /dev/null +++ b/code_sandboxes/environments/resolve_conda.py @@ -0,0 +1,1078 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""Conda resolution: an ``environment.yml`` to one explicit lock (E3-02). + +A ``build.source: dependencyFile`` version whose ``sourceFormat`` is ``conda`` +brings a conda ``environment.yml`` — the form advanced users already have for a +project that pulls, say, ``gdal`` from ``conda-forge`` rather than from a wheel. +It is resolved the way every other source is (PLAN_ENV.md §5, D-9): **once**, +into one lock, and every variant of the version then builds from that one lock, +so four sandboxes of the same version carry the same packages. + +Two decisions this item makes, both recorded here rather than left implicit: + +- **micromamba, not conda-lock.** The solve runs ``micromamba`` — the same tool + the Datalayer, E2B and Daytona builders install the lock with, and the tool + Modal's ``micromamba_install`` wraps — so the interpreter that resolves is the + one that installs, with no second solver's opinion in between. The lock it + produces is a conda **explicit** file (``@EXPLICIT``): one ``package-url#hash`` + line per package, which ``micromamba create --file`` installs without + re-solving. That is the "explicit lock with hashes" this box asks for. +- **The protected pins still apply to the pip layer.** A conda environment needs + the same kernel stack every sandbox needs to connect (``ipykernel`` and its + siblings, ``constraints/sandbox-contract-v1.txt``). They are merged *over* the + ``pip:`` section of the ``environment.yml`` exactly as :func:`merge_requirements` + merges them over a ``packages`` source's dependencies — a pip requirement that + contradicts a protected pin is refused with the range that is supported, and + every pin is forced in whether or not the environment named it, because a + ``--constraint`` alone never installs what nothing else already depends on. + +Where the solve runs is the :class:`CondaResolveRunner`'s business, mirroring +:mod:`code_sandboxes.environments.resolve`: + +- :class:`BuildkitCondaResolveRunner` is D-9's: a BuildKit solve ``FROM`` the + resolved base digest, which gives resolution the builder's isolation and its + egress allowlist. +- :class:`MicromambaResolveRunner` runs ``micromamba`` where it is called, for + ``plane local`` and for tests. + +@module code_sandboxes.environments.resolve_conda +""" + +from __future__ import annotations + +import hashlib +import re +import shlex +import shutil +import subprocess +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Protocol + +from .errors import ( + CAPABILITY_UNSUPPORTED, + PACKAGE_NOT_FOUND, + PROVIDER_ERROR, + RESOLVE_CONFLICT, + SPEC_INVALID, + EnvironmentsError, +) +from .resolve import ( + WHEELHOUSE_IMAGE_PATH, + WHEELHOUSE_PATH, + MergedRequirements, + ProtectedPin, + buildkit_proxy, + buildkit_proxy_options, + egress_findings, + egress_hosts_text, + egress_refused_hosts, + merge_requirements, +) + +__all__ = [ + "CONDA_LOCK_FORMAT", + "BuildkitCondaResolveRunner", + "CondaEnvironment", + "CondaResolveOutcome", + "CondaResolveRequest", + "CondaResolveRunner", + "MicromambaResolveRunner", + "conda_lock_document", + "conda_lock_pip_requirements", + "explicit_lock_packages", + "is_conda_lock", + "merge_conda_pip", + "parse_conda_environment", + "parse_conda_failure", + "resolve_conda_environment", +] + +#: What a conda lock is, as the version records it: a conda **explicit** file, +#: the ``@EXPLICIT`` form ``micromamba create --file`` installs without solving. +CONDA_LOCK_FORMAT = "conda-explicit" + +#: The platform a Phase-2 artifact is built for (D-9): every variant is +#: ``linux/amd64``, so the solve is for ``linux-64`` in conda's own naming. +CONDA_PLATFORM = "linux-64" + +#: The marker the ``@EXPLICIT`` conda lock body opens with, and the key the +#: ``environment.yml`` names its pip layer under. +_EXPLICIT_MARKER = "@EXPLICIT" +_PIP_SECTION_KEY = "pip" + +#: How the whole pip layer is recorded in the lock's header — the user's pip +#: requirements *and* the protected pins Datalayer forces over them, pinned to +#: the versions the solve resolved — one ``# datalayer-pip: `` line each, +#: above the ``@EXPLICIT`` body. A builder installs the conda layer from the +#: body and then this pip layer, so the whole of what a version resolved to is +#: in the one document and nothing resolved in the solve is lost from the build. +CONDA_PIP_PREFIX = "# datalayer-pip: " + +#: The ``micromamba`` the conda solve and every conda build use, pinned so the +#: tool that resolves is the tool that installs (E3-02): the approved base bakes +#: uv, the wheelhouse and the doctor, but not micromamba, so it is brought in +#: here rather than assumed. A BuildKit build copies the binary from this image; +#: a builder driving an SDK installs the same pinned release. +MICROMAMBA_VERSION = "2.0.5" +MICROMAMBA_IMAGE = f"mambaorg/micromamba:{MICROMAMBA_VERSION}" +MICROMAMBA_BINARY = "/usr/local/bin/micromamba" + +#: A channel URL that carries a credential in its userinfo — the same shape +#: :func:`code_sandboxes.environments.spec._index_findings` refuses in an index +#: URL, so a token is caught the same way whichever field names it. +_URL_CREDENTIALS = re.compile(r"^[a-z][a-z0-9+.-]*://[^/@\s]+:[^/@\s]*@", re.IGNORECASE) + + +def micromamba_bootstrap_dockerfile_line() -> str: + """The Dockerfile line that brings the pinned micromamba into a build. + + ``COPY --from`` the pinned micromamba image, so the binary is present and + reproducible without a network fetch inside the build itself. Used by the + resolver's own solve image and by the Datalayer (BuildKit) builder. + """ + return f"COPY --from={MICROMAMBA_IMAGE} /bin/micromamba {MICROMAMBA_BINARY}" + + +def micromamba_bootstrap_command() -> str: + """The shell command that installs the pinned micromamba into a build. + + For a builder that drives an SDK (E2B, Daytona) rather than emitting a + Dockerfile: the same pinned release ``COPY --from`` brings, fetched into + ``/usr/local/bin`` so a later ``micromamba install`` finds it on the PATH. + """ + return ( + f"curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/{MICROMAMBA_VERSION} " + "| tar -xj -C /usr/local/bin --strip-components=1 bin/micromamba" + ) + + +#: Where an approved Datalayer base keeps its interpreter: `python` on the +#: PATH is `/opt/conda/bin/python` (E1-05), and that is the environment a +#: sandbox's kernel runs in. +CONDA_PREFIX = "/opt/conda" + + +def micromamba_install_command(lock_path: str, *, micromamba: str = "micromamba") -> str: + """The command that installs an explicit lock into the base's own interpreter. + + **Into `/opt/conda`, by name.** The base sets no `MAMBA_ROOT_PREFIX`, and + left to itself `micromamba install --name base` makes a *new* `base` under + `~/.local/share/mamba` — for a Datalayer base, inside the very home the + runtime mounts a person's content over, and nowhere `python` looks. The + first real conda build (2026-09-18) installed all 81 packages there, and + then failed its own `postInstall` with `No module named 'osgeo'`. Both the + root and the target prefix are named, so nothing depends on which user the + step happens to run as. + """ + return ( + f"{micromamba} install --yes --root-prefix {CONDA_PREFIX} " + f"--prefix {CONDA_PREFIX} --file {lock_path}" + ) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +# -- Reading the environment.yml ---------------------------------------------- + + +@dataclass(frozen=True) +class CondaEnvironment: + """An ``environment.yml``, as the resolver reads it.""" + + name: str + channels: tuple[str, ...] + conda_dependencies: tuple[str, ...] + pip_dependencies: tuple[str, ...] + + +def parse_conda_environment(text: str) -> CondaEnvironment: + """One ``environment.yml``'s channels, conda packages and pip packages. + + A conda ``dependencies`` list mixes plain conda specs with a single + ``{"pip": [...]}`` mapping for the pip layer; both are separated here so the + protected pins merge over the pip layer alone (:func:`merge_conda_pip`) and + the conda layer is passed through untouched. Anything that is not a conda + spec or the one pip mapping — a nested list, a bare number — is refused with + its position, because a solve is not the place to discover a malformed file. + """ + import yaml + + try: + document = yaml.safe_load(text) + except yaml.YAMLError as error: + raise EnvironmentsError( + SPEC_INVALID, + f"the environment.yml is not valid YAML: {error}", + detail={"field": "spec.build.dependencyFile.content"}, + ) from error + if not isinstance(document, Mapping): + raise EnvironmentsError( + SPEC_INVALID, + "the environment.yml is empty or not a mapping", + detail={"field": "spec.build.dependencyFile.content"}, + ) + raw_dependencies = document.get("dependencies") + if raw_dependencies is None: + raise EnvironmentsError( + SPEC_INVALID, + "the environment.yml names no `dependencies`", + detail={"field": "spec.build.dependencyFile.content"}, + ) + if not isinstance(raw_dependencies, Sequence) or isinstance(raw_dependencies, (str, bytes)): + raise EnvironmentsError( + SPEC_INVALID, + "`dependencies` in the environment.yml is not a list", + detail={"field": "spec.build.dependencyFile.content.dependencies"}, + ) + conda: list[str] = [] + pip: list[str] = [] + seen_pip = False + for index, entry in enumerate(raw_dependencies): + field_name = f"spec.build.dependencyFile.content.dependencies[{index}]" + if isinstance(entry, str): + spec = entry.strip() + if spec: + conda.append(spec) + continue + if isinstance(entry, Mapping) and set(entry) == {_PIP_SECTION_KEY}: + if seen_pip: + raise EnvironmentsError( + SPEC_INVALID, + "the environment.yml names more than one `pip:` section", + detail={"field": field_name}, + ) + seen_pip = True + pip.extend(_pip_requirements(entry[_PIP_SECTION_KEY], field_name)) + continue + raise EnvironmentsError( + SPEC_INVALID, + "a `dependencies` entry in the environment.yml is neither a conda " + "spec nor a single `pip:` section", + detail={"field": field_name}, + ) + channels = _channels(document) + return CondaEnvironment( + name=str(document.get("name") or "environment"), + channels=channels, + conda_dependencies=tuple(conda), + pip_dependencies=tuple(pip), + ) + + +def _pip_requirements(entries: Any, field_name: str) -> list[str]: + """The strings of a ``pip:`` section, or a refusal naming its position.""" + if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)): + raise EnvironmentsError( + SPEC_INVALID, + "the `pip:` section in the environment.yml is not a list", + detail={"field": f"{field_name}.pip"}, + ) + requirements: list[str] = [] + for pip_index, requirement in enumerate(entries): + if not isinstance(requirement, str): + raise EnvironmentsError( + SPEC_INVALID, + "a `pip:` entry in the environment.yml is not a string", + detail={"field": f"{field_name}.pip[{pip_index}]"}, + ) + spec = requirement.strip() + if spec: + requirements.append(spec) + return requirements + + +def _channels(document: Mapping[str, Any]) -> tuple[str, ...]: + raw = document.get("channels") + if raw is None: + return () + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)): + raise EnvironmentsError( + SPEC_INVALID, + "`channels` in the environment.yml is not a list", + detail={"field": "spec.build.dependencyFile.content.channels"}, + ) + channels: list[str] = [] + for index, channel in enumerate(raw): + if not isinstance(channel, str): + raise EnvironmentsError( + SPEC_INVALID, + "a `channels` entry in the environment.yml is not a string", + detail={"field": f"spec.build.dependencyFile.content.channels[{index}]"}, + ) + text = channel.strip() + if text: + if _URL_CREDENTIALS.match(text): + # The same refusal `spec.packages.python.indexes` gives a + # credential-bearing index URL: a token in the channel is + # copied into the solve context and can reappear in the + # explicit lock, so it belongs in the build's secrets, not here. + raise EnvironmentsError( + SPEC_INVALID, + "a `channels` entry carries a credential in its URL; reference the " + "credential in `buildSecrets`", + detail={"field": f"spec.build.dependencyFile.content.channels[{index}]"}, + ) + channels.append(text) + return tuple(channels) + + +def merge_conda_pip( + environment: CondaEnvironment, + pins: Sequence[ProtectedPin] | None = None, +) -> MergedRequirements: + """Datalayer's protected pins merged over the environment's ``pip:`` layer. + + The conda layer is Datalayer's to leave alone — the kernel stack is a pip + concern — but the pip layer is merged exactly as a ``packages`` source's + dependencies are (:func:`merge_requirements`): a pip requirement that + contradicts a protected pin is refused, and every pin is forced in whether + or not the environment named it. The one that has no index — the + jupyter-server fork wheel (E1-04) — is satisfied from the wheelhouse the + same ``--find-links`` reaches in the pip step below. + """ + return merge_requirements(environment.pip_dependencies, pins=pins) + + +def rendered_environment( + environment: CondaEnvironment, + merged: MergedRequirements, + *, + python_version: str, +) -> str: + """The ``environment.yml`` the solve is actually given. + + The user's conda packages and channels, with two things settled: the + interpreter is pinned to the base's ``python`` (D-9, never silently + replaced), and the ``pip:`` section is the merged one — the user's pip + requirements with the protected pins forced over them. + """ + import yaml + + conda_without_python = [ + spec for spec in environment.conda_dependencies if _conda_package_name(spec) != "python" + ] + dependencies: list[Any] = [f"python={python_version}", *conda_without_python] + pip_layer = list(merged.requirements) + if pip_layer: + dependencies.append({_PIP_SECTION_KEY: pip_layer}) + document = { + "name": environment.name, + "channels": list(environment.channels) or ["conda-forge"], + "dependencies": dependencies, + } + return yaml.safe_dump(document, sort_keys=False, default_flow_style=False) + + +def _conda_package_name(spec: str) -> str: + """The package a conda spec names, lowercased: ``python`` from + ``python=3.13``, ``python >=3.11`` or ``python[version='3.13']``.""" + return re.split(r"[\s=<>!~\[]", spec.strip(), maxsplit=1)[0].strip().lower() + + +# -- What a runner is asked, and what it answers ------------------------------ + + +@dataclass(frozen=True) +class CondaResolveRequest: + """One conda solve: what to resolve, for which interpreter, from where.""" + + environment_yml: str + """The rendered ``environment.yml`` — interpreter pinned, pins merged.""" + + python_version: str + platform: str = CONDA_PLATFORM + base_reference: str = "" + """The base the solve runs inside, pinned by digest (D-9).""" + + registry_auth: Mapping[str, str] | None = None + """What the registry needs to be read, when the runner pulls the base.""" + + +@dataclass +class CondaResolveOutcome: + """A conda solve's answer: the explicit lock, and the pip layer it resolved. + + ``lock_text`` is the ``@EXPLICIT`` conda lock, verbatim from ``micromamba``. + ``pip_lock`` is the pip layer the same solve installed — the user's pip + requirements and the protected pins forced over them — pinned to the + versions it resolved, read back from ``micromamba env export`` so the + artifact carries the whole of what the solve produced, not the conda layer + alone (E3-02). A runner that cannot read the prefix back leaves it empty, + and :func:`conda_lock_document` falls back to the merged requirements. + """ + + lock_text: str + pip_lock: tuple[str, ...] = () + + +def pip_requirements_from_env_yaml(text: str) -> tuple[str, ...]: + """The pip layer a ``micromamba env export`` names, pinned, in order. + + A conda ``env export`` (the YAML form, not ``--explicit``) lists the pip + packages it installed under a single ``{"pip": [...]}`` entry of its + ``dependencies``, each ``name==version`` — cleanly separated from the conda + packages, which are their own strings. This reads that section back, so the + solve's resolved pip versions become the lock's pip layer. A malformed or + pip-less export is an empty layer, never a raised error: the explicit lock + is what a solve is judged by, and its own marker is checked elsewhere. + """ + import yaml + + try: + document = yaml.safe_load(text) + except yaml.YAMLError: + return () + if not isinstance(document, Mapping): + return () + dependencies = document.get("dependencies") + if not isinstance(dependencies, Sequence) or isinstance(dependencies, (str, bytes)): + return () + requirements: list[str] = [] + for entry in dependencies: + if isinstance(entry, Mapping) and _PIP_SECTION_KEY in entry: + for requirement in entry[_PIP_SECTION_KEY] or []: + if isinstance(requirement, str) and requirement.strip(): + requirements.append(requirement.strip()) + return tuple(requirements) + + +#: What the solved prefix's own interpreter is asked, to list the pip layer +#: whole. Standard library only, and it runs inside the prefix. +#: +#: ``micromamba env export`` names the pip packages the *file* asked for and +#: nothing they pulled in: the first real solve (2026-09-18) exported 7 pip +#: pins where pip had installed several dozen distributions, so every build +#: would have resolved ``tornado``, ``pyzmq``, ``traitlets`` and the rest +#: afresh — on each variant, on whichever day it ran — under a lock that +#: claimed to say the whole of what a build installs. A distribution records +#: who installed it in its own ``INSTALLER`` file, which is what tells the pip +#: layer from the conda packages that also carry Python metadata. +PIP_LAYER_SCRIPT = """\ +import importlib.metadata as metadata + +layer = {} +for distribution in metadata.distributions(): + installer = (distribution.read_text("INSTALLER") or "").strip().lower() + name = distribution.metadata["Name"] + if installer in ("pip", "uv") and name: + layer[name.lower().replace("_", "-")] = distribution.version +for name in sorted(layer): + print(f"{name}=={layer[name]}") +""" + + +def pip_requirements_from_listing(text: str) -> tuple[str, ...]: + """The pip layer :data:`PIP_LAYER_SCRIPT` printed: ``name==version`` lines. + + Anything else on a line — a warning an interpreter wrote to the same + stream — is dropped rather than installed. + """ + requirements: list[str] = [] + for line in text.splitlines(): + entry = line.strip() + name, separator, version = entry.partition("==") + if separator and name and version and " " not in entry: + requirements.append(entry) + return tuple(requirements) + + +class CondaResolveRunner(Protocol): + """Where a conda solve runs.""" + + name: str + + def solve( + self, request: CondaResolveRequest, log: Callable[[str], None] | None = None + ) -> CondaResolveOutcome: ... + + +# -- Reading micromamba's refusals -------------------------------------------- + +_NOTHING_PROVIDES = re.compile( + r"nothing provides (?:requested )?(?P[A-Za-z0-9][A-Za-z0-9._-]*)" +) +_PACKAGE_NOT_FOUND = re.compile( + r"(?:package|libmamba).*?(?P[A-Za-z0-9][A-Za-z0-9._-]*) is not available" +) + + +def parse_conda_failure(output: str) -> EnvironmentsError: + """``micromamba``'s refusal as one of section 10's codes. + + A package no channel serves, and an unsatisfiable set of packages, are the + version's own to fix and are reported as such. Anything else — the channel + unreachable, ``micromamba`` missing — is ``DL_ENV_PROVIDER_ERROR``, which is + retryable, because nothing about the version is wrong. + """ + text = " ".join(line.strip() for line in output.splitlines() if line.strip()) + detail: dict[str, Any] = {"output": text[:2000]} + + provides = _NOTHING_PROVIDES.search(text) + if provides: + name = provides.group("name") + return EnvironmentsError( + PACKAGE_NOT_FOUND, + f"`{name}` is not in any channel this environment may read", + detail={**detail, "package": name}, + ) + unavailable = _PACKAGE_NOT_FOUND.search(text) + if unavailable: + name = unavailable.group("name") + return EnvironmentsError( + PACKAGE_NOT_FOUND, + f"`{name}` is not in any channel this environment may read", + detail={**detail, "package": name}, + ) + # A channel the build pool may not reach: solving again is refused again, + # so it is said with the host rather than retried (E2-19). + refused = egress_refused_hosts(output) + if refused: + return EnvironmentsError( + PACKAGE_NOT_FOUND, + f"The build pool's egress proxy refused {egress_hosts_text(refused)}: a version " + "solves only against the channels the build pool allows. Use one of them, or " + "ask for the host to be allowed", + detail={**detail, "refused_hosts": refused, "findings": egress_findings(refused)}, + retryable=False, + ) + lowered = text.lower() + if ( + "could not solve" in lowered + or "unsolvable" in lowered + or "encountered problems while solving" in lowered + or "no solution" in lowered + ): + return EnvironmentsError( + RESOLVE_CONFLICT, + "The conda dependencies cannot be satisfied together: " + text[:400], + detail=detail, + ) + return EnvironmentsError( + PROVIDER_ERROR, + "The conda resolver did not finish: " + text[:400], + detail=detail, + ) + + +# -- Running the solve -------------------------------------------------------- + + +class MicromambaResolveRunner: + """``micromamba`` where this runs: for ``plane local`` and for tests. + + Creates the environment the ``environment.yml`` asks for into a scratch + prefix and exports it as an explicit lock. The prefix is thrown away; only + the lock is kept, which is the one thing every variant then builds from. + """ + + name = "micromamba" + + def __init__(self, micromamba: str | None = None, timeout: float = 900.0) -> None: + # `None` means "find it"; an empty string means "there is none", which + # is how a test says so without hiding micromamba from the process. + self._micromamba = (shutil.which("micromamba") or "") if micromamba is None else micromamba + self._timeout = timeout + + def solve( + self, request: CondaResolveRequest, log: Callable[[str], None] | None = None + ) -> CondaResolveOutcome: + say = log or (lambda _line: None) + if not self._micromamba: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "No `micromamba` to resolve with: the local conda resolver needs it on the PATH", + detail={"missing": "micromamba", "runner": self.name}, + ) + with tempfile.TemporaryDirectory(prefix="dl-conda-") as directory: + root = Path(directory) + spec_file = root / "environment.yml" + spec_file.write_text(request.environment_yml, encoding="utf-8") + prefix = root / "prefix" + create = [ + self._micromamba, + "create", + "--yes", + "--prefix", + str(prefix), + "--platform", + request.platform, + "--file", + str(spec_file), + ] + say(f"Resolving the conda environment for {request.platform}") + try: + created = subprocess.run( # noqa: S603 - the argv is built here + create, + capture_output=True, + text=True, + timeout=self._timeout, + check=False, + env={**_os_environ(), "PIP_FIND_LINKS": str(WHEELHOUSE_PATH)}, + ) + except subprocess.TimeoutExpired as expired: + raise EnvironmentsError( + PROVIDER_ERROR, + f"The conda solve did not finish within {self._timeout:.0f}s", + detail={"runner": self.name, "timeout": self._timeout}, + ) from expired + if created.returncode != 0: + for line in (created.stderr or "").splitlines(): + say(line) + raise parse_conda_failure(created.stderr or created.stdout or "") + export = self._export( + [self._micromamba, "env", "export", "--explicit", "--prefix", str(prefix)], + say, + ) + # The pip layer the same solve installed, pinned, read back from the + # YAML export's own `pip:` section (E3-02): the explicit export above + # carries conda packages alone, so without this the user's resolved + # pip requirements would be absent from the artifact. + pip_export = self._export( + [self._micromamba, "env", "export", "--prefix", str(prefix)], + say, + ) + # Every distribution pip installed, the transitive ones included: + # the export above names only what the file asked for. + listing = self._export( + [str(prefix / "bin" / "python"), "-c", PIP_LAYER_SCRIPT], + say, + ) + return CondaResolveOutcome( + lock_text=export.stdout, + pip_lock=pip_requirements_from_listing(listing.stdout) + or pip_requirements_from_env_yaml(pip_export.stdout), + ) + + def _export( + self, argv: list[str], say: Callable[[str], None] + ) -> subprocess.CompletedProcess[str]: + """One ``micromamba env export``, its timeout handled the same as the solve. + + The ``create`` above and both exports share the one refusal so a timeout + anywhere becomes ``DL_ENV_PROVIDER_ERROR`` rather than a raw + :class:`subprocess.TimeoutExpired` a caller cannot classify or retry. + """ + try: + result = subprocess.run( # noqa: S603 - the argv is built here + argv, + capture_output=True, + text=True, + timeout=self._timeout, + check=False, + ) + except subprocess.TimeoutExpired as expired: + raise EnvironmentsError( + PROVIDER_ERROR, + f"The conda solve did not finish within {self._timeout:.0f}s", + detail={"runner": self.name, "timeout": self._timeout}, + ) from expired + if result.returncode != 0: + for line in (result.stderr or "").splitlines(): + say(line) + raise parse_conda_failure(result.stderr or result.stdout or "") + return result + + +class BuildkitCondaResolveRunner: + """D-9's conda solve: ``micromamba`` inside the resolved base, under BuildKit. + + The solve gets the builder's isolation, its egress allowlist and the exact + interpreter the artifact will have. It needs ``buildctl`` and a reachable + ``buildkitd``, neither of which exists until the build pool is deployed + (E1-06), so until then it refuses by name rather than resolving elsewhere. + """ + + name = "buildkit-conda" + + def __init__( + self, + buildctl: str | None = None, + address: str | None = None, + tlscert: str | None = None, + tlskey: str | None = None, + tlscacert: str | None = None, + timeout: float = 1200.0, + proxy: str | None = None, + ) -> None: + self._buildctl = (shutil.which("buildctl") or "") if buildctl is None else buildctl + #: The build pool's egress proxy (E1-06), or `DATALAYER_BUILDKIT_PROXY`. + self._proxy = buildkit_proxy(proxy) + self._tlscert = tlscert or "" + self._tlskey = tlskey or "" + self._tlscacert = tlscacert or "" + self._address = address or "" + self._timeout = timeout + + def _tls_options(self) -> list[str]: + if not (self._tlscert and self._tlskey and self._tlscacert): + return [] + return [ + f"--tlscert={self._tlscert}", + f"--tlskey={self._tlskey}", + f"--tlscacert={self._tlscacert}", + ] + + def dockerfile(self, request: CondaResolveRequest) -> str: + """The solve, as the frontend reads it: create the env, then export it. + + The ``environment.yml`` is a spec field a user wrote, so its path is the + only thing that reaches the ``RUN`` line — its content is a file copied + into the context, never interpolated into a shell command — and the + wheelhouse is brought along for the one protected pin no index has + (E1-04), reached through ``PIP_FIND_LINKS`` the same way the local + runner reaches it. ``micromamba`` is copied in from its pinned image + first, because the approved base bakes uv and the wheelhouse but not it. + The explicit lock and the pip layer are both exported, so the artifact + carries the whole of what the solve resolved, not the conda layer alone. + """ + micromamba = shlex.quote(MICROMAMBA_BINARY) + return ( + "\n".join( + [ + f"FROM {request.base_reference} AS solve", + "USER root", + "WORKDIR /solve", + micromamba_bootstrap_dockerfile_line(), + "COPY environment.yml ./environment.yml", + "ENV PIP_FIND_LINKS=" + shlex.quote(WHEELHOUSE_IMAGE_PATH), + "RUN --mount=type=cache,target=/opt/conda/pkgs " + f"{micromamba} create --yes --prefix /solve/prefix " + f"--platform {shlex.quote(request.platform)} --file environment.yml", + f"RUN {micromamba} env export --explicit " + "--prefix /solve/prefix > /solve/lock.txt", + f"RUN {micromamba} env export --prefix /solve/prefix > /solve/pip-env.yml", + # The pip layer whole, from the prefix's own interpreter: + # a file of this package's, never a spec field. + "COPY pip_layer.py ./pip_layer.py", + "RUN /solve/prefix/bin/python pip_layer.py > /solve/pip-lock.txt", + "FROM scratch", + "COPY --from=solve /solve/lock.txt /lock.txt", + "COPY --from=solve /solve/pip-env.yml /pip-env.yml", + "COPY --from=solve /solve/pip-lock.txt /pip-lock.txt", + ] + ) + + "\n" + ) + + def solve( + self, request: CondaResolveRequest, log: Callable[[str], None] | None = None + ) -> CondaResolveOutcome: + say = log or (lambda _line: None) + if not self._buildctl: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "No `buildctl` to resolve with: the build pool is not deployed here", + detail={"missing": "buildctl", "runner": self.name, "item": "E1-06"}, + ) + if "@sha256:" not in request.base_reference: + raise EnvironmentsError( + SPEC_INVALID, + "The base is resolved to a digest before anything is solved", + detail={"base": request.base_reference}, + ) + with tempfile.TemporaryDirectory(prefix="dl-conda-solve-") as directory: + root = Path(directory) + (root / "environment.yml").write_text(request.environment_yml, encoding="utf-8") + (root / "pip_layer.py").write_text(PIP_LAYER_SCRIPT, encoding="utf-8") + (root / "Dockerfile").write_text(self.dockerfile(request), encoding="utf-8") + out = root / "out" + command = [ + self._buildctl, + *(["--addr", self._address] if self._address else []), + *self._tls_options(), + "build", + "--frontend", + "dockerfile.v0", + "--local", + f"context={root}", + "--local", + f"dockerfile={root}", + "--output", + f"type=local,dest={out}", + *buildkit_proxy_options(self._proxy), + ] + say(f"Solving the conda lock in {request.base_reference}") + try: + finished = subprocess.run( # noqa: S603 - the argv is built here + command, + capture_output=True, + text=True, + timeout=self._timeout, + check=False, + env=self._environment(request), + ) + except subprocess.TimeoutExpired as expired: + raise EnvironmentsError( + PROVIDER_ERROR, + f"The conda solve did not finish within {self._timeout:.0f}s", + detail={"runner": self.name, "timeout": self._timeout}, + ) from expired + for line in (finished.stderr or "").splitlines(): + say(line) + if finished.returncode != 0: + raise parse_conda_failure(finished.stderr or finished.stdout or "") + lock = (out / "lock.txt").read_text(encoding="utf-8") + pip_env = out / "pip-env.yml" + pip_listing = out / "pip-lock.txt" + pip_lock = ( + pip_requirements_from_listing(pip_listing.read_text(encoding="utf-8")) + if pip_listing.exists() + else () + ) or ( + pip_requirements_from_env_yaml(pip_env.read_text(encoding="utf-8")) + if pip_env.exists() + else () + ) + return CondaResolveOutcome(lock_text=lock, pip_lock=pip_lock) + + def _environment(self, request: CondaResolveRequest) -> dict[str, str] | None: + auth = dict(request.registry_auth or {}) + if not auth: + return None + return {**_os_environ(), **{str(key): str(value) for key, value in auth.items()}} + + +def _os_environ() -> dict[str, str]: + import os + + return dict(os.environ) + + +# -- The lock ----------------------------------------------------------------- + + +def explicit_lock_packages(lock_text: str) -> list[str]: + """Every package an explicit lock installs, one URL per line. + + An ``@EXPLICIT`` file is comments, the ``@EXPLICIT`` marker, and then one + ``https://…/pkg.conda#hash`` line per package; the URLs are what a build + installs and what this counts. + """ + packages: list[str] = [] + for raw in lock_text.splitlines(): + line = raw.strip() + if not line or line.startswith("#") or line == _EXPLICIT_MARKER: + continue + packages.append(line) + return packages + + +#: `name-version-build.conda` (or `.tar.bz2`), as the last segment of an +#: explicit lock's URL. A conda name may itself contain `-`, so the version and +#: the build are the last two dash-separated fields, never the first two. +_EXPLICIT_PACKAGE = re.compile( + r"/(?P[^/]+)-(?P[^-/]+)-(?P[^-/]+)\.(?:conda|tar\.bz2)(?:#.*)?$" +) + + +def conda_lock_python_packages(lock_text: str) -> dict[str, str]: + """The Python distributions an explicit lock installs, by name, with their versions. + + Only the ones a build string marks as Python packages — `py313h…` for a + compiled one, `pyh…`/`pyhd8ed…` for a noarch one. A conda lock is mostly + libraries with no Python metadata at all (`libgdal-core`, `proj`, `openssl`), + and Appendix B check 5 asks the interpreter for a distribution's version: + handing it `proj` would fail a check with nothing wrong to report. + """ + found: dict[str, str] = {} + for url in explicit_lock_packages(lock_text): + match = _EXPLICIT_PACKAGE.search(url) + if match and match["build"].startswith("py"): + found[match["name"].lower()] = match["version"] + return found + + +def conda_expected_packages(environment_yml: str, lock_text: str) -> dict[str, str]: + """What an `environment.yml` names at its top level, pinned to what its lock resolved. + + Check 5's question, for a conda source (E3-02): the file's own `pip:` + requirements, and the conda packages it names that are Python + distributions. The interpreter is left out — check 3 asks about it, and + `python` is not a distribution the interpreter reports about itself. + Empty for a file that cannot be read: the spec's own validation refuses + one long before a build, and a smoke test is not where to say so again. + """ + from packaging.requirements import InvalidRequirement, Requirement + from packaging.utils import canonicalize_name + + try: + environment = parse_conda_environment(environment_yml) + except EnvironmentsError: + return {} + expected: dict[str, str] = {} + pythons = conda_lock_python_packages(lock_text) + for spec in environment.conda_dependencies: + name = _conda_package_name(spec) + if name != "python" and name in pythons: + expected[canonicalize_name(name)] = pythons[name] + pinned: dict[str, str] = {} + for requirement in conda_lock_pip_requirements(lock_text): + name, separator, version = requirement.partition("==") + if separator: + pinned[canonicalize_name(name.strip())] = version.strip() + for text in environment.pip_dependencies: + try: + name = canonicalize_name(Requirement(text).name) + except InvalidRequirement: + continue + if name in pinned: + expected[name] = pinned[name] + return expected + + +def is_conda_lock(lock_text: str | None) -> bool: + """Whether a lock is a conda explicit lock, and not the pip one. + + A conda lock carries the ``@EXPLICIT`` marker; a pip lock never does. A + builder reads this to install with ``micromamba`` rather than ``uv pip + sync`` — the one signal that travels with the lock text itself, so a + builder handed only :attr:`BuildRequest.lock_text` still knows which it is. + """ + if not lock_text: + return False + return any(line.strip() == _EXPLICIT_MARKER for line in lock_text.splitlines()) + + +def conda_lock_pip_requirements(lock_text: str) -> list[str]: + """The whole pip layer a conda lock's header records, in order. + + :func:`conda_lock_document` writes the pip layer the solve resolved as + ``# datalayer-pip: `` lines above the ``@EXPLICIT`` body: the user's + own pip requirements and the protected pins Datalayer forced over them, + pinned to the versions the solve produced. A builder installs the conda + layer from the body and then this pip layer, so the whole of what the + version resolved to is built and nothing the solve installed is lost (E3-02). + """ + prefix = CONDA_PIP_PREFIX.strip() + requirements: list[str] = [] + for raw in lock_text.splitlines(): + line = raw.strip() + if line.startswith(prefix): + requirement = line[len(prefix) :].strip() + if requirement: + requirements.append(requirement) + return requirements + + +def conda_lock_document( + outcome: CondaResolveOutcome, + *, + python_version: str, + base_reference: str, + merged: MergedRequirements, + platform: str = CONDA_PLATFORM, +) -> dict[str, Any]: + """The stored conda lock: its text, its digest, and what a reader needs. + + The pip layer the solve resolved is written as comments above the explicit + lock — the ``# datalayer-pip:`` lines a builder installs after the conda + packages — so the one document says the whole of what a build installs: the + conda packages by URL and hash, and the user's pip requirements with the + protected pins Datalayer forced over them, pinned to the versions the solve + produced. It stays a file ``micromamba create --file`` reads unchanged. + The pip layer is the solve's own (``outcome.pip_lock``) when the runner + could read the prefix back, and the merged requirements otherwise, so it is + always complete rather than the protected pins alone. + """ + header = [ + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.", + f"# python: {python_version}", + f"# platform: {platform}", + f"# base: {base_reference}", + ] + pip_layer = list(outcome.pip_lock) or list(merged.requirements) + for requirement in pip_layer: + header.append(f"{CONDA_PIP_PREFIX}{requirement}") + body = outcome.lock_text.lstrip("\n") + if _EXPLICIT_MARKER not in {line.strip() for line in body.splitlines()}: + raise EnvironmentsError( + PROVIDER_ERROR, + "micromamba did not produce an explicit lock (no @EXPLICIT marker)", + detail={"format": CONDA_LOCK_FORMAT}, + ) + text = "\n".join(header) + "\n" + body + if not text.endswith("\n"): + text += "\n" + packages = explicit_lock_packages(text) + return { + "digest": "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest(), + "format": CONDA_LOCK_FORMAT, + "content": text, + "python_version": python_version, + "package_count": len(packages), + } + + +def resolve_conda_environment( + *, + environment_yml: str, + python_version: str, + resolved_bases: Mapping[str, str], + platform: str = CONDA_PLATFORM, + credential: Any = None, + log: Callable[[str], None] | None = None, + runner: CondaResolveRunner | None = None, +) -> dict[str, Any]: + """A conda version's lock, from its ``environment.yml`` (E3-02). + + This is the conda seam of :func:`code_sandboxes.environments.resolve.resolve_environment`: + it takes the ``environment.yml`` a ``dependencyFile`` source carries and the + bases already resolved for the wanted variants, and answers the same lock + document every other source answers — ``digest``, ``format``, ``content``, + ``python_version``, ``package_count`` — so the workflow stores it the same way. + + Raises + ------ + EnvironmentsError + Everything a person can act on: a malformed ``environment.yml``, a pip + requirement that contradicts a protected pin, a conflict, or a package + no channel serves. + """ + say = log or (lambda _line: None) + environment = parse_conda_environment(environment_yml) + merged = merge_conda_pip(environment) + for note in merged.notes: + say(note) + rendered = rendered_environment(environment, merged, python_version=python_version) + solving_in = resolved_bases.get("datalayer") or next(iter(resolved_bases.values())) + request = CondaResolveRequest( + environment_yml=rendered, + python_version=python_version, + platform=platform, + base_reference=solving_in, + registry_auth=_registry_auth(credential), + ) + outcome = (runner or BuildkitCondaResolveRunner()).solve(request, say) + document = conda_lock_document( + outcome, + python_version=python_version, + base_reference=solving_in, + merged=merged, + platform=platform, + ) + say(f"Locked {document['package_count']} conda packages as {document['digest']}") + return {**document, "resolved_bases": dict(resolved_bases)} + + +def _registry_auth(credential: Any) -> Mapping[str, str] | None: + """The credential's registry auth, however it carries it — as + :func:`code_sandboxes.environments.resolve._registry_auth` reads it, kept + in step so both resolvers accept the one credential shape durable mints.""" + if credential is None: + return None + for attribute in ("registry_auth", "environment", "env"): + value = getattr(credential, attribute, None) + if callable(value): + value = value() + if isinstance(value, Mapping): + return {str(key): str(item) for key, item in value.items()} + return None diff --git a/code_sandboxes/environments/spec.py b/code_sandboxes/environments/spec.py index f0d8f3c..08277fe 100644 --- a/code_sandboxes/environments/spec.py +++ b/code_sandboxes/environments/spec.py @@ -49,12 +49,14 @@ parse_image_reference, ) from .lifecycle import VersionState +from .policy import UNRESTRICTED_POLICY, EnvironmentsPolicy __all__ = [ "API_VERSION", "BUILD_SOURCES", "GPU_SIZE_CLASSES", "KIND", + "PUBLIC_PACKAGE_INDEX_HOSTS", "SIZE_CLASSES", "SUPPORTED_BUILD_SOURCES", "VARIANTS", @@ -84,6 +86,7 @@ "VersionStatus", "assert_publishable", "command_names_secret", + "index_is_public", "parse_environment", "parse_requirements_txt", "publication_findings", @@ -105,7 +108,7 @@ BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "dockerfile", "image") #: What builds today; `dockerfile` is the one source still to come. -SUPPORTED_BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "image") +SUPPORTED_BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "dockerfile", "image") SUPPORTED_PACKAGE_MANAGERS: tuple[str, ...] = ("uv", "pip") #: `requirements.txt` and `pyproject.toml`/`uv.lock` are archived on the #: version they resolved (E3-01); this bounds what a spec may carry inline, @@ -167,6 +170,68 @@ class Platform(_Model): architecture: Literal["linux/amd64"] = "linux/amd64" +#: The package indexes D-12 counts as public: a version may be published only +#: when every index it resolves from is one of these, since a private index is +#: reached with a credential the public does not hold. Matched on host, so the +#: trailing `/simple` or its absence never decides it. `pypi.org` is the index; +#: `files.pythonhosted.org` is where its wheels are served from. +PUBLIC_PACKAGE_INDEX_HOSTS = frozenset({"pypi.org", "files.pythonhosted.org"}) + + +def _package_index_host(url: str) -> str: + """The host an index URL names, lower-cased and without its port, or `""`.""" + from urllib.parse import urlsplit + + try: + return (urlsplit(url).hostname or "").lower() + except ValueError: + return "" + + +def index_is_public(url: str) -> bool: + """Whether an index URL is one D-12 lets a published version resolve from.""" + return _package_index_host(url) in PUBLIC_PACKAGE_INDEX_HOSTS + + +#: The conda channels D-12 counts as public: a published conda version +#: (E3-02's ``dependencyFile``) may resolve only from these, since a private +#: channel is reached with a token no public reader holds — the same boundary +#: :data:`PUBLIC_PACKAGE_INDEX_HOSTS` draws for pip indexes. The bare names +#: anaconda.org serves openly, and the hosts a channel URL may name; any other +#: name or host is private and blocks publication. +PUBLIC_CONDA_CHANNELS = frozenset( + { + "conda-forge", + "bioconda", + "defaults", + "nodefaults", + "main", + "r", + "anaconda", + "pkgs/main", + "pkgs/r", + "msys2", + } +) +PUBLIC_CONDA_CHANNEL_HOSTS = frozenset({"conda.anaconda.org", "repo.anaconda.com", "anaconda.org"}) + + +def channel_is_public(channel: str) -> bool: + """Whether a conda channel is one D-12 lets a published version resolve from. + + A channel is a URL, whose host must be a public conda host, or a bare name, + which is public only when it is one of the well-known open channels — an + unlisted name (say a private org's) is treated as private, since a bare name + on anaconda.org may still need a token the public does not have. + """ + text = channel.strip() + if not text: + return True + if "://" in text: + return _package_index_host(text) in PUBLIC_CONDA_CHANNEL_HOSTS + return text.lower() in PUBLIC_CONDA_CHANNELS + + class PythonPackages(_Model): manager: Literal["uv", "pip", "conda"] = "uv" dependencies: list[str] = Field(default_factory=list) @@ -191,6 +256,21 @@ class FileEntry(_Model): sha256: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$") +class ContentsBuildEntry(_Model): + """One immutable file the Environment bakes from an external source. + + Unlike ``files`` — which a user uploads, referenced by ``contentRef`` — a + ``contentsBuild`` entry names an external ``source`` URL fetched at build + time and verified against ``sha256`` (required: the build verifies every + byte it bakes). Both are baked the same way, into every provider artifact. + """ + + source: str + path: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + size_bytes: int | None = Field(default=None, ge=0) + + class Commands(_Model): post_install: list[str] = Field(default_factory=list) @@ -242,22 +322,44 @@ class Compatibility(_Model): class DependencyFileSpec(_Model): - """A `requirements.txt`, or a `pyproject.toml` with its `uv.lock` (E3-01). + """A `requirements.txt`, a `pyproject.toml` with its `uv.lock`, or a conda + `environment.yml` (E3-01, E3-02). ``requirements`` resolves the way ``packages`` does — the protected constraints merged in, the same solve. ``pyproject`` does not resolve at all: its own ``uv.lock`` is verified against the current ``pyproject.toml`` and exported, never re-solved, because a lock the - author already made is the whole point of bringing one. + author already made is the whole point of bringing one. ``conda`` resolves + the ``environment.yml`` in its own ``micromamba`` solve into an explicit + lock, with the protected constraints merged over its ``pip:`` layer. """ - source_format: Literal["requirements", "pyproject"] = "requirements" - #: The `requirements.txt` text, or the `pyproject.toml` text. + source_format: Literal["requirements", "pyproject", "conda"] = "requirements" + #: The `requirements.txt`, `pyproject.toml` or conda `environment.yml` text. content: str = "" #: The `uv.lock` text. Required, and only meaningful, for `pyproject`. lock_content: str = "" +class DockerfileSpec(_Model): + """The Dockerfile a `dockerfile`-sourced version builds from (E3-03). + + Inline, the way ``DependencyFileSpec`` carries a ``requirements.txt``: a + file the author brings lives in the spec, so ``check_dockerfile`` and + every variant's capability report can read it **before** anything is + queued — which is the whole point of refusing an instruction a variant + does not implement at ``validate`` rather than halfway through a build. + + The build *context* — the extra files a ``COPY`` needs — is the separate + upload E3-03 describes, bounded and checked by ``validate_build_context``. + A Dockerfile with no context is the common case and needs no upload at + all. + """ + + #: The Dockerfile text. + content: str = "" + + class ImageSourceSpec(_Model): """An existing OCI image, imported as the build's base (E3-04). @@ -284,6 +386,7 @@ class ImageSourceSpec(_Model): class BuildSpec(_Model): source: Literal["packages", "dependencyFile", "dockerfile", "image"] = "packages" dependency_file: DependencyFileSpec | None = None + dockerfile: DockerfileSpec | None = None image: ImageSourceSpec | None = None @@ -294,6 +397,7 @@ class EnvironmentSpec(_Model): platform: Platform = Field(default_factory=Platform) packages: Packages = Field(default_factory=Packages) files: list[FileEntry] = Field(default_factory=list) + contents_build: list[ContentsBuildEntry] = Field(default_factory=list) env: dict[str, str] = Field(default_factory=dict) commands: Commands = Field(default_factory=Commands) build_secrets: list[BuildSecret] = Field(default_factory=list) @@ -362,6 +466,24 @@ def _requirement_problem(text: str) -> str | None: return None +def _requirement_name(text: str) -> str: + """The package name a dependency line asks for, or `""` when it cannot be read. + + `_requirement_problem` already refused a line this cannot parse; a + package-policy check that ran on the same line anyway would raise the + same problem a second time under a different finding, so this answers + empty rather than raising and lets the caller skip it. + """ + try: + from packaging.requirements import InvalidRequirement, Requirement + except ImportError: # pragma: no cover - packaging ships with pip and jupyter + return "" + try: + return Requirement(text).name + except InvalidRequirement: + return "" + + def _index_findings(field: str, url: str) -> list[SpecFinding]: if _URL_CREDENTIALS.match(url): return [ @@ -375,6 +497,12 @@ def _index_findings(field: str, url: str) -> list[SpecFinding]: return [] +def _same_cuda(asked: str, carried: str) -> bool: + """Whether a spec's CUDA names the base's: `12` and `12.8` both name 12.8.""" + asked_parts = asked.strip().split(".") + return carried.strip().split(".")[: len(asked_parts)] == asked_parts + + def parse_requirements_txt(text: str) -> list[str]: """The requirement lines of a `requirements.txt`, comments and blanks dropped. @@ -399,6 +527,40 @@ def parse_requirements_txt(text: str) -> list[str]: return lines +def _dockerfile_findings(dockerfile: DockerfileSpec | None) -> list[SpecFinding]: + """What a `dockerfile`-sourced version must carry, and what the contract refuses. + + The contract's own refusals are reported here, at `validate`, rather than + at build time: a `FROM` that is not an approved base, a privileged build, + the host network, a Docker socket mount, a host bind mount, and every + instruction §6 does not allow. Each is named with its line, because a + Dockerfile is somebody's file and "it was refused" is not a reason. + """ + field = "spec.build.dockerfile" + if dockerfile is None: + return [SpecFinding(field, "is required when `spec.build.source` is `dockerfile`")] + if not dockerfile.content.strip(): + return [SpecFinding(f"{field}.content", "is empty; it is the Dockerfile text")] + from .contract import dockerfile_findings_for_build, validate_dockerfile + + # The contract's refusals, then what building it needs on top (E3-03): + # one approved base named by its channel, and no build context yet. The + # second set is not the contract's — a variant could honour both — so it + # is its own list, and a line the contract already refused is not named + # twice. + findings = validate_dockerfile(dockerfile.content) + refused = {finding.line for finding in findings} + findings += [ + finding + for finding in dockerfile_findings_for_build(dockerfile.content) + if finding.line not in refused + ] + return [ + SpecFinding(f"{field}.content", f"line {finding.line}: {finding.message}") + for finding in sorted(findings, key=lambda finding: finding.line) + ] + + def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> list[SpecFinding]: field = "spec.build.dependencyFile" if dependency_file is None: @@ -409,7 +571,11 @@ def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> lis findings: list[SpecFinding] = [] if not dependency_file.content.strip(): name = ( - "pyproject.toml" if dependency_file.source_format == "pyproject" else "requirements.txt" + "pyproject.toml" + if dependency_file.source_format == "pyproject" + else "environment.yml" + if dependency_file.source_format == "conda" + else "requirements.txt" ) findings.append(SpecFinding(f"{field}.content", f"is empty; it is the {name} text")) elif dependency_file.source_format == "requirements": @@ -419,6 +585,8 @@ def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> lis findings.append( SpecFinding(f"{field}.content[{index}]", f"`{requirement}`: {problem}") ) + elif dependency_file.source_format == "conda": + findings.extend(_conda_environment_findings(dependency_file.content)) if len(dependency_file.content.encode("utf-8")) > MAX_DEPENDENCY_FILE_BYTES: findings.append( SpecFinding(f"{field}.content", f"is over {MAX_DEPENDENCY_FILE_BYTES} bytes") @@ -438,13 +606,39 @@ def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> lis findings.append( SpecFinding( f"{field}.lockContent", - "is only read for a `pyproject` source; a `requirements` source resolves fresh", + "is only read for a `pyproject` source; a `requirements` or `conda` source " + "resolves fresh", ) ) return findings -def _image_findings(image: ImageSourceSpec | None) -> list[SpecFinding]: +def _conda_environment_findings(content: str) -> list[SpecFinding]: + """An `environment.yml`'s own shape, before it reaches the conda solver (E3-02). + + The same validate-before-resolve rule every source follows: a malformed + `environment.yml` is the version's to fix, and refusing it here — with the + field the resolver would have named — is cheaper than a solve that fails on + it after taking a worker. + """ + from .resolve_conda import parse_conda_environment + + field = "spec.build.dependencyFile.content" + try: + parse_conda_environment(content) + except EnvironmentsError as error: + return [SpecFinding(error.detail.get("field", field), error.message, error.code)] + return [] + + +def _image_findings( + image: ImageSourceSpec | None, *, registries: tuple[str, ...] = DEFAULT_ALLOWED_REGISTRIES +) -> list[SpecFinding]: + """`registries` is the platform bootstrap, widened with an organization's + own private ones when its policy names any (E3-06) — never narrowed to + just the organization's list, since the public bootstrap stays reachable + whatever an organization's own policy adds, the same reading + `refuse_unless_allowed`'s own docstring already gives this list.""" field = "spec.build.image" if image is None: return [SpecFinding(field, "is required when `spec.build.source` is `image`")] @@ -454,12 +648,12 @@ def _image_findings(image: ImageSourceSpec | None) -> list[SpecFinding]: parsed = parse_image_reference(image.reference) except EnvironmentsError as error: return [SpecFinding(f"{field}.reference", error.message)] - if not image_registry_allowed(parsed) and not image.credential_secret_id: + if not image_registry_allowed(parsed, registries) and not image.credential_secret_id: return [ SpecFinding( f"{field}.reference", f"`{parsed.registry}` is not an allowed registry; allowed: " - + ", ".join(DEFAULT_ALLOWED_REGISTRIES) + + ", ".join(registries) + ", or reference a credential for a private one", POLICY_DENIED, ) @@ -467,8 +661,76 @@ def _image_findings(image: ImageSourceSpec | None) -> list[SpecFinding]: return [] +def _organization_findings( + environment: Environment, policy: EnvironmentsPolicy +) -> list[SpecFinding]: + """What an organization's own policy denies, beyond the platform's own + rules above (E3-06). A no-op field by field for a dimension the + organization has not narrowed — :data:`UNRESTRICTED_POLICY` produces no + findings here at all, which is every spec's reading until an + organization writes one. + + Bases and packages are checked against the organization's own list + alone: an organization can only ever *forbid* an approved base (the + platform decides which bases exist at all) and there is no platform + allowlist of packages to widen. Indexes and licences are the same, and + licences are checked at attestation instead, once the SBOM names which + package actually carries one (`policy.refuse_unlicensed`) — a spec names + no licences of its own to check here. Registries are the one dimension + `_image_findings` widens rather than narrows; see its own docstring. + """ + findings: list[SpecFinding] = [] + spec = environment.spec + if ( + spec.build.source not in ("image", "dockerfile") + and policy.allowed_bases is not None + and not policy.allows("bases", spec.base.ref) + ): + findings.append( + SpecFinding( + "spec.base.ref", + f"`{spec.base.ref}` is not an allowed base under this organization's " + "policy (the approved bases it allows: " + + (", ".join(policy.allowed_bases) or "(none)") + + ")", + POLICY_DENIED, + ) + ) + if policy.allowed_indexes is not None: + for index, url in enumerate(spec.packages.python.indexes): + if not policy.allows("indexes", url): + findings.append( + SpecFinding( + f"spec.packages.python.indexes[{index}]", + f"`{url}` is not an allowed index under this organization's policy " + "(the indexes it allows: " + + (", ".join(policy.allowed_indexes) or "(none)") + + ")", + POLICY_DENIED, + ) + ) + if policy.allowed_packages is not None: + for group_index, requirement in enumerate(spec.packages.python.dependencies): + name = _requirement_name(requirement) + if name and not policy.allows("packages", name): + findings.append( + SpecFinding( + f"spec.packages.python.dependencies[{group_index}]", + f"`{name}` is not an allowed package under this organization's " + "policy (the packages it allows: " + + (", ".join(policy.allowed_packages) or "(none)") + + ")", + POLICY_DENIED, + ) + ) + return findings + + def spec_findings( - environment: Environment, *, bases: Mapping[str, ApprovedBase] = APPROVED_BASES + environment: Environment, + *, + bases: Mapping[str, ApprovedBase] = APPROVED_BASES, + policy: EnvironmentsPolicy = UNRESTRICTED_POLICY, ) -> list[SpecFinding]: """Every rule of the specification the environment breaks.""" findings: list[SpecFinding] = [] @@ -497,10 +759,12 @@ def spec_findings( ) ) - # An `image` source brings its own base (E3-04): `spec.base` names - # nothing Datalayer approved, so checking it against the table would - # refuse every import for the one reason imports exist to avoid. - if spec.build.source != "image": + # An `image` source brings its own base (E3-04), and a `dockerfile` + # source's base is the `FROM` its uploaded Dockerfile names (E3-03, + # validated against the approved bases by `check_dockerfile`, not here): + # `spec.base` names nothing Datalayer approved for either, so checking it + # against the table would refuse every one for the reason they exist. + if spec.build.source not in ("image", "dockerfile"): base = bases.get(spec.base.ref) if base is None: findings.append( @@ -530,8 +794,15 @@ def spec_findings( ) if spec.build.source == "dependencyFile": findings.extend(_dependency_file_findings(spec.build.dependency_file)) + elif spec.build.source == "dockerfile": + findings.extend(_dockerfile_findings(spec.build.dockerfile)) elif spec.build.source == "image": - findings.extend(_image_findings(spec.build.image)) + registries = ( + DEFAULT_ALLOWED_REGISTRIES + policy.allowed_registries + if policy.allowed_registries + else DEFAULT_ALLOWED_REGISTRIES + ) + findings.extend(_image_findings(spec.build.image, registries=registries)) python = spec.packages.python if python.manager not in SUPPORTED_PACKAGE_MANAGERS: @@ -695,6 +966,18 @@ def spec_findings( "spec.base.ref", f"`{base.ref}` has no CUDA; a GPU size class needs a CUDA base" ) ) + # The CUDA a spec asks for is the base's toolkit, which a channel pins + # (E2-17): a version that asks for another would build and then fail + # check 11 in every sandbox it starts. + cuda = resources.accelerator.cuda if wants_gpu else None + if cuda and base is not None and base.cuda and not _same_cuda(cuda, base.cuda): + findings.append( + SpecFinding( + "spec.resources.accelerator.cuda", + f"`{base.ref}` carries CUDA {base.cuda}, not {cuda}; ask for {base.cuda} " + "or leave `cuda` out", + ) + ) variants = spec.compatibility.variants if not variants.required: @@ -730,6 +1013,7 @@ def spec_findings( f"spec.compatibility.regions[{index}]", f"`{region}` is not a region name" ) ) + findings.extend(_organization_findings(environment, policy)) return findings @@ -776,6 +1060,7 @@ def validate_environment( document: Mapping[str, Any] | str | Environment, *, bases: Mapping[str, ApprovedBase] = APPROVED_BASES, + policy: EnvironmentsPolicy = UNRESTRICTED_POLICY, ) -> Environment: """The Environment, or the error its findings amount to. @@ -783,11 +1068,18 @@ def validate_environment( field always outranks the rest, whatever else the spec also asks for. Failing that, the first finding's own code is what is raised: valid but unbuildable yet is ``DL_ENV_CAPABILITY_UNSUPPORTED``, an image off the - allowlist is ``DL_ENV_POLICY_DENIED`` (E3-04), and so on for whatever a - future rule adds. Either way every finding is listed. + allowlist is ``DL_ENV_POLICY_DENIED`` (E3-04), an index or a base or a + package an organization's own policy denies is the same code (E3-06), + and so on for whatever a future rule adds. Either way every finding is + listed. + + `policy` is the caller's own organization's environments policy, or + :data:`UNRESTRICTED_POLICY` for a personal owner, or an organization + that has not written this section — the reading every other MCP policy + rule gives an unset one. """ environment = parse_environment(document) - findings = spec_findings(environment, bases=bases) + findings = spec_findings(environment, bases=bases, policy=policy) if findings: invalid = [finding for finding in findings if finding.code is SPEC_INVALID] first = invalid[0] if invalid else findings[0] @@ -810,17 +1102,15 @@ def publication_findings(environment: Environment) -> list[SpecFinding]: D-12: *"A promoted version becomes public only by being published, and only when every input is public — public indexes, no ``files``, no - ``buildSecrets``, an approved base."* This function holds only the - ``buildSecrets`` half of that boundary — a build secret is IAM-held and - fetched for one build's own use, so it is never public by definition, - whether the version is otherwise made of nothing but public inputs or - not. The rest of D-12's boundary (public indexes, no baked ``files``, an - approved base) belongs to the publish route itself once it exists - (E2-15, not built yet as of this writing — there is no - ``services/library`` "environment" artifact type and no publish endpoint - in ``services/runtimes/datalayer_runtimes/services/environments.py`` to - call this from today). This is the seam that route calls when it lands, - named the way every other rule of the specification is. + ``buildSecrets``, an approved base."* This function holds the input half + of that boundary that the spec alone decides against a credential the + public does not hold: a build secret is IAM-held and so never public, and + a private package index is reached with a credential no public reader has. + The parts of D-12 that depend on a version's *status* rather than its + spec — a passing scan, a signed artifact, and that the datalayer variant's + base is an approved one — are the publish route's own to check against the + artifact it publishes (E2-15), since ``publication_findings`` is handed + the spec and nothing built from it. Deliberately never applied to `promote()` (the private, per-owner lifecycle step that makes a version an environment's active one): a @@ -828,17 +1118,66 @@ def publication_findings(environment: Environment) -> list[SpecFinding]: ever builds or launches it (D-12's own words). Only the act of making a version world-visible is refused. """ - if not environment.spec.build_secrets: - return [] - ids = ", ".join(secret.id for secret in environment.spec.build_secrets) - return [ - SpecFinding( - "spec.buildSecrets", - f"a version with a build secret ({ids}) can never be published to the " - "public Library (D-12); remove it, or keep the version private", - PUBLICATION_BLOCKED, + findings: list[SpecFinding] = [] + if environment.spec.build_secrets: + ids = ", ".join(secret.id for secret in environment.spec.build_secrets) + findings.append( + SpecFinding( + "spec.buildSecrets", + f"a version with a build secret ({ids}) can never be published to the " + "public Library (D-12); remove it, or keep the version private", + PUBLICATION_BLOCKED, + ) + ) + private = [url for url in environment.spec.packages.python.indexes if not index_is_public(url)] + if private: + findings.append( + SpecFinding( + "spec.packages.python.indexes", + f"a version that resolves from a private index ({', '.join(private)}) " + "can never be published to the public Library (D-12); publish only from " + f"public indexes ({', '.join(sorted(PUBLIC_PACKAGE_INDEX_HOSTS))})", + PUBLICATION_BLOCKED, + ) ) + private_channels = [ + channel for channel in _conda_channels(environment) if not channel_is_public(channel) ] + if private_channels: + findings.append( + SpecFinding( + "spec.build.dependencyFile.content.channels", + "a version that resolves from a private conda channel " + f"({', '.join(private_channels)}) can never be published to the public " + "Library (D-12); publish only from public channels " + f"({', '.join(sorted(PUBLIC_CONDA_CHANNELS))})", + PUBLICATION_BLOCKED, + ) + ) + return findings + + +def _conda_channels(environment: Environment) -> tuple[str, ...]: + """The channels a conda ``dependencyFile`` names, or none for any other source. + + A conda ``environment.yml``'s ``channels`` are package inputs the same as a + pip source's indexes, so publication weighs them the same (D-12). A file + that will not parse has no channels to weigh here — validation refuses it + before it is ever published — so a parse failure is an empty tuple, not a + raise. + """ + build = environment.spec.build + dependency_file = build.dependency_file + if build.source != "dependencyFile" or dependency_file is None: + return () + if dependency_file.source_format != "conda": + return () + from .resolve_conda import parse_conda_environment + + try: + return parse_conda_environment(dependency_file.content).channels + except EnvironmentsError: + return () def assert_publishable(environment: Environment) -> None: diff --git a/code_sandboxes/jupyter_server_sandbox.py b/code_sandboxes/jupyter_server_sandbox.py index cbb0aec..8465a98 100644 --- a/code_sandboxes/jupyter_server_sandbox.py +++ b/code_sandboxes/jupyter_server_sandbox.py @@ -607,6 +607,62 @@ def _do_interrupt(self) -> bool: logger.warning(f"Failed to interrupt Jupyter kernel: {e}") return False + def restart_kernel(self) -> bool: + """Restart the kernel itself, through the server's own REST API. + + Not `stop()` then `start()`: those are this *client's* lifecycle, and + when the server is somebody else's — a Datalayer runtime pod, which + is every attached sandbox — stopping the client only drops the + websocket. The kernel is a process on the server and keeps running, + so a reconnect lands back in the same interpreter with every global + still set. + + That is what Appendix B check 7 ("nothing is assumed to persist + across restarts") measures, and it read `state survived the restart` + for exactly this reason — found live on r1, 2026-09-16, the first + drill whose smoke test reached the check. `POST + /api/kernels/{id}/restart` is the one that restarts the kernel, the + same way `_do_interrupt` already uses the API rather than the + client's own lifecycle. + + Answers whether the server accepted it; never raises, so a caller + that cannot restart reports a failed check rather than an error. + """ + if not self._server_url or not self._client: + return False + kernel_id = getattr(self._client, "id", None) + if not kernel_id: + return False + try: + response = requests.post( + f"{self._server_url}/api/kernels/{kernel_id}/restart", + params={"token": self._token}, + headers=self._headers or None, + timeout=30, + ) + except Exception as error: + # A restart that cannot even be asked for is a failed check, not + # an error to raise at the caller, exactly as `_do_interrupt` is. + logger.warning(f"Failed to restart Jupyter kernel: {error}") + return False + if not response.ok: + logger.warning( + "Failed to restart Jupyter kernel: the server answered %s", response.status_code + ) + return False + # **The websocket is kept, never reconnected.** jupyter-server binds a + # kernel's websocket to the kernel id, not to its process, and moves + # it onto the restarted kernel itself — which is how JupyterLab's own + # restart works. Reconnecting here broke every restart instead: + # `jupyter_kernel_client` caches its shell and IOPub channels bound to + # the socket they were built with, so after `stop()` and `start()` the + # client held a new socket while its channels still wrote to the + # closed one. The next execution answered `ok` with no output — found + # on r1, 2026-09-18, as checks 7, 8 and 9 all failing together, and + # reproduced against the approved base itself, with the deployed + # client. Swallowing both calls' errors is what hid it. + return True + @marks_execution def run_code( # noqa: C901 self, diff --git a/code_sandboxes/manage.py b/code_sandboxes/manage.py index 2e1f96a..0570fce 100644 --- a/code_sandboxes/manage.py +++ b/code_sandboxes/manage.py @@ -42,7 +42,7 @@ from abc import ABC, abstractmethod from typing import Any -from .models import SandboxInfo, SandboxStatus, normalize_variant +from .models import SandboxEnvironment, SandboxInfo, SandboxStatus, normalize_variant __all__ = [ "SandboxManagementError", @@ -95,6 +95,69 @@ def _unsupported(self, verb: str, reason: str) -> SandboxManagementError: f"The {self.variant} variant does not support {verb}: {reason}" ) + def environments(self) -> list[SandboxEnvironment]: + """The environments this manager's provider ships, or none when it cannot say.""" + from .providers import get_provider + + provider = get_provider(self.variant) + return list(provider.environments()) if provider is not None else [] + + def _configure(self, kwargs: dict[str, Any]) -> SandboxEnvironment | None: + """Move `create`'s own options onto the sandbox's config; answer the environment. + + A `Sandbox` reads the environment and the name from its + `SandboxConfig`. Passed as bare keywords they land in its `**kwargs` + and are dropped there in silence, so `create` answers `running` for a + default sandbox under a generated name — and says nothing about + either. `DatalayerSandboxManager` learned this once, in the words of + its own `create`; every other manager still had the hole. Found on + 2026-09-17: `sandboxes create daytona -e eric/daytona-drift -n x` + launched `daytonaio/sandbox:0.8.0` with no name, and so did + `-e this-environment-does-not-exist-at-all`. + + The option is `environment`, spelled the way `SandboxConfig` spells + it, everywhere and by every caller. An environment the provider does + not ship is **refused**, naming what it does ship: a person who asks + for `daytona-gpu` and is handed a CPU sandbox has been told the wrong + thing, which is worse than being told no. + """ + from .models import SandboxConfig + + asked = kwargs.pop("environment", None) + name = kwargs.pop("name", None) + environment: SandboxEnvironment | None = None + updates: dict[str, Any] = {} + if name: + updates["name"] = name + if asked: + environment = self._shipped(str(asked)) + updates["environment"] = environment.name + # The card the environment names, which is what every adapter's + # own resource shaping reads. What each provider takes *besides* + # a card — Daytona's `spot`, for one — stays its own to apply. + if environment.gpu: + updates["gpu"] = environment.gpu + if updates: + config = kwargs.pop("config", None) or SandboxConfig() + kwargs["config"] = config.model_copy(update=updates) + return environment + + def _shipped(self, name: str) -> SandboxEnvironment: + """The shipped environment of that name, or a refusal naming the rest.""" + shipped = self.environments() + for environment in shipped: + if environment.name == name: + return environment + offered = ( + f"it ships {', '.join(sorted(item.name for item in shipped))}" + if shipped + else "it ships none that can be named — `sandboxes environments` " + "says what any provider offers" + ) + raise SandboxManagementError( + f"The {self.variant} variant ships no environment {name!r}: {offered}" + ) + class _EphemeralManager(SandboxManager): """The in-process variants: nothing outlives the interpreter. @@ -226,6 +289,7 @@ def create(self, **kwargs: Any) -> SandboxInfo: # auto_remove would erase the container the moment this process lets # go of it — the opposite of a detached create. + self._configure(kwargs) sandbox = DockerSandbox(auto_remove=False, **kwargs) sandbox.start() info = sandbox.info @@ -598,6 +662,7 @@ def update(self, sandbox_id: str, tags: dict[str, str] | None = None, **_: Any) def create(self, **kwargs: Any) -> SandboxInfo: from .modal_sandbox import ModalSandbox + self._configure(kwargs) sandbox = ModalSandbox(app_name=self._app_name, **kwargs) sandbox.start() info = sandbox.info @@ -721,6 +786,16 @@ def create(self, **kwargs: Any) -> SandboxInfo: from .daytona_sandbox import DaytonaSandbox given = {key: value for key, value in self._settings.items() if value} + environment = self._configure(kwargs) + # Preemptible capacity is Daytona's own argument, not a card, so the + # base helper cannot set it: `daytona-gpu-spot` differs from + # `daytona-gpu` by exactly this and by nothing the config carries. + if environment is not None: + metadata = environment.metadata or {} + if metadata.get("spot"): + kwargs.setdefault("spot", True) + if environment.gpu_count: + kwargs.setdefault("gpu_count", int(environment.gpu_count)) # Detached, so it outlives this call: stopping it is `delete`. sandbox = DaytonaSandbox(delete_on_stop=False, **given, **kwargs) sandbox.start() @@ -832,17 +907,13 @@ def create(self, **kwargs: Any) -> SandboxInfo: `SandboxConfig`. Passed as keywords, they fell into its extra arguments, so both CLIs started every runtime in `ai-agents-env` under a generated name: asked for `python-cpu-env`, the platform claimed an agents pod. - `environment_name` is the `datalayer sandboxes create` spelling, and - `environment` the `code-sandboxes create` one. """ from .datalayer_sandbox import DatalayerSandbox from .models import SandboxConfig config = kwargs.pop("config", None) or SandboxConfig() - environment_name = kwargs.pop("environment_name", None) - environment = kwargs.pop("environment", None) chosen = { - "environment": environment_name or environment, + "environment": kwargs.pop("environment", None), # The version of a user environment, read from the config by the # sandbox exactly as the environment is (PLAN_ENV.md, E1-19); left # in the extra arguments it would be dropped, and the CLI would @@ -947,6 +1018,7 @@ def delete(self, sandbox_id: str) -> bool: def create(self, **kwargs: Any) -> SandboxInfo: from .e2b_sandbox import E2BSandbox + self._configure(kwargs) sandbox = E2BSandbox(**self._opts(), **kwargs) sandbox.start() info = sandbox.info @@ -1044,6 +1116,7 @@ def create(self, **kwargs: Any) -> SandboxInfo: given = {key: value for key, value in self._settings.items() if value} # No session process: a sandbox nothing is holding open should not be # paying for a driver waiting on a stdin that will never be written. + self._configure(kwargs) sandbox = CoreWeaveSandbox(stateful=False, **given, **kwargs) sandbox.start() info = sandbox.info @@ -1123,6 +1196,7 @@ def create(self, **kwargs: Any) -> SandboxInfo: from .cloudflare_sandbox import CloudflareSandbox given = {key: value for key, value in self._settings.items() if value} + self._configure(kwargs) sandbox = CloudflareSandbox(**given, **kwargs) sandbox.start() info = sandbox.info diff --git a/code_sandboxes/modal_sandbox.py b/code_sandboxes/modal_sandbox.py index a65c683..f2cf697 100644 --- a/code_sandboxes/modal_sandbox.py +++ b/code_sandboxes/modal_sandbox.py @@ -72,7 +72,7 @@ _CONTRACT_UID = "1000" _CONTRACT_GID = "100" _CONTRACT_HOME = "/home/datalayer" -_CONTRACT_CONTENT_DIR = "/home/datalayer/content" +_CONTRACT_CONTENT_DIR = "/home/datalayer" def _resolve_modal_gpu(gpu_flavor: str, modal_module: Any) -> Any: @@ -215,6 +215,12 @@ class ModalSandbox(Sandbox): ``debian_slim`` image is used, optionally extended with ``pip_packages``. pip_packages: Optional list of pip packages to install in the default image. python_executable: Executable used to run snippets (default ``python``). + client: The ``modal.Client`` to act as. The app, the image and the + sandbox are all looked up and created with it, so a sandbox runs + in the workspace that client opens. With none, the SDK's own + ambient credentials decide — right for a person's own machine, + wrong for a worker acting for an owner (D-8), which passes the + owner's. """ def __init__( @@ -227,9 +233,11 @@ def __init__( python_version: str = DEFAULT_MODAL_PYTHON_VERSION, python_executable: str = "python", features: list[str] | None = None, + client: Any | None = None, **kwargs, ): super().__init__(config) + self._client = client self._app_name = app_name self._image = image #: A Modal image id, `im-…`: what an Environment build of this variant @@ -396,13 +404,16 @@ def start(self) -> None: "modal is required for ModalSandbox. Install it with: pip install modal" ) from exc - self._app = modal.App.lookup(self._app_name, create_if_missing=True) + # The client, when given, on every call that names a workspace: the + # app, the image and the sandbox are all the owner's or none are. + on_client = {"client": self._client} if self._client is not None else {} + self._app = modal.App.lookup(self._app_name, create_if_missing=True, **on_client) image = self._image if image is None and self._image_id: # `Image.from_id` in the Python SDK; `images.fromId` is the # JavaScript one, which an earlier note had here (correction 42). - image = modal.Image.from_id(self._image_id) + image = modal.Image.from_id(self._image_id, **on_client) if image is None: image = modal.Image.debian_slim(python_version=self._python_version) if self._pip_packages: @@ -416,6 +427,7 @@ def start(self) -> None: "app": self._app, "image": image, "timeout": int(self.config.max_lifetime), + **on_client, } if self.config.gpu: create_kwargs["gpu"] = _resolve_modal_gpu(self.config.gpu, modal) @@ -429,7 +441,16 @@ def start(self) -> None: for mount_path, volume_id in self._volume_mounts.requested.items() } - self._sandbox = modal.Sandbox.create(**create_kwargs) + # An Environments artifact's main process only holds the container: + # everything this class runs in it is an exec, Jupyter included + # (`prepare_jupyter_server`). Named here rather than left to the + # image, since an artifact built before code-sandboxes 1.9.36 carries + # its base's `start-jupyter.sh` as its CMD, which Modal runs under the + # entrypoint; that server exits within a minute and ends the sandbox + # (found live on 2026-09-18: a smoke test's restarted sandbox died + # between checks 8 and 9). + command = ("sleep", "infinity") if self._image_id else () + self._sandbox = modal.Sandbox.create(*command, **create_kwargs) self._volume_mounts.created() self._start_driver() diff --git a/docs/docs/environments/contract.mdx b/docs/docs/environments/contract.mdx index 1f6f43c..0365f65 100644 --- a/docs/docs/environments/contract.mdx +++ b/docs/docs/environments/contract.mdx @@ -15,12 +15,12 @@ This page is generated from `code_sandboxes/environments/contract.py`: change th |---|---|---|---| | Architecture | `linux/amd64`. | Modal pulls registry images for `linux/amd64` only; adopted everywhere for parity. | `doctor:architecture` | | Base OS | Debian-derived: Debian or Ubuntu. | E2B builds templates only from Debian-derived images; adopted everywhere for parity. | `doctor:os` | -| User | Non-root user `datalayer`, uid 1000, gid 100, home `/home/datalayer`, working directory `/home/datalayer/content`. | Runtime pods run as 1000:100 and home folders on the shared filesystem are owned that way. Modal ignores `USER`, so its adapter asserts the user in the run command. | `doctor:user`, `doctor:uid`, `doctor:gid`, `doctor:home`, `doctor:workdir`, `conformance:2` | +| User | Non-root user `datalayer`, uid 1000, gid 100, home `/home/datalayer`, working directory `/home/datalayer` — the home itself, which is where a person's folders are mounted and what the file browser shows, so `pwd` and the browser agree. | Runtime pods run as 1000:100 and home folders on the shared filesystem are owned that way. Modal ignores `USER`, so its adapter asserts the user in the run command. | `doctor:user`, `doctor:uid`, `doctor:gid`, `doctor:home`, `doctor:workdir`, `conformance:2` | | Python | `python3` and `pip` on `PATH`. | The kernel, the doctor and the package installers all need them. | `doctor:python`, `doctor:pip`, `conformance:3` | | Kernel stack | `ipykernel`, `jupyter_client` and the Datalayer runtime agent, at versions inside the range Datalayer's protected constraints allow. | A pin outside that range builds, then gives a sandbox that never connects. | `doctor:kernel`, `conformance:4` | | Entrypoint | A long-running process that `exec`s the arguments it is given. | Modal requires it, Daytona defaults to `sleep infinity` and E2B uses a start command; the adapters make all three the same. | `conformance:8` | | Signals | PID 1 reaps children and forwards `SIGTERM`; shutdown finishes within 10 seconds. | A sandbox that ignores `SIGTERM` is killed with its work unsaved, and zombies pile up. | `doctor:init`, `conformance:8` | -| Filesystem | `/opt/datalayer` is reserved and read-only to the user; `/home/datalayer/content` is writable; nothing is assumed to persist across restarts. | Datalayer's tools live under `/opt/datalayer`; user code writes where its content is. | `doctor:reserved_path`, `doctor:workdir`, `conformance:6` | +| Filesystem | `/opt/datalayer` is reserved and read-only to the user; `/home/datalayer` is writable; nothing is assumed to persist across restarts. | Datalayer's tools live under `/opt/datalayer`; user code writes where its content is. | `doctor:reserved_path`, `doctor:workdir`, `conformance:6` | | Network | Egress is governed at runtime by policy, not baked into the image; no build-time credentials on disk. | A credential left in a layer is readable by anyone who can pull the image. | `conformance:9`, `conformance:10` | | Locale and time | `C.UTF-8`, UTC, and current CA certificates. | Tools behave the same on every provider, and HTTPS works. | `doctor:locale`, `doctor:timezone`, `doctor:ca_certificates` | | Health | `datalayer-sandbox doctor --json` exits 0 and reports the contract version, the Python version, the kernel versions, the user and the writable paths. | This command is the first check of every smoke test. | `doctor`, `conformance:1` | @@ -35,7 +35,7 @@ This page is generated from `code_sandboxes/environments/contract.py`: change th | uid | `1000` | | gid | `100` | | Home | `/home/datalayer` | -| Working directory | `/home/datalayer/content` | +| Working directory | `/home/datalayer` | | Reserved, read-only | `/opt/datalayer` | | Doctor | `/opt/datalayer/bin/datalayer-sandbox` | | Locale | `C.UTF-8` | diff --git a/docs/docs/environments/specification.mdx b/docs/docs/environments/specification.mdx index bd048b4..410cd35 100644 --- a/docs/docs/environments/specification.mdx +++ b/docs/docs/environments/specification.mdx @@ -27,7 +27,7 @@ A document is refused with `DL_ENV_SPEC_INVALID` when a field is wrong, and with | `base.ref` | — | An approved base: `datalayer/python-cpu` or `datalayer/python-cuda`. | | `base.channel` | — | The release channel, resolved to a digest per variant when the version is resolved. | | `platform.architecture` | `linux/amd64` | Only `linux/amd64`. | -| `packages.python.manager` | `uv` | `uv` or `pip`; both are resolved with uv. `conda` is not buildable yet. | +| `packages.python.manager` | `uv` | `uv` or `pip`; both are resolved with uv. A `conda` **`packages`** list is not built yet — bring a conda `environment.yml` as a `dependencyFile` instead (see `build.source`). | | `packages.python.dependencies` | none | PEP 508 requirements. | | `packages.python.constraints` | none | PEP 508 requirements, applied under Datalayer's protected constraints, which win. | | `packages.python.indexes` | `https://pypi.org/simple` | `https` URLs, with no credential in them. | @@ -37,12 +37,12 @@ A document is refused with `DL_ENV_SPEC_INVALID` when a field is wrong, and with | `commands.postInstall` | none | At most 32 non-empty commands, run after the packages are installed. | | `buildSecrets` | none | `id` is a Datalayer secret id, `dlsec_...`; `mountAs` is `env` or `file`; `name` is the variable or file name. Ids and names are unique. Secrets are injected into the build step that needs them and never written into the artifact. | | `resources.sizeClass` | `small` | `small`, `medium`, `large`, `gpu-small` or `gpu-large`. A GPU class needs an accelerator and a CUDA base; an accelerator needs a GPU class. Datalayer runs no GPU nodes yet, so a GPU class is built for Modal and Daytona, which bring their own GPUs; E2B has none. | -| `resources.accelerator` | `none` | `none`, or `type`, `count` and `cuda`. | +| `resources.accelerator` | `none` | `none`, or `type`, `count` and `cuda`. `type` is the provider's own name for the GPU (Modal: `T4`, `L4`, `A10G`, `L40S`, `A100`, `A100-40GB`, `A100-80GB`, `H100`, `H200`, `B200`; Daytona: `H100`, `H200`, `RTX-PRO-6000`, `RTX-4090`, `RTX-5090`), and a GPU a variant does not offer is refused for it at `validate`. `cuda` is the base's toolkit: `datalayer/python-cuda:2026.09` carries 12.8, and another version is refused. | | `resources.hints` | none | `cpu`, `memoryGi`, `diskGi`: positive numbers, within the size class. | | `compatibility.variants.required` | `[datalayer]` | At least one of `datalayer`, `e2b`, `daytona`, `modal`. | | `compatibility.variants.optional` | none | The same variants, none of them also required. | | `compatibility.regions` | none | Region names. | -| `build.source` | `packages` | `packages`, `dependencyFile` (a `requirements.txt`, or a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved) and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base) all resolve on the Datalayer variant; `image` from a private registry, and `dockerfile`, are refused until they do. | +| `build.source` | `packages` | `packages`; `dependencyFile` (a `requirements.txt`; a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved; or a conda `environment.yml`, resolved in its own `micromamba` solve into an explicit lock with the protected pins merged over its `pip:` layer); `dockerfile` (`build.dockerfile.content`, inline: its `FROM` names an approved base with its channel as the tag, which is what resolves, and what its own instructions install is not locked, as the lock's header says); and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base). `packages`, a `requirements.txt`, a `pyproject.toml` with its `uv.lock`, a conda `environment.yml` and `dockerfile` build on the Datalayer and Daytona variants. A `pyproject.toml`'s `uv.lock` has to lock Datalayer's `jupyter-server` fork, which no package index serves: the project takes it by URL from [the fork's release](https://github.com/datalayer-externals/jupyter-server/releases/tag/v2.21.0-datalayer.1), in `[tool.uv.sources]`, which is also what makes `uv` record its hash. `image` resolves but does not yet start as a sandbox, because the imported image has no Datalayer runtime layer; `image` from a private registry, and a `COPY` from a Dockerfile's build context, are refused. | ## The spec digest diff --git a/docs/docs/providers/daytona.mdx b/docs/docs/providers/daytona.mdx index 7c14471..e888056 100644 --- a/docs/docs/providers/daytona.mdx +++ b/docs/docs/providers/daytona.mdx @@ -229,15 +229,33 @@ sandbox.files.read_bytes("/tmp/data.parquet") ## Environments -An [Environment](/docs/environments) built for Daytona is a **snapshot** — -built from the approved Datalayer base, `uv pip sync --require-hashes` -against the resolved lock, your files and `postInstall` commands, and an -explicit `tini -- sleep infinity` entrypoint (Daytona's own default, -otherwise, is a bare `sleep infinity`, with no init to reap orphans). The -base already carries `uv`, the doctor and the sandbox contract's user and -locale, so none of that is reinstalled — only the lock is genuinely -per-build. The snapshot's own `id` is the artifact; a build also names it, -for operability, but a launch never uses the name. +An [Environment](/environments) built for Daytona is a **snapshot** — +built from the approved Datalayer base, the resolved lock, your files and +`postInstall` commands, and an explicit `tini -- sleep infinity` entrypoint +(Daytona's own default, otherwise, is a bare `sleep infinity`, with no init +to reap orphans). The base already carries `uv`, the doctor and the sandbox +contract's user and locale, so none of that is reinstalled — only the lock is +genuinely per-build. The snapshot's own `id` is the artifact; a build also +names it, for operability, but a launch never uses the name. + +How the lock is installed depends on where the environment came from: + +| Source | Installed with | +|---|---| +| A package list, a `requirements.txt`, a `pyproject.toml` and its `uv.lock` | `uv pip sync --require-hashes`: the snapshot holds exactly the lock | +| A conda `environment.yml` | `micromamba install` of the explicit lock into the base's own interpreter, then the pip layer the solve recorded | +| A Dockerfile | Daytona's `Image.from_dockerfile`, its approved `FROM` pinned to the resolved digest, then `uv pip install --require-hashes` of the lock — `install`, not `sync`, so what the Dockerfile installed stays | + +**Deleted by id, and never stranded.** Retention deletes a snapshot nothing +keeps any more, by its `id` — a name is reused once its snapshot is gone — +and a snapshot already gone counts as deleted. A build that fails *after* +Daytona built its snapshot, at the core tier or at the record, deletes that +snapshot before it reports the failure, so nothing is left in your account +that no version names. + +`make env-build-daytona`, in the `environments` folder of the +[examples repository](https://github.com/datalayer/examples), builds a +version for Daytona from the command line. **Bring your own account:** a build runs with the environment owner's own Daytona organization, not this package's ambient credentials — either @@ -252,6 +270,16 @@ for, and the `target` a build's snapshot lands in is the one a launch starts from — a version that changes size class or region rebuilds a second, distinct artifact rather than resizing the first. +**A GPU is part of the snapshot too.** A GPU class (`gpu-small`, +`gpu-large`) on the CUDA base bakes the spec's `accelerator` into it: its +`type`, one of Daytona's GPUs, and its `count`. CPU and memory come from +`resources.hints`, or Daytona's own size for that GPU, and the disk holds the +CUDA base. Every sandbox of a GPU snapshot is ephemeral, as Daytona requires: +`DaytonaSandbox` reads the snapshot's `gpu` and asks for it. The build's +smoke test adds check 11 to the core tier: the GPUs asked for are visible, +CUDA is the base's, and the host's driver is new enough for it. GPU compute +is billed by Daytona to the owner's organization, which needs GPU credits. + Of the four variants, Daytona is the one whose build-time doctor check and live launch agree in full: the doctor is *not* run at build time (a build-time `RUN` step's PID 1 is Daytona's own build agent, not the diff --git a/docs/docs/providers/e2b.mdx b/docs/docs/providers/e2b.mdx index 3094edc..344ffed 100644 --- a/docs/docs/providers/e2b.mdx +++ b/docs/docs/providers/e2b.mdx @@ -190,7 +190,7 @@ sandbox.files.read_bytes("/tmp/data.parquet") ## Environments -An [Environment](/docs/environments) built for E2B is a **template**, not +An [Environment](/environments) built for E2B is a **template**, not built from the Datalayer base the other three variants share: E2B's own code-interpreter server (the FastAPI service `run_code` and contexts actually talk to) is proprietary and ships baked into E2B's own diff --git a/docs/docs/providers/modal.mdx b/docs/docs/providers/modal.mdx index 1f3e2c2..5af135c 100644 --- a/docs/docs/providers/modal.mdx +++ b/docs/docs/providers/modal.mdx @@ -74,12 +74,14 @@ with Sandbox.create( ## Environments -An [Environment](/docs/environments) built for Modal is an **image id**, +An [Environment](/environments) built for Modal is an **image id**, `im-…` — pulled `from_aws_ecr` off the approved Datalayer base, with your lock installed by `uv pip sync --require-hashes` on top. A published name -is worth having for operability, but a launch always pins the id: each -build gets a fresh credential, so no layer cache survives between builds, -and the id is the only reference that names one build and not another. +is worth having for operability, but a launch always pins the id: a name +moves when the same build is published again, and the id is the only +reference that names one build's image. Modal answers an identical image +definition with the image it already has, so two builds of the same lock +can share one id. **Bring your own account:** a build runs in the environment owner's own Modal workspace, from the same `MODAL_TOKEN_ID`/`MODAL_TOKEN_SECRET` pair @@ -96,8 +98,22 @@ with no image id — general Modal sandbox usage here is unaffected). Confirmed live: a launched sandbox's identity check now passes, where it used to report `uid: 0`. -**A known, open gap:** two of the core tier's checks — imports and -filesystem — still fail on a launched Environment sandbox, for a reason -not yet root-caused: reproduced on a real, contract-built image, not on a -plain `ModalSandbox`, so it is specific to something about that image -under repeated `exec`, not the identity fix above. +**The sandbox is held by `sleep infinity`, not the base's Jupyter.** +Modal keeps the base image's CMD, `start-jupyter.sh`, under the builder's +`ENTRYPOINT`, where Docker would reset it, so a sandbox started with no +command ran a Jupyter server as its main process. That server exits +within a minute and the sandbox ends with it: "Modal Sandbox is shutting +down", a few commands in. Since 1.9.36 the artifact's CMD is +`sleep infinity`, and `ModalSandbox` names `sleep infinity` when it +launches an artifact, which covers images built by earlier releases too. +Jupyter is started by exec when it is asked for +(`prepare_jupyter_server`). + +**GPU versions.** On Modal a GPU is a launch option, not part of the +image: a GPU version's image builds like any other on the CUDA base, and +its sandboxes start on the GPU the version names. The build's smoke test +does that (`gpu="L4"`, or `"H100:2"` for two) and adds the contract's +check 11: the GPUs asked for are visible, and the image's CUDA is the +version's. A GPU Modal does not offer — it offers `T4`, `L4`, `A10G`, +`L40S`, `A100`, `A100-40GB`, `A100-80GB`, `H100`, `H200`, `B200` — is +refused at `validate`, before anything is built. diff --git a/pyproject.toml b/pyproject.toml index b005720..7936d4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "jupyter-server", "jupyter-server-client", "pydantic>=2.0", + "pyyaml", "rich", "typer>=0.12.0", ] @@ -75,7 +76,7 @@ test = [ "pytest-cov>=4.0", ] lint = ["mdformat>0.7", "mdformat-gfm>=0.3.5", "ruff"] -typing = ["mypy>=0.990"] +typing = ["mypy>=0.990", "types-PyYAML"] [project.license] file = "LICENSE" @@ -104,6 +105,10 @@ filterwarnings = [ # use of the client in a test became an exception. It took the live matrix # against a real Datalayer sandbox down before the sandbox was reached. "module:Datalayer is migrating its paths to use standard platformdirs:DeprecationWarning", + # The Daytona SDK (0.205) reads its own deprecated `DaytonaConfig.server_url` + # inside `Daytona(config)`: with `error` in force, every live Daytona test + # failed before reaching Daytona, and a nightly that skipped hid it. + "ignore:`server_url` is deprecated:DeprecationWarning", ] [tool.mypy] diff --git a/schemas/environment-v1alpha1.json b/schemas/environment-v1alpha1.json index d23b194..4b7040a 100644 --- a/schemas/environment-v1alpha1.json +++ b/schemas/environment-v1alpha1.json @@ -95,6 +95,17 @@ ], "default": null }, + "dockerfile": { + "anyOf": [ + { + "$ref": "#/$defs/DockerfileSpec" + }, + { + "type": "null" + } + ], + "default": null + }, "image": { "anyOf": [ { @@ -152,9 +163,48 @@ "title": "Compatibility", "type": "object" }, + "ContentsBuildEntry": { + "additionalProperties": false, + "description": "One immutable file the Environment bakes from an external source.\n\nUnlike ``files`` \u2014 which a user uploads, referenced by ``contentRef`` \u2014 a\n``contentsBuild`` entry names an external ``source`` URL fetched at build\ntime and verified against ``sha256`` (required: the build verifies every\nbyte it bakes). Both are baked the same way, into every provider artifact.", + "properties": { + "path": { + "title": "Path", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + }, + "sizeBytes": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sizebytes" + }, + "source": { + "title": "Source", + "type": "string" + } + }, + "required": [ + "source", + "path", + "sha256" + ], + "title": "ContentsBuildEntry", + "type": "object" + }, "DependencyFileSpec": { "additionalProperties": false, - "description": "A `requirements.txt`, or a `pyproject.toml` with its `uv.lock` (E3-01).\n\n``requirements`` resolves the way ``packages`` does \u2014 the protected\nconstraints merged in, the same solve. ``pyproject`` does not resolve at\nall: its own ``uv.lock`` is verified against the current\n``pyproject.toml`` and exported, never re-solved, because a lock the\nauthor already made is the whole point of bringing one.", + "description": "A `requirements.txt`, a `pyproject.toml` with its `uv.lock`, or a conda\n`environment.yml` (E3-01, E3-02).\n\n``requirements`` resolves the way ``packages`` does \u2014 the protected\nconstraints merged in, the same solve. ``pyproject`` does not resolve at\nall: its own ``uv.lock`` is verified against the current\n``pyproject.toml`` and exported, never re-solved, because a lock the\nauthor already made is the whole point of bringing one. ``conda`` resolves\nthe ``environment.yml`` in its own ``micromamba`` solve into an explicit\nlock, with the protected constraints merged over its ``pip:`` layer.", "properties": { "content": { "default": "", @@ -170,7 +220,8 @@ "default": "requirements", "enum": [ "requirements", - "pyproject" + "pyproject", + "conda" ], "title": "Sourceformat", "type": "string" @@ -179,6 +230,19 @@ "title": "DependencyFileSpec", "type": "object" }, + "DockerfileSpec": { + "additionalProperties": false, + "description": "The Dockerfile a `dockerfile`-sourced version builds from (E3-03).\n\nInline, the way ``DependencyFileSpec`` carries a ``requirements.txt``: a\nfile the author brings lives in the spec, so ``check_dockerfile`` and\nevery variant's capability report can read it **before** anything is\nqueued \u2014 which is the whole point of refusing an instruction a variant\ndoes not implement at ``validate`` rather than halfway through a build.\n\nThe build *context* \u2014 the extra files a ``COPY`` needs \u2014 is the separate\nupload E3-03 describes, bounded and checked by ``validate_build_context``.\nA Dockerfile with no context is the common case and needs no upload at\nall.", + "properties": { + "content": { + "default": "", + "title": "Content", + "type": "string" + } + }, + "title": "DockerfileSpec", + "type": "object" + }, "EnvironmentSpec": { "additionalProperties": false, "properties": { @@ -201,6 +265,13 @@ "compatibility": { "$ref": "#/$defs/Compatibility" }, + "contentsBuild": { + "items": { + "$ref": "#/$defs/ContentsBuildEntry" + }, + "title": "Contentsbuild", + "type": "array" + }, "contract": { "default": "sandbox-contract/v1", "title": "Contract", diff --git a/tests/egress_logs/buildctl-base.log b/tests/egress_logs/buildctl-base.log new file mode 100644 index 0000000..81b6748 --- /dev/null +++ b/tests/egress_logs/buildctl-base.log @@ -0,0 +1,16 @@ +#1 [internal] load build definition from Dockerfile +#1 transferring dockerfile: 80B done +#1 DONE 0.0s + +#2 [internal] load metadata for ghcr.io/astral-sh/uv:0.12.11 +#2 ERROR: failed to do request: Head "https://ghcr.io/v2/astral-sh/uv/manifests/0.12.11": Forbidden +------ + > [internal] load metadata for ghcr.io/astral-sh/uv:0.12.11: +------ +Dockerfile:1 +-------------------- + 1 | >>> FROM ghcr.io/astral-sh/uv:0.12.11 + 2 | RUN true + 3 | +-------------------- +error: failed to solve: ghcr.io/astral-sh/uv:0.12.11: failed to resolve source metadata for ghcr.io/astral-sh/uv:0.12.11: failed to do request: Head "https://ghcr.io/v2/astral-sh/uv/manifests/0.12.11": Forbidden diff --git a/tests/egress_logs/buildctl-curl.log b/tests/egress_logs/buildctl-curl.log new file mode 100644 index 0000000..54d5fec --- /dev/null +++ b/tests/egress_logs/buildctl-curl.log @@ -0,0 +1,41 @@ +#1 [internal] load build definition from Dockerfile +#1 transferring dockerfile: 325B done +#1 DONE 0.0s + +#2 [internal] load metadata for docker.io/library/buildpack-deps:bookworm-scm +#2 DONE 0.2s + +#3 [internal] load .dockerignore +#3 transferring context: 2B done +#3 DONE 0.0s + +#4 [1/4] FROM docker.io/library/buildpack-deps:bookworm-scm@sha256:bb8654d03bd8341e702f1ae30adca91a11f4164da2d37dbe7a0bf49e4901bd03 +#4 CACHED + +#5 [2/4] RUN git clone -q https://github.com/psf/requests.git /tmp/r || true +#5 0.063 fatal: unable to access 'https://github.com/psf/requests.git/': CONNECT tunnel failed, response 403 +#5 DONE 0.1s + +#6 [3/4] RUN curl -fsSL https://raw.githubusercontent.com/psf/requests/main/README.md -o /dev/null || true +#6 0.065 curl: (22) The requested URL returned error: 403 +#6 DONE 0.1s + +#7 [4/4] RUN wget https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz -O /dev/null +#7 0.060 --2026-09-19 08:44:09-- https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz +#7 0.066 Connecting to 172.18.0.2:3128... connected. +#7 0.067 Proxy tunneling failed: ForbiddenUnable to establish SSL connection. +#7 ERROR: process "/bin/sh -c wget https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz -O /dev/null" did not complete successfully: exit code: 4 +------ + > [4/4] RUN wget https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz -O /dev/null: +0.060 --2026-09-19 08:44:09-- https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz +0.066 Connecting to 172.18.0.2:3128... connected. +0.067 Proxy tunneling failed: ForbiddenUnable to establish SSL connection. +------ +Dockerfile:4 +-------------------- + 2 | RUN git clone -q https://github.com/psf/requests.git /tmp/r || true + 3 | RUN curl -fsSL https://raw.githubusercontent.com/psf/requests/main/README.md -o /dev/null || true + 4 | >>> RUN wget https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz -O /dev/null + 5 | +-------------------- +error: failed to solve: process "/bin/sh -c wget https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz -O /dev/null" did not complete successfully: exit code: 4 diff --git a/tests/egress_logs/buildctl-pip.log b/tests/egress_logs/buildctl-pip.log new file mode 100644 index 0000000..b43290e --- /dev/null +++ b/tests/egress_logs/buildctl-pip.log @@ -0,0 +1,43 @@ +#1 [internal] load build definition from Dockerfile +#1 transferring dockerfile: 192B done +#1 DONE 0.0s + +#2 [internal] load metadata for docker.io/library/python:3.12-slim +#2 DONE 0.2s + +#3 [internal] load .dockerignore +#3 transferring context: 2B done +#3 DONE 0.0s + +#4 [1/3] FROM docker.io/library/python:3.12-slim@sha256:2f17fc044b579bab302c2e8054d3a686e2cb9a83de48e70534b94cd8ebbe06a9 +#4 CACHED + +#5 [2/3] RUN pip install --no-cache-dir --index-url https://pypi.example-private.io/simple requests +#5 0.887 Looking in indexes: https://pypi.example-private.io/simple +#5 0.890 WARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +#5 1.391 WARNING: Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +#5 2.393 WARNING: Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +#5 4.395 WARNING: Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +#5 8.397 WARNING: Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +#5 8.401 ERROR: Could not find a version that satisfies the requirement requests (from versions: none) +#5 8.413 ERROR: No matching distribution found for requests +#5 ERROR: process "/bin/sh -c pip install --no-cache-dir --index-url https://pypi.example-private.io/simple requests" did not complete successfully: exit code: 1 +------ + > [2/3] RUN pip install --no-cache-dir --index-url https://pypi.example-private.io/simple requests: +0.887 Looking in indexes: https://pypi.example-private.io/simple +0.890 WARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +1.391 WARNING: Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +2.393 WARNING: Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +4.395 WARNING: Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +8.397 WARNING: Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 403 Forbidden'))': /simple/requests/ +8.401 ERROR: Could not find a version that satisfies the requirement requests (from versions: none) +8.413 ERROR: No matching distribution found for requests +------ +Dockerfile:2 +-------------------- + 1 | FROM python:3.12-slim + 2 | >>> RUN pip install --no-cache-dir --index-url https://pypi.example-private.io/simple requests + 3 | RUN pip install --no-cache-dir requests + 4 | +-------------------- +error: failed to solve: process "/bin/sh -c pip install --no-cache-dir --index-url https://pypi.example-private.io/simple requests" did not complete successfully: exit code: 1 diff --git a/tests/egress_logs/buildctl-uv.log b/tests/egress_logs/buildctl-uv.log new file mode 100644 index 0000000..9df2a80 --- /dev/null +++ b/tests/egress_logs/buildctl-uv.log @@ -0,0 +1,54 @@ +#1 [internal] load build definition from Dockerfile +#1 transferring dockerfile: 193B done +#1 DONE 0.0s + +#2 [internal] load metadata for docker.io/library/python:3.12-slim +#2 DONE 0.2s + +#3 [internal] load .dockerignore +#3 transferring context: 2B done +#3 DONE 0.0s + +#4 [1/3] FROM docker.io/library/python:3.12-slim@sha256:2f17fc044b579bab302c2e8054d3a686e2cb9a83de48e70534b94cd8ebbe06a9 +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 26B done +#5 DONE 0.0s + +#6 [2/3] COPY uv /usr/local/bin/uv +#6 CACHED + +#7 [3/3] RUN uv pip install --system "requests @ https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz" +#7 0.175 Using Python 3.12.14 environment at: /usr/local +#7 9.090 × Failed to download and build `requests @ +#7 9.090 │ https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz` +#7 9.090 ├─▶ Request failed after 3 retries in 8.9s +#7 9.090 ├─▶ Failed to fetch: +#7 9.090 │ `https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz` +#7 9.090 ├─▶ error sending request for url +#7 9.090 │ (https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz) +#7 9.090 ├─▶ client error (Connect) +#7 9.090 ╰─▶ tunnel error: unsuccessful +#7 ERROR: process "/bin/sh -c uv pip install --system \"requests @ https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz\"" did not complete successfully: exit code: 1 +------ + > [3/3] RUN uv pip install --system "requests @ https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz": +0.175 Using Python 3.12.14 environment at: /usr/local +9.090 × Failed to download and build `requests @ +9.090 │ https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz` +9.090 ├─▶ Request failed after 3 retries in 8.9s +9.090 ├─▶ Failed to fetch: +9.090 │ `https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz` +9.090 ├─▶ error sending request for url +9.090 │ (https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz) +9.090 ├─▶ client error (Connect) +9.090 ╰─▶ tunnel error: unsuccessful +------ +Dockerfile:3 +-------------------- + 1 | FROM python:3.12-slim + 2 | COPY uv /usr/local/bin/uv + 3 | >>> RUN uv pip install --system "requests @ https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz" + 4 | +-------------------- +error: failed to solve: process "/bin/sh -c uv pip install --system \"requests @ https://github.com/psf/requests/archive/refs/tags/v2.32.3.tar.gz\"" did not complete successfully: exit code: 1 diff --git a/tests/egress_logs/micromamba.log b/tests/egress_logs/micromamba.log new file mode 100644 index 0000000..11a35f4 --- /dev/null +++ b/tests/egress_logs/micromamba.log @@ -0,0 +1,30 @@ +warning libmamba Download error (56) Failure when receiving data from the peer [https://conda.anaconda.org/conda-forge/noarch/repodata.json.zst] + CONNECT tunnel failed, response 403 +warning libmamba Retrying in 2 seconds +warning libmamba Download error (56) Failure when receiving data from the peer [https://conda.anaconda.org/conda-forge/linux-64/repodata.json.zst] + CONNECT tunnel failed, response 403 +warning libmamba Retrying in 2 seconds +warning libmamba Download error (56) Failure when receiving data from the peer [https://conda.anaconda.org/conda-forge/noarch/repodata.json.zst] + CONNECT tunnel failed, response 403 +warning libmamba Retrying in 2 seconds +warning libmamba Download error (56) Failure when receiving data from the peer [https://conda.anaconda.org/conda-forge/linux-64/repodata.json.zst] + CONNECT tunnel failed, response 403 +warning libmamba Retrying in 2 seconds +warning libmamba Download error (56) Failure when receiving data from the peer [https://conda.anaconda.org/conda-forge/noarch/repodata.json.zst] + CONNECT tunnel failed, response 403 +warning libmamba Retrying in 2 seconds +warning libmamba Download error (56) Failure when receiving data from the peer [https://conda.anaconda.org/conda-forge/linux-64/repodata.json.zst] + CONNECT tunnel failed, response 403 +warning libmamba Retrying in 2 seconds +warning libmamba Download error (56) Failure when receiving data from the peer [https://conda.anaconda.org/conda-forge/noarch/repodata.json.zst] + CONNECT tunnel failed, response 403 +warning libmamba Retrying in 2 seconds +critical libmamba Multiple errors occurred: + Download error (56) Failure when receiving data from the peer [https://conda.anaconda.org/conda-forge/noarch/repodata.json.zst] + CONNECT tunnel failed, response 403 + Subdir conda-forge/noarch not loaded! + If you run into this error repeatedly, your package cache may be corrupted. + Please try running `mamba clean -a` to remove this cache before retrying the operation. + + If you still are having issues, please report the error on `mamba-org/mamba`'s issue tracker: + https://github.com/mamba-org/mamba/issues/new?assignees=&labels=&projects=&template=bug.yml diff --git a/tests/test_client.py b/tests/test_client.py index c75c86e..34df743 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -157,6 +157,62 @@ def tool_caller(): assert client.is_alive() is True +class TestRestartingWhatIsActuallyHoldingTheState: + """A restart has to clear the interpreter, not just this client's socket. + + Appendix B check 7 is "nothing is assumed to persist across restarts". A + sandbox this process owns is cleared by its own lifecycle; one attached + to somebody else's Jupyter server is not, because the kernel outlives + the websocket. Found live on r1, 2026-09-16: `state survived the + restart`. + """ + + def test_a_sandbox_that_can_restart_its_kernel_is_asked_to(self): + class _AttachedSandbox(_FakeSandbox): + def __init__(self): + super().__init__() + self.kernel_restarts = 0 + + def restart_kernel(self): + self.kernel_restarts += 1 + return True + + sandbox = _AttachedSandbox() + sandbox.start() + CodeSandboxClient(sandbox).restart() + + assert sandbox.kernel_restarts == 1 + # And the lifecycle was left alone: stopping it would have dropped a + # connection the restarted kernel is still reachable on. + assert sandbox.is_started is True + + def test_a_kernel_restart_that_fails_falls_back_to_the_lifecycle(self): + class _RefusingSandbox(_FakeSandbox): + def __init__(self): + super().__init__() + self.stops = 0 + + def restart_kernel(self): + return False + + def stop(self): + self.stops += 1 + super().stop() + + sandbox = _RefusingSandbox() + sandbox.start() + CodeSandboxClient(sandbox).restart() + + assert sandbox.stops == 1, "a refused kernel restart still restarts the sandbox" + assert sandbox.is_started is True + + def test_a_sandbox_with_no_kernel_of_its_own_restarts_as_it_always_did(self): + sandbox = _FakeSandbox() + sandbox.start() + CodeSandboxClient(sandbox).restart() + assert sandbox.is_started is True + + def test_a_sandbox_that_knows_it_is_gone_is_believed(): """`is_alive` reports what the sandbox can find out, not the local start flag, so a backend that died under us is not reported as ready.""" diff --git a/tests/test_daytona.py b/tests/test_daytona.py index 89166d7..f3c06dd 100644 --- a/tests/test_daytona.py +++ b/tests/test_daytona.py @@ -700,6 +700,44 @@ def test_a_snapshot_and_an_image_together_are_refused(): sandbox._create_params(daytona) +class _SnapshotsNamed: + """`daytona.snapshot`, answering the records it was given by id.""" + + def __init__(self, **records) -> None: + self.records = records + + def get(self, snapshot_id): + if snapshot_id not in self.records: + raise KeyError(snapshot_id) + return self.records[snapshot_id] + + +def test_a_sandbox_of_a_gpu_snapshot_is_ephemeral(): + """E2-17: a GPU is baked into an Environment's snapshot, so its sandbox asks + for a GPU without `gpu=`, and Daytona refuses it unless it is ephemeral.""" + import types + + daytona = pytest.importorskip("daytona") + sandbox = _started(SandboxConfig(), snapshot="dl-gpu-v1") + sandbox._daytona = types.SimpleNamespace( + snapshot=_SnapshotsNamed(**{"dl-gpu-v1": types.SimpleNamespace(gpu=2)}) + ) + params = sandbox._create_params(daytona) + assert isinstance(params, daytona.CreateSandboxFromSnapshotParams) + assert params.auto_delete_interval == 0 + + +def test_a_sandbox_of_a_cpu_snapshot_is_kept_as_asked(): + import types + + daytona = pytest.importorskip("daytona") + for records in ({"dl-geo-v3": types.SimpleNamespace(gpu=0)}, {}): + sandbox = _started(SandboxConfig(), snapshot="dl-geo-v3") + sandbox._daytona = types.SimpleNamespace(snapshot=_SnapshotsNamed(**records)) + params = sandbox._create_params(daytona) + assert getattr(params, "auto_delete_interval", None) != 0 + + def test_a_snapshot_alone_still_creates_from_the_snapshot(): daytona = pytest.importorskip("daytona") diff --git a/tests/test_environment_attest.py b/tests/test_environment_attest.py index 90038b7..d1d9165 100644 --- a/tests/test_environment_attest.py +++ b/tests/test_environment_attest.py @@ -19,14 +19,22 @@ import json import subprocess +from types import SimpleNamespace import pytest -from code_sandboxes.environments.attest import Attestor, attest_artifact, signature_tag +from code_sandboxes.environments.attest import ( + Attestor, + attest_artifact, + licenses_by_package, + licenses_of, + signature_tag, +) from code_sandboxes.environments.builders import ArtifactReference -from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.errors import PROVIDER_ERROR, EnvironmentsError from code_sandboxes.environments.policy import ( DEFAULT_POLICY, + EnvironmentsPolicy, Finding, ScanPolicy, decide, @@ -80,7 +88,7 @@ def basic(identifier: str, severity: str, *, package: str = "openssl") -> dict: class FakeEcr: - """The two ECR calls the attestor makes, and nothing else.""" + """The three ECR calls the attestor makes, and nothing else.""" def __init__( self, @@ -90,6 +98,7 @@ def __init__( enhanced_findings=True, manifest: dict | None = None, pages: list[list[dict]] | None = None, + size_bytes: int | None = 2_147_483_648, ) -> None: self.statuses = list(statuses) self.findings = list(findings) @@ -99,8 +108,12 @@ def __init__( self.pages = pages self.enhanced_findings = enhanced_findings self.manifest = manifest + #: What `describe_images` answers for the digest; None makes it raise + #: `ImageNotFoundException`, the way a registry that lost it would. + self.size_bytes = size_bytes self.asked = 0 self.scanned: list[str] = [] + self.sized: list[str] = [] def batch_get_image(self, repositoryName, imageIds, acceptedMediaTypes): # noqa: N803 - boto3's spelling manifest = self.manifest or { @@ -117,6 +130,14 @@ def batch_get_image(self, repositoryName, imageIds, acceptedMediaTypes): # noqa ] } + def describe_images(self, repositoryName, imageIds): # noqa: N803 - boto3's spelling + self.sized.append(imageIds[0]["imageDigest"]) + if self.size_bytes is None: + error = Exception("ImageNotFoundException") + error.response = {"Error": {"Code": "ImageNotFoundException"}} + raise error + return {"imageDetails": [{"imageSizeInBytes": self.size_bytes}]} + def describe_image_scan_findings(self, repositoryName, imageId, nextToken=None): # noqa: N803 - boto3's spelling self.asked += 1 self.scanned.append(imageId["imageDigest"]) @@ -599,28 +620,68 @@ def test_a_blocked_artifact_is_never_signed(self) -> None: assert "CVE-2026-1234" in raised.value.message assert cosign.argv == [], "a blocked artifact must not be signed" - def test_a_reference_that_is_not_a_digest_cannot_be_attested(self) -> None: - artifact = ArtifactReference( - variant="modal", - immutable_reference="im-1234567890", - provider_artifact_id="im-1234567890", - contract_version="sandbox-contract/v1", + def test_a_denied_licence_is_never_signed(self) -> None: + """The order is the point, the same as a blocked scan: a signature is + Datalayer's word, and this is the second word that goes into it.""" + cosign = Cosign() + sbom = {"packages": [{"name": "gpl-lib", "licenseConcluded": "GPL-3.0"}]} + policy = EnvironmentsPolicy(allowed_licenses=("MIT", "Apache-2.0")) + with pytest.raises(EnvironmentsError) as raised: + attest_artifact( + artifact=self.an_artifact(), + attestor=an_attestor(run=cosign), + sbom=sbom, + environments_policy=policy, + ) + assert raised.value.code.code == "DL_ENV_POLICY_DENIED" + assert "GPL-3.0" in raised.value.message and "gpl-lib" in raised.value.message + assert cosign.argv == [], "a denied licence must not be signed" + + def test_an_allowed_licence_is_signed_as_usual(self) -> None: + cosign = Cosign() + sbom = {"packages": [{"name": "requests", "licenseConcluded": "Apache-2.0"}]} + policy = EnvironmentsPolicy(allowed_licenses=("MIT", "Apache-2.0")) + answer = attest_artifact( + artifact=self.an_artifact(), + attestor=an_attestor(run=cosign), + sbom=sbom, + environments_policy=policy, + ) + assert answer["licenses"] == ["Apache-2.0"] + assert cosign.argv, "an allowed licence is signed" + + def test_with_no_organization_policy_nothing_about_licences_is_refused(self) -> None: + """The default: no organization has written this section, so a build + with any licence at all is unrestricted, the same as every artifact + before this box existed.""" + cosign = Cosign() + sbom = {"packages": [{"name": "gpl-lib", "licenseConcluded": "GPL-3.0"}]} + answer = attest_artifact( + artifact=self.an_artifact(), attestor=an_attestor(run=cosign), sbom=sbom ) + assert answer["licenses"] == ["GPL-3.0"] + assert cosign.argv, "unrestricted by default" + + def test_a_reference_that_is_not_a_digest_cannot_be_attested(self) -> None: + """A `datalayer` artifact must be a digest in this platform's registry. + + `attest_artifact` takes `artifact: Any`, so nothing guarantees every + caller went through the model validator that would have refused this. + """ + artifact = SimpleNamespace(variant="datalayer", immutable_reference="im-1234567890") with pytest.raises(EnvironmentsError) as raised: attest_artifact(artifact=artifact, attestor=an_attestor()) assert raised.value.code.code == "DL_ENV_PROVIDER_ERROR" def test_a_malformed_digest_cannot_be_attested_either(self) -> None: """`sha256:bad` starts with `sha256:` too: only a whole one is - accepted (found on PR #27's Copilot review). `e2b` rather than - `datalayer`, whose own model validator already refuses a malformed - digest before this function ever sees it — `attest_artifact` takes - `artifact: Any`, so nothing guarantees every caller went through it.""" - artifact = ArtifactReference( - variant="e2b", + accepted (found on PR #27's Copilot review). Asked of `datalayer`, + since that is the variant this check is for — `attest_artifact` takes + `artifact: Any`, so nothing guarantees every caller went through the + model validator that would have refused it.""" + artifact = SimpleNamespace( + variant="datalayer", immutable_reference=f"{REGISTRY}/{REPOSITORY}@sha256:bad", - provider_artifact_id="sha256:bad", - contract_version="sandbox-contract/v1", ) with pytest.raises(EnvironmentsError) as raised: attest_artifact(artifact=artifact, attestor=an_attestor()) @@ -630,6 +691,33 @@ def test_the_policy_it_was_decided_under_is_part_of_the_record(self) -> None: answer = attest_artifact(artifact=self.an_artifact(), attestor=an_attestor()) assert answer["scan_summary"]["policy"] == DEFAULT_POLICY.body() + def test_the_size_is_read_from_the_registry_when_nobody_hands_one_down(self) -> None: + """Which is every real call: the builder answers a reference, not a + weight, so `environments.artifact.bytes` — section 14's artifact size + — had no point in it although artifacts had been recorded (E1-25).""" + ecr = FakeEcr(size_bytes=2_147_483_648) + answer = attest_artifact(artifact=self.an_artifact(), attestor=an_attestor(ecr=ecr)) + assert answer["size_bytes"] == 2_147_483_648 + assert ecr.sized == [DIGEST] + + def test_a_size_handed_down_is_kept_and_the_registry_is_not_asked(self) -> None: + ecr = FakeEcr() + answer = attest_artifact( + artifact=self.an_artifact(), size_bytes=116_183_040, attestor=an_attestor(ecr=ecr) + ) + assert answer["size_bytes"] == 116_183_040 + assert ecr.sized == [] + + def test_a_size_that_cannot_be_read_is_not_a_reason_to_refuse(self) -> None: + """A missing number on a dashboard, against an artifact nothing can + launch: the artifact is signed and the attestation stands.""" + said: list[str] = [] + attestor = an_attestor(ecr=FakeEcr(size_bytes=None), log=said.append) + answer = attest_artifact(artifact=self.an_artifact(), attestor=attestor) + assert answer["size_bytes"] is None + assert answer["signature_ref"] == f"{REGISTRY}/{REPOSITORY}@{DIGEST}" + assert any("size could not be read" in line for line in said) + def test_nothing_reaches_a_registry_when_nothing_could_sign() -> None: """The order that keeps a refusal cheap and honest (E1-08, E1-09). @@ -649,3 +737,138 @@ def test_nothing_reaches_a_registry_when_nothing_could_sign() -> None: attest_artifact(artifact=artifact, attestor=an_attestor(ecr=ecr, key="")) assert raised.value.detail["missing"] == "DATALAYER_ENVIRONMENTS_KMS_KEY" assert ecr.asked == 0, "the registry was asked before anything could have been signed" + + +# -- the licences a publication carries --------------------------------------------------------- + + +class TestLicencesFromTheSbom: + """What `licenses_of` reads, and what it refuses to guess. + + A published version's page says it shows the licences its SBOM names + (D-12, E2-16). They had nowhere to come from: the snapshot read the scan + summary, and the registry's scanner reports vulnerabilities. + """ + + def test_it_reads_spdx_which_is_what_buildkit_writes(self) -> None: + document = { + "packages": [ + {"name": "gdal", "licenseConcluded": "MIT"}, + {"name": "numpy", "licenseConcluded": "BSD-3-Clause"}, + {"name": "again", "licenseConcluded": "MIT"}, + ] + } + assert licenses_of(document) == ["BSD-3-Clause", "MIT"] + + def test_a_concluded_licence_wins_over_a_declared_one(self) -> None: + """`licenseConcluded` is what the tool decided; `licenseDeclared` is the claim.""" + document = { + "packages": [ + {"licenseConcluded": "Apache-2.0", "licenseDeclared": "MIT"}, + ] + } + assert licenses_of(document) == ["Apache-2.0"] + + def test_noassertion_is_not_a_licence_and_falls_through(self) -> None: + """SPDX writes NOASSERTION when it could not tell, which must not be shown.""" + document = { + "packages": [ + {"licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-3-Clause"}, + {"licenseConcluded": "NONE", "licenseDeclared": ""}, + ] + } + assert licenses_of(document) == ["BSD-3-Clause"] + + def test_it_reads_cyclonedx_by_id_by_name_and_by_expression(self) -> None: + document = { + "components": [ + {"licenses": [{"license": {"id": "Apache-2.0"}}]}, + {"licenses": [{"license": {"name": "Public Domain"}}]}, + {"licenses": [{"expression": "MIT OR Apache-2.0"}]}, + ] + } + assert licenses_of(document) == [ + "Apache-2.0", + "MIT OR Apache-2.0", + "Public Domain", + ] + + def test_a_document_it_does_not_understand_names_nothing(self) -> None: + """Never a reason to fail a build: a licence list is worth having, not dying for.""" + for document in (None, {}, {"packages": None}, {"components": [1, 2]}, "spdx"): + assert licenses_of(document) == [] + + +class TestWhatIsAttestedAndWhatIsNot: + """D-11 is about the Datalayer artifact, not every artifact. + + It lives in this platform's registry: the scanner reads it there, cosign + signs that digest, and the Operator refuses to start what is unsigned. A + managed artifact is none of those things — it lives in the owner's own + provider account (D-8), named the way that provider names it. + """ + + def _artifact(self, variant: str, reference: str): + return SimpleNamespace(variant=variant, immutable_reference=reference) + + def test_a_daytona_snapshot_is_not_attested(self) -> None: + """Its reference is a uuid, and attesting it anyway failed the first + real Daytona build *after* the snapshot was already built + (2026-09-17).""" + answer = attest_artifact( + artifact=self._artifact("daytona", "51d10ab0-d98d-4117-bdb5-918e98646c92"), + size_bytes=1234, + ) + assert answer["scan_summary"] == {} + assert answer["signature_ref"] == "" + assert answer["signed_now"] is False + # What the provider told us is still recorded. + assert answer["size_bytes"] == 1234 + + def test_an_e2b_build_id_and_a_modal_image_id_are_not_either(self) -> None: + for variant, reference in (("e2b", "bld-123"), ("modal", "im-abc123")): + answer = attest_artifact(artifact=self._artifact(variant, reference)) + assert answer["signed_now"] is False, variant + assert answer["scan_summary"] == {}, variant + + def test_a_datalayer_artifact_that_is_not_a_digest_still_fails(self) -> None: + """The check that matters is kept where it means something.""" + with pytest.raises(EnvironmentsError) as raised: + attest_artifact(artifact=self._artifact("datalayer", "not-a-digest")) + assert raised.value.code is PROVIDER_ERROR + assert "cannot be attested" in str(raised.value) + + +class TestLicencesByPackage: + """`licenses_by_package`: the attribution `licenses_of` itself throws away + (E3-06 needs to name which package carries a denied licence).""" + + def test_it_pairs_each_spdx_package_with_its_licence(self) -> None: + document = { + "packages": [ + {"name": "gdal", "licenseConcluded": "MIT"}, + {"name": "numpy", "licenseConcluded": "BSD-3-Clause"}, + ] + } + assert licenses_by_package(document) == [ + ("gdal", "MIT"), + ("numpy", "BSD-3-Clause"), + ] + + def test_it_pairs_each_cyclonedx_component_with_its_licence(self) -> None: + document = { + "components": [ + {"name": "requests", "licenses": [{"license": {"id": "Apache-2.0"}}]}, + ] + } + assert licenses_by_package(document) == [("requests", "Apache-2.0")] + + def test_an_unnamed_package_still_carries_its_licence(self) -> None: + """`licenses_of` must not lose a licence just because this test does not name one.""" + document = {"packages": [{"licenseConcluded": "MIT"}]} + assert licenses_by_package(document) == [("", "MIT")] + assert licenses_of(document) == ["MIT"] + + def test_a_document_it_does_not_understand_names_nothing(self) -> None: + for document in (None, {}, {"packages": None}, {"components": [1, 2]}, "spdx"): + assert licenses_by_package(document) == [] diff --git a/tests/test_environment_bases.py b/tests/test_environment_bases.py index c2a185a..df0beb2 100644 --- a/tests/test_environment_bases.py +++ b/tests/test_environment_bases.py @@ -25,14 +25,25 @@ @pytest.mark.parametrize("variant", VARIANTS) -def test_the_2026_09_channel_of_python_cuda_has_no_digest_until_it_is_pushed( +def test_the_2026_09_channel_of_python_cuda_resolves_the_digest_its_release_pushed( variant: str, ) -> None: - """PLAN_ENV.md, E2-17: the CUDA channel is not in ECR yet, so nothing may be made up.""" - ref = "datalayer/python-cuda" - assert APPROVED_BASES[ref].channels == {"2026.09": {}} + """PLAN_ENV.md, E2-17: released 2026-09-18, one image for every variant, + carrying the CUDA 12.8 toolkit a spec's `accelerator.cuda` may ask for.""" + base = APPROVED_BASES["datalayer/python-cuda"] + assert resolve_base(base.ref, "2026.09", variant) == ( + "sha256:dc8f0015b4f7dbca92a88d2d6a4a96714b76f9a1af9dddd86835812493e40f08" + ) + assert (base.accelerator, base.cuda) == (True, "12.8") + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_a_channel_with_no_digest_yet_is_refused_rather_than_made_up(variant: str) -> None: + """A channel is approved before its release pushes it: nothing may be made up.""" + ref = "datalayer/python-next" + bases = {ref: ApprovedBase(ref=ref, python_versions=("3.13",), channels={"2026.10": {}})} with pytest.raises(BaseChannelUnpublishedError) as refused: - resolve_base(ref, "2026.09", variant) + resolve_base(ref, "2026.10", variant, bases) error = refused.value assert isinstance(error, EnvironmentsError) assert error.code is errors.ARTIFACT_MISSING @@ -40,9 +51,9 @@ def test_the_2026_09_channel_of_python_cuda_has_no_digest_until_it_is_pushed( assert error.detail == { "reason": "base_channel_unpublished", "base": ref, - "channel": "2026.09", + "channel": "2026.10", "variant": variant, - "repository": "environments/base/" + ref.rsplit("/", 1)[-1], + "repository": "environments/base/python-next", } assert "sha256:" not in str(error) assert error.to_body()["code"] == "DL_ENV_ARTIFACT_MISSING" @@ -52,10 +63,17 @@ def test_the_2026_09_channel_of_python_cuda_has_no_digest_until_it_is_pushed( def test_the_2026_09_channel_of_python_cpu_resolves_the_digest_its_release_pushed( variant: str, ) -> None: - """PLAN_ENV.md, E1-05: released 2026-09-14 (jupyter-python 0.2.2, E1-08's - scan fix), same digest for every variant.""" + """PLAN_ENV.md, E1-05: the digest the channel's last release pushed, the + same one for every variant. + + **This moves with every base release**, and `bases.py` is where it moves + first: two releases (2026-09-15's and 2026-09-16's) changed the channel + and left this assertion on 2026-09-14's digest, so it sat red rather than + catching anything. Current: released 2026-09-16, the contract layer that + starts kernels in `/home/datalayer` (E1-05, Appendix B check 2). + """ ref = "datalayer/python-cpu" - digest = "sha256:334adf6c2714c8919ef60beeca1db12e3531a391c9dde41932c782f81c432b36" + digest = "sha256:122d3e31f5e2507251457cbf47871c39ac1753adb1d83777ab0743fa11cd6148" assert APPROVED_BASES[ref].channels == {"2026.09": dict.fromkeys(VARIANTS, digest)} assert resolve_base(ref, "2026.09", variant) == digest @@ -114,9 +132,13 @@ def test_each_base_is_published_under_its_own_repository() -> None: def test_the_2026_09_channel_of_python_cpu_pins_apt_to_its_snapshot() -> None: - """D-9: the moment just after the channel's image upgraded its packages.""" - assert channel_snapshot("datalayer/python-cpu", "2026.09") == "20260914T150000Z" - assert channel_snapshot("datalayer/python-cuda", "2026.09") == "" + """D-9: the moment just after the channel's image upgraded its packages. + + Moves with the channel, like the digest above, and had rotted the same + way — left on 2026-09-14's id after the channel moved to 2026-09-16's. + """ + assert channel_snapshot("datalayer/python-cpu", "2026.09") == "20260916T120000Z" + assert channel_snapshot("datalayer/python-cuda", "2026.09") == "20260918T150000Z" assert channel_snapshot("datalayer/nothing", "2026.09") == "" diff --git a/tests/test_environment_build_secrets.py b/tests/test_environment_build_secrets.py index e28cf0d..85393c0 100644 --- a/tests/test_environment_build_secrets.py +++ b/tests/test_environment_build_secrets.py @@ -18,8 +18,13 @@ IAM_API_KEY_VARIABLE, IAM_URL_VARIABLE, resolve_build_secret, + resolve_provider_credential, +) +from code_sandboxes.environments.errors import ( + BUILD_SECRET_UNAVAILABLE, + CAPABILITY_UNSUPPORTED, + EnvironmentsError, ) -from code_sandboxes.environments.errors import BUILD_SECRET_UNAVAILABLE, EnvironmentsError from code_sandboxes.environments.spec import BuildSecret SECRET = BuildSecret(id="dlsec_01J9BUILDSECRET0000000000", name="PIP_TOKEN") @@ -215,3 +220,114 @@ def handler(request: httpx.Request) -> httpx.Response: value = resolve_build_secret(SECRET, owner_uid=OWNER, transport=httpx.MockTransport(handler)) assert value == "tok-1" + + +# -- the owner's own provider credential (E2-01, D-8) ------------------------------------------- + + +def _credential_transport(handler) -> httpx.MockTransport: + return httpx.MockTransport(handler) + + +def test_the_credential_is_asked_for_by_variant_and_owner() -> None: + """A build knows whose it is and which variant it is building, never a secret id.""" + seen: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response( + 200, + json={"provider": "daytona", "value": {"DAYTONA_API_KEY": "k"}}, + ) + + secrets = resolve_provider_credential( + "daytona", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="worker-key", + transport=_credential_transport(handler), + ) + assert secrets == {"DAYTONA_API_KEY": "k"} + (request,) = seen + assert request.url.path == "/api/iam/v1/secrets/provider/daytona/value" + assert request.url.params["owner_uid"] == "owner-1" + # The worker's own key, never a person's token. + assert request.headers["X-API-Key"] == "worker-key" + assert "authorization" not in {name.lower() for name in request.headers} + + +def test_the_account_names_come_through_beside_the_key() -> None: + """`environments/accounts.py` fingerprints the account from these, so a + credential is more than the key that opens it.""" + + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"value": {"E2B_API_KEY": "k", "E2B_TEAM_ID": "team"}}) + + assert resolve_provider_credential( + "e2b", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="k", + transport=_credential_transport(handler), + ) == {"E2B_API_KEY": "k", "E2B_TEAM_ID": "team"} + + +def test_an_owner_with_no_credential_is_refused_by_name_and_not_retried() -> None: + """Never a fallback to whatever keys the worker holds: that is the one + thing D-8 exists to prevent.""" + + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(404, json={"detail": "No credential for that provider"}) + + with pytest.raises(EnvironmentsError) as raised: + resolve_provider_credential( + "modal", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="k", + transport=_credential_transport(handler), + ) + assert raised.value.code is CAPABILITY_UNSUPPORTED + assert raised.value.retryable is False + assert "their own account" in str(raised.value) + + +def test_iam_being_away_is_retryable() -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(503, json={"detail": "away"}) + + with pytest.raises(EnvironmentsError) as raised: + resolve_provider_credential( + "daytona", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="k", + transport=_credential_transport(handler), + ) + assert raised.value.retryable is True + + +def test_datalayer_is_not_a_variant_a_credential_is_kept_for() -> None: + """The platform's own builder uses the platform's own registry.""" + with pytest.raises(EnvironmentsError) as raised: + resolve_provider_credential( + "datalayer", owner_uid="owner-1", iam_url="https://iam.example", api_key="k" + ) + assert raised.value.code is CAPABILITY_UNSUPPORTED + + +def test_a_credential_that_names_nothing_is_refused() -> None: + for value in ({}, "a string", None, {"": "v"}): + + def handler(_request: httpx.Request, value=value) -> httpx.Response: + return httpx.Response(200, json={"value": value}) + + with pytest.raises(EnvironmentsError) as raised: + resolve_provider_credential( + "daytona", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="k", + transport=_credential_transport(handler), + ) + assert raised.value.code is CAPABILITY_UNSUPPORTED, value diff --git a/tests/test_environment_builders.py b/tests/test_environment_builders.py index 4596ecc..bf737ee 100644 --- a/tests/test_environment_builders.py +++ b/tests/test_environment_builders.py @@ -280,6 +280,58 @@ def test_a_file_is_not_baked_without_its_digest() -> None: files_step(environment(), variant="kaggle") +CONTENTS_SHA = "c" * 64 + + +def test_the_files_step_bakes_a_contents_build_manifest_from_an_external_source() -> None: + env = environment( + files=[], + contents_build=[ + { + "source": "https://data.example/iris.csv", + "path": "/opt/datalayer/contents/iris.csv", + "sha256": CONTENTS_SHA, + } + ], + ) + commands = files_step(env, variant="modal") + # The external source is fetched as it is (no contentRef to sign) and verified. + assert ( + "curl -fsSL https://data.example/iris.csv -o /opt/datalayer/contents/iris.csv" + in commands[0] + ) + assert f'echo "{CONTENTS_SHA} /opt/datalayer/contents/iris.csv" | sha256sum -c' in commands[0] + assert "environment-contents.json" in commands[-1] + + +def test_files_and_contents_build_are_baked_together() -> None: + env = environment( + contents_build=[ + { + "source": "https://data.example/iris.csv", + "path": "/opt/datalayer/contents/iris.csv", + "sha256": CONTENTS_SHA, + } + ] + ) + entries = build_entries(env, source_of=lambda entry: "https://signed.example/notes.md") + # The uploaded file first, then the external build entry — both baked, one engine. + assert [entry.destination_path for entry in entries] == [ + "/home/datalayer/content/notes.md", + "/opt/datalayer/contents/iris.csv", + ] + assert entries[0].source_uri == "https://signed.example/notes.md" + assert entries[1].source_uri == "https://data.example/iris.csv" + assert entries[1].sha256 == CONTENTS_SHA + + +def test_a_contents_build_entry_without_its_digest_will_not_parse() -> None: + with pytest.raises(EnvironmentsError) as raised: + environment(contents_build=[{"source": "https://data.example/x", "path": "/opt/x"}]) + assert raised.value.code.code == "DL_ENV_SPEC_INVALID" + assert "sha256" in raised.value.message + + def test_the_neutral_modules_import_no_provider_sdk() -> None: """What a service may import must not drag a provider in.""" code = ( diff --git a/tests/test_environment_conformance.py b/tests/test_environment_conformance.py index a0e8c30..49655d2 100644 --- a/tests/test_environment_conformance.py +++ b/tests/test_environment_conformance.py @@ -40,7 +40,7 @@ "gid": 100, "user": "datalayer", "home": "/home/datalayer", - "cwd": "/home/datalayer/content", + "cwd": "/home/datalayer", }, 3: {"version": "3.12", "full": "3.12.6", "pip": "/opt/conda/bin/pip", "uv": None}, 5: { @@ -282,6 +282,78 @@ def test_egress_and_gpu_are_judged_against_what_was_asked() -> None: assert "CUDA is 12.2, not 12.4" in by_id(result, 11).detail +def test_check_eleven_gates_a_gpu_version_that_cannot_see_its_gpu() -> None: + """E2-17: the GPU check is the one extended check that gates, and only + for a version that asked for an accelerator — a GPU version whose GPU is + not visible is not the version its spec describes.""" + sandbox = ScriptedSandbox({11: {"returncode": 0, "gpus": [], "cuda": None}}) + result = run_extended_tier(sandbox, accelerator_requested=True, cuda_version="12.4") + gpu = by_id(result, 11) + assert gpu.gating + assert not gpu.passed and not result.passed + assert "no GPU is visible" in gpu.detail + + +def test_check_eleven_gates_when_a_gpu_version_sees_its_gpu() -> None: + """The same version, its GPU and CUDA as the spec asked: the gating check + passes, so the extended tier passes.""" + sandbox = ScriptedSandbox() + result = run_extended_tier(sandbox, accelerator_requested=True, cuda_version="12.4") + gpu = by_id(result, 11) + assert gpu.gating and gpu.passed and result.passed + + +def test_check_eleven_reads_the_images_cuda_not_the_drivers() -> None: + """`nvidia-smi` says the newest CUDA the host's driver runs, which the image + does not choose: a 12.8 image on a 13.0 driver is what its spec asked for + (found on 2026-09-18, E2-17).""" + sandbox = ScriptedSandbox( + {11: {"returncode": 0, "gpus": ["H100"], "driver": "13.0", "cuda": "12.8"}} + ) + result = run_extended_tier(sandbox, accelerator_requested=True, cuda_version="12.8") + assert by_id(result, 11).passed, by_id(result, 11).detail + + +def test_check_eleven_refuses_a_driver_older_than_the_image() -> None: + sandbox = ScriptedSandbox( + {11: {"returncode": 0, "gpus": ["H100"], "driver": "12.4", "cuda": "12.8"}} + ) + gpu = by_id(run_extended_tier(sandbox, accelerator_requested=True), 11) + assert not gpu.passed + assert "the driver runs CUDA up to 12.4, older than the image's 12.8" in gpu.detail + + +def test_check_eleven_counts_the_gpus_asked_for() -> None: + sandbox = ScriptedSandbox({11: {"returncode": 0, "gpus": ["H100"], "cuda": "12.8"}}) + gpu = by_id(run_extended_tier(sandbox, accelerator_requested=True, accelerator_count=2), 11) + assert not gpu.passed + assert "1 GPU(s) visible, not the 2 asked for" in gpu.detail + + +def test_check_eleven_alone_is_what_a_smoke_test_adds() -> None: + from code_sandboxes.environments.conformance import run_accelerator_check + + passed = run_accelerator_check( + ScriptedSandbox({11: {"returncode": 0, "gpus": ["H100", "H100"], "cuda": "12.8.93"}}), + cuda="12.8", + count=2, + ) + assert passed.id == "conformance:11" and passed.gating and passed.passed + failed = run_accelerator_check( + ScriptedSandbox({11: {"returncode": 9, "gpus": []}}), cuda="12.8" + ) + assert failed.gating and not failed.passed + + +def test_a_cpu_version_never_gates_on_the_gpu_check() -> None: + """No accelerator asked for: check 11 is the trivial recorded pass, and + the extended tier still gates nothing.""" + result = run_extended_tier(ScriptedSandbox()) + gpu = by_id(result, 11) + assert not gpu.gating and gpu.passed + assert not any(item.gating for item in result.checks) + + def test_the_probes_run_for_real_in_a_local_sandbox( tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/test_environment_constraints.py b/tests/test_environment_constraints.py index 06dcef9..0aa1f0f 100644 --- a/tests/test_environment_constraints.py +++ b/tests/test_environment_constraints.py @@ -20,6 +20,7 @@ "jupyter-client", "jupyter-server", "jupyter-server-nbmodel", + "jupyter-kernels", "datalayer", } diff --git a/tests/test_environment_contract.py b/tests/test_environment_contract.py index b669e0a..452b9b1 100644 --- a/tests/test_environment_contract.py +++ b/tests/test_environment_contract.py @@ -16,11 +16,14 @@ from code_sandboxes.environments.contract import ( SANDBOX_CONTRACT_V1, SUPPORTED_CONTRACTS, + BuildContextEntry, + check_build_context, check_dockerfile, contract_markdown, get_contract, main, parse_dockerfile, + validate_build_context, validate_dockerfile, ) from code_sandboxes.environments.doctor.datalayer_sandbox import ROW_IDS @@ -35,7 +38,7 @@ def test_the_contract_carries_the_identity_the_owner_took() -> None: """PLAN_ENV.md, D-6: gid 100 is the one departure from section 3.""" contract = SANDBOX_CONTRACT_V1 assert (contract.user, contract.uid, contract.gid) == ("datalayer", 1000, 100) - assert (contract.home, contract.workdir) == ("/home/datalayer", "/home/datalayer/content") + assert (contract.home, contract.workdir) == ("/home/datalayer", "/home/datalayer") assert contract.reserved_path == "/opt/datalayer" assert contract.doctor_path == "/opt/datalayer/bin/datalayer-sandbox" assert SUPPORTED_CONTRACTS == ("sandbox-contract/v1",) @@ -134,6 +137,52 @@ def test_the_parser_joins_continuations_and_skips_comments_inside_them() -> None assert instructions[1].arguments == "apt-get update && apt-get install -y gdal-bin" +# -- The build context (E3-03) ------------------------------------------------- + + +def test_a_plain_build_context_passes() -> None: + entries = [ + BuildContextEntry("Dockerfile", 200), + BuildContextEntry("src/app.py", 1024), + BuildContextEntry("data/model.bin", 5 * 1024 * 1024), + ] + assert validate_build_context(entries) == [] + check_build_context(entries) + + +@pytest.mark.parametrize( + ("entry", "message"), + [ + (BuildContextEntry("/etc/passwd", 10), "is an absolute path, not a context path"), + (BuildContextEntry("../secret", 10), "escapes the context with `..`"), + (BuildContextEntry("a/../../secret", 10), "escapes the context with `..`"), + (BuildContextEntry("link", is_symlink=True), "is a symlink, which could read a host file"), + ( + BuildContextEntry("big.bin", 50 * 1024 * 1024 + 1), + "is over the 52428800-byte per-file limit", + ), + ], +) +def test_a_forbidden_context_member_is_refused(entry: BuildContextEntry, message: str) -> None: + findings = validate_build_context([entry]) + assert any(finding.message == message for finding in findings), findings + with pytest.raises(EnvironmentsError) as refused: + check_build_context([entry]) + assert refused.value.code is errors.SPEC_INVALID + + +def test_too_many_files_is_refused() -> None: + entries = [BuildContextEntry(f"f{index}", 1) for index in range(2001)] + findings = validate_build_context(entries) + assert any("more than 2000 files" in finding.message for finding in findings) + + +def test_too_many_bytes_in_all_is_refused() -> None: + entries = [BuildContextEntry(f"f{index}", 40 * 1024 * 1024) for index in range(3)] + findings = validate_build_context(entries) + assert any("total limit" in finding.message for finding in findings) + + @pytest.mark.parametrize( ("image", "approved"), [ diff --git a/tests/test_environment_datalayer_builder.py b/tests/test_environment_datalayer_builder.py index 5b92936..11b40b1 100644 --- a/tests/test_environment_datalayer_builder.py +++ b/tests/test_environment_datalayer_builder.py @@ -43,6 +43,29 @@ ) LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02): the `@EXPLICIT` marker, one conda package +#: URL, and the pip layer the solve resolved (the user's pip requirements and +#: the protected pins over them). +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-pip: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) + +#: The section 4.1 example as a conda `dependencyFile` source. +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's registry credential, as the workflow mints one (D-17).""" @@ -161,9 +184,12 @@ def a_builder(**changes) -> Builder: class Buildctl: """A `buildctl` that writes the metadata a successful build writes.""" - def __init__(self, *, digest: str | None = DIGEST, returncode: int = 0) -> None: + def __init__( + self, *, digest: str | None = DIGEST, returncode: int = 0, stderr: str = "solved\n" + ) -> None: self.digest = digest self.returncode = returncode + self.stderr = stderr self.argv: list[str] = [] self.context: Path | None = None self.env: dict[str, str] | None = None @@ -178,7 +204,7 @@ def __call__(self, argv, **kwargs) -> subprocess.CompletedProcess[str]: Path(self.argv[position + 1]).write_text( json.dumps({"containerimage.digest": self.digest}), encoding="utf-8" ) - return subprocess.CompletedProcess(self.argv, self.returncode, "", "solved\n") + return subprocess.CompletedProcess(self.argv, self.returncode, "", self.stderr) # -- The Dockerfile ------------------------------------------------------------ @@ -189,7 +215,7 @@ def test_it_is_what_the_section_4_1_example_builds(self) -> None: """The snapshot. Every line of it is a decision; read the diff, not just the failure.""" dockerfile = a_builder().dockerfile(a_request()) assert dockerfile == ( - "# syntax=docker/dockerfile:1.7\n" + "# syntax=docker/dockerfile:1.10\n" "# Generated by Datalayer for geospatial-analysis v3, build bld-1.\n" f"# Lock: {LOCK_DIGEST}\n" f"FROM {BASE}\n" @@ -212,10 +238,10 @@ def test_it_is_what_the_section_4_1_example_builds(self) -> None: "uv pip sync --system --require-hashes --find-links /opt/datalayer/wheelhouse " "/opt/datalayer/lock.txt\n" "USER 1000:100\n" - "WORKDIR /home/datalayer/content\n" + "WORKDIR /home/datalayer\n" "RUN --network=none python -c 'import geopandas'\n" "USER 1000:100\n" - "WORKDIR /home/datalayer/content\n" + "WORKDIR /home/datalayer\n" "RUN /opt/datalayer/bin/datalayer-sandbox doctor --json > /tmp/doctor.json\n" ) @@ -248,6 +274,31 @@ def test_it_installs_from_the_lock_with_hashes(self) -> None: # Never the loose list: that is the whole point of resolving once. assert "geopandas==1.1.1" not in dockerfile + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): the `@EXPLICIT` lock installs with a pinned + `micromamba` copied in first, and the pip layer the solve resolved + follows, so the kernel stack (E1-04) is present the same.""" + request = a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC) + dockerfile = a_builder().dockerfile(request) + # Into the base's own interpreter, by name: left to itself micromamba + # made a new `base` under the content home, where `python` never looks + # (found on the first real conda build, 2026-09-18). + assert ( + "micromamba install --yes --root-prefix /opt/conda --prefix /opt/conda " + "--file /opt/datalayer/lock.txt" + ) in dockerfile + assert "--name base" not in dockerfile + # micromamba is copied in from its pinned image before it is invoked. + bootstrap = dockerfile.index("COPY --from=mambaorg/micromamba") + micromamba = dockerfile.index("micromamba install") + assert bootstrap < micromamba + # The header's own pip layer, installed with pip after the conda layer. + pip = dockerfile.index("uv pip install --system") + assert micromamba < pip + assert "ipykernel==7.3.0" in dockerfile + # A conda source never runs the pip-lock `uv pip sync`. + assert "uv pip sync" not in dockerfile + def test_apt_installs_from_the_snapshot_the_lock_pinned_it_at(self) -> None: """A pinned version can leave the live mirror: the build installs from the snapshot the resolver pinned against (D-9).""" @@ -278,6 +329,33 @@ def test_a_build_secret_is_mounted_for_the_command_that_names_it_alone(self) -> assert not any(line.startswith(("ARG", "ENV")) and "PIP_TOKEN" in line for line in lines) assert sum(1 for line in lines if "dlsec_01J9BUILDSECRET0000000000" in line) == 1 + def test_the_frontend_is_new_enough_for_the_mount_it_emits(self) -> None: + """The `env=` mount and the `# syntax=` pin have to agree. + + `--mount=type=secret,env=NAME` does not exist before frontend 1.10: + 1.7, which this pinned for its first year, answers `unexpected key + 'env' in 'env=NAME'` and fails the whole build. Nothing caught it + because no build had ever carried a build secret — the first one that + did, on r1 on 2026-09-17, died on exactly that. + + Asserted against the rendered Dockerfile rather than the constant, so + lowering the pin fails here whatever else is refactored. + """ + from code_sandboxes.environments.adapters.datalayer import ( + SECRET_ENV_FRONTEND, + ) + + request = a_request(spec=A_SECRET, build_secret_ids=("dlsec_01J9BUILDSECRET0000000000",)) + dockerfile = a_builder().dockerfile(request) + assert "env=PIP_TOKEN" in dockerfile, "this test needs an env-mounted secret" + syntax = dockerfile.splitlines()[0] + assert syntax.startswith("# syntax=docker/dockerfile:") + pinned = tuple(int(part) for part in syntax.split(":")[-1].split(".")[:2]) + assert pinned >= SECRET_ENV_FRONTEND, ( + f"frontend {syntax} has no `env=` secret mount; it needs at least " + f"{'.'.join(str(n) for n in SECRET_ENV_FRONTEND)}" + ) + def test_a_file_mounted_build_secret_targets_run_secrets(self) -> None: request = a_request( spec={ @@ -354,11 +432,6 @@ def test_a_gpu_class_points_at_the_providers_that_run_one(self) -> None: assert any("modal or daytona" in finding.message for finding in report.findings) assert all(finding.field == "spec.resources.sizeClass" for finding in report.findings) - def test_a_form_that_is_not_built_yet_is_a_finding(self) -> None: - request = a_request(spec={"build": {"source": "dockerfile"}}) - report = a_builder().validate(request.environment) - assert [finding.field for finding in report.findings] == ["spec.build.source"] - def test_a_dependency_file_source_is_supported(self) -> None: """E3-01 resolves it the same way `packages` does; this builder never refused it on its own — until this box, its own `validate` still did.""" @@ -400,7 +473,7 @@ def test_the_section_4_1_example_is_supported(self) -> None: def test_this_variant_has_no_gpu_yet(self) -> None: capabilities = a_builder().capabilities() assert capabilities.supports_gpu is False - assert capabilities.build_sources == ("packages", "dependencyFile", "image") + assert capabilities.build_sources == ("packages", "dependencyFile", "image", "dockerfile") assert capabilities.package_managers == ("uv", "pip") @@ -656,6 +729,27 @@ def test_a_partial_set_of_tls_options_is_treated_as_none(self) -> None: a_builder(run=buildctl, tlscert="/certs/client/tls.crt").build(a_request()) assert not any(arg.startswith("--tls") for arg in buildctl.argv) + def test_the_steps_go_through_the_pools_proxy(self, monkeypatch) -> None: + """The build pool lets a step reach the network only through its + proxy (E1-06), so every step is handed it, in both spellings.""" + monkeypatch.setenv("DATALAYER_BUILDKIT_PROXY", "http://127.0.0.1:3128") + buildctl = Buildctl() + a_builder(run=buildctl).build(a_request()) + for name in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"): + assert f"build-arg:{name}=http://127.0.0.1:3128" in buildctl.argv + assert "build-arg:NO_PROXY=127.0.0.1,localhost" in buildctl.argv + + def test_no_proxy_when_none_is_set(self, monkeypatch) -> None: + """A `buildkitd` whose steps reach the network directly needs none.""" + monkeypatch.delenv("DATALAYER_BUILDKIT_PROXY", raising=False) + buildctl = Buildctl() + a_builder(run=buildctl).build(a_request()) + assert not any("proxy" in arg.lower() for arg in buildctl.argv) + + def test_a_proxy_that_is_not_http_host_port_is_refused(self) -> None: + with pytest.raises(ValueError, match="DATALAYER_BUILDKIT_PROXY"): + a_builder(proxy="socks5://127.0.0.1:1080") + def test_the_docker_config_does_not_outlive_the_build(self) -> None: """A registry password base64'd into a file the worker keeps forever is a credential leak on disk, whichever way the build ends (found on diff --git a/tests/test_environment_daytona_builder.py b/tests/test_environment_daytona_builder.py index 3ac05ab..a5cbb53 100644 --- a/tests/test_environment_daytona_builder.py +++ b/tests/test_environment_daytona_builder.py @@ -17,13 +17,14 @@ from __future__ import annotations +import enum from datetime import datetime, timezone from pathlib import Path from typing import Any, ClassVar import pytest -from code_sandboxes.environments.adapters.daytona import Builder +from code_sandboxes.environments.adapters.daytona import DAYTONA_GPUS, Builder, daytona_gpu from code_sandboxes.environments.builders import ArtifactReference, BuildRequest from code_sandboxes.environments.errors import ( ARTIFACT_MISSING, @@ -45,6 +46,25 @@ APT_LOCK = LOCK + "# datalayer-apt: gdal-bin=3.8.4+dfsg-3build2\n" LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02) and its `dependencyFile` source. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-pip: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's owner secrets, as the workflow mints them (D-8, D-17, E2-01).""" @@ -95,6 +115,11 @@ def __init__(self, calls: list[Call]) -> None: def base(cls, ref: str) -> FakeImage: return cls([Call("base", (ref,))]) + @classmethod + def from_dockerfile(cls, path: str) -> FakeImage: + """The text is read now: the builder's scratch file is gone by the time a test looks.""" + return cls([Call("from_dockerfile", (path,), {"content": Path(path).read_text()})]) + def env(self, env_vars: dict[str, str]) -> FakeImage: self.calls.append(Call("env", (env_vars,))) return self @@ -138,6 +163,10 @@ def __init__( self.gpu_type = gpu_type +#: The SDK's `GpuType`, by value, as the builder asks for one. +FakeGpuType = enum.Enum("GpuType", {name.replace("-", "_"): name for name in DAYTONA_GPUS}) + + class FakeCreateSnapshotParams: def __init__( self, @@ -189,9 +218,12 @@ def __init__( create_error: Exception | None = None, get_results: dict[str, FakeSnapshot] | None = None, get_errors: dict[str, Exception] | None = None, + delete_errors: dict[str, Exception] | None = None, ) -> None: self.create_calls: list[Call] = [] self.get_calls: list[Call] = [] + self.delete_calls: list[Call] = [] + self._delete_errors = delete_errors or {} self._create_result = create_result self._create_error = create_error self._get_results = get_results or {} @@ -217,6 +249,15 @@ def get(self, name_or_id: str) -> FakeSnapshot: return self._get_results[name_or_id] raise FakeDaytonaNotFoundError(f"no such snapshot {name_or_id}") + def delete(self, snapshot: Any) -> None: + """As the SDK's: an id or a name, and a missing one is not found.""" + self.delete_calls.append(Call("delete", (snapshot,))) + if snapshot in self._delete_errors: + raise self._delete_errors[snapshot] + if snapshot not in self._get_results: + raise FakeDaytonaNotFoundError(f"no such snapshot {snapshot}") + del self._get_results[snapshot] + class FakeDaytonaClient: def __init__(self, *, snapshot_service: FakeSnapshotService | None = None) -> None: @@ -247,6 +288,7 @@ def __init__(self, *, client: FakeDaytonaClient | None = None) -> None: self.DaytonaNotFoundError = FakeDaytonaNotFoundError self.Image = FakeImage self.Resources = FakeResources + self.GpuType = FakeGpuType self.CreateSnapshotParams = FakeCreateSnapshotParams self.DaytonaConfig = FakeDaytonaConfig self.Daytona = _DaytonaFactory(self) @@ -459,6 +501,24 @@ def test_uv_pip_sync_reaches_the_bases_own_shared_wheelhouse(self) -> None: assert "--require-hashes" in sync.args[0] assert "--find-links /opt/datalayer/wheelhouse" in sync.args[0] + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): micromamba is bootstrapped, `micromamba + install --file` reads the `@EXPLICIT` lock, and the pip layer the solve + resolved follows — never the pip-lock `uv pip sync`.""" + daytona = FakeDaytonaModule() + a_builder(daytona=daytona).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + image = daytona.client.snapshot.create_calls[0].args[0].image + runs = calls_named(image, "run_commands") + bootstrap = next(i for i, call in enumerate(runs) if "micro.mamba.pm" in call.args[0]) + micromamba = next(i for i, call in enumerate(runs) if "micromamba install" in call.args[0]) + pip = next(i for i, call in enumerate(runs) if "ipykernel==7.3.0" in call.args[0]) + assert bootstrap < micromamba < pip + assert not any("uv pip sync" in call.args[0] for call in runs) + # The interpreter a sandbox runs, not a new `base` under the content + # home (2026-09-18): the same command the Datalayer builder emits. + assert "--root-prefix /opt/conda --prefix /opt/conda" in runs[micromamba].args[0] + assert "--name base" not in runs[micromamba].args[0] + def test_user_root_brackets_the_install_steps(self) -> None: """Daytona honours the base's `USER`, unlike E2B (E0-04): no synthetic account, just `USER root` around what needs it.""" @@ -513,7 +573,7 @@ def test_the_chain_ends_with_workdir(self) -> None: a_builder(daytona=daytona).build(a_request()) image = daytona.client.snapshot.create_calls[0].args[0].image assert image.calls[-1].name == "workdir" - assert image.calls[-1].args[0] == "/home/datalayer/content" + assert image.calls[-1].args[0] == "/home/datalayer" def test_the_entrypoint_is_always_set(self) -> None: """Daytona's own default, unset, is `sleep infinity` with no PID 1 (§11.3 @@ -535,12 +595,28 @@ def test_resources_come_from_the_size_class( resources = daytona.client.snapshot.create_calls[0].args[0].resources assert (resources.cpu, resources.memory, resources.disk) == (cpu, memory, disk) - def test_the_region_is_passed_through(self) -> None: + def test_the_region_sent_is_daytonas_own_not_this_platforms(self) -> None: + """`request.region` is Datalayer's — `r1` — and Daytona answered + "Region not found" for it on the first real build (2026-09-17). + + The owner names a Daytona region in `compatibility.regions`; that is + the one that scopes the snapshot. + """ daytona = FakeDaytonaModule() - a_builder(daytona=daytona).build(a_request(region="eu")) + a_builder(daytona=daytona).build( + a_request(region="r1", spec={"compatibility": {"regions": ["eu"]}}) + ) params = daytona.client.snapshot.create_calls[0].args[0] assert params.region_id == "eu" + def test_with_no_region_named_the_account_default_decides(self) -> None: + """The field is left out rather than filled with something Daytona + does not know — which is what every snapshot in a real account has.""" + daytona = FakeDaytonaModule() + a_builder(daytona=daytona).build(a_request(region="r1")) + params = daytona.client.snapshot.create_calls[0].args[0] + assert params.region_id is None + def test_the_snapshot_is_named_after_the_environment_and_version(self) -> None: daytona = FakeDaytonaModule() a_builder(daytona=daytona).build(a_request()) @@ -651,18 +727,68 @@ def test_a_registry_delete_failure_does_not_hide_a_successful_build(self) -> Non assert any("Could not delete the Daytona registry entry" in line for line in logged) -class TestWhatDaytonaCannotBuildYet: - def test_a_gpu_size_class_is_refused_at_build_time_naming_e2_17(self) -> None: +class TestAGpuSnapshot: + """E2-17: the spec's GPU is baked into the snapshot with its other resources.""" + + GPU: ClassVar[dict[str, Any]] = { + "sizeClass": "gpu-large", + "accelerator": {"type": "h100", "count": 2, "cuda": "12.8"}, + } + + def gpu_request(self, **resources: Any) -> BuildRequest: + return a_request( + spec={ + "base": {"ref": "datalayer/python-cuda", "channel": "2026.09"}, + "resources": {**self.GPU, **resources}, + }, + size_class="gpu-large", + ) + + def test_the_gpu_type_and_count_are_the_specs(self) -> None: daytona = FakeDaytonaModule() - registry = FakeRegistrySdk() - with pytest.raises(EnvironmentsError) as raised: - a_builder(daytona=daytona, registry=registry).build(a_request(size_class="gpu-large")) - assert raised.value.code.code == CAPABILITY_UNSUPPORTED.code - assert raised.value.detail["missing"] == "E2-17" - # Refused before any provider is touched: no registry, no snapshot. - assert registry.client.create_calls == [] - assert daytona.client.snapshot.create_calls == [] + a_builder(daytona=daytona).build(self.gpu_request()) + resources = daytona.client.snapshot.create_calls[0].args[0].resources + assert (resources.gpu, resources.gpu_type) == (2, FakeGpuType("H100")) + # D-4 gives a GPU class no CPU or memory: Daytona sizes the machine + # for the GPU, and the disk holds the CUDA base. + assert (resources.cpu, resources.memory, resources.disk) == (None, None, 50) + + def test_the_hints_size_the_rest_of_the_machine(self) -> None: + daytona = FakeDaytonaModule() + request = self.gpu_request(hints={"cpu": 8, "memoryGi": 64, "diskGi": 120}) + a_builder(daytona=daytona).build(request) + resources = daytona.client.snapshot.create_calls[0].args[0].resources + assert (resources.cpu, resources.memory, resources.disk, resources.gpu) == (8, 64, 120, 2) + + def test_a_gpu_daytona_does_not_offer_is_refused_before_any_build(self) -> None: + request = a_request( + spec={ + "base": {"ref": "datalayer/python-cuda", "channel": "2026.09"}, + "resources": {"sizeClass": "gpu-large", "accelerator": {"type": "A100-80GB"}}, + } + ) + report = a_builder().validate(request.environment) + assert report.supported is False + [finding] = [item for item in report.findings if "A100-80GB" in item.message] + assert finding.field == "spec.resources.accelerator.type" + assert all(name in finding.message for name in DAYTONA_GPUS) + + def test_a_name_is_read_the_way_people_write_it(self) -> None: + assert [daytona_gpu(name) for name in ("h100", "RTX_4090", " rtx-pro-6000 ", "T4")] == [ + "H100", + "RTX-4090", + "RTX-PRO-6000", + None, + ] + + def test_the_names_are_the_sdks(self) -> None: + """Spelled out because `validate` runs where the SDK may not be; held to it here.""" + sdk = pytest.importorskip("daytona") + offered = {g.value for g in sdk.GpuType if not g.value.lower().startswith("unknown")} + assert set(DAYTONA_GPUS) == offered + +class TestWhatDaytonaCannotBuildYet: def test_a_build_secret_is_refused_before_anything_is_queued(self) -> None: """Daytona has no per-step secret mechanism E0-04 could find (found in review: this chain consumed no build secret at all, and nothing @@ -784,3 +910,252 @@ def test_the_client_is_built_once_and_reused(self) -> None: builder.build(a_request()) builder.exists(an_artifact(provider_artifact_id="snp-999")) assert len(daytona.daytona_calls) == 1 + + +class TestCancellingABuild: + """E2-18: the build step lets a cancelled build's thread finish unheard, so + a snapshot Daytona goes on building would be recorded by nobody. Found by + E2-14's drill on 2026-09-18: `dl-backfill-drill-v2-…` kept building after + its build was cancelled.""" + + NAME = "dl-geospatial-analysis-v3-bld-1" + + def test_a_snapshot_being_made_is_deleted_by_id(self) -> None: + service = FakeSnapshotService( + get_results={ + self.NAME: FakeSnapshot(id="snp-9", name=self.NAME), + "snp-9": FakeSnapshot(id="snp-9", name=self.NAME), + } + ) + builder = a_builder( + daytona=FakeDaytonaModule(client=FakeDaytonaClient(snapshot_service=service)) + ) + builder.cancel(a_request()) + assert [call.args for call in service.get_calls] == [(self.NAME,)] + assert [call.args for call in service.delete_calls] == [("snp-9",)] + + def test_one_made_after_the_cancel_is_deleted_by_the_build(self) -> None: + """Not made yet when the cancel came: the build deletes it the moment + Daytona hands it back, and does not answer with it.""" + made = FakeSnapshot(id="snp-late", name=self.NAME) + service = FakeSnapshotService(create_result=made, get_results={"snp-late": made}) + builder = a_builder( + daytona=FakeDaytonaModule(client=FakeDaytonaClient(snapshot_service=service)) + ) + builder.cancel(a_request()) + assert service.delete_calls == [] + with pytest.raises(EnvironmentsError) as raised: + builder.build(a_request()) + assert raised.value.code.code == BUILD_FAILED.code + assert "cancelled" in raised.value.message + assert [call.args for call in service.delete_calls] == [("snp-late",)] + + def test_a_snapshot_daytona_refused_is_deleted_by_the_build(self) -> None: + """Daytona keeps a failed snapshot, in `error`, under the build's name: + one over its 20 GB limit was left there (E2-17, 2026-09-18).""" + failed = FakeSnapshot(id="snp-err", name=self.NAME, state="error") + service = FakeSnapshotService( + create_error=RuntimeError( + "Snapshot size (28.66GB) exceeds maximum allowed size of 20GB" + ), + get_results={self.NAME: failed, "snp-err": failed}, + ) + builder = a_builder( + daytona=FakeDaytonaModule(client=FakeDaytonaClient(snapshot_service=service)) + ) + with pytest.raises(EnvironmentsError) as raised: + builder.build(a_request()) + assert raised.value.code.code == BUILD_FAILED.code + assert "20GB" in raised.value.message + assert [call.args for call in service.delete_calls] == [("snp-err",)] + + def test_another_build_of_the_same_builder_is_not_touched(self) -> None: + builder = a_builder() + builder.cancel(a_request(build_uid="bld-other")) + assert builder.build(a_request()).provider_artifact_id + + +class TestDeletingASnapshot: + """E2-18: retention and a failed build both need a snapshot to go.""" + + def test_exists_is_false_after_delete(self) -> None: + service = FakeSnapshotService(get_results={"snp-123": FakeSnapshot(id="snp-123")}) + builder = a_builder( + daytona=FakeDaytonaModule(client=FakeDaytonaClient(snapshot_service=service)) + ) + artifact = an_artifact(provider_artifact_id="snp-123") + assert builder.exists(artifact) is True + builder.delete(artifact) + assert builder.exists(artifact) is False + + def test_it_deletes_by_id_never_by_the_name(self) -> None: + """A name is reused once its snapshot is deleted (E0-04), so deleting + by name could remove a later build's snapshot that inherited it.""" + service = FakeSnapshotService(get_results={"snp-123": FakeSnapshot(id="snp-123")}) + builder = a_builder( + daytona=FakeDaytonaModule(client=FakeDaytonaClient(snapshot_service=service)) + ) + builder.delete(an_artifact(provider_artifact_id="snp-123", mutable_alias="dl-geo-v3")) + assert [call.args for call in service.delete_calls] == [("snp-123",)] + + def test_deleting_what_is_already_gone_is_a_success(self) -> None: + """The collector deletes first and marks second (E1-17): a sweep + that died in between deletes again, and must not be refused for it.""" + service = FakeSnapshotService(get_results={"snp-123": FakeSnapshot(id="snp-123")}) + builder = a_builder( + daytona=FakeDaytonaModule(client=FakeDaytonaClient(snapshot_service=service)) + ) + artifact = an_artifact(provider_artifact_id="snp-123") + builder.delete(artifact) + builder.delete(artifact) + assert len(service.delete_calls) == 2 + + def test_any_other_failure_is_a_provider_error_and_not_a_success(self) -> None: + service = FakeSnapshotService( + get_results={"snp-123": FakeSnapshot(id="snp-123")}, + delete_errors={"snp-123": RuntimeError("the snapshot is in use")}, + ) + builder = a_builder( + daytona=FakeDaytonaModule(client=FakeDaytonaClient(snapshot_service=service)) + ) + with pytest.raises(EnvironmentsError) as raised: + builder.delete(an_artifact(provider_artifact_id="snp-123")) + assert raised.value.code.code == PROVIDER_ERROR.code + assert "in use" in raised.value.message + + +class TestSmokeTestingASnapshot: + """E2-04's own `Done when`: a sandbox launched from its id passes the core tier. + + It refused through `ManagedBuilder` until 2026-09-17, and the build + workflow calls this step — so no Daytona build could reach `succeeded`: + the snapshot was built and live at the provider, and the build was + recorded failed. + """ + + def _environment(self): + return parse_environment(a_request().environment.model_dump(by_alias=True)) + + def test_it_launches_by_id_and_runs_the_core_tier(self, monkeypatch) -> None: + """A Daytona sandbox record keeps the snapshot's *name*, and a name is + republished, so only the id says which artifact ran (E0-04).""" + made: dict = {} + ran: dict = {} + + class FakeSandbox: + def __init__(self, **kwargs): + made.update(kwargs) + self.events: list[str] = [] + + def start(self): + self.events.append("start") + + def stop(self): + self.events.append("stop") + + builder = a_builder() + monkeypatch.setattr( + "code_sandboxes.daytona_sandbox.DaytonaSandbox", FakeSandbox, raising=False + ) + monkeypatch.setattr( + "code_sandboxes.environments.conformance.run_core_tier", + lambda sandbox, **kwargs: ran.update(kwargs) or "the-result", + ) + + answer = builder.smoke_test( + an_artifact( + variant="daytona", + immutable_reference="snap-1", + provider_artifact_id="snap-1", + ), + environment=self._environment(), + lock_text="", + ) + assert answer == "the-result" + assert made["snapshot"] == "snap-1" + # A smoke test that leaves a sandbox running bills the owner for a check. + assert made["delete_on_stop"] is True + assert "restart" in ran and ran["python_version"] + + def test_a_gpu_version_also_passes_check_eleven(self, monkeypatch) -> None: + """E2-17: the core tier alone passes on a machine with no GPU, so a GPU + version's smoke test adds check 11, gating, with the spec's CUDA and count.""" + from code_sandboxes.environments.builders import CheckResult, ValidationResult + + asked: dict = {} + + class FakeSandbox: + def __init__(self, **_kwargs): + pass + + def start(self): + pass + + def stop(self): + pass + + monkeypatch.setattr( + "code_sandboxes.daytona_sandbox.DaytonaSandbox", FakeSandbox, raising=False + ) + monkeypatch.setattr( + "code_sandboxes.environments.conformance.run_core_tier", + lambda sandbox, **kwargs: ValidationResult(contract_version="sandbox-contract/v1"), + ) + monkeypatch.setattr( + "code_sandboxes.environments.conformance.run_accelerator_check", + lambda sandbox, **kwargs: asked.update(kwargs) + or CheckResult(id="conformance:11", name="gpu", passed=False, gating=True), + ) + request = TestAGpuSnapshot().gpu_request() + answer = a_builder().smoke_test( + an_artifact( + variant="daytona", immutable_reference="snap-gpu", provider_artifact_id="snap-gpu" + ), + environment=request.environment, + lock_text="", + ) + assert asked == {"cuda": "12.8", "count": 2} + assert [check.id for check in answer.checks] == ["conformance:11"] + assert answer.passed is False + + def test_the_sandbox_is_deleted_even_when_the_tier_raises(self, monkeypatch) -> None: + events: list[str] = [] + + class FakeSandbox: + def __init__(self, **_kwargs): + pass + + def start(self): + events.append("start") + + def stop(self): + events.append("stop") + + monkeypatch.setattr( + "code_sandboxes.daytona_sandbox.DaytonaSandbox", FakeSandbox, raising=False + ) + monkeypatch.setattr( + "code_sandboxes.environments.conformance.run_core_tier", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("the tier blew up")), + ) + with pytest.raises(EnvironmentsError): + a_builder().smoke_test( + an_artifact( + variant="daytona", immutable_reference="snap-1", provider_artifact_id="snap-1" + ), + environment=self._environment(), + lock_text="", + ) + assert events == ["start", "stop"] + + def test_without_a_spec_it_says_what_it_needs(self) -> None: + """The core tier asks for the Python version and the pinned packages, + and an artifact carries neither.""" + with pytest.raises(EnvironmentsError) as raised: + a_builder().smoke_test( + an_artifact( + variant="daytona", immutable_reference="snap-1", provider_artifact_id="snap-1" + ) + ) + assert raised.value.code is CAPABILITY_UNSUPPORTED + assert "needs the version's spec" in str(raised.value) diff --git a/tests/test_environment_dockerfile_source.py b/tests/test_environment_dockerfile_source.py new file mode 100644 index 0000000..b7feea1 --- /dev/null +++ b/tests/test_environment_dockerfile_source.py @@ -0,0 +1,229 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""A `dockerfile` source, from validate to both builders (PLAN_ENV.md E3-03). + +The author's Dockerfile names its base in `FROM`; that base, named by its +channel, is what the resolver pins and solves in, and what both builders pin +the `FROM` to. The contract's own lines come after the author's, and install +the lock rather than sync to it, so the author's own installs are kept. +""" + +from __future__ import annotations + +import pytest + +from code_sandboxes.environments.bases import ApprovedBase +from code_sandboxes.environments.contract import ( + dockerfile_base, + dockerfile_findings_for_build, + pin_dockerfile_base, +) +from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.resolve import COVERAGE_PREFIX, resolve_environment +from code_sandboxes.environments.spec import validate_environment + +from .test_environment_datalayer_builder import a_builder as a_datalayer_builder +from .test_environment_datalayer_builder import a_request as a_datalayer_request +from .test_environment_daytona_builder import FakeDaytonaModule, calls_named +from .test_environment_daytona_builder import a_builder as a_daytona_builder +from .test_environment_daytona_builder import a_request as a_daytona_request +from .test_environment_resolve import A_LOCK, BASES, RecordedRunner, a_spec + +PINNED = "registry.example/environments/base/python-cpu@sha256:" + "aa" * 32 + +AUTHORED = ( + "# syntax=docker/dockerfile:1\n" + "FROM datalayer/python-cpu:2026.09 AS build\n" + "RUN pip install --no-cache-dir rich==14.1.0\n" + "FROM build\n" + "COPY --from=build /etc/hostname /tmp/built-from\n" + "COPY < list[tuple[int, str]]: + return [(finding.line, finding.message) for finding in dockerfile_findings_for_build(text)] + + +class TestWhatIsBuiltFromADockerfile: + def test_a_channel_named_base_a_stage_a_heredoc_and_a_stage_copy_are_all_fine(self) -> None: + assert findings(AUTHORED) == [] + + @pytest.mark.parametrize( + ("line", "why"), + [ + ("FROM datalayer/python-cpu", "channel as its tag"), + ("FROM datalayer/python-cpu@sha256:" + "bb" * 32, "by its channel, not a digest"), + ("FROM datalayer/python-cpu:1999.01", "has no channel `1999.01`"), + ], + ids=["no-tag", "digest", "unknown-channel"], + ) + def test_the_base_must_be_named_by_one_of_its_channels(self, line: str, why: str) -> None: + [(number, message)] = findings(line + "\nRUN true\n") + assert number == 1 and why in message + + def test_two_different_approved_bases_are_refused_on_the_second(self) -> None: + text = "FROM datalayer/python-cpu:2026.09 AS a\nFROM datalayer/python-cpu:2026.10\n" + bases = { + "datalayer/python-cpu": ApprovedBase( + ref="datalayer/python-cpu", + python_versions=("3.13",), + channels={"2026.09": {"datalayer": "sha256:" + "11" * 32}, "2026.10": {}}, + ) + } + [finding] = dockerfile_findings_for_build(text, bases) + assert finding.line == 2 and "same base and channel" in finding.message + + @pytest.mark.parametrize("keyword", ["COPY", "ADD"]) + def test_a_file_from_the_build_context_is_refused_until_it_can_be_uploaded( + self, keyword: str + ) -> None: + [(number, message)] = findings( + f"FROM datalayer/python-cpu:2026.09\n{keyword} app.py /app\n" + ) + assert number == 2 and "`app.py` from the build context" in message + + def test_add_of_a_url_needs_no_context(self) -> None: + assert ( + findings("FROM datalayer/python-cpu:2026.09\nADD https://example.org/a.tgz /a\n") == [] + ) + + def test_a_different_escape_character_is_refused(self) -> None: + [(number, message)] = findings("# escape=`\nFROM datalayer/python-cpu:2026.09\n") + assert number == 1 and "escape" in message + + +class TestItsBase: + def test_is_the_approved_base_the_from_names(self) -> None: + base = dockerfile_base(AUTHORED) + assert (base.ref, base.channel, base.lines) == ("datalayer/python-cpu", "2026.09", (2,)) + + def test_is_refused_naming_the_line_when_it_cannot_be_pinned(self) -> None: + with pytest.raises(EnvironmentsError) as raised: + dockerfile_base("RUN true\nFROM datalayer/python-cpu\n") + assert raised.value.code.code == "DL_ENV_SPEC_INVALID" + assert raised.value.message.startswith("line 2:") + + def test_is_pinned_where_it_is_named_and_nowhere_else(self) -> None: + pinned = pin_dockerfile_base(AUTHORED, PINNED) + assert pinned.splitlines()[0] == f"FROM {PINNED} AS build" + # The stage stays a stage, and the author's own lines are untouched. + assert "FROM build\n" in pinned + assert "RUN pip install --no-cache-dir rich==14.1.0\n" in pinned + # The author's directive goes: the build states its own frontend first. + assert "# syntax=docker/dockerfile:1" not in pinned + + +def a_dockerfile_spec(content: str = AUTHORED) -> dict[str, object]: + return a_spec(build={"source": "dockerfile", "dockerfile": {"content": content}}) + + +class TestValidatingIt: + def test_a_buildable_dockerfile_validates(self) -> None: + validate_environment(a_dockerfile_spec()) + + def test_what_cannot_be_built_is_refused_at_validate_with_its_line(self) -> None: + with pytest.raises(EnvironmentsError) as raised: + validate_environment( + a_dockerfile_spec("FROM datalayer/python-cpu:2026.09\nCOPY a /a\n") + ) + assert "line 2" in raised.value.message + assert "build context" in raised.value.message + + +class TestResolvingIt: + def test_the_base_is_the_one_the_from_names_not_spec_base(self) -> None: + """`spec.base` is still required by the schema; the Dockerfile's + `FROM` is what the build starts from, so it is what is pinned.""" + spec = a_dockerfile_spec() + spec["spec"]["base"] = {"ref": "datalayer/python-cpu", "channel": "2026.10"} + runner = RecordedRunner(A_LOCK) + answer = resolve_environment(spec=spec, variants=["datalayer"], runner=runner, bases=BASES) + assert answer["resolved_bases"]["datalayer"].endswith("11" * 32) + assert runner.request is not None and runner.request.base_reference.endswith("11" * 32) + + def test_the_lock_says_what_it_does_not_cover(self) -> None: + answer = resolve_environment( + spec=a_dockerfile_spec(), + variants=["datalayer"], + runner=RecordedRunner(A_LOCK), + bases=BASES, + ) + [line] = [ + text for text in answer["content"].splitlines() if text.startswith(COVERAGE_PREFIX) + ] + assert "Dockerfile's own instructions install is not locked" in line + + def test_a_packages_lock_claims_no_such_thing(self) -> None: + answer = resolve_environment( + spec=a_spec(), variants=["datalayer"], runner=RecordedRunner(A_LOCK), bases=BASES + ) + assert COVERAGE_PREFIX not in answer["content"] + + +class TestTheDatalayerBuilder: + def request(self): + return a_datalayer_request( + spec={"build": {"source": "dockerfile", "dockerfile": {"content": AUTHORED}}} + ) + + def test_accepts_the_source(self) -> None: + assert "dockerfile" in a_datalayer_builder().capabilities().build_sources + + def test_its_own_validate_accepts_it_too(self) -> None: + """`validate` kept a list of its own, and refused the source on r1 + after the capability set above had learned it.""" + request = self.request() + report = a_datalayer_builder().validate(request.environment, lock_text=request.lock_text) + assert report.supported, [finding.message for finding in report.findings] + + def test_builds_from_the_authors_dockerfile_pinned_to_the_resolved_base(self) -> None: + request = self.request() + text = a_datalayer_builder().dockerfile(request) + assert f"FROM {request.resolved_base} AS build" in text + assert "RUN pip install --no-cache-dir rich==14.1.0" in text + # The author's lines come before the contract's own. + assert text.index("rich==14.1.0") < text.index("COPY lock.txt") + + def test_installs_the_lock_rather_than_syncing_the_authors_installs_away(self) -> None: + text = a_datalayer_builder().dockerfile(self.request()) + assert "uv pip install --system --require-hashes" in text + assert "-r /opt/datalayer/lock.txt" in text + assert "uv pip sync" not in text + + def test_a_packages_source_still_syncs(self) -> None: + assert "uv pip sync" in a_datalayer_builder().dockerfile(a_datalayer_request()) + + +class TestTheDaytonaBuilder: + def image(self, request): + daytona = FakeDaytonaModule() + a_daytona_builder(daytona=daytona).build(request) + return daytona.client.snapshot.create_calls[0].args[0].image + + def test_starts_from_the_authors_dockerfile_pinned_to_the_resolved_base(self) -> None: + request = a_daytona_request( + spec={"build": {"source": "dockerfile", "dockerfile": {"content": AUTHORED}}} + ) + image = self.image(request) + [start] = calls_named(image, "from_dockerfile") + assert f"FROM {request.resolved_base} AS build" in start.kwargs["content"] + assert not calls_named(image, "base") + + def test_installs_the_lock_rather_than_syncing_it(self) -> None: + request = a_daytona_request( + spec={"build": {"source": "dockerfile", "dockerfile": {"content": AUTHORED}}} + ) + runs = [call.args[0] for call in calls_named(self.image(request), "run_commands")] + assert any(run.startswith("uv pip install --system --require-hashes") for run in runs) + assert not any("uv pip sync" in run for run in runs) + + def test_a_packages_source_still_starts_from_the_base_and_syncs(self) -> None: + image = self.image(a_daytona_request()) + assert calls_named(image, "base") and not calls_named(image, "from_dockerfile") + runs = [call.args[0] for call in calls_named(image, "run_commands")] + assert any("uv pip sync" in run for run in runs) diff --git a/tests/test_environment_doctor.py b/tests/test_environment_doctor.py index 1455f54..df4623c 100644 --- a/tests/test_environment_doctor.py +++ b/tests/test_environment_doctor.py @@ -32,7 +32,7 @@ class CompliantHost(doctor.Host): "/home/datalayer/content", "/opt/datalayer", } - writable: ClassVar[set[str]] = {"/home/datalayer/content"} + writable: ClassVar[set[str]] = {"/home/datalayer"} def machine(self): return "x86_64" @@ -56,7 +56,7 @@ def environ(self): return self.environment def cwd(self): - return "/home/datalayer/content" + return "/home/datalayer" def which(self, name): return "/opt/conda/bin/" + name @@ -124,7 +124,7 @@ def test_a_compliant_sandbox_passes_every_row() -> None: assert report["ok"] is True and report["failed"] is None assert [row["id"] for row in report["rows"]] == list(doctor.ROW_IDS) assert report["paths"] == { - "writable": ["/home/datalayer/content"], + "writable": ["/home/datalayer"], "readOnly": ["/opt/datalayer"], } @@ -139,7 +139,7 @@ def test_the_image_before_the_move_fails_exactly_what_the_move_and_the_contract_ def test_a_writable_reserved_path_fails() -> None: class Writable(CompliantHost): - writable: ClassVar[set[str]] = {"/home/datalayer/content", "/opt/datalayer"} + writable: ClassVar[set[str]] = {"/home/datalayer", "/opt/datalayer"} report = doctor.check(Writable()) assert report["failed"] == "reserved_path" diff --git a/tests/test_environment_e2b_builder.py b/tests/test_environment_e2b_builder.py index ce93277..f17d25c 100644 --- a/tests/test_environment_e2b_builder.py +++ b/tests/test_environment_e2b_builder.py @@ -35,6 +35,25 @@ APT_LOCK = LOCK + "# datalayer-apt: gdal-bin=3.8.4+dfsg-3build2\n" LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02) and its `dependencyFile` source. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-pip: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's owner secrets, as the workflow mints them (D-8, E2-01).""" @@ -291,7 +310,7 @@ def test_user_and_workdir_are_set_last(self) -> None: names = [call.name for call in fake.calls] assert names[-2:] == ["set_user", "set_workdir"] assert fake.calls[-2].args == ("datalayer",) - assert fake.calls[-1].args == ("/home/datalayer/content",) + assert fake.calls[-1].args == ("/home/datalayer",) def test_env_is_set_when_the_spec_has_any(self) -> None: fake = FakeTemplate() @@ -390,6 +409,32 @@ def test_no_apt_step_when_the_lock_pins_none(self) -> None: a_builder(fake).build(a_request()) assert not any(call.name == "run_cmd" and "apt-get" in call.args[0] for call in fake.calls) + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): micromamba is bootstrapped, `micromamba + install --file` reads the `@EXPLICIT` lock, and the pip layer the solve + resolved follows — never the pip-lock `uv pip sync`.""" + fake = FakeTemplate() + a_builder(fake).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + bootstrap = next( + i + for i, call in enumerate(fake.calls) + if call.name == "run_cmd" and "micro.mamba.pm" in call.args[0] + ) + micromamba = next( + i + for i, call in enumerate(fake.calls) + if call.name == "run_cmd" and "micromamba install" in call.args[0] + ) + pip = next( + i + for i, call in enumerate(fake.calls) + if call.name == "run_cmd" and "ipykernel==7.3.0" in call.args[0] + ) + assert bootstrap < micromamba < pip + assert not any( + call.name == "run_cmd" and "uv pip sync" in call.args[0] for call in fake.calls + ) + def test_the_owners_credential_is_passed_to_the_sdk(self) -> None: """D-8: a build must run in the *environment owner's* team, never whichever team the worker process itself happens to be configured diff --git a/tests/test_environment_egress_refusals.py b/tests/test_environment_egress_refusals.py new file mode 100644 index 0000000..4577609 --- /dev/null +++ b/tests/test_environment_egress_refusals.py @@ -0,0 +1,146 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# Distributed under the terms of the Modified BSD License. + +"""A host the build pool's egress proxy refused, named as a finding (E2-19). + +The logs under `egress_logs/` are real: `buildctl --progress=plain` against +`moby/buildkit:v0.33.0`, its steps sent through the build pool chart's own +Squid (`ubuntu/squid:6.6`, the chart's `squid.conf`, allowing `pypi.org` +alone), on 2026-09-19. Only the image layers' progress lines are left out. +`micromamba.log` is `micromamba create` 2.3.3 through the same proxy. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.resolve import ( + egress_hosts_text, + egress_refused_hosts, + parse_resolver_failure, +) +from code_sandboxes.environments.resolve_conda import parse_conda_failure + +from .test_environment_datalayer_builder import Buildctl, a_builder, a_request + +LOGS = Path(__file__).parent / "egress_logs" + + +def a_log(name: str) -> str: + return (LOGS / name).read_text(encoding="utf-8") + + +class TestTheRefusedHostIsReadFromWhatEachToolWrites: + def test_pip_names_its_index_a_line_before_it_is_refused(self) -> None: + # pip's retries say `Tunnel connection failed: 403` with a path only; + # the host is the index it said it was looking in. + assert egress_refused_hosts(a_log("buildctl-pip.log")) == ["pypi.example-private.io"] + + def test_uv_names_the_url_four_lines_above_its_tunnel_error(self) -> None: + assert egress_refused_hosts(a_log("buildctl-uv.log")) == ["github.com"] + + def test_git_and_wget_are_named_and_curl_f_is_left_to_the_log(self) -> None: + # `curl -f` says only "returned error: 403", which a host answering + # 403 itself says too: not a finding. + assert egress_refused_hosts(a_log("buildctl-curl.log"), proxy="http://172.18.0.2:3128") == [ + "github.com" + ] + + def test_buildkitd_s_own_pull_of_a_base_is_named(self) -> None: + assert egress_refused_hosts(a_log("buildctl-base.log")) == ["ghcr.io"] + + def test_micromamba_names_the_channel_a_line_above(self) -> None: + # The issue tracker's URL after the refusal is not what was refused. + assert egress_refused_hosts(a_log("micromamba.log")) == ["conda.anaconda.org"] + + def test_a_log_with_no_refusal_names_nothing(self) -> None: + assert ( + egress_refused_hosts( + "#5 [2/2] RUN pip install requests\n#5 1.2 Successfully installed\n" + ) + == [] + ) + + def test_a_host_s_own_403_over_its_own_tls_is_not_a_refusal(self) -> None: + log = ( + "#6 [3/4] RUN curl -fsSL https://example.org/private -o /dev/null\n" + "#6 0.065 curl: (22) The requested URL returned error: 403\n" + ) + assert egress_refused_hosts(log) == [] + + def test_steps_side_by_side_are_read_apart(self) -> None: + # Step 8's URL is not step 7's refusal. + log = ( + "#7 [3/4] RUN wget https://blocked.example/a.tar.gz\n" + "#8 [4/4] RUN curl https://allowed.example/b\n" + "#8 0.1 ok\n" + "#7 0.067 Proxy tunneling failed: ForbiddenUnable to establish SSL connection.\n" + ) + assert egress_refused_hosts(log) == ["blocked.example"] + + def test_the_proxy_itself_is_never_a_refused_host(self) -> None: + log = ( + "#7 0.066 Connecting to proxy http://10.0.0.5:3128\n" + "#7 0.067 Proxy tunneling failed: Forbidden\n" + ) + assert egress_refused_hosts(log, proxy="http://10.0.0.5:3128") == [] + + def test_each_host_once_in_the_order_refused(self) -> None: + log = a_log("buildctl-uv.log") + a_log("buildctl-base.log") + a_log("buildctl-uv.log") + assert egress_refused_hosts(log) == ["github.com", "ghcr.io"] + + def test_the_hosts_read_as_a_sentence(self) -> None: + assert egress_hosts_text(["a.io"]) == "`a.io`" + assert egress_hosts_text(["a.io", "b.io", "c.io"]) == "`a.io`, `b.io` and `c.io`" + + +class TestARefusalIsAFindingOfTheFailure: + def test_a_build_refused_a_host_names_it_rather_than_the_log(self) -> None: + run = Buildctl(returncode=1, digest=None, stderr=a_log("buildctl-uv.log")) + with pytest.raises(EnvironmentsError) as raised: + a_builder(run=run).build(a_request()) + error = raised.value + assert error.code.code == "DL_ENV_BUILD_FAILED" + assert error.retryable is False + assert "`github.com`" in error.message + assert error.detail["refused_hosts"] == ["github.com"] + assert error.detail["findings"] == [{"kind": "egress_refused", "subject": "github.com"}] + + def test_a_build_that_failed_otherwise_still_says_to_read_the_log(self) -> None: + run = Buildctl( + returncode=1, digest=None, stderr=a_log("buildctl-curl.log").split("#7 [4/4]")[0] + ) + with pytest.raises(EnvironmentsError) as raised: + a_builder(run=run).build(a_request()) + # git's refusal in step 5 is in this log too: it is named. + assert raised.value.detail["refused_hosts"] == ["github.com"] + with pytest.raises(EnvironmentsError) as plain: + a_builder(run=Buildctl(returncode=1, digest=None, stderr="#5 0.1 exit 1\n")).build( + a_request() + ) + assert "findings" not in plain.value.detail + assert plain.value.message == "The build failed; its log says where" + + def test_a_resolve_refused_an_index_is_not_retried_as_an_outage(self) -> None: + error = parse_resolver_failure(a_log("buildctl-pip.log")) + assert error.code.code == "DL_ENV_PACKAGE_NOT_FOUND" + assert error.retryable is False + assert "`pypi.example-private.io`" in error.message + assert error.detail["findings"] == [ + {"kind": "egress_refused", "subject": "pypi.example-private.io"} + ] + + def test_a_conda_solve_refused_a_channel_names_it(self) -> None: + error = parse_conda_failure(a_log("micromamba.log")) + assert error.code.code == "DL_ENV_PACKAGE_NOT_FOUND" + assert error.retryable is False + assert error.detail["refused_hosts"] == ["conda.anaconda.org"] + + def test_an_unreachable_index_that_the_proxy_did_not_refuse_is_still_an_outage(self) -> None: + error = parse_resolver_failure( + "error: Failed to fetch: `https://pypi.org/simple/x/`\n Caused by: dns error\n" + ) + assert error.code.code == "DL_ENV_PROVIDER_ERROR" diff --git a/tests/test_environment_live_matrix.py b/tests/test_environment_live_matrix.py index 2637f20..d54acf6 100644 --- a/tests/test_environment_live_matrix.py +++ b/tests/test_environment_live_matrix.py @@ -145,13 +145,31 @@ def __init__(self, *, registry: str, password: str) -> None: class _ModalCredential: + """The shape durable's `_mint_credential` gives a Modal build: it reads + `aws_session` (D-18), never `username`/`password` — this used to put the + AWS keys there, a shape the worker never produces, so this test passed + while the worker's Modal builds failed their base pull. + + In production `aws_session` is always the `modal_base_reader` IAM user's + own real key: Modal's own `from_aws_ecr` never reads `AWS_SESSION_TOKEN` + anywhere in its SDK, so an assumed session fails the same way a wrong key + would ("The security token included in the request is invalid", found + live 2026-09-18). This double carries whatever AWS credential the person + running the live test has ambient — a session token is passed on if one + is present, but a live Modal run only actually succeeds against a real, + static key, the same requirement production has. + """ + def __init__(self) -> None: self.provider_secrets = { "MODAL_TOKEN_ID": os.environ.get("MODAL_TOKEN_ID", ""), "MODAL_TOKEN_SECRET": os.environ.get("MODAL_TOKEN_SECRET", ""), } - self.username = os.environ.get("AWS_ACCESS_KEY_ID", "") - self.password = os.environ.get("AWS_SECRET_ACCESS_KEY", "") + self.aws_session = { + name: os.environ[name] + for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN") + if os.environ.get(name) + } def _build_request(variant: str, resolved_base: str, lock_text: str) -> BuildRequest: @@ -225,25 +243,12 @@ def test_build_launch_and_the_core_tier(self, real_lock: tuple[str, str]) -> Non class TestModal: - """`xfail(strict=False)`: checks 5 (imports) and 6 (filesystem) fail — - confirmed live, 2026-09-13, the session driver refusing to start - against a sandbox that reports itself already shutting down, while 1, - 2, 3, 4, 7, 8 and 9 all pass. This used to be attributed to the - identity gap (checks 1/2 failed too, then) — closed since, live: a - contract-built artifact's `ModalSandbox` now drops its driver to - `1000:100` (`_start_driver`, gated on `image_id` so a plain - `ModalSandbox` is unaffected), and checks 1/2 pass with it. 5/6 turned - out to be a separate, still-unexplained issue: a plain, non-Environments - `ModalSandbox` runs several sequential snippets with no trouble at all - over the same span, so this is specific to a contract-built artifact's - image under repeated `exec`. An `XPASS` here means that's closed too.""" + """No `xfail` since 2026-09-18: a real artifact, built by the production + builder and launched by image id, passes all nine checks, check 9 included + with a secret to scan for, right after check 8. The break recorded under + E2-05 (the exec channel gone after check 8) no longer reproduces; the + kernel-restart fix of 1.9.28, check 7's own path, is the likeliest cause.""" - @pytest.mark.xfail( - reason="checks 5 and 6 fail on a session driver restart the sandbox refuses, " - "specific to a contract-built artifact and not yet root-caused (identity, " - "checks 1 and 2, is fixed)", - strict=False, - ) def test_build_launch_and_the_core_tier(self, real_lock: tuple[str, str]) -> None: _skip_unless_available("modal") import modal diff --git a/tests/test_environment_managed_builders.py b/tests/test_environment_managed_builders.py index 977e508..5752e0c 100644 --- a/tests/test_environment_managed_builders.py +++ b/tests/test_environment_managed_builders.py @@ -52,6 +52,26 @@ def environment(**spec: Any) -> Environment: return parse_environment(data) +#: A `dependencyFile` conda source: an `environment.yml` a managed variant +#: builds with `micromamba` (E3-02). +CONDA_ENVIRONMENT_YML = ( + "name: geo\nchannels: [conda-forge]\ndependencies:\n - python=3.13\n - gdal\n" +) + + +def a_conda_environment(**spec: Any) -> Environment: + return environment( + build={ + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": CONDA_ENVIRONMENT_YML, + }, + }, + **spec, + ) + + def messages(report: CapabilityReport) -> str: return " | ".join(finding.message for finding in report.findings) @@ -78,15 +98,33 @@ def test_e2b_has_no_gpu_and_the_other_two_do(self) -> None: assert get_builder("modal").capabilities().supports_gpu is True assert get_builder("daytona").capabilities().supports_gpu is True - def test_only_modal_forbids_instructions_its_builder_never_implemented(self) -> None: - """Modal implements its own Dockerfile builder; the others hand a - Dockerfile to BuildKit, which implements all of them.""" + def test_each_variant_forbids_what_its_own_builder_will_not_honour(self) -> None: + """Modal implements its own Dockerfile builder, and E2B parses a + Dockerfile into Template SDK calls; Daytona hands the text to a real + Docker builder, which implements all of them. + + E2B's list is read from its SDK (E3-03, 2026-09-17): + `e2b.template.dockerfile_parser` branches on FROM, RUN, COPY, ADD, + WORKDIR, USER, ENV, ARG, CMD and ENTRYPOINT, and for anything else + **prints `Unsupported instruction` and carries on** — so a template + built from a Dockerfile naming one of these comes back without it and + reports success. That is the case a capability report exists for. + """ assert set(get_builder("modal").capabilities().forbidden_instructions) == { "ONBUILD", "STOPSIGNAL", "VOLUME", } - assert get_builder("e2b").capabilities().forbidden_instructions == () + assert set(get_builder("e2b").capabilities().forbidden_instructions) == { + "VOLUME", + "EXPOSE", + "HEALTHCHECK", + "SHELL", + "ONBUILD", + "STOPSIGNAL", + "LABEL", + "MAINTAINER", + } assert get_builder("daytona").capabilities().forbidden_instructions == () def test_each_one_bounds_how_long_a_build_may_take(self) -> None: @@ -96,9 +134,13 @@ def test_each_one_bounds_how_long_a_build_may_take(self) -> None: seconds = get_builder(variant).capabilities().max_build_seconds assert seconds and 0 < seconds <= 60 * 60, variant - def test_this_phase_builds_a_package_list_and_nothing_else(self) -> None: + def test_this_phase_builds_a_package_list_a_conda_file_and_a_dockerfile(self) -> None: + """All three take a Dockerfile (E3-03), each through its own door: + E2B parses one into Template SDK calls, Daytona hands the text to a + real Docker builder, Modal builds it with its own frontend.""" for variant in MANAGED: - assert get_builder(variant).capabilities().build_sources == ("packages",) + sources = get_builder(variant).capabilities().build_sources + assert sources == ("packages", "dependencyFile", "dockerfile"), variant # -- what each one refuses ------------------------------------------------------ @@ -130,8 +172,9 @@ def test_an_accelerator_alone_is_the_same_ask(self) -> None: assert "spec.resources.accelerator" in fields(report) def test_a_gpu_spec_is_buildable_on_modal_and_daytona(self) -> None: - spec = {"sizeClass": "gpu-large", "accelerator": {"type": "A100", "cuda": "12.4"}} - for variant in ("modal", "daytona"): + """Each in its own GPU's name (E2-17): an A100 is Modal's, an H100 Daytona's.""" + for variant, gpu in (("modal", "A100"), ("daytona", "H100")): + spec = {"sizeClass": "gpu-large", "accelerator": {"type": gpu, "cuda": "12.8"}} report = get_builder(variant).validate(environment(resources=spec)) assert report.supported is True, f"{variant}: {messages(report)}" @@ -205,17 +248,72 @@ def test_other_variables_are_left_alone(self) -> None: def test_a_source_this_phase_does_not_build_is_refused_by_name(self) -> None: for variant in MANAGED: - report = get_builder(variant).validate(environment(build={"source": "dockerfile"})) + report = get_builder(variant).validate(environment(build={"source": "image"})) assert report.supported is False, variant - assert "`dockerfile` is not built for" in messages(report) - assert "it builds packages" in messages(report) + assert "`image` is not built for" in messages(report) + assert "it builds packages, dependencyFile, dockerfile" in messages(report) + + def test_an_instruction_a_variant_would_drop_is_refused_with_its_line(self) -> None: + """The case this exists for: E2B's parser prints `Unsupported + instruction` and carries on, so without this the template comes back + missing what the Dockerfile asked for and the build reports success.""" + dockerfile = "FROM datalayer/python-cpu:2026.09\nRUN true\nVOLUME /data\n" + report = get_builder("e2b").validate( + environment(build={"source": "dockerfile", "dockerfile": {"content": dockerfile}}) + ) + assert report.supported is False + assert "line 3: E2B does not implement `VOLUME`" in messages(report) + + def test_a_dockerfile_a_variant_can_honour_is_accepted(self) -> None: + """Daytona builds the text as it is, so Docker's own grammar is the limit.""" + dockerfile = "FROM datalayer/python-cpu:2026.09\nVOLUME /data\nEXPOSE 8888\n" + report = get_builder("daytona").validate( + environment(build={"source": "dockerfile", "dockerfile": {"content": dockerfile}}) + ) + assert report.supported is True, messages(report) - def test_conda_is_not_resolved_for_a_managed_variant_yet(self) -> None: + def test_a_conda_dependency_file_is_buildable_on_every_managed_variant(self) -> None: + for variant in MANAGED: + report = get_builder(variant).validate(a_conda_environment()) + assert report.supported is True, f"{variant}: {messages(report)}" + + def test_a_pyproject_dependency_file_is_not_built_for_a_managed_variant_yet(self) -> None: report = get_builder("e2b").validate( - environment(packages={"python": {"manager": "conda", "dependencies": ["numpy"]}}) + environment( + build={ + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "pyproject", + "content": "[project]\nname='x'\nversion='0'\n", + "lockContent": "# lock\n", + }, + } + ) ) assert report.supported is False - assert "`conda` is not resolved for E2B yet" in messages(report) + assert "a `pyproject` dependency file is not built for E2B yet" in messages(report) + assert "spec.build.dependencyFile.sourceFormat" in fields(report) + + @pytest.mark.parametrize( + "dependency_file", + [ + {"sourceFormat": "requirements", "content": "geopandas==1.1.1\n"}, + { + "sourceFormat": "pyproject", + "content": "[project]\nname='x'\nversion='0'\n", + "lockContent": "version = 1\n", + }, + ], + ids=["requirements", "pyproject"], + ) + def test_daytona_builds_a_pip_dependency_file(self, dependency_file: dict) -> None: + """Both resolve to the pip lock a `packages` list does (E3-01), and + Daytona's build installs that lock whatever wrote it. Refusing them + kept E3-08's first two examples and half of E3-09 off Daytona.""" + report = get_builder("daytona").validate( + environment(build={"source": "dependencyFile", "dependencyFile": dependency_file}) + ) + assert report.supported is True, messages(report) def test_e2b_and_daytona_refuse_a_build_secret_e0_04_found_no_mechanism_for(self) -> None: """E0-04's spike found only a registry login for the private base on @@ -273,14 +371,14 @@ class TestTheHalfThatIsNotBuiltYet: `ManagedBuilder` methods on all three.""" def test_modal_still_refuses_what_e2_05_did_not_build(self) -> None: - """`build`/`inspect`/`exists` are E2-05's; `smoke_test`, `resolve` and - `delete` are not — see test_environment_modal_builder.py for what - is built.""" + """`build`/`inspect`/`exists` are E2-05's, `delete` is too — it + collects the intermediate layers a build recorded (E2-05, E2-09) — + and so is `smoke_test`, since without it no Modal build could reach + `succeeded`. `resolve` is still not built — see + test_environment_modal_builder.py for what is.""" builder = get_builder("modal") calls = { - "smoke_test": lambda: builder.smoke_test(None), # type: ignore[arg-type] "resolve": lambda: builder.resolve("geo@1"), - "delete": lambda: builder.delete(None), # type: ignore[arg-type] } for operation, call in calls.items(): with pytest.raises(EnvironmentsError) as raised: @@ -303,14 +401,14 @@ def test_e2b_still_refuses_what_e2_03_did_not_build(self) -> None: assert raised.value.detail["operation"], f"e2b.{operation}" def test_daytona_still_refuses_what_e2_04_did_not_build(self) -> None: - """`build`/`inspect`/`exists` are E2-04's; `smoke_test`, `resolve` and - `delete` are not — see test_environment_daytona_builder.py for what - is built.""" + """`build`, `inspect`, `exists` and now `smoke_test` are E2-04's — + the last of them because this box's own `Done when` asks for "a + sandbox launched from its id passes the core tier", and until it was + built no Daytona build could reach `succeeded`. `delete` is E2-18's. + `resolve` is still not built — see test_environment_daytona_builder.py.""" builder = get_builder("daytona") calls = { - "smoke_test": lambda: builder.smoke_test(None), # type: ignore[arg-type] "resolve": lambda: builder.resolve("geo@1"), - "delete": lambda: builder.delete(None), # type: ignore[arg-type] } for operation, call in calls.items(): with pytest.raises(EnvironmentsError) as raised: @@ -375,8 +473,11 @@ def test_every_variant_validates_with_no_provider_sdk_installed() -> None: assert builder.capabilities().variant == variant report = builder.validate(environment()) assert report.supported is True, f"{variant}: {messages(report)}" + # A refusal each variant still makes, and makes without an SDK. + # `dockerfile` stopped being one when E3-03 gave all three a door + # into it, so a managed variant is asked about `image` instead. refused = get_builder(variant).validate( - environment(build={"source": "dockerfile"}) + environment(build={"source": "image"}) if variant != "datalayer" else environment( resources={"sizeClass": "gpu-small", "accelerator": {"type": "A10G"}} diff --git a/tests/test_environment_modal_builder.py b/tests/test_environment_modal_builder.py index ac8e90d..4493991 100644 --- a/tests/test_environment_modal_builder.py +++ b/tests/test_environment_modal_builder.py @@ -23,7 +23,7 @@ import pytest -from code_sandboxes.environments.adapters.modal import Builder +from code_sandboxes.environments.adapters.modal import MODAL_GPUS, Builder, modal_gpu from code_sandboxes.environments.builders import ArtifactReference, BuildRequest from code_sandboxes.environments.errors import ( ARTIFACT_MISSING, @@ -46,16 +46,52 @@ APT_LOCK = LOCK + "# datalayer-apt: gdal-bin=3.8.4+dfsg-3build2\n" LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02) and its `dependencyFile` source. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-pip: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: - """The build's owner secrets, as the workflow mints them (D-8, D-17, E2-01).""" + """The build's credential in exactly the shape durable's `_mint_credential` + returns for Modal (D-8, D-17, D-18, E2-01). + + `aws_session` is a *real* IAM user's key — `modal_base_reader`'s own, + read-only on the bases — never an assumed session's: Modal's own + `from_aws_ecr` never reads `AWS_SESSION_TOKEN` anywhere in its SDK, so a + session handed to it the way E2B and Daytona happily take one is exactly + "The security token included in the request is invalid" (found live, + 2026-09-18, after this double's own first version put IAM keys in + `username`/`password` instead and hid the same defect a first time). + `username`/`password` are unrelated here — the ECR login pair minted + from that same static key, which only E2B/Daytona and the resolver's own + pull read; the Modal adapter never touches them. + """ provider_secrets: ClassVar[dict[str, str]] = { "MODAL_TOKEN_ID": "owners-modal-token-id", "MODAL_TOKEN_SECRET": "owners-modal-token-secret", } - username: ClassVar[str] = "AKIA-owners-access-key" - password: ClassVar[str] = "owners-secret-key" + username: ClassVar[str] = "AWS" + password: ClassVar[str] = "ecr-login-token-from-the-static-key" + aws_session: ClassVar[dict[str, str]] = { + "AWS_ACCESS_KEY_ID": "AKIA-modal-base-reader", + "AWS_SECRET_ACCESS_KEY": "modal-base-reader-secret", + } class NoRegistryCredential: @@ -114,6 +150,14 @@ def run_commands(self, *commands: str, secrets: Any = None) -> FakeImage: self.calls.append(Call("run_commands", commands, kwargs)) return self + def micromamba_install(self, *, spec_file: str) -> FakeImage: + self.calls.append(Call("micromamba_install", (), {"spec_file": spec_file})) + return self + + def pip_install(self, *packages: str, find_links: str | None = None) -> FakeImage: + self.calls.append(Call("pip_install", packages, {"find_links": find_links})) + return self + def workdir(self, path: str) -> FakeImage: self.calls.append(Call("workdir", (path,))) return self @@ -122,6 +166,10 @@ def entrypoint(self, commands: list[str]) -> FakeImage: self.calls.append(Call("entrypoint", (commands,))) return self + def cmd(self, command: list[str]) -> FakeImage: + self.calls.append(Call("cmd", (command,))) + return self + def build(self, app: Any) -> FakeImage: self.calls.append(Call("build", (app,))) # Simulates Modal's own `enable_output()` printing straight to @@ -210,20 +258,41 @@ class FakeStub: `def`-ined, the same way `test_environment_daytona_builder.py` handles the same clash for `DockerRegistryApi`.""" - def __init__(self, *, delete_error: Exception | None = None) -> None: + def __init__( + self, + *, + delete_error: Exception | None = None, + image_delete_errors: dict[str, Exception] | None = None, + ) -> None: self.secret_delete_calls: list[Any] = [] self._delete_error = delete_error self.SecretDelete = self._secret_delete + #: Every image this stub was asked to delete, in order. + self.image_delete_calls: list[str] = [] + self._image_delete_errors = dict(image_delete_errors or {}) + self.ImageDelete = self._image_delete async def _secret_delete(self, request: Any) -> None: self.secret_delete_calls.append(request) if self._delete_error: raise self._delete_error + async def _image_delete(self, request: Any) -> None: + image_id = request.kwargs["image_id"] + self.image_delete_calls.append(image_id) + error = self._image_delete_errors.get(image_id) + if error: + raise error + class FakeClient: - def __init__(self, *, delete_error: Exception | None = None) -> None: - self.stub = FakeStub(delete_error=delete_error) + def __init__( + self, + *, + delete_error: Exception | None = None, + image_delete_errors: dict[str, Exception] | None = None, + ) -> None: + self.stub = FakeStub(delete_error=delete_error, image_delete_errors=image_delete_errors) class FakeClientFactory: @@ -311,10 +380,14 @@ class FakeApiPb2: def __init__(self) -> None: self.SecretDeleteRequest = self._secret_delete_request + self.ImageDeleteRequest = self._image_delete_request def _secret_delete_request(self, *, secret_id: str) -> Call: return Call("SecretDeleteRequest", (), {"secret_id": secret_id}) + def _image_delete_request(self, *, image_id: str) -> Call: + return Call("ImageDeleteRequest", (), {"image_id": image_id}) + class FakeSynchronizer: def wrap(self, fn: Any) -> Any: @@ -413,13 +486,25 @@ def test_it_pulls_the_base_through_ecr(self) -> None: [call] = modal.Image.from_aws_ecr_calls assert call.args[0] == BASE - def test_the_ecr_secret_carries_the_credential(self) -> None: + def test_the_ecr_secret_carries_the_base_reader_session(self) -> None: + """Modal takes the session itself — keys, token, region (D-18). + + Not the ECR login pair: `from_aws_ecr` calls AWS with what the secret + holds, and `AWS`/ as IAM keys is "The security token included + in the request is invalid" on the base pull. This test asserted that + exact mapping until 2026-09-18. + """ modal = FakeModalModule() a_builder(modal=modal).build(a_request()) [secret] = modal.Secret.from_dict_calls - assert secret.env_dict["AWS_ACCESS_KEY_ID"] == Credential.username - assert secret.env_dict["AWS_SECRET_ACCESS_KEY"] == Credential.password - assert secret.env_dict["AWS_REGION"] == "us-east-1" + assert secret.env_dict == { + "AWS_ACCESS_KEY_ID": Credential.aws_session["AWS_ACCESS_KEY_ID"], + "AWS_SECRET_ACCESS_KEY": Credential.aws_session["AWS_SECRET_ACCESS_KEY"], + "AWS_REGION": "us-east-1", + } + # No session token: this is a real key, not one it could carry. + assert "AWS_SESSION_TOKEN" not in secret.env_dict + assert Credential.password not in secret.env_dict.values() def test_no_secret_with_no_pull_credential(self) -> None: modal = FakeModalModule() @@ -488,6 +573,22 @@ def test_uv_pip_sync_reaches_the_bases_own_shared_wheelhouse(self) -> None: assert "--require-hashes" in sync assert "--find-links /opt/datalayer/wheelhouse" in sync + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): Modal's own `micromamba_install` reads the + `@EXPLICIT` lock, and `pip_install` layers the pip layer the solve + resolved — never the pip-lock `uv pip sync`.""" + modal = FakeModalModule() + a_builder(modal=modal).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + [image] = modal.Image.created + [mamba] = calls_named(image, "micromamba_install") + assert mamba.kwargs["spec_file"] == "/opt/datalayer/lock.txt" + [pip] = calls_named(image, "pip_install") + assert "ipykernel==7.3.0" in pip.args + mamba_at = image.calls.index(mamba) + pip_at = image.calls.index(pip) + assert mamba_at < pip_at + assert not run_commands_containing(image, "uv pip sync") + def test_no_user_line_is_ever_emitted(self) -> None: """Modal ignores `USER` entirely (found live): writing one would be dead code, so this builder never calls `dockerfile_commands` at all.""" @@ -504,14 +605,24 @@ def test_the_doctor_check_is_not_run_at_build_time(self) -> None: [image] = modal.Image.created assert not any("doctor" in call.args[0] for call in calls_named(image, "run_commands")) - def test_the_chain_ends_with_workdir_then_entrypoint(self) -> None: + def test_the_chain_ends_with_workdir_entrypoint_and_cmd(self) -> None: modal = FakeModalModule() a_builder(modal=modal).build(a_request()) [image] = modal.Image.created - # `build` is appended by `FakeImage.build` itself; the two before it - # are the chain's own last words. + # `build` is appended by `FakeImage.build` itself; the three before + # it are the chain's own last words. names = [call.name for call in image.calls] - assert names[-3:] == ["workdir", "entrypoint", "build"] + assert names[-4:] == ["workdir", "entrypoint", "cmd", "build"] + + def test_the_bases_jupyter_cmd_is_replaced_by_sleep(self) -> None: + """Modal keeps the base's `start-jupyter.sh` under a new ENTRYPOINT, + and a Jupyter server as the main process exits and ends the sandbox + (found live on 2026-09-18). The artifact says what holds it.""" + modal = FakeModalModule() + a_builder(modal=modal).build(a_request()) + [image] = modal.Image.created + [cmd_call] = calls_named(image, "cmd") + assert cmd_call.args[0] == ["sleep", "infinity"] def test_the_entrypoint_execs_its_arguments(self) -> None: """A bare script path, nothing for `entrypoint()`'s own Dockerfile @@ -651,16 +762,84 @@ def test_a_partially_hydrated_secret_is_still_cleaned_up_on_failure(self) -> Non assert deleted.kwargs["secret_id"] == secret.object_id -class TestWhatModalCannotBuildYet: - def test_a_gpu_size_class_is_refused_at_build_time_naming_e2_17(self) -> None: +class TestAGpuVersion: + """E2-17: on Modal a GPU is a launch option, not part of the image.""" + + GPU: ClassVar[dict[str, Any]] = { + "base": {"ref": "datalayer/python-cuda", "channel": "2026.09"}, + "resources": { + "sizeClass": "gpu-small", + "accelerator": {"type": "l4", "count": 2, "cuda": "12.8"}, + }, + } + + def test_it_builds_the_image_any_version_builds(self) -> None: modal = FakeModalModule() - with pytest.raises(EnvironmentsError) as raised: - a_builder(modal=modal).build(a_request(size_class="gpu-large")) - assert raised.value.code.code == CAPABILITY_UNSUPPORTED.code - assert raised.value.detail["missing"] == "E2-17" - # Refused before any provider is touched. - assert modal.Secret.from_dict_calls == [] - assert modal.Image.from_aws_ecr_calls == [] + artifact = a_builder(modal=modal).build(a_request(spec=self.GPU, size_class="gpu-small")) + assert artifact.provider_artifact_id.startswith("im-") + assert modal.Image.from_aws_ecr_calls, "built from the CUDA base like any image" + + def test_a_gpu_modal_does_not_offer_is_refused_before_any_build(self) -> None: + spec = { + **self.GPU, + "resources": {"sizeClass": "gpu-small", "accelerator": {"type": "RTX-4090"}}, + } + report = a_builder().validate(a_request(spec=spec).environment) + assert report.supported is False + [finding] = [item for item in report.findings if "RTX-4090" in item.message] + assert finding.field == "spec.resources.accelerator.type" + assert all(name in finding.message for name in MODAL_GPUS) + + def test_a_name_is_modal_s_gpu_argument(self) -> None: + assert [ + modal_gpu("t4"), + modal_gpu("a100_80gb"), + modal_gpu("H100", 2), + modal_gpu("RTX-4090"), + ] == [ + "T4", + "A100-80GB", + "H100:2", + None, + ] + + def test_the_smoke_test_runs_on_the_gpu_and_adds_check_eleven(self, monkeypatch) -> None: + from code_sandboxes.environments.builders import CheckResult, ValidationResult + + made: dict = {} + asked: dict = {} + + class FakeSandbox: + def __init__(self, **kwargs): + made.update(kwargs) + + def start(self): + pass + + def stop(self): + pass + + monkeypatch.setattr("code_sandboxes.modal_sandbox.ModalSandbox", FakeSandbox) + monkeypatch.setattr( + "code_sandboxes.environments.conformance.run_core_tier", + lambda sandbox, **kwargs: ValidationResult(contract_version="sandbox-contract/v1"), + ) + monkeypatch.setattr( + "code_sandboxes.environments.conformance.run_accelerator_check", + lambda sandbox, **kwargs: asked.update(kwargs) + or CheckResult(id="conformance:11", name="gpu", passed=True, gating=True), + ) + environment = a_request(spec=self.GPU, size_class="gpu-small").environment + artifact = ArtifactReference( + variant="modal", + immutable_reference="im-1", + provider_artifact_id="im-1", + contract_version="sandbox-contract/v1", + ) + answer = a_builder().smoke_test(artifact, environment=environment, lock_text=LOCK) + assert made["config"].gpu == "L4:2" + assert asked == {"cuda": "12.8", "count": 2} + assert [check.id for check in answer.checks] == ["conformance:11"] class TestABuildSecret: @@ -811,3 +990,215 @@ def test_an_authentication_failure_is_a_provider_error_not_a_raw_exception(self) with pytest.raises(EnvironmentsError) as raised: a_builder(modal=modal).exists(an_artifact(provider_artifact_id="im-abc123")) assert raised.value.code.code == PROVIDER_ERROR.code + + +class _Layer: + """One image in a chain, as `_intermediates_of` reads it.""" + + def __init__(self, *, object_id: Any = None, deps: Any = None) -> None: + self.object_id = object_id + self.deps = deps if deps is not None else (lambda: ()) + + +def _builder_with_client( + *, image_delete_errors: dict[str, Exception] | None = None +) -> tuple[Builder, FakeClient]: + """A builder whose client this test can read the delete calls back from.""" + client = FakeClient(image_delete_errors=image_delete_errors) + modal = FakeModalModule() + modal.Client = type( + "_C", + (), + { + "from_credentials": staticmethod(lambda *_: client), + "from_env": staticmethod(lambda: client), + }, + )() + return a_builder(modal=modal), client + + +# -- the layers a build leaves behind ----------------------------------------------------------- + + +class TestTheIntermediateLayers: + """What `build` records and `delete` collects (E2-05, E2-09). + + Each chained builder call leaves an image with an id of its own, and + deleting the artifact does not delete them. Modal offers no call that + lists an account's images — `ImageGetOrCreate`, `ImageFromId`, + `ImageGetByTag`, `ImageListTags`, `ImageTagRevisions`, `ImagePublish` and + `ImageDelete`, and nothing that enumerates — so an intermediate nobody + wrote down at build time can never be found again. + """ + + def test_every_layer_under_the_artifact_is_recorded_deepest_first(self) -> None: + from code_sandboxes.environments.adapters.modal import _intermediates_of + + base = _Layer(object_id="im-base", deps=lambda: ()) + middle = _Layer(object_id="im-middle", deps=lambda: (base,)) + built = _Layer(object_id="im-built", deps=lambda: (middle,)) + # The built image is the artifact, not an intermediate. + assert _intermediates_of(built) == ("im-base", "im-middle") + + def test_the_builds_ecr_secret_is_not_an_intermediate(self) -> None: + """It sits in `deps()` beside the layers, with an `st-` id `ImageDelete` + refuses as "not a valid Image ID" (found live, 2026-09-18).""" + from code_sandboxes.environments.adapters.modal import _intermediates_of + + secret = _Layer(object_id="st-ecr", deps=lambda: ()) + base = _Layer(object_id="im-base", deps=lambda: (secret,)) + built = _Layer(object_id="im-built", deps=lambda: (base,)) + assert _intermediates_of(built) == ("im-base",) + + def test_a_layer_with_no_id_is_not_recorded(self) -> None: + """Only a hydrated layer has an id worth writing down.""" + from code_sandboxes.environments.adapters.modal import _intermediates_of + + unbuilt = _Layer(object_id=None, deps=lambda: ()) + built = _Layer(object_id="im-built", deps=lambda: (unbuilt,)) + assert _intermediates_of(built) == () + + def test_a_chain_that_cannot_be_walked_is_no_layers_not_a_failure(self) -> None: + """A layer list is never worth failing a build over.""" + from code_sandboxes.environments.adapters.modal import _intermediates_of + + def _explode() -> Any: + raise RuntimeError("the SDK changed shape") + + assert _intermediates_of(_Layer(object_id="im-1", deps=_explode)) == () + + def test_delete_removes_the_intermediates_then_the_artifact(self) -> None: + """The artifact last: an intermediate is only reachable while the + record naming it survives.""" + builder, client = _builder_with_client() + builder.delete( + an_artifact( + intermediates=("im-base", "im-middle"), + provider_artifact_id="im-built", + immutable_reference="im-built", + ) + ) + assert client.stub.image_delete_calls == ["im-base", "im-middle", "im-built"] + + def test_a_layer_already_gone_is_success(self) -> None: + """A replay of a collection deletes the same set again with no harm.""" + builder, client = _builder_with_client( + image_delete_errors={"im-base": FakeNotFoundError("gone")} + ) + builder.delete( + an_artifact( + intermediates=("im-base", "im-middle"), + provider_artifact_id="im-built", + immutable_reference="im-built", + ) + ) + assert client.stub.image_delete_calls == ["im-base", "im-middle", "im-built"] + + def test_a_layer_that_is_not_ours_is_stepped_over(self) -> None: + """Modal's own `debian_slim` answers PermissionDenied, live on + 2026-09-17: an image somebody else owns was never ours to collect.""" + builder, client = _builder_with_client( + image_delete_errors={ + "im-base": RuntimeError("You don't have permission to modify Image 'im-base'") + } + ) + builder.delete( + an_artifact( + intermediates=("im-base",), + provider_artifact_id="im-built", + immutable_reference="im-built", + ) + ) + assert client.stub.image_delete_calls == ["im-base", "im-built"] + + def test_the_artifacts_own_refusal_is_raised(self) -> None: + """A layer's refusal is survivable; the artifact's is the whole point.""" + builder, _ = _builder_with_client( + image_delete_errors={"im-built": RuntimeError("modal is away")} + ) + with pytest.raises(EnvironmentsError) as raised: + builder.delete( + an_artifact(provider_artifact_id="im-built", immutable_reference="im-built") + ) + assert raised.value.code is PROVIDER_ERROR + + +class TestSmokeTestingAnImage: + """E2-05's own `Done when`: "the live test launches by image id and + passes the core tier". It refused through `ManagedBuilder`, and the + build workflow calls this step — so no Modal build could reach + `succeeded`, the state Daytona was in until 2026-09-17.""" + + def _environment(self): + return parse_environment(a_request().environment.model_dump(by_alias=True)) + + def _artifact(self) -> ArtifactReference: + return ArtifactReference( + variant="modal", + immutable_reference="im-1", + provider_artifact_id="im-1", + contract_version="sandbox-contract/v1", + ) + + def test_it_launches_by_image_id_as_the_owner_and_runs_the_core_tier(self, monkeypatch) -> None: + """By id, since a published name is mutable by design; and with the + owner's own client, so the sandbox runs in the workspace the image + is in (D-8) — not whatever token the worker happens to hold.""" + made: dict = {} + ran: dict = {} + modal = FakeModalModule() + + class FakeSandbox: + def __init__(self, **kwargs): + made.update(kwargs) + + def start(self): + pass + + def stop(self): + pass + + monkeypatch.setattr("code_sandboxes.modal_sandbox.ModalSandbox", FakeSandbox) + monkeypatch.setattr( + "code_sandboxes.environments.conformance.run_core_tier", + lambda sandbox, **kwargs: ran.update(kwargs) or "the-result", + ) + answer = a_builder(modal=modal).smoke_test( + self._artifact(), environment=self._environment(), lock_text=LOCK + ) + assert answer == "the-result" + assert made["image_id"] == "im-1" + assert made["client"] is modal.client + assert modal.Client.from_credentials_calls, "the owner's token, not the ambient one" + assert made["app_name"] == "dl-geospatial-analysis" + assert ran["python_version"] == "3.13" and "restart" in ran + + def test_the_sandbox_is_stopped_even_when_the_tier_raises(self, monkeypatch) -> None: + """A smoke test that leaves a sandbox running bills the owner for a check.""" + events: list[str] = [] + + class FakeSandbox: + def __init__(self, **_kwargs): + pass + + def start(self): + events.append("start") + + def stop(self): + events.append("stop") + + monkeypatch.setattr("code_sandboxes.modal_sandbox.ModalSandbox", FakeSandbox) + monkeypatch.setattr( + "code_sandboxes.environments.conformance.run_core_tier", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("the tier blew up")), + ) + with pytest.raises(EnvironmentsError) as raised: + a_builder().smoke_test(self._artifact(), environment=self._environment()) + assert raised.value.code is PROVIDER_ERROR + assert events == ["start", "stop"] + + def test_without_a_spec_it_says_what_it_needs(self) -> None: + with pytest.raises(EnvironmentsError) as raised: + a_builder().smoke_test(self._artifact()) + assert raised.value.code is CAPABILITY_UNSUPPORTED + assert "needs the version's spec" in str(raised.value) diff --git a/tests/test_environment_organization_policy.py b/tests/test_environment_organization_policy.py new file mode 100644 index 0000000..faabd5f --- /dev/null +++ b/tests/test_environment_organization_policy.py @@ -0,0 +1,165 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""An organization's own narrowing of what a build may do (PLAN_ENV.md E3-06). + +`EnvironmentsPolicy` and its reading from an organization's raw MCP policy +document — the shape `iam/datalayer_iam/services/mcp_policies.py` stores under +the policy's own `environments` key. The spec-level checks that read it +(bases, indexes, registries, packages) live with `spec_findings` in +`test_environment_spec.py`; the licence check lives with attestation in +`test_environment_attest.py`, since a licence is only known once the SBOM is. +This file is the shape and the parsing alone. +""" + +from __future__ import annotations + +import pytest + +from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.policy import ( + DEFAULT_POLICY, + UNRESTRICTED_POLICY, + EnvironmentsPolicy, + environments_policy_from_rules, + refuse_unlicensed, +) + + +class TestEnvironmentsPolicyFromRules: + """Reading the `environments` section of a stored MCP policy document.""" + + def test_no_rules_is_unrestricted(self) -> None: + assert environments_policy_from_rules(None) == UNRESTRICTED_POLICY + assert environments_policy_from_rules({}) == UNRESTRICTED_POLICY + + def test_something_that_is_not_an_object_is_unrestricted_too(self) -> None: + """A stored policy is trusted, not re-validated here (IAM already + did, at the write) — but a caller handing this the wrong shape by + accident must not be refused everything for it.""" + for rules in ("environments", 12, ["a", "list"]): + assert environments_policy_from_rules(rules) == UNRESTRICTED_POLICY # type: ignore[arg-type] + + def test_every_allowlist_is_read_by_its_camelcase_name(self) -> None: + rules = { + "allowedBases": ["datalayer/python-cpu"], + "allowedIndexes": ["https://pypi.org/simple"], + "allowedRegistries": ["docker.io"], + "allowedPackages": ["numpy", "pandas"], + "allowedLicenses": ["MIT", "Apache-2.0"], + } + policy = environments_policy_from_rules(rules) + assert policy.allowed_bases == ("datalayer/python-cpu",) + assert policy.allowed_indexes == ("https://pypi.org/simple",) + assert policy.allowed_registries == ("docker.io",) + assert policy.allowed_packages == ("numpy", "pandas") + assert policy.allowed_licenses == ("MIT", "Apache-2.0") + + def test_a_dimension_not_written_stays_unrestricted(self) -> None: + """One allowlist narrowed, the rest of the platform's own defaults.""" + policy = environments_policy_from_rules({"allowedLicenses": ["MIT"]}) + assert policy.allowed_licenses == ("MIT",) + assert policy.allowed_bases is None + assert policy.allowed_indexes is None + assert policy.allowed_registries is None + assert policy.allowed_packages is None + + def test_an_allowlist_written_empty_denies_everything_on_it(self) -> None: + """A real, if probably unintended, policy — not read the same as unset.""" + policy = environments_policy_from_rules({"allowedLicenses": []}) + assert policy.allowed_licenses == () + assert policy.allowed_licenses is not None + assert policy.allows("licenses", "MIT") is False + + def test_the_scan_threshold_narrows_the_platform_default(self) -> None: + policy = environments_policy_from_rules( + {"scan": {"blocksAt": "HIGH", "onlyFixable": False, "allowed": ["CVE-1"]}} + ) + assert policy.scan.blocks_at == "HIGH" + assert policy.scan.only_fixable is False + assert policy.scan.allowed == ("CVE-1",) + + def test_no_scan_section_keeps_the_platform_default(self) -> None: + policy = environments_policy_from_rules({"allowedLicenses": ["MIT"]}) + assert policy.scan == DEFAULT_POLICY + + def test_an_unrecognized_scan_threshold_falls_back_to_the_default(self) -> None: + """A stored value outside `SEVERITIES` is not trusted blindly — the + platform default is the honest reading of a value it cannot place.""" + policy = environments_policy_from_rules({"scan": {"blocksAt": "NOT-A-SEVERITY"}}) + assert policy.scan.blocks_at == DEFAULT_POLICY.blocks_at + + def test_the_version_is_carried_through_for_the_decision_record(self) -> None: + policy = environments_policy_from_rules({"allowedLicenses": ["MIT"]}, version=7) + assert policy.version == 7 + + +class TestAllows: + """`EnvironmentsPolicy.allows`: the one question every check asks it.""" + + def test_unrestricted_allows_anything(self) -> None: + assert UNRESTRICTED_POLICY.allows("bases", "anything") is True + assert UNRESTRICTED_POLICY.allows("licenses", "GPL-3.0") is True + + def test_a_value_on_the_list_is_allowed(self) -> None: + policy = EnvironmentsPolicy(allowed_bases=("datalayer/python-cpu",)) + assert policy.allows("bases", "datalayer/python-cpu") is True + + def test_a_value_off_the_list_is_not(self) -> None: + policy = EnvironmentsPolicy(allowed_bases=("datalayer/python-cpu",)) + assert policy.allows("bases", "datalayer/python-cuda") is False + + def test_the_comparison_is_exact_not_a_prefix_or_a_host(self) -> None: + """An index is compared the way `image_registry_allowed` already + compares a registry — the whole value, never a substring of it.""" + policy = EnvironmentsPolicy(allowed_indexes=("https://pypi.org/simple",)) + assert policy.allows("indexes", "https://pypi.org/simple/extra") is False + assert policy.allows("indexes", "pypi.org") is False + + +class TestRefuseUnlicensed: + """`refuse_unlicensed`: E3-06's own licence check, naming the package.""" + + def test_unrestricted_refuses_nothing(self) -> None: + refuse_unlicensed([("gpl-lib", "GPL-3.0")], UNRESTRICTED_POLICY) + + def test_the_default_argument_is_unrestricted_too(self) -> None: + """A caller passing no policy at all gets today's behaviour: nothing + about licences was ever refused before this box, and a bare call + must not start refusing by accident.""" + refuse_unlicensed([("gpl-lib", "GPL-3.0")]) + + def test_a_licence_off_the_allowlist_is_refused_naming_the_package(self) -> None: + policy = EnvironmentsPolicy(allowed_licenses=("MIT",)) + with pytest.raises(EnvironmentsError) as raised: + refuse_unlicensed([("gpl-lib", "GPL-3.0")], policy) + assert raised.value.code.code == "DL_ENV_POLICY_DENIED" + assert "gpl-lib" in raised.value.message + assert "GPL-3.0" in raised.value.message + assert raised.value.detail["package"] == "gpl-lib" + assert raised.value.detail["license"] == "GPL-3.0" + + def test_every_licence_on_the_allowlist_passes(self) -> None: + policy = EnvironmentsPolicy(allowed_licenses=("MIT", "Apache-2.0")) + refuse_unlicensed([("gdal", "MIT"), ("requests", "Apache-2.0")], policy) + + def test_the_first_offender_in_sbom_order_is_the_one_named(self) -> None: + """Deterministic for the same artifact: the SBOM's own order, not + sorted — a rebuild of the same lock names the same package first.""" + policy = EnvironmentsPolicy(allowed_licenses=("MIT",)) + with pytest.raises(EnvironmentsError) as raised: + refuse_unlicensed([("first-bad", "GPL-3.0"), ("second-bad", "AGPL-3.0")], policy) + assert raised.value.detail["package"] == "first-bad" + + def test_an_unnamed_package_is_still_refused_and_says_so(self) -> None: + policy = EnvironmentsPolicy(allowed_licenses=("MIT",)) + with pytest.raises(EnvironmentsError) as raised: + refuse_unlicensed([("", "GPL-3.0")], policy) + assert "an unnamed package" in raised.value.message + + def test_the_policy_version_is_carried_into_the_refusal(self) -> None: + policy = EnvironmentsPolicy(allowed_licenses=("MIT",), version=4) + with pytest.raises(EnvironmentsError) as raised: + refuse_unlicensed([("gpl-lib", "GPL-3.0")], policy) + assert raised.value.detail["policyVersion"] == 4 diff --git a/tests/test_environment_resolve.py b/tests/test_environment_resolve.py index 43af27e..2f8ed21 100644 --- a/tests/test_environment_resolve.py +++ b/tests/test_environment_resolve.py @@ -13,7 +13,6 @@ from __future__ import annotations import subprocess -from datetime import datetime, timezone import httpx import pytest @@ -141,10 +140,12 @@ def test_they_are_the_kernel_stack_at_one_version_each(self) -> None: "jupyter-client", "jupyter-server", "jupyter-server-nbmodel", + "jupyter-kernels", "datalayer", } assert all(pin.version for pin in pins.values()) assert pins["jupyter-server"].version == "2.21.0+datalayer.1" + assert pins["jupyter-kernels"].version == "1.2.23" def test_a_requirement_that_agrees_with_a_pin_is_dropped_for_it(self) -> None: # The fork satisfies `>=2.19`, which is what jupyterlab asks for, so @@ -264,7 +265,6 @@ def test_it_is_a_requirements_file_with_what_else_is_installed_in_comments(self) python_version="3.13", base_reference="environments/base/python-cpu@sha256:" + "11" * 32, merged=merge_requirements(["geopandas==1.1.1"]), - resolved_at=datetime(2026, 9, 12, 8, 30, tzinfo=timezone.utc), ) assert document["format"] == LOCK_FORMAT assert document["digest"].startswith("sha256:") @@ -273,7 +273,6 @@ def test_it_is_a_requirements_file_with_what_else_is_installed_in_comments(self) content = document["content"] assert f"{APT_PIN_PREFIX}gdal-bin=3.8.4+dfsg-3build2" in content assert "# datalayer-protected: ipykernel==7.3.0" in content - assert "# resolved-at: 2026-09-12T08:30:00+00:00" in content # The snapshot the pins came from, which the builder installs from. from code_sandboxes.environments.resolve import apt_pins_in, apt_snapshot_in @@ -287,15 +286,24 @@ def test_it_is_a_requirements_file_with_what_else_is_installed_in_comments(self) } def test_the_same_lock_digests_the_same_and_a_changed_one_does_not(self) -> None: + """And nothing here may pin a clock to make it true. + + The header carried a `# resolved-at:` line until 2026-09-16, and this + test passed only because it froze the moment. Resolving the same spec + twice on r1 produced two digests differing in that one line out of + 5,388, and section 5's cache key is over the lock digest — so D-12's + build cache had never hit, 12 lookups out of 12. + """ arguments = { "python_version": "3.13", "base_reference": "environments/base/python-cpu@sha256:" + "11" * 32, "merged": MergedRequirements((), (), ()), - "resolved_at": datetime(2026, 9, 12, tzinfo=timezone.utc), } first = lock_document(A_LOCK, **arguments) again = lock_document(A_LOCK, **arguments) assert first["digest"] == again["digest"] + assert first["content"] == again["content"] + assert "resolved-at" not in first["content"] moved = lock_document( ResolveOutcome(lock_text=A_LOCK.lock_text.replace("1.1.1", "1.1.2")), **arguments ) @@ -432,17 +440,6 @@ def test_a_refusal_from_the_solve_is_the_refusal_the_caller_gets(self) -> None: resolve_environment(spec=a_spec(), variants=["datalayer"], runner=runner, bases=BASES) assert raised.value.code.code == "DL_ENV_RESOLVE_CONFLICT" - def test_a_form_that_is_not_packages_is_refused_by_name(self) -> None: - with pytest.raises(EnvironmentsError) as raised: - resolve_environment( - spec=a_spec(build={"source": "dockerfile"}), - variants=["datalayer"], - runner=RecordedRunner(A_LOCK), - bases=BASES, - ) - assert raised.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" - assert raised.value.detail["source"] == "dockerfile" - def test_conda_waits_for_its_own_solver(self) -> None: spec = a_spec( packages={"python": {"manager": "conda", "dependencies": ["geopandas=1.1.1"]}} @@ -454,6 +451,54 @@ def test_conda_waits_for_its_own_solver(self) -> None: assert raised.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" assert raised.value.detail["manager"] == "conda" + def test_a_conda_dependency_file_uses_the_conda_runner_it_is_given(self) -> None: + """The main API forwards its `conda_runner` to the conda solve, so a + local or test solver reaches it the same way `runner` reaches pip.""" + from code_sandboxes.environments.resolve_conda import ( + CondaResolveOutcome, + CondaResolveRequest, + ) + + class RecordedConda: + name = "recorded-conda" + + def __init__(self, outcome: CondaResolveOutcome) -> None: + self._outcome = outcome + self.request: CondaResolveRequest | None = None + + def solve(self, request, log=None): # type: ignore[no-untyped-def] + self.request = request + return self._outcome + + conda_runner = RecordedConda( + CondaResolveOutcome( + lock_text=( + "# platform: linux-64\n@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2.conda#" + + "bb" * 32 + + "\n" + ) + ) + ) + spec = a_spec( + packages={}, + build={ + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "channels:\n - conda-forge\ndependencies:\n - gdal=3.9\n", + }, + }, + ) + resolve_environment( + spec=spec, + variants=["datalayer"], + conda_runner=conda_runner, + bases=BASES, + ) + assert conda_runner.request is not None + assert "gdal=3.9" in conda_runner.request.environment_yml + def test_the_credentials_registry_auth_reaches_the_runner(self) -> None: class Credential: def registry_auth(self) -> dict[str, str]: @@ -546,6 +591,19 @@ def test_the_same_list_produces_the_same_lock_as_packages(self) -> None: assert from_packages["content"] == from_file["content"] assert from_packages["digest"] == from_file["digest"] + def test_check_five_is_handed_what_the_file_names(self) -> None: + """Read from `packages.python`, which a file source leaves empty, it + was handed nothing and check 5 passed for having nothing to check + (found live on 2026-09-18, E3-08).""" + from code_sandboxes.environments.conformance import expected_packages + from code_sandboxes.environments.spec import validate_environment + + environment = validate_environment( + a_dependency_file_spec(content="geopandas==1.1.1 # inline\n-r other.txt\nsix>=1\n") + ) + lock = "geopandas==1.1.1\nsix==1.16.0\nshapely==2.1.2\n" + assert expected_packages(environment, lock) == {"geopandas": "1.1.1", "six": "1.16.0"} + # -- A pyproject.toml and its uv.lock (E3-01) ---------------------------------- @@ -561,9 +619,11 @@ class FakeUv: def __init__(self, *answers: tuple[int, str, str]) -> None: self.answers = list(answers) self.calls: list[list[str]] = [] + self.environments: list[dict[str, str]] = [] - def __call__(self, argv, **_kwargs): + def __call__(self, argv, **kwargs): self.calls.append(list(argv)) + self.environments.append(dict(kwargs.get("env") or {})) code, out, err = self.answers[min(len(self.calls), len(self.answers)) - 1] return subprocess.CompletedProcess(list(argv), code, out, err) @@ -595,10 +655,18 @@ def a_pyproject_spec(**dependency_file: object) -> dict[str, object]: "jupyter-client==8.9.1 \\\n --hash=sha256:" + "cc" * 32 + "\n" "jupyter-server==2.21.0+datalayer.1 \\\n --hash=sha256:" + "dd" * 32 + "\n" "jupyter-server-nbmodel==0.2.8 \\\n --hash=sha256:" + "ee" * 32 + "\n" + "jupyter-kernels==1.2.23 \\\n --hash=sha256:" + "a7" * 32 + "\n" "datalayer==1.7.4 \\\n --hash=sha256:" + "ff" * 32 + "\n" ) +#: Where a `pyproject` author locks Datalayer's `jupyter-server` fork from. +FORK_WHEEL_URL = ( + "https://github.com/datalayer-externals/jupyter-server/releases/download/" + "v2.21.0-datalayer.1/jupyter_server-2.21.0%2Bdatalayer.1-py3-none-any.whl" +) + + class TestAPyprojectFile: def test_a_current_lock_is_exported_rather_than_resolved(self) -> None: uv = FakeUv((0, "", "Resolved 2 packages in 1ms\n"), (0, EXPORTED, "")) @@ -610,7 +678,7 @@ def test_a_current_lock_is_exported_rather_than_resolved(self) -> None: pyproject_run=uv, ) assert answer["content"] == EXPORTED - assert answer["package_count"] == 6 + assert answer["package_count"] == 7 assert answer["python_version"] == "3.13" assert uv.calls[0][:2] == ["/usr/bin/uv", "lock"] assert uv.calls[1][:2] == ["/usr/bin/uv", "export"] @@ -745,6 +813,70 @@ def hangs(*_args, **kwargs): ) assert raised.value.code.code == "DL_ENV_PROVIDER_ERROR" + def test_check_five_is_handed_the_projects_own_dependencies(self) -> None: + """`[project].dependencies`, not every package the export pins: most of + a lock is transitive, and check 5 imports what it is handed.""" + from code_sandboxes.environments.conformance import expected_packages + from code_sandboxes.environments.spec import validate_environment + + environment = validate_environment(a_pyproject_spec()) + assert expected_packages(environment, EXPORTED) == {"six": "1.16.0"} + + def test_the_fork_brought_by_url_is_a_locked_protected_pin(self) -> None: + """No index serves `2.21.0+datalayer.1`, and `uv` hashes a wheel only + when it fetched it by URL: so a real author's export names the fork + `jupyter-server @ https://…whl`, with no `==` (E3-08). Its version is + in the wheel's name, and that is what the pin is checked against.""" + by_url = EXPORTED.replace( + "jupyter-server==2.21.0+datalayer.1 \\\n", + f"jupyter-server @ {FORK_WHEEL_URL} \\\n", + ) + uv = FakeUv((0, "", "Resolved 7 packages in 1ms\n"), (0, by_url, "")) + answer = resolve_environment( + spec=a_pyproject_spec(), + variants=["datalayer"], + bases=BASES, + uv="/usr/bin/uv", + pyproject_run=uv, + ) + assert answer["content"] == by_url + assert locked_versions(by_url)["jupyter-server"] == "2.21.0+datalayer.1" + + def test_uv_keeps_its_cache_in_the_scratch_directory_and_downloads_no_python(self) -> None: + """The durable worker's user has no home: `uv`'s default cache could not + be made, nor the interpreter it would download (E3-08).""" + uv = FakeUv((0, "", "Resolved 7 packages in 1ms\n"), (0, EXPORTED, "")) + resolve_environment( + spec=a_pyproject_spec(), + variants=["datalayer"], + bases=BASES, + uv="/usr/bin/uv", + pyproject_run=uv, + ) + assert len(uv.environments) == 2 + for environment in uv.environments: + assert environment["UV_PYTHON_DOWNLOADS"] == "never" + assert environment["UV_CACHE_DIR"].endswith("/.uv-cache") + assert "/dl-pyproject-" in environment["UV_CACHE_DIR"] + assert environment["PATH"] + + def test_a_url_that_is_not_that_distributions_wheel_pins_nothing(self) -> None: + assert ( + locked_versions( + "jupyter-server @ https://example.org/jupyter_server-2.21.0.tar.gz\n" + f"ipykernel @ {FORK_WHEEL_URL}\n" + "six @ https://example.org/not-a-wheel-name.whl\n" + ) + == {} + ) + + def test_a_pyproject_check_five_cannot_read_hands_it_nothing(self) -> None: + from code_sandboxes.environments.conformance import expected_packages + from code_sandboxes.environments.spec import validate_environment + + environment = validate_environment(a_pyproject_spec(content="[project\n")) + assert expected_packages(environment, EXPORTED) == {} + # -- An imported image (E3-04) ------------------------------------------------- @@ -947,8 +1079,12 @@ def test_the_solve_is_asked_to_pin_apt_at_the_base_channels_snapshot() -> None: runner = RecordedRunner(A_LOCK) resolve_environment(spec=a_spec(), variants=["datalayer"], runner=runner) assert runner.request is not None + # What this test is about: the solve is pinned at *the channel's* snapshot, + # whatever that is. The id itself is pinned once, in + # `test_environment_bases.py`, where it moves with the channel — repeating + # it here only meant a base release left two tests red instead of one. assert runner.request.apt_snapshot == channel_snapshot("datalayer/python-cpu", "2026.09") - assert runner.request.apt_snapshot == "20260914T150000Z" + assert runner.request.apt_snapshot class TestTheBuildkitRunner: @@ -1179,6 +1315,34 @@ def fake_run(command, **kwargs): ) assert not any(arg.startswith("--tls") for arg in seen["argv"]) + def test_the_solve_goes_through_the_pools_proxy(self, monkeypatch) -> None: + """The solve reaches the index the way a build does, through the + build pool's proxy (E1-06).""" + import subprocess as subprocess_module + + from code_sandboxes.environments import resolve as resolve_module + from code_sandboxes.environments.resolve import BuildkitResolveRunner + + seen: dict[str, list[str]] = {} + + def fake_run(command, **kwargs): + seen["argv"] = list(command) + return subprocess_module.CompletedProcess(command, 1, "", "boom") + + monkeypatch.setattr(resolve_module.subprocess, "run", fake_run) + with pytest.raises(EnvironmentsError): + BuildkitResolveRunner(buildctl="/usr/bin/true", proxy="http://127.0.0.1:3128").solve( + ResolveRequest( + python_version="3.13", + requirements=("ipykernel==7.3.0",), + constraints=(), + indexes=(), + base_reference="environments/base/python-cpu@sha256:" + "11" * 32, + ) + ) + assert "build-arg:HTTPS_PROXY=http://127.0.0.1:3128" in seen["argv"] + assert "build-arg:https_proxy=http://127.0.0.1:3128" in seen["argv"] + def test_it_refuses_a_base_that_is_not_pinned_by_digest(self) -> None: from code_sandboxes.environments.resolve import BuildkitResolveRunner diff --git a/tests/test_environment_resolve_conda.py b/tests/test_environment_resolve_conda.py new file mode 100644 index 0000000..72d4ee1 --- /dev/null +++ b/tests/test_environment_resolve_conda.py @@ -0,0 +1,587 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""Conda resolution: an environment.yml to one explicit lock (E3-02). + +Like the pip resolver's suite, micromamba's refusals below are **recorded** — +each is what ``micromamba`` actually writes for that input — so the parser is +tested against the solver's own words rather than a paraphrase of them. +""" + +from __future__ import annotations + +import subprocess + +import pytest + +from code_sandboxes.environments.bases import ApprovedBase +from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.resolve import protected_pins +from code_sandboxes.environments.resolve_conda import ( + CONDA_LOCK_FORMAT, + PIP_LAYER_SCRIPT, + BuildkitCondaResolveRunner, + CondaResolveOutcome, + CondaResolveRequest, + MicromambaResolveRunner, + conda_expected_packages, + conda_lock_document, + conda_lock_python_packages, + explicit_lock_packages, + merge_conda_pip, + parse_conda_environment, + parse_conda_failure, + pip_requirements_from_env_yaml, + pip_requirements_from_listing, + rendered_environment, + resolve_conda_environment, +) +from code_sandboxes.environments.spec import validate_environment + +# -- What micromamba wrote ---------------------------------------------------- + +#: An explicit lock, as ``micromamba env export --explicit`` writes one. +EXPLICIT_LOCK = """\ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/python-3.13.0.conda#{} +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2.conda#{} +""".format("aa" * 32, "bb" * 32) + +#: A package no channel serves. +CONDA_MISSING = """\ +critical libmamba Could not solve for environment specs +The following package could not be found: + - nothing provides no-such-conda-pkg-xyzzy needed by requested +""" + +#: An unsatisfiable set. +CONDA_CONFLICT = """\ +critical libmamba Could not solve for environment specs +The following packages are incompatible +encountered problems while solving: + - package gdal-3.9.2 requires libgdal 3.9.*, but none of the providers can be installed +""" + +#: Not a resolution failure at all — the channel is unreachable. +CONDA_UNREACHABLE = """\ +critical libmamba Download error (6) Could not resolve host: conda.anaconda.org +""" + +BASES = { + "datalayer/python-cpu": ApprovedBase( + ref="datalayer/python-cpu", + python_versions=("3.13",), + channels={ + "2026.09": {"datalayer": "sha256:" + "11" * 32, "modal": "sha256:" + "22" * 32}, + "2026.10": {}, + }, + ) +} + +A_YAML = """\ +name: geospatial +channels: + - conda-forge +dependencies: + - gdal=3.9 + - pip: + - shapely==2.0.6 +""" + + +def a_conda_spec(content: str = A_YAML, **spec: object) -> dict[str, object]: + """A section 4.1 example whose build source is a conda environment.yml.""" + return { + "apiVersion": "environments.datalayer.io/v1alpha1", + "kind": "Environment", + "metadata": {"name": "geospatial", "title": "Geospatial"}, + "spec": { + "language": {"name": "python", "version": "3.13"}, + "base": {"ref": "datalayer/python-cpu", "channel": "2026.09"}, + "build": { + "source": "dependencyFile", + "dependencyFile": {"sourceFormat": "conda", "content": content}, + }, + "resources": {"sizeClass": "medium"}, + "compatibility": {"variants": {"required": ["datalayer"], "optional": ["modal"]}}, + **spec, + }, + } + + +class RecordedRunner: + """A conda solve that answers what it was given, remembering the request.""" + + name = "recorded-conda" + + def __init__(self, outcome: CondaResolveOutcome | Exception) -> None: + self._outcome = outcome + self.request: CondaResolveRequest | None = None + + def solve(self, request: CondaResolveRequest, log=None) -> CondaResolveOutcome: + self.request = request + if log is not None: + log("solving conda") + if isinstance(self._outcome, Exception): + raise self._outcome + return self._outcome + + +# -- Reading the environment.yml --------------------------------------------- + + +class TestParsingTheEnvironmentYml: + def test_it_separates_the_conda_and_pip_layers(self) -> None: + env = parse_conda_environment(A_YAML) + assert env.channels == ("conda-forge",) + assert env.conda_dependencies == ("gdal=3.9",) + assert env.pip_dependencies == ("shapely==2.0.6",) + + def test_no_pip_section_is_an_empty_pip_layer(self) -> None: + env = parse_conda_environment("dependencies:\n - gdal=3.9\n") + assert env.conda_dependencies == ("gdal=3.9",) + assert env.pip_dependencies == () + + def test_malformed_yaml_is_refused_with_its_field(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment("dependencies: [\n") + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + assert caught.value.detail["field"] == "spec.build.dependencyFile.content" + + def test_a_document_that_is_not_a_mapping_is_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("- just\n- a\n- list\n") + + def test_missing_dependencies_is_refused(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment("name: env\nchannels: [conda-forge]\n") + assert "dependencies" in caught.value.message + + def test_two_pip_sections_are_refused(self) -> None: + text = "dependencies:\n - pip:\n - a\n - pip:\n - b\n" + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment(text) + assert "more than one" in caught.value.message + + def test_a_nested_list_entry_is_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("dependencies:\n - [nested]\n") + + def test_a_non_string_pip_entry_is_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("dependencies:\n - pip:\n - 3\n") + + def test_channels_that_are_not_a_list_are_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("channels: conda-forge\ndependencies:\n - gdal\n") + + def test_a_channel_url_carrying_a_credential_is_refused(self) -> None: + # The same refusal a credential-bearing pip index gets: the token + # belongs in the build's secrets, never verbatim in the spec. + text = ( + "channels:\n" + " - https://user:tok@conda.example.com/private\n" + "dependencies:\n - gdal\n" + ) + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment(text) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + assert "credential" in caught.value.message + assert caught.value.detail["field"].endswith("channels[0]") + + def test_a_plain_channel_url_without_a_credential_is_kept(self) -> None: + env = parse_conda_environment( + "channels:\n - https://conda.anaconda.org/conda-forge\n" "dependencies:\n - gdal\n" + ) + assert env.channels == ("https://conda.anaconda.org/conda-forge",) + + +class TestReadingThePipExport: + def test_it_reads_the_pip_section_in_order(self) -> None: + export = ( + "name: solved\n" + "channels:\n - conda-forge\n" + "dependencies:\n" + " - python=3.13\n" + " - gdal=3.9.2\n" + " - pip:\n" + " - shapely==2.0.6\n" + " - ipykernel==7.3.0\n" + ) + assert pip_requirements_from_env_yaml(export) == ( + "shapely==2.0.6", + "ipykernel==7.3.0", + ) + + def test_an_export_without_a_pip_section_is_an_empty_layer(self) -> None: + export = "dependencies:\n - python=3.13\n - gdal=3.9.2\n" + assert pip_requirements_from_env_yaml(export) == () + + def test_a_malformed_export_is_an_empty_layer_never_a_raise(self) -> None: + assert pip_requirements_from_env_yaml("dependencies: [\n") == () + assert pip_requirements_from_env_yaml("- just\n- a\n- list\n") == () + + +# -- Datalayer's pins over the pip layer ------------------------------------- + + +class TestMergingThePipLayer: + def test_the_protected_pins_are_forced_into_the_pip_layer(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + names = {req.split("==")[0] for req in merged.requirements} + assert {"shapely", "ipykernel", "jupyter-server"} <= names + + def test_a_pip_requirement_contradicting_a_pin_is_refused(self) -> None: + env = parse_conda_environment("dependencies:\n - pip:\n - ipykernel==6.0.0\n") + with pytest.raises(EnvironmentsError) as caught: + merge_conda_pip(env) + assert caught.value.code.code == "DL_ENV_PROTECTED_PACKAGE" + + def test_the_interpreter_is_pinned_and_never_doubled(self) -> None: + env = parse_conda_environment("dependencies:\n - python=3.11\n - gdal=3.9\n") + merged = merge_conda_pip(env) + rendered = rendered_environment(env, merged, python_version="3.13") + assert rendered.count("python=3.13") == 1 + assert "python=3.11" not in rendered + + def test_the_rendered_pip_layer_carries_the_pins(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + rendered = rendered_environment(env, merged, python_version="3.13") + assert "ipykernel==7.3.0" in rendered + assert "shapely==2.0.6" in rendered + + +# -- Reading micromamba's refusals ------------------------------------------- + + +class TestReadingRefusals: + def test_a_missing_package_is_package_not_found(self) -> None: + error = parse_conda_failure(CONDA_MISSING) + assert error.code.code == "DL_ENV_PACKAGE_NOT_FOUND" + assert "no-such-conda-pkg-xyzzy" in error.message + + def test_an_unsatisfiable_set_is_resolve_conflict(self) -> None: + error = parse_conda_failure(CONDA_CONFLICT) + assert error.code.code == "DL_ENV_RESOLVE_CONFLICT" + + def test_an_unreachable_channel_is_a_provider_error(self) -> None: + error = parse_conda_failure(CONDA_UNREACHABLE) + assert error.code.code == "DL_ENV_PROVIDER_ERROR" + assert error.code.retryable is True + + +# -- The lock ----------------------------------------------------------------- + + +class TestTheLock: + def test_it_counts_only_the_package_urls(self) -> None: + assert explicit_lock_packages(EXPLICIT_LOCK) == [ + "https://conda.anaconda.org/conda-forge/linux-64/python-3.13.0.conda#" + "aa" * 32, + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2.conda#" + "bb" * 32, + ] + + def test_the_document_records_the_pins_and_is_deterministic(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + document = conda_lock_document( + CondaResolveOutcome(lock_text=EXPLICIT_LOCK), + python_version="3.13", + base_reference="registry/base@sha256:" + "11" * 32, + merged=merged, + ) + assert document["format"] == CONDA_LOCK_FORMAT + assert document["package_count"] == 2 + # The header carries the complete pip layer: the user's own pip + # requirement and the protected pin the resolver forced over it. + assert "# datalayer-pip: shapely==2.0.6" in document["content"] + assert "# datalayer-pip: ipykernel==7.3.0" in document["content"] + assert "@EXPLICIT" in document["content"] + again = conda_lock_document( + CondaResolveOutcome(lock_text=EXPLICIT_LOCK), + python_version="3.13", + base_reference="registry/base@sha256:" + "11" * 32, + merged=merged, + ) + # Deterministic without anybody pinning a clock: the header carried a + # `# resolved-at:` line until 2026-09-16, which made two resolves of + # one spec two different digests and kept D-12's cache from ever + # hitting. + assert document["digest"] == again["digest"] + assert "resolved-at" not in document["content"] + + def test_an_export_without_the_marker_is_a_provider_error(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + with pytest.raises(EnvironmentsError) as caught: + conda_lock_document( + CondaResolveOutcome(lock_text="just some lines\nno marker\n"), + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + merged=merged, + ) + assert caught.value.code.code == "DL_ENV_PROVIDER_ERROR" + + +# -- The runners refuse honestly when their tool is absent ------------------- + + +class TestTheRunners: + def test_the_local_runner_refuses_without_micromamba(self) -> None: + runner = MicromambaResolveRunner(micromamba="") + with pytest.raises(EnvironmentsError) as caught: + runner.solve(CondaResolveRequest(environment_yml=A_YAML, python_version="3.13")) + assert caught.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" + + def test_an_export_that_times_out_is_a_provider_error( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # The solve finishes, but the `env export` that reads the lock back + # exceeds the deadline: its timeout is classified the same as the + # solve's, never left as a raw subprocess.TimeoutExpired. + runner = MicromambaResolveRunner(micromamba="/usr/local/bin/micromamba", timeout=5.0) + calls = {"n": 0} + + def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + calls["n"] += 1 + if calls["n"] == 1: + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + raise subprocess.TimeoutExpired(argv, 5.0) + + monkeypatch.setattr("code_sandboxes.environments.resolve_conda.subprocess.run", fake_run) + with pytest.raises(EnvironmentsError) as caught: + runner.solve(CondaResolveRequest(environment_yml=A_YAML, python_version="3.13")) + assert caught.value.code.code == "DL_ENV_PROVIDER_ERROR" + + def test_the_buildkit_runner_refuses_without_buildctl(self) -> None: + runner = BuildkitCondaResolveRunner(buildctl="") + with pytest.raises(EnvironmentsError) as caught: + runner.solve( + CondaResolveRequest( + environment_yml=A_YAML, + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + ) + ) + assert caught.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" + + def test_the_buildkit_runner_goes_through_the_pools_proxy( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # The conda solve reaches its channels the way a build does, through + # the build pool's proxy (E1-06). + seen: dict[str, list[str]] = {} + + def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + seen["argv"] = list(argv) + return subprocess.CompletedProcess(argv, 1, stdout="", stderr="boom") + + monkeypatch.setenv("DATALAYER_BUILDKIT_PROXY", "http://127.0.0.1:3128") + monkeypatch.setattr("code_sandboxes.environments.resolve_conda.subprocess.run", fake_run) + runner = BuildkitCondaResolveRunner(buildctl="/usr/bin/buildctl") + with pytest.raises(EnvironmentsError): + runner.solve( + CondaResolveRequest( + environment_yml=A_YAML, + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + ) + ) + assert "build-arg:https_proxy=http://127.0.0.1:3128" in seen["argv"] + + def test_the_buildkit_runner_refuses_an_unpinned_base(self) -> None: + runner = BuildkitCondaResolveRunner(buildctl="/usr/bin/buildctl") + with pytest.raises(EnvironmentsError) as caught: + runner.solve( + CondaResolveRequest( + environment_yml=A_YAML, python_version="3.13", base_reference="base:latest" + ) + ) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + + def test_the_buildkit_dockerfile_brings_the_wheelhouse_and_solves(self) -> None: + runner = BuildkitCondaResolveRunner(buildctl="/usr/bin/buildctl") + dockerfile = runner.dockerfile( + CondaResolveRequest( + environment_yml=A_YAML, + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + ) + ) + assert "micromamba create" in dockerfile + assert "PIP_FIND_LINKS" in dockerfile + assert "micromamba env export --explicit" in dockerfile + # The pinned micromamba is copied in (the base bakes uv, not it), and + # the pip layer is exported alongside the explicit lock. + assert "COPY --from=mambaorg/micromamba" in dockerfile + assert "env export --prefix /solve/prefix > /solve/pip-env.yml" in dockerfile + assert "COPY --from=solve /solve/pip-env.yml /pip-env.yml" in dockerfile + # And the pip layer whole, asked of the prefix's own interpreter: the + # export names what the file asked for and nothing it pulled in. + assert "COPY pip_layer.py ./pip_layer.py" in dockerfile + assert "RUN /solve/prefix/bin/python pip_layer.py > /solve/pip-lock.txt" in dockerfile + assert "COPY --from=solve /solve/pip-lock.txt /pip-lock.txt" in dockerfile + + +class TestThePipLayerIsWhole: + """`micromamba env export` named 7 pip pins where pip had installed 310 + distributions (the first real solve, 2026-09-18): everything they pulled + in would have been resolved afresh by each build, on each variant.""" + + def test_the_script_lists_what_pip_installed_and_not_what_conda_did(self, tmp_path) -> None: + import subprocess as process + import sys + + site = tmp_path / "site" + for name, version, installer in ( + ("Tornado", "6.5.10", "pip"), + ("jupyter_server", "2.21.0+datalayer.1", "uv"), + ("GDAL", "3.11.5", "conda"), + ("unknown", "1.0", ""), + ): + info = site / f"{name}-{version}.dist-info" + info.mkdir(parents=True) + (info / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" + ) + (info / "INSTALLER").write_text(installer + "\n") + listed = process.run( # noqa: S603 - this interpreter, this package's script + [ + sys.executable, + "-S", + "-c", + f"import sys; sys.path[:] = [{str(site)!r}] + sys.path\n" + PIP_LAYER_SCRIPT, + ], + capture_output=True, + text=True, + check=True, + ).stdout + assert pip_requirements_from_listing(listed) == ( + "jupyter-server==2.21.0+datalayer.1", + "tornado==6.5.10", + ) + + def test_a_line_that_is_not_a_pin_is_never_installed(self) -> None: + listing = "a==1\nWARNING: something was said\n\nb-c==2.0\n== \n" + assert pip_requirements_from_listing(listing) == ("a==1", "b-c==2.0") + + +class TestWhatCheckFiveExpectsOfACondaLock: + LOCK = ( + "# datalayer-pip: shapely==2.1.2\n# datalayer-pip: tornado==6.5.10\n@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/libgdal-core-3.11.5-h4f65170_7.conda#aa\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.11.5-py313h1ee8c46_7.conda#bb\n" + "https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-pyhcf101f3_0.conda#cc\n" + "https://conda.anaconda.org/conda-forge/linux-64/python-3.13.15-h2b335a9_0_cp313.conda#dd\n" + ) + FILE = ( + "channels: [conda-forge]\n" + "dependencies:\n" + " - python=3.13\n" + " - gdal=3.11\n" + " - libgdal-core\n" + " - pip:\n" + " - shapely==2.1.2\n" + ) + + def test_only_python_distributions_are_read_out_of_the_lock(self) -> None: + """`libgdal-core` has no Python metadata: asking the interpreter for + its version would fail a check with nothing wrong to report.""" + assert conda_lock_python_packages(self.LOCK) == { + "gdal": "3.11.5", + "typing-extensions": "4.15.0", + } + + def test_it_is_what_the_file_names_pinned_to_what_the_lock_resolved(self) -> None: + assert conda_expected_packages(self.FILE, self.LOCK) == { + "gdal": "3.11.5", + "shapely": "2.1.2", + } + + def test_check_five_is_handed_them_for_a_conda_version(self) -> None: + """Read from `packages.python`, as a pip source is, it was handed + nothing, and check 5 passed for having nothing to check.""" + from code_sandboxes.environments.conformance import expected_packages + + environment = validate_environment(a_conda_spec(content=self.FILE)) + assert expected_packages(environment, self.LOCK) == {"gdal": "3.11.5", "shapely": "2.1.2"} + + +# -- The whole resolve, through the recorded runner -------------------------- + + +class TestResolvingACondaVersion: + def test_it_answers_the_lock_and_the_bases(self) -> None: + runner = RecordedRunner(CondaResolveOutcome(lock_text=EXPLICIT_LOCK)) + document = resolve_conda_environment( + environment_yml=A_YAML, + python_version="3.13", + resolved_bases={"datalayer": "base@sha256:" + "11" * 32}, + runner=runner, + ) + assert document["format"] == CONDA_LOCK_FORMAT + assert document["package_count"] == 2 + assert document["resolved_bases"] == {"datalayer": "base@sha256:" + "11" * 32} + + def test_it_sends_the_rendered_environment_with_the_pins(self) -> None: + runner = RecordedRunner(CondaResolveOutcome(lock_text=EXPLICIT_LOCK)) + resolve_conda_environment( + environment_yml=A_YAML, + python_version="3.13", + resolved_bases={"datalayer": "base@sha256:" + "11" * 32}, + runner=runner, + ) + assert runner.request is not None + assert "python=3.13" in runner.request.environment_yml + assert "ipykernel==7.3.0" in runner.request.environment_yml + + def test_a_missing_package_propagates_as_package_not_found(self) -> None: + runner = RecordedRunner(parse_conda_failure(CONDA_MISSING)) + with pytest.raises(EnvironmentsError) as caught: + resolve_conda_environment( + environment_yml=A_YAML, + python_version="3.13", + resolved_bases={"datalayer": "base@sha256:" + "11" * 32}, + runner=runner, + ) + assert caught.value.code.code == "DL_ENV_PACKAGE_NOT_FOUND" + + +# -- The spec validates a conda dependency file ------------------------------ + + +class TestValidatingTheSpec: + def test_a_conda_environment_file_validates(self) -> None: + # Does not raise: a well-formed conda source is authorable. + validate_environment(a_conda_spec(), bases=BASES) + + def test_an_empty_environment_file_is_refused(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + validate_environment(a_conda_spec(content=" \n"), bases=BASES) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + + def test_a_malformed_environment_file_is_refused(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + validate_environment(a_conda_spec(content="dependencies: [\n"), bases=BASES) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + + def test_a_lock_content_is_refused_for_conda(self) -> None: + spec = a_conda_spec() + spec["spec"]["build"]["dependencyFile"]["lockContent"] = "irrelevant" # type: ignore[index] + with pytest.raises(EnvironmentsError) as caught: + validate_environment(spec, bases=BASES) + assert "pyproject" in caught.value.message + + +class TestTheProtectedPinsAreTheKernelStack: + def test_merge_uses_the_same_pins_the_pip_resolver_does(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + pin_names = {pin.name for pin in protected_pins()} + forced = {req.split("==")[0].replace("_", "-") for req in merged.requirements} + assert pin_names <= forced diff --git a/tests/test_environment_spec.py b/tests/test_environment_spec.py index 81d03d7..f84c1f7 100644 --- a/tests/test_environment_spec.py +++ b/tests/test_environment_spec.py @@ -196,7 +196,6 @@ def _codes(data: dict[str, Any]) -> dict[str, str]: ("spec.contract", "sandbox-contract/v9", "spec.contract", UNSUPPORTED), ("spec.base.ref", "python", "spec.base.ref", INVALID), ("spec.language.version", "3.9", "spec.language.version", INVALID), - ("spec.build.source", "dockerfile", "spec.build.source", UNSUPPORTED), ("spec.packages.python.manager", "conda", "spec.packages.python.manager", UNSUPPORTED), # E3-05: a secret no postInstall command names would be mounted nowhere. ( @@ -293,6 +292,18 @@ def test_a_gpu_class_needs_a_cuda_base() -> None: assert _codes(data) == {} +def test_a_gpu_asks_for_the_cuda_its_base_carries() -> None: + """The CUDA channel pins its toolkit (E2-17): a spec asking for another + would build and then fail check 11 in every sandbox it started.""" + data = mutated("spec.resources.sizeClass", "gpu-small") + data["spec"]["base"]["ref"] = "datalayer/python-cuda" + for cuda in ("12.8", "12", None): + data["spec"]["resources"]["accelerator"] = {"type": "H100", "cuda": cuda} + assert _codes(data) == {}, cuda + data["spec"]["resources"]["accelerator"] = {"type": "H100", "cuda": "12.4"} + assert _codes(data) == {"spec.resources.accelerator.cuda": INVALID} + + def test_all_baked_files_together_are_capped() -> None: data = document() data["spec"]["files"] = [ @@ -307,8 +318,8 @@ def test_all_baked_files_together_are_capped() -> None: def test_an_invalid_field_outranks_something_unsupported() -> None: - data = mutated("spec.build.source", "dockerfile") - assert _codes(data) == {"spec.build.source": UNSUPPORTED} + data = mutated("spec.packages.python.manager", "conda") + assert _codes(data) == {"spec.packages.python.manager": UNSUPPORTED} with pytest.raises(EnvironmentsError) as unsupported: validate_environment(data) assert unsupported.value.code is errors.CAPABILITY_UNSUPPORTED @@ -319,10 +330,50 @@ def test_an_invalid_field_outranks_something_unsupported() -> None: assert invalid.value.code is errors.SPEC_INVALID assert {finding["field"] for finding in invalid.value.detail["findings"]} == { "metadata.name", - "spec.build.source", + "spec.packages.python.manager", } +def a_dockerfile_document(content: str) -> dict[str, Any]: + data = mutated("spec.build.source", "dockerfile") + data["spec"]["build"]["dockerfile"] = {"content": content} + return data + + +def test_a_dockerfile_source_is_accepted_and_keeps_its_own_base() -> None: + # E3-03: the base is the `FROM` the Dockerfile names, so `spec.base` is + # not checked against the approved table (as for `image`). + data = a_dockerfile_document("FROM datalayer/python-cpu:2026.09\nRUN true\n") + data["spec"]["base"]["ref"] = "python" + assert _codes(data) == {} + + +def test_a_dockerfile_source_without_a_dockerfile_is_refused() -> None: + """`source: dockerfile` with nowhere to read the file from is not a spec.""" + data = mutated("spec.build.source", "dockerfile") + assert _codes(data) == {"spec.build.dockerfile": "DL_ENV_SPEC_INVALID"} + + +def test_an_empty_dockerfile_is_refused() -> None: + assert _codes(a_dockerfile_document(" \n")) == { + "spec.build.dockerfile.content": "DL_ENV_SPEC_INVALID" + } + + +def test_the_contract_refuses_a_dockerfile_at_validate_naming_the_line() -> None: + """A Dockerfile is somebody's file: "it was refused" is not a reason. + + The refusals are the contract's own — an unapproved base, the host + network, a privileged build, a Docker socket mount — and they are made + here, before anything is queued, rather than partway through a build. + """ + data = a_dockerfile_document("FROM ubuntu:22.04\nRUN --network=host apt-get update\n") + assert _codes(data) == {"spec.build.dockerfile.content": "DL_ENV_SPEC_INVALID"} + messages = [finding.message for finding in spec_findings(parse_environment(data))] + assert any("line 1" in message and "approved" in message for message in messages) + assert any("line 2" in message and "host network" in message for message in messages) + + # -- Dependency files (E3-01) -------------------------------------------------- @@ -586,3 +637,59 @@ def test_a_private_build_with_a_secret_is_untouched(self) -> None: accepting the same spec `publication_findings` refuses to publish.""" environment = parse_environment(document()) assert spec_findings(environment) == [] + + def test_a_private_index_blocks_publication(self) -> None: + """D-12: a published version resolves only from public indexes, since a + private one is reached with a credential the public does not hold.""" + data = document() + del data["spec"]["buildSecrets"] + data["spec"]["packages"]["python"]["indexes"] = [ + "https://pypi.org/simple", + "https://pypi.mycorp.internal/simple", + ] + environment = parse_environment(data) + findings = publication_findings(environment) + assert len(findings) == 1 + assert findings[0].field == "spec.packages.python.indexes" + assert findings[0].code is errors.PUBLICATION_BLOCKED + assert "pypi.mycorp.internal" in findings[0].message + + def test_only_public_indexes_are_publishable(self) -> None: + """The public index and its wheel host are both accepted; nothing else.""" + data = document() + del data["spec"]["buildSecrets"] + data["spec"]["packages"]["python"]["indexes"] = [ + "https://pypi.org/simple", + "https://files.pythonhosted.org/", + ] + environment = parse_environment(data) + assert publication_findings(environment) == [] + + def test_a_private_conda_channel_blocks_publication(self) -> None: + """D-12 the same for a conda source: a channel reached with a + credential the public does not hold can never be published.""" + data = a_dependency_file_document( + sourceFormat="conda", + content=( + "channels:\n" + " - conda-forge\n" + " - https://conda.mycorp.internal/private\n" + "dependencies:\n - gdal\n" + ), + ) + del data["spec"]["buildSecrets"] + environment = parse_environment(data) + findings = publication_findings(environment) + assert len(findings) == 1 + assert findings[0].field == "spec.build.dependencyFile.content.channels" + assert findings[0].code is errors.PUBLICATION_BLOCKED + assert "conda.mycorp.internal" in findings[0].message + + def test_public_conda_channels_are_publishable(self) -> None: + data = a_dependency_file_document( + sourceFormat="conda", + content="channels:\n - conda-forge\n - bioconda\ndependencies:\n - gdal\n", + ) + del data["spec"]["buildSecrets"] + environment = parse_environment(data) + assert publication_findings(environment) == [] diff --git a/tests/test_environment_spec_organization_policy.py b/tests/test_environment_spec_organization_policy.py new file mode 100644 index 0000000..f8ba856 --- /dev/null +++ b/tests/test_environment_spec_organization_policy.py @@ -0,0 +1,198 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""An organization's own policy, enforced on the spec at `validate` (PLAN_ENV.md E3-06). + +The box's own `Done when`: "an index an organization admin denied yields +`DL_ENV_POLICY_DENIED` at `validate`". Bases, registries and packages are the +same shape and covered alongside it; licences are checked once an artifact's +SBOM exists, not here — see `test_environment_attest.py` and +`test_environment_organization_policy.py`. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +import yaml + +from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.policy import UNRESTRICTED_POLICY, EnvironmentsPolicy +from code_sandboxes.environments.spec import ( + parse_environment, + spec_findings, + validate_environment, +) + +EXAMPLE = """ +apiVersion: environments.datalayer.io/v1alpha1 +kind: Environment +metadata: + name: geospatial-analysis +spec: + contract: sandbox-contract/v1 + language: + name: python + version: "3.13" + base: + ref: datalayer/python-cpu + channel: "2026.08" + platform: + architecture: linux/amd64 + packages: + python: + manager: uv + dependencies: + - geopandas==1.1.1 + indexes: + - https://pypi.org/simple + compatibility: + variants: + required: [datalayer] + build: + source: packages +""" + + +def document() -> dict[str, Any]: + return yaml.safe_load(EXAMPLE) + + +def mutated(path: str, value: Any) -> dict[str, Any]: + data = document() + target: Any = data + parts = path.split(".") + for part in parts[:-1]: + target = target[int(part)] if part.isdigit() else target[part] + target[parts[-1]] = value + return data + + +class TestUnrestrictedByDefault: + """No organization has written this section: every check here is a no-op.""" + + def test_the_example_validates_with_no_policy_at_all(self) -> None: + environment = validate_environment(EXAMPLE) + assert spec_findings(environment) == [] + + def test_the_default_policy_argument_is_unrestricted(self) -> None: + environment = validate_environment(EXAMPLE, policy=UNRESTRICTED_POLICY) + assert environment.metadata.name == "geospatial-analysis" + + +class TestAnIndexAnOrganizationDenies: + """The box's own `Done when`, verbatim.""" + + def test_an_undenied_index_still_validates(self) -> None: + policy = EnvironmentsPolicy(allowed_indexes=("https://pypi.org/simple",)) + environment = validate_environment(EXAMPLE, policy=policy) + assert spec_findings(environment, policy=policy) == [] + + def test_an_index_not_on_the_allowlist_is_policy_denied_at_validate(self) -> None: + policy = EnvironmentsPolicy(allowed_indexes=("https://pypi.org/simple",)) + doc = mutated("spec.packages.python.indexes", ["https://evil.example/simple"]) + with pytest.raises(EnvironmentsError) as raised: + validate_environment(doc, policy=policy) + assert raised.value.code.code == "DL_ENV_POLICY_DENIED" + assert "https://evil.example/simple" in raised.value.message + + def test_the_finding_is_on_the_right_field_and_names_the_allowed_ones(self) -> None: + policy = EnvironmentsPolicy(allowed_indexes=("https://pypi.org/simple",)) + doc = mutated( + "spec.packages.python.indexes", + ["https://pypi.org/simple", "https://evil.example/simple"], + ) + environment = parse_environment(doc) + findings = spec_findings(environment, policy=policy) + [finding] = [f for f in findings if f.code.code == "DL_ENV_POLICY_DENIED"] + assert finding.field == "spec.packages.python.indexes[1]" + assert "https://pypi.org/simple" in finding.message + + +class TestABaseAnOrganizationDenies: + def test_an_approved_base_the_organization_also_allows_passes(self) -> None: + policy = EnvironmentsPolicy(allowed_bases=("datalayer/python-cpu",)) + environment = validate_environment(EXAMPLE, policy=policy) + assert spec_findings(environment, policy=policy) == [] + + def test_an_approved_base_the_organization_does_not_list_is_denied(self) -> None: + """The platform approves it; the organization has not, and its + policy narrows what the platform allows rather than widening it.""" + policy = EnvironmentsPolicy(allowed_bases=("datalayer/python-cuda",)) + with pytest.raises(EnvironmentsError) as raised: + validate_environment(EXAMPLE, policy=policy) + assert raised.value.code.code == "DL_ENV_POLICY_DENIED" + assert "datalayer/python-cpu" in raised.value.message + + def test_an_image_or_dockerfile_source_is_not_checked_against_it(self) -> None: + """`spec.base` names nothing for either source (the platform's own + rule, above this one) — an organization's base policy has nothing to + apply to and must not invent a refusal for a field these sources + never use.""" + policy = EnvironmentsPolicy(allowed_bases=("datalayer/python-cuda",)) + doc = mutated("spec.build.source", "dockerfile") + doc["spec"]["build"]["dockerfile"] = {"content": "FROM datalayer/python-cpu:2026.08\n"} + environment = parse_environment(doc) + findings = spec_findings(environment, policy=policy) + assert not any(f.field == "spec.base.ref" for f in findings) + + +class TestARegistryAnOrganizationAdds: + """The one dimension that widens rather than narrows (`_image_findings`' + own docstring): the platform's public bootstrap stays reachable, and an + organization's own private registries are added to it, never instead of + it.""" + + def _image_doc(self, reference: str) -> dict[str, Any]: + doc = mutated("spec.build.source", "image") + doc["spec"]["build"]["image"] = {"reference": reference} + del doc["spec"]["packages"] + return doc + + def test_the_platform_bootstrap_still_passes_with_no_organization_list(self) -> None: + environment = parse_environment(self._image_doc("docker.io/library/python:3.12-slim")) + assert not any(f.code.code == "DL_ENV_POLICY_DENIED" for f in spec_findings(environment)) + + def test_an_organizations_own_private_registry_is_added_not_substituted(self) -> None: + policy = EnvironmentsPolicy(allowed_registries=("registry.acme.internal",)) + environment = parse_environment(self._image_doc("docker.io/library/python:3.12-slim")) + findings = spec_findings(environment, policy=policy) + assert not any(f.code.code == "DL_ENV_POLICY_DENIED" for f in findings) + + def test_a_registry_on_neither_list_is_denied_naming_both(self) -> None: + policy = EnvironmentsPolicy(allowed_registries=("registry.acme.internal",)) + environment = parse_environment( + self._image_doc("registry.other.example/someone/image:latest") + ) + findings = spec_findings(environment, policy=policy) + [finding] = [f for f in findings if f.code.code == "DL_ENV_POLICY_DENIED"] + assert "registry.acme.internal" in finding.message + assert "docker.io" in finding.message # the bootstrap default, still named + + +class TestAPackageAnOrganizationDenies: + def test_an_allowed_package_passes(self) -> None: + policy = EnvironmentsPolicy(allowed_packages=("geopandas",)) + environment = validate_environment(EXAMPLE, policy=policy) + assert spec_findings(environment, policy=policy) == [] + + def test_a_package_off_the_allowlist_is_denied_naming_it(self) -> None: + policy = EnvironmentsPolicy(allowed_packages=("numpy",)) + with pytest.raises(EnvironmentsError) as raised: + validate_environment(EXAMPLE, policy=policy) + assert raised.value.code.code == "DL_ENV_POLICY_DENIED" + assert "geopandas" in raised.value.message + + def test_constraints_are_not_checked_the_dependency_list_is(self) -> None: + """`spec.packages.python.constraints` narrows a version, never adds a + package that is actually installed on its own — nothing to deny + there that is not already covered by the dependency it constrains.""" + policy = EnvironmentsPolicy(allowed_packages=("geopandas",)) + doc = mutated("spec.packages.python.constraints", ["numpy<2"]) + environment = parse_environment(doc) + findings = spec_findings(environment, policy=policy) + assert not any( + f.code.code == "DL_ENV_POLICY_DENIED" and "numpy" in f.message for f in findings + ) diff --git a/tests/test_jupyter_server.py b/tests/test_jupyter_server.py index 9c0be8c..e582ee2 100644 --- a/tests/test_jupyter_server.py +++ b/tests/test_jupyter_server.py @@ -526,3 +526,105 @@ def test_stop_forgets_the_temporary_workdir_it_removed(tmp_path: Path, monkeypat second = Path(sandbox._resolve_workdir()) assert second.is_dir() assert second != first + + +class TestRestartingTheKernelRatherThanTheConnection: + """Appendix B check 7, against a server this process does not own. + + `stop()` then `start()` is this client's lifecycle: when the Jupyter + server belongs to somebody else — a Datalayer runtime pod, which is every + attached sandbox — it drops the websocket and leaves the kernel process + running, so the next execution lands in the same interpreter with every + global still set. Found live on r1, 2026-09-16: the smoke test read + `state survived the restart`. + """ + + def test_the_kernel_is_restarted_through_the_server_s_own_api(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + asked: dict = {} + + class _Response: + ok = True + status_code = 200 + + def _post(url, params=None, headers=None, timeout=None): + asked["url"] = url + asked["params"] = params + return _Response() + + monkeypatch.setattr("code_sandboxes.jupyter_server_sandbox.requests.post", _post) + try: + assert sandbox.restart_kernel() is True + assert asked["url"].endswith("/api/kernels/kernel-1/restart") + finally: + sandbox.stop() + + def test_the_connection_is_kept_across_the_restart(self, monkeypatch): + """jupyter-server moves a kernel's websocket onto the restarted kernel. + + Reconnecting broke it: `jupyter_kernel_client`'s channels stay bound + to the socket they were built with, so after a `stop()` and `start()` + every execution answered `ok` with no output (r1, 2026-09-18). + """ + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + lifecycle: list[str] = [] + client = sandbox._client + monkeypatch.setattr(client, "stop", lambda *a, **k: lifecycle.append("stop")) + monkeypatch.setattr(client, "start", lambda *a, **k: lifecycle.append("start")) + + class _Response: + ok = True + status_code = 200 + + monkeypatch.setattr( + "code_sandboxes.jupyter_server_sandbox.requests.post", + lambda *args, **kwargs: _Response(), + ) + try: + assert sandbox.restart_kernel() is True + assert lifecycle == [] + assert sandbox._client is client + finally: + sandbox.stop() + + def test_a_server_that_refuses_the_restart_is_reported_not_raised(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + + class _Response: + ok = False + status_code = 503 + + monkeypatch.setattr( + "code_sandboxes.jupyter_server_sandbox.requests.post", + lambda *args, **kwargs: _Response(), + ) + try: + assert sandbox.restart_kernel() is False + finally: + sandbox.stop() + + def test_a_server_that_cannot_be_reached_is_reported_not_raised(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + + def _explode(*args, **kwargs): + raise OSError("no route to host") + + monkeypatch.setattr("code_sandboxes.jupyter_server_sandbox.requests.post", _explode) + try: + assert sandbox.restart_kernel() is False + finally: + sandbox.stop() + + def test_a_sandbox_with_no_kernel_id_asks_nothing(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id=None) + + def _should_not_be_called(*args, **kwargs): + raise AssertionError("the server must not be asked without a kernel id") + + monkeypatch.setattr( + "code_sandboxes.jupyter_server_sandbox.requests.post", _should_not_be_called + ) + try: + assert sandbox.restart_kernel() is False + finally: + sandbox.stop() diff --git a/tests/test_manage.py b/tests/test_manage.py index 8febcfe..ccb5483 100644 --- a/tests/test_manage.py +++ b/tests/test_manage.py @@ -11,6 +11,7 @@ from code_sandboxes import cli as sandbox_cli from code_sandboxes.manage import ( + DaytonaSandboxManager, DockerSandboxManager, KaggleSandboxManager, SandboxManagementError, @@ -259,3 +260,70 @@ def _raise(*a, **k): result = CliRunner().invoke(sandbox_cli.app, ["list", "-v", "docker"]) assert result.exit_code == 1 assert "no backend today" in result.output + + +class TestCreateOptionsReachTheSandbox: + """`--environment` and `--name` configure the sandbox, or are refused. + + A `Sandbox` reads both from its `SandboxConfig`. Passed as bare keywords + they land in its `**kwargs` and are dropped there in silence, so `create` + answers `running` for a default sandbox under a generated name. + + Found on 2026-09-17 against a real Daytona account: + `sandboxes create daytona -e eric/daytona-drift -n envs-drill-launch` + launched `daytonaio/sandbox:0.8.0` with a blank name, and so did + `-e this-environment-does-not-exist-at-all`. + """ + + def test_a_name_reaches_the_config(self): + manager = DaytonaSandboxManager() + kwargs = {"name": "a-named-sandbox"} + manager._configure(kwargs) + assert kwargs["config"].name == "a-named-sandbox" + assert "name" not in kwargs, "the bare keyword would be dropped downstream" + + def test_an_environment_the_provider_ships_reaches_the_config(self): + manager = DaytonaSandboxManager() + kwargs = {"environment": "daytona-gpu"} + environment = manager._configure(kwargs) + assert environment is not None and environment.name == "daytona-gpu" + assert kwargs["config"].environment == "daytona-gpu" + # The card the environment names, which is what the adapter's own + # resource shaping reads. + assert kwargs["config"].gpu == "H100" + + def test_the_spot_environment_differs_from_the_plain_gpu_one(self): + """`daytona-gpu-spot` differs from `daytona-gpu` by a Daytona argument. + + Nothing the config carries says preemptible, so a manager that only + set the config would make the two names mean the same machine. + """ + manager = DaytonaSandboxManager() + kwargs = {"environment": "daytona-gpu-spot"} + environment = manager._configure(kwargs) + assert (environment.metadata or {}).get("spot") is True + + def test_an_environment_the_provider_does_not_ship_is_refused(self): + """Rather than handing back a default sandbox and saying `running`.""" + manager = DaytonaSandboxManager() + with pytest.raises(SandboxManagementError) as refused: + manager._configure({"environment": "eric/daytona-drift"}) + message = str(refused.value) + assert "eric/daytona-drift" in message + # The refusal says what there is instead, so it is actionable. + assert "daytona-gpu" in message + + def test_the_option_leaves_the_keywords(self): + """It has to: left bare it reaches the sandbox's `**kwargs` and is dropped.""" + manager = DaytonaSandboxManager() + kwargs = {"environment": "daytona-default"} + manager._configure(kwargs) + assert "environment" not in kwargs + assert kwargs["config"].environment == "daytona-default" + + def test_nothing_asked_for_changes_nothing(self): + """A create with no options still builds no config of its own.""" + manager = DaytonaSandboxManager() + kwargs = {"spot": True} + assert manager._configure(kwargs) is None + assert kwargs == {"spot": True} diff --git a/tests/test_manage_datalayer.py b/tests/test_manage_datalayer.py index 6ca0bcb..7992b3a 100644 --- a/tests/test_manage_datalayer.py +++ b/tests/test_manage_datalayer.py @@ -49,9 +49,8 @@ def fake_sandbox(monkeypatch): monkeypatch.setattr(datalayer_sandbox, "DatalayerSandbox", _Sandbox) -@pytest.mark.parametrize("key", ["environment_name", "environment"]) -def test_create_starts_the_environment_asked_for_under_its_name(key): - DatalayerSandboxManager(token="t").create(**{key: "python-cpu-env"}, name="e0-11-check") +def test_create_starts_the_environment_asked_for_under_its_name(): + DatalayerSandboxManager(token="t").create(environment="python-cpu-env", name="e0-11-check") sandbox = _Sandbox.started[-1] assert sandbox.config.environment == "python-cpu-env" assert sandbox.config.name == "e0-11-check" diff --git a/tests/test_modal_google_colab_sandbox.py b/tests/test_modal_google_colab_sandbox.py index ebc7c5b..3f2859e 100644 --- a/tests/test_modal_google_colab_sandbox.py +++ b/tests/test_modal_google_colab_sandbox.py @@ -493,6 +493,57 @@ def create(**kwargs): sandbox.stop() +def test_an_environments_artifact_is_held_by_sleep_not_its_cmd(monkeypatch): + """An artifact's CMD may be its base's `start-jupyter.sh`, which exits and + ends the sandbox (found live on 2026-09-18): the launch names the command + that holds the container, and Jupyter is started by exec when asked.""" + + class _FakeApp: + pass + + class _FakeSandboxObj: + object_id = "modal-object-id" + + def exec(self, *_args, **_kwargs): + raise RuntimeError("no driver in this test") + + def terminate(self): + return None + + def detach(self): + return None + + captured: dict = {} + + class _FakeModal: + class App: + @staticmethod + def lookup(_name, create_if_missing=False): + return _FakeApp() + + class Image: + @staticmethod + def from_id(image_id): + captured["image_id"] = image_id + return object() + + class Sandbox: + @staticmethod + def create(*args, **kwargs): + captured["args"] = args + return _FakeSandboxObj() + + monkeypatch.setitem(sys.modules, "modal", _FakeModal) + + sandbox = ModalSandbox(config=SandboxConfig(timeout=10.0), image_id="im-artifact") + sandbox.start() + + assert captured["image_id"] == "im-artifact" + assert captured["args"] == ("sleep", "infinity") + + sandbox.stop() + + class _FakeExecSandbox: """Records what `_start_driver` asks `sandbox.exec` for, nothing more.""" @@ -528,7 +579,7 @@ def test_a_contract_artifact_drops_privileges_and_sets_the_workdir(): "DATALAYER_SANDBOX_CONTRACT_GID": "100", "DATALAYER_SANDBOX_CONTRACT_HOME": "/home/datalayer", } - assert call["kwargs"]["workdir"] == "/home/datalayer/content" + assert call["kwargs"]["workdir"] == "/home/datalayer" def test_a_plain_sandbox_asks_for_nothing_extra():