From 25a372848b69bda584bdb7e2154ce5e677cb111d Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Tue, 20 Oct 2026 18:37:26 -0400 Subject: [PATCH] fix: vendor hosted lockfile patches without installed packages --- crates/socket-patch-cli/CLI_CONTRACT.md | 20 +- .../socket-patch-cli/src/commands/vendor.rs | 171 +++++++++++------- .../src/commands/vendored_backend/mod.rs | 3 + crates/socket-patch-cli/src/lib.rs | 9 +- .../tests/in_process_vendor_pnpm_takeover.rs | 16 +- .../tests/vendor/npm_fixture.rs | 26 ++- .../tests/vendor/vendor_failure_paths.rs | 38 ++-- .../tests/vendor/vendor_render.rs | 15 +- crates/socket-patch-cli/tests/vendor_eject.rs | 42 ++++- .../tests/vendor_eject_bun_lockb.rs | 25 +-- .../tests/vendor_eject_fresh_checkout.rs | 159 ++++++++++++++-- docs/usage.md | 10 +- 12 files changed, 381 insertions(+), 153 deletions(-) diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 3756e7abf..df15a4edd 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -720,15 +720,15 @@ Not covered: the agent-mode blob cache and manifest are checked from `.socket` d ## Vendor command contract -`vendor` is `apply`'s committable sibling: instead of patching installed packages in place -(machine-local state), it ejects each patched package into `.socket/vendor/` and rewires the +`vendor` converts the hosted patches pinned in the project's lockfiles into committable +`.socket/vendor/` artifacts, without installing dependencies first. It rewires the ecosystem's lockfile/config so the project consumes the vendored copy. After committing `.socket/vendor/` + the lockfile edits, a fresh checkout builds with the patched dependency on machines with **no socket-patch installed and no Socket API access** (registry access for other, unvendored dependencies may still be needed). Every mechanism below was validated against the real package managers (`spikes/PHASE0-FINDINGS.txt`). -**Eject a hosted project (v5.0)**: standalone `vendor` with NO manifest in a project whose lockfiles +**Eject a hosted project (v5.0)**: standalone `vendor` in a project whose lockfiles pin hosted patches (not under `--global`; `--ecosystems` narrows the pins) takes its patch set from those pins — each pin's purl plus the patch uuid in its hosted URL — fetches each record from the patch API (`GET …/patches/view/`, the same client and public-proxy fallback as `get`), vendors @@ -736,15 +736,23 @@ into `.socket/vendor/` exactly like `scan --mode vendored`, and rewires each pac vendored (the upstream registry entry is restored first, so a later `vendor --revert` returns the project to upstream, never to hosted). The human output opens with `Ejecting N hosted package(s) into .socket/vendor/...` (`Would eject …` under `--dry-run`); the -JSON is the vendor envelope. It needs no installed `node_modules` / site-packages: the sources are -fetched from the upstream registry, so a fresh hosted checkout ejects. +JSON is the vendor envelope. Hosted pins take precedence even when `.socket/manifest.json` +exists: the manifest is neither used to select patches nor rewritten. The conversion skips +installed-package discovery and variant probes, so an unrelated virtualenv or global cache +cannot reject a lockfile-selected patch. It uses the shared lockfile readers, without VEX +attestation or installed-byte verification; downloaded artifacts still pass integrity and +patched-member verification. Upstream registry metadata is resolved for reversal, while the +patched archives come from the artifact service. No installed `node_modules`, site-packages, +or upstream source download is needed. When no hosted pins are in scope, legacy manifest-driven +vendoring remains available. The eject is ONE planned transition, all-or-nothing: (1) every record is fetched first — a failed (or 404) view fetch is a `failed` event with `errorCode: "patch_fetch_failed"` and the run stops with `status: "error"`, `error.code: "eject_refused"`, exit 1, nothing touched; (2) the upstream restore of every pin is resolved against staged copies — a refused pin (offline registry, a non-derivable field; a binary `bun.lockb` record is rebuilt like the takeover's) is `eject_refused` with the `git checkout -- ` -remedy, nothing touched; (3) `--dry-run` stops here and reports each pin as an `applied` event with +remedy, nothing touched. Backend refusals are also previewed against the restored text in memory +(for example, unsupported pnpm catalog entries); (3) `--dry-run` stops here and reports each pin as an `applied` event with reason `eject_planned` — no file is written and no `.socket/` is created; (4) the wet run snapshots every file the eject may touch under one `apply.lock`, restores upstream, then vendors. If any package then fails, exactly the files the eject wrote are put back (the pins' files, the upstream diff --git a/crates/socket-patch-cli/src/commands/vendor.rs b/crates/socket-patch-cli/src/commands/vendor.rs index 3516e7c49..415055afa 100644 --- a/crates/socket-patch-cli/src/commands/vendor.rs +++ b/crates/socket-patch-cli/src/commands/vendor.rs @@ -1,7 +1,7 @@ //! `socket-patch vendor` — committable vendoring of patched dependencies. //! -//! Works like `apply`, but instead of patching installed packages in place it -//! ejects each patched package into `.socket/vendor///…` and +//! Converts the hosted patches pinned in the project's lockfiles into +//! `.socket/vendor///…` without an installed package tree, and //! rewires the ecosystem's lockfile/config so the project consumes the //! vendored copy. After committing `.socket/vendor/` + the lockfile edits, a //! fresh checkout builds with the patched dependency on machines with no @@ -15,7 +15,8 @@ //! vendored` drive this module's [`vendor_records`] engine directly in //! DETACHED mode: every entry they write carries its patch record embedded //! in the ledger and no `.socket/manifest.json` is ever written — this -//! command is the one manifest-driven writer (`detached: false`). See +//! command also supports legacy manifest-driven vendoring when no hosted +//! pins are in scope (`detached: false`). See //! CLI_CONTRACT.md "Ownership, state, and reversal". use clap::Args; @@ -987,15 +988,10 @@ pub async fn run(args: VendorArgs) -> i32 { let manifest_path = args.common.resolved_manifest_path(); let socket_dir = crate::args::socket_dir_of(&manifest_path, &args.common.cwd); - // `--revert` derives everything from state.json + the vendor tree; it - // must work after the manifest was deleted. Plain vendor needs the - // manifest and exits clean without one (same contract as apply). This - // is a MANIFEST check, not a `.socket/` check: `scan`/`get --mode - // vendored` projects have `.socket/` but never a manifest. Nothing is - // locked or written on this path. - if !args.revert && tokio::fs::metadata(&manifest_path).await.is_err() { - // A hosted project (no manifest, hosted pins in its lockfiles) - // ejects: its patch set is the lockfiles' hosted pins. + // Hosted pins are the input, even when an agent manifest also exists. + // The raw lockfile inventory does not require VEX eligibility or an + // installed copy of a patch. Revert needs only the vendor ledger. + if !args.revert { let inventory = crate::commands::hosted_inventory(&args.common, &args.common.cwd).await; // Contested hosted wiring: the patch set cannot be read off the // lockfiles, and a "nothing to vendor" answer would hide it. @@ -1020,6 +1016,12 @@ pub async fn run(args: VendorArgs) -> i32 { } return run_eject(&args, pins).await; } + } + + // Preserve legacy manifest-only vendoring when there are no hosted + // pins. A missing manifest is a no-op; detached vendored projects keep + // their records in the ledger instead. + if !args.revert && tokio::fs::metadata(&manifest_path).await.is_err() { // A requested `--vex` still attests what the `.socket/vendor` // ledgers and lockfiles already wire. Same contract as `apply --vex` // with no manifest: nothing referenced anywhere keeps exit 0; any @@ -1719,9 +1721,9 @@ impl EjectSnapshot { } } -/// Standalone `vendor` in a hosted project — no manifest, hosted pins in the -/// lockfiles: EJECT. The patch set is the pins themselves (purl + the uuid -/// in each hosted URL); each record is fetched from the API, vendored into +/// Standalone `vendor` in a hosted project: EJECT. The patch set comes only +/// from the lockfile pins (purl + the uuid in each hosted URL); each record +/// is fetched from the API, vendored into /// `.socket/vendor/` exactly like `scan --mode vendored`, and the lock is /// rewired from hosted to vendored (the engine's takeover restores each /// pin's upstream registry entry first, so `vendor --revert` later returns @@ -1731,6 +1733,7 @@ async fn run_eject(args: &VendorArgs, pins: Vec) -> i32 { let (client, use_public_proxy) = get_api_client_with_overrides(common.api_client_overrides()).await; let telemetry = TelemetryAuth::for_client(&client); + let service = common.vendor_service_config(Some(client.clone()), use_public_proxy); if !common.json && !common.silent { println!( "{} {} into .socket/vendor/...", @@ -1827,11 +1830,39 @@ async fn run_eject(args: &VendorArgs, pins: Vec) -> i32 { .refused() .map(|(pin, why)| (pin.purl.clone(), "redirect_revert_failed", why.to_string())) .collect(); - // An sbt / scala-cli pin is vendored only past the build-evidence gate - // (no `target/` evidence in a fresh clone, stale evidence, …). Run it - // now, while the hosted pin still serves the patch: a pin the gate - // would stop after the restore would end neither hosted nor vendored. + // Reuse the takeover's read-only backend preview over the restored + // lockfile text. A manifest's presence must not change whether a dry + // run catches an unsupported lock entry (for example a pnpm catalog). + let uninstalled = common.cwd.join(".socket/vendor/.uninstalled"); + let blobs = common.socket_dir().join("blobs"); + let sources = PatchSources { + blobs_path: &blobs, + mem_blobs: None, + }; + let pipenv_version = tokio::sync::OnceCell::new(); + let installed_sites = vendor::pypi::InstalledSiteListings::default(); + let vendored_at = now_rfc3339(); for pin in &pins { + if let Some((code, detail)) = takeover_dry_refusal( + &plan, + &pin.purl, + PackageSource::Installed(&uninstalled), + &common.cwd, + &records[&pin.purl], + &sources, + &vendored_at, + args.force, + Some(&service), + &pipenv_version, + &installed_sites, + ) + .await + { + refused.push((pin.purl.clone(), code, detail)); + } + // sbt / scala-cli also require build evidence. The backend treats + // that gate as a benign skip, but an eject must retain the hosted + // pin if it cannot vendor it. if let Err((code, detail)) = socket_patch_core::vendor::maven_repo::jvm_gate_preflight(&common.cwd, &pin.purl).await { @@ -1912,11 +1943,10 @@ async fn run_eject(args: &VendorArgs, pins: Vec) -> i32 { } // One transaction under one apply lock: snapshot what the eject can - // touch, restore every pin upstream (so the vendor engine resolves the - // pristine registry package even in a fresh checkout with nothing - // installed), vendor, and on ANY failure put back what the eject wrote - // — a failed eject leaves the project hosted, exactly as it was, and - // every file it never wrote untouched. + // touch, restore upstream metadata for the vendor ledger's reversal, + // download and wire the prebuilt patches, and on ANY failure put back + // what the eject wrote. A failed eject leaves the project hosted, + // exactly as it was, and every file it never wrote untouched. let socket_dir = common.socket_dir(); let timeout = Duration::from_secs(common.lock_timeout.unwrap_or(0)); let guard = match crate::commands::lock_cli::acquire_with_status(&socket_dir, timeout) { @@ -1987,9 +2017,8 @@ async fn run_eject(args: &VendorArgs, pins: Vec) -> i32 { } let manifest = crate::commands::vendored_backend::records_manifest(records); // The same vendored apply as `scan --mode vendored`: detached (the - // eject is manifest-free) over this run's client and the shared - // service-config assembler. - let service = common.vendor_service_config(Some(client.clone()), use_public_proxy); + // eject does not use the agent manifest) over this run's client + // and the shared service-config assembler. let applied = VendoredBackend::new(common, Some(&service)) .apply( ApplyRequest { @@ -1997,7 +2026,7 @@ async fn run_eject(args: &VendorArgs, pins: Vec) -> i32 { socket_dir: &socket_dir, ledger: load_state(&common.cwd).await, detached: true, - force: false, + force: args.force, prior: None, eject: Some(&mut capture), }, @@ -2082,7 +2111,7 @@ async fn run_eject(args: &VendorArgs, pins: Vec) -> i32 { // Embedded VEX: same contract as the manifest-driven arm — only on // success, never on a dry run, and a requested-but-failed VEX flips the - // exit code. The ejected project has no manifest. + // exit code. The ejected patches are recorded in the vendor ledger. if exit == 0 { if let Some(vex_path) = args.vex.vex.as_ref() { if common.dry_run { @@ -2769,41 +2798,48 @@ pub(crate) async fn vendor_records_reusing( vendor::prestage::sweep_stale(&common.cwd).await; } - let crawler_options = common.crawler_options(); - // Resolve installed packages with the qualified-purl-aware resolver, never - // a base-keyed one: the manifest keys release-variant ecosystems (gem - // `?platform=`, pypi `?artifact_id=`, maven `?classifier=&ext=`) by - // *qualified* purls, but the crawler only knows the *base* purl, so a - // base-keyed map would misclassify every installed qualified purl as - // "not installed". The rollback variant fans each base path back out to - // every qualified manifest purl. - let mut all_packages: HashMap = find_packages_for_rollback_reusing( - &vendorable_partition, - &crawler_options, - common.silent || common.json, - prior, - ) - .await - .into_iter() - .map(|(purl, dir)| (purl, StagedSource::Installed(dir))) - .collect(); - - // An npm alias is installed under its dependency key, not its actual - // package name. The targeted resolver probes canonical paths; before - // fetching a supposedly missing source, resolve aliases by the installed - // package.json identity. This also permits offline binary Bun vendoring. - let missing_npm: Vec<&String> = vendorable_partition - .get(&Ecosystem::Npm) + // An eject already selected exact patch UUIDs from the lockfiles. Its + // artifacts come from the service: installed copies (including an + // unrelated global Python/Ruby environment) cannot select or reject + // them. Leave their sources missing and let the backends read the locks. + let mut all_packages: HashMap = HashMap::new(); + if eject.is_none() { + let crawler_options = common.crawler_options(); + // Resolve installed packages with the qualified-purl-aware resolver, never + // a base-keyed one: the manifest keys release-variant ecosystems (gem + // `?platform=`, pypi `?artifact_id=`, maven `?classifier=&ext=`) by + // *qualified* purls, but the crawler only knows the *base* purl, so a + // base-keyed map would misclassify every installed qualified purl as + // "not installed". The rollback variant fans each base path back out to + // every qualified manifest purl. + all_packages = find_packages_for_rollback_reusing( + &vendorable_partition, + &crawler_options, + common.silent || common.json, + prior, + ) + .await .into_iter() - .flatten() - .filter(|p| !all_packages.contains_key(*p)) + .map(|(purl, dir)| (purl, StagedSource::Installed(dir))) .collect(); - let by_identity = match prior.and_then(|p| p.packages_for(&crawler_options)) { - Some(installed) => npm_paths_by_identity_in(installed, &missing_npm), - None => npm_paths_by_identity(&crawler_options, &missing_npm).await, - }; - for (purl, paths) in by_identity { - all_packages.insert(purl, StagedSource::Installed(paths[0].clone())); + + // An npm alias is installed under its dependency key, not its actual + // package name. The targeted resolver probes canonical paths; before + // fetching a supposedly missing source, resolve aliases by the installed + // package.json identity. This also permits offline binary Bun vendoring. + let missing_npm: Vec<&String> = vendorable_partition + .get(&Ecosystem::Npm) + .into_iter() + .flatten() + .filter(|p| !all_packages.contains_key(*p)) + .collect(); + let by_identity = match prior.and_then(|p| p.packages_for(&crawler_options)) { + Some(installed) => npm_paths_by_identity_in(installed, &missing_npm), + None => npm_paths_by_identity(&crawler_options, &missing_npm).await, + }; + for (purl, paths) in by_identity { + all_packages.insert(purl, StagedSource::Installed(paths[0].clone())); + } } let vendored_installs = drop_vendored_installs_by(&common.cwd, &mut all_packages, |source| match source { @@ -2884,10 +2920,13 @@ pub(crate) async fn vendor_records_reusing( // the lockfiles still pin hosted must restore its upstream registry // entry FIRST (see the dispatch loop below). Discovered once, before any // write of this run. - let hosted_pins: Vec = - socket_patch_core::patch::redirect::upstream::HostedPin::all( - &crate::commands::discover_wiring(common, &common.cwd).await, - ); + // Eject restored all its pins in one transaction before calling us; + // there is no per-package takeover or second discovery to perform. + let hosted_pins = if eject.is_some() { + Vec::new() + } else { + HostedPin::all(&crate::commands::discover_wiring(common, &common.cwd).await) + }; let hosted_pin_of = |purl: &str| { hosted_pins .iter() diff --git a/crates/socket-patch-cli/src/commands/vendored_backend/mod.rs b/crates/socket-patch-cli/src/commands/vendored_backend/mod.rs index 0830ec6b1..edf085bdb 100644 --- a/crates/socket-patch-cli/src/commands/vendored_backend/mod.rs +++ b/crates/socket-patch-cli/src/commands/vendored_backend/mod.rs @@ -72,6 +72,9 @@ pub(crate) struct ApplyRequest<'a> { /// rollback undoes exactly these), and the wired flavors. With it, the /// run's human summary and "Next steps:" are left to the eject, which /// prints them only once it knows the eject stands. + /// The records are exact lockfile-selected patches with their hosted + /// wiring already restored upstream; skip installed-package discovery + /// and per-package hosted takeover. pub(crate) eject: Option<&'a mut EjectCapture>, } diff --git a/crates/socket-patch-cli/src/lib.rs b/crates/socket-patch-cli/src/lib.rs index 9b04330af..66d68faad 100644 --- a/crates/socket-patch-cli/src/lib.rs +++ b/crates/socket-patch-cli/src/lib.rs @@ -87,11 +87,12 @@ pub enum Commands { /// Generate OpenVEX for vulnerabilities addressed by verified patches Vex(commands::vex::VexArgs), - /// Store patched dependencies in .socket/vendor/ for offline installs + /// Download hosted patches from lockfiles into .socket/vendor/ /// - /// Rewire dependency files to use the committed artifacts. Unpatched - /// dependencies still need their normal registry or cache. Use --revert - /// to undo vendoring. + /// Works before installing dependencies and rewires dependency files to + /// use the committed artifacts. Falls back to the agent manifest when no + /// hosted patches are in scope. Unpatched dependencies still need their + /// normal registry or cache. Use --revert to undo vendoring. Vendor(commands::vendor::VendorArgs), /// Reapply agent-mode patches after installing dependencies diff --git a/crates/socket-patch-cli/tests/in_process_vendor_pnpm_takeover.rs b/crates/socket-patch-cli/tests/in_process_vendor_pnpm_takeover.rs index d8b9c172a..7d4bcc49d 100644 --- a/crates/socket-patch-cli/tests/in_process_vendor_pnpm_takeover.rs +++ b/crates/socket-patch-cli/tests/in_process_vendor_pnpm_takeover.rs @@ -637,7 +637,21 @@ async fn vendor_dry_run_over_hosted_pnpm_catalog_dep_previews_the_refusal() { .unwrap(); let hosted = snapshot(root); let vendor = |extra: &[&str]| { - let mut args = vec!["vendor", "--json", "--cwd", root.to_str().unwrap()]; + let uri = server.uri(); + let mut args = vec![ + "vendor", + "--json", + "--cwd", + root.to_str().unwrap(), + "--api-url", + &uri, + "--api-token", + "fake", + "--org", + ORG, + "--patch-server-url", + &uri, + ]; args.extend_from_slice(extra); run_json(root, &server.uri(), &args) }; diff --git a/crates/socket-patch-cli/tests/vendor/npm_fixture.rs b/crates/socket-patch-cli/tests/vendor/npm_fixture.rs index c0c6ba9c1..bb9b7c6f1 100644 --- a/crates/socket-patch-cli/tests/vendor/npm_fixture.rs +++ b/crates/socket-patch-cli/tests/vendor/npm_fixture.rs @@ -289,6 +289,22 @@ pub(crate) fn pin_hosted(fx: &NpmFixture) -> Vec { /// A wiremock npm registry answering left-pad@1.3.0's version document. pub(crate) async fn mock_npm_registry() -> MockServer { let server = MockServer::start().await; + let mut view = patch_record( + &compute_git_sha256_from_bytes(ORIG_INDEX), + &compute_git_sha256_from_bytes(PATCHED_INDEX), + ); + view["purl"] = json!(PURL); + view["publishedAt"] = json!("2026-01-01T00:00:00Z"); + view["files"]["package/index.js"]["blobContent"] = json!(base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + PATCHED_INDEX + )); + prebuilt_common::mount_view(&server, &view, None).await; + Mock::given(method("GET")) + .and(path(format!("/v0/orgs/test-org/patches/view/{UUID}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(view)) + .mount(&server) + .await; Mock::given(method("GET")) .and(path("/left-pad/1.3.0")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ @@ -306,15 +322,14 @@ pub(crate) async fn mock_npm_registry() -> MockServer { } /// `vendor --json --patch-server-url ` through the binary, -/// ONLINE against the mock registry (the upstream restore needs it) but -/// anonymous, so no other network path opens. +/// ONLINE against the mock registry and patch service. pub(crate) fn vendor_online(fx: &NpmFixture, registry: &str, extra: &[&str]) -> (i32, Value) { let root = fx.root().to_str().unwrap(); let mut args = vec![ "vendor", "--json", "--patch-server-url", - HOSTED_ORIGIN, + registry, "--cwd", root, ]; @@ -323,7 +338,10 @@ pub(crate) fn vendor_online(fx: &NpmFixture, registry: &str, extra: &[&str]) -> fx.root(), &args, &[ - ("SOCKET_NO_API_TOKEN", "1"), + ("SOCKET_API_URL", registry), + ("SOCKET_VENDOR_URL", registry), + ("SOCKET_API_TOKEN", "fake-token"), + ("SOCKET_ORG_SLUG", "test-org"), ("SOCKET_NPM_REGISTRY", registry), ], ); diff --git a/crates/socket-patch-cli/tests/vendor/vendor_failure_paths.rs b/crates/socket-patch-cli/tests/vendor/vendor_failure_paths.rs index 2530d937e..8934b7608 100644 --- a/crates/socket-patch-cli/tests/vendor/vendor_failure_paths.rs +++ b/crates/socket-patch-cli/tests/vendor/vendor_failure_paths.rs @@ -207,23 +207,21 @@ fn corrupt_pre_v5_redirect_ledger_is_ignored_by_vendor() { ); } -/// Dry run over a purl the lockfile pins hosted: the run warns -/// `vendor_would_revert_redirect` (an UNCOUNTED advisory — dry/wet takeover -/// parity) after resolving the upstream entry exactly like a wet run -/// would, and writes nothing — no lock edit, no artifact, no ledger. +/// Dry run over a hosted pin with an agent manifest uses the same eject +/// preview as a manifest-free checkout, and writes nothing. #[tokio::test] -async fn dry_run_over_hosted_pin_warns_and_writes_nothing() { +async fn dry_run_over_hosted_pin_previews_eject_and_writes_nothing() { let fx = npm_fixture(); let hosted_lock = pin_hosted(&fx); let registry = mock_npm_registry().await; let (code, env) = vendor_online(&fx, ®istry.uri(), &["--dry-run"]); assert_eq!(code, 0, "the dry run itself succeeds: {env:#}"); - let warned = find_event(&env, "skipped", Some("vendor_would_revert_redirect")); + let warned = find_event(&env, "applied", Some("eject_planned")); assert_eq!(warned["purl"], PURL); assert_eq!( env["summary"]["skipped"], 0, - "the takeover advisory is uncounted: {env:#}" + "no package is skipped: {env:#}" ); assert_eq!( fx.lock_bytes(), @@ -242,7 +240,7 @@ async fn dry_run_over_hosted_pin_warns_and_writes_nothing() { } /// The wet twin: vendoring over the hosted pin first restores the upstream -/// registry entry (`vendor_takeover_reverted_redirect`), then vendors; the +/// registry entry, then vendors; the /// vendor ledger records the UPSTREAM entry as the original, so `vendor /// --revert` lands on the registry entry — never back on the hosted URL. #[tokio::test] @@ -254,12 +252,11 @@ async fn vendor_over_hosted_pin_restores_upstream_then_revert_returns_to_registr let (code, env) = vendor_online(&fx, ®istry.uri(), &[]); assert_eq!(code, 0, "{env:#}"); find_event(&env, "applied", None); - let warned = find_event(&env, "skipped", Some("vendor_takeover_reverted_redirect")); assert!( - warned["reason"] - .as_str() - .is_some_and(|m| m.contains("package-lock.json")), - "the advisory names the restored lockfile: {warned:#}" + !events(&env) + .iter() + .any(|e| e["errorCode"] == "vendor_takeover_reverted_redirect"), + "eject restores upstream as its own step: {env:#}" ); let lock: Value = serde_json::from_slice(&fx.lock_bytes()).unwrap(); let entry = &lock["packages"]["node_modules/left-pad"]; @@ -288,10 +285,8 @@ async fn vendor_over_hosted_pin_restores_upstream_then_revert_returns_to_registr assert_eq!(entry["integrity"], UPSTREAM_INTEGRITY, "{lock:#}"); } -/// A hosted pin whose upstream entry cannot be restored (here: `--offline`, -/// so the registry lookup is impossible) fails the purl CLOSED with -/// `redirect_revert_failed` naming the checkout remedy — vendoring over a -/// live hosted pin would record the hosted fragment as the "original". +/// An offline hosted conversion refuses before fetching records or +/// restoring upstream, even when an agent manifest is present. #[test] fn unrestorable_hosted_pin_fails_closed() { let fx = npm_fixture(); @@ -302,12 +297,11 @@ fn unrestorable_hosted_pin_fails_closed() { &["--offline", "--patch-server-url", HOSTED_ORIGIN], ); assert_eq!(code, 1, "{env:#}"); - let failed = find_event(&env, "failed", Some("redirect_revert_failed")); - assert_eq!(failed["purl"], PURL); - let detail = failed["error"].as_str().unwrap_or_default(); + assert_eq!(env["error"]["code"], "offline_eject_unavailable"); + let detail = env["error"]["message"].as_str().unwrap_or_default(); assert!( - detail.contains("git checkout") && detail.contains("package-lock.json"), - "the refusal names the checkout remedy for the lockfile: {env:#}" + detail.contains("offline") && detail.contains("re-run without --offline"), + "{env:#}" ); assert_eq!( fx.lock_bytes(), diff --git a/crates/socket-patch-cli/tests/vendor/vendor_render.rs b/crates/socket-patch-cli/tests/vendor/vendor_render.rs index 471f9871a..bfba61ca1 100644 --- a/crates/socket-patch-cli/tests/vendor/vendor_render.rs +++ b/crates/socket-patch-cli/tests/vendor/vendor_render.rs @@ -464,11 +464,9 @@ fn human_corrupt_pre_v5_redirect_ledger_vendors_normally() { ); } -/// Human unrestorable-hosted-pin surface: an offline run over a hosted pin -/// prints the `Cannot vendor …: cannot restore the upstream entry` line and -/// counts the refusal. +/// Human offline-hosted surface: an early refusal names the offline gate. #[test] -fn human_unrestorable_hosted_pin_prints_cannot_restore() { +fn human_offline_hosted_pin_reports_eject_refusal() { let fx = npm_fixture(); let hosted_lock = pin_hosted(&fx); @@ -476,13 +474,12 @@ fn human_unrestorable_hosted_pin_prints_cannot_restore() { human_vendor(&fx, &["--offline", "--patch-server-url", HOSTED_ORIGIN]); assert_eq!(code, 1, "stdout:\n{stdout}\nstderr:\n{stderr}"); assert!( - stderr.contains("Cannot vendor pkg:npm/left-pad@1.3.0:") - && stderr.contains("cannot restore the upstream entry"), - "the human takeover-refusal line: {stderr}" + stderr.contains("offline_eject_unavailable") && stderr.contains("re-run without --offline"), + "the human eject-refusal line: {stderr}" ); assert!( - stdout.contains("Vendored 0 packages; 1 failed."), - "the fail-closed refusal is counted: {stdout}" + !stdout.contains("Vendored"), + "an early refusal never starts vendoring: {stdout}" ); assert_eq!(fx.lock_bytes(), hosted_lock, "lock untouched"); } diff --git a/crates/socket-patch-cli/tests/vendor_eject.rs b/crates/socket-patch-cli/tests/vendor_eject.rs index 8218d4ef4..3c72cf75b 100644 --- a/crates/socket-patch-cli/tests/vendor_eject.rs +++ b/crates/socket-patch-cli/tests/vendor_eject.rs @@ -10,8 +10,7 @@ //! Every run goes through the built binary with a scrubbed environment: the //! API, the npm registry (`SOCKET_NPM_REGISTRY`) and the patch-server origin //! (`SOCKET_PATCH_SERVER_URL`, what makes the mock hosted URL count as -//! hosted) all point at a wiremock; `SOCKET_VENDOR_SOURCE=build` keeps the -//! vendoring service out of it. +//! hosted) all point at a wiremock, including the prebuilt artifact service. #[path = "common/envelope.rs"] mod envelope; @@ -271,6 +270,45 @@ async fn eject_vendors_hosted_pins_and_revert_returns_to_upstream() { assert!(!p.redirect_state().exists()); } +/// Hosted pins remain the input when an agent manifest is present. The +/// manifest does not list this patch and nothing is installed locally. +#[tokio::test] +async fn eject_reads_hosted_lockfiles_even_with_a_manifest() { + let p = hosted_project(true).await; + std::fs::remove_dir_all(p.root().join("node_modules")).unwrap(); + std::fs::create_dir(p.root().join(".socket")).unwrap(); + let manifest_path = p.root().join(".socket/manifest.json"); + let manifest = b"{\"patches\": {}}\n"; + std::fs::write(&manifest_path, manifest).unwrap(); + mock_registry(&p).await; + mock_view(&p).await; + + let original_lock = p.lock_bytes(); + let (code, preview) = p.run_json(&["vendor", "--dry-run"]); + assert_eq!(code, 0, "{preview:#}"); + assert_eq!( + find_event(&preview, "applied", Some("eject_planned"))["purl"], + PURL + ); + assert_eq!(p.lock_bytes(), original_lock); + assert!(!p.artifact().exists()); + + let (code, env) = p.run_json(&["vendor"]); + assert_eq!(code, 0, "{env:#}"); + assert_eq!(find_event(&env, "applied", None)["purl"], PURL); + assert!(p.artifact().is_file()); + assert!(p.lock()["packages"]["node_modules/left-pad"]["resolved"] + .as_str() + .unwrap() + .contains(".socket/vendor/")); + assert_eq!(std::fs::read(&manifest_path).unwrap(), manifest); + assert!(!p.root().join("node_modules").exists()); + let state: Value = + serde_json::from_slice(&std::fs::read(p.root().join(".socket/vendor/state.json")).unwrap()) + .unwrap(); + assert_eq!(state["entries"][PURL]["detached"], true); +} + /// Human eject: the first line announces the eject. #[tokio::test] async fn eject_human_output_announces_the_eject() { diff --git a/crates/socket-patch-cli/tests/vendor_eject_bun_lockb.rs b/crates/socket-patch-cli/tests/vendor_eject_bun_lockb.rs index 6214e7314..5b9c6f763 100644 --- a/crates/socket-patch-cli/tests/vendor_eject_bun_lockb.rs +++ b/crates/socket-patch-cli/tests/vendor_eject_bun_lockb.rs @@ -199,8 +199,8 @@ async fn hosted_project(writer: &str) -> Project { project } -/// `.socket/manifest.json` + the after-hash blob: the per-purl takeover -/// (instead of the manifest-less eject). +/// Legacy agent state alongside the hosted pins. Vendor still selects +/// the lockfile patches and runs the same eject as a manifest-free repo. fn stage_record(root: &Path) { let socket = root.join(".socket"); std::fs::create_dir_all(socket.join("blobs")).unwrap(); @@ -279,9 +279,7 @@ async fn takeover_vendors_over_a_hosted_bun_lockb_and_reverts_exactly() { let (code, env) = p.run_json(&["vendor", "--dry-run"]); assert_eq!(code, 0, "{writer}: dry run: {env:#}"); assert!( - codes(&env) - .iter() - .any(|c| c == "vendor_would_revert_redirect"), + codes(&env).iter().any(|c| c == "eject_planned"), "{writer}: {env:#}" ); assert_eq!(p.lock(), hosted, "{writer}: a dry run writes nothing"); @@ -292,7 +290,7 @@ async fn takeover_vendors_over_a_hosted_bun_lockb_and_reverts_exactly() { assert_eq!(env["summary"]["applied"], 1, "{writer}: {env:#}"); let codes = codes(&env); assert!( - codes + !codes .iter() .any(|c| c == "vendor_takeover_reverted_redirect"), "{writer}: {env:#}" @@ -395,18 +393,11 @@ async fn rollback_and_offline_vendor_refuse_with_the_checkout_remedy() { let (code, env) = p.run_json(&["vendor", "--offline"]); assert_eq!(code, 1, "{env:#}"); - let refused = env["events"] - .as_array() - .and_then(|events| { - events - .iter() - .find(|e| e["errorCode"] == "redirect_revert_failed") - }) - .unwrap_or_else(|| panic!("expected redirect_revert_failed: {env:#}")); + assert_eq!(env["error"]["code"], "offline_eject_unavailable", "{env:#}"); assert!( - refused["error"] + env["error"]["message"] .as_str() - .is_some_and(|e| e.contains("offline") && e.contains("git checkout -- bun.lockb")), + .is_some_and(|e| e.contains("re-run without --offline")), "{env:#}" ); assert_eq!(p.lock(), hosted, "a refused vendor writes nothing"); @@ -487,7 +478,7 @@ async fn takeover_and_revert_keep_the_bunfig_registry_tarball_url() { assert_eq!(env["summary"]["applied"], 1, "round {round}: {env:#}"); let codes = codes(&env); assert!( - codes + !codes .iter() .any(|c| c == "vendor_takeover_reverted_redirect"), "round {round}: {env:#}" diff --git a/crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs b/crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs index dce1abede..28e0d3244 100644 --- a/crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs +++ b/crates/socket-patch-cli/tests/vendor_eject_fresh_checkout.rs @@ -2,10 +2,8 @@ //! and lockfiles exist — no `node_modules`, no Cargo home, no `.socket/`. //! //! The eject is lockfile-only by contract: it restores each hosted pin's -//! upstream registry entry first, so the vendor engine sees an ordinary -//! registry pin with a registry checksum and fetches the pristine source -//! from the registry (verified against that checksum) instead of requiring -//! an installed tree. Every registry and API endpoint is a wiremock. +//! upstream registry metadata for reversal, then downloads the prebuilt +//! patched artifact. Every registry and API endpoint is a wiremock. #[path = "common/hermetic.rs"] mod hermetic; @@ -135,8 +133,7 @@ fn applied(env: &Value, purl: &str) -> bool { }) } -/// npm: a hosted package-lock.json with NO `node_modules`. The pristine -/// tarball comes from the registry the restored entry names. +/// npm: a hosted package-lock.json with NO `node_modules`. #[tokio::test] async fn npm_eject_needs_no_installed_tree() { const UUID: &str = "9f6b2c4e-1d3a-4f6b-8c2d-7e5a9b1c3d5f"; @@ -215,9 +212,114 @@ async fn npm_eject_needs_no_installed_tree() { assert!(!lock.contains(&hosted), "no hosted residue: {lock}"); } +#[tokio::test] +async fn npm_eject_downloads_every_hosted_patch_from_the_lockfile() { + let server = MockServer::start().await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + std::fs::create_dir(&root).unwrap(); + let patches = [ + ("left-pad", "1.3.0", "9f6b2c4e-1d3a-4f6b-8c2d-7e5a9b1c3d5f"), + ("ms", "2.1.3", "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d11"), + ]; + let mut lock = json!({ + "lockfileVersion": 3, + "packages": { + "": { "dependencies": { "left-pad": "1.3.0", "ms": "2.1.3", "unpatched": "1.0.0" } }, + "node_modules/unpatched": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpatched/-/unpatched-1.0.0.tgz" } + } + }); + let untouched = lock["packages"]["node_modules/unpatched"].clone(); + for (name, version, uuid) in patches { + let purl = format!("pkg:npm/{name}@{version}"); + mock_view( + &server, + uuid, + &purl, + "package/index.js", + b"original\n", + b"patched\n", + ) + .await; + Mock::given(method("GET")) + .and(path(format!("/{name}/{version}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "name": name, "version": version, + "dist": { + "tarball": format!("{}/{name}/-/{name}-{version}.tgz", server.uri()), + "integrity": "sha512-UPSTREAMupstreamUPSTREAM==" + } + }))) + .mount(&server) + .await; + // No upstream tarball endpoint: vendor needs only its metadata + // for --revert, and downloads the patched service artifact. + lock["packages"][format!("node_modules/{name}")] = json!({ + "version": version, + "resolved": format!("{}/patch/npm/{name}/{version}/55555555-5555-4555-8555-555555555555/{uuid}/{name}-{version}.tgz", server.uri()), + "integrity": "sha512-HOSTEDpatchedHOSTEDpatched==" + }); + } + std::fs::write(root.join("package.json"), serde_json::to_vec(&json!({ + "name": "fixture", "version": "1.0.0", "dependencies": lock["packages"][""]["dependencies"] + })).unwrap()).unwrap(); + std::fs::write( + root.join("package-lock.json"), + serde_json::to_vec(&lock).unwrap(), + ) + .unwrap(); + + let (code, env) = run_json(&root, &server, &["vendor"]); + assert_eq!(code, 0, "{env:#}"); + assert_eq!(env["summary"]["applied"], 2, "{env:#}"); + let lock: Value = + serde_json::from_slice(&std::fs::read(root.join("package-lock.json")).unwrap()).unwrap(); + let requests = server.received_requests().await.unwrap(); + for (name, version, uuid) in patches { + assert!( + applied(&env, &format!("pkg:npm/{name}@{version}")), + "{env:#}" + ); + let relative = format!(".socket/vendor/npm/{uuid}/{name}-{version}.tgz"); + let bytes = std::fs::read(root.join(&relative)).unwrap(); + let entry = &lock["packages"][format!("node_modules/{name}")]; + assert_eq!(entry["resolved"], format!("file:{relative}")); + assert_eq!( + entry["integrity"], + format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(Sha512::digest(&bytes)) + ) + ); + let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(bytes.as_slice())); + let mut patched = archive + .entries() + .unwrap() + .map(Result::unwrap) + .find(|file| file.path().unwrap() == Path::new("package/index.js")) + .unwrap(); + let mut content = Vec::new(); + std::io::Read::read_to_end(&mut patched, &mut content).unwrap(); + assert_eq!(content, b"patched\n"); + assert_eq!( + requests + .iter() + .filter(|r| r.url.path() == format!("/v0/orgs/{ORG}/patches/view/{uuid}")) + .count(), + 1 + ); + assert!(!requests + .iter() + .any(|r| r.url.path() == format!("/{name}/-/{name}-{version}.tgz"))); + } + assert_eq!(lock["packages"]["node_modules/unpatched"], untouched); + assert!(!root.join("node_modules").exists()); + assert!(!root.join(".socket/manifest.json").exists()); +} + /// cargo: the hosted `Cargo.lock` / `Cargo.toml` pin with an EMPTY Cargo /// home. The restore re-resolves the crates.io checksum from the sparse -/// index; the pristine `.crate` is downloaded and verified against it. +/// index for reversal; the service supplies the patched crate. #[tokio::test] async fn cargo_eject_needs_no_cargo_home() { const UUID: &str = "55555555-5555-5555-5555-555555555555"; @@ -308,10 +410,21 @@ fn wheel(files: &[(&str, &[u8])]) -> Vec { } /// pypi: a hash-pinned hosted `requirements.txt` with NO virtualenv. The -/// restore re-resolves the release file hash from the PyPI JSON API; the -/// pristine wheel is downloaded and verified against it. +/// restore re-resolves the release file hash from the PyPI JSON API for +/// reversal; the service supplies the patched wheel. #[tokio::test] async fn pypi_eject_needs_no_virtualenv() { + pypi_eject_with_virtualenv(false).await; +} + +/// An unrelated environment must not select or reject a patch already +/// identified by the project's lockfile, even if it has the same release. +#[tokio::test] +async fn pypi_eject_ignores_unrelated_virtualenv() { + pypi_eject_with_virtualenv(true).await; +} + +async fn pypi_eject_with_virtualenv(unrelated_install: bool) { const UUID: &str = "2b1f6c1e-8d3a-4f6b-9c2d-7e5a9b1c3d11"; const PURL: &str = "pkg:pypi/six@1.16.0"; const WHEEL: &str = "six-1.16.0-py2.py3-none-any.whl"; @@ -374,28 +487,38 @@ async fn pypi_eject_needs_no_virtualenv() { ) .unwrap(); - // Nothing installed for the project: VIRTUAL_ENV names an EMPTY - // virtualenv. With no venv at all, a Python project falls back to the - // global interpreters, and on Ubuntu those carry apt's python3-six - // 1.16.0 (`six-1.16.0.egg-info`), whose bytes are not this fixture's. - let empty_venv = tmp.path().join("empty-venv"); - std::fs::create_dir_all(empty_venv.join(if cfg!(windows) { + // Nothing is installed in the project. An ambient VIRTUAL_ENV may + // belong to another project and contain unrelated bytes for six. + let venv = tmp.path().join("ambient-venv"); + let site = venv.join(if cfg!(windows) { "Lib/site-packages" } else { "lib/python3.11/site-packages" - })) - .unwrap(); + }); + std::fs::create_dir_all(&site).unwrap(); + if unrelated_install { + let metadata = site.join("six-1.16.0.dist-info"); + std::fs::create_dir(&metadata).unwrap(); + std::fs::write(metadata.join("METADATA"), "Name: six\nVersion: 1.16.0\n").unwrap(); + std::fs::write(site.join("six.py"), b"# unrelated installation\n").unwrap(); + } let (code, env) = run_json_with( &root, &server, &["vendor"], &[ ("SOCKET_PYPI_JSON_API", format!("{}/pypi", server.uri())), - ("VIRTUAL_ENV", empty_venv.display().to_string()), + ("VIRTUAL_ENV", venv.display().to_string()), ], ); assert_eq!(code, 0, "a fresh hosted pypi checkout ejects: {env:#}"); assert!(applied(&env, PURL), "{env:#}"); + if unrelated_install { + assert_eq!( + std::fs::read(site.join("six.py")).unwrap(), + b"# unrelated installation\n" + ); + } let reqs = std::fs::read_to_string(root.join("requirements.txt")).unwrap(); assert!( !reqs.contains("patch.socket.dev"), diff --git a/docs/usage.md b/docs/usage.md index 5f9841df8..bd316b637 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -78,16 +78,18 @@ existing patches. See [gradual rollout](configuration.md#gradual-rollout). ## Vendoring and offline installs -For an existing hosted project without an agent manifest: +For an existing hosted project, including a clean checkout with no dependencies installed: ```sh socket-patch vendor --dry-run socket-patch vendor ``` -`vendor` ejects the patches referenced by the project's hosted dependency files. -When an agent manifest exists, it uses that manifest's patch set. To discover and -vendor available patches in one run, use: +`vendor` reads the hosted patch pins from the project's lockfiles, downloads their +prebuilt patched artifacts, and rewrites the dependency files to use `.socket/vendor/`. +An agent manifest or an installed package tree does not change that selection. +When no hosted patches are in scope, it retains the legacy fallback to the agent +manifest. To discover and vendor available patches in one run, use: ```sh socket-patch scan --mode vendored