From 7d789fb7956cb8716d2e8bafbf6e0330d041821f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 19:08:05 +0000 Subject: [PATCH 1/2] apt proxies: serve pool files below an area of the pool A proxy answered 404 for every pool path but `pool////`. The Debian security archive keeps its packages one level deeper, `pool/updates/main/o/openssl/...`, listed in `dists//main/binary-/Packages`, so a proxy of security.debian.org, as docs/apt.md sets one up, served no security updates. A pool path may now have an area before its component. Proxies serve it and check the file against the index of the component, the directory before the name's prefix (`main`). Hosted repositories keep their one layout: a path below an area is neither found nor deleted there. Fixes #750 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJb6V3DHD5wQrsYhKda --- .github/scripts/apt-e2e.sh | 15 ++++-- crates/formats/apt/src/layout.rs | 44 ++++++++++++++-- crates/formats/apt/src/lib.rs | 6 ++- crates/formats/apt/src/proxy.rs | 42 +++++++++++++++- crates/server/tests/apt.rs | 86 ++++++++++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 11 deletions(-) diff --git a/.github/scripts/apt-e2e.sh b/.github/scripts/apt-e2e.sh index 62a3a8a7..625a6457 100755 --- a/.github/scripts/apt-e2e.sh +++ b/.github/scripts/apt-e2e.sh @@ -12,7 +12,8 @@ # verify with either key alone, and apt-get update trusting only the # old key still has no warning; the old key is then deleted; # 3. proxy: a mock Debian mirror (dpkg-scanpackages, signed with its own -# key, by-hash copies) behind a proxy repository; apt-get installs +# key, by-hash copies, a package below pool/updates/ as in the security +# archive) behind a proxy repository; apt-get installs # through it, then installs again in a fresh container while the mirror # is down. # @@ -274,8 +275,13 @@ gpg --batch --passphrase '' --quick-gen-key 'Mock Mirror ' e gpg --export > "$work/client/mirror.gpg" mirror="$work/mirror/debian" deb mirror-pkg 3.0-1 amd64 -mkdir -p "$mirror/pool/main/m/mirror-pkg" "$mirror/dists/bookworm/main/binary-amd64" +# A second package where the Debian security archive keeps its packages, +# below `pool/updates/` and listed in `main`'s Packages. +deb security-pkg 1.0-1+deb12u1 amd64 +mkdir -p "$mirror/pool/main/m/mirror-pkg" "$mirror/pool/updates/main/s/security-pkg" \ + "$mirror/dists/bookworm/main/binary-amd64" cp "$work/debs/mirror-pkg_3.0-1_amd64.deb" "$mirror/pool/main/m/mirror-pkg/" +cp "$work/debs/security-pkg_1.0-1+deb12u1_amd64.deb" "$mirror/pool/updates/main/s/security-pkg/" (cd "$mirror" && dpkg-scanpackages --arch amd64 pool > dists/bookworm/main/binary-amd64/Packages 2>/dev/null) gzip -9kn "$mirror/dists/bookworm/main/binary-amd64/Packages" { @@ -310,8 +316,9 @@ proxy_auth="machine $base/repository/debian-proxy\nlogin admin@example.com\npass proxy_source=$(apt_source "deb [signed-by=/client/mirror.gpg] $proxy/ bookworm main" "$proxy_auth") apt_container proxy-install "$proxy_source apt-get update -apt-get install -y mirror-pkg -mirror-pkg | grep -qx 'mirror-pkg 3.0-1'" +apt-get install -y mirror-pkg security-pkg +mirror-pkg | grep -qx 'mirror-pkg 3.0-1' +security-pkg | grep -qx 'security-pkg 1.0-1+deb12u1'" step "proxy: installs again while the mirror is down" kill "$mirror_pid" diff --git a/crates/formats/apt/src/layout.rs b/crates/formats/apt/src/layout.rs index 6b864b00..91453d26 100644 --- a/crates/formats/apt/src/layout.rs +++ b/crates/formats/apt/src/layout.rs @@ -5,7 +5,9 @@ //! `.xz`), and their `by-hash/SHA256/` copies; //! * `pool////__.deb`, the //! prefix being the name's first letter (`lib` and the next letter for a -//! `lib` name), the version the normalized one (an epoch keeps its `:`); +//! `lib` name), the version the normalized one (an epoch keeps its `:`). +//! An upstream may put more before the component: the Debian security +//! archive's pool is `pool/updates/main/…`, which proxies serve; //! * `key.asc` and `key.gpg`, the repository's signing keys (hosted). //! //! Request paths are parsed with [`forge_plugin_api::files`]: a path that @@ -42,10 +44,16 @@ pub enum Target { Other(String), } -/// A file under `pool/`: `pool////`. +/// A file under `pool/`: +/// `pool/[/]///`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct PoolPath { pub path: String, + /// What comes between `pool/` and the component: nothing in the layout + /// hosted repositories and `dpkg`'s tools write, `updates` in the Debian + /// security archive's (`pool/updates/main/o/openssl/…`, listed in + /// `dists//main/binary-/Packages`). + pub area: String, pub component: String, pub prefix: String, pub directory: String, @@ -80,7 +88,8 @@ fn classify(path: String) -> Target { rest: rest.join("/"), path: path.clone(), }, - ["pool", component, prefix, directory, file] => Target::Pool(PoolPath { + ["pool", area @ .., component, prefix, directory, file] => Target::Pool(PoolPath { + area: area.join("/"), component: (*component).to_string(), prefix: (*prefix).to_string(), directory: (*directory).to_string(), @@ -190,9 +199,10 @@ pub fn by_hash_sha256(rest: &str) -> Option { } /// Whether a pool path is where a hosted package of `component` and `name` -/// lives (its prefix and directory). +/// lives (its prefix and directory, directly under `pool//`). pub fn pool_matches(pool: &PoolPath, component: &str, name: &str) -> bool { - valid_component(&pool.component) + pool.area.is_empty() + && valid_component(&pool.component) && pool.component == component && pool.prefix == prefix(name) && pool.directory == name @@ -222,12 +232,29 @@ mod tests { parse("/pool/main/h/hello/hello_1%3a2.0-1_amd64.deb").unwrap(), Target::Pool(PoolPath { path: "pool/main/h/hello/hello_1:2.0-1_amd64.deb".into(), + area: String::new(), component: "main".into(), prefix: "h".into(), directory: "hello".into(), file: "hello_1:2.0-1_amd64.deb".into(), }) ); + // The Debian security archive's pool: `updates` before the component. + assert_eq!( + parse("/pool/updates/main/o/openssl/libssl3_3.0.17-1~deb12u3_amd64.deb").unwrap(), + Target::Pool(PoolPath { + path: "pool/updates/main/o/openssl/libssl3_3.0.17-1~deb12u3_amd64.deb".into(), + area: "updates".into(), + component: "main".into(), + prefix: "o".into(), + directory: "openssl".into(), + file: "libssl3_3.0.17-1~deb12u3_amd64.deb".into(), + }) + ); + assert_eq!( + parse("/pool/main/h/hello").unwrap(), + Target::Other("pool/main/h/hello".into()) + ); assert_eq!(parse("/dists/").unwrap(), Target::Other("dists/".into())); assert_eq!(parse("/dists/x").unwrap(), Target::Other("dists/x".into())); } @@ -297,6 +324,13 @@ mod tests { panic!() }; assert_eq!(pool_package(&pool).as_deref(), Some("hello:source")); + // Below an area of the pool: a package's file, never a hosted one's. + let Target::Pool(pool) = parse("/pool/updates/main/h/hello/hello_1.0_amd64.deb").unwrap() + else { + panic!() + }; + assert_eq!(pool_package(&pool).as_deref(), Some("hello:amd64")); + assert!(!pool_matches(&pool, "main", "hello")); assert_eq!(split_id("hello:amd64"), Some(("hello", "amd64"))); assert_eq!(split_id("hello"), None); } diff --git a/crates/formats/apt/src/lib.rs b/crates/formats/apt/src/lib.rs index cf525906..8d0ae3fe 100644 --- a/crates/formats/apt/src/lib.rs +++ b/crates/formats/apt/src/lib.rs @@ -298,8 +298,12 @@ impl FormatPlugin for Apt { } } -/// The version a pool path names, at the path a hosted package of it has. +/// The version a pool path names, at the path a hosted package of it has: +/// directly under `pool//`, never below an area of the pool. fn pool_ref(pool: &PoolPath) -> Option { + if !pool.area.is_empty() { + return None; + } let deb = layout::deb_name(&pool.file)?; if !pool.file.ends_with(".deb") { return None; diff --git a/crates/formats/apt/src/proxy.rs b/crates/formats/apt/src/proxy.rs index 6fc1b6e5..dd2e3e87 100644 --- a/crates/formats/apt/src/proxy.rs +++ b/crates/formats/apt/src/proxy.rs @@ -9,7 +9,10 @@ //! package is cached the host checks it against the size and SHA-256 the //! upstream's `Packages` index of its component and architecture gives it //! (`Packages.gz`, else `.xz`, else the plain one), when there is such an -//! index and it lists the file. +//! index and it lists the file. The component is the pool directory's +//! last before the name's prefix: `main` for the Debian security +//! archive's `pool/updates/main/o/openssl/…`, which +//! `dists//main/binary-/Packages` lists. use std::io::{BufRead, BufReader, Read}; @@ -269,6 +272,43 @@ mod tests { assert_eq!((at.size, at.sha256), (None, None)); } + #[test] + fn security_archive_packages_are_checked_against_their_components_index() { + // security.debian.org: `Filename: pool/updates/main/…`, listed in + // `dists/bookworm-security/main/binary-amd64/Packages`. + let mut remote = Remote::new("http://security.example.test/debian-security"); + remote + .options + .push(("distribution".into(), "bookworm-security".into())); + let path = "pool/updates/main/o/openssl/libssl3_3.0.17-1~deb12u3_amd64.deb"; + let UpstreamStep::Fetch(fetches) = file(&remote, path, &[]) else { + panic!() + }; + assert_eq!( + fetches[0].url, + "http://security.example.test/debian-security/dists/bookworm-security/main/binary-amd64/Packages.gz" + ); + let index = format!( + "Package: libssl3\nFilename: {path}\nSize: 7\nSHA256: {}\n", + "d".repeat(64) + ); + let found = FetchResult { + fetch: Fetch::new(&fetches[0].url, Freshness::Metadata), + outcome: Fetched::Found(index.into_bytes()), + status: None, + headers: Vec::new(), + }; + let UpstreamStep::Done(UpstreamAnswer::FileAt(at)) = file(&remote, path, &[found]) else { + panic!() + }; + assert_eq!( + at.url, + format!("http://security.example.test/debian-security/{path}") + ); + assert!(at.immutable); + assert_eq!((at.size, at.sha256), (Some(7), Some("d".repeat(64)))); + } + #[test] fn urls_append_encoded_segments() { assert_eq!( diff --git a/crates/server/tests/apt.rs b/crates/server/tests/apt.rs index 23eff281..c4632ab8 100644 --- a/crates/server/tests/apt.rs +++ b/crates/server/tests/apt.rs @@ -858,3 +858,89 @@ async fn proxies_serve_the_upstream_unchanged() { 1 ); } + +/// The Debian security archive keeps its packages under `pool/updates/…` +/// (`Filename: pool/updates/main/o/openssl/…`), listed in the component's +/// own `dists//main/binary-/Packages` (#750). A hosted +/// repository's packages are only at their own path. +#[tokio::test] +async fn proxies_serve_the_security_archives_pool() { + let srv = common::start().await; + let state = Arc::new(Mirror::default()); + let url = mirror(state.clone()).await; + let libssl = deb("libssl3", "3.0.17-1~deb12u3", "amd64", "libs"); + let bad = deb("bad", "1.0", "amd64", "devel"); + let packages = format!( + "Package: libssl3\nVersion: 3.0.17-1~deb12u3\n\ + Filename: pool/updates/main/o/openssl/libssl3_3.0.17-1~deb12u3_amd64.deb\nSize: {}\nSHA256: {}\n\n\ + Package: bad\nVersion: 1.0\nFilename: pool/updates/main/b/bad/bad_1.0_amd64.deb\nSize: {}\nSHA256: {}\n", + libssl.len(), + sha256(&libssl), + bad.len(), + "0".repeat(64) + ); + { + let mut files = state.files.lock().unwrap(); + files.insert( + "debian-security/dists/bookworm-security/main/binary-amd64/Packages".into(), + packages.into_bytes(), + ); + files.insert( + "debian-security/pool/updates/main/o/openssl/libssl3_3.0.17-1~deb12u3_amd64.deb" + .into(), + libssl.clone(), + ); + files.insert( + "debian-security/pool/updates/main/b/bad/bad_1.0_amd64.deb".into(), + bad, + ); + } + let name = format!("aptsecurity{}", unique()); + create( + &srv, + json!({"name": name, "kind": "proxy", "format": "apt", + "remote_url": format!("{url}/debian-security"), + "options": {"distribution": "bookworm-security"}}), + ) + .await; + let apt = Apt::new(&srv, &name).await; + + let (status, body) = apt + .get("pool/updates/main/o/openssl/libssl3_3.0.17-1~deb12u3_amd64.deb") + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, libssl); + // Checked against the index of `main`, the component under `updates/`. + assert_eq!( + apt.get("pool/updates/main/b/bad/bad_1.0_amd64.deb").await.0, + StatusCode::BAD_GATEWAY + ); + assert!(state + .seen + .lock() + .unwrap() + .contains_key("debian-security/dists/bookworm-security/main/binary-amd64/Packages")); + + // A hosted package is not found below an area of the pool, and is not + // deleted through one. + let hosted = format!("apt{}", unique()); + create( + &srv, + json!({"name": hosted, "kind": "hosted", "format": "apt", "options": {"distribution": "bookworm"}}), + ) + .await; + let apt = Apt::new(&srv, &hosted).await; + let hello = deb("hello", "1.0-1", "amd64", "utils"); + assert_eq!(apt.post(&hello).await.0, StatusCode::CREATED); + let elsewhere = "pool/updates/main/h/hello/hello_1.0-1_amd64.deb"; + assert_eq!(apt.get(elsewhere).await.0, StatusCode::NOT_FOUND); + let resp = apt + .req(reqwest::Method::DELETE, elsewhere) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + let (status, body) = apt.get("pool/main/h/hello/hello_1.0-1_amd64.deb").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, hello); +} From 28f3fd4c7f5c6a84f466ce45a17532c505d5e3b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 19:12:23 +0000 Subject: [PATCH 2/2] Format the security archive proxy test Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJb6V3DHD5wQrsYhKda --- crates/server/tests/apt.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/server/tests/apt.rs b/crates/server/tests/apt.rs index c4632ab8..c8a586ca 100644 --- a/crates/server/tests/apt.rs +++ b/crates/server/tests/apt.rs @@ -886,8 +886,7 @@ async fn proxies_serve_the_security_archives_pool() { packages.into_bytes(), ); files.insert( - "debian-security/pool/updates/main/o/openssl/libssl3_3.0.17-1~deb12u3_amd64.deb" - .into(), + "debian-security/pool/updates/main/o/openssl/libssl3_3.0.17-1~deb12u3_amd64.deb".into(), libssl.clone(), ); files.insert(