diff --git a/src/astc/astcenc_bin.zig b/src/astc/astcenc_bin.zig index 50832543..66476573 100644 --- a/src/astc/astcenc_bin.zig +++ b/src/astc/astcenc_bin.zig @@ -11,6 +11,7 @@ const std = @import("std"); const builtin = @import("builtin"); const util = @import("../cli/util.zig"); +const cache_publish = @import("../cli/cache_publish.zig"); /// Pinned astcenc release. ARM-software tags releases WITHOUT a `v` prefix /// (e.g. `5.5.0`), and the archive embeds the same version in its name. @@ -18,6 +19,10 @@ pub const DEFAULT_VERSION = "5.5.0"; const RELEASE_BASE = "https://github.com/ARM-software/astc-encoder/releases/download"; +/// Test-only: serve releases from here (a `file://` fixture) instead of +/// `RELEASE_BASE`. Production code never sets it. +var release_base_override: ?[]const u8 = null; + /// The platform fragment in an astcenc release asset name. macOS ships a single /// universal binary; Linux/Windows split by arch. Pure — caller passes the /// target so tests don't depend on the host. @@ -60,7 +65,7 @@ pub fn candidateNames(os: std.Target.Os.Tag, arch: std.Target.Cpu.Arch) []const /// `https://...//astcenc--.zip`. Caller owns the slice. pub fn releaseUrl(allocator: std.mem.Allocator, version: []const u8, suffix: []const u8) ![]u8 { - return std.fmt.allocPrint(allocator, "{s}/{s}/astcenc-{s}-{s}.zip", .{ RELEASE_BASE, version, version, suffix }); + return std.fmt.allocPrint(allocator, "{s}/{s}/astcenc-{s}-{s}.zip", .{ release_base_override orelse RELEASE_BASE, version, version, suffix }); } /// `/astcenc/` — the archive extracts its own `bin/...` @@ -83,7 +88,17 @@ fn resolveExisting(allocator: std.mem.Allocator, ver_dir: []const u8) !?[]u8 { /// Resolve a usable astcenc path: the cached binary if present, else download + /// extract the release archive into the cache and return that. `cache_root` is /// the labelle home (e.g. from `asm_cache.getCacheRoot`). Caller owns the slice. +/// +/// Concurrency-safe across processes sharing one labelle home (CI shell runners +/// run several jobs at once): the zip is downloaded and extracted into a UNIQUE +/// staging sibling of the version dir, and the finished tree is published with +/// one directory rename (`cache_publish.publishDir`). The version dir therefore +/// either does not exist or holds a complete, executable install — a reader +/// never sees a half-extracted `bin/`. An `InstallLock` makes one process do +/// the download while the others wait and then reuse it. pub fn ensure(allocator: std.mem.Allocator, cache_root: []const u8, version: []const u8) ![]u8 { + const io = util_io(); + const cwd = std.Io.Dir.cwd(); const ver_dir = try versionDir(allocator, cache_root, version); defer allocator.free(ver_dir); @@ -96,13 +111,26 @@ pub fn ensure(allocator: std.mem.Allocator, cache_root: []const u8, version: []c const url = try releaseUrl(allocator, version, suffix); defer allocator.free(url); - const zip_path = try std.fmt.allocPrint(allocator, "{s}/astcenc.zip", .{ver_dir}); + var lock = cache_publish.InstallLock.acquire(allocator, ver_dir) catch return error.AstcencDownloadFailed; + defer lock.release(); + // Double-checked: a concurrent labelle may have published it while we + // waited for the lock. + if (try resolveExisting(allocator, ver_dir)) |p| return p; + + const staging = try cache_publish.tempSibling(allocator, ver_dir, "tmp"); + defer allocator.free(staging); + cwd.createDirPath(io, staging) catch |err| { + std.debug.print("labelle: could not create {s}: {s}\n", .{ staging, @errorName(err) }); + return error.AstcencDownloadFailed; + }; + // Removed on every exit; after a successful publish it no longer exists. + defer cwd.deleteTree(io, staging) catch {}; + + const zip_path = try std.fs.path.join(allocator, &.{ staging, "astcenc.zip" }); defer allocator.free(zip_path); std.debug.print("labelle: downloading astcenc {s}...\n url: {s}\n", .{ version, url }); - std.Io.Dir.cwd().createDirPath(util_io(), ver_dir) catch {}; - - if (!runOk(allocator, &.{ "curl", "-fSL", "-o", zip_path, url })) { + if (!runOk(allocator, &.{ "curl", "-fSL", "--connect-timeout", "30", "--max-time", "600", "-o", zip_path, url })) { std.debug.print("labelle: astcenc download failed (curl). Install astcenc under {s}/bin/\n", .{ver_dir}); return error.AstcencDownloadFailed; } @@ -114,7 +142,8 @@ pub fn ensure(allocator: std.mem.Allocator, cache_root: []const u8, version: []c // shell, that environment's GNU `tar` shadows the System32 bsdtar on PATH, and // GNU tar can't read a zip ("This does not look like a tar archive") — so the // extract would fail only for those users. The absolute path always hits the - // libarchive-backed system tar. Falls back to bare `tar` if SystemRoot is unset. + // libarchive-backed system tar. If SystemRoot is unset it tries the stock + // C:\Windows\System32\tar.exe, then bare `tar` (`unsetRootTar`). const extracted = if (builtin.os.tag == .windows) blk: { const tar_exe = windowsTarPath(allocator) catch |err| switch (err) { // The bare-`tar` fallback exists ONLY for an unset/invalid @@ -122,31 +151,59 @@ pub fn ensure(allocator: std.mem.Allocator, cache_root: []const u8, version: []c // to null). Everything else — OOM first and foremost — propagates, // so a real failure isn't silently masked by running whatever `tar` // is on PATH (reintroducing the Git Bash GNU-tar shadowing bug). - error.EnvironmentVariableMissing, error.InvalidWtf8 => try allocator.dupe(u8, "tar"), + error.EnvironmentVariableMissing, error.InvalidWtf8 => try unsetRootTar(allocator), else => return err, }; defer allocator.free(tar_exe); - break :blk runOk(allocator, &.{ tar_exe, "-xf", zip_path, "-C", ver_dir }); - } else runOk(allocator, &.{ "unzip", "-o", "-q", zip_path, "-d", ver_dir }); + break :blk runOk(allocator, &.{ tar_exe, "-xf", zip_path, "-C", staging }); + } else runOk(allocator, &.{ "unzip", "-o", "-q", zip_path, "-d", staging }); if (!extracted) { std.debug.print("labelle: astcenc extract failed. Install astcenc under {s}/bin/\n", .{ver_dir}); return error.AstcencExtractFailed; } + // Only the extracted tree is published, not the archive. + cwd.deleteFile(io, zip_path) catch {}; + + { + const staged_bin = (try resolveExisting(allocator, staging)) orelse { + std.debug.print("labelle: astcenc binary not found after extracting {s}\n", .{url}); + return error.AstcencExtractFailed; + }; + defer allocator.free(staged_bin); + // Executable BEFORE publishing: once the version dir exists another + // labelle may run the binary straight away. + if (builtin.os.tag != .windows) { + if (cwd.openFile(io, staged_bin, .{})) |f| { + defer f.close(io); + f.setPermissions(io, .fromMode(0o755)) catch {}; + } else |_| {} + } + } + _ = cache_publish.publishDir(allocator, staging, ver_dir, HasBinary{}) catch |err| { + std.debug.print("labelle: could not move the extracted astcenc into {s}: {s}\n", .{ ver_dir, @errorName(err) }); + return error.AstcencExtractFailed; + }; const bin = (try resolveExisting(allocator, ver_dir)) orelse { - std.debug.print("labelle: astcenc binary not found after extracting {s}\n", .{zip_path}); + std.debug.print("labelle: astcenc binary missing from {s} after install\n", .{ver_dir}); return error.AstcencExtractFailed; }; - if (builtin.os.tag != .windows) { - if (std.Io.Dir.cwd().openFile(util_io(), bin, .{})) |f| { - defer f.close(util_io()); - f.setPermissions(util_io(), .fromMode(0o755)) catch {}; - } else |_| {} - } std.debug.print(" cached at {s}\n", .{bin}); return bin; } +/// `publishDir` validator: the version dir holds one of the candidate binaries. +const HasBinary = struct { + pub fn valid(_: HasBinary, dir: []const u8) bool { + const p = resolveExisting(std.heap.page_allocator, dir) catch return false; + if (p) |path| { + std.heap.page_allocator.free(path); + return true; + } + return false; + } +}; + fn util_io() std.Io { return @import("../cli/config.zig").globalIo(); } @@ -170,6 +227,16 @@ fn windowsTarPath(allocator: std.mem.Allocator) ![]u8 { return std.fmt.allocPrint(allocator, "{s}\\System32\\tar.exe", .{root}); } +/// The tar to use when SystemRoot/windir are unset: the stock +/// `C:\Windows\System32\tar.exe` bsdtar if it is there (the usual location, +/// and what an empty environment — e.g. under `zig build test` — still has), +/// else bare `tar`. Caller owns the slice. +fn unsetRootTar(allocator: std.mem.Allocator) ![]u8 { + const stock = "C:\\Windows\\System32\\tar.exe"; + if (util.fileExists(stock)) return allocator.dupe(u8, stock); + return allocator.dupe(u8, "tar"); +} + fn runOk(allocator: std.mem.Allocator, argv: []const []const u8) bool { const r = util.runCmd(allocator, argv) catch return false; defer allocator.free(r.stdout); @@ -213,3 +280,126 @@ test "releaseUrl + versionDir shapes" { try std.testing.expect(std.mem.endsWith(u8, vd, "5.5.0")); try std.testing.expect(std.mem.indexOf(u8, vd, "astcenc") != null); } + +// ── Concurrency (shared labelle home, CI shell runners) ───────────────────── + +const ConcurrentEnsure = struct { + const version = "0.0.1-fixture"; + + fn fileUrl(a: std.mem.Allocator, abs: []const u8) ![]u8 { + const fwd = try a.dupe(u8, abs); + defer a.free(fwd); + std.mem.replaceScalar(u8, fwd, '\\', '/'); + return if (fwd.len > 0 and fwd[0] == '/') + std.fmt.allocPrint(a, "file://{s}", .{fwd}) + else + std.fmt.allocPrint(a, "file:///{s}", .{fwd}); + } + + /// Zip `/bin` into `zip_path`: bsdtar on Windows/macOS, `zip` on + /// Linux (GNU tar cannot write zips). Null if no tool is available. + fn makeZip(a: std.mem.Allocator, src: []const u8, zip_path: []const u8) !bool { + if (builtin.os.tag == .windows) { + const tar_exe = windowsTarPath(a) catch try unsetRootTar(a); + defer a.free(tar_exe); + return runOk(a, &.{ tar_exe, "-a", "-cf", zip_path, "-C", src, "bin" }); + } + if (builtin.os.tag == .macos) return runOk(a, &.{ "tar", "-a", "-cf", zip_path, "-C", src, "bin" }); + // `zip` stores paths relative to its cwd; -j would flatten, so cd via sh. + const script = try std.fmt.allocPrint(a, "cd '{s}' && zip -q -r '{s}' bin", .{ src, zip_path }); + defer a.free(script); + return runOk(a, &.{ "sh", "-c", script }); + } + + const Worker = struct { + root: []const u8, + failures: *std.atomic.Value(u32), + fn run(w: Worker) void { + const p = ensure(std.heap.page_allocator, w.root, version) catch |err| { + std.debug.print("concurrent astcenc ensure failed: {s}\n", .{@errorName(err)}); + _ = w.failures.fetchAdd(1, .monotonic); + return; + }; + std.heap.page_allocator.free(p); + } + }; +}; + +test "astcenc ensure: parallel installs into one labelle home all succeed with a complete tree" { + const a = std.testing.allocator; + const io = util_io(); + const cwd = std.Io.Dir.cwd(); + const version = ConcurrentEnsure.version; + const suffix = assetSuffix(builtin.os.tag, builtin.cpu.arch) catch return error.SkipZigTest; + if (!runOk(a, &.{ "curl", "--version" })) return error.SkipZigTest; + if (builtin.os.tag == .linux and !runOk(a, &.{ "unzip", "-v" })) return error.SkipZigTest; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(root); + + // Fixture archive: bin/, served as + // //astcenc--.zip. + const src = try std.fs.path.join(a, &.{ root, "src" }); + defer a.free(src); + const src_bin = try std.fs.path.join(a, &.{ src, "bin" }); + defer a.free(src_bin); + try cwd.createDirPath(io, src_bin); + const payload = try a.alloc(u8, 2 << 20); + defer a.free(payload); + for (payload, 0..) |*b, i| b.* = @truncate(i *% 2654435761 >> 5); + for (candidateNames(builtin.os.tag, builtin.cpu.arch)) |name| { + const p = try std.fs.path.join(a, &.{ src_bin, name }); + defer a.free(p); + try cwd.writeFile(io, .{ .sub_path = p, .data = payload }); + } + const rel_ver = try std.fs.path.join(a, &.{ root, "rel", version }); + defer a.free(rel_ver); + try cwd.createDirPath(io, rel_ver); + const zip_name = try std.fmt.allocPrint(a, "astcenc-{s}-{s}.zip", .{ version, suffix }); + defer a.free(zip_name); + const zip_path = try std.fs.path.join(a, &.{ rel_ver, zip_name }); + defer a.free(zip_path); + if (!(try ConcurrentEnsure.makeZip(a, src, zip_path))) return error.SkipZigTest; + + const rel = try std.fs.path.join(a, &.{ root, "rel" }); + defer a.free(rel); + const base = try ConcurrentEnsure.fileUrl(a, rel); + defer a.free(base); + release_base_override = base; + defer release_base_override = null; + + const home = try std.fs.path.join(a, &.{ root, "home" }); + defer a.free(home); + var failures: std.atomic.Value(u32) = .init(0); + var threads: [6]std.Thread = undefined; + for (&threads) |*t| t.* = try std.Thread.spawn(.{}, ConcurrentEnsure.Worker.run, .{ConcurrentEnsure.Worker{ .root = home, .failures = &failures }}); + for (threads) |t| t.join(); + try std.testing.expectEqual(@as(u32, 0), failures.load(.monotonic)); + + // Every candidate is complete (no truncated binary from a racing extract). + const ver_dir = try versionDir(a, home, version); + defer a.free(ver_dir); + for (candidateNames(builtin.os.tag, builtin.cpu.arch)) |name| { + const p = try std.fs.path.join(a, &.{ ver_dir, "bin", name }); + defer a.free(p); + const got = try cwd.readFileAlloc(io, p, a, .limited(8 << 20)); + defer a.free(got); + try std.testing.expectEqualSlices(u8, payload, got); + } + // Only the version dir and its lock remain under /astcenc, and the + // archive was not published into the version dir. + const parent = std.fs.path.dirname(ver_dir).?; + var dir = try cwd.openDir(io, parent, .{ .iterate = true }); + defer dir.close(io); + var it = dir.iterate(); + while (try it.next(io)) |e| { + const ok = std.mem.eql(u8, e.name, version) or std.mem.eql(u8, e.name, version ++ ".lock"); + if (!ok) std.debug.print("leftover under astcenc/: {s}\n", .{e.name}); + try std.testing.expect(ok); + } + const zip_in_cache = try std.fs.path.join(a, &.{ ver_dir, "astcenc.zip" }); + defer a.free(zip_in_cache); + try std.testing.expect(!util.fileExists(zip_in_cache)); +} diff --git a/src/cli.zig b/src/cli.zig index 9e05a410..f11b54f1 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -471,6 +471,14 @@ test { _ = @import("cli/project_lock.zig"); } +// Concurrency-safe cache publication (shared LABELLE_HOME on CI runners) and +// the downloaders built on it. +test { + _ = @import("cli/cache_publish.zig"); + _ = @import("cli/assembler.zig"); + _ = @import("astc/astcenc_bin.zig"); +} + // `labelle init` stamps the running CLI as `--labelle-version`. pub const InitCliVersionSpec = init.InitCliVersionSpec; diff --git a/src/cli/assembler.zig b/src/cli/assembler.zig index a66f0384..1f894bbc 100644 --- a/src/cli/assembler.zig +++ b/src/cli/assembler.zig @@ -33,10 +33,19 @@ const config = @import("config.zig"); const launcher_manifest = @import("launcher_manifest.zig"); const util = @import("util.zig"); const runner = @import("runner.zig"); +const cache_publish = @import("cache_publish.zig"); /// GitHub release URL template for assembler binaries. const ASSEMBLER_RELEASE_BASE = "https://github.com/labelle-toolkit/labelle-assembler/releases/download"; +/// Test-only: serve releases from here (a `file://` fixture) instead of +/// `ASSEMBLER_RELEASE_BASE`. Production code never sets it. +var release_base_override: ?[]const u8 = null; + +fn releaseBase() []const u8 { + return release_base_override orelse ASSEMBLER_RELEASE_BASE; +} + /// The LAST assembler release published WITHOUT a `SHA256SUMS` manifest. /// /// labelle-assembler#718/#720 added the manifest to the release workflow; @@ -204,15 +213,19 @@ fn digestFor(manifest: []const u8, asset: []const u8) ?[]const u8 { /// manifest that does not list this asset, fails. That way the older /// releases start being verified the moment someone backfills a manifest /// onto them, with no code change here. +/// +/// Returns the verified digest (lowercase hex), or null for a pre-manifest +/// release that could not be verified. `downloadAssembler` uses it to check a +/// binary a concurrent process published while we downloaded. fn verifyAgainstManifest( allocator: std.mem.Allocator, version: []const u8, asset: []const u8, path: []const u8, -) !void { +) !?[64]u8 { const io = config.globalIo(); const manifest_url = try std.fmt.allocPrint(allocator, "{s}/v{s}/SHA256SUMS", .{ - ASSEMBLER_RELEASE_BASE, + releaseBase(), version, }); defer allocator.free(manifest_url); @@ -225,7 +238,7 @@ fn verifyAgainstManifest( .olderThan(compatibility.parseVersion(version)); const fetched = fetch: { - const r = util.runCmd(allocator, &.{ "curl", "-fSL", "-o", manifest_path, manifest_url }) catch break :fetch false; + const r = util.runCmd(allocator, &.{ "curl", "-fSL", "--connect-timeout", "30", "--max-time", "600", "-o", manifest_path, manifest_url }) catch break :fetch false; defer allocator.free(r.stdout); defer allocator.free(r.stderr); break :fetch switch (r.term) { @@ -244,7 +257,7 @@ fn verifyAgainstManifest( } std.debug.print("labelle: WARNING — assembler v{s} predates published checksums; the download\n", .{version}); std.debug.print(" could NOT be verified. Releases after v{s} are verified automatically.\n", .{LAST_UNVERIFIED_ASSEMBLER_VERSION}); - return; + return null; } const manifest = std.Io.Dir.cwd().readFileAlloc(io, manifest_path, allocator, .limited(64 << 10)) catch { @@ -272,6 +285,7 @@ fn verifyAgainstManifest( } std.debug.print(" checksum ok (SHA256SUMS)\n", .{}); + return expected[0..64].*; } /// Download an assembler binary from GitHub releases and cache it. @@ -288,47 +302,72 @@ fn verifyAgainstManifest( /// the CLI would `chmod +x` and execute whatever arrived /// (labelle-assembler#616). /// -/// Staging matters as much as the check. The bytes land on a `.part` sibling -/// and are moved to `dest_path` ONLY after they verify, because `dest_path` -/// is not a scratch location: it is the cache slot, and every caller decides -/// "already downloaded?" by `access()`ing exactly that path. Writing the -/// unverified download there directly — as this function used to — means a -/// failed verification would leave a file that the NEXT run treats as a good -/// cache hit and never re-downloads or re-checks. So a rejected download -/// must leave nothing at `dest_path`: nothing cached, nothing executable. +/// Staging matters as much as the check. The bytes land on a UNIQUE sibling +/// (`.tmp--`) and are moved to `dest_path` ONLY after they +/// verify, because `dest_path` is not a scratch location: it is the cache +/// slot, and every caller decides "already downloaded?" by `access()`ing +/// exactly that path. Writing the unverified download there directly — as +/// this function once did — means a failed verification would leave a file +/// that the NEXT run treats as a good cache hit and never re-downloads or +/// re-checks. So a rejected download must leave nothing at `dest_path`: +/// nothing cached, nothing executable. +/// +/// The staging name must be unique, not merely different from `dest_path`: +/// `LABELLE_HOME` is shared by every labelle process, and a CI shell runner +/// runs several jobs at once. With a fixed `.part`, two jobs downloaded +/// into the same file; one renamed it into place and the other's verification +/// found nothing to read ("could not read the download to verify it" → +/// `AssemblerNotCached`). Now each process stages privately, an +/// `InstallLock` lets only one of them download, and the publish keeps a +/// concurrent winner's verified binary instead of failing +/// (`cache_publish.zig`). pub fn downloadAssembler(allocator: std.mem.Allocator, version: []const u8, dest_path: []const u8) !void { const io = config.globalIo(); + const cwd = std.Io.Dir.cwd(); const asset = try std.fmt.allocPrint(allocator, "labelle-assembler-{s}-{s}", .{ try osName(), try archName(), }); defer allocator.free(asset); const url = try std.fmt.allocPrint(allocator, "{s}/v{s}/{s}", .{ - ASSEMBLER_RELEASE_BASE, + releaseBase(), version, asset, }); defer allocator.free(url); - std.debug.print("labelle: downloading assembler v{s}...\n", .{version}); - std.debug.print(" url: {s}\n", .{url}); - // Ensure the parent directory exists. if (std.fs.path.dirname(dest_path)) |dir| { - std.Io.Dir.cwd().createDirPath(io, dir) catch |err| { + cwd.createDirPath(io, dir) catch |err| { std.debug.print("labelle: could not create directory {s}: {any}\n", .{ dir, err }); return error.AssemblerDownloadFailed; }; } + // One download per version across processes: a concurrent labelle that + // got here first holds the lock until its binary is published (or its + // attempt failed), and we then find it in place instead of fetching again. + var lock = cache_publish.InstallLock.acquire(allocator, dest_path) catch return error.AssemblerDownloadFailed; + defer lock.release(); + if (cwd.access(io, dest_path, .{})) |_| { + std.debug.print("labelle: assembler v{s} was installed by a concurrent labelle: {s}\n", .{ version, dest_path }); + return; + } else |_| {} + + std.debug.print("labelle: downloading assembler v{s}...\n", .{version}); + std.debug.print(" url: {s}\n", .{url}); + // Staging path: everything below writes HERE, never to dest_path, until - // the bytes have been verified. See the doc comment above. - const part_path = try std.fmt.allocPrint(allocator, "{s}.part", .{dest_path}); + // the bytes have been verified. See the doc comment above. Removed on + // every exit (after a successful publish it no longer exists). + const part_path = try cache_publish.tempSibling(allocator, dest_path, "tmp"); defer allocator.free(part_path); - defer std.Io.Dir.cwd().deleteFile(io, part_path) catch {}; + defer cwd.deleteFile(io, part_path) catch {}; // Use curl with -L to follow redirects (GitHub releases redirect to S3). - const result = util.runCmd(allocator, &.{ "curl", "-fSL", "-o", part_path, url }) catch { + // Bounded so a stalled mirror cannot hang this process — or every other + // labelle waiting on the install lock — forever. + const result = util.runCmd(allocator, &.{ "curl", "-fSL", "--connect-timeout", "30", "--max-time", "600", "-o", part_path, url }) catch { std.debug.print("labelle: download failed (is curl installed?)\n", .{}); std.debug.print(" url: {s}\n", .{url}); std.debug.print(" you can download manually and place it at: {s}\n", .{dest_path}); @@ -358,30 +397,37 @@ pub fn downloadAssembler(allocator: std.mem.Allocator, version: []const u8, dest // own manifest rather than a pinned constant (a pinned constant per // assembler version would need a CLI release for every assembler // release). - try verifyAgainstManifest(allocator, version, asset, part_path); + const digest = try verifyAgainstManifest(allocator, version, asset, part_path); - // Verified: move the staged bytes into the cache slot. Only now does - // `dest_path` exist, so a caller's `access()` cache-hit check can never - // see an unverified binary. - std.Io.Dir.cwd().rename(part_path, std.Io.Dir.cwd(), dest_path, io) catch |err| { + // Executable BEFORE it is published: the moment `dest_path` exists a + // concurrent labelle may run it, so it must never appear without +x. + if (builtin.os.tag != .windows) { + if (cwd.openFile(io, part_path, .{})) |file| { + defer file.close(io); + file.setPermissions(io, .fromMode(0o755)) catch |err| { + std.debug.print("labelle: chmod failed for {s}: {any}\n", .{ part_path, err }); + }; + } else |err| { + std.debug.print("labelle: could not open {s} for chmod: {any}\n", .{ part_path, err }); + } + } + + // Verified: move the staged bytes into the cache slot with one atomic + // rename. Only now does `dest_path` exist, so a caller's `access()` + // cache-hit check can never see an unverified or partial binary. If a + // concurrent process (e.g. an older CLI that does not take the lock) + // already published a binary that verifies, theirs is kept. + const outcome = cache_publish.publishFile(part_path, dest_path, cache_publish.Sha256File{ + .sha256 = if (digest) |*d| d else null, + }) catch |err| { std.debug.print("labelle: could not move the verified download into place: {any}\n", .{err}); - std.debug.print(" staged at: {s}\n", .{part_path}); + std.debug.print(" destination: {s}\n", .{dest_path}); return error.AssemblerDownloadFailed; }; - - // Make executable on Unix. - if (builtin.os.tag != .windows) { - const file = std.Io.Dir.cwd().openFile(io, dest_path, .{}) catch |err| { - std.debug.print("labelle: could not open {s} for chmod: {any}\n", .{ dest_path, err }); - return; - }; - defer file.close(io); - file.setPermissions(io, .fromMode(0o755)) catch |err| { - std.debug.print("labelle: chmod failed for {s}: {any}\n", .{ dest_path, err }); - }; + switch (outcome) { + .published => std.debug.print(" cached at {s}\n", .{dest_path}), + .kept_existing => std.debug.print(" a concurrent labelle already cached a verified copy at {s}\n", .{dest_path}), } - - std.debug.print(" cached at {s}\n", .{dest_path}); } /// Resolve the default assembler version. Downloads it if not cached. @@ -981,3 +1027,149 @@ pub const DownloadVerification = struct { try std.testing.expect(!util.sha256Matches("abc", "BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD")); } }; + +/// Concurrent installs into ONE shared cache (FP pipeline #57: two CI jobs on +/// one shell runner fetched the same assembler into the same `.part`, and the +/// loser failed `AssemblerNotCached`). Drives the real `downloadAssembler` +/// — curl, SHA256SUMS verification, staging, publish — against a `file://` +/// release fixture, from several threads at once. +pub const ConcurrentDownload = struct { + const fixture_version = "9.9.9"; + + fn fileUrl(a: std.mem.Allocator, abs: []const u8) ![]u8 { + const fwd = try a.dupe(u8, abs); + defer a.free(fwd); + std.mem.replaceScalar(u8, fwd, '\\', '/'); + // `file:///C:/x` on Windows, `file:///tmp/x` elsewhere. + return if (fwd.len > 0 and fwd[0] == '/') + std.fmt.allocPrint(a, "file://{s}", .{fwd}) + else + std.fmt.allocPrint(a, "file:///{s}", .{fwd}); + } + + fn haveCurl() bool { + const r = util.runCmd(std.testing.allocator, &.{ "curl", "--version" }) catch return false; + defer std.testing.allocator.free(r.stdout); + defer std.testing.allocator.free(r.stderr); + return r.term == .exited and r.term.exited == 0; + } + + const Worker = struct { + dest: []const u8, + failures: *std.atomic.Value(u32), + fn run(w: Worker) void { + downloadAssembler(std.heap.page_allocator, fixture_version, w.dest) catch |err| { + std.debug.print("concurrent downloadAssembler failed: {s}\n", .{@errorName(err)}); + _ = w.failures.fetchAdd(1, .monotonic); + }; + } + }; + + test "parallel installs of one version into one cache all succeed and the binary verifies" { + if (!haveCurl()) return error.SkipZigTest; + const a = std.testing.allocator; + const io = config.globalIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(root); + + // Release fixture: /rel/v9.9.9/{, SHA256SUMS}. + const asset = try std.fmt.allocPrint(a, "labelle-assembler-{s}-{s}", .{ try osName(), try archName() }); + defer a.free(asset); + const rel_dir = try std.fs.path.join(a, &.{ root, "rel", "v" ++ fixture_version }); + defer a.free(rel_dir); + try std.Io.Dir.cwd().createDirPath(io, rel_dir); + // A few MiB so unsynchronized writers would genuinely interleave. + const payload = try a.alloc(u8, 6 << 20); + defer a.free(payload); + for (payload, 0..) |*b, i| b.* = @truncate((i *% 2654435761) >> 7); + const asset_path = try std.fs.path.join(a, &.{ rel_dir, asset }); + defer a.free(asset_path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = asset_path, .data = payload }); + const sha = try @import("provider_github/files.zig").sha256Hex(a, payload); + defer a.free(sha); + const sums = try std.fmt.allocPrint(a, "{s} {s}\n", .{ sha, asset }); + defer a.free(sums); + const sums_path = try std.fs.path.join(a, &.{ rel_dir, "SHA256SUMS" }); + defer a.free(sums_path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = sums_path, .data = sums }); + + const rel_root = try std.fs.path.join(a, &.{ root, "rel" }); + defer a.free(rel_root); + const base = try fileUrl(a, rel_root); + defer a.free(base); + release_base_override = base; + defer release_base_override = null; + + const ver_dir = try std.fs.path.join(a, &.{ root, "home", "assembler", fixture_version }); + defer a.free(ver_dir); + const dest = try std.fs.path.join(a, &.{ ver_dir, exe_name }); + defer a.free(dest); + + var failures: std.atomic.Value(u32) = .init(0); + var threads: [6]std.Thread = undefined; + for (&threads) |*t| t.* = try std.Thread.spawn(.{}, Worker.run, .{Worker{ .dest = dest, .failures = &failures }}); + for (threads) |t| t.join(); + + try std.testing.expectEqual(@as(u32, 0), failures.load(.monotonic)); + const got = try std.Io.Dir.cwd().readFileAlloc(io, dest, a, .limited(64 << 20)); + defer a.free(got); + try std.testing.expect(util.sha256Matches(got, sha)); + + // Nothing but the binary and its lock file is left in the slot. + var dir = try std.Io.Dir.cwd().openDir(io, ver_dir, .{ .iterate = true }); + defer dir.close(io); + var it = dir.iterate(); + while (try it.next(io)) |e| { + const ok = std.mem.eql(u8, e.name, exe_name) or std.mem.eql(u8, e.name, exe_name ++ ".lock"); + if (!ok) std.debug.print("leftover in cache slot: {s}\n", .{e.name}); + try std.testing.expect(ok); + } + } + + test "a rejected download leaves nothing at the final path, even with a stale lock file" { + if (!haveCurl()) return error.SkipZigTest; + const a = std.testing.allocator; + const io = config.globalIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(root); + const asset = try std.fmt.allocPrint(a, "labelle-assembler-{s}-{s}", .{ try osName(), try archName() }); + defer a.free(asset); + const rel_dir = try std.fs.path.join(a, &.{ root, "rel", "v" ++ fixture_version }); + defer a.free(rel_dir); + try std.Io.Dir.cwd().createDirPath(io, rel_dir); + const asset_path = try std.fs.path.join(a, &.{ rel_dir, asset }); + defer a.free(asset_path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = asset_path, .data = "tampered" }); + const sums = try std.fmt.allocPrint(a, "{s} {s}\n", .{ "0" ** 64, asset }); + defer a.free(sums); + const sums_path = try std.fs.path.join(a, &.{ rel_dir, "SHA256SUMS" }); + defer a.free(sums_path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = sums_path, .data = sums }); + const rel_root = try std.fs.path.join(a, &.{ root, "rel" }); + defer a.free(rel_root); + const base = try fileUrl(a, rel_root); + defer a.free(base); + release_base_override = base; + defer release_base_override = null; + + const ver_dir = try std.fs.path.join(a, &.{ root, "home", "assembler", fixture_version }); + defer a.free(ver_dir); + const dest = try std.fs.path.join(a, &.{ ver_dir, exe_name }); + defer a.free(dest); + try std.Io.Dir.cwd().createDirPath(io, ver_dir); + const lock_path = try std.fmt.allocPrint(a, "{s}.lock", .{dest}); + defer a.free(lock_path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = lock_path, .data = "" }); + + try std.testing.expectError(error.AssemblerChecksumMismatch, downloadAssembler(a, fixture_version, dest)); + try std.testing.expect(!util.fileExists(dest)); + var dir = try std.Io.Dir.cwd().openDir(io, ver_dir, .{ .iterate = true }); + defer dir.close(io); + var it = dir.iterate(); + while (try it.next(io)) |e| try std.testing.expectEqualStrings(exe_name ++ ".lock", e.name); + } +}; diff --git a/src/cli/cache_publish.zig b/src/cli/cache_publish.zig new file mode 100644 index 00000000..819cd636 --- /dev/null +++ b/src/cli/cache_publish.zig @@ -0,0 +1,402 @@ +//! Concurrency-safe publication into the shared labelle cache. +//! +//! `LABELLE_HOME` (default `~/.labelle`) is shared by every labelle process on +//! the machine — and on a CI shell runner, by every CONCURRENT JOB. A download +//! that writes through a fixed path (`.part`, `/astcenc.zip`, +//! the version directory itself) lets two processes clobber each other: a +//! Flying Platform pipeline ran two jobs that fetched assembler v0.126.0 into +//! the same `.part`; one renamed it into place and the other failed with +//! "could not read the download to verify it" → `AssemblerNotCached`. +//! +//! Every cache writer follows the same rules: +//! +//! 1. Stage on a UNIQUE sibling of the final path (`tempSibling`). Same +//! directory → same filesystem → the publishing rename is atomic. The +//! name carries the pid and 64 random bits, so neither two processes nor +//! two threads of one process can pick the same staging path. +//! 2. Verify the STAGED bytes/tree, never the final path. +//! 3. Publish with one rename (`publishFile` / `publishDir`). If the final +//! path already holds a valid install when we finish — a concurrent +//! process won the race — keep theirs and discard ours. +//! 4. Remove the staging path on every exit. Nothing partial is ever visible +//! at the final path, so "exists" keeps meaning "complete and verified" +//! for every reader that only checks existence. +//! +//! `InstallLock` additionally serializes installs of ONE item across processes +//! so only one of them downloads (the others wait, then find it published). +//! For a single file it is only an optimisation — rules 1–4 are the safety +//! baseline. For a directory it also makes replacing a broken leftover tree +//! safe (see `publishDir`). +const std = @import("std"); +const builtin = @import("builtin"); +const config = @import("config.zig"); +const util = @import("util.zig"); + +/// The current process id (0 where the platform offers none). Only used to +/// make staging names readable and unique across processes; the random part +/// keeps them unique across threads of one process. +pub fn processId() u64 { + return switch (builtin.os.tag) { + .windows => std.os.windows.GetCurrentProcessId(), + .linux => @intCast(std.os.linux.getpid()), + else => if (builtin.link_libc) @intCast(std.c.getpid()) else 0, + }; +} + +/// `.--<16 hex>` — a unique sibling of `dest` (same directory, +/// so a rename from it onto `dest` is atomic). `tag` is `tmp` for staging and +/// `stale` for a broken tree moved aside. Caller owns the slice. +pub fn tempSibling(a: std.mem.Allocator, dest: []const u8, tag: []const u8) ![]u8 { + var random: [8]u8 = undefined; + config.globalIo().random(&random); + return std.fmt.allocPrint(a, "{s}.{s}-{d}-{s}", .{ + std.mem.trimEnd(u8, dest, "/\\"), + tag, + processId(), + std.fmt.bytesToHex(random, .lower), + }); +} + +/// True if `name` (a bare file name) is a staging/stale sibling of `base` +/// made by `tempSibling`. Tests use it to assert nothing was left behind. +pub fn isSiblingName(name: []const u8, base: []const u8) bool { + if (!std.mem.startsWith(u8, name, base) or name.len == base.len) return false; + const rest = name[base.len..]; + return std.mem.startsWith(u8, rest, ".tmp-") or std.mem.startsWith(u8, rest, ".stale-"); +} + +/// An exclusive advisory lock on `.lock`, held across a download + +/// verify + publish so concurrent labelle processes install one item once. +/// Blocking; the OS releases it if the holder dies. The lock file itself is +/// empty and harmless, and is left in place (deleting it would race a waiter +/// that already opened it). +pub const InstallLock = struct { + file: std.Io.File, + + pub fn acquire(a: std.mem.Allocator, dest: []const u8) !InstallLock { + const io = config.globalIo(); + const cwd = std.Io.Dir.cwd(); + const lock_path = try std.fmt.allocPrint(a, "{s}.lock", .{std.mem.trimEnd(u8, dest, "/\\")}); + defer a.free(lock_path); + if (std.fs.path.dirname(lock_path)) |dir| try cwd.createDirPath(io, dir); + const file = cwd.createFile(io, lock_path, .{ .truncate = false, .lock = .exclusive }) catch |err| { + std.debug.print("labelle: could not acquire install lock {s}: {s}\n", .{ lock_path, @errorName(err) }); + return err; + }; + return .{ .file = file }; + } + + pub fn release(self: *InstallLock) void { + self.file.close(config.globalIo()); + } +}; + +pub const Outcome = enum { + /// Our staged copy is now at the final path. + published, + /// The final path already held a valid install (a concurrent process won + /// the race); ours was discarded. + kept_existing, +}; + +/// A validator that accepts any existing path. For callers whose final path +/// is only ever produced by a verified publish (so existence implies valid). +pub const Exists = struct { + pub fn valid(_: Exists, path: []const u8) bool { + return util.fileExists(path); + } +}; + +/// A validator for a published FILE: it exists and, when `sha256` (lowercase +/// hex) is given, its bytes hash to it. +pub const Sha256File = struct { + sha256: ?[]const u8, + max_bytes: usize = 512 << 20, + + pub fn valid(self: Sha256File, path: []const u8) bool { + const expected = self.sha256 orelse return util.fileExists(path); + const data = std.Io.Dir.cwd().readFileAlloc(config.globalIo(), path, std.heap.page_allocator, .limited(self.max_bytes)) catch return false; + defer std.heap.page_allocator.free(data); + return util.sha256Matches(data, expected); + } +}; + +/// Move the verified file `staged` to `dest` with one atomic rename (replacing +/// a damaged `dest`). If `dest` already validates — before we start, or after a +/// rename that failed because a concurrent process holds it (Windows cannot +/// replace a running executable) — ours is discarded and theirs is used. +/// `staged` never survives this call. +pub fn publishFile(staged: []const u8, dest: []const u8, validator: anytype) !Outcome { + const io = config.globalIo(); + const cwd = std.Io.Dir.cwd(); + if (util.fileExists(dest) and validator.valid(dest)) { + cwd.deleteFile(io, staged) catch {}; + return .kept_existing; + } + cwd.rename(staged, cwd, dest, io) catch |err| { + cwd.deleteFile(io, staged) catch {}; + if (validator.valid(dest)) return .kept_existing; + return err; + }; + return .published; +} + +/// Move the verified directory `staged` to `dest` with one atomic rename. If +/// `dest` already validates, ours is deleted and theirs is used. If `dest` +/// exists but does NOT validate (a partial tree left by a crash, or by an +/// older CLI that extracted in place), it is moved aside in one rename, ours +/// takes its place, and the old tree is deleted — so at no point is a mix of +/// the two visible at `dest`. Callers hold the item's `InstallLock` so no +/// other labelle process replaces `dest` at the same time. `staged` never +/// survives this call. +pub fn publishDir(a: std.mem.Allocator, staged: []const u8, dest: []const u8, validator: anytype) !Outcome { + const io = config.globalIo(); + const cwd = std.Io.Dir.cwd(); + defer cwd.deleteTree(io, staged) catch {}; + if (validator.valid(dest)) return .kept_existing; + cwd.rename(staged, cwd, dest, io) catch |first_err| { + if (validator.valid(dest)) return .kept_existing; + if (!util.fileExists(dest)) return first_err; + const stale = try tempSibling(a, dest, "stale"); + defer a.free(stale); + cwd.rename(dest, cwd, stale, io) catch |err| { + if (validator.valid(dest)) return .kept_existing; + return err; + }; + defer cwd.deleteTree(io, stale) catch {}; + cwd.rename(staged, cwd, dest, io) catch |err| { + if (validator.valid(dest)) return .kept_existing; + return err; + }; + return .published; + }; + return .published; +} + +// ── Tests ──────────────────────────────────────────────────────────── + +const testing = std.testing; + +fn testRoot(tmp: *std.testing.TmpDir) ![:0]u8 { + return tmp.dir.realPathFileAlloc(config.globalIo(), ".", testing.allocator); +} + +/// Names in `dir` that are staging/stale siblings of `base`. +fn countSiblings(dir_path: []const u8, base: []const u8) !usize { + const io = config.globalIo(); + var dir = try std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true }); + defer dir.close(io); + var it = dir.iterate(); + var n: usize = 0; + while (try it.next(io)) |e| { + if (isSiblingName(e.name, base)) n += 1; + } + return n; +} + +test "tempSibling: same directory, tagged, pid-stamped and unique per call" { + const a = testing.allocator; + const dest = try std.fs.path.join(a, &.{ "cache", "assembler", "1.2.3", "labelle-assembler" }); + defer a.free(dest); + const one = try tempSibling(a, dest, "tmp"); + defer a.free(one); + const two = try tempSibling(a, dest, "tmp"); + defer a.free(two); + + try testing.expect(!std.mem.eql(u8, one, two)); + try testing.expectEqualStrings(std.fs.path.dirname(dest).?, std.fs.path.dirname(one).?); + try testing.expect(std.mem.startsWith(u8, one, dest)); + const pid = try std.fmt.allocPrint(a, ".tmp-{d}-", .{processId()}); + defer a.free(pid); + try testing.expect(std.mem.indexOf(u8, one, pid) != null); + try testing.expect(isSiblingName(std.fs.path.basename(one), "labelle-assembler")); + try testing.expect(!isSiblingName("labelle-assembler", "labelle-assembler")); + try testing.expect(!isSiblingName("labelle-assembler.lock", "labelle-assembler")); + + // A trailing separator on a directory destination does not leak in. + const dir_dest = try std.fmt.allocPrint(a, "{s}{c}", .{ std.fs.path.dirname(dest).?, std.fs.path.sep }); + defer a.free(dir_dest); + const staged_dir = try tempSibling(a, dir_dest, "stale"); + defer a.free(staged_dir); + try testing.expect(isSiblingName(std.fs.path.basename(staged_dir), "1.2.3")); +} + +test "publishFile: publishes, keeps a valid existing file, replaces a damaged one" { + const a = testing.allocator; + const io = config.globalIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try testRoot(&tmp); + defer a.free(root); + const dest = try std.fs.path.join(a, &.{ root, "item" }); + defer a.free(dest); + const good = "good bytes"; + const good_sha = try @import("provider_github/files.zig").sha256Hex(a, good); + defer a.free(good_sha); + const v: Sha256File = .{ .sha256 = good_sha }; + + // Absent → published. + const s1 = try tempSibling(a, dest, "tmp"); + defer a.free(s1); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = s1, .data = good }); + try testing.expectEqual(Outcome.published, try publishFile(s1, dest, v)); + try testing.expect(v.valid(dest)); + + // Valid existing → ours discarded, theirs kept. + const s2 = try tempSibling(a, dest, "tmp"); + defer a.free(s2); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = s2, .data = good }); + try testing.expectEqual(Outcome.kept_existing, try publishFile(s2, dest, v)); + try testing.expect(!util.fileExists(s2)); + + // Damaged existing → replaced in one rename. + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = dest, .data = "trunc" }); + const s3 = try tempSibling(a, dest, "tmp"); + defer a.free(s3); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = s3, .data = good }); + try testing.expectEqual(Outcome.published, try publishFile(s3, dest, v)); + try testing.expect(v.valid(dest)); + try testing.expectEqual(@as(usize, 0), try countSiblings(root, "item")); +} + +test "publishDir: publishes, keeps a valid tree, replaces a partial leftover" { + const a = testing.allocator; + const io = config.globalIo(); + const cwd = std.Io.Dir.cwd(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try testRoot(&tmp); + defer a.free(root); + const dest = try std.fs.path.join(a, &.{ root, "5.5.0" }); + defer a.free(dest); + const marker = try std.fs.path.join(a, &.{ dest, "bin", "tool" }); + defer a.free(marker); + const HasTool = struct { + pub fn valid(_: @This(), dir: []const u8) bool { + var buf: [1024]u8 = undefined; + const p = std.fmt.bufPrint(&buf, "{s}{c}bin{c}tool", .{ dir, std.fs.path.sep, std.fs.path.sep }) catch return false; + return util.fileExists(p); + } + }; + + const stage = struct { + fn make(alloc: std.mem.Allocator, d: []const u8, content: []const u8) ![]u8 { + const s = try tempSibling(alloc, d, "tmp"); + const bin = try std.fs.path.join(alloc, &.{ s, "bin" }); + defer alloc.free(bin); + try std.Io.Dir.cwd().createDirPath(config.globalIo(), bin); + const f = try std.fs.path.join(alloc, &.{ bin, "tool" }); + defer alloc.free(f); + try std.Io.Dir.cwd().writeFile(config.globalIo(), .{ .sub_path = f, .data = content }); + return s; + } + }.make; + + const s1 = try stage(a, dest, "first"); + defer a.free(s1); + try testing.expectEqual(Outcome.published, try publishDir(a, s1, dest, HasTool{})); + try testing.expect(!util.fileExists(s1)); + + const s2 = try stage(a, dest, "second"); + defer a.free(s2); + try testing.expectEqual(Outcome.kept_existing, try publishDir(a, s2, dest, HasTool{})); + try testing.expect(!util.fileExists(s2)); + var buf: [16]u8 = undefined; + try testing.expectEqualStrings("first", try cwd.readFile(io, marker, &buf)); + + // A partial tree (no bin/tool) — what the old in-place extract left on a + // crash — is replaced wholesale. + try cwd.deleteTree(io, dest); + const junk = try std.fs.path.join(a, &.{ dest, "half-extracted" }); + defer a.free(junk); + try cwd.createDirPath(io, dest); + try cwd.writeFile(io, .{ .sub_path = junk, .data = "x" }); + const s3 = try stage(a, dest, "third"); + defer a.free(s3); + try testing.expectEqual(Outcome.published, try publishDir(a, s3, dest, HasTool{})); + try testing.expectEqualStrings("third", try cwd.readFile(io, marker, &buf)); + try testing.expect(!util.fileExists(junk)); + try testing.expectEqual(@as(usize, 0), try countSiblings(root, "5.5.0")); +} + +const RaceCtx = struct { + dest: []const u8, + payload: []const u8, + sha: []const u8, + dir_mode: bool, + failed: std.atomic.Value(u32) = .init(0), + + fn run(ctx: *RaceCtx) void { + ctx.once() catch |err| { + std.debug.print("race worker failed: {s}\n", .{@errorName(err)}); + _ = ctx.failed.fetchAdd(1, .monotonic); + }; + } + + fn once(ctx: *RaceCtx) !void { + const a = std.heap.page_allocator; + const io = config.globalIo(); + const cwd = std.Io.Dir.cwd(); + const staged = try tempSibling(a, ctx.dest, "tmp"); + defer a.free(staged); + if (!ctx.dir_mode) { + // Unlocked on purpose: atomic staging alone must be safe for files. + try cwd.writeFile(io, .{ .sub_path = staged, .data = ctx.payload }); + _ = try publishFile(staged, ctx.dest, Sha256File{ .sha256 = ctx.sha }); + return; + } + var lock = try InstallLock.acquire(a, ctx.dest); + defer lock.release(); + const f = try std.fs.path.join(a, &.{ staged, "payload" }); + defer a.free(f); + try cwd.createDirPath(io, staged); + try cwd.writeFile(io, .{ .sub_path = f, .data = ctx.payload }); + _ = try publishDir(a, staged, ctx.dest, DirPayload{ .sha = ctx.sha }); + } +}; + +const DirPayload = struct { + sha: []const u8, + pub fn valid(self: DirPayload, dir: []const u8) bool { + const p = std.fs.path.join(std.heap.page_allocator, &.{ dir, "payload" }) catch return false; + defer std.heap.page_allocator.free(p); + return util.fileExists(p) and (Sha256File{ .sha256 = self.sha }).valid(p); + } +}; + +fn raceTest(dir_mode: bool) !void { + const a = testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const root = try testRoot(&tmp); + defer a.free(root); + const dest = try std.fs.path.join(a, &.{ root, "shared" }); + defer a.free(dest); + // Big enough that unsynchronized writers would interleave. + const payload = try a.alloc(u8, 4 << 20); + defer a.free(payload); + for (payload, 0..) |*b, i| b.* = @truncate(i *% 131); + const sha = try @import("provider_github/files.zig").sha256Hex(a, payload); + defer a.free(sha); + + var ctx: RaceCtx = .{ .dest = dest, .payload = payload, .sha = sha, .dir_mode = dir_mode }; + var threads: [8]std.Thread = undefined; + for (&threads) |*t| t.* = try std.Thread.spawn(.{}, RaceCtx.run, .{&ctx}); + for (threads) |t| t.join(); + + try testing.expectEqual(@as(u32, 0), ctx.failed.load(.monotonic)); + if (dir_mode) { + try testing.expect((DirPayload{ .sha = sha }).valid(dest)); + } else { + try testing.expect((Sha256File{ .sha256 = sha }).valid(dest)); + } + try testing.expectEqual(@as(usize, 0), try countSiblings(root, "shared")); +} + +test "publishFile: 8 concurrent unlocked publishers all succeed and the file verifies" { + try raceTest(false); +} + +test "publishDir: 8 concurrent locked publishers all succeed and the tree verifies" { + try raceTest(true); +} diff --git a/src/cli/provider_github/archive.zig b/src/cli/provider_github/archive.zig index 719cc84b..cdc291d0 100644 --- a/src/cli/provider_github/archive.zig +++ b/src/cli/provider_github/archive.zig @@ -4,6 +4,7 @@ const std = @import("std"); const config = @import("../config.zig"); const contract = @import("../provider_contract.zig"); const util = @import("../util.zig"); +const cache_publish = @import("../cache_publish.zig"); const Pin = @import("pin.zig").Pin; const files_mod = @import("files.zig"); const read = files_mod.read; @@ -55,7 +56,9 @@ pub fn archive(a: std.mem.Allocator, pin: Pin, allow_download: bool) ![]u8 { try download(a, try pin.archiveUrl(a), tmp, max_archive_size); const downloaded = try read(a, tmp, max_archive_size); if (!util.sha256Matches(downloaded, pin.sha256)) return error.ProviderArchiveHashMismatch; - try std.Io.Dir.renameAbsolute(tmp, path, io); + // One rename publishes it; a concurrent labelle that cached the same + // verified archive first wins and ours is discarded (cache_publish). + _ = try cache_publish.publishFile(tmp, path, cache_publish.Sha256File{ .sha256 = pin.sha256, .max_bytes = max_archive_size }); break :blk downloaded; }; if (!util.sha256Matches(bytes, pin.sha256)) return error.ProviderArchiveHashMismatch; diff --git a/src/cli/provider_github/fetch.zig b/src/cli/provider_github/fetch.zig index 9e10de17..acd16a6a 100644 --- a/src/cli/provider_github/fetch.zig +++ b/src/cli/provider_github/fetch.zig @@ -21,6 +21,7 @@ const std = @import("std"); const config = @import("../config.zig"); const util = @import("../util.zig"); +const cache_publish = @import("../cache_publish.zig"); const pin_mod = @import("pin.zig"); const Pin = pin_mod.Pin; const Document = pin_mod.Document; @@ -157,7 +158,9 @@ pub fn fetch(a: std.mem.Allocator, root: []const u8, offline: bool, download: Do // back, and the temporaries not yet renamed are removed. committed = true; for (temps.items) |t| { - std.Io.Dir.renameAbsolute(t.tmp, t.path, io) catch |err| { + // `publishFile` keeps an archive a concurrent `providers fetch` + // already cached and verified, and replaces a damaged one. + _ = cache_publish.publishFile(t.tmp, t.path, cache_publish.Sha256File{ .sha256 = t.pin.sha256, .max_bytes = max_archive_size }) catch |err| { std.debug.print("labelle: provider '{s}' {s}: the verified archive could not be moved into the cache at {s} ({s}); the {d} archive(s) cached before it are verified and kept. Run `labelle providers fetch` again.\n", .{ t.pin.package, t.pin.version, t.path, @errorName(err), summary.fetched }); for (temps.items) |left| cwd.deleteFile(io, left.tmp) catch {}; return err; diff --git a/src/cli/python_provision.zig b/src/cli/python_provision.zig index 0c182f5c..7830cced 100644 --- a/src/cli/python_provision.zig +++ b/src/cli/python_provision.zig @@ -21,6 +21,7 @@ const builtin = @import("builtin"); const config = @import("config.zig"); const util = @import("util.zig"); const asm_cache = @import("asm_cache.zig"); +const cache_publish = @import("cache_publish.zig"); const is_windows = builtin.os.tag == .windows; const path_sep: u8 = if (is_windows) ';' else ':'; @@ -163,11 +164,27 @@ fn provisionInto(gpa: std.mem.Allocator) Result { std.debug.print(" provisioned Python at {s} is broken — reprovisioning...\n", .{py_dir}); } - // Clean start so a partial prior extract can't leave a broken tree. - std.Io.Dir.cwd().deleteTree(io, py_dir) catch {}; - std.Io.Dir.cwd().createDirPath(io, py_dir) catch return .failed; + // Serialize provisioning across processes sharing this labelle home (CI + // shell runners run several jobs at once). A waiter re-checks after the + // lock: the holder may have just provisioned it. + var lock = cache_publish.InstallLock.acquire(a, py_dir) catch return .failed; + defer lock.release(); + if (util.fileExists(py_exe) and runOk(a, &.{ py_exe, "--version" })) { + std.debug.print(" Python {s} provisioned by a concurrent labelle: {s}\n", .{ PY_VERSION, py_dir }); + return .ready; + } + + // Everything is staged on UNIQUE siblings and published with one + // directory rename, so `py_dir` never holds a half-extracted tree and two + // processes never write through the same archive path. A broken prior + // install at `py_dir` is replaced wholesale by the publish. + const cwd = std.Io.Dir.cwd(); + const archive = cache_publish.tempSibling(a, join(a, &.{ cache_root, "python-download.tar.gz" }) orelse return .failed, "tmp") catch return .failed; + defer cwd.deleteFile(io, archive) catch {}; + const staging = cache_publish.tempSibling(a, py_dir, "tmp") catch return .failed; + defer cwd.deleteTree(io, staging) catch {}; + cwd.createDirPath(io, staging) catch return .failed; - const archive = join(a, &.{ cache_root, "python-download.tar.gz" }) orelse return .failed; const url = std.fmt.allocPrint( a, "https://github.com/astral-sh/python-build-standalone/releases/download/" ++ @@ -184,23 +201,33 @@ fn provisionInto(gpa: std.mem.Allocator) Result { } // Integrity: verify the download against the pinned hash before extracting. - const arc_data = std.Io.Dir.cwd().readFileAlloc(io, archive, a, .limited(256 << 20)) catch { + const arc_data = cwd.readFileAlloc(io, archive, a, .limited(256 << 20)) catch { std.debug.print(" could not read the download to verify it.\n", .{}); return .failed; }; if (!util.sha256Matches(arc_data, dist.sha256)) { std.debug.print(" checksum mismatch — refusing the Python download (expected {s}).\n", .{dist.sha256}); - std.Io.Dir.cwd().deleteFile(io, archive) catch {}; return .failed; } std.debug.print(" extracting...\n", .{}); - if (!extractArchive(a, archive, py_dir)) { + if (!extractArchive(a, archive, staging)) { std.debug.print(" extract failed.\n", .{}); return .failed; } - std.Io.Dir.cwd().deleteFile(io, archive) catch {}; + cwd.deleteFile(io, archive) catch {}; + // python-build-standalone is relocatable: validate the staged tree, then + // publish it. + const staged_exe = interpreterPath(a, staging) orelse return .failed; + if (!util.fileExists(staged_exe) or !runOk(a, &.{ staged_exe, "--version" })) { + std.debug.print(" provisioning incomplete (python missing or won't run).\n", .{}); + return .failed; + } + _ = cache_publish.publishDir(a, staging, py_dir, RunnablePython{}) catch |err| { + std.debug.print(" could not move the provisioned Python into {s}: {s}\n", .{ py_dir, @errorName(err) }); + return .failed; + }; if (!util.fileExists(py_exe) or !runOk(a, &.{ py_exe, "--version" })) { std.debug.print(" provisioning incomplete (python missing or won't run).\n", .{}); return .failed; @@ -210,6 +237,17 @@ fn provisionInto(gpa: std.mem.Allocator) Result { return .ready; } +/// `publishDir` validator: ``'s interpreter exists and runs. +const RunnablePython = struct { + pub fn valid(_: RunnablePython, dir: []const u8) bool { + var arena_inst = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena_inst.deinit(); + const a = arena_inst.allocator(); + const exe = interpreterPath(a, dir) orelse return false; + return util.fileExists(exe) and runOk(a, &.{ exe, "--version" }); + } +}; + /// Extract the `.tar.gz` so its contents land directly in `py_dir` /// (`--strip-components=1` drops the leading `python/` wrapper). fn extractArchive(a: std.mem.Allocator, archive: []const u8, py_dir: []const u8) bool { diff --git a/src/cli/update.zig b/src/cli/update.zig index 53f1883a..8cd34baf 100644 --- a/src/cli/update.zig +++ b/src/cli/update.zig @@ -2,6 +2,7 @@ const std = @import("std"); const project_config = @import("project_config.zig"); const asm_cache = @import("asm_cache.zig"); const util = @import("util.zig"); +const cache_publish = @import("cache_publish.zig"); const config = @import("config.zig"); const update_check = @import("update_check.zig"); @@ -290,12 +291,36 @@ pub fn cmdUpdate(allocator: std.mem.Allocator, cmd_args: []const []const u8) !vo }); defer allocator.free(download_url); - const tmp_path = try util.getTempFilePath(allocator, "labelle-update"); + const cache_root = asm_cache.getCacheRoot(allocator) catch { + std.debug.print("labelle: could not determine cache root\n", .{}); + return; + }; + defer allocator.free(cache_root); + + const bin_dir = try std.fs.path.join(allocator, &.{ cache_root, "bin" }); + defer allocator.free(bin_dir); + + const bin_name: []const u8 = if (builtin.os.tag == .windows) "labelle.exe" else "labelle"; + const bin_path = try std.fs.path.join(allocator, &.{ bin_dir, bin_name }); + defer allocator.free(bin_path); + + std.Io.Dir.cwd().createDirPath(config.globalIo(), bin_dir) catch |err| { + std.debug.print("labelle: could not create {s}: {any}\n", .{ bin_dir, err }); + return; + }; + + // Stage the download NEXT TO the installed binary under a unique name + // (`.tmp--`): same filesystem, so the final move is one + // atomic rename and a concurrently starting `labelle` never runs a + // partially written binary; unique, so two updates cannot clobber each + // other's download. + const tmp_path = try cache_publish.tempSibling(allocator, bin_path, "tmp"); defer allocator.free(tmp_path); std.debug.print(" downloading {s}...\n", .{download_url}); const dl_result = util.runCmd(allocator, &.{ "curl", "-s", "-f", "-o", tmp_path, download_url }) catch { std.debug.print("labelle: download failed (is curl installed?)\n", .{}); + std.Io.Dir.cwd().deleteFile(config.globalIo(), tmp_path) catch {}; printManualUpdateInstructions(target_version); return; }; @@ -305,11 +330,13 @@ pub fn cmdUpdate(allocator: std.mem.Allocator, cmd_args: []const []const u8) !vo switch (dl_result.term) { .exited => |code| if (code != 0) { std.debug.print("labelle: download failed (HTTP error)\n", .{}); + std.Io.Dir.cwd().deleteFile(config.globalIo(), tmp_path) catch {}; printManualUpdateInstructions(target_version); return; }, else => { std.debug.print("labelle: download process terminated abnormally\n", .{}); + std.Io.Dir.cwd().deleteFile(config.globalIo(), tmp_path) catch {}; return; }, } @@ -318,24 +345,6 @@ pub fn cmdUpdate(allocator: std.mem.Allocator, cmd_args: []const []const u8) !vo _ = util.runCmd(allocator, &.{ "chmod", "+x", tmp_path }) catch {}; } - const cache_root = asm_cache.getCacheRoot(allocator) catch { - std.debug.print("labelle: could not determine cache root\n", .{}); - return; - }; - defer allocator.free(cache_root); - - const bin_dir = try std.fs.path.join(allocator, &.{ cache_root, "bin" }); - defer allocator.free(bin_dir); - - const bin_name: []const u8 = if (builtin.os.tag == .windows) "labelle.exe" else "labelle"; - const bin_path = try std.fs.path.join(allocator, &.{ bin_dir, bin_name }); - defer allocator.free(bin_path); - - std.Io.Dir.cwd().createDirPath(config.globalIo(), bin_dir) catch |err| { - std.debug.print("labelle: could not create {s}: {any}\n", .{ bin_dir, err }); - return; - }; - if (builtin.os.tag == .windows) { const bat_path = try std.fs.path.join(allocator, &.{ bin_dir, "labelle-update.bat" }); defer allocator.free(bat_path); diff --git a/src/cli/zig_toolchain.zig b/src/cli/zig_toolchain.zig index c2482483..cc68ca91 100644 --- a/src/cli/zig_toolchain.zig +++ b/src/cli/zig_toolchain.zig @@ -41,6 +41,7 @@ const zig_cache = @import("zig_cache.zig"); const config = @import("config.zig"); const launcher_manifest = @import("launcher_manifest.zig"); const util = @import("util.zig"); +const cache_publish = @import("cache_publish.zig"); const is_windows = builtin.os.tag == .windows; @@ -512,12 +513,11 @@ fn ensureInstalledWithSeed(allocator: std.mem.Allocator, version: []const u8, se // touch the published dir. if (cwd.access(io, bin_path, .{})) |_| return else |_| {} - // Stage everything under a temp sibling of the version dir. The suffix - // only needs to avoid collisions with a concurrent/stale staging dir, not - // be cryptographic — 0.16 has no std.crypto.random, so seed a PRNG from a - // stack address mixed with the dest path. - var prng = std.Random.DefaultPrng.init(@intFromPtr(&dest_dir) ^ std.hash.Wyhash.hash(0, dest_dir)); - const staging = try std.fmt.allocPrint(allocator, "{s}.tmp-{d}", .{ dest_dir, prng.random().int(u32) }); + // Stage everything under a UNIQUE temp sibling of the version dir + // (`.tmp--`, cache_publish.zig): same filesystem, so + // the publishing rename below is atomic, and distinct across processes + // and threads even if a crashed holder left its staging dir behind. + const staging = try cache_publish.tempSibling(allocator, dest_dir, "tmp"); defer allocator.free(staging); cwd.deleteTree(io, staging) catch {}; try cwd.createDirPath(io, staging);