From 570a56f74bf79aa3fa383e708d4741043c1e4f6f Mon Sep 17 00:00:00 2001 From: Yash Mandaviya Date: Thu, 23 Jul 2026 23:14:47 -0400 Subject: [PATCH] Add stream-based copyIn/copyOut for tar stream cp support Adds file-handle/stream-based copy entrypoints so callers can transfer a tar stream directly to/from a container's filesystem without staging the contents on the host, enabling true `docker cp -` / `podman cp -` parity in downstream CLIs (apple/container#1908). - LinuxContainer.copyIn(archive:to:) splices a tar stream from a file handle straight to the guest over vsock; the guest extracts it in place, honoring the ownership/mode/symlink metadata in the tar headers. - LinuxContainer.copyOut(from:to:) streams the guest-produced tar archive to a file handle; it always archives, even a single regular file, to match `docker cp CONTAINER:/path -`. - ArchiveReader.init(fileHandle:) auto-detects archive format and compression filter, so an externally supplied tar stream (uncompressed or gzip/bzip2/xz) is accepted, not only the internal pax+gzip form. - vminitd copy handlers use the auto-detecting reader for copyIn and honor CopyRequest.is_archive to force single-file archiving on copyOut. No host-side temp staging, so host permission and path-length limits no longer constrain the transfer. Existing path-based copyIn/copyOut behavior is unchanged (the new is_archive-on-copyOut path is opt-in). Co-Authored-By: Claude Opus 4.8 (1M context) --- Sources/Containerization/LinuxContainer.swift | 173 ++++++++++++++++++ .../ArchiveReader.swift | 23 +++ .../ArchiveReaderTests.swift | 60 ++++++ vminitd/Sources/VminitdCore/Server+GRPC.swift | 21 ++- 4 files changed, 274 insertions(+), 3 deletions(-) diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index e24bdf67a..68357a057 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -1439,6 +1439,179 @@ extension LinuxContainer { } } } + + /// Copy the contents of a tar stream into the container. + /// + /// The `archive` file handle is expected to yield a tar stream, either + /// uncompressed or compressed (gzip/bzip2/xz), matching `docker cp -` / + /// `podman cp -` input semantics. The bytes are streamed directly to the + /// guest, which extracts them in place honoring the ownership, mode and + /// symlink metadata recorded in the tar headers. No intermediate files are + /// created on the host, so host-side permission and path-length constraints + /// do not apply. + /// + /// - Parameters: + /// - archive: A file handle yielding the tar stream to extract. + /// - destination: The guest directory to extract the archive into. + /// - createParents: Create parent directories of `destination` if missing. + /// - chunkSize: The transfer chunk size in bytes. + public func copyIn( + archive: FileHandle, + to destination: URL, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { + let state = try $0.startedState("copyIn") + + let guestPath = URL(filePath: self.root).appending(path: destination.path) + let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue + let listener = try state.vm.listen(port) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await state.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyIn, + guestPath: guestPath, + vsockPort: port, + createParents: createParents, + isArchive: true + ) + } + } + + group.addTask { + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.copyQueue.async { + do { + defer { conn.closeFile() } + try Self.spliceFd( + from: archive.fileDescriptor, + to: conn.fileDescriptor, + chunkSize: chunkSize, + label: "copyIn" + ) + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + } + + /// Stream the contents of a container path as a tar archive to a file handle. + /// + /// Matches `docker cp CONTAINER:/path -` / `podman cp` output semantics: the + /// output is always a tar archive, even for a single regular file, and + /// preserves the ownership and mode recorded in the guest filesystem. The + /// guest performs the archiving natively and streams the result directly; + /// no intermediate files are created on the host. + /// + /// - Parameters: + /// - source: The guest path (file or directory) to archive. + /// - archive: A file handle the tar stream is written to. + /// - chunkSize: The transfer chunk size in bytes. + public func copyOut( + from source: URL, + to archive: FileHandle, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { + let state = try $0.startedState("copyOut") + + let guestPath = URL(filePath: self.root).appending(path: source.path) + let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue + let listener = try state.vm.listen(port) + + let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await state.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyOut, + guestPath: guestPath, + vsockPort: port, + isArchive: true, + onMetadata: { meta in + metadataCont.yield(meta) + metadataCont.finish() + } + ) + } + } + + group.addTask { + // Wait for the guest to report metadata (and thus that the + // source exists) before accepting the data connection. + _ = await metadataStream.first(where: { _ in true }) + + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.copyQueue.async { + do { + defer { conn.closeFile() } + try Self.spliceFd( + from: conn.fileDescriptor, + to: archive.fileDescriptor, + chunkSize: chunkSize, + label: "copyOut" + ) + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + } + + /// Copy raw bytes from one file descriptor to another until EOF. + private static func spliceFd(from srcFd: Int32, to dstFd: Int32, chunkSize: Int, label: String) throws { + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(srcFd, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError(.internalError, message: "\(label): read error: \(String(cString: strerror(errno)))") + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(dstFd, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError(.internalError, message: "\(label): vsock write error: \(String(cString: strerror(errno)))") + } + written += w + } + } + } } extension VirtualMachineInstance { diff --git a/Sources/ContainerizationArchive/ArchiveReader.swift b/Sources/ContainerizationArchive/ArchiveReader.swift index 0477d8e4a..01ac8d674 100644 --- a/Sources/ContainerizationArchive/ArchiveReader.swift +++ b/Sources/ContainerizationArchive/ArchiveReader.swift @@ -106,6 +106,29 @@ public final class ArchiveReader { .checkOk(elseThrow: { .unableToOpenArchive($0) }) } + /// Initializes an `ArchiveReader` to read from the provided file descriptor, + /// auto-detecting the archive `Format` and compression `Filter`. + /// + /// Use this when the incoming stream's format and compression are not known + /// ahead of time, e.g. a tar stream supplied on stdin that may be + /// uncompressed or compressed (gzip, bzip2, xz, ...), matching + /// `docker cp -` / `podman cp -` input semantics. zstd is not auto-detected + /// from a stream because it requires seeking; use + /// ``init(format:filter:fileHandle:)`` with ``Filter/zstd`` for that case. + public init(fileHandle: FileHandle) throws { + self.underlying = archive_read_new() + self.fileHandle = fileHandle + + try archive_read_support_filter_all(underlying) + .checkOk(elseThrow: .failedToDetectFilter) + try archive_read_support_format_all(underlying) + .checkOk(elseThrow: .failedToDetectFormat) + + let fd = fileHandle.fileDescriptor + try archive_read_open_fd(underlying, fd, 4096) + .checkOk(elseThrow: { .unableToOpenArchive($0) }) + } + /// Initialize the `ArchiveReader` to read from a specified file URL /// by trying to auto determine the archives `Format` and `Filter`. public init(file: URL) throws { diff --git a/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift b/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift index c7c0e563a..83aff4cbe 100644 --- a/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift +++ b/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift @@ -658,6 +658,66 @@ struct ArchiveReaderTests { } } + // MARK: - FileHandle Stream Auto-Detect Tests + + /// The stream-based auto-detecting reader must accept an uncompressed tar + /// stream, matching `docker cp -` / `podman cp -` input semantics. + @Test func readUncompressedTarFromFileHandle() throws { + let archiveURL = try createTestArchive( + name: "stream-plain", + entries: [ + ("dir/", .directory, nil), + ("dir/file.txt", .regular("streamed plain"), nil), + ]) + defer { try? FileManager.default.removeItem(at: archiveURL.deletingLastPathComponent()) } + + let extractDir = try createExtractionDirectory(name: "stream-plain") + defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) } + + let fh = try FileHandle(forReadingFrom: archiveURL) + defer { try? fh.close() } + + let reader = try ArchiveReader(fileHandle: fh) + let rejected = try reader.extractContents(to: extractDir) + + #expect(rejected.isEmpty) + let content = try String(contentsOf: extractDir.appendingPathComponent("dir/file.txt"), encoding: .utf8) + #expect(content == "streamed plain") + } + + /// The stream-based auto-detecting reader must also accept a gzip-compressed + /// tar stream (the format the guest emits internally for directory copies), + /// without being told the filter up front. + @Test func readGzipTarFromFileHandle() throws { + let testDirectory = createTemporaryDirectory(baseName: "ArchiveReaderTests")! + defer { try? FileManager.default.removeItem(at: testDirectory) } + let archiveURL = testDirectory.appendingPathComponent("stream-gzip.tar.gz") + + let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) + try writer.open(file: archiveURL) + let entry = WriteEntry() + entry.path = "hello.txt" + entry.fileType = .regular + entry.permissions = 0o644 + let data = "streamed gzip".data(using: .utf8)! + entry.size = numericCast(data.count) + try writer.writeEntry(entry: entry, data: data) + try writer.finishEncoding() + + let extractDir = try createExtractionDirectory(name: "stream-gzip") + defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) } + + let fh = try FileHandle(forReadingFrom: archiveURL) + defer { try? fh.close() } + + let reader = try ArchiveReader(fileHandle: fh) + let rejected = try reader.extractContents(to: extractDir) + + #expect(rejected.isEmpty) + let content = try String(contentsOf: extractDir.appendingPathComponent("hello.txt"), encoding: .utf8) + #expect(content == "streamed gzip") + } + // MARK: - Zstd Compression Tests @Test func readZstdCompressedArchive() throws { diff --git a/vminitd/Sources/VminitdCore/Server+GRPC.swift b/vminitd/Sources/VminitdCore/Server+GRPC.swift index dd07ef54f..feff5307f 100644 --- a/vminitd/Sources/VminitdCore/Server+GRPC.swift +++ b/vminitd/Sources/VminitdCore/Server+GRPC.swift @@ -535,7 +535,11 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ try FileManager.default.createDirectory(at: destURL, withIntermediateDirectories: true) let fileHandle = FileHandle(fileDescriptor: sockFd, closeOnDealloc: false) - let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fileHandle) + // Auto-detect format and compression filter so both the internal + // pax+gzip archives (directory copyIn) and externally supplied tar + // streams (uncompressed or compressed, e.g. `docker cp -`) are + // accepted. + let reader = try ArchiveReader(fileHandle: fileHandle) return try reader.extractContents(to: destURL) } @@ -562,7 +566,10 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ guard FileManager.default.fileExists(atPath: path, isDirectory: &isDirectory) else { throw RPCError(code: .notFound, message: "copy: path not found '\(path)'") } - let isArchive = isDirectory.boolValue + // `request.isArchive` forces archive output even for a single regular + // file, matching `docker cp CONTAINER:/path -` which always emits a tar + // stream. Directories are always archived. + let isArchive = isDirectory.boolValue || request.isArchive // Determine total size for single files. var totalSize: UInt64 = 0 @@ -593,7 +600,15 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ let fileURL = URL(fileURLWithPath: path) let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) try writer.open(fileDescriptor: sock.fileDescriptor) - try writer.archiveDirectory(fileURL) + if isDirectory.boolValue { + try writer.archiveDirectory(fileURL) + } else { + // Forced single-file archive: emit one entry named after the + // file's basename, relative to its parent directory. + let filePath = FilePath(path) + let base = filePath.removingLastComponent() + try writer.archive([filePath], base: base) + } try writer.finishEncoding() } else { let srcFd = open(path, O_RDONLY)