diff --git a/Package.resolved b/Package.resolved index 0bcfc149..300edd90 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "80db0cddb0e78cf6a4c79f94537cdfebd8004679161532fbae8a57ee9d1d8cd1", + "originHash" : "d3996bd48500c7697baf2f1a62200c73ee6079c9d22b6898ee370bbc1a748aea", "pins" : [ { "identity" : "aexml", diff --git a/Package.swift b/Package.swift index dab231df..4c272ea5 100644 --- a/Package.swift +++ b/Package.swift @@ -181,6 +181,7 @@ let package = Package( name: "PackLib", dependencies: [ "XUtils", + .product(name: "Superutils", package: "xtool-core"), .product(name: "Yams", package: "Yams"), .product(name: "XcodeGenKit", package: "XcodeGen", condition: .when(platforms: [.macOS])), ], diff --git a/Sources/PackLib/DarwinSDK.swift b/Sources/PackLib/DarwinSDK.swift index 2386e8b2..659bec7f 100644 --- a/Sources/PackLib/DarwinSDK.swift +++ b/Sources/PackLib/DarwinSDK.swift @@ -1,10 +1,42 @@ import Foundation import XUtils import Subprocess +import Superutils public struct DarwinSDK { + public enum Flavor { + // can't be updated in place + case slim + // can be updated in place, includes a whole copy of Xcode.app + case normal + // from before the slim/normal split existed (version "develop") + case legacy + } + public let bundle: URL public let version: String + public let flavor: Flavor + + static func swiftPMDirectory( + environment: [String: String] = ProcessInfo.processInfo.environment, + homeDirectory: URL = FileManager.default.homeDirectoryForCurrentUser + ) throws -> URL { + if let configurationDirectory = environment["XDG_CONFIG_HOME"] { + guard (configurationDirectory as NSString).isAbsolutePath else { + throw StringError("XDG_CONFIG_HOME must be an absolute path: '\(configurationDirectory)'") + } + return URL(fileURLWithPath: configurationDirectory, isDirectory: true) + .appendingPathComponent("swiftpm", isDirectory: true) + } else { + return homeDirectory.appendingPathComponent(".swiftpm", isDirectory: true) + } + } + + private static var swiftSDKsDirectory: URL { + get throws { + try swiftPMDirectory().appendingPathComponent("swift-sdks", isDirectory: true) + } + } public init?(bundle: URL) { self.bundle = bundle @@ -12,31 +44,61 @@ public struct DarwinSDK { self.version = String(decoding: version, as: UTF8.self) .trimmingCharacters(in: .whitespacesAndNewlines) } else if ["darwin.xtoolsdk", "darwin.artifactbundle"].contains(bundle.lastPathComponent) { - self.version = "unknown" + self.version = "develop" } else { return nil } + + if version == "develop" { + self.flavor = .legacy + } else if bundle.appendingPathComponent("Xcode.app").dirExists { + self.flavor = .normal + } else { + self.flavor = .slim + } } public static func install(from path: String) async throws { - // we can't just move into ~/.swiftpm/swift-sdks because the swiftpm directory - // location depends on factors like $XDG_CONFIG_HOME. Rather than replicating - // SwiftPM's logic, which may change, it's more reliable to directly invoke - // `swift sdk install`. See: https://github.com/xtool-org/xtool/pull/40 - let url = URL(fileURLWithPath: path) guard DarwinSDK(bundle: url) != nil else { throw StringError("Invalid Darwin SDK at '\(path)'")} try await addHostClangResourceDir(to: url) + let sdksDirectory = try swiftSDKsDirectory + try FileManager.default.createDirectory( + at: sdksDirectory, + withIntermediateDirectories: true + ) + let destination = sdksDirectory.appendingPathComponent("darwin.artifactbundle", isDirectory: true) + try await movePreservingHardLinks(from: url, to: destination) + } + + private static func copyPreservingHardLinks(from source: URL, to destination: URL) async throws { try await Subprocess.run( - .name("swift"), - arguments: ["sdk", "install", url.path], + .name("cp"), + arguments: ["-a", source.path, destination.path], output: .discarded ) .checkSuccess() } + private static func movePreservingHardLinks(from source: URL, to destination: URL) async throws { + let fileManager = FileManager.default + let sourceAttributes = try fileManager.attributesOfItem(atPath: source.path) + let destinationAttributes = try fileManager.attributesOfItem( + atPath: destination.deletingLastPathComponent().path + ) + let sourceSystem = sourceAttributes[.systemNumber] as? NSNumber + let destinationSystem = destinationAttributes[.systemNumber] as? NSNumber + + if let sourceSystem, let destinationSystem, sourceSystem == destinationSystem { + try fileManager.moveItem(at: source, to: destination) + } else { + try await copyPreservingHardLinks(from: source, to: destination) + try fileManager.removeItem(at: source) + } + } + private static func addHostClangResourceDir(to sdk: URL) async throws { let clangURL = try await ToolRegistry.locate("clang") let process = try await Subprocess.run( @@ -51,38 +113,10 @@ public struct DarwinSDK { try await FileManager.default.copyItem(at: hostInclude, to: sdkInclude, preserveOwner: false) } - public static func current() async throws -> DarwinSDK? { - let outputString: String - do { - outputString = try await Subprocess.run( - .name("swift"), - arguments: ["sdk", "configure", "darwin", "arm64-apple-ios", "--show-configuration"], - output: .string(limit: .max) - ) - .checkSuccess() - .standardOutput - ?? "" - } catch SubprocessFailure.exited { - return nil - } - - // should be something like - // swiftResourcesPath: /home/user/.swiftpm/swift-sdks/darwin.artifactbundle/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift - // swiftlint:disable:previous line_length - let resourcesPathPrefix = "swiftResourcesPath: " - - guard let resourcesPath = outputString - .split(separator: "\n") - .first(where: { $0.hasPrefix(resourcesPathPrefix) })? - .dropFirst(resourcesPathPrefix.count) - else { return nil } - - var resourcesURL = URL(fileURLWithPath: String(resourcesPath)) - for _ in 0..<6 { - resourcesURL = resourcesURL.deletingLastPathComponent() - } - - return DarwinSDK(bundle: resourcesURL) + public static func current() throws -> DarwinSDK? { + let bundle = try swiftSDKsDirectory.appendingPathComponent("darwin.artifactbundle", isDirectory: true) + guard bundle.dirExists else { return nil } + return DarwinSDK(bundle: bundle) } public func remove() throws { diff --git a/Sources/XToolSupport/DevCommand.swift b/Sources/XToolSupport/DevCommand.swift index a8273540..f9a43e0f 100644 --- a/Sources/XToolSupport/DevCommand.swift +++ b/Sources/XToolSupport/DevCommand.swift @@ -27,6 +27,8 @@ struct PackOperation { @discardableResult func run() async throws -> URL { + try await EnsureSDKOperation(quiet: true).run() + print("Planning...") let schema: PackSchema diff --git a/Sources/XToolSupport/SDKBuilder.swift b/Sources/XToolSupport/SDKBuilder.swift index 96e7f6f6..bda60ea7 100644 --- a/Sources/XToolSupport/SDKBuilder.swift +++ b/Sources/XToolSupport/SDKBuilder.swift @@ -52,13 +52,41 @@ struct SDKBuilder { } } + enum Mode { + /// Create a slim SDK with just the files that this version of xtool uses + case buildSlim + /// Create an SDK that retains a full copy of Xcode.app. Larger but allows in-place updates. + case buildNormal + /// Update a normal SDK in-place. + case update + + var usesHardLinks: Bool { + switch self { + case .buildSlim: false + case .buildNormal, .update: true + } + } + } + let input: Input let output: URL let arch: Arch + let mode: Mode + + // bump this when the sdk builder logic changes + static let sdkEpoch = 1 + + // tag from https://github.com/xtool-org/darwin-tools-linux-llvm + static let darwinToolsVersion = "1.0.1" + + static var currentSDKVersion: String { + """ + epoch=\(sdkEpoch),darwinTools=\(darwinToolsVersion) + """ + } func buildSDK() async throws { - // TODO: store relevant info for staleness check - let sdkVersion = "develop" + let sdkVersion = Self.currentSDKVersion try? FileManager.default.removeItem(at: output) try FileManager.default.createDirectory( @@ -170,9 +198,6 @@ struct SDKBuilder { } private func installToolset(in output: URL) async throws { - // tag from https://github.com/xtool-org/darwin-tools-linux-llvm - let darwinToolsVersion = "1.0.1" - let toolsetDir = output.appendingPathComponent("toolset") try FileManager.default.createDirectory( @@ -183,7 +208,7 @@ struct SDKBuilder { @Dependency(\.httpClient) var httpClient let url = URL(string: """ https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/\ - v\(darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz + v\(Self.darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz """)! let (response, body) = try await httpClient.send(HTTPRequest(url: url)) guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") } @@ -224,23 +249,19 @@ struct SDKBuilder { private func installDeveloper(in output: URL) async throws -> URL { let dev = output.appendingPathComponent("Developer") + let expectedAppDir = output.appendingPathComponent("Xcode.app") let appDir: URL let cleanupStageDir: URL? let wanted: Int? - switch input { - case .xip(let inputPath): - let devStage = output.appendingPathComponent("DeveloperStage") - try FileManager.default.createDirectory(at: devStage, withIntermediateDirectories: false) - // unxip doesn't like cooperative cancellation atm so shield it. - // if the user does a ^C during unxip, we'll just wait until extraction - // is over before bailing - wanted = try await Task { - try await extractXIP(inputPath: inputPath, outDir: devStage.path) - }.value + switch (input, mode) { + case (.xip(let inputPath), .buildSlim): + let stage = output.appendingPathComponent("DeveloperStage") + try FileManager.default.createDirectory(at: stage, withIntermediateDirectories: false) + wanted = try await extractXIP(inputPath: inputPath, outDir: stage.path) try Task.checkCancellation() let contents = try FileManager.default.contentsOfDirectory( - at: devStage, + at: stage, includingPropertiesForKeys: nil ) let apps = contents.filter { $0.pathExtension == "app" } @@ -252,12 +273,25 @@ struct SDKBuilder { default: throw Console.Error("Unrecognized xip layout (multiple apps found)") } - cleanupStageDir = devStage - case .app(let appPath): - wanted = nil + cleanupStageDir = stage + case (.xip(let inputPath), .buildNormal): + wanted = try await extractXIP(inputPath: inputPath, outDir: output.path) + appDir = expectedAppDir + cleanupStageDir = nil + case (.xip, .update): + throw Console.Error("Can't update with xip input") + case (.app(let appPath), .buildSlim), (.app(let appPath), .update): appDir = URL(fileURLWithPath: appPath) + wanted = nil + cleanupStageDir = nil + case (.app(let appPath), .buildNormal): + let source = URL(fileURLWithPath: appPath) + try await FileManager.default.copyItem(at: source, to: expectedAppDir, preserveOwner: false) + appDir = expectedAppDir + wanted = nil cleanupStageDir = nil } + try Task.checkCancellation() try FileManager.default.createDirectory(at: dev, withIntermediateDirectories: false) @@ -280,7 +314,7 @@ struct SDKBuilder { } if count % 100 == 0 { if wanted == nil { - print("\r[Installing SDKs] Copied \(count) files", terminator: "") + print("\r[Installing SDKs] Installed \(count) files", terminator: "") fflush(stdoutSafe) } await Task.yield() @@ -293,7 +327,11 @@ struct SDKBuilder { if try child.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true { toDoDirs.append(path) try FileManager.default.createDirectory(at: dest, withIntermediateDirectories: false) + } else if mode.usesHardLinks { + // Installed SDKs retain Xcode.app, so avoid storing their developer files twice. + try FileManager.default.linkItem(at: child, to: dest) } else { + // Slim SDKs omit Xcode.app and contain independent copies. try FileManager.default.copyItem(at: child, to: dest) } } @@ -305,9 +343,9 @@ struct SDKBuilder { } print() - print("[Cleaning up]") if let cleanupStageDir { - try? FileManager.default.removeItem(at: cleanupStageDir) + print("[Cleaning up]") + try FileManager.default.removeItem(at: cleanupStageDir) } print("[Finalizing SDKs]") @@ -357,9 +395,18 @@ struct SDKBuilder { return dev } + private func extractXIP(inputPath: String, outDir: String) async throws -> Int { + // unxip doesn't like cooperative cancellation atm so shield it. + // if the user does a ^C during unxip, we'll just wait until extraction + // is over before bailing + try await Task { + try await _extractXIP(inputPath: inputPath, outDir: outDir) + }.value + } + // returns the number of files we actually want to keep, // useful for computing progress % during fs traversal - private func extractXIP(inputPath: String, outDir: String) async throws -> Int { + private func _extractXIP(inputPath: String, outDir: String) async throws -> Int { let fd = try FileDescriptor.open(inputPath, .readOnly) defer { try? fd.close() } diff --git a/Sources/XToolSupport/SDKCommand.swift b/Sources/XToolSupport/SDKCommand.swift index 3b0b7202..bcb2bace 100644 --- a/Sources/XToolSupport/SDKCommand.swift +++ b/Sources/XToolSupport/SDKCommand.swift @@ -13,6 +13,7 @@ struct SDKCommand: AsyncParsableCommand { abstract: "Manage the Darwin Swift SDK", subcommands: [ DevSDKInstallCommand.self, + DevSDKUpdateCommand.self, DevSDKRemoveCommand.self, DevSDKBuildCommand.self, DevSDKStatusCommand.self, @@ -49,7 +50,7 @@ struct DevSDKBuildCommand: AsyncParsableCommand { let builderArch = try arch.sdkBuilderArch let input = try SDKBuilder.Input(path: path) let output = URL(fileURLWithPath: outputDir, isDirectory: true).appending(path: "darwin.xtoolsdk") - let builder = SDKBuilder(input: input, output: output, arch: builderArch) + let builder = SDKBuilder(input: input, output: output, arch: builderArch, mode: .buildSlim) try await builder.buildSDK() print("Built SDK at \(output.path). You can install it with `xtool sdk install`.") } @@ -90,8 +91,24 @@ struct DevSDKInstallCommand: AsyncParsableCommand { ) var path: String + @Flag( + help: "Install a slim SDK (uses less disk space). Slim SDKs cannot be updated in place." + ) + var slim = false + func run() async throws { - try await InstallSDKOperation(path: path).run() + try await InstallSDKOperation(path: path, slim: slim).run() + } +} + +struct DevSDKUpdateCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "update", + abstract: "Update the installed Darwin Swift SDK" + ) + + func run() async throws { + try await UpdateSDKOperation().run() } } @@ -102,7 +119,7 @@ struct DevSDKRemoveCommand: AsyncParsableCommand { ) func run() async throws { - guard let sdk = try await DarwinSDK.current() else { + guard let sdk = try DarwinSDK.current() else { throw Console.Error("Cannot remove SDK: no Darwin SDK installed") } try sdk.remove() @@ -117,8 +134,11 @@ struct DevSDKStatusCommand: AsyncParsableCommand { ) func run() async throws { - if let sdk = try await DarwinSDK.current() { - print("Installed at \(sdk.bundle.path)") + if let sdk = try DarwinSDK.current() { + print("Darwin SDK is installed") + print(" Path: \(sdk.bundle.path)") + print(" Flavor: \(sdk.flavor)") + print(" Version: \(sdk.version)") } else { print("Not installed") } @@ -127,7 +147,7 @@ struct DevSDKStatusCommand: AsyncParsableCommand { extension DarwinSDK { func isUpToDate() -> Bool { - true + version == SDKBuilder.currentSDKVersion } } @@ -165,8 +185,70 @@ extension SwiftVersion { } } +struct EnsureSDKOperation { + let quiet: Bool + + func run() async throws { + #if os(macOS) + if !quiet { + print("Skipping Darwin SDK setup since we're on macOS.") + } + #else + let sdk = try DarwinSDK.current() + switch sdk.map({ ($0.isUpToDate(), $0.flavor) }) { + case (true, _)?: // swiftlint:disable:this optional_enum_case_matching + if !quiet { + print("Darwin SDK is up to date.") + } + case (false, .slim)?: // swiftlint:disable:this optional_enum_case_matching + throw Console.Error(""" + Darwin SDK is out of date, and was installed in 'slim' mode. + + Slim SDKs take less disk space, but can't be auto-updated. + Please install a new SDK with + xtool sdk install [--slim] + """) + case (false, .normal)?: // swiftlint:disable:this optional_enum_case_matching + print("Darwin SDK is out of date. Rebuilding...") + try await UpdateSDKOperation().run() + case (false, .legacy)?: // swiftlint:disable:this optional_enum_case_matching + print(""" + Darwin SDK is incompatible: built with an older version of xtool. + + Requesting re-install. This is a one-time rebuild; after this, + xtool will be able to resolve incompatibilities automatically. + + """) + try await generateSDK() + case nil: + print("Now generating the Darwin SDK.\n") + try await generateSDK() + } + + func generateSDK() async throws { + let path = try await Console.prompt(""" + Please download Xcode from http://developer.apple.com/download/all/?q=Xcode + and enter the path to the downloaded Xcode.xip. + + Path to Xcode.xip: \("" /* pacify swiftlint trailing_whitespace */) + """) + + let expanded = (path as NSString).expandingTildeInPath + + try await InstallSDKOperation(path: expanded).run() + } + #endif + } +} + struct InstallSDKOperation { let path: String + let slim: Bool + + init(path: String, slim: Bool = false) { + self.path = path + self.slim = slim + } func run() async throws { #if os(macOS) @@ -184,11 +266,12 @@ struct InstallSDKOperation { let input = try SDKBuilder.Input(path: path) let arch = try ArchSelection.auto.sdkBuilderArch - let builder = SDKBuilder(input: input, output: sdkPath, arch: arch) + let mode: SDKBuilder.Mode = slim ? .buildSlim : .buildNormal + let builder = SDKBuilder(input: input, output: sdkPath, arch: arch, mode: mode) try await builder.buildSDK() } - if let sdk = try await DarwinSDK.current() { + if let sdk = try DarwinSDK.current() { print("Removing existing SDK...") try sdk.remove() } @@ -200,3 +283,46 @@ struct InstallSDKOperation { #endif } } + +struct UpdateSDKOperation { + func run() async throws { + #if os(macOS) + print("Skipping SDK install; the iOS SDK ships with Xcode on macOS") + #else + guard let existing = try DarwinSDK.current() else { + throw Console.Error("Could not locate existing SDK; cannot perform update.") + } + let xcode = existing.bundle.appendingPathComponent("Xcode.app") + guard xcode.dirExists else { + // This includes prebuilt .xtoolsdk installs and installs created with --slim. + throw Console.Error(""" + The installed SDK was built in 'slim' mode and cannot be updated in place. \ + Please install a new copy with `xtool sdk install`. + """) + } + + let input = try SDKBuilder.Input(path: xcode.path) + let arch = try ArchSelection.auto.sdkBuilderArch + + let tempDir = try TemporaryDirectory(name: "DarwinSDKBuild") + let sdkURL = tempDir.url.appending(path: "darwin.artifactbundle") + let builder = SDKBuilder(input: input, output: sdkURL, arch: arch, mode: .update) + try await builder.buildSDK() + + guard DarwinSDK(bundle: sdkURL) != nil else { + throw Console.Error("Invalid Darwin SDK at '\(sdkURL.path)'") + } + + try FileManager.default.moveItem( + at: xcode, + to: sdkURL.appendingPathComponent("Xcode.app") + ) + try existing.remove() + try await DarwinSDK.install(from: sdkURL.path) + + print("Updated SDK") + + withExtendedLifetime(tempDir) {} + #endif + } +} diff --git a/Sources/XToolSupport/SetupCommand.swift b/Sources/XToolSupport/SetupCommand.swift index cff7c770..65749f31 100644 --- a/Sources/XToolSupport/SetupCommand.swift +++ b/Sources/XToolSupport/SetupCommand.swift @@ -24,36 +24,6 @@ struct SetupOperation { func run() async throws { try await AuthOperation(logoutFromExisting: false, quiet: quiet).run() - - #if os(macOS) - if !quiet { - print("Skipping Darwin SDK setup since we're on macOS.") - } - #else - switch try await DarwinSDK.current()?.isUpToDate() { - case true?: - if !quiet { - print("Darwin SDK is up to date.") - } - case false?: - if !quiet { - print("Darwin SDK is outdated.") - } - fallthrough - case nil: - let path = try await Console.prompt(""" - Now generating the Darwin SDK. - - Please download Xcode from http://developer.apple.com/download/all/?q=Xcode - and enter the path to the downloaded Xcode.xip. - - Path to Xcode.xip: \("" /* pacify swiftlint trailing_whitespace */) - """) - - let expanded = (path as NSString).expandingTildeInPath - - try await InstallSDKOperation(path: expanded).run() - } - #endif + try await EnsureSDKOperation(quiet: quiet).run() } } diff --git a/Tests/XToolTests/DarwinSDKTests.swift b/Tests/XToolTests/DarwinSDKTests.swift new file mode 100644 index 00000000..c480c42f --- /dev/null +++ b/Tests/XToolTests/DarwinSDKTests.swift @@ -0,0 +1,30 @@ +import Foundation +import Testing +@testable import PackLib + +@Test func swiftPMDirectoryUsesXDGConfigHome() throws { + let directory = try DarwinSDK.swiftPMDirectory( + environment: ["XDG_CONFIG_HOME": "/xdg/config"], + homeDirectory: URL(fileURLWithPath: "/home/test") + ) + + #expect(directory.path == "/xdg/config/swiftpm") +} + +@Test func swiftPMDirectoryFallsBackToHomeDirectory() throws { + let directory = try DarwinSDK.swiftPMDirectory( + environment: [:], + homeDirectory: URL(fileURLWithPath: "/home/test") + ) + + #expect(directory.path == "/home/test/.swiftpm") +} + +@Test func swiftPMDirectoryRejectsRelativeXDGConfigHome() { + #expect(throws: StringError.self) { + try DarwinSDK.swiftPMDirectory( + environment: ["XDG_CONFIG_HOME": "relative/config"], + homeDirectory: URL(fileURLWithPath: "/home/test") + ) + } +}