sdk: self-updating darwin tools

Package.swift

@@ -181,6 +181,7 @@ let package = Package(
             name: "PackLib",
             dependencies: [
                 "XUtils",
+                .product(name: "Superutils", package: "xtool-core"),
                 .product(name: "Yams", package: "Yams"),
                 .product(name: "XcodeGenKit", package: "XcodeGen", condition: .when(platforms: [.macOS])),
             ],

Sources/PackLib/DarwinSDK.swift

@@ -1,42 +1,104 @@
 import Foundation
 import XUtils
 import Subprocess
+import Superutils
 
 public struct DarwinSDK {
+    public enum Flavor {
+        // can't be updated in place
+        case slim
+        // can be updated in place, includes a whole copy of Xcode.app
+        case normal
+        // from before the slim/normal split existed (version "develop")
+        case legacy
+    }
+
     public let bundle: URL
     public let version: String
+    public let flavor: Flavor
+
+    static func swiftPMDirectory(
+        environment: [String: String] = ProcessInfo.processInfo.environment,
+        homeDirectory: URL = FileManager.default.homeDirectoryForCurrentUser
+    ) throws -> URL {
+        if let configurationDirectory = environment["XDG_CONFIG_HOME"] {
+            guard (configurationDirectory as NSString).isAbsolutePath else {
+                throw StringError("XDG_CONFIG_HOME must be an absolute path: '\(configurationDirectory)'")
+            }
+            return URL(fileURLWithPath: configurationDirectory, isDirectory: true)
+                .appendingPathComponent("swiftpm", isDirectory: true)
+        } else {
+            return homeDirectory.appendingPathComponent(".swiftpm", isDirectory: true)
+        }
+    }
+
+    private static var swiftSDKsDirectory: URL {
+        get throws {
+            try swiftPMDirectory().appendingPathComponent("swift-sdks", isDirectory: true)
+        }
+    }
 
     public init?(bundle: URL) {
         self.bundle = bundle
         if let version = try? Data(contentsOf: bundle.appendingPathComponent("darwin-sdk-version.txt")) {
             self.version = String(decoding: version, as: UTF8.self)
                 .trimmingCharacters(in: .whitespacesAndNewlines)
         } else if ["darwin.xtoolsdk", "darwin.artifactbundle"].contains(bundle.lastPathComponent) {
-            self.version = "unknown"
+            self.version = "develop"
         } else {
             return nil
         }
+
+        if version == "develop" {
+            self.flavor = .legacy
+        } else if bundle.appendingPathComponent("Xcode.app").dirExists {
+            self.flavor = .normal
+        } else {
+            self.flavor = .slim
+        }
     }
 
     public static func install(from path: String) async throws {
-        // we can't just move into ~/.swiftpm/swift-sdks because the swiftpm directory
-        // location depends on factors like $XDG_CONFIG_HOME. Rather than replicating
-        // SwiftPM's logic, which may change, it's more reliable to directly invoke
-        // `swift sdk install`. See: https://github.com/xtool-org/xtool/pull/40
-
         let url = URL(fileURLWithPath: path)
         guard DarwinSDK(bundle: url) != nil else { throw StringError("Invalid Darwin SDK at '\(path)'")}
 
         try await addHostClangResourceDir(to: url)
 
+        let sdksDirectory = try swiftSDKsDirectory
+        try FileManager.default.createDirectory(
+            at: sdksDirectory,
+            withIntermediateDirectories: true
+        )
+        let destination = sdksDirectory.appendingPathComponent("darwin.artifactbundle", isDirectory: true)
+        try await movePreservingHardLinks(from: url, to: destination)
+    }
+
+    private static func copyPreservingHardLinks(from source: URL, to destination: URL) async throws {
         try await Subprocess.run(
-            .name("swift"),
-            arguments: ["sdk", "install", url.path],
+            .name("cp"),
+            arguments: ["-a", source.path, destination.path],
             output: .discarded
         )
         .checkSuccess()
     }
 
+    private static func movePreservingHardLinks(from source: URL, to destination: URL) async throws {
+        let fileManager = FileManager.default
+        let sourceAttributes = try fileManager.attributesOfItem(atPath: source.path)
+        let destinationAttributes = try fileManager.attributesOfItem(
+            atPath: destination.deletingLastPathComponent().path
+        )
+        let sourceSystem = sourceAttributes[.systemNumber] as? NSNumber
+        let destinationSystem = destinationAttributes[.systemNumber] as? NSNumber
+
+        if let sourceSystem, let destinationSystem, sourceSystem == destinationSystem {
+            try fileManager.moveItem(at: source, to: destination)
+        } else {
+            try await copyPreservingHardLinks(from: source, to: destination)
+            try fileManager.removeItem(at: source)
+        }
+    }
+
     private static func addHostClangResourceDir(to sdk: URL) async throws {
         let clangURL = try await ToolRegistry.locate("clang")
         let process = try await Subprocess.run(
@@ -51,38 +113,10 @@ public struct DarwinSDK {
         try await FileManager.default.copyItem(at: hostInclude, to: sdkInclude, preserveOwner: false)
     }
 
-    public static func current() async throws -> DarwinSDK? {
-        let outputString: String
-        do {
-            outputString = try await Subprocess.run(
-                .name("swift"),
-                arguments: ["sdk", "configure", "darwin", "arm64-apple-ios", "--show-configuration"],
-                output: .string(limit: .max)
-            )
-            .checkSuccess()
-            .standardOutput
-            ?? ""
-        } catch SubprocessFailure.exited {
-            return nil
-        }
-
-        // should be something like
-        // swiftResourcesPath: /home/user/.swiftpm/swift-sdks/darwin.artifactbundle/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift
-        // swiftlint:disable:previous line_length
-        let resourcesPathPrefix = "swiftResourcesPath: "
-
-        guard let resourcesPath = outputString
-            .split(separator: "\n")
-            .first(where: { $0.hasPrefix(resourcesPathPrefix) })?
-            .dropFirst(resourcesPathPrefix.count)
-            else { return nil }
-
-        var resourcesURL = URL(fileURLWithPath: String(resourcesPath))
-        for _ in 0..<6 {
-            resourcesURL = resourcesURL.deletingLastPathComponent()
-        }
-
-        return DarwinSDK(bundle: resourcesURL)
+    public static func current() throws -> DarwinSDK? {
+        let bundle = try swiftSDKsDirectory.appendingPathComponent("darwin.artifactbundle", isDirectory: true)
+        guard bundle.dirExists else { return nil }
+        return DarwinSDK(bundle: bundle)
     }
 
     public func remove() throws {

Sources/XToolSupport/DevCommand.swift

@@ -27,6 +27,8 @@ struct PackOperation {
 
     @discardableResult
     func run() async throws -> URL {
+        try await EnsureSDKOperation(quiet: true).run()
+
         print("Planning...")
 
         let schema: PackSchema

Sources/XToolSupport/SDKBuilder.swift

@@ -52,13 +52,41 @@ struct SDKBuilder {
         }
     }
 
+    enum Mode {
+        /// Create a slim SDK with just the files that this version of xtool uses
+        case buildSlim
+        /// Create an SDK that retains a full copy of Xcode.app. Larger but allows in-place updates.
+        case buildNormal
+        /// Update a normal SDK in-place.
+        case update
+
+        var usesHardLinks: Bool {
+            switch self {
+            case .buildSlim: false
+            case .buildNormal, .update: true
+            }
+        }
+    }
+
     let input: Input
     let output: URL
     let arch: Arch
+    let mode: Mode
+
+    // bump this when the sdk builder logic changes
+    static let sdkEpoch = 1
+
+    // tag from https://github.com/xtool-org/darwin-tools-linux-llvm
+    static let darwinToolsVersion = "1.0.1"
+
+    static var currentSDKVersion: String {
+        """
+        epoch=\(sdkEpoch),darwinTools=\(darwinToolsVersion)
+        """
+    }
 
     func buildSDK() async throws {
-        // TODO: store relevant info for staleness check
-        let sdkVersion = "develop"
+        let sdkVersion = Self.currentSDKVersion
 
         try? FileManager.default.removeItem(at: output)
         try FileManager.default.createDirectory(
@@ -170,9 +198,6 @@ struct SDKBuilder {
     }
 
     private func installToolset(in output: URL) async throws {
-        // tag from https://github.com/xtool-org/darwin-tools-linux-llvm
-        let darwinToolsVersion = "1.0.1"
-
         let toolsetDir = output.appendingPathComponent("toolset")
 
         try FileManager.default.createDirectory(
@@ -183,7 +208,7 @@ struct SDKBuilder {
         @Dependency(\.httpClient) var httpClient
         let url = URL(string: """
         https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/\
-        v\(darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz
+        v\(Self.darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz
         """)!
         let (response, body) = try await httpClient.send(HTTPRequest(url: url))
         guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") }
@@ -224,23 +249,19 @@ struct SDKBuilder {
     private func installDeveloper(in output: URL) async throws -> URL {
         let dev = output.appendingPathComponent("Developer")
 
+        let expectedAppDir = output.appendingPathComponent("Xcode.app")
         let appDir: URL
         let cleanupStageDir: URL?
         let wanted: Int?
 
-        switch input {
-        case .xip(let inputPath):
-            let devStage = output.appendingPathComponent("DeveloperStage")
-            try FileManager.default.createDirectory(at: devStage, withIntermediateDirectories: false)
-            // unxip doesn't like cooperative cancellation atm so shield it.
-            // if the user does a ^C during unxip, we'll just wait until extraction
-            // is over before bailing
-            wanted = try await Task {
-                try await extractXIP(inputPath: inputPath, outDir: devStage.path)
-            }.value
+        switch (input, mode) {
+        case (.xip(let inputPath), .buildSlim):
+            let stage = output.appendingPathComponent("DeveloperStage")
+            try FileManager.default.createDirectory(at: stage, withIntermediateDirectories: false)
+            wanted = try await extractXIP(inputPath: inputPath, outDir: stage.path)
             try Task.checkCancellation()
             let contents = try FileManager.default.contentsOfDirectory(
-                at: devStage,
+                at: stage,
                 includingPropertiesForKeys: nil
             )
             let apps = contents.filter { $0.pathExtension == "app" }
@@ -252,12 +273,25 @@ struct SDKBuilder {
             default:
                 throw Console.Error("Unrecognized xip layout (multiple apps found)")
             }
-            cleanupStageDir = devStage
-        case .app(let appPath):
-            wanted = nil
+            cleanupStageDir = stage
+        case (.xip(let inputPath), .buildNormal):
+            wanted = try await extractXIP(inputPath: inputPath, outDir: output.path)
+            appDir = expectedAppDir
+            cleanupStageDir = nil
+        case (.xip, .update):
+            throw Console.Error("Can't update with xip input")
+        case (.app(let appPath), .buildSlim), (.app(let appPath), .update):
             appDir = URL(fileURLWithPath: appPath)
+            wanted = nil
+            cleanupStageDir = nil
+        case (.app(let appPath), .buildNormal):
+            let source = URL(fileURLWithPath: appPath)
+            try await FileManager.default.copyItem(at: source, to: expectedAppDir, preserveOwner: false)
+            appDir = expectedAppDir
+            wanted = nil
             cleanupStageDir = nil
         }
+        try Task.checkCancellation()
 
         try FileManager.default.createDirectory(at: dev, withIntermediateDirectories: false)
 
@@ -280,7 +314,7 @@ struct SDKBuilder {
                 }
                 if count % 100 == 0 {
                     if wanted == nil {
-                        print("\r[Installing SDKs] Copied \(count) files", terminator: "")
+                        print("\r[Installing SDKs] Installed \(count) files", terminator: "")
                         fflush(stdoutSafe)
                     }
                     await Task.yield()
@@ -293,7 +327,11 @@ struct SDKBuilder {
                 if try child.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true {
                     toDoDirs.append(path)
                     try FileManager.default.createDirectory(at: dest, withIntermediateDirectories: false)
+                } else if mode.usesHardLinks {
+                    // Installed SDKs retain Xcode.app, so avoid storing their developer files twice.
+                    try FileManager.default.linkItem(at: child, to: dest)
                 } else {
+                    // Slim SDKs omit Xcode.app and contain independent copies.
                     try FileManager.default.copyItem(at: child, to: dest)
                 }
             }
@@ -305,9 +343,9 @@ struct SDKBuilder {
         }
         print()
 
-        print("[Cleaning up]")
         if let cleanupStageDir {
-            try? FileManager.default.removeItem(at: cleanupStageDir)
+            print("[Cleaning up]")
+            try FileManager.default.removeItem(at: cleanupStageDir)
         }
 
         print("[Finalizing SDKs]")
@@ -357,9 +395,18 @@ struct SDKBuilder {
         return dev
     }
 
+    private func extractXIP(inputPath: String, outDir: String) async throws -> Int {
+        // unxip doesn't like cooperative cancellation atm so shield it.
+        // if the user does a ^C during unxip, we'll just wait until extraction
+        // is over before bailing
+        try await Task {
+            try await _extractXIP(inputPath: inputPath, outDir: outDir)
+        }.value
+    }
+
     // returns the number of files we actually want to keep,
     // useful for computing progress % during fs traversal
-    private func extractXIP(inputPath: String, outDir: String) async throws -> Int {
+    private func _extractXIP(inputPath: String, outDir: String) async throws -> Int {
         let fd = try FileDescriptor.open(inputPath, .readOnly)
         defer { try? fd.close() }