fix(connections): write connection files that iPhone and older Macs can open

apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle.swift

@@ -128,6 +128,7 @@ public struct BundleSavedQuery: Codable, Sendable, Equatable {
 }
 
 public struct ConnectionBundle: Sendable, Equatable {
+    /// The newest format this build reads. A file is written in it only when format 1 cannot hold the bundle.
     public static let formatVersion = 2
 
     public let exportedAt: Date

apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundleCodec.swift

@@ -16,8 +16,10 @@ public enum ConnectionBundleCodec {
         let json: Data
         do {
             json = try await ConnectionExportCrypto.decrypt(data: data, passphrase: passphrase)
+        } catch ConnectionExportCryptoError.invalidPassphrase {
+            throw ConnectionBundleError.decryptionFailed(ConnectionExportCryptoError.invalidPassphrase.localizedDescription)
         } catch {
-            throw ConnectionBundleError.decryptionFailed(error.localizedDescription)
+            throw ConnectionBundleError.decodingFailed(error.localizedDescription)
         }
         return try decodeJSON(json)
     }
@@ -50,7 +52,7 @@ public enum ConnectionBundleCodec {
         guard probe.formatVersion <= ConnectionBundle.formatVersion else {
             throw ConnectionBundleError.unsupportedVersion(probe.formatVersion)
         }
-        if probe.formatVersion == 1 {
+        if probe.formatVersion == ConnectionBundleV1.formatVersion {
             let envelope = try decodeBody(ConnectionBundleV1.Envelope.self, from: data)
             return try envelope.upgraded().sanitizedForImport()
         }
@@ -68,11 +70,15 @@ public enum ConnectionBundleCodec {
         }
     }
 
+    /// Format 2 only when the bundle needs it, so a file without saved queries or nested groups opens in older builds.
     private static func encodeJSON(_ bundle: ConnectionBundle) throws -> Data {
         let encoder = JSONEncoder()
         encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
         encoder.dateEncodingStrategy = .iso8601
         do {
+            if let envelope = ConnectionBundleV1.Envelope(bundle) {
+                return try encoder.encode(envelope)
+            }
             return try encoder.encode(bundle)
         } catch {
             throw ConnectionBundleError.encodingFailed

apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundleError.swift

@@ -6,6 +6,7 @@ public enum ConnectionBundleError: LocalizedError, Equatable {
     case decodingFailed(String)
     case invalidBundle(String)
     case requiresPassphrase
+    /// The passphrase does not open the file. A file that cannot be decrypted for any other reason is `decodingFailed`.
     case decryptionFailed(String)
     case encodingFailed
     case credentialsRequireEncryption

apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundleV1.swift

@@ -1,8 +1,11 @@
 import Foundation
 
-/// The format 1 file shape, kept frozen: the only place a v1 file is read.
+/// The format 1 file shape, kept frozen: the only place a v1 file is read or written. Builds before format 2
+/// read nothing else, so a bundle that fits it is written in it.
 enum ConnectionBundleV1 {
-    struct Envelope: Decodable {
+    static let formatVersion = 1
+
+    struct Envelope: Codable {
         let formatVersion: Int
         let exportedAt: Date
         let appVersion: String
@@ -54,7 +57,7 @@ enum ConnectionBundleV1 {
         }
     }
 
-    struct Connection: Decodable {
+    struct Connection: Codable {
         let settings: ExportableConnection
         let groupName: String?
         let tagName: String?
@@ -65,6 +68,15 @@ enum ConnectionBundleV1 {
             case groupName, tagName, tagNames, credentialProfileName
         }
 
+        /// `tagName` repeats the first tag for builds that read only it.
+        init(settings: ExportableConnection, groupName: String?, tagNames: [String], credentialProfileName: String?) {
+            self.settings = settings
+            self.groupName = groupName
+            self.tagName = tagNames.first
+            self.tagNames = tagNames.isEmpty ? nil : tagNames
+            self.credentialProfileName = credentialProfileName
+        }
+
         init(from decoder: any Decoder) throws {
             settings = try ExportableConnection(from: decoder)
             let container = try decoder.container(keyedBy: CodingKeys.self)
@@ -73,20 +85,29 @@ enum ConnectionBundleV1 {
             tagNames = try container.decodeIfPresent([String].self, forKey: .tagNames)
             credentialProfileName = try container.decodeIfPresent(String.self, forKey: .credentialProfileName)
         }
+
+        func encode(to encoder: any Encoder) throws {
+            try settings.encode(to: encoder)
+            var container = encoder.container(keyedBy: CodingKeys.self)
+            try container.encodeIfPresent(groupName, forKey: .groupName)
+            try container.encodeIfPresent(tagName, forKey: .tagName)
+            try container.encodeIfPresent(tagNames, forKey: .tagNames)
+            try container.encodeIfPresent(credentialProfileName, forKey: .credentialProfileName)
+        }
     }
 
-    struct Group: Decodable {
+    struct Group: Codable {
         let name: String
         let color: String?
         let iconName: String?
     }
 
-    struct Tag: Decodable {
+    struct Tag: Codable {
         let name: String
         let color: String?
     }
 
-    struct CredentialProfile: Decodable {
+    struct CredentialProfile: Codable {
         let name: String
         let username: String
         let passwordMode: String
@@ -106,3 +127,58 @@ enum ConnectionBundleV1 {
         return values
     }
 }
+
+extension ConnectionBundleV1.Envelope {
+    /// Nil unless the envelope reads back as the same bundle, connection `i` as ref "i". Saved queries, query
+    /// folders, a group inside a group, and names that differ only in case fail that, and so does any field
+    /// format 2 gains later, so format 1 never drops what format 2 keeps.
+    init?(_ bundle: ConnectionBundle) {
+        var credentials: [String: ExportableCredentials] = [:]
+        for (index, connection) in bundle.connections.enumerated() {
+            credentials[String(index)] = bundle.credentials[connection.ref]
+        }
+        let envelope = Self(
+            formatVersion: ConnectionBundleV1.formatVersion,
+            exportedAt: bundle.exportedAt,
+            appVersion: bundle.appVersion,
+            connections: bundle.connections.map { connection in
+                ConnectionBundleV1.Connection(
+                    settings: connection.settings,
+                    groupName: bundle.groupChain(connection.groupRef).last?.name,
+                    tagNames: connection.tagNames,
+                    credentialProfileName: bundle.credentialProfile(connection.credentialProfileRef)?.name
+                )
+            },
+            groups: bundle.groups.isEmpty ? nil : bundle.groups.map {
+                ConnectionBundleV1.Group(name: $0.name, color: $0.color, iconName: $0.iconName)
+            },
+            tags: bundle.tags.isEmpty ? nil : bundle.tags.map { ConnectionBundleV1.Tag(name: $0.name, color: $0.color) },
+            credentials: credentials.isEmpty ? nil : credentials,
+            credentialProfiles: bundle.credentialProfiles.isEmpty ? nil : bundle.credentialProfiles.map {
+                ConnectionBundleV1.CredentialProfile(
+                    name: $0.name,
+                    username: $0.username,
+                    passwordMode: $0.passwordMode.rawValue,
+                    secureFieldIds: $0.secureFieldIds.isEmpty ? nil : $0.secureFieldIds
+                )
+            }
+        )
+        let renumbered = ConnectionBundle(
+            unchecked: bundle.exportedAt,
+            appVersion: bundle.appVersion,
+            connections: bundle.connections.enumerated().map { index, connection in
+                var copy = connection
+                copy.ref = BundleRef(String(index))
+                return copy
+            },
+            groups: bundle.groups,
+            tags: bundle.tags,
+            credentialProfiles: bundle.credentialProfiles,
+            credentials: Dictionary(uniqueKeysWithValues: credentials.map { (BundleRef($0.key), $0.value) }),
+            queryFolders: bundle.queryFolders,
+            savedQueries: bundle.savedQueries
+        )
+        guard (try? envelope.upgraded()) == renumbered else { return nil }
+        self = envelope
+    }
+}

apple/Packages/TableProCore/Tests/TableProImportTests/ConnectionBundleCodecTests.swift

@@ -37,7 +37,7 @@ struct ConnectionBundleCodecTests {
 
         let object = try Self.jsonObject(ConnectionBundleCodec.encode(bundle))
 
-        #expect(object["formatVersion"] as? Int == 2)
+        #expect(object["formatVersion"] as? Int == 1)
         #expect(object["exportedAt"] as? String == "2023-11-14T22:13:20Z")
         #expect(object["appVersion"] as? String == "0.70.0")
         #expect((object["connections"] as? [Any])?.isEmpty == true)
@@ -56,6 +56,89 @@ struct ConnectionBundleCodecTests {
         #expect(credentials["c1"]?["password"] as? String == "hunter2")
     }
 
+    @Test("A bundle without saved queries or nested groups is written in format 1, linked by name")
+    func flatBundleIsWrittenInFormatOne() async throws {
+        let sealed = try await ConnectionBundleCodec.encode(Self.flatBundle(), passphrase: Self.passphrase)
+        let object = try await Self.jsonObject(ConnectionExportCrypto.decrypt(data: sealed, passphrase: Self.passphrase))
+        let connections = try #require(object["connections"] as? [[String: Any]])
+        let orders = try #require(connections.first)
+
+        #expect(object["formatVersion"] as? Int == 1)
+        #expect(orders["name"] as? String == "Orders")
+        #expect(orders["groupName"] as? String == "Production")
+        #expect(orders["tagName"] as? String == "prod")
+        #expect(orders["tagNames"] as? [String] == ["prod", "team"])
+        #expect(orders["credentialProfileName"] as? String == "reader")
+        #expect(connections.allSatisfy { $0["ref"] == nil && $0["groupRef"] == nil && $0["credentialProfileRef"] == nil })
+        #expect((object["groups"] as? [[String: Any]])?.first?["ref"] == nil)
+        #expect((object["credentialProfiles"] as? [[String: Any]])?.first?["passwordMode"] as? String == "pgpass")
+        let credentials = try #require(object["credentials"] as? [String: Any])
+        #expect(Set(credentials.keys) == ["1"])
+    }
+
+    @Test("A format 1 file reads back as the bundle it was written from")
+    func formatOneRoundTrips() async throws {
+        let bundle = try Self.flatBundle()
+
+        let sealed = try await ConnectionBundleCodec.encode(bundle, passphrase: Self.passphrase)
+        let decoded = try await ConnectionBundleCodec.decode(sealed, passphrase: Self.passphrase)
+
+        #expect(decoded == bundle)
+        #expect(decoded.credentials["1"]?.password == "hunter2")
+    }
+
+    @Test("Saved queries, a query folder or a nested group keep format 2")
+    func formatTwoOnlyWhenNeeded() throws {
+        let settings = BundleConnection(ref: "c1", settings: Self.settings())
+        let bundles = try [
+            ConnectionBundle(
+                exportedAt: Self.exportedAt,
+                appVersion: "0.70.0",
+                connections: [settings],
+                savedQueries: [BundleSavedQuery(ref: "q1", name: "Locks", sql: "SELECT 1")]
+            ),
+            ConnectionBundle(
+                exportedAt: Self.exportedAt,
+                appVersion: "0.70.0",
+                connections: [settings],
+                queryFolders: [BundleQueryFolder(ref: "f1", name: "Reports")]
+            ),
+            ConnectionBundle(
+                exportedAt: Self.exportedAt,
+                appVersion: "0.70.0",
+                connections: [BundleConnection(ref: "c1", settings: Self.settings(), groupRef: "g2")],
+                groups: [BundleGroup(ref: "g1", name: "Client A"), BundleGroup(ref: "g2", name: "Production", parentRef: "g1")]
+            )
+        ]
+
+        for bundle in bundles {
+            let data = try ConnectionBundleCodec.encode(bundle)
+            #expect(try Self.jsonObject(data)["formatVersion"] as? Int == 2)
+            #expect(try ConnectionBundleCodec.decode(data) == bundle)
+        }
+    }
+
+    @Test("Names format 1 would merge keep format 2")
+    func caseOnlyNamesKeepFormatTwo() throws {
+        let bundle = try ConnectionBundle(
+            exportedAt: Self.exportedAt,
+            appVersion: "0.70.0",
+            connections: [
+                BundleConnection(ref: "c1", settings: Self.settings(), credentialProfileRef: "p1"),
+                BundleConnection(ref: "c2", settings: Self.settings(name: "Second"), credentialProfileRef: "p2")
+            ],
+            credentialProfiles: [
+                BundleCredentialProfile(ref: "p1", name: "reader", username: "alice", passwordMode: .prompt),
+                BundleCredentialProfile(ref: "p2", name: "Reader", username: "bob", passwordMode: .pgpass)
+            ]
+        )
+
+        let data = try ConnectionBundleCodec.encode(bundle)
+
+        #expect(try Self.jsonObject(data)["formatVersion"] as? Int == 2)
+        #expect(try ConnectionBundleCodec.decode(data) == bundle)
+    }
+
     @Test("A plain file never carries credentials into the app")
     func plainDecodeDropsCredentials() throws {
         let json = Data("""
@@ -123,6 +206,31 @@ struct ConnectionBundleCodecTests {
         }
     }
 
+    @Test("An encrypted file in a newer format reports the version once the passphrase opens it")
+    func encryptedNewerFormatIsUnsupported() async throws {
+        let json = Data(#"{"formatVersion":3,"exportedAt":"2023-11-14T22:13:20Z","appVersion":"9.0","connections":[]}"#.utf8)
+        let sealed = try await ConnectionExportCrypto.encrypt(data: json, passphrase: Self.passphrase)
+
+        await #expect(throws: ConnectionBundleError.unsupportedVersion(3)) {
+            _ = try await ConnectionBundleCodec.decode(sealed, passphrase: Self.passphrase)
+        }
+    }
+
+    @Test("An encrypted file no passphrase can open fails to parse instead of asking again")
+    func unreadableContainerIsNotAPassphraseFailure() async {
+        let saltAndNonce = Data(repeating: 0, count: 44)
+        let cases: [(container: Data, reason: ConnectionExportCryptoError)] = [
+            (Data("TPRO".utf8) + Data([1]) + saltAndNonce + Data(repeating: 0, count: 10), .corruptData),
+            (Data("TPRO".utf8) + Data([2]) + saltAndNonce + Data(repeating: 0, count: 40), .unsupportedVersion(2))
+        ]
+        for entry in cases {
+            #expect(ConnectionBundleCodec.isEncrypted(entry.container))
+            await #expect(throws: ConnectionBundleError.decodingFailed(entry.reason.localizedDescription)) {
+                _ = try await ConnectionBundleCodec.decode(entry.container, passphrase: Self.passphrase)
+            }
+        }
+    }
+
     @Test("A newer format is unsupported whatever its shape")
     func newerFormatIsUnsupported() {
         let json = Data(#"{"formatVersion":3,"bundle":{"anything":[1,2,3]}}"#.utf8)
@@ -308,6 +416,7 @@ struct ConnectionBundleCodecTests {
         )
     }
 
+    /// The saved query keeps it in format 2, where credentials are keyed by connection ref.
     private static func bundleWithCredentials() throws -> ConnectionBundle {
         try ConnectionBundle(
             exportedAt: exportedAt,
@@ -320,7 +429,8 @@ struct ConnectionBundleCodecTests {
                 sslClientKeyPassphrase: nil,
                 totpSecret: nil,
                 pluginSecureFields: ["token": "secret"]
-            )]
+            )],
+            savedQueries: [BundleSavedQuery(ref: "q1", name: "Locks", sql: "SELECT 1", connectionRef: "c1")]
         )
     }
 
@@ -358,6 +468,33 @@ struct ConnectionBundleCodecTests {
         )
     }
 
+    /// Built the way a format 1 file is read back: connection `i` has ref "i".
+    static func flatBundle() throws -> ConnectionBundle {
+        var builder = ConnectionBundleBuilder(appVersion: "0.70.0", exportedAt: exportedAt)
+        builder.addConnection(
+            fullSettings(),
+            ref: "0",
+            groupPath: [.init(name: "Production", color: "Blue", iconName: "person.3")],
+            tags: [BundleTag(name: "prod", color: "Red"), BundleTag(name: "team")],
+            credentialProfile: .init(name: "reader", username: "ro", passwordMode: .pgpass, secureFieldIds: ["token"])
+        )
+        builder.addConnection(
+            settings(name: "Cache"),
+            ref: "1",
+            groupPath: [.init(name: "Production")],
+            credentialProfile: .init(name: "reader", username: "ro", passwordMode: .pgpass),
+            credentials: ExportableCredentials(
+                password: "hunter2",
+                sshPassword: "ssh",
+                keyPassphrase: nil,
+                sslClientKeyPassphrase: "ssl",
+                totpSecret: "totp",
+                pluginSecureFields: ["token": "secret"]
+            )
+        )
+        return try builder.build()
+    }
+
     static func fullBundle() throws -> ConnectionBundle {
         try ConnectionBundle(
             exportedAt: exportedAt,