From e0a0b045517f056cf1de93df805b7b430691e9cf Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 24 Sep 2026 03:32:01 +0700 Subject: [PATCH] fix: keep edits made during an iCloud sync and SQL file encodings on save, and move diff work off the main thread --- CHANGELOG.md | 5 + .../CloudKitSyncEngine.swift | 20 +- .../TableProSyncTransport/PushOutcome.swift | 24 + .../TableProSyncTransport/SyncError.swift | 4 + .../TableProSyncTests/PushOutcomeTests.swift | 55 ++ .../StructureDefinitionDiffPresentation.swift | 30 + .../Infrastructure/EditorTabOpener.swift | 1 + .../Infrastructure/SQLFileService.swift | 17 +- .../Services/Infrastructure/TabRouter.swift | 37 +- .../SQL/LinkedSQLFavoriteWriter.swift | 12 +- TablePro/Core/Storage/ConnectionStorage.swift | 8 +- .../Storage/CredentialProfileStorage.swift | 3 +- .../Storage/FavoriteDatabasesStorage.swift | 5 +- .../Core/Storage/FavoriteTablesStorage.swift | 20 +- TablePro/Core/Storage/GroupStorage.swift | 11 +- .../Core/Storage/SQLFavoriteManager.swift | 199 ++++-- .../SQLFavoriteStorage+SyncReads.swift | 51 ++ .../Core/Storage/SQLFavoriteStorage.swift | 14 +- TablePro/Core/Storage/SSHProfileStorage.swift | 3 +- TablePro/Core/Storage/TagStorage.swift | 9 +- .../SyncCoordinator+PushCollection.swift | 175 +++++ .../SyncCoordinator+RemoteDeletions.swift | 163 +++++ .../Extensions/SyncCoordinator+Settings.swift | 64 ++ TablePro/Core/Sync/SyncChangeTracker.swift | 43 ++ TablePro/Core/Sync/SyncCoordinator.swift | 669 ++++-------------- TablePro/Core/Sync/SyncEchoGuard.swift | 29 + TablePro/Core/Sync/SyncEditSnapshot.swift | 29 + TablePro/Core/Sync/SyncRecordChanges.swift | 13 + TablePro/Core/Sync/SyncTransport.swift | 14 + .../Utilities/File/FileTextEncoding.swift | 59 ++ .../Core/Utilities/File/FileTextLoader.swift | 85 ++- .../Core/Utilities/File/FileTextWriter.swift | 85 +++ .../File/TextEncodingAttribute.swift | 51 ++ .../Utilities/File/TextPrefixDecoder.swift | 31 +- .../Core/Utilities/MainActorSerialQueue.swift | 20 + .../Core/Utilities/SQL/SQLChunkDecoder.swift | 27 +- .../Core/Utilities/Text/ByteOrderMark.swift | 29 +- .../LinkedFileVersionHistoryProvider.swift | 3 +- TablePro/Models/Query/EditorTabPayload.swift | 9 +- TablePro/Models/Query/FileTabBaseline.swift | 30 +- TablePro/Models/Query/LinkedSQLFavorite.swift | 13 + TablePro/Models/Query/QueryTabManager.swift | 19 +- TablePro/Models/Query/QueryTabState.swift | 4 + TablePro/Resources/Localizable.xcstrings | 204 +++++- .../ViewModels/QueryPlanComparisonModel.swift | 4 +- .../Compare/StructureDefinitionDiffView.swift | 24 +- ...MainContentCommandActions+FileSaving.swift | 102 ++- .../MainContentCoordinator+Favorites.swift | 6 +- .../MainContentCommandActions+BulkClose.swift | 6 +- .../Main/MainContentCommandActions.swift | 17 +- TablePro/Views/Shared/ObjectSourceView.swift | 2 +- .../Views/Sidebar/LinkedFavoriteRowView.swift | 11 +- ...ctureDefinitionDiffPresentationTests.swift | 82 +++ .../Core/Git/GitIntegrationTests.swift | 45 ++ .../Core/Git/GitOutputParserTests.swift | 16 + .../AppSettingsCategoryParityTests.swift | 2 +- .../Infrastructure/SQLFileOpeningTests.swift | 70 ++ .../Services/PersistedTabRoundTripTests.swift | 2 +- .../SQL/LinkedSQLFavoriteWriterTests.swift | 47 ++ .../Storage/FavoriteTablesStorageTests.swift | 1 + .../Core/Storage/GroupStorageTests.swift | 10 +- .../Storage/SQLFavoriteStorageOpenTests.swift | 70 ++ .../Core/Storage/SyncDirtyMarkingTests.swift | 173 +++++ .../Core/Storage/TagStorageTests.swift | 2 - .../Core/Sync/SyncChangeTrackerTests.swift | 88 +++ .../Core/Sync/SyncCoordinatorEchoTests.swift | 555 +++++++++++++++ TableProTests/Core/Sync/SyncScopeTests.swift | 3 +- .../Core/Utilities/FileTextLoaderTests.swift | 84 +++ .../Core/Utilities/FileTextWriterTests.swift | 158 +++++ .../Utilities/MainActorSerialQueueTests.swift | 58 ++ .../Utilities/Text/ByteOrderMarkTests.swift | 15 +- .../Utilities/TextPrefixDecoderTests.swift | 36 + .../Helpers/EncodedSQLFileFixture.swift | 117 +++ .../Models/FileTabBaselineTests.swift | 31 + .../Models/FileTabExternalChangeTests.swift | 29 +- .../LinkedSQLFavoriteEncodingTests.swift | 49 ++ .../Query/TabQueryContentEqualityTests.swift | 12 + .../SourceFileDiskChangeHandlingTests.swift | 10 +- .../Main/SourceFileEncodingSaveTests.swift | 320 +++++++++ docs/features/favorites.mdx | 2 +- docs/features/sql-files.mdx | 10 + 81 files changed, 3839 insertions(+), 826 deletions(-) create mode 100644 TablePro/Core/Diff/StructureDefinitionDiffPresentation.swift create mode 100644 TablePro/Core/Storage/SQLFavoriteStorage+SyncReads.swift create mode 100644 TablePro/Core/Sync/Extensions/SyncCoordinator+PushCollection.swift create mode 100644 TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift create mode 100644 TablePro/Core/Sync/Extensions/SyncCoordinator+Settings.swift create mode 100644 TablePro/Core/Sync/SyncEchoGuard.swift create mode 100644 TablePro/Core/Sync/SyncEditSnapshot.swift create mode 100644 TablePro/Core/Sync/SyncRecordChanges.swift create mode 100644 TablePro/Core/Sync/SyncTransport.swift create mode 100644 TablePro/Core/Utilities/File/FileTextEncoding.swift create mode 100644 TablePro/Core/Utilities/File/FileTextWriter.swift create mode 100644 TablePro/Core/Utilities/File/TextEncodingAttribute.swift create mode 100644 TablePro/Core/Utilities/MainActorSerialQueue.swift create mode 100644 TableProTests/Core/Diff/StructureDefinitionDiffPresentationTests.swift create mode 100644 TableProTests/Core/Services/Infrastructure/SQLFileOpeningTests.swift create mode 100644 TableProTests/Core/Storage/SQLFavoriteStorageOpenTests.swift create mode 100644 TableProTests/Core/Storage/SyncDirtyMarkingTests.swift create mode 100644 TableProTests/Core/Sync/SyncCoordinatorEchoTests.swift create mode 100644 TableProTests/Core/Utilities/FileTextWriterTests.swift create mode 100644 TableProTests/Core/Utilities/MainActorSerialQueueTests.swift create mode 100644 TableProTests/Helpers/EncodedSQLFileFixture.swift create mode 100644 TableProTests/Models/LinkedSQLFavoriteEncodingTests.swift create mode 100644 TableProTests/Views/Main/SourceFileEncodingSaveTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eb5060fe5..e5482aabf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Edits made while an iCloud sync was running reverted by that sync and never uploaded. +- Saved queries unavailable until relaunch after their store failed to open once. +- SQL files saved as UTF-8 whatever their encoding, and non-UTF-8 SQL files not opening from Finder or **File > Open File…**. +- Unresponsive app while comparing large query plans. +- Slow definition diff in Compare & Sync for large tables. - Autocomplete offering another schema's tables without their schema once that schema was completed or expanded. - Stale column and MongoDB field suggestions when a refresh ran while they were loading. - Tables in an expanded Oracle or Snowflake schema missing from Open Quickly until the next refresh. diff --git a/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift b/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift index 194cc697c9..6b3de40b61 100644 --- a/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift +++ b/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift @@ -88,11 +88,15 @@ public actor CloudKitSyncEngine { ) for plan in plans { - outcome.merge(try await push( - plan: plan, - records: publishableRecords, - deletions: publishableDeletions - )) + do { + outcome.merge(try await push( + plan: plan, + records: publishableRecords, + deletions: publishableDeletions + )) + } catch { + throw SyncPushInterruption.after(outcome, failingWith: error) + } } let saved = outcome.savedRecords.count @@ -129,7 +133,11 @@ public actor CloudKitSyncEngine { var outcome = PushOutcome() for half in halves { - outcome.merge(try await push(plan: half, records: records, deletions: deletions)) + do { + outcome.merge(try await push(plan: half, records: records, deletions: deletions)) + } catch { + throw SyncPushInterruption.after(outcome, failingWith: error) + } } return outcome } diff --git a/Packages/TableProCore/Sources/TableProSyncTransport/PushOutcome.swift b/Packages/TableProCore/Sources/TableProSyncTransport/PushOutcome.swift index 256d4fcc46..7b576cc641 100644 --- a/Packages/TableProCore/Sources/TableProSyncTransport/PushOutcome.swift +++ b/Packages/TableProCore/Sources/TableProSyncTransport/PushOutcome.swift @@ -42,6 +42,10 @@ public struct PushOutcome: Sendable { public var hasFailures: Bool { !failures.isEmpty } + public var isEmpty: Bool { + savedRecords.isEmpty && deletedRecordIDs.isEmpty && failures.isEmpty + } + public var conflicts: [CKRecord.ID: SyncItemFailure] { failures.filter(\.value.isConflict) } @@ -90,3 +94,23 @@ public struct PushOutcome: Sendable { } } } + +public struct SyncPushInterruption: Error, Sendable { + public let completed: PushOutcome + public let cause: any Error + + public init(completed: PushOutcome, cause: any Error) { + self.completed = completed + self.cause = cause + } + + public static func after(_ completed: PushOutcome, failingWith error: any Error) -> any Error { + if let interruption = error as? SyncPushInterruption { + var merged = completed + merged.merge(interruption.completed) + return SyncPushInterruption(completed: merged, cause: interruption.cause) + } + guard !completed.isEmpty else { return error } + return SyncPushInterruption(completed: completed, cause: error) + } +} diff --git a/Packages/TableProCore/Sources/TableProSyncTransport/SyncError.swift b/Packages/TableProCore/Sources/TableProSyncTransport/SyncError.swift index 46269449d4..cb545a924b 100644 --- a/Packages/TableProCore/Sources/TableProSyncTransport/SyncError.swift +++ b/Packages/TableProCore/Sources/TableProSyncTransport/SyncError.swift @@ -50,6 +50,10 @@ public enum SyncError: Error, LocalizedError, Equatable, Sendable { return syncError } + if let interruption = error as? SyncPushInterruption { + return from(interruption.cause) + } + if let ckError = error as? CKError { switch ckError.code { case .networkUnavailable, .networkFailure: diff --git a/Packages/TableProCore/Tests/TableProSyncTests/PushOutcomeTests.swift b/Packages/TableProCore/Tests/TableProSyncTests/PushOutcomeTests.swift index f96d927407..e920d1624e 100644 --- a/Packages/TableProCore/Tests/TableProSyncTests/PushOutcomeTests.swift +++ b/Packages/TableProCore/Tests/TableProSyncTests/PushOutcomeTests.swift @@ -144,4 +144,59 @@ struct PushOutcomeTests { #expect(outcome.failures[rejected] != nil) #expect(!outcome.didDelete(rejected)) } + + @Test("A push that stopped after saving some records carries those saves with the error") + func interruptionKeepsWhatWasSaved() throws { + var completed = PushOutcome() + let saved = makeRecord("Connection_Saved") + completed.recordSave(saved) + + let error = SyncPushInterruption.after(completed, failingWith: CKError(.networkFailure)) + + let interruption = try #require(error as? SyncPushInterruption) + #expect(interruption.completed.didSave(saved.recordID)) + #expect((interruption.cause as? CKError)?.code == .networkFailure) + } + + @Test("A push that stopped before saving anything throws its own error unchanged") + func interruptionWithoutProgressIsTheRawError() { + let error = SyncPushInterruption.after(PushOutcome(), failingWith: CKError(.networkFailure)) + + #expect(!(error is SyncPushInterruption)) + #expect((error as? CKError)?.code == .networkFailure) + } + + @Test("An interruption from a later batch keeps the saves of the batches before it") + func nestedInterruptionsMerge() throws { + var earlier = PushOutcome() + let first = makeRecord("Connection_First") + earlier.recordSave(first) + var later = PushOutcome() + let second = makeRecord("Connection_Second") + later.recordSave(second) + let inner = SyncPushInterruption(completed: later, cause: CKError(.networkFailure)) + + let error = SyncPushInterruption.after(earlier, failingWith: inner) + + let interruption = try #require(error as? SyncPushInterruption) + #expect(interruption.completed.didSave(first.recordID)) + #expect(interruption.completed.didSave(second.recordID)) + } + + @Test("An interrupted push reports the error that stopped it") + func interruptionMapsToItsCause() { + let interruption = SyncPushInterruption(completed: PushOutcome(), cause: CKError(.networkFailure)) + + #expect(SyncError.from(interruption) == .networkUnavailable) + } + + @Test("An outcome with nothing saved, deleted or rejected is empty") + func emptyOutcome() { + var outcome = PushOutcome() + #expect(outcome.isEmpty) + + outcome.recordDeletion(recordID("Connection_Gone")) + + #expect(!outcome.isEmpty) + } } diff --git a/TablePro/Core/Diff/StructureDefinitionDiffPresentation.swift b/TablePro/Core/Diff/StructureDefinitionDiffPresentation.swift new file mode 100644 index 0000000000..ac5eace13a --- /dev/null +++ b/TablePro/Core/Diff/StructureDefinitionDiffPresentation.swift @@ -0,0 +1,30 @@ +// +// StructureDefinitionDiffPresentation.swift +// TablePro +// + +import Foundation + +internal struct StructureDefinitionDiffInput: Equatable, Sendable { + let sourceLines: [String] + let targetLines: [String] +} + +internal struct StructureDefinitionDiffPresentation: Equatable, Sendable { + let input: StructureDefinitionDiffInput + let pairs: [DiffPair] + + init(input: StructureDefinitionDiffInput) { + self.input = input + pairs = DiffComputer.computeSplit(before: input.targetLines, after: input.sourceLines) + } + + @concurrent + static func load(_ input: StructureDefinitionDiffInput) async -> StructureDefinitionDiffPresentation { + StructureDefinitionDiffPresentation(input: input) + } + + func isCurrent(for input: StructureDefinitionDiffInput) -> Bool { + self.input == input + } +} diff --git a/TablePro/Core/Services/Infrastructure/EditorTabOpener.swift b/TablePro/Core/Services/Infrastructure/EditorTabOpener.swift index ff6e873caf..bfa46837db 100644 --- a/TablePro/Core/Services/Infrastructure/EditorTabOpener.swift +++ b/TablePro/Core/Services/Infrastructure/EditorTabOpener.swift @@ -70,6 +70,7 @@ internal enum EditorTabOpener { databaseName: payload.databaseName ?? browseDatabaseName, sourceFileURL: payload.sourceFileURL, sourceFileStamp: payload.sourceFileStamp, + sourceFileEncoding: payload.sourceFileEncoding, claimFocus: true ) case .createTable: diff --git a/TablePro/Core/Services/Infrastructure/SQLFileService.swift b/TablePro/Core/Services/Infrastructure/SQLFileService.swift index 5c31a314d3..ac4df6a37e 100644 --- a/TablePro/Core/Services/Infrastructure/SQLFileService.swift +++ b/TablePro/Core/Services/Infrastructure/SQLFileService.swift @@ -20,26 +20,21 @@ enum SQLFileService { return types.isEmpty ? [.plainText] : Array(types) } - /// Reads a SQL file from disk. - static func readFile(url: URL) async throws -> String { + static func writeFile(content: String, to url: URL, encoding: FileTextEncoding) async throws { try await Task.detached { - try String(contentsOf: url, encoding: .utf8) + try FileTextWriter.write(content, to: url, as: encoding) }.value } - /// Writes content to a SQL file atomically. - static func writeFile(content: String, to url: URL) async throws { - try await Task.detached { - guard let data = content.data(using: .utf8) else { - throw CocoaError(.fileWriteInapplicableStringEncoding) - } - try data.write(to: url, options: .atomic) + static func encodingOnDisk(of url: URL) async -> FileTextEncoding? { + await Task.detached { + FileTextLoader.load(url)?.textEncoding }.value } static func writeData(_ data: Data, to url: URL) async throws { try await Task.detached { - try data.write(to: url, options: .atomic) + try FileTextWriter.replaceContents(of: url, with: data, attribute: TextEncodingAttribute.read(from: url)) }.value } diff --git a/TablePro/Core/Services/Infrastructure/TabRouter.swift b/TablePro/Core/Services/Infrastructure/TabRouter.swift index 01e15860a5..2452a40c40 100644 --- a/TablePro/Core/Services/Infrastructure/TabRouter.swift +++ b/TablePro/Core/Services/Infrastructure/TabRouter.swift @@ -488,22 +488,7 @@ internal final class TabRouter { } if let session = DatabaseManager.shared.lastActiveSession { - let read = await Task.detached(priority: .userInitiated) { () -> (content: String, stamp: FileStamp?)? in - let stamp = FileStamp.read(url) - guard let content = try? String(contentsOf: url, encoding: .utf8) else { return nil } - return (content, stamp) - }.value - guard let read else { - Self.logger.error("Failed to read SQL file: \(url.lastPathComponent, privacy: .private(mask: .hash))") - return - } - let payload = EditorTabPayload( - connectionId: session.connection.id, - tabType: .query, - initialQuery: read.content, - sourceFileURL: url, - sourceFileStamp: read.stamp - ) + let payload = try await Self.sqlFileTabPayload(for: url, connectionId: session.connection.id) WindowManager.shared.openTab(payload: payload) AppActivationPolicyController.shared.activate(ignoringOtherApps: true) } else { @@ -511,6 +496,26 @@ internal final class TabRouter { } } + internal static func sqlFileTabPayload(for url: URL, connectionId: UUID) async throws -> EditorTabPayload { + let read: FileTextLoader.LoadedText + do { + read = try await Task.detached(priority: .userInitiated) { + try FileTextLoader.read(url) + }.value + } catch { + logger.error("Failed to read SQL file: \(url.lastPathComponent, privacy: .private(mask: .hash))") + throw error + } + return EditorTabPayload( + connectionId: connectionId, + tabType: .query, + initialQuery: read.content, + sourceFileURL: url, + sourceFileStamp: read.stamp, + sourceFileEncoding: read.textEncoding + ) + } + // MARK: - Helpers internal func bringConnectionWindowToFront(_ connectionId: UUID) { diff --git a/TablePro/Core/Services/SQL/LinkedSQLFavoriteWriter.swift b/TablePro/Core/Services/SQL/LinkedSQLFavoriteWriter.swift index 8718f68abe..ef75ae44d9 100644 --- a/TablePro/Core/Services/SQL/LinkedSQLFavoriteWriter.swift +++ b/TablePro/Core/Services/SQL/LinkedSQLFavoriteWriter.swift @@ -13,7 +13,7 @@ internal enum LinkedSQLFavoriteWriter { enum WriteError: Error { case readFailed - case encodingMismatch(String.Encoding) + case encodingMismatch(FileTextEncoding) case writeFailed } @@ -33,12 +33,10 @@ internal enum LinkedSQLFavoriteWriter { let newContent = rewrite(loaded.content, with: metadata) do { - try newContent.write(to: url, atomically: true, encoding: loaded.encoding) - } catch let error as NSError where - error.domain == NSCocoaErrorDomain && - error.code == NSFileWriteInapplicableStringEncodingError { - Self.logger.error("Encoding \(loaded.encoding.rawValue) cannot represent edited content at \(url.path, privacy: .private(mask: .hash))") - throw WriteError.encodingMismatch(loaded.encoding) + try FileTextWriter.write(newContent, to: url, as: loaded.textEncoding) + } catch FileTextWriter.WriteError.unrepresentable(let encoding) { + Self.logger.error("Encoding \(encoding.encoding.rawValue) cannot represent edited content at \(url.path, privacy: .private(mask: .hash))") + throw WriteError.encodingMismatch(encoding) } catch { Self.logger.error("Failed to write metadata to \(url.path, privacy: .private(mask: .hash)): \(error.publicLogShape, privacy: .public)") throw WriteError.writeFailed diff --git a/TablePro/Core/Storage/ConnectionStorage.swift b/TablePro/Core/Storage/ConnectionStorage.swift index 782580a30b..2435bb3717 100644 --- a/TablePro/Core/Storage/ConnectionStorage.swift +++ b/TablePro/Core/Storage/ConnectionStorage.swift @@ -27,6 +27,8 @@ final class ConnectionStorage { /// In-memory cache to avoid re-decoding JSON from file on every access private var cachedConnections: [DatabaseConnection]? + private(set) var lastLoadFailed = false + /// Whether the file on disk is the one TablePro last wrote. False once it has been edited by /// something else, which is the signal to refuse to run a connection's password source. var storeIsTrusted: Bool { file.isTrusted } @@ -90,7 +92,11 @@ final class ConnectionStorage { func loadConnections() -> [DatabaseConnection] { if let cached = cachedConnections { return cached } - guard let storedConnections = file.load() else { return [] } + guard let storedConnections = file.load() else { + lastLoadFailed = true + return [] + } + lastLoadFailed = false let connections = storedConnections.map { stored in stored.toConnection() diff --git a/TablePro/Core/Storage/CredentialProfileStorage.swift b/TablePro/Core/Storage/CredentialProfileStorage.swift index 1964f0ce34..07b66b64be 100644 --- a/TablePro/Core/Storage/CredentialProfileStorage.swift +++ b/TablePro/Core/Storage/CredentialProfileStorage.swift @@ -87,8 +87,9 @@ final class CredentialProfileStorage { @discardableResult func saveProfiles(_ profiles: [CredentialProfile]) -> Bool { + let previous = loadProfiles() guard saveProfilesWithoutSync(profiles) else { return false } - syncTracker.markDirty(.credentialProfile, ids: profiles.map { $0.id.uuidString }) + syncTracker.markDirty(.credentialProfile, ids: SyncRecordChanges.changedIds(from: previous, to: profiles)) return true } diff --git a/TablePro/Core/Storage/FavoriteDatabasesStorage.swift b/TablePro/Core/Storage/FavoriteDatabasesStorage.swift index 28aa98a9e8..6dad69db05 100644 --- a/TablePro/Core/Storage/FavoriteDatabasesStorage.swift +++ b/TablePro/Core/Storage/FavoriteDatabasesStorage.swift @@ -117,6 +117,7 @@ internal final class FavoriteDatabasesStorage { persist(favorites) guard !skipSync else { + syncTracker.discardDirty(.favoriteDatabase, ids: removed.map(Self.syncId(for:))) postChangeNotification() return } @@ -169,7 +170,9 @@ internal final class FavoriteDatabasesStorage { } postChangeNotification() case .removed(let entry): - if !skipSync { + if skipSync { + syncTracker.discardDirty(.favoriteDatabase, ids: [Self.syncId(for: entry)]) + } else { syncTracker.markDeleted(.favoriteDatabase, id: Self.syncId(for: entry)) } postChangeNotification() diff --git a/TablePro/Core/Storage/FavoriteTablesStorage.swift b/TablePro/Core/Storage/FavoriteTablesStorage.swift index 9b2af15860..021d73fee7 100644 --- a/TablePro/Core/Storage/FavoriteTablesStorage.swift +++ b/TablePro/Core/Storage/FavoriteTablesStorage.swift @@ -48,6 +48,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { ) } + @MainActor func toggle(name: String, schema: String?, database: String?, connectionId: UUID) { let entry = FavoriteEntry(connectionId: connectionId, database: database, schema: schema, name: name) let action: TrackedAction = mutate { favorites in @@ -61,6 +62,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { notify(after: action) } + @MainActor @discardableResult func addFavorite(name: String, schema: String?, database: String?, connectionId: UUID) -> Bool { let entry = FavoriteEntry(connectionId: connectionId, database: database, schema: schema, name: name) @@ -72,6 +74,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { return action.changed } + @MainActor @discardableResult func addFavoriteWithoutSync(_ entry: FavoriteEntry) -> Bool { let action = mutate { favorites in @@ -81,6 +84,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { return action.changed } + @MainActor func removeFavorite(name: String, schema: String?, database: String?, connectionId: UUID) { let entry = FavoriteEntry(connectionId: connectionId, database: database, schema: schema, name: name) let action = mutate { favorites in @@ -89,6 +93,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { notify(after: action) } + @MainActor func removeFavoriteWithoutSync(_ entry: FavoriteEntry) { let action = mutate { favorites in favorites.remove(entry) != nil ? .removed(entry) : .noChange @@ -96,6 +101,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { notify(after: action, skipSync: true) } + @MainActor func removeFavoriteWithoutSync(id: String) { let action = mutate { favorites in guard let entry = favorites.first(where: { Self.syncId(for: $0) == id }) else { return .noChange } @@ -103,12 +109,14 @@ final class FavoriteTablesStorage: @unchecked Sendable { return .removed(entry) } notify(after: action, skipSync: true) + syncTracker.discardDirty(.tableFavorite, ids: [id]) } /// Drops every favorite inside a database, or inside one schema of it when a schema is named. /// /// The container is gone, so each entry names a table that no longer exists. Removed one at a /// time through the syncing path, because the tables are gone on every device, not only this one. + @MainActor @discardableResult func removeFavorites(inDatabase database: String?, schema: String?, connectionId: UUID) -> [FavoriteEntry] { let doomed = favorites(for: connectionId).filter { entry in @@ -124,6 +132,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { return Array(doomed) } + @MainActor @discardableResult func removeFavorites(for connectionId: UUID) -> [FavoriteEntry] { removeFavorites(for: connectionId, skipSync: false) @@ -131,11 +140,13 @@ final class FavoriteTablesStorage: @unchecked Sendable { /// Used when another device deleted the connection. Marking tombstones here would push its own /// deletion straight back at it. + @MainActor @discardableResult func removeFavoritesWithoutSync(for connectionId: UUID) -> [FavoriteEntry] { removeFavorites(for: connectionId, skipSync: true) } + @MainActor @discardableResult private func removeFavorites(for connectionId: UUID, skipSync: Bool) -> [FavoriteEntry] { var removed: [FavoriteEntry] = [] @@ -150,7 +161,9 @@ final class FavoriteTablesStorage: @unchecked Sendable { lock.unlock() guard !removed.isEmpty else { return [] } - if !skipSync { + if skipSync { + syncTracker.discardDirty(.tableFavorite, ids: removed.map(Self.syncId(for:))) + } else { for entry in removed { syncTracker.markDeleted(.tableFavorite, id: Self.syncId(for: entry)) } @@ -188,6 +201,7 @@ final class FavoriteTablesStorage: @unchecked Sendable { return action } + @MainActor private func notify(after action: TrackedAction, skipSync: Bool = false) { switch action { case .noChange: @@ -198,7 +212,9 @@ final class FavoriteTablesStorage: @unchecked Sendable { } NotificationCenter.default.post(name: .favoriteTablesDidChange, object: nil) case .removed(let entry): - if !skipSync { + if skipSync { + syncTracker.discardDirty(.tableFavorite, ids: [Self.syncId(for: entry)]) + } else { syncTracker.markDeleted(.tableFavorite, id: Self.syncId(for: entry)) } NotificationCenter.default.post(name: .favoriteTablesDidChange, object: nil) diff --git a/TablePro/Core/Storage/GroupStorage.swift b/TablePro/Core/Storage/GroupStorage.swift index 90d5642a41..c26f330227 100644 --- a/TablePro/Core/Storage/GroupStorage.swift +++ b/TablePro/Core/Storage/GroupStorage.swift @@ -54,7 +54,7 @@ internal final class GroupStorage { /// Set when the stored payload could not be understood at all. Every mutation rewrites the /// whole array, so continuing over an unreadable store would replace the user's groups with /// whatever the caller happened to be holding. - private var storeIsUnreadable = false + internal private(set) var storeIsUnreadable = false internal init( userDefaults: UserDefaults = AppStorageEnvironment.shared.defaults, @@ -108,6 +108,7 @@ internal final class GroupStorage { /// that failed leaves the store holding the previous set. @discardableResult internal func saveGroups(_ groups: [ConnectionGroup]) -> Bool { + let previous = loadGroups() guard !storeIsUnreadable else { Self.logger.error("Refusing to overwrite an unreadable group store") return false @@ -117,7 +118,7 @@ internal final class GroupStorage { let data = try encoder.encode(groups) defaults.set(data, forKey: groupsKey) cachedGroups = nil - syncTracker.markDirty(.group, ids: groups.map { $0.id.uuidString }) + syncTracker.markDirty(.group, ids: SyncRecordChanges.changedIds(from: previous, to: groups)) return true } catch { Self.logger.error("Failed to save groups: \(error)") @@ -239,10 +240,8 @@ internal final class GroupStorage { /// /// The pull that calls this raises one change notification for the batch, so this raises none. /// - /// A record identical to the one already stored is skipped, because `saveGroups` marks every - /// group dirty and the push uploads every dirty group. Writing an unchanged record therefore - /// re-uploads the whole list, which the other device receives and writes back, and two Macs - /// trade the same records forever. The iOS coordinator has always had this guard. + /// A record identical to the one already stored is skipped. The iOS coordinator has always had + /// this guard. @discardableResult internal func applyRemoteGroup(_ group: ConnectionGroup) -> RemoteApplyOutcome { var groups = loadGroups() diff --git a/TablePro/Core/Storage/SQLFavoriteManager.swift b/TablePro/Core/Storage/SQLFavoriteManager.swift index 21134090df..85d91d4e04 100644 --- a/TablePro/Core/Storage/SQLFavoriteManager.swift +++ b/TablePro/Core/Storage/SQLFavoriteManager.swift @@ -15,6 +15,7 @@ internal final class SQLFavoriteManager: @unchecked Sendable { private let storage: SQLFavoriteStorage private let syncTracker: SyncChangeTracker + private let operations = MainActorSerialQueue() init(storage: SQLFavoriteStorage = SQLFavoriteStorage(), syncTracker: SyncChangeTracker = .shared) { self.storage = storage @@ -24,34 +25,36 @@ internal final class SQLFavoriteManager: @unchecked Sendable { // MARK: - Favorites func addFavorite(_ favorite: SQLFavorite) async -> Bool { - let result = await storage.addFavorite(favorite) - if result { + await operations.run { [self] in + guard await storage.addFavorite(favorite) else { return false } syncTracker.markDirty(.favorite, id: favorite.id.uuidString) postUpdateNotification(connectionId: favorite.connectionId) + return true } - return result } func updateFavorite(_ favorite: SQLFavorite) async -> Bool { - let result = await storage.updateFavorite(favorite) - guard result.succeeded else { return false } - syncTracker.markDirty(.favorite, id: favorite.id.uuidString) - postUpdateNotification(for: result, newConnectionId: favorite.connectionId) - return true + await operations.run { [self] in + let result = await storage.updateFavorite(favorite) + guard result.succeeded else { return false } + syncTracker.markDirty(.favorite, id: favorite.id.uuidString) + postUpdateNotification(for: result, newConnectionId: favorite.connectionId) + return true + } } func deleteFavorite(id: UUID) async -> Bool { - let result = await storage.deleteFavorite(id: id) - if result { + await operations.run { [self] in + guard await storage.deleteFavorite(id: id) else { return false } syncTracker.markDeleted(.favorite, id: id.uuidString) postUpdateNotification(connectionId: nil) + return true } - return result } func deleteFavorites(ids: [UUID]) async { - let result = await storage.deleteFavorites(ids: ids) - if result { + await operations.run { [self] in + guard await storage.deleteFavorites(ids: ids) else { return } for id in ids { syncTracker.markDeleted(.favorite, id: id.uuidString) } @@ -68,16 +71,18 @@ internal final class SQLFavoriteManager: @unchecked Sendable { /// posts a change notification that can start a sync, and a sync that reads a file still /// holding the record re-uploads what was just deleted. func removeFavoritesAndFolders(for connectionId: UUID) async { - let removed = await storage.deleteFavoritesAndFolders(connectionId: connectionId) - guard !removed.isEmpty else { return } - for id in removed.favorites { - syncTracker.markDeleted(.favorite, id: id.uuidString) - } - for id in removed.folders { - syncTracker.markDeleted(.favoriteFolder, id: id.uuidString) + await operations.run { [self] in + let removed = await storage.deleteFavoritesAndFolders(connectionId: connectionId) + guard !removed.isEmpty else { return } + for id in removed.favorites { + syncTracker.markDeleted(.favorite, id: id.uuidString) + } + for id in removed.folders { + syncTracker.markDeleted(.favoriteFolder, id: id.uuidString) + } + markDetachedDirty(removed.detached) + postUpdateNotification(connectionId: nil) } - markDetachedDirty(removed.detached) - postUpdateNotification(connectionId: nil) } /// A row that survived the delete holding a reference the delete had to clear is a local edit @@ -87,6 +92,7 @@ internal final class SQLFavoriteManager: @unchecked Sendable { /// Only on the path that owns the deletion. When another device deleted the connection it runs /// the same cleanup over the same rows and pushes the result itself, and the caller that exists /// for that case deliberately does not mark anything. + @MainActor private func markDetachedDirty(_ detached: DetachedFavoriteRecords) { guard !detached.isEmpty else { return } syncTracker.markDirty(.favorite, ids: detached.favorites.map(\.uuidString)) @@ -97,11 +103,13 @@ internal final class SQLFavoriteManager: @unchecked Sendable { /// deletion straight back at it, which is the reason `FavoriteTablesStorage` splits the same /// way. func removeFavoritesAndFoldersWithoutSync(for connectionId: UUID) async { - let removed = await storage.deleteFavoritesAndFolders(connectionId: connectionId) - guard !removed.isEmpty else { return } - syncTracker.discardDirty(.favorite, ids: removed.favorites.map(\.uuidString)) - syncTracker.discardDirty(.favoriteFolder, ids: removed.folders.map(\.uuidString)) - postUpdateNotification(connectionId: nil) + await operations.run { [self] in + let removed = await storage.deleteFavoritesAndFolders(connectionId: connectionId) + guard !removed.isEmpty else { return } + syncTracker.discardDirty(.favorite, ids: removed.favorites.map(\.uuidString)) + syncTracker.discardDirty(.favoriteFolder, ids: removed.folders.map(\.uuidString)) + postUpdateNotification(connectionId: nil) + } } func hasFavorites(for connectionIds: [UUID]) async -> Bool { @@ -126,6 +134,14 @@ internal final class SQLFavoriteManager: @unchecked Sendable { ) } + func favoritesForSync() async -> [SQLFavorite]? { + await storage.readAllFavorites() + } + + func foldersForSync() async -> [SQLFavoriteFolder]? { + await storage.readAllFolders() + } + // MARK: - Versions func fetchVersions(favoriteId: UUID) async -> [SQLFavoriteVersion] { @@ -137,30 +153,38 @@ internal final class SQLFavoriteManager: @unchecked Sendable { } func restore(_ version: SQLFavoriteVersion) async -> Bool { - let result = await storage.replaceQuery(favoriteId: version.favoriteId, query: version.query, updatedAt: Date()) - guard result.succeeded else { return false } - syncTracker.markDirty(.favorite, id: version.favoriteId.uuidString) - postUpdateNotification(connectionId: result.retainedScope) - return true + await operations.run { [self] in + let result = await storage.replaceQuery( + favoriteId: version.favoriteId, + query: version.query, + updatedAt: Date() + ) + guard result.succeeded else { return false } + syncTracker.markDirty(.favorite, id: version.favoriteId.uuidString) + postUpdateNotification(connectionId: result.retainedScope) + return true + } } // MARK: - Folders func addFolder(_ folder: SQLFavoriteFolder) async -> Bool { - let result = await storage.addFolder(folder) - if result { + await operations.run { [self] in + guard await storage.addFolder(folder) else { return false } syncTracker.markDirty(.favoriteFolder, id: folder.id.uuidString) postUpdateNotification(connectionId: folder.connectionId) + return true } - return result } func updateFolder(_ folder: SQLFavoriteFolder) async -> Bool { - let result = await storage.updateFolder(folder) - guard result.succeeded else { return false } - syncTracker.markDirty(.favoriteFolder, id: folder.id.uuidString) - postUpdateNotification(for: result, newConnectionId: folder.connectionId) - return true + await operations.run { [self] in + let result = await storage.updateFolder(folder) + guard result.succeeded else { return false } + syncTracker.markDirty(.favoriteFolder, id: folder.id.uuidString) + postUpdateNotification(for: result, newConnectionId: folder.connectionId) + return true + } } /// The records the delete moved up a level are marked dirty alongside the folder's tombstone. @@ -169,12 +193,14 @@ internal final class SQLFavoriteManager: @unchecked Sendable { /// to sync. The moved records kept the id of the deleted folder on the wire, so a device /// fetching the account fresh stored a `folderId` matching no folder. func deleteFolder(id: UUID) async -> Bool { - guard let deletion = await storage.deleteFolder(id: id) else { return false } - syncTracker.markDeleted(.favoriteFolder, id: id.uuidString) - syncTracker.markDirty(.favorite, ids: deletion.movedFavorites.map(\.uuidString)) - syncTracker.markDirty(.favoriteFolder, ids: deletion.movedFolders.map(\.uuidString)) - postUpdateNotification(connectionId: nil) - return true + await operations.run { [self] in + guard let deletion = await storage.deleteFolder(id: id) else { return false } + syncTracker.markDeleted(.favoriteFolder, id: id.uuidString) + syncTracker.markDirty(.favorite, ids: deletion.movedFavorites.map(\.uuidString)) + syncTracker.markDirty(.favoriteFolder, ids: deletion.movedFolders.map(\.uuidString)) + postUpdateNotification(connectionId: nil) + return true + } } func fetchFolders(connectionId: UUID? = nil) async -> [SQLFavoriteFolder] { @@ -186,46 +212,78 @@ internal final class SQLFavoriteManager: @unchecked Sendable { } func renameFolder(id: UUID, name: String) async -> Bool { - let result = await storage.renameFolder(id: id, name: name) - guard result.succeeded else { return false } - syncTracker.markDirty(.favoriteFolder, id: id.uuidString) - postUpdateNotification(connectionId: result.retainedScope) - return true + await operations.run { [self] in + let result = await storage.renameFolder(id: id, name: name) + guard result.succeeded else { return false } + syncTracker.markDirty(.favoriteFolder, id: id.uuidString) + postUpdateNotification(connectionId: result.retainedScope) + return true + } } /// The mark runs after the write has committed, per the sync ordering rule: `markDirty` posts a /// change notification that can start a sync, and a sync reading the database before the write /// lands pushes the scope the folder is leaving. func setFolderScope(id: UUID, connectionId: UUID?) async -> Bool { - let result = await storage.setFolderScope(id: id, connectionId: connectionId) - guard result.succeeded else { return false } - syncTracker.markDirty(.favoriteFolder, id: id.uuidString) - postUpdateNotification(for: result, newConnectionId: connectionId) - return true + await operations.run { [self] in + let result = await storage.setFolderScope(id: id, connectionId: connectionId) + guard result.succeeded else { return false } + syncTracker.markDirty(.favoriteFolder, id: id.uuidString) + postUpdateNotification(for: result, newConnectionId: connectionId) + return true + } } func setFavoriteFolder(id: UUID, folderId: UUID?) async -> Bool { - let result = await storage.setFavoriteFolder(id: id, folderId: folderId) - guard result.succeeded else { return false } - syncTracker.markDirty(.favorite, id: id.uuidString) - postUpdateNotification(connectionId: result.retainedScope) - return true + await operations.run { [self] in + let result = await storage.setFavoriteFolder(id: id, folderId: folderId) + guard result.succeeded else { return false } + syncTracker.markDirty(.favorite, id: id.uuidString) + postUpdateNotification(connectionId: result.retainedScope) + return true + } } // MARK: - Remote Apply - func applyRemote(_ batch: RemoteSQLFavoriteBatch) async -> RemoteApplyOutcome { + func applyRemote( + _ batch: RemoteSQLFavoriteBatch, + echoGuard: SyncEchoGuard? = nil + ) async -> RemoteApplyOutcome { guard !batch.isEmpty else { return .skipped } - guard await applyRemoteFavoriteDeletions(batch.deletedFavoriteIds), - await applyRemoteFolders(batch.folders), - await applyRemoteFavorites(batch.favoritesToUpsert), - await applyRemoteFolderDeletions(batch.deletedFolderIds) - else { - return .failed + return await operations.run { [self] in + let admitted = admitting(batch, echoGuard: echoGuard) + guard !admitted.isEmpty else { return .skipped } + guard await applyRemoteFavoriteDeletions(admitted.deletedFavoriteIds), + await applyRemoteFolders(admitted.folders), + await applyRemoteFavorites(admitted.favoritesToUpsert), + await applyRemoteFolderDeletions(admitted.deletedFolderIds) + else { + return .failed + } + return .applied + } + } + + @MainActor + private func admitting(_ batch: RemoteSQLFavoriteBatch, echoGuard: SyncEchoGuard?) -> RemoteSQLFavoriteBatch { + let deletedFavoriteIds = syncTracker.tombstonedIds(for: .favorite) + let deletedFolderIds = syncTracker.tombstonedIds(for: .favoriteFolder) + var admitted = batch + admitted.favorites = batch.favorites.filter { favorite in + let id = favorite.id.uuidString + guard !deletedFavoriteIds.contains(id) else { return false } + return echoGuard?.withholds(.favorite, id: id, tracker: syncTracker) != true + } + admitted.folders = batch.folders.filter { folder in + let id = folder.id.uuidString + guard !deletedFolderIds.contains(id) else { return false } + return echoGuard?.withholds(.favoriteFolder, id: id, tracker: syncTracker) != true } - return .applied + return admitted } + @MainActor private func applyRemoteFavoriteDeletions(_ ids: Set) async -> Bool { guard !ids.isEmpty else { return true } guard await storage.deleteFavorites(ids: Array(ids)) else { return false } @@ -234,6 +292,7 @@ internal final class SQLFavoriteManager: @unchecked Sendable { return true } + @MainActor private func applyRemoteFolders(_ folders: [SQLFavoriteFolder]) async -> Bool { for folder in folders { let write = await storage.upsertFolder(folder) @@ -243,6 +302,7 @@ internal final class SQLFavoriteManager: @unchecked Sendable { return true } + @MainActor private func applyRemoteFavorites(_ favorites: [SQLFavorite]) async -> Bool { guard !favorites.isEmpty else { return true } guard let result = await storage.applyRemoteFavorites(favorites) else { return false } @@ -256,6 +316,7 @@ internal final class SQLFavoriteManager: @unchecked Sendable { return true } + @MainActor private func applyRemoteFolderDeletions(_ ids: Set) async -> Bool { guard !ids.isEmpty else { return true } defer { postUpdateNotification(connectionId: nil) } diff --git a/TablePro/Core/Storage/SQLFavoriteStorage+SyncReads.swift b/TablePro/Core/Storage/SQLFavoriteStorage+SyncReads.swift new file mode 100644 index 0000000000..aed76b8573 --- /dev/null +++ b/TablePro/Core/Storage/SQLFavoriteStorage+SyncReads.swift @@ -0,0 +1,51 @@ +import Foundation +import os +import SQLite3 + +extension SQLFavoriteStorage { + private static let syncReadLogger = Logger(subsystem: "com.TablePro", category: "SQLFavoriteSyncReads") + + func readAllFavorites() -> [SQLFavorite]? { + readAllRows( + "SELECT id, name, query, keyword, folder_id, connection_id, sort_order, created_at, updated_at FROM favorites;", + parse: parseFavorite(from:) + ) + } + + func readAllFolders() -> [SQLFavoriteFolder]? { + readAllRows( + "SELECT id, name, parent_id, connection_id, sort_order, created_at, updated_at FROM folders;", + parse: parseFolder(from:) + ) + } + + private func readAllRows(_ sql: String, parse: (OpaquePointer?) -> Row?) -> [Row]? { + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + let prepareResult = sqlite3_prepare_v2(db, sql, -1, &statement, nil) + guard prepareResult == SQLITE_OK else { + Self.syncReadLogger.error( + "Saved queries could not be read: \(String(cString: sqlite3_errstr(prepareResult)), privacy: .public)" + ) + return nil + } + + var rows: [Row] = [] + while true { + let stepResult = sqlite3_step(statement) + switch stepResult { + case SQLITE_ROW: + if let row = parse(statement) { + rows.append(row) + } + case SQLITE_DONE: + return rows + default: + Self.syncReadLogger.error( + "Saved queries could not be read: \(String(cString: sqlite3_errstr(stepResult)), privacy: .public)" + ) + return nil + } + } + } +} diff --git a/TablePro/Core/Storage/SQLFavoriteStorage.swift b/TablePro/Core/Storage/SQLFavoriteStorage.swift index c600185b3f..227e61238d 100644 --- a/TablePro/Core/Storage/SQLFavoriteStorage.swift +++ b/TablePro/Core/Storage/SQLFavoriteStorage.swift @@ -15,11 +15,9 @@ internal actor SQLFavoriteStorage { } private var dbHandle = DatabaseHandle() - private var isPrepared = false internal var db: OpaquePointer? { - if !isPrepared { - isPrepared = true + if dbHandle.pointer == nil { setupDatabase() } return dbHandle.pointer @@ -65,10 +63,14 @@ internal actor SQLFavoriteStorage { let dbPath = databaseURL.path(percentEncoded: false) - if sqlite3_open(dbPath, &dbHandle.pointer) != SQLITE_OK { - Self.logger.error("Error opening database") + var pointer: OpaquePointer? + let openResult = sqlite3_open(dbPath, &pointer) + guard openResult == SQLITE_OK else { + Self.logger.error("Error opening database: \(String(cString: sqlite3_errstr(openResult)), privacy: .public)") + sqlite3_close_v2(pointer) return } + dbHandle.pointer = pointer execute("PRAGMA journal_mode=WAL;") execute("PRAGMA synchronous=NORMAL;") @@ -1156,7 +1158,7 @@ internal actor SQLFavoriteStorage { // MARK: - Parsing Helpers - private func parseFavorite(from statement: OpaquePointer?) -> SQLFavorite? { + internal func parseFavorite(from statement: OpaquePointer?) -> SQLFavorite? { guard let statement = statement else { return nil } guard let idString = sqlite3_column_text(statement, 0).map({ String(cString: $0) }), diff --git a/TablePro/Core/Storage/SSHProfileStorage.swift b/TablePro/Core/Storage/SSHProfileStorage.swift index 2cc53d23b6..4044b6d4f3 100644 --- a/TablePro/Core/Storage/SSHProfileStorage.swift +++ b/TablePro/Core/Storage/SSHProfileStorage.swift @@ -63,8 +63,9 @@ final class SSHProfileStorage { /// tombstones can abort instead of stranding them against a profile that was never persisted. @discardableResult func saveProfiles(_ profiles: [SSHProfile]) -> Bool { + let previous = loadProfiles() guard saveProfilesWithoutSync(profiles) else { return false } - syncTracker.markDirty(.sshProfile, ids: profiles.map { $0.id.uuidString }) + syncTracker.markDirty(.sshProfile, ids: SyncRecordChanges.changedIds(from: previous, to: profiles)) return true } diff --git a/TablePro/Core/Storage/TagStorage.swift b/TablePro/Core/Storage/TagStorage.swift index 775e78b94b..f9593993aa 100644 --- a/TablePro/Core/Storage/TagStorage.swift +++ b/TablePro/Core/Storage/TagStorage.swift @@ -40,7 +40,7 @@ internal final class TagStorage { /// Set when the stored payload could not be understood at all. Every mutation rewrites the /// whole array, so continuing over an unreadable store would replace a user's own tags with /// the preset list this falls back to for display. - private var storeIsUnreadable = false + internal private(set) var storeIsUnreadable = false internal init( userDefaults: UserDefaults = AppStorageEnvironment.shared.defaults, @@ -97,6 +97,7 @@ internal final class TagStorage { /// that goes on to write related state must check the result. @discardableResult internal func saveTags(_ tags: [ConnectionTag]) -> Bool { + let previous = defaults.data(forKey: tagsKey) == nil ? [] : loadTags() guard !storeIsUnreadable else { Self.logger.error("Refusing to overwrite an unreadable tag store") return false @@ -106,7 +107,7 @@ internal final class TagStorage { let data = try encoder.encode(tags) defaults.set(data, forKey: tagsKey) cachedTags = nil - syncTracker.markDirty(.tag, ids: tags.map { $0.id.uuidString }) + syncTracker.markDirty(.tag, ids: SyncRecordChanges.changedIds(from: previous, to: tags)) return true } catch { Self.logger.error("Failed to save tags: \(error)") @@ -128,9 +129,7 @@ internal final class TagStorage { /// Apply a tag that arrived from another device. /// - /// Written as it arrived, and skipped when it matches what is already stored: `saveTags` marks - /// every tag dirty and the push uploads every dirty tag, so writing an unchanged record - /// re-uploads the whole library to the device it came from, which writes it back. + /// Written as it arrived, and skipped when it matches what is already stored. @discardableResult internal func applyRemoteTag(_ tag: ConnectionTag) -> RemoteApplyOutcome { var tags = loadTags() diff --git a/TablePro/Core/Sync/Extensions/SyncCoordinator+PushCollection.swift b/TablePro/Core/Sync/Extensions/SyncCoordinator+PushCollection.swift new file mode 100644 index 0000000000..04f0e2d1d0 --- /dev/null +++ b/TablePro/Core/Sync/Extensions/SyncCoordinator+PushCollection.swift @@ -0,0 +1,175 @@ +import CloudKit +import Foundation +import os +import TableProSyncTransport + +struct SyncPushBatch { + var records: [CKRecord] = [] + var deletions: [CKRecord.ID] = [] + + var uniqueDeletions: [CKRecord.ID] { + Array(Set(deletions)) + } +} + +extension SyncCoordinator { + func collectPushBatch( + snapshot: SyncEditSnapshot, + settings: SyncSettings, + zoneID: CKRecordZone.ID + ) async -> SyncPushBatch { + var batch = SyncPushBatch() + + if settings.syncConnections { + let storage = services.connectionStorage + await collectRecords(of: .connection, snapshot: snapshot, into: &batch, zoneID: zoneID) { + let connections = storage.loadConnections() + return storage.lastLoadFailed ? nil : connections + } record: { (connection: DatabaseConnection) -> CKRecord? in + guard !connection.localOnly else { return nil } + let recordID = SyncRecordMapper.recordID(type: .connection, id: connection.id.uuidString, in: zoneID) + return SyncRecordMapper.toCKRecord(connection, in: zoneID, base: recordCache.record(for: recordID)) + } + } + + if settings.syncGroupsAndTags { + let groupStorage = services.groupStorage + await collectRecords(of: .group, snapshot: snapshot, into: &batch, zoneID: zoneID) { + let groups = groupStorage.loadGroups() + return groupStorage.storeIsUnreadable ? nil : groups + } record: { (group: ConnectionGroup) in SyncRecordMapper.toCKRecord(group, in: zoneID) } + + let tagStorage = services.tagStorage + await collectRecords(of: .tag, snapshot: snapshot, into: &batch, zoneID: zoneID) { + let tags = tagStorage.loadTags() + return tagStorage.storeIsUnreadable ? nil : tags + } record: { (tag: ConnectionTag) in SyncRecordMapper.toCKRecord(tag, in: zoneID) } + } + + if settings.syncSSHProfiles { + let storage = services.sshProfileStorage + await collectRecords(of: .sshProfile, snapshot: snapshot, into: &batch, zoneID: zoneID) { + let profiles = storage.loadProfiles() + return storage.lastLoadFailed ? nil : profiles + } record: { (profile: SSHProfile) in SyncRecordMapper.toCKRecord(profile, in: zoneID) } + } + + if settings.syncCredentialProfiles { + let storage = services.credentialProfileStorage + await collectRecords(of: .credentialProfile, snapshot: snapshot, into: &batch, zoneID: zoneID) { + let profiles = storage.loadProfiles() + return storage.lastLoadFailed ? nil : profiles + } record: { (profile: CredentialProfile) in SyncRecordMapper.toCKRecord(profile, in: zoneID) } + } + + if settings.syncSettings { + collectSettings(snapshot: snapshot, into: &batch, zoneID: zoneID) + } + + if settings.syncTableFavorites { + collectTableFavorites(snapshot: snapshot, into: &batch, zoneID: zoneID) + } + + if settings.syncDatabaseFavorites { + collectDatabaseFavorites(snapshot: snapshot, into: &batch, zoneID: zoneID) + } + + if settings.syncSQLFavorites { + await collectSQLFavorites(snapshot: snapshot, into: &batch, zoneID: zoneID) + } + + return batch + } + + private func collectRecords( + of type: SyncRecordType, + snapshot: SyncEditSnapshot, + into batch: inout SyncPushBatch, + zoneID: CKRecordZone.ID, + loaded: () async -> [Record]?, + record: (Record) -> CKRecord? + ) async where Record.ID == UUID { + appendTombstones(of: type, to: &batch, zoneID: zoneID) + let dirtyIds = snapshot.dirtyIds(for: type) + guard !dirtyIds.isEmpty, let records = await loaded() else { return } + + var pushable: Set = [] + for item in records where dirtyIds.contains(item.id.uuidString) { + let id = item.id.uuidString + guard !pushable.contains(id), let built = record(item) else { continue } + pushable.insert(id) + batch.records.append(built) + } + discardUnpushable(type, dirtyIds: dirtyIds, pushable: pushable) + } + + private func discardUnpushable(_ type: SyncRecordType, dirtyIds: Set, pushable: Set) { + let unpushable = dirtyIds.subtracting(pushable) + guard !unpushable.isEmpty else { return } + Self.logger.info( + "Dropping \(unpushable.count, privacy: .public) \(type.rawValue, privacy: .public) marks this Mac cannot push" + ) + changeTracker.discardDirty(type, ids: Array(unpushable)) + } + + private func appendTombstones(of type: SyncRecordType, to batch: inout SyncPushBatch, zoneID: CKRecordZone.ID) { + for tombstone in metadataStorage.tombstones(for: type) { + batch.deletions.append(SyncRecordMapper.recordID(type: type, id: tombstone.id, in: zoneID)) + } + } + + private func collectSettings(snapshot: SyncEditSnapshot, into batch: inout SyncPushBatch, zoneID: CKRecordZone.ID) { + for category in snapshot.dirtyIds(for: .settings) { + guard let data = settingsData(for: category) else { continue } + batch.records.append(SyncRecordMapper.toCKRecord(category: category, settingsData: data, in: zoneID)) + } + } + + private func collectTableFavorites( + snapshot: SyncEditSnapshot, + into batch: inout SyncPushBatch, + zoneID: CKRecordZone.ID + ) { + appendTombstones(of: .tableFavorite, to: &batch, zoneID: zoneID) + let dirtyIds = snapshot.dirtyIds(for: .tableFavorite) + guard !dirtyIds.isEmpty else { return } + for entry in services.favoriteTablesStorage.loadFavorites() + where dirtyIds.contains(FavoriteTablesStorage.syncId(for: entry)) { + batch.records.append(SyncRecordMapper.toCKRecord(favoriteEntry: entry, in: zoneID)) + } + } + + /// A connection the user marked local only never reaches iCloud, and neither do the database + /// names hanging off it. Tombstones are not filtered: a deletion only ever removes something, + /// and a connection can be marked local only after its favorites were already pushed. + private func collectDatabaseFavorites( + snapshot: SyncEditSnapshot, + into batch: inout SyncPushBatch, + zoneID: CKRecordZone.ID + ) { + appendTombstones(of: .favoriteDatabase, to: &batch, zoneID: zoneID) + let dirtyIds = snapshot.dirtyIds(for: .favoriteDatabase) + guard !dirtyIds.isEmpty else { return } + let localOnlyIds = Set(services.connectionStorage.loadConnections().filter(\.localOnly).map(\.id)) + for entry in services.favoriteDatabasesStorage.loadFavorites() + where dirtyIds.contains(FavoriteDatabasesStorage.syncId(for: entry)) + && !localOnlyIds.contains(entry.connectionId) { + batch.records.append(SyncRecordMapper.toCKRecord(favoriteDatabase: entry, in: zoneID)) + } + } + + private func collectSQLFavorites( + snapshot: SyncEditSnapshot, + into batch: inout SyncPushBatch, + zoneID: CKRecordZone.ID + ) async { + let manager = services.sqlFavoriteManager + await collectRecords(of: .favorite, snapshot: snapshot, into: &batch, zoneID: zoneID) { + await manager.favoritesForSync() + } record: { (favorite: SQLFavorite) in SyncRecordMapper.toCKRecord(sqlFavorite: favorite, in: zoneID) } + + await collectRecords(of: .favoriteFolder, snapshot: snapshot, into: &batch, zoneID: zoneID) { + await manager.foldersForSync() + } record: { (folder: SQLFavoriteFolder) in SyncRecordMapper.toCKRecord(sqlFavoriteFolder: folder, in: zoneID) } + } +} diff --git a/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift b/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift new file mode 100644 index 0000000000..d8b235f35f --- /dev/null +++ b/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift @@ -0,0 +1,163 @@ +import CloudKit +import Foundation +import os +import TableProSyncTransport + +struct SyncPendingDeletions: Equatable { + var connections: Set = [] + var groups: Set = [] + var tags: Set = [] + var sshProfiles: Set = [] + var credentialProfiles: Set = [] + var tableFavorites: Set = [] + var sqlFavorites: Set = [] + var sqlFolders: Set = [] + + static func parse(_ recordIDs: [CKRecord.ID], settings: SyncSettings) -> SyncPendingDeletions { + var pending = SyncPendingDeletions() + for recordID in recordIDs { + guard let parsed = SyncRecordType.parse(recordName: recordID.recordName) else { continue } + pending.insert(parsed.id, of: parsed.type, settings: settings) + } + return pending + } + + private mutating func insert(_ id: String, of type: SyncRecordType, settings: SyncSettings) { + let uuid = UUID(uuidString: id) + switch type { + case .connection: + if let uuid { connections.insert(uuid) } + case .group: + if let uuid { groups.insert(uuid) } + case .tag: + if let uuid { tags.insert(uuid) } + case .sshProfile: + if let uuid { sshProfiles.insert(uuid) } + case .credentialProfile: + if settings.syncCredentialProfiles, let uuid { credentialProfiles.insert(uuid) } + case .tableFavorite: + tableFavorites.insert(id) + case .favoriteDatabase, .settings: + return + case .favorite: + if settings.syncSQLFavorites, let uuid { sqlFavorites.insert(uuid) } + case .favoriteFolder: + if settings.syncSQLFavorites, let uuid { sqlFolders.insert(uuid) } + } + } +} + +struct SyncRemoteDeletionEffects { + var connectionsChanged = false + var groupsOrTagsChanged = false + var persistenceFailed = false +} + +extension SyncCoordinator { + func applyRemoteDeletions(_ pending: SyncPendingDeletions) -> SyncRemoteDeletionEffects { + var effects = SyncRemoteDeletionEffects() + effects.connectionsChanged = !pending.connections.isEmpty + effects.groupsOrTagsChanged = !pending.groups.isEmpty || !pending.tags.isEmpty + + let persisted = [ + applyRemoteConnectionDeletions(pending.connections), + applyRemoteGroupDeletions(pending.groups), + applyRemoteTagDeletions(pending.tags), + applyRemoteSSHProfileDeletions(pending.sshProfiles), + applyRemoteCredentialProfileDeletions(pending.credentialProfiles) + ] + effects.persistenceFailed = persisted.contains(false) + + for id in pending.tableFavorites { + services.favoriteTablesStorage.removeFavoriteWithoutSync(id: id) + } + return effects + } + + private func applyRemoteConnectionDeletions(_ ids: Set) -> Bool { + guard !ids.isEmpty else { return true } + var connections = services.connectionStorage.loadConnections() + connections.removeAll { ids.contains($0.id) } + guard services.connectionStorage.saveConnections(connections) else { + Self.logger.error("Failed to apply remote connection deletions: persistence error") + return false + } + changeTracker.discardDirty(.connection, ids: ids.map(\.uuidString)) + ConnectionLocalState.purge( + connectionIds: ids, + origin: .remote, + sqlFavorites: services.sqlFavoriteManager, + queryHistory: services.queryHistoryManager + ) + return true + } + + private func applyRemoteGroupDeletions(_ ids: Set) -> Bool { + guard !ids.isEmpty else { return true } + var groups = services.groupStorage.loadGroups() + groups.removeAll { ids.contains($0.id) } + guard services.groupStorage.saveGroups(groups) else { return false } + changeTracker.discardDirty(.group, ids: ids.map(\.uuidString)) + return true + } + + private func applyRemoteTagDeletions(_ ids: Set) -> Bool { + guard !ids.isEmpty else { return true } + var tags = services.tagStorage.loadTags() + tags.removeAll { ids.contains($0.id) } + guard services.tagStorage.saveTags(tags) else { return false } + changeTracker.discardDirty(.tag, ids: ids.map(\.uuidString)) + return true + } + + /// Unlinking reads a profile's secrets, so it runs while they still exist. Dropping the record + /// alone left every connection using it addressing a profile id that no longer resolved, and + /// left its keychain items with nothing able to name them. + /// + /// Returns false when a conversion could not be persisted, which withholds the pull token: the + /// alternative is acknowledging a delete whose connections still point at the profile, with the + /// conversion lost and no tombstone left to replay it. + private func applyRemoteSSHProfileDeletions(_ profileIds: Set) -> Bool { + guard !profileIds.isEmpty else { return true } + + var profiles = services.sshProfileStorage.loadProfiles() + let deleted = profiles.filter { profileIds.contains($0.id) } + profiles.removeAll { profileIds.contains($0.id) } + + for profile in deleted where !services.sshProfileStorage.unlinkConnections(fromProfile: profile) { + Self.logger.error( + "Kept SSH profile \(profile.id.uuidString, privacy: .public): its connections could not be converted" + ) + return false + } + guard services.sshProfileStorage.saveProfilesWithoutSync(profiles) else { return false } + changeTracker.discardDirty(.sshProfile, ids: profileIds.map(\.uuidString)) + for profile in deleted { + services.sshProfileStorage.deleteSecrets(for: profile.id) + } + return true + } + + /// A profile deleted on another Mac hands its credentials to the connections using it here + /// too, so they keep connecting rather than losing the password the link took away. + private func applyRemoteCredentialProfileDeletions(_ profileIds: Set) -> Bool { + guard !profileIds.isEmpty else { return true } + + var profiles = services.credentialProfileStorage.loadProfiles() + let deleted = profiles.filter { profileIds.contains($0.id) } + profiles.removeAll { profileIds.contains($0.id) } + + for profile in deleted where !services.credentialProfileStorage.unlinkConnections(from: profile) { + Self.logger.error( + "Kept credential profile \(profile.id.uuidString, privacy: .public): its connections could not be converted" + ) + return false + } + guard services.credentialProfileStorage.saveProfilesWithoutSync(profiles) else { return false } + changeTracker.discardDirty(.credentialProfile, ids: profileIds.map(\.uuidString)) + for profile in deleted { + services.credentialProfileStorage.deleteSecrets(for: profile) + } + return true + } +} diff --git a/TablePro/Core/Sync/Extensions/SyncCoordinator+Settings.swift b/TablePro/Core/Sync/Extensions/SyncCoordinator+Settings.swift new file mode 100644 index 0000000000..fec376d2f0 --- /dev/null +++ b/TablePro/Core/Sync/Extensions/SyncCoordinator+Settings.swift @@ -0,0 +1,64 @@ +import Foundation +import os + +extension SyncCoordinator { + func settingsData(for category: String) -> Data? { + let storage = services.appSettingsStorage + let encoder = JSONEncoder() + + do { + switch category { + case AppSettingsCategory.general: return try encoder.encode(storage.loadGeneral()) + case AppSettingsCategory.appearance: return try encoder.encode(storage.loadAppearance()) + case AppSettingsCategory.editor: return try encoder.encode(storage.loadEditor()) + case AppSettingsCategory.dataGrid: return try encoder.encode(storage.loadDataGrid()) + case AppSettingsCategory.history: return try encoder.encode(storage.loadHistory()) + case AppSettingsCategory.tabs: return try encoder.encode(storage.loadTabs()) + case AppSettingsCategory.keyboard: return try encoder.encode(storage.loadKeyboard()) + case AppSettingsCategory.ai: return try encoder.encode(storage.loadAI()) + case AppSettingsCategory.notifications: return try encoder.encode(storage.loadNotifications()) + case CustomSlashCommandStorage.syncCategory: + return try encoder.encode(CustomSlashCommandStorage.shared.commands) + case let category where category.hasPrefix(FileColumnLayoutPersister.syncCategoryPrefix): + return columnLayouts().rawData( + forStorageKey: String(category.dropFirst(FileColumnLayoutPersister.syncCategoryPrefix.count)) + ) + default: return nil + } + } catch { + Self.logger.error("Failed to encode settings category '\(category)': \(error.localizedDescription)") + return nil + } + } + + func applySettingsData(_ data: Data, for category: String) throws { + let manager = services.appSettings + let decoder = JSONDecoder() + + do { + switch category { + case AppSettingsCategory.general: manager.general = try decoder.decode(GeneralSettings.self, from: data) + case AppSettingsCategory.appearance: + manager.appearance = try decoder.decode(AppearanceSettings.self, from: data) + case AppSettingsCategory.editor: manager.editor = try decoder.decode(EditorSettings.self, from: data) + case AppSettingsCategory.dataGrid: manager.dataGrid = try decoder.decode(DataGridSettings.self, from: data) + case AppSettingsCategory.history: manager.history = try decoder.decode(HistorySettings.self, from: data) + case AppSettingsCategory.tabs: manager.tabs = try decoder.decode(TabSettings.self, from: data) + case AppSettingsCategory.keyboard: manager.keyboard = try decoder.decode(KeyboardSettings.self, from: data) + case AppSettingsCategory.ai: manager.ai = try decoder.decode(AISettings.self, from: data) + case AppSettingsCategory.notifications: + manager.notifications = try decoder.decode(NotificationSettings.self, from: data) + case CustomSlashCommandStorage.syncCategory: + CustomSlashCommandStorage.shared.applyRemote(try decoder.decode([CustomSlashCommand].self, from: data)) + case let category where category.hasPrefix(FileColumnLayoutPersister.syncCategoryPrefix): + columnLayouts().applyRemote( + storageKey: String(category.dropFirst(FileColumnLayoutPersister.syncCategoryPrefix.count)), + data: data + ) + default: return + } + } catch { + throw SyncDecodeError.decodeFailure(field: category, underlying: error) + } + } +} diff --git a/TablePro/Core/Sync/SyncChangeTracker.swift b/TablePro/Core/Sync/SyncChangeTracker.swift index 09c7643ee4..5eb1ee9bb1 100644 --- a/TablePro/Core/Sync/SyncChangeTracker.swift +++ b/TablePro/Core/Sync/SyncChangeTracker.swift @@ -16,6 +16,7 @@ final class SyncChangeTracker: Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SyncChangeTracker") private let metadataStorage: SyncMetadataStorage + private let editGenerations = OSAllocatedUnfairLock(initialState: SyncEditGenerations()) /// When true, changes are not tracked (used during remote apply to avoid sync loops) private let suppressionLock = OSAllocatedUnfairLock(initialState: false) @@ -31,9 +32,11 @@ final class SyncChangeTracker: Sendable { // MARK: - Mark Dirty + @MainActor func markDirty(_ type: SyncRecordType, id: String) { guard !isSuppressed, type.syncScope == .synced else { return } metadataStorage.markDirty(id, type: type) + recordEdits(type, ids: [id]) Self.logger.info("Marked dirty: \(type.rawValue)/\(id)") postChangeNotification() } @@ -44,19 +47,23 @@ final class SyncChangeTracker: Sendable { /// in-flight sync and awaits it before scheduling the next, so a few hundred of them in a row /// build a chain of tasks each waiting on its predecessor. Always prefer this when the caller /// already holds the whole set. + @MainActor func markDirty(_ type: SyncRecordType, ids: [String]) { guard !isSuppressed, !ids.isEmpty, type.syncScope == .synced else { return } metadataStorage.markDirty(ids, type: type) + recordEdits(type, ids: ids) Self.logger.trace("Marked dirty: \(type.rawValue) x\(ids.count)") postChangeNotification() } // MARK: - Mark Deleted + @MainActor func markDeleted(_ type: SyncRecordType, id: String) { guard !isSuppressed else { return } metadataStorage.removeDirty(id, type: type) metadataStorage.addTombstone(id, type: type) + recordEdits(type, ids: [id]) Self.logger.trace("Marked deleted: \(type.rawValue)/\(id)") postChangeNotification() } @@ -66,15 +73,18 @@ final class SyncChangeTracker: Sendable { /// For records another device already deleted: a tombstone would send its own deletion back at /// it, but leaving the dirty ids behind is not free either. The next push looks for records /// that are gone, skips them, and never drains the entries. + @MainActor func discardDirty(_ type: SyncRecordType, ids: [String]) { guard !ids.isEmpty else { return } metadataStorage.removeDirty(ids, type: type) } + @MainActor func markDeleted(_ type: SyncRecordType, ids: [String]) { guard !isSuppressed, !ids.isEmpty else { return } metadataStorage.removeDirty(ids, type: type) metadataStorage.addTombstones(ids, type: type) + recordEdits(type, ids: ids) Self.logger.trace("Marked deleted: \(type.rawValue) x\(ids.count)") postChangeNotification() } @@ -85,18 +95,51 @@ final class SyncChangeTracker: Sendable { metadataStorage.dirtyIds(for: type) } + func tombstonedIds(for type: SyncRecordType) -> Set { + Set(metadataStorage.tombstones(for: type).map(\.id)) + } + + func editSnapshot() -> SyncEditSnapshot { + let dirty = Set(SyncRecordType.allCases.flatMap { type in + metadataStorage.dirtyIds(for: type).map { SyncRecordIdentity(type: type, id: $0) } + }) + return editGenerations.withLock { state in + SyncEditSnapshot( + dirty: dirty, + generations: Dictionary(uniqueKeysWithValues: dirty.map { ($0, state.generation(of: $0)) }) + ) + } + } + + func hasEdit(_ identity: SyncRecordIdentity, since snapshot: SyncEditSnapshot) -> Bool { + guard let recorded = snapshot.generations[identity] else { return true } + return editGenerations.withLock { $0.generation(of: identity) } != recorded + } + // MARK: - Clear func clearDirty(_ type: SyncRecordType, id: String) { metadataStorage.removeDirty(id, type: type) } + @discardableResult + func clearDirty(_ identity: SyncRecordIdentity, unlessEditedSince snapshot: SyncEditSnapshot) -> Bool { + guard !hasEdit(identity, since: snapshot) else { return false } + clearDirty(identity.type, id: identity.id) + return true + } + func clearAllDirty(_ type: SyncRecordType) { metadataStorage.clearDirty(type: type) } // MARK: - Private + private func recordEdits(_ type: SyncRecordType, ids: [String]) { + let identities = ids.map { SyncRecordIdentity(type: type, id: $0) } + editGenerations.withLock { $0.recordEdits(of: identities) } + } + private func postChangeNotification() { Task { @MainActor in AppEvents.shared.syncChangeTracked.send(()) diff --git a/TablePro/Core/Sync/SyncCoordinator.swift b/TablePro/Core/Sync/SyncCoordinator.swift index a519a679de..a468c71f45 100644 --- a/TablePro/Core/Sync/SyncCoordinator.swift +++ b/TablePro/Core/Sync/SyncCoordinator.swift @@ -15,32 +15,43 @@ import TableProSyncTransport @MainActor final class SyncCoordinator: ObservableObject { static let shared = SyncCoordinator() - nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "SyncCoordinator") + nonisolated static let logger = Logger(subsystem: "com.TablePro", category: "SyncCoordinator") @Published private(set) var syncStatus: SyncStatus = .disabled(.userDisabled) @Published private(set) var lastSyncDate: Date? @Published private(set) var iCloudAccountAvailable: Bool = false - private let services: AppServices - private let engine = CloudKitSyncEngine() - private let changeTracker: SyncChangeTracker - private let metadataStorage: SyncMetadataStorage - private let recordCache: SyncRecordCache + let services: AppServices + private let transport: any SyncTransport + let changeTracker: SyncChangeTracker + let metadataStorage: SyncMetadataStorage + let recordCache: SyncRecordCache + let columnLayouts: () -> FileColumnLayoutPersister private let accountObserver = OSAllocatedUnfairLock<(any NSObjectProtocol)?>(uncheckedState: nil) private var changeCancellable: AnyCancellable? private var licenseCancellable: AnyCancellable? private var syncTask: Task? private var hasStarted = false + private var isRunningCycle = false + private var hasDeferredNotificationPull = false + private var notificationPull: Task? /// Bumped every time something other than a sync run decides the status, so a run that has been /// suspended across the network can tell whether its outcome is still the current answer. private var statusGeneration = 0 - init(services: AppServices = .live, recordCache: SyncRecordCache = SyncCoordinator.makeRecordCache()) { + init( + services: AppServices = .live, + recordCache: SyncRecordCache = SyncCoordinator.makeRecordCache(), + transport: any SyncTransport = CloudKitSyncEngine(), + columnLayouts: @escaping @autoclosure () -> FileColumnLayoutPersister = .shared + ) { self.services = services + self.transport = transport self.changeTracker = services.syncTracker self.metadataStorage = services.syncMetadataStorage self.recordCache = recordCache + self.columnLayouts = columnLayouts lastSyncDate = metadataStorage.lastSyncDate } @@ -108,40 +119,50 @@ final class SyncCoordinator: ObservableObject { let generation = statusGeneration syncStatus = .syncing - do { - try await engine.ensureZoneExists() - - var pushError: Error? - do { - try await performPush() - } catch { - pushError = error - Self.logger.error("Push failed: \(error.localizedDescription)") - } - - let pullError = await performPull() + if let syncError = await runSyncCycle() { + settle(.error(syncError), from: generation) + return + } - if let pushError { - settle(.error(SyncError.from(pushError)), from: generation) - return - } + lastSyncDate = Date() + metadataStorage.lastSyncDate = lastSyncDate + settle(.idle, from: generation) + metadataStorage.pruneTombstones(olderThan: 30) - if let pullError { - settle(.error(pullError), from: generation) - return - } + Self.logger.info("Sync completed successfully") + } - lastSyncDate = Date() - metadataStorage.lastSyncDate = lastSyncDate - settle(.idle, from: generation) - metadataStorage.pruneTombstones(olderThan: 30) + internal func runSyncCycle() async -> SyncError? { + isRunningCycle = true + await notificationPull?.value + let failure = await pushThenPull() + isRunningCycle = false + if hasDeferredNotificationPull { + hasDeferredNotificationPull = false + await pullForRemoteNotification() + } + return failure + } - Self.logger.info("Sync completed successfully") + private func pushThenPull() async -> SyncError? { + do { + try await transport.ensureZoneExists() } catch { - let syncError = SyncError.from(error) - settle(.error(syncError), from: generation) Self.logger.error("Sync failed: \(error.localizedDescription)") + return SyncError.from(error) + } + + let push = await performPush() + let pullError = await performPull(echoGuard: push.echoGuard) + while hasDeferredNotificationPull { + hasDeferredNotificationPull = false + await performPull(echoGuard: push.echoGuard) + } + + if let pushError = push.error { + return SyncError.from(pushError) } + return pullError } /// Publishes the outcome of a sync run, unless something decided the status while it was in @@ -164,8 +185,25 @@ final class SyncCoordinator: ObservableObject { guard syncStatus.isEnabled else { return } Task { + await pullForRemoteNotification() + } + } + + internal func pullForRemoteNotification() async { + guard !isRunningCycle else { + hasDeferredNotificationPull = true + return + } + let previous = notificationPull + let pull = Task { + await previous?.value await performPull() } + notificationPull = pull + await pull.value + if notificationPull == pull { + notificationPull = nil + } } /// Called when user enables sync in settings @@ -239,7 +277,7 @@ final class SyncCoordinator: ObservableObject { ) let settingsCategories = AppSettingsCategory.synced + [CustomSlashCommandStorage.syncCategory] - let columnLayoutCategories = FileColumnLayoutPersister.shared.customizedStorageKeys() + let columnLayoutCategories = columnLayouts().customizedStorageKeys() .map { FileColumnLayoutPersister.syncCategory(for: $0) } changeTracker.markDirty(.settings, ids: settingsCategories + columnLayoutCategories) @@ -337,90 +375,37 @@ final class SyncCoordinator: ObservableObject { // MARK: - Push - private func performPush() async throws { - let settings = services.appSettingsStorage.loadSync() - var recordsToSave: [CKRecord] = [] - var recordIDsToDelete: [CKRecord.ID] = [] - let zoneID = await engine.currentZoneID - - if settings.syncConnections { - let dirtyConnectionIds = changeTracker.dirtyRecords(for: .connection) - if !dirtyConnectionIds.isEmpty { - let connections = services.connectionStorage.loadConnections() - for id in dirtyConnectionIds { - if let connection = connections.first(where: { $0.id.uuidString == id }), - !connection.localOnly { - let recordID = SyncRecordMapper.recordID(type: .connection, id: id, in: zoneID) - recordsToSave.append( - SyncRecordMapper.toCKRecord( - connection, - in: zoneID, - base: recordCache.record(for: recordID) - ) - ) - } - } - } - - let connectionTombstones = metadataStorage.tombstones(for: .connection) - for tombstone in connectionTombstones { - recordIDsToDelete.append( - SyncRecordMapper.recordID(type: .connection, id: tombstone.id, in: zoneID) - ) - } - } - - if settings.syncGroupsAndTags { - collectDirtyGroups(into: &recordsToSave, deletions: &recordIDsToDelete, zoneID: zoneID) - collectDirtyTags(into: &recordsToSave, deletions: &recordIDsToDelete, zoneID: zoneID) - } - - if settings.syncSSHProfiles { - collectDirtySSHProfiles(into: &recordsToSave, deletions: &recordIDsToDelete, zoneID: zoneID) - } - - if settings.syncCredentialProfiles { - collectDirtyCredentialProfiles(into: &recordsToSave, deletions: &recordIDsToDelete, zoneID: zoneID) - } - - if settings.syncSettings { - let dirtySettingsIds = changeTracker.dirtyRecords(for: .settings) - for category in dirtySettingsIds { - if let data = settingsData(for: category) { - recordsToSave.append( - SyncRecordMapper.toCKRecord(category: category, settingsData: data, in: zoneID) - ) - } - } - } + private struct PushReport { + var echoGuard: SyncEchoGuard? + var error: Error? + } - if settings.syncTableFavorites { - collectDirtyTableFavorites(into: &recordsToSave, deletions: &recordIDsToDelete, zoneID: zoneID) - } + private func performPush() async -> PushReport { + let snapshot = changeTracker.editSnapshot() + let settings = services.appSettingsStorage.loadSync() + let zoneID = await transport.currentZoneID + let batch = await collectPushBatch(snapshot: snapshot, settings: settings, zoneID: zoneID) + let deletions = batch.uniqueDeletions - if settings.syncDatabaseFavorites { - collectDirtyDatabaseFavorites(into: &recordsToSave, deletions: &recordIDsToDelete, zoneID: zoneID) - } + guard !batch.records.isEmpty || !deletions.isEmpty else { return PushReport() } - if settings.syncSQLFavorites { - await collectDirtySQLFavorites(into: &recordsToSave, deletions: &recordIDsToDelete, zoneID: zoneID) + let identities = SyncRecordMapper.identities(for: pushedLocalIds(snapshot), in: zoneID) + let outcome: PushOutcome + var interruption: Error? + do { + outcome = try await transport.push(records: batch.records, deletions: deletions) + } catch let interrupted as SyncPushInterruption { + outcome = interrupted.completed + interruption = interrupted.cause + } catch { + Self.logger.error("Push failed: \(error.localizedDescription)") + return PushReport(error: error) } - // Deduplicate deletion IDs to prevent CloudKit "can't delete same record twice" error - let uniqueDeletions = Array(Set(recordIDsToDelete)) - - guard !recordsToSave.isEmpty || !uniqueDeletions.isEmpty else { return } - - let identities = SyncRecordMapper.identities(for: pushedLocalIds(), in: zoneID) - let outcome = try await engine.push(records: recordsToSave, deletions: uniqueDeletions) - recordCache.store(Array(outcome.savedRecords.values)) recordCache.remove(Array(outcome.deletedRecordIDs)) - for recordID in outcome.savedRecords.keys { - guard let identity = identities[recordID] else { continue } - changeTracker.clearDirty(identity.type, id: identity.id) - } + let savedRecords = settleSavedRecords(outcome, batch: batch, identities: identities, snapshot: snapshot) for recordID in outcome.deletedRecordIDs { guard let identity = identities[recordID] else { continue } @@ -432,19 +417,38 @@ final class SyncCoordinator: ObservableObject { let rejectedCount = outcome.failures.count Self.logger.info("Push completed: \(savedCount) saved, \(deletedCount) deleted, \(rejectedCount) rejected") - guard outcome.hasFailures else { return } + let echoGuard = SyncEchoGuard(snapshot: snapshot, savedRecords: savedRecords) + if let interruption { + Self.logger.error("Push stopped part way: \(interruption.localizedDescription)") + return PushReport(echoGuard: echoGuard, error: interruption) + } + guard outcome.hasFailures, let firstFailure = outcome.failures.values.first else { + return PushReport(echoGuard: echoGuard) + } + let rejection = SyncError.pushRejected(count: outcome.failures.count, detail: firstFailure.message) + Self.logger.error("Push failed: \(rejection.localizedDescription)") + return PushReport(echoGuard: echoGuard, error: rejection) + } - guard let firstFailure = outcome.failures.values.first else { return } - throw SyncError.pushRejected(count: outcome.failures.count, detail: firstFailure.message) + private func settleSavedRecords( + _ outcome: PushOutcome, + batch: SyncPushBatch, + identities: [CKRecord.ID: SyncRecordIdentity], + snapshot: SyncEditSnapshot + ) -> [CKRecord.ID: SyncRecordIdentity] { + var savedRecords: [CKRecord.ID: SyncRecordIdentity] = [:] + for recordID in outcome.savedRecords.keys { + guard let identity = identities[recordID] else { continue } + savedRecords[recordID] = identity + changeTracker.clearDirty(identity, unlessEditedSince: snapshot) + } + return savedRecords } - /// Every local identifier this push can have sent. `SyncChangeTracker` is not isolated to this - /// actor, so the sets can move under an await; a record whose identifier is missing from the - /// snapshot is left dirty and pushed again rather than cleared against the wrong entry. - private func pushedLocalIds() -> [SyncRecordType: Set] { + private func pushedLocalIds(_ snapshot: SyncEditSnapshot) -> [SyncRecordType: Set] { var localIds: [SyncRecordType: Set] = [:] for type in SyncRecordType.allCases { - let ids = changeTracker.dirtyRecords(for: type) + let ids = snapshot.dirtyIds(for: type) .union(metadataStorage.tombstones(for: type).map(\.id)) guard !ids.isEmpty else { continue } localIds[type] = ids @@ -459,20 +463,20 @@ final class SyncCoordinator: ObservableObject { } @discardableResult - private func performPull() async -> SyncError? { + private func performPull(echoGuard: SyncEchoGuard? = nil) async -> SyncError? { let token = metadataStorage.loadToken() let tokenStatus = token == nil ? "nil (full fetch)" : "present (delta)" Self.logger.info("Pull starting, token: \(tokenStatus)") do { - let result = try await engine.pull(since: token) - return await applyPullResult(result) ? nil : .pullNotSaved + let result = try await transport.pull(since: token) + return await applyPullResult(result, echoGuard: echoGuard) ? nil : .pullNotSaved } catch let error where Self.isTokenExpired(error) { Self.logger.warning("Change token expired, clearing and retrying with full fetch") metadataStorage.saveToken(nil) do { - let result = try await engine.pull(since: nil) - return await applyPullResult(result) ? nil : .pullNotSaved + let result = try await transport.pull(since: nil) + return await applyPullResult(result, echoGuard: echoGuard) ? nil : .pullNotSaved } catch { Self.logger.error("Full fetch after token expiry failed: \(error.localizedDescription)") return nil @@ -484,11 +488,12 @@ final class SyncCoordinator: ObservableObject { } @discardableResult - internal func applyPullResult(_ result: PullResult) async -> Bool { + internal func applyPullResult(_ result: PullResult, echoGuard: SyncEchoGuard? = nil) async -> Bool { let settings = services.appSettingsStorage.loadSync() - let storesPersisted = applyRemoteChanges(result, settings: settings) + let storesPersisted = applyRemoteChanges(result, settings: settings, echoGuard: echoGuard) let favoritesOutcome = await services.sqlFavoriteManager.applyRemote( - remoteSQLFavoriteBatch(from: result, settings: settings) + remoteSQLFavoriteBatch(from: result, settings: settings), + echoGuard: echoGuard ) /// The token and the cache are the record of what this device holds, so neither is @@ -505,7 +510,9 @@ final class SyncCoordinator: ObservableObject { metadataStorage.saveToken(newToken) } - recordCache.store(result.changedRecords) + recordCache.store(result.changedRecords.filter { record in + echoGuard?.withholds(record.recordID, tracker: changeTracker) != true + }) recordCache.remove(result.deletedRecordIDs) Self.logger.info( @@ -519,7 +526,7 @@ final class SyncCoordinator: ObservableObject { // for large payloads. /// Reports whether every record that can say so was persisted. A pull that answers false must /// not commit its token: the batch has to arrive again. - private func applyRemoteChanges(_ result: PullResult, settings: SyncSettings) -> Bool { + private func applyRemoteChanges(_ result: PullResult, settings: SyncSettings, echoGuard: SyncEchoGuard?) -> Bool { services.connectionStorage.invalidateCache() changeTracker.isSuppressed = true @@ -540,6 +547,10 @@ final class SyncCoordinator: ObservableObject { let databaseFavoriteTombstoneIds = Set(metadataStorage.tombstones(for: .favoriteDatabase).map(\.id)) for record in result.changedRecords { + if let echoGuard, echoGuard.withholds(record.recordID, tracker: changeTracker) { + Self.logger.info("Kept a local edit made while its record was being pushed") + continue + } switch record.recordType { case SyncRecordType.connection.rawValue where settings.syncConnections: switch applyRemoteConnection(record, tombstoneIds: connectionTombstoneIds) { @@ -576,51 +587,10 @@ final class SyncCoordinator: ObservableObject { } } - let pendingDeletions = Self.parseDeletions(result.deletedRecordIDs, settings: settings) - let connectionIdsToDelete = pendingDeletions.connections - let groupIdsToDelete = pendingDeletions.groups - let tagIdsToDelete = pendingDeletions.tags - let sshProfileIdsToDelete = pendingDeletions.sshProfiles - let credentialProfileIdsToDelete = pendingDeletions.credentialProfiles - let tableFavoriteIdsToDelete = pendingDeletions.tableFavorites - actualConnectionChanges = actualConnectionChanges || !connectionIdsToDelete.isEmpty - groupsOrTagsChanged = groupsOrTagsChanged - || !groupIdsToDelete.isEmpty - || !tagIdsToDelete.isEmpty - - if !connectionIdsToDelete.isEmpty { - var connections = services.connectionStorage.loadConnections() - connections.removeAll { connectionIdsToDelete.contains($0.id) } - if !services.connectionStorage.saveConnections(connections) { - Self.logger.error("Failed to apply remote connection deletions: persistence error") - } else { - ConnectionLocalState.purge( - connectionIds: connectionIdsToDelete, - origin: .remote, - sqlFavorites: services.sqlFavoriteManager, - queryHistory: services.queryHistoryManager - ) - } - } - if !groupIdsToDelete.isEmpty { - var groups = services.groupStorage.loadGroups() - groups.removeAll { groupIdsToDelete.contains($0.id) } - services.groupStorage.saveGroups(groups) - } - if !tagIdsToDelete.isEmpty { - var tags = services.tagStorage.loadTags() - tags.removeAll { tagIdsToDelete.contains($0.id) } - services.tagStorage.saveTags(tags) - } - if !applyRemoteSSHProfileDeletions(sshProfileIdsToDelete) { - persistenceFailed = true - } - if !applyRemoteCredentialProfileDeletions(credentialProfileIdsToDelete) { - persistenceFailed = true - } - for id in tableFavoriteIdsToDelete { - services.favoriteTablesStorage.removeFavoriteWithoutSync(id: id) - } + let deletions = applyRemoteDeletions(SyncPendingDeletions.parse(result.deletedRecordIDs, settings: settings)) + actualConnectionChanges = actualConnectionChanges || deletions.connectionsChanged + groupsOrTagsChanged = groupsOrTagsChanged || deletions.groupsOrTagsChanged + persistenceFailed = persistenceFailed || deletions.persistenceFailed /// After the batch, never per record: a pull carries no dependency order, so a legal /// hierarchy change spread over two records passes through a state that reads as a cycle @@ -639,9 +609,7 @@ final class SyncCoordinator: ObservableObject { private func remoteSQLFavoriteBatch(from result: PullResult, settings: SyncSettings) -> RemoteSQLFavoriteBatch { guard settings.syncSQLFavorites else { return RemoteSQLFavoriteBatch() } - let favoriteTombstoneIds = Set(metadataStorage.tombstones(for: .favorite).map(\.id)) - let folderTombstoneIds = Set(metadataStorage.tombstones(for: .favoriteFolder).map(\.id)) - let deletions = Self.parseDeletions(result.deletedRecordIDs, settings: settings) + let deletions = SyncPendingDeletions.parse(result.deletedRecordIDs, settings: settings) var batch = RemoteSQLFavoriteBatch( deletedFavoriteIds: deletions.sqlFavorites, deletedFolderIds: deletions.sqlFolders @@ -650,12 +618,10 @@ final class SyncCoordinator: ObservableObject { for record in result.changedRecords { switch record.recordType { case SyncRecordType.favorite.rawValue: - guard let favorite = try? SyncRecordMapper.sqlFavorite(from: record), - !favoriteTombstoneIds.contains(favorite.id.uuidString) else { continue } + guard let favorite = try? SyncRecordMapper.sqlFavorite(from: record) else { continue } batch.favorites.append(favorite) case SyncRecordType.favoriteFolder.rawValue: - guard let folder = try? SyncRecordMapper.sqlFavoriteFolder(from: record), - !folderTombstoneIds.contains(folder.id.uuidString) else { continue } + guard let folder = try? SyncRecordMapper.sqlFavoriteFolder(from: record) else { continue } batch.folders.append(folder) default: continue @@ -746,81 +712,6 @@ final class SyncCoordinator: ObservableObject { return services.tagStorage.applyRemoteTag(remoteTag) } - /// Unlinking reads a profile's secrets, so it runs while they still exist. Dropping the record - /// alone left every connection using it addressing a profile id that no longer resolved, and - /// left its keychain items with nothing able to name them. - /// - /// Returns false when a conversion could not be persisted, which withholds the pull token: the - /// alternative is acknowledging a delete whose connections still point at the profile, with the - /// conversion lost and no tombstone left to replay it. - private func applyRemoteSSHProfileDeletions(_ profileIds: Set) -> Bool { - guard !profileIds.isEmpty else { return true } - - var profiles = services.sshProfileStorage.loadProfiles() - let deleted = profiles.filter { profileIds.contains($0.id) } - profiles.removeAll { profileIds.contains($0.id) } - - for profile in deleted where !services.sshProfileStorage.unlinkConnections(fromProfile: profile) { - Self.logger.error( - "Kept SSH profile \(profile.id.uuidString, privacy: .public): its connections could not be converted" - ) - return false - } - guard services.sshProfileStorage.saveProfilesWithoutSync(profiles) else { return false } - for profile in deleted { - services.sshProfileStorage.deleteSecrets(for: profile.id) - } - return true - } - - /// The tombstones a pull carried, sorted by the record type their name encodes. - /// - /// Pure and lifted out of `applyRemoteChanges`, which is the function every new record type - /// grows and which SwiftLint caps. - private struct PendingDeletions { - var connections: Set = [] - var groups: Set = [] - var tags: Set = [] - var sshProfiles: Set = [] - var credentialProfiles: Set = [] - var tableFavorites: Set = [] - var sqlFavorites: Set = [] - var sqlFolders: Set = [] - } - - private static func parseDeletions( - _ recordIDs: [CKRecord.ID], - settings: SyncSettings - ) -> PendingDeletions { - var pending = PendingDeletions() - for recordID in recordIDs { - let name = recordID.recordName - func identifier(after prefix: String) -> UUID? { - guard name.hasPrefix(prefix) else { return nil } - return UUID(uuidString: String(name.dropFirst(prefix.count))) - } - - if let uuid = identifier(after: "Connection_") { - pending.connections.insert(uuid) - } else if let uuid = identifier(after: "Group_") { - pending.groups.insert(uuid) - } else if let uuid = identifier(after: "Tag_") { - pending.tags.insert(uuid) - } else if let uuid = identifier(after: "SSHProfile_") { - pending.sshProfiles.insert(uuid) - } else if settings.syncCredentialProfiles, let uuid = identifier(after: "CredentialProfile_") { - pending.credentialProfiles.insert(uuid) - } else if name.hasPrefix("FavoriteTable_") { - pending.tableFavorites.insert(String(name.dropFirst("FavoriteTable_".count))) - } else if settings.syncSQLFavorites, let uuid = identifier(after: "FavoriteFolder_") { - pending.sqlFolders.insert(uuid) - } else if settings.syncSQLFavorites, let uuid = identifier(after: "Favorite_") { - pending.sqlFavorites.insert(uuid) - } - } - return pending - } - private static func availableProfileName( basedOn name: String, taken profiles: [CredentialProfile] @@ -873,28 +764,6 @@ final class SyncCoordinator: ObservableObject { return true } - /// A profile deleted on another Mac hands its credentials to the connections using it here - /// too, so they keep connecting rather than losing the password the link took away. - private func applyRemoteCredentialProfileDeletions(_ profileIds: Set) -> Bool { - guard !profileIds.isEmpty else { return true } - - var profiles = services.credentialProfileStorage.loadProfiles() - let deleted = profiles.filter { profileIds.contains($0.id) } - profiles.removeAll { profileIds.contains($0.id) } - - for profile in deleted where !services.credentialProfileStorage.unlinkConnections(from: profile) { - Self.logger.error( - "Kept credential profile \(profile.id.uuidString, privacy: .public): its connections could not be converted" - ) - return false - } - guard services.credentialProfileStorage.saveProfilesWithoutSync(profiles) else { return false } - for profile in deleted { - services.credentialProfileStorage.deleteSecrets(for: profile) - } - return true - } - private func applyRemoteSSHProfile(_ record: CKRecord, tombstoneIds: Set) { let remoteProfile: SSHProfile do { @@ -1024,7 +893,7 @@ final class SyncCoordinator: ObservableObject { private func checkAccountStatus() async { do { - let status = try await engine.accountStatus() + let status = try await transport.accountStatus() iCloudAccountAvailable = (status == .available) if iCloudAccountAvailable { @@ -1039,246 +908,6 @@ final class SyncCoordinator: ObservableObject { } private func currentAccountId() async throws -> String? { - try await engine.currentAccountId() - } - - // MARK: - Conflict Handling - - // MARK: - Settings Helpers - - private func settingsData(for category: String) -> Data? { - let storage = services.appSettingsStorage - let encoder = JSONEncoder() - - do { - switch category { - case AppSettingsCategory.general: return try encoder.encode(storage.loadGeneral()) - case AppSettingsCategory.appearance: return try encoder.encode(storage.loadAppearance()) - case AppSettingsCategory.editor: return try encoder.encode(storage.loadEditor()) - case AppSettingsCategory.dataGrid: return try encoder.encode(storage.loadDataGrid()) - case AppSettingsCategory.history: return try encoder.encode(storage.loadHistory()) - case AppSettingsCategory.tabs: return try encoder.encode(storage.loadTabs()) - case AppSettingsCategory.keyboard: return try encoder.encode(storage.loadKeyboard()) - case AppSettingsCategory.ai: return try encoder.encode(storage.loadAI()) - case AppSettingsCategory.notifications: return try encoder.encode(storage.loadNotifications()) - case CustomSlashCommandStorage.syncCategory: - return try encoder.encode(CustomSlashCommandStorage.shared.commands) - case let category where category.hasPrefix(FileColumnLayoutPersister.syncCategoryPrefix): - return FileColumnLayoutPersister.shared.rawData( - forStorageKey: String(category.dropFirst(FileColumnLayoutPersister.syncCategoryPrefix.count)) - ) - default: return nil - } - } catch { - Self.logger.error("Failed to encode settings category '\(category)': \(error.localizedDescription)") - return nil - } - } - - private func applySettingsData(_ data: Data, for category: String) throws { - let manager = services.appSettings - let decoder = JSONDecoder() - - do { - switch category { - case AppSettingsCategory.general: manager.general = try decoder.decode(GeneralSettings.self, from: data) - case AppSettingsCategory.appearance: - manager.appearance = try decoder.decode(AppearanceSettings.self, from: data) - case AppSettingsCategory.editor: manager.editor = try decoder.decode(EditorSettings.self, from: data) - case AppSettingsCategory.dataGrid: manager.dataGrid = try decoder.decode(DataGridSettings.self, from: data) - case AppSettingsCategory.history: manager.history = try decoder.decode(HistorySettings.self, from: data) - case AppSettingsCategory.tabs: manager.tabs = try decoder.decode(TabSettings.self, from: data) - case AppSettingsCategory.keyboard: manager.keyboard = try decoder.decode(KeyboardSettings.self, from: data) - case AppSettingsCategory.ai: manager.ai = try decoder.decode(AISettings.self, from: data) - case AppSettingsCategory.notifications: - manager.notifications = try decoder.decode(NotificationSettings.self, from: data) - case CustomSlashCommandStorage.syncCategory: - CustomSlashCommandStorage.shared.applyRemote(try decoder.decode([CustomSlashCommand].self, from: data)) - case let category where category.hasPrefix(FileColumnLayoutPersister.syncCategoryPrefix): - FileColumnLayoutPersister.shared.applyRemote( - storageKey: String(category.dropFirst(FileColumnLayoutPersister.syncCategoryPrefix.count)), - data: data - ) - default: return - } - } catch { - throw SyncDecodeError.decodeFailure(field: category, underlying: error) - } - } - - // MARK: - Group/Tag Collection Helpers - - private func collectDirtyGroups( - into records: inout [CKRecord], - deletions: inout [CKRecord.ID], - zoneID: CKRecordZone.ID - ) { - let dirtyGroupIds = changeTracker.dirtyRecords(for: .group) - if !dirtyGroupIds.isEmpty { - let groups = services.groupStorage.loadGroups() - for id in dirtyGroupIds { - if let group = groups.first(where: { $0.id.uuidString == id }) { - records.append(SyncRecordMapper.toCKRecord(group, in: zoneID)) - } - } - } - - for tombstone in metadataStorage.tombstones(for: .group) { - deletions.append( - SyncRecordMapper.recordID(type: .group, id: tombstone.id, in: zoneID) - ) - } - } - - private func collectDirtyTags( - into records: inout [CKRecord], - deletions: inout [CKRecord.ID], - zoneID: CKRecordZone.ID - ) { - let dirtyTagIds = changeTracker.dirtyRecords(for: .tag) - if !dirtyTagIds.isEmpty { - let tags = services.tagStorage.loadTags() - for id in dirtyTagIds { - if let tag = tags.first(where: { $0.id.uuidString == id }) { - records.append(SyncRecordMapper.toCKRecord(tag, in: zoneID)) - } - } - } - - for tombstone in metadataStorage.tombstones(for: .tag) { - deletions.append( - SyncRecordMapper.recordID(type: .tag, id: tombstone.id, in: zoneID) - ) - } - } - - private func collectDirtyCredentialProfiles( - into records: inout [CKRecord], - deletions: inout [CKRecord.ID], - zoneID: CKRecordZone.ID - ) { - let dirtyIds = changeTracker.dirtyRecords(for: .credentialProfile) - if !dirtyIds.isEmpty { - let profiles = services.credentialProfileStorage.loadProfiles() - for id in dirtyIds { - if let profile = profiles.first(where: { $0.id.uuidString == id }) { - records.append(SyncRecordMapper.toCKRecord(profile, in: zoneID)) - } - } - } - - for tombstone in metadataStorage.tombstones(for: .credentialProfile) { - deletions.append( - SyncRecordMapper.recordID(type: .credentialProfile, id: tombstone.id, in: zoneID) - ) - } - } - - private func collectDirtySSHProfiles( - into records: inout [CKRecord], - deletions: inout [CKRecord.ID], - zoneID: CKRecordZone.ID - ) { - let dirtyProfileIds = changeTracker.dirtyRecords(for: .sshProfile) - if !dirtyProfileIds.isEmpty { - let profiles = services.sshProfileStorage.loadProfiles() - for id in dirtyProfileIds { - if let profile = profiles.first(where: { $0.id.uuidString == id }) { - records.append(SyncRecordMapper.toCKRecord(profile, in: zoneID)) - } - } - } - - for tombstone in metadataStorage.tombstones(for: .sshProfile) { - deletions.append( - SyncRecordMapper.recordID(type: .sshProfile, id: tombstone.id, in: zoneID) - ) - } - } - - private func collectDirtySQLFavorites( - into records: inout [CKRecord], - deletions: inout [CKRecord.ID], - zoneID: CKRecordZone.ID - ) async { - let dirtyFavoriteIds = changeTracker.dirtyRecords(for: .favorite) - if !dirtyFavoriteIds.isEmpty { - let favorites = await services.sqlFavoriteManager.fetchFavorites() - let favoritesById = Dictionary(favorites.map { ($0.id.uuidString, $0) }, uniquingKeysWith: { first, _ in first }) - for id in dirtyFavoriteIds { - if let favorite = favoritesById[id] { - records.append(SyncRecordMapper.toCKRecord(sqlFavorite: favorite, in: zoneID)) - } - } - } - for tombstone in metadataStorage.tombstones(for: .favorite) { - deletions.append( - SyncRecordMapper.recordID(type: .favorite, id: tombstone.id, in: zoneID) - ) - } - - let dirtyFolderIds = changeTracker.dirtyRecords(for: .favoriteFolder) - if !dirtyFolderIds.isEmpty { - let folders = await services.sqlFavoriteManager.fetchFolders() - let foldersById = Dictionary(folders.map { ($0.id.uuidString, $0) }, uniquingKeysWith: { first, _ in first }) - for id in dirtyFolderIds { - if let folder = foldersById[id] { - records.append(SyncRecordMapper.toCKRecord(sqlFavoriteFolder: folder, in: zoneID)) - } - } - } - for tombstone in metadataStorage.tombstones(for: .favoriteFolder) { - deletions.append( - SyncRecordMapper.recordID(type: .favoriteFolder, id: tombstone.id, in: zoneID) - ) - } - } - - private func collectDirtyTableFavorites( - into records: inout [CKRecord], - deletions: inout [CKRecord.ID], - zoneID: CKRecordZone.ID - ) { - let dirtyIds = changeTracker.dirtyRecords(for: .tableFavorite) - if !dirtyIds.isEmpty { - let favorites = services.favoriteTablesStorage.loadFavorites() - for entry in favorites where dirtyIds.contains(FavoriteTablesStorage.syncId(for: entry)) { - records.append(SyncRecordMapper.toCKRecord(favoriteEntry: entry, in: zoneID)) - } - } - - for tombstone in metadataStorage.tombstones(for: .tableFavorite) { - deletions.append( - SyncRecordMapper.recordID(type: .tableFavorite, id: tombstone.id, in: zoneID) - ) - } - } - - /// A connection the user marked local only never reaches iCloud, and neither do the database - /// names hanging off it. Tombstones are not filtered: a deletion only ever removes something, - /// and a connection can be marked local only after its favorites were already pushed. - private func collectDirtyDatabaseFavorites( - into records: inout [CKRecord], - deletions: inout [CKRecord.ID], - zoneID: CKRecordZone.ID - ) { - let dirtyIds = changeTracker.dirtyRecords(for: .favoriteDatabase) - if !dirtyIds.isEmpty { - let localOnlyIds = Set( - services.connectionStorage.loadConnections().filter(\.localOnly).map(\.id) - ) - let favorites = services.favoriteDatabasesStorage.loadFavorites() - for entry in favorites - where dirtyIds.contains(FavoriteDatabasesStorage.syncId(for: entry)) - && !localOnlyIds.contains(entry.connectionId) { - records.append(SyncRecordMapper.toCKRecord(favoriteDatabase: entry, in: zoneID)) - } - } - - for tombstone in metadataStorage.tombstones(for: .favoriteDatabase) { - deletions.append( - SyncRecordMapper.recordID(type: .favoriteDatabase, id: tombstone.id, in: zoneID) - ) - } + try await transport.currentAccountId() } } diff --git a/TablePro/Core/Sync/SyncEchoGuard.swift b/TablePro/Core/Sync/SyncEchoGuard.swift new file mode 100644 index 0000000000..7a319c4947 --- /dev/null +++ b/TablePro/Core/Sync/SyncEchoGuard.swift @@ -0,0 +1,29 @@ +import CloudKit +import Foundation +import TableProSyncTransport + +struct SyncEchoGuard: Sendable { + static let typesMergedOnPull: Set = [.connection] + + let snapshot: SyncEditSnapshot + let savedRecords: [CKRecord.ID: SyncRecordIdentity] + private let savedIdentities: Set + + init(snapshot: SyncEditSnapshot, savedRecords: [CKRecord.ID: SyncRecordIdentity]) { + let guarded = savedRecords.filter { !Self.typesMergedOnPull.contains($0.value.type) } + self.snapshot = snapshot + self.savedRecords = guarded + self.savedIdentities = Set(guarded.values) + } + + func withholds(_ recordID: CKRecord.ID, tracker: SyncChangeTracker) -> Bool { + guard let identity = savedRecords[recordID] else { return false } + return tracker.hasEdit(identity, since: snapshot) + } + + func withholds(_ type: SyncRecordType, id: String, tracker: SyncChangeTracker) -> Bool { + let identity = SyncRecordIdentity(type: type, id: id) + guard savedIdentities.contains(identity) else { return false } + return tracker.hasEdit(identity, since: snapshot) + } +} diff --git a/TablePro/Core/Sync/SyncEditSnapshot.swift b/TablePro/Core/Sync/SyncEditSnapshot.swift new file mode 100644 index 0000000000..621ae2cfdb --- /dev/null +++ b/TablePro/Core/Sync/SyncEditSnapshot.swift @@ -0,0 +1,29 @@ +import Foundation +import TableProSyncTransport + +struct SyncEditSnapshot: Equatable, Sendable { + static let empty = SyncEditSnapshot(dirty: [], generations: [:]) + + let dirty: Set + let generations: [SyncRecordIdentity: UInt64] + + func dirtyIds(for type: SyncRecordType) -> Set { + Set(dirty.lazy.filter { $0.type == type }.map(\.id)) + } +} + +struct SyncEditGenerations: Sendable { + private var lastGeneration: UInt64 = 0 + private var generations: [SyncRecordIdentity: UInt64] = [:] + + func generation(of identity: SyncRecordIdentity) -> UInt64 { + generations[identity] ?? 0 + } + + mutating func recordEdits(of identities: [SyncRecordIdentity]) { + for identity in identities { + lastGeneration += 1 + generations[identity] = lastGeneration + } + } +} diff --git a/TablePro/Core/Sync/SyncRecordChanges.swift b/TablePro/Core/Sync/SyncRecordChanges.swift new file mode 100644 index 0000000000..21bee383e7 --- /dev/null +++ b/TablePro/Core/Sync/SyncRecordChanges.swift @@ -0,0 +1,13 @@ +import Foundation + +enum SyncRecordChanges { + static func changedIds( + from previous: [Record], + to current: [Record] + ) -> [String] where Record.ID == UUID { + let previousById = Dictionary(previous.map { ($0.id, $0) }, uniquingKeysWith: { _, last in last }) + return current + .filter { previousById[$0.id] != $0 } + .map { $0.id.uuidString } + } +} diff --git a/TablePro/Core/Sync/SyncTransport.swift b/TablePro/Core/Sync/SyncTransport.swift new file mode 100644 index 0000000000..981faad1a7 --- /dev/null +++ b/TablePro/Core/Sync/SyncTransport.swift @@ -0,0 +1,14 @@ +import CloudKit +import Foundation +import TableProSyncTransport + +protocol SyncTransport: Sendable { + var currentZoneID: CKRecordZone.ID { get async } + func accountStatus() async throws -> CKAccountStatus + func currentAccountId() async throws -> String + func ensureZoneExists() async throws + func push(records: [CKRecord], deletions: [CKRecord.ID]) async throws -> PushOutcome + func pull(since token: CKServerChangeToken?) async throws -> PullResult +} + +extension CloudKitSyncEngine: SyncTransport {} diff --git a/TablePro/Core/Utilities/File/FileTextEncoding.swift b/TablePro/Core/Utilities/File/FileTextEncoding.swift new file mode 100644 index 0000000000..c18c5abadb --- /dev/null +++ b/TablePro/Core/Utilities/File/FileTextEncoding.swift @@ -0,0 +1,59 @@ +// +// FileTextEncoding.swift +// TablePro +// + +import Foundation + +internal struct FileTextEncoding: Hashable, Sendable { + let encoding: String.Encoding + let byteOrderMark: ByteOrderMark? + let attribute: TextEncodingAttribute? + + static let utf8 = FileTextEncoding(encoding: .utf8) + + init( + encoding: String.Encoding, + byteOrderMark: ByteOrderMark? = nil, + attributeOnDisk: TextEncodingAttribute? = nil + ) { + let savedEncoding: String.Encoding = encoding == .ascii ? .utf8 : encoding + self.encoding = savedEncoding + self.byteOrderMark = byteOrderMark + let attributeNamesEncoding = byteOrderMark == nil && attributeOnDisk?.encoding == savedEncoding + self.attribute = attributeNamesEncoding ? attributeOnDisk : nil + } + + var displayName: String { + String.localizedName(of: encoding) + } + + func bytes(of text: String) -> Data? { + let byteOrderedEncoding = byteOrderMark?.byteOrderedEncoding ?? encoding.unmarkedByteOrder + guard let body = text.data(using: byteOrderedEncoding, allowLossyConversion: false) else { return nil } + guard let byteOrderMark else { return body } + return Data(byteOrderMark.bytes) + body + } +} + +extension FileTextEncoding: Codable { + private enum CodingKeys: String, CodingKey { + case encoding, byteOrderMark, attribute + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.init( + encoding: String.Encoding(rawValue: try container.decode(UInt.self, forKey: .encoding)), + byteOrderMark: try container.decodeIfPresent(ByteOrderMark.self, forKey: .byteOrderMark), + attributeOnDisk: try container.decodeIfPresent(TextEncodingAttribute.self, forKey: .attribute) + ) + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(encoding.rawValue, forKey: .encoding) + try container.encodeIfPresent(byteOrderMark, forKey: .byteOrderMark) + try container.encodeIfPresent(attribute, forKey: .attribute) + } +} diff --git a/TablePro/Core/Utilities/File/FileTextLoader.swift b/TablePro/Core/Utilities/File/FileTextLoader.swift index 246039404d..d89de3728d 100644 --- a/TablePro/Core/Utilities/File/FileTextLoader.swift +++ b/TablePro/Core/Utilities/File/FileTextLoader.swift @@ -8,7 +8,7 @@ import Foundation internal enum FileTextLoader { struct LoadedText: Sendable { let content: String - let encoding: String.Encoding + let textEncoding: FileTextEncoding /// What the file was, as of just before this text was read. /// /// Read here rather than by the caller, because a caller that stats afterwards records a @@ -17,39 +17,56 @@ internal enum FileTextLoader { /// fails the other way, leaving the baseline older than the text, so the changed-on-disk /// notice can fire once too often but never go missing. let stamp: FileStamp? + var encoding: String.Encoding { textEncoding.encoding } var isUTF8: Bool { encoding == .utf8 } } static func load(_ url: URL) -> LoadedText? { + try? read(url) + } + + static func read(_ url: URL) throws -> LoadedText { let stamp = FileStamp.read(url) if startsWithByteOrderMark(url) { - return loadByteOrderMarked(url, stamp: stamp) + return try readByteOrderMarked(url, stamp: stamp) } + let attribute = TextEncodingAttribute.read(from: url) var detected: String.Encoding = .utf8 if let content = try? String(contentsOf: url, usedEncoding: &detected) { - return LoadedText(content: content, encoding: detected, stamp: stamp) + let textEncoding = FileTextEncoding(encoding: detected, attributeOnDisk: attribute) + return LoadedText(content: content, textEncoding: textEncoding, stamp: stamp) } if let content = try? String(contentsOf: url, encoding: .utf8) { - return LoadedText(content: content, encoding: .utf8, stamp: stamp) + return LoadedText(content: content, textEncoding: .utf8, stamp: stamp) } - if let content = try? String(contentsOf: url, encoding: .isoLatin1) { - return LoadedText(content: content, encoding: .isoLatin1, stamp: stamp) - } - return nil + let content = try String(contentsOf: url, encoding: .isoLatin1) + return LoadedText(content: content, textEncoding: FileTextEncoding(encoding: .isoLatin1), stamp: stamp) } - static func decode(_ data: Data) -> String? { + static func decode(_ data: Data, declaredEncoding: String.Encoding? = nil) -> String? { + if declaredEncoding != nil, ByteOrderMark.leading(data) == nil, let utf8 = String(data: data, encoding: .utf8) { + return utf8 + } guard !data.isEmpty else { return "" } - return TextPrefixDecoder.decode(data, prefixLength: data.count)?.content + return TextPrefixDecoder.decode(data, prefixLength: data.count, declaredEncoding: declaredEncoding)?.content } static func loadHeader(_ url: URL, maxBytes: Int = 4_096) -> LoadedText? { let stamp = FileStamp.read(url) + let attribute = TextEncodingAttribute.read(from: url) guard let handle = try? FileHandle(forReadingFrom: url) else { return nil } defer { try? handle.close() } guard let bytes = try? handle.read(upToCount: maxBytes + TextPrefixDecoder.lookaheadLength), - let decoded = TextPrefixDecoder.decode(bytes, prefixLength: maxBytes) else { return nil } - return LoadedText(content: decoded.content, encoding: decoded.encoding, stamp: stamp) + let decoded = TextPrefixDecoder.decode( + bytes, + prefixLength: maxBytes, + declaredEncoding: attribute?.encoding + ) else { return nil } + return LoadedText( + content: decoded.content, + textEncoding: textEncoding(of: decoded, attribute: attribute), + stamp: stamp + ) } private static func startsWithByteOrderMark(_ url: URL) -> Bool { @@ -59,14 +76,50 @@ internal enum FileTextLoader { return ByteOrderMark.leading(bytes) != nil } - private static func loadByteOrderMarked(_ url: URL, stamp: FileStamp?) -> LoadedText? { - guard let bytes = try? Data(contentsOf: url), - let decoded = TextPrefixDecoder.decode(bytes, prefixLength: bytes.count) else { return nil } - return LoadedText(content: decoded.content, encoding: decoded.encoding, stamp: stamp) + private static func readByteOrderMarked(_ url: URL, stamp: FileStamp?) throws -> LoadedText { + let bytes = try Data(contentsOf: url) + guard let decoded = TextPrefixDecoder.decode(bytes, prefixLength: bytes.count) else { + throw CocoaError(.fileReadCorruptFile, userInfo: [NSFilePathErrorKey: url.path]) + } + return LoadedText( + content: decoded.content, + textEncoding: textEncoding(of: decoded, attribute: nil), + stamp: stamp + ) + } + + private static func textEncoding( + of decoded: TextPrefixDecoder.Decoded, + attribute: TextEncodingAttribute? + ) -> FileTextEncoding { + FileTextEncoding(encoding: decoded.encoding, byteOrderMark: decoded.byteOrderMark, attributeOnDisk: attribute) } } internal extension String.Encoding { + init?(coreFoundationEncoding: CFStringEncoding) { + guard coreFoundationEncoding != kCFStringEncodingInvalidId, + CFStringIsEncodingAvailable(coreFoundationEncoding) else { return nil } + self.init(rawValue: CFStringConvertEncodingToNSStringEncoding(coreFoundationEncoding)) + } + + init?(ianaCharacterSetName name: String) { + self.init(coreFoundationEncoding: CFStringConvertIANACharSetNameToEncoding(name as CFString)) + } + + private static let gb18030 = String.Encoding( + coreFoundationEncoding: CFStringEncoding(CFStringEncodings.GB_18030_2000.rawValue) + ) + + var representsAllOfUnicode: Bool { + switch self { + case .utf8, .utf16, .utf16BigEndian, .utf16LittleEndian, .utf32, .utf32BigEndian, .utf32LittleEndian: + return true + default: + return self == Self.gb18030 + } + } + var displayName: String { switch self { case .utf8: return "UTF-8" diff --git a/TablePro/Core/Utilities/File/FileTextWriter.swift b/TablePro/Core/Utilities/File/FileTextWriter.swift new file mode 100644 index 0000000000..c11191dd89 --- /dev/null +++ b/TablePro/Core/Utilities/File/FileTextWriter.swift @@ -0,0 +1,85 @@ +// +// FileTextWriter.swift +// TablePro +// + +import Foundation + +internal enum FileTextWriter { + enum WriteError: Error, Equatable { + case unrepresentable(FileTextEncoding) + } + + typealias AttributeApplier = (TextEncodingAttribute, URL) throws -> Void + + private static let stagingPrefix = ".tablepro-save-" + + static func write(_ text: String, to url: URL, as encoding: FileTextEncoding) throws { + guard let bytes = encoding.bytes(of: text) else { + throw WriteError.unrepresentable(encoding) + } + try replaceContents(of: url, with: bytes, attribute: encoding.attribute) + } + + static func replaceContents( + of url: URL, + with bytes: Data, + attribute: TextEncodingAttribute?, + applyingAttribute applyAttribute: AttributeApplier = { attribute, url in try attribute.write(to: url) } + ) throws { + let destination = url.resolvingSymlinksInPath() + let staging = destination.deletingLastPathComponent() + .appendingPathComponent(stagingPrefix + UUID().uuidString) + do { + try bytes.write(to: staging, options: .withoutOverwriting) + if let attribute { + try applyAttribute(attribute, staging) + } + try carryPermissions(of: destination, to: staging) + try move(staging, over: destination) + } catch { + try? FileManager.default.removeItem(at: staging) + throw describing(error, at: destination) + } + } + + private static func carryPermissions(of destination: URL, to staging: URL) throws { + guard let attributes = try? FileManager.default.attributesOfItem(atPath: destination.path), + let permissions = attributes[.posixPermissions] else { return } + try FileManager.default.setAttributes([.posixPermissions: permissions], ofItemAtPath: staging.path) + } + + private static func move(_ staging: URL, over destination: URL) throws { + let failure = staging.withUnsafeFileSystemRepresentation { stagingPath -> POSIXErrorCode? in + destination.withUnsafeFileSystemRepresentation { destinationPath -> POSIXErrorCode? in + guard let stagingPath, let destinationPath else { return .ENOENT } + guard rename(stagingPath, destinationPath) != 0 else { return nil } + return POSIXErrorCode(rawValue: errno) ?? .EIO + } + } + if let failure { + throw POSIXError(failure) + } + } + + private static func describing(_ error: Error, at destination: URL) -> Error { + CocoaError(writeErrorCode(for: error), userInfo: [NSFilePathErrorKey: destination.path, NSUnderlyingErrorKey: error]) + } + + private static func writeErrorCode(for error: Error) -> CocoaError.Code { + if let cocoaError = error as? CocoaError { + return cocoaError.code + } + guard let posixError = error as? POSIXError else { return .fileWriteUnknown } + switch posixError.code { + case .EACCES, .EPERM: + return .fileWriteNoPermission + case .ENOSPC, .EDQUOT: + return .fileWriteOutOfSpace + case .EROFS: + return .fileWriteVolumeReadOnly + default: + return .fileWriteUnknown + } + } +} diff --git a/TablePro/Core/Utilities/File/TextEncodingAttribute.swift b/TablePro/Core/Utilities/File/TextEncodingAttribute.swift new file mode 100644 index 0000000000..db73764ccb --- /dev/null +++ b/TablePro/Core/Utilities/File/TextEncodingAttribute.swift @@ -0,0 +1,51 @@ +// +// TextEncodingAttribute.swift +// TablePro +// + +import Foundation + +internal struct TextEncodingAttribute: Codable, Hashable, Sendable { + private static let name = "com.apple.TextEncoding" + private static let separator: Character = ";" + + let value: Data + + var encoding: String.Encoding? { + guard let text = String(data: value, encoding: .utf8) else { return nil } + let fields = text + .trimmingCharacters(in: CharacterSet(charactersIn: "\0").union(.whitespaces)) + .split(separator: Self.separator, maxSplits: 1, omittingEmptySubsequences: false) + if fields.count == 2, let number = UInt32(fields[1]) { + return String.Encoding(coreFoundationEncoding: number) + } + guard let ianaName = fields.first, !ianaName.isEmpty else { return nil } + return String.Encoding(ianaCharacterSetName: String(ianaName)) + } + + static func read(from url: URL) -> TextEncodingAttribute? { + url.withUnsafeFileSystemRepresentation { path -> TextEncodingAttribute? in + guard let path else { return nil } + let size = getxattr(path, name, nil, 0, 0, 0) + guard size > 0 else { return nil } + var bytes = [UInt8](repeating: 0, count: size) + let read = getxattr(path, name, &bytes, size, 0, 0) + guard read > 0 else { return nil } + return TextEncodingAttribute(value: Data(bytes.prefix(read))) + } + } + + func write(to url: URL) throws { + let failure = url.withUnsafeFileSystemRepresentation { path -> POSIXErrorCode? in + guard let path else { return .ENOENT } + let status = value.withUnsafeBytes { buffer in + setxattr(path, Self.name, buffer.baseAddress, buffer.count, 0, 0) + } + guard status != 0 else { return nil } + return POSIXErrorCode(rawValue: errno) ?? .EIO + } + if let failure { + throw POSIXError(failure) + } + } +} diff --git a/TablePro/Core/Utilities/File/TextPrefixDecoder.swift b/TablePro/Core/Utilities/File/TextPrefixDecoder.swift index f559c4f613..004795816c 100644 --- a/TablePro/Core/Utilities/File/TextPrefixDecoder.swift +++ b/TablePro/Core/Utilities/File/TextPrefixDecoder.swift @@ -9,36 +9,45 @@ internal enum TextPrefixDecoder { struct Decoded { let content: String let encoding: String.Encoding + let byteOrderMark: ByteOrderMark? } static let lookaheadLength = 3 - static func decode(_ bytes: Data, prefixLength: Int) -> Decoded? { + static func decode(_ bytes: Data, prefixLength: Int, declaredEncoding: String.Encoding? = nil) -> Decoded? { guard !bytes.isEmpty, prefixLength > 0 else { return nil } - let mark = ByteOrderMark.leading(bytes) - let encoding = mark?.encoding ?? .utf8 - let codeUnitLength = mark?.codeUnitLength ?? 1 - if let content = decode(bytes, prefixLength: prefixLength, as: encoding, codeUnitLength: codeUnitLength) { - return Decoded(content: content, encoding: encoding) + if let mark = ByteOrderMark.leading(bytes) { + return decode(bytes, prefixLength: prefixLength, as: mark.encoding, byteOrderMark: mark) + ?? decodeAsLatin1(bytes, prefixLength: prefixLength) } - guard let content = String(data: bytes.prefix(prefixLength), encoding: .isoLatin1) else { return nil } - return Decoded(content: content, encoding: .isoLatin1) + if let declaredEncoding, + let decoded = decode(bytes, prefixLength: prefixLength, as: declaredEncoding, byteOrderMark: nil) { + return decoded + } + return decode(bytes, prefixLength: prefixLength, as: .utf8, byteOrderMark: nil) + ?? decodeAsLatin1(bytes, prefixLength: prefixLength) } private static func decode( _ bytes: Data, prefixLength: Int, as encoding: String.Encoding, - codeUnitLength: Int - ) -> String? { + byteOrderMark: ByteOrderMark? + ) -> Decoded? { + let codeUnitLength = encoding.codeUnitLength for end in candidateEnds(of: bytes, prefixLength: prefixLength) where end.isMultiple(of: codeUnitLength) { if let content = String(data: bytes.prefix(end), encoding: encoding) { - return content + return Decoded(content: content, encoding: encoding, byteOrderMark: byteOrderMark) } } return nil } + private static func decodeAsLatin1(_ bytes: Data, prefixLength: Int) -> Decoded? { + guard let content = String(data: bytes.prefix(prefixLength), encoding: .isoLatin1) else { return nil } + return Decoded(content: content, encoding: .isoLatin1, byteOrderMark: nil) + } + private static func candidateEnds(of bytes: Data, prefixLength: Int) -> ClosedRange { guard bytes.count > prefixLength else { return bytes.count...bytes.count } return prefixLength...min(bytes.count, prefixLength + lookaheadLength) diff --git a/TablePro/Core/Utilities/MainActorSerialQueue.swift b/TablePro/Core/Utilities/MainActorSerialQueue.swift new file mode 100644 index 0000000000..b4b453f4b2 --- /dev/null +++ b/TablePro/Core/Utilities/MainActorSerialQueue.swift @@ -0,0 +1,20 @@ +import Foundation + +@MainActor +internal final class MainActorSerialQueue { + private var tail: Task? + + nonisolated internal init() {} + + internal func run(_ operation: @escaping @MainActor () async -> Value) async -> Value { + let previous = tail + let current = Task { @MainActor in + await previous?.value + return await operation() + } + tail = Task { @MainActor in + _ = await current.value + } + return await current.value + } +} diff --git a/TablePro/Core/Utilities/SQL/SQLChunkDecoder.swift b/TablePro/Core/Utilities/SQL/SQLChunkDecoder.swift index 68a452b4ba..0bb7e683b1 100644 --- a/TablePro/Core/Utilities/SQL/SQLChunkDecoder.swift +++ b/TablePro/Core/Utilities/SQL/SQLChunkDecoder.swift @@ -79,7 +79,7 @@ struct SQLChunkDecoder { let resolution = Self.resolve(declaredEncoding, startingWith: data) resolvedEncoding = resolution.encoding - unitSize = Self.unitSize(of: resolution.encoding) + unitSize = resolution.encoding.codeUnitLength maximumTrim = Self.maximumTrim(of: resolution.encoding, unitSize: unitSize) if resolution.byteOrderMarkLength > 0, data.count >= resolution.byteOrderMarkLength { data = Data(data.dropFirst(resolution.byteOrderMarkLength)) @@ -92,34 +92,11 @@ struct SQLChunkDecoder { let byteOrderMarkLength: Int } - /// `.utf8` is absent on purpose: Foundation consumes a UTF-8 mark itself, measured. private static func resolve(_ encoding: String.Encoding, startingWith data: Data) -> Resolution { if let mark = ByteOrderMark.leading(data, allowedBy: encoding) { return Resolution(encoding: mark.byteOrderedEncoding, byteOrderMarkLength: mark.length) } - return Resolution(encoding: unmarkedByteOrder(of: encoding), byteOrderMarkLength: 0) - } - - private static func unmarkedByteOrder(of encoding: String.Encoding) -> String.Encoding { - switch encoding { - case .utf16: - return .utf16BigEndian - case .utf32: - return .utf32BigEndian - default: - return encoding - } - } - - private static func unitSize(of encoding: String.Encoding) -> Int { - switch encoding { - case .utf16, .utf16LittleEndian, .utf16BigEndian: - return 2 - case .utf32, .utf32LittleEndian, .utf32BigEndian: - return 4 - default: - return 1 - } + return Resolution(encoding: encoding.unmarkedByteOrder, byteOrderMarkLength: 0) } /// `CFStringGetMaximumSizeForEncoding` counts bytes per UTF-16 code unit, so it is asked for diff --git a/TablePro/Core/Utilities/Text/ByteOrderMark.swift b/TablePro/Core/Utilities/Text/ByteOrderMark.swift index 17e2a83ffe..bf30f676c6 100644 --- a/TablePro/Core/Utilities/Text/ByteOrderMark.swift +++ b/TablePro/Core/Utilities/Text/ByteOrderMark.swift @@ -5,14 +5,15 @@ import Foundation -internal enum ByteOrderMark { +internal enum ByteOrderMark: String, Codable, Sendable { case utf32LittleEndian case utf32BigEndian + case utf8 case utf16LittleEndian case utf16BigEndian private static let longestFirst: [ByteOrderMark] = [ - .utf32LittleEndian, .utf32BigEndian, .utf16LittleEndian, .utf16BigEndian + .utf32LittleEndian, .utf32BigEndian, .utf8, .utf16LittleEndian, .utf16BigEndian ] static var longestLength: Int { longestFirst.map(\.length).max() ?? 0 } @@ -29,6 +30,7 @@ internal enum ByteOrderMark { switch self { case .utf32LittleEndian: return [0xFF, 0xFE, 0x00, 0x00] case .utf32BigEndian: return [0x00, 0x00, 0xFE, 0xFF] + case .utf8: return [0xEF, 0xBB, 0xBF] case .utf16LittleEndian: return [0xFF, 0xFE] case .utf16BigEndian: return [0xFE, 0xFF] } @@ -39,6 +41,7 @@ internal enum ByteOrderMark { var encoding: String.Encoding { switch self { case .utf32LittleEndian, .utf32BigEndian: return .utf32 + case .utf8: return .utf8 case .utf16LittleEndian, .utf16BigEndian: return .utf16 } } @@ -47,19 +50,31 @@ internal enum ByteOrderMark { switch self { case .utf32LittleEndian: return .utf32LittleEndian case .utf32BigEndian: return .utf32BigEndian + case .utf8: return .utf8 case .utf16LittleEndian: return .utf16LittleEndian case .utf16BigEndian: return .utf16BigEndian } } - var codeUnitLength: Int { + private func isAllowed(by declaredEncoding: String.Encoding) -> Bool { + declaredEncoding == encoding || declaredEncoding == byteOrderedEncoding + } +} + +internal extension String.Encoding { + var unmarkedByteOrder: String.Encoding { switch self { - case .utf32LittleEndian, .utf32BigEndian: return 4 - case .utf16LittleEndian, .utf16BigEndian: return 2 + case .utf16: return .utf16BigEndian + case .utf32: return .utf32BigEndian + default: return self } } - private func isAllowed(by declaredEncoding: String.Encoding) -> Bool { - declaredEncoding == encoding || declaredEncoding == byteOrderedEncoding + var codeUnitLength: Int { + switch self { + case .utf16, .utf16LittleEndian, .utf16BigEndian: return 2 + case .utf32, .utf32LittleEndian, .utf32BigEndian: return 4 + default: return 1 + } } } diff --git a/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift b/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift index 98a27cd75a..bd1d98084c 100644 --- a/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift +++ b/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift @@ -64,7 +64,8 @@ internal struct LinkedFileVersionHistoryProvider: VersionHistoryProvider { case .gitRevision(let commit, let path): let client = try makeClient() let data = try await gitCall { try await client.blob(revision: commit, path: path, in: directory) } - guard let content = FileTextLoader.decode(data) else { + let workingCopyEncoding = TextEncodingAttribute.read(from: fileURL)?.encoding + guard let content = FileTextLoader.decode(data, declaredEncoding: workingCopyEncoding) else { throw VersionHistoryError.undecodableContent } return content diff --git a/TablePro/Models/Query/EditorTabPayload.swift b/TablePro/Models/Query/EditorTabPayload.swift index 7da703fe61..0f4325b5a6 100644 --- a/TablePro/Models/Query/EditorTabPayload.swift +++ b/TablePro/Models/Query/EditorTabPayload.swift @@ -54,6 +54,7 @@ internal struct EditorTabPayload: Codable, Hashable { /// Source file URL for .sql files opened from disk (used for deduplication) internal let sourceFileURL: URL? internal let sourceFileStamp: FileStamp? + internal let sourceFileEncoding: FileTextEncoding? /// Schema key for ER diagram tabs internal let erDiagramSchemaKey: String? /// The routine or trigger a .objectSource tab shows @@ -69,7 +70,8 @@ internal struct EditorTabPayload: Codable, Hashable { case initialQuery, isView, objectType, showStructure, skipAutoExecute, isPreview case forcesNewTab case tabTitle - case initialFilterState, sourceFileURL, sourceFileStamp, erDiagramSchemaKey, objectRef, versionHistorySubject, intent + case initialFilterState, sourceFileURL, sourceFileStamp, sourceFileEncoding + case erDiagramSchemaKey, objectRef, versionHistorySubject, intent // Legacy key for backward decoding only case isNewTab } @@ -91,6 +93,7 @@ internal struct EditorTabPayload: Codable, Hashable { initialFilterState: TabFilterState? = nil, sourceFileURL: URL? = nil, sourceFileStamp: FileStamp? = nil, + sourceFileEncoding: FileTextEncoding? = nil, erDiagramSchemaKey: String? = nil, objectRef: DatabaseObjectRef? = nil, versionHistorySubject: VersionHistorySubject? = nil, @@ -113,6 +116,7 @@ internal struct EditorTabPayload: Codable, Hashable { self.initialFilterState = initialFilterState self.sourceFileURL = sourceFileURL self.sourceFileStamp = sourceFileStamp + self.sourceFileEncoding = sourceFileEncoding self.erDiagramSchemaKey = erDiagramSchemaKey self.objectRef = objectRef self.versionHistorySubject = versionHistorySubject @@ -141,6 +145,7 @@ internal struct EditorTabPayload: Codable, Hashable { initialFilterState = try container.decodeIfPresent(TabFilterState.self, forKey: .initialFilterState) sourceFileURL = try container.decodeIfPresent(URL.self, forKey: .sourceFileURL) sourceFileStamp = try container.decodeIfPresent(FileStamp.self, forKey: .sourceFileStamp) + sourceFileEncoding = try container.decodeIfPresent(FileTextEncoding.self, forKey: .sourceFileEncoding) erDiagramSchemaKey = try container.decodeIfPresent(String.self, forKey: .erDiagramSchemaKey) objectRef = try container.decodeIfPresent(DatabaseObjectRef.self, forKey: .objectRef) versionHistorySubject = try container.decodeIfPresent(VersionHistorySubject.self, forKey: .versionHistorySubject) @@ -171,6 +176,7 @@ internal struct EditorTabPayload: Codable, Hashable { try container.encodeIfPresent(initialFilterState, forKey: .initialFilterState) try container.encodeIfPresent(sourceFileURL, forKey: .sourceFileURL) try container.encodeIfPresent(sourceFileStamp, forKey: .sourceFileStamp) + try container.encodeIfPresent(sourceFileEncoding, forKey: .sourceFileEncoding) try container.encodeIfPresent(erDiagramSchemaKey, forKey: .erDiagramSchemaKey) try container.encodeIfPresent(objectRef, forKey: .objectRef) try container.encodeIfPresent(versionHistorySubject, forKey: .versionHistorySubject) @@ -196,6 +202,7 @@ internal struct EditorTabPayload: Codable, Hashable { self.initialFilterState = nil self.sourceFileURL = tab.content.sourceFileURL self.sourceFileStamp = nil + self.sourceFileEncoding = nil self.erDiagramSchemaKey = tab.display.erDiagramSchemaKey self.objectRef = tab.display.objectRef self.versionHistorySubject = tab.display.versionHistorySubject diff --git a/TablePro/Models/Query/FileTabBaseline.swift b/TablePro/Models/Query/FileTabBaseline.swift index f7debef478..aabd259c75 100644 --- a/TablePro/Models/Query/FileTabBaseline.swift +++ b/TablePro/Models/Query/FileTabBaseline.swift @@ -18,7 +18,7 @@ import Foundation internal enum FileTabBaseline { internal static func hydrate(_ tab: inout QueryTab) { guard let url = tab.content.sourceFileURL, let loaded = FileTextLoader.load(url) else { return } - record(loaded.content, stamp: loaded.stamp, in: &tab.content) + record(loaded.content, stamp: loaded.stamp, encoding: loaded.textEncoding, in: &tab.content) } internal static func hydrate(_ tabs: inout [QueryTab]) { @@ -28,16 +28,26 @@ internal enum FileTabBaseline { } internal static func adopt(_ loaded: FileTextLoader.LoadedText, into content: inout TabQueryContent) { - adopt(text: loaded.content, stamp: loaded.stamp, into: &content) + adopt(text: loaded.content, stamp: loaded.stamp, encoding: loaded.textEncoding, into: &content) } - internal static func adopt(text: String, stamp: FileStamp?, into content: inout TabQueryContent) { + internal static func adopt( + text: String, + stamp: FileStamp?, + encoding: FileTextEncoding?, + into content: inout TabQueryContent + ) { content.query = text - record(text, stamp: stamp, in: &content) + record(text, stamp: stamp, encoding: encoding, in: &content) } - internal static func recordWrite(of text: String, to url: URL, in content: inout TabQueryContent) { - record(text, stamp: FileStamp.read(url), in: &content) + internal static func recordWrite( + of text: String, + to url: URL, + as encoding: FileTextEncoding, + in content: inout TabQueryContent + ) { + record(text, stamp: FileStamp.read(url), encoding: encoding, in: &content) } internal static func diskChange(in content: TabQueryContent) -> SourceFileDiskChange? { @@ -69,9 +79,15 @@ internal enum FileTabBaseline { content.diskChange = nil } - private static func record(_ text: String, stamp: FileStamp?, in content: inout TabQueryContent) { + private static func record( + _ text: String, + stamp: FileStamp?, + encoding: FileTextEncoding?, + in content: inout TabQueryContent + ) { content.savedFileContent = text content.savedFileStamp = stamp + content.sourceFileEncoding = encoding content.diskChange = nil content.dismissedDiskChange = nil } diff --git a/TablePro/Models/Query/LinkedSQLFavorite.swift b/TablePro/Models/Query/LinkedSQLFavorite.swift index 6be2d54eda..d3ff54bbf8 100644 --- a/TablePro/Models/Query/LinkedSQLFavorite.swift +++ b/TablePro/Models/Query/LinkedSQLFavorite.swift @@ -22,6 +22,19 @@ internal struct LinkedSQLFavorite: Identifiable, Hashable { encodingName.lowercased() == "utf-8" } + var encodingDisplayName: String { + encoding.map(String.localizedName(of:)) ?? encodingName + } + + var encodingCannotRepresentEveryCharacter: Bool { + guard let encoding else { return !isUTF8 } + return !encoding.representsAllOfUnicode + } + + private var encoding: String.Encoding? { + String.Encoding(ianaCharacterSetName: encodingName) + } + init( folderId: UUID, fileURL: URL, diff --git a/TablePro/Models/Query/QueryTabManager.swift b/TablePro/Models/Query/QueryTabManager.swift index 5e99d37668..26d569a2c6 100644 --- a/TablePro/Models/Query/QueryTabManager.swift +++ b/TablePro/Models/Query/QueryTabManager.swift @@ -263,12 +263,18 @@ final class QueryTabManager: ObservableObject { databaseName: String = "", sourceFileURL: URL? = nil, sourceFileStamp: FileStamp? = nil, + sourceFileEncoding: FileTextEncoding? = nil, claimFocus: Bool = false ) { if let sourceFileURL, let existingIndex = tabs.firstIndex(where: { $0.content.sourceFileURL == sourceFileURL }) { if let query = initialQuery { - adoptReopenedFile(at: existingIndex, content: query, stamp: sourceFileStamp) + adoptReopenedFile( + at: existingIndex, + content: query, + stamp: sourceFileStamp, + encoding: sourceFileEncoding + ) } selectedTabId = tabs[existingIndex].id return @@ -292,7 +298,12 @@ final class QueryTabManager: ObservableObject { newTab.tableContext.databaseName = databaseName newTab.content.sourceFileURL = sourceFileURL if sourceFileURL != nil { - FileTabBaseline.adopt(text: newTab.content.query, stamp: sourceFileStamp, into: &newTab.content) + FileTabBaseline.adopt( + text: newTab.content.query, + stamp: sourceFileStamp, + encoding: sourceFileEncoding, + into: &newTab.content + ) } tabs.append(newTab) selectedTabId = newTab.id @@ -311,12 +322,12 @@ final class QueryTabManager: ObservableObject { /// The baseline moves with the buffer. Writing the text without it left the tab reading as /// dirty against content it had just loaded, and armed the same banner for a change it had /// already taken. - private func adoptReopenedFile(at index: Int, content: String, stamp: FileStamp?) { + private func adoptReopenedFile(at index: Int, content: String, stamp: FileStamp?, encoding: FileTextEncoding?) { /// An unknown baseline is not a licence to replace what the tab holds. `isFileDirty` reads a /// missing one as clean, so without this a tab that never learned what its file said would /// be overwritten by the very check meant to protect it. guard tabs[index].content.savedFileContent != nil, !tabs[index].content.isFileDirty else { return } - FileTabBaseline.adopt(text: content, stamp: stamp, into: &tabs[index].content) + FileTabBaseline.adopt(text: content, stamp: stamp, encoding: encoding, into: &tabs[index].content) } /// Take an already-built tab, such as one rebuilt from the recently closed history, rather than diff --git a/TablePro/Models/Query/QueryTabState.swift b/TablePro/Models/Query/QueryTabState.swift index 78c8daa90f..037435c831 100644 --- a/TablePro/Models/Query/QueryTabState.swift +++ b/TablePro/Models/Query/QueryTabState.swift @@ -629,6 +629,7 @@ struct TabQueryContent: Equatable { var sourceFileURL: URL? var savedFileContent: String? var savedFileStamp: FileStamp? + var sourceFileEncoding: FileTextEncoding? var diskChange: SourceFileDiskChange? var dismissedDiskChange: SourceFileDiskChange? @@ -641,6 +642,7 @@ struct TabQueryContent: Equatable { sourceFileURL: URL? = nil, savedFileContent: String? = nil, savedFileStamp: FileStamp? = nil, + sourceFileEncoding: FileTextEncoding? = nil, diskChange: SourceFileDiskChange? = nil, dismissedDiskChange: SourceFileDiskChange? = nil ) { @@ -650,6 +652,7 @@ struct TabQueryContent: Equatable { self.sourceFileURL = sourceFileURL self.savedFileContent = savedFileContent self.savedFileStamp = savedFileStamp + self.sourceFileEncoding = sourceFileEncoding self.diskChange = diskChange self.dismissedDiskChange = dismissedDiskChange } @@ -671,6 +674,7 @@ struct TabQueryContent: Equatable { && lhs.dismissedDiskChange == rhs.dismissedDiskChange && lhs.sourceFileURL == rhs.sourceFileURL && lhs.savedFileStamp == rhs.savedFileStamp + && lhs.sourceFileEncoding == rhs.sourceFileEncoding && lhs.queryParameters == rhs.queryParameters && (lhs.queryStorage === rhs.queryStorage || sameText(lhs.query, rhs.query)) && sameText(lhs.savedFileContent, rhs.savedFileContent) diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 2f5d4edb78..98afa992bc 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -1070,6 +1070,176 @@ } } }, + "Encoded as %@. Saving keeps this encoding, and a character it can't represent stops the save." : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "%@(으)로 인코딩됨. 저장해도 이 인코딩이 유지되며, 표현할 수 없는 문자가 있으면 저장이 중단됩니다." + } + }, + "tr" : { + "stringUnit" : { + "state" : "translated", + "value" : "%@ ile kodlanmış. Kaydetme bu kodlamayı korur; gösterilemeyen bir karakter kaydetmeyi durdurur." + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Mã hoá %@. Khi lưu, mã hoá này được giữ nguyên, và ký tự không biểu diễn được sẽ khiến việc lưu dừng lại." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "编码为 %@。存储时保留此编码，遇到无法表示的字符时会停止存储。" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "編碼為 %@。儲存時保留此編碼，遇到無法表示的字元時會停止儲存。" + } + } + } + }, + "“%1$@” is encoded as %2$@, which can't represent some of the text in its tab. The file was not changed. Use Save As to save the text as UTF-8." : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "“%1$@”은(는) %2$@(으)로 인코딩되어 있어 탭의 일부 텍스트를 나타낼 수 없습니다. 파일은 변경되지 않았습니다. 텍스트를 UTF-8로 저장하려면 다른 이름으로 저장을 사용하세요." + } + }, + "tr" : { + "stringUnit" : { + "state" : "translated", + "value" : "“%1$@” %2$@ ile kodlanmış ve bu kodlama sekmedeki metnin bir kısmını gösteremiyor. Dosya değiştirilmedi. Metni UTF-8 olarak kaydetmek için Farklı Kaydet'i kullanın." + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "“%1$@” được mã hoá bằng %2$@, không thể biểu diễn một phần văn bản trong tab. Tệp không bị thay đổi. Dùng Lưu thành để lưu văn bản dưới dạng UTF-8." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "“%1$@”的编码为 %2$@，无法表示标签页中的部分文本。文件未被更改。请使用“存储为”将文本存储为 UTF-8。" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "「%1$@」的編碼為 %2$@，無法表示標籤頁中的部分文字。檔案未被更改。請使用「儲存為」將文字儲存為 UTF-8。" + } + } + } + }, + "“%1$@” could not be written. %2$@" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "“%1$@”을(를) 쓸 수 없습니다. %2$@" + } + }, + "tr" : { + "stringUnit" : { + "state" : "translated", + "value" : "“%1$@” yazılamadı. %2$@" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Không thể ghi “%1$@”. %2$@" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "无法写入“%1$@”。%2$@" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "無法寫入「%1$@」。%2$@" + } + } + } + }, + "Couldn't Save Files" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "파일을 저장할 수 없음" + } + }, + "tr" : { + "stringUnit" : { + "state" : "translated", + "value" : "Dosyalar Kaydedilemedi" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Không thể lưu các tệp" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "无法保存这些文件" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "無法儲存這些檔案" + } + } + } + }, + "Couldn't Save File" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "파일을 저장할 수 없음" + } + }, + "tr" : { + "stringUnit" : { + "state" : "translated", + "value" : "Dosya Kaydedilemedi" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Không thể lưu tệp" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "无法保存文件" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "無法儲存檔案" + } + } + } + }, "\"%@\" was deleted or moved." : { "localizations" : { "ko" : { @@ -103198,40 +103368,6 @@ } } }, - "Non-UTF-8 file (%@). Saving may change the encoding." : { - "localizations" : { - "ko" : { - "stringUnit" : { - "state" : "translated", - "value" : "UTF-8이 아닌 파일(%@)입니다. 저장하면 인코딩이 변경될 수 있습니다." - } - }, - "tr" : { - "stringUnit" : { - "state" : "translated", - "value" : "UTF-8 olmayan dosya (%@). Kaydetmek kodlamayı değiştirebilir." - } - }, - "vi" : { - "stringUnit" : { - "state" : "translated", - "value" : "Tệp không phải UTF-8 (%@). Lưu có thể làm thay đổi bảng mã." - } - }, - "zh-Hans" : { - "stringUnit" : { - "state" : "translated", - "value" : "非 UTF-8 文件（%@）。保存可能会改变编码。" - } - }, - "zh-Hant" : { - "stringUnit" : { - "state" : "translated", - "value" : "非 UTF-8 檔案（%@）。儲存可能會改變編碼。" - } - } - } - }, "None" : { "localizations" : { "ko" : { diff --git a/TablePro/ViewModels/QueryPlanComparisonModel.swift b/TablePro/ViewModels/QueryPlanComparisonModel.swift index 45a3bbdbdc..9d98afbe5a 100644 --- a/TablePro/ViewModels/QueryPlanComparisonModel.swift +++ b/TablePro/ViewModels/QueryPlanComparisonModel.swift @@ -199,9 +199,7 @@ final class QueryPlanComparisonModel: ObservableObject { } } - /// Parsing and diffing are the only expensive part, and neither touches the model, so they run - /// off the main actor. `nonisolated async` is what moves them there; a detached task would - /// escape the model's isolation for no benefit. + @concurrent nonisolated private static func makeContent( baselineRawText: String, format: ExplainPlanFormat, diff --git a/TablePro/Views/Compare/StructureDefinitionDiffView.swift b/TablePro/Views/Compare/StructureDefinitionDiffView.swift index c18a9b8501..42d6152dd1 100644 --- a/TablePro/Views/Compare/StructureDefinitionDiffView.swift +++ b/TablePro/Views/Compare/StructureDefinitionDiffView.swift @@ -17,9 +17,10 @@ internal struct StructureDefinitionDiffView: View { internal let targetLines: [String] @State private var isUnified = false + @State private var presentation: StructureDefinitionDiffPresentation? - private var pairs: [DiffPair] { - DiffComputer.computeSplit(before: targetLines, after: sourceLines) + private var input: StructureDefinitionDiffInput { + StructureDefinitionDiffInput(sourceLines: sourceLines, targetLines: targetLines) } internal var body: some View { @@ -38,12 +39,29 @@ internal struct StructureDefinitionDiffView: View { } .padding(.bottom, 6) + diffBody + } + .task(id: input) { + let loaded = await StructureDefinitionDiffPresentation.load(input) + guard !Task.isCancelled else { return } + presentation = loaded + } + } + + @ViewBuilder + private var diffBody: some View { + if let presentation, presentation.isCurrent(for: input) { TextDiffView( - pairs: pairs, + pairs: presentation.pairs, beforeLabel: targetLabel, afterLabel: sourceLabel, layout: isUnified ? .unified : .split ) + } else { + ProgressView() + .controlSize(.small) + .frame(maxWidth: .infinity) + .padding(.vertical, 8) } } } diff --git a/TablePro/Views/Main/Extensions/MainContentCommandActions+FileSaving.swift b/TablePro/Views/Main/Extensions/MainContentCommandActions+FileSaving.swift index 8d874f88d9..e9e1a44be9 100644 --- a/TablePro/Views/Main/Extensions/MainContentCommandActions+FileSaving.swift +++ b/TablePro/Views/Main/Extensions/MainContentCommandActions+FileSaving.swift @@ -46,42 +46,90 @@ extension MainContentCommandActions { @discardableResult private func writeOrSaveAs(tabId: UUID, content: String, to url: URL) async -> Bool { - guard await writeTabContentAwaiting(tabId: tabId, content: content, to: url) else { - return await saveFileAsAwaiting() - } - return true - } - - /// The write itself, awaited and answering whether it landed. - /// - /// A batch close has to know: `writeTabContent` starts a detached Task and returns, so a caller - /// that closed the tab on its say-so closed it before the write had happened or failed. Its - /// failure path is the selected tab's Save As panel, which is also wrong for a victim that is - /// not the tab on screen, so the batch takes this instead and keeps a failed victim open. - @discardableResult - func writeTabContentAwaiting(tabId: UUID, content: String, to url: URL) async -> Bool { do { - try await SQLFileService.writeFile(content: content, to: url) - coordinator?.tabManager.mutate(tabId: tabId) { tab in - FileTabBaseline.recordWrite(of: content, to: url, in: &tab.content) - } + try await writeSourceFile(tabId: tabId, content: content, to: url) return true - } catch { - Self.fileLogger.error("Failed to save file: \(error.localizedDescription)") + } catch let error as FileTextWriter.WriteError { + Self.fileLogger.info( + "Save refused text the file's encoding cannot represent: \(url.lastPathComponent, privacy: .private(mask: .hash))" + ) + reportFileSaveFailures([Self.saveFailureMessage(for: error, fileName: url.lastPathComponent)]) return false + } catch { + Self.fileLogger.error("Failed to save file: \(error.publicLogShape, privacy: .public)") + return await saveFileAsAwaiting() } } - /// One victim's file, for a batch close. A file whose copy on disk moved under the tab is left + /// Each victim's file, for a batch close. A file whose copy on disk moved under the tab is left /// alone: resolving that needs the conflict sheet, which is a single window-level slot with no /// queue, so the batch keeps the tab open and the user answers it there. - func saveFile(of tab: QueryTab, to url: URL) async -> Bool { - if let change = FileTabBaseline.diskChange(in: tab.content) { - Self.fileLogger.info("Batch save skipped a file changed on disk: \(url.lastPathComponent, privacy: .private(mask: .hash))") - coordinator?.tabManager.mutate(tabId: tab.id) { FileTabBaseline.showDiskChange(change, in: &$0.content) } - return false + func saveFiles(_ targets: [(tab: QueryTab, url: URL)]) async -> Set { + var saved: Set = [] + var failures: [String] = [] + for target in targets { + if let change = FileTabBaseline.diskChange(in: target.tab.content) { + Self.fileLogger.info( + "Batch save skipped a file changed on disk: \(target.url.lastPathComponent, privacy: .private(mask: .hash))" + ) + coordinator?.tabManager.mutate(tabId: target.tab.id) { + FileTabBaseline.showDiskChange(change, in: &$0.content) + } + continue + } + do { + try await writeSourceFile(tabId: target.tab.id, content: target.tab.content.query, to: target.url) + saved.insert(target.tab.id) + } catch { + Self.fileLogger.error("Batch save failed to write a file: \(error.publicLogShape, privacy: .public)") + failures.append(Self.saveFailureMessage(for: error, fileName: target.url.lastPathComponent)) + } + } + reportFileSaveFailures(failures) + return saved + } + + func reportFileSaveFailures(_ messages: [String]) { + guard !messages.isEmpty else { return } + let title = messages.count == 1 + ? String(localized: "Couldn't Save File") + : String(localized: "Couldn't Save Files") + coordinator?.presentError(title, messages.joined(separator: "\n\n"), closeAnchorWindow) + } + + static func saveFailureMessage(for error: Error, fileName: String) -> String { + guard case FileTextWriter.WriteError.unrepresentable(let encoding) = error else { + return String( + format: String(localized: "“%1$@” could not be written. %2$@"), + fileName, + error.localizedDescription + ) + } + return String( + format: String( + localized: """ + “%1$@” is encoded as %2$@, which can't represent some of the text in its tab. \ + The file was not changed. Use Save As to save the text as UTF-8. + """ + ), + fileName, + encoding.displayName + ) + } + + private func writeSourceFile(tabId: UUID, content: String, to url: URL) async throws { + let encoding = await sourceFileEncoding(ofTab: tabId, at: url) + try await SQLFileService.writeFile(content: content, to: url, encoding: encoding) + coordinator?.tabManager.mutate(tabId: tabId) { tab in + FileTabBaseline.recordWrite(of: content, to: url, as: encoding, in: &tab.content) + } + } + + private func sourceFileEncoding(ofTab tabId: UUID, at url: URL) async -> FileTextEncoding { + if let recorded = coordinator?.tabManager.tabs.first(where: { $0.id == tabId })?.content.sourceFileEncoding { + return recorded } - return await writeTabContentAwaiting(tabId: tab.id, content: tab.content.query, to: url) + return await SQLFileService.encodingOnDisk(of: url) ?? .utf8 } private func requestConflictResolution(tab: QueryTab, url: URL) { diff --git a/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift b/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift index e495221a6a..c826920b1e 100644 --- a/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift +++ b/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift @@ -60,7 +60,8 @@ extension MainContentCoordinator { tabManager.addTab( initialQuery: loaded.content, sourceFileURL: favorite.fileURL, - sourceFileStamp: loaded.stamp + sourceFileStamp: loaded.stamp, + sourceFileEncoding: loaded.textEncoding ) registerWindowForSourceFile(favorite.fileURL) return @@ -87,7 +88,8 @@ extension MainContentCoordinator { databaseName: browseDatabaseName, initialQuery: loaded.content, sourceFileURL: favorite.fileURL, - sourceFileStamp: loaded.stamp + sourceFileStamp: loaded.stamp, + sourceFileEncoding: loaded.textEncoding ) WindowManager.shared.openTab(payload: payload) } diff --git a/TablePro/Views/Main/MainContentCommandActions+BulkClose.swift b/TablePro/Views/Main/MainContentCommandActions+BulkClose.swift index 2052d5d945..f15681f4c8 100644 --- a/TablePro/Views/Main/MainContentCommandActions+BulkClose.swift +++ b/TablePro/Views/Main/MainContentCommandActions+BulkClose.swift @@ -97,7 +97,7 @@ extension MainContentCommandActions { let everything = Set(victims.map(\.id)) guard hasUnsavedWork(among: victims) else { return .close(everything) } - switch await AlertHelper.confirmSaveChanges(message: unsavedWorkMessage(for: victims), window: closeAnchorWindow) { + switch await confirmSaveChanges(unsavedWorkMessage(for: victims), closeAnchorWindow) { case .save: return .close(await saveVictims(victims)) case .dontSave: @@ -150,6 +150,7 @@ extension MainContentCommandActions { guard await saveWorkOnScreen(among: victims) else { return [] } var closable: Set = [] + var files: [(tab: QueryTab, url: URL)] = [] for victim in victims where !coordinator.isSelectedTab(victim) { switch coordinator.savability(of: victim) { case .nothingAtRisk: @@ -161,9 +162,10 @@ extension MainContentCommandActions { closable.insert(victim.id) continue } - if await saveFile(of: victim, to: url) { closable.insert(victim.id) } + files.append((tab: victim, url: url)) } } + closable.formUnion(await saveFiles(files)) if let selected = coordinator.tabManager.selectedTab, victims.contains(where: { $0.id == selected.id }) { closable.insert(selected.id) } diff --git a/TablePro/Views/Main/MainContentCommandActions.swift b/TablePro/Views/Main/MainContentCommandActions.swift index ee11c3b9bd..26aa8c6fa7 100644 --- a/TablePro/Views/Main/MainContentCommandActions.swift +++ b/TablePro/Views/Main/MainContentCommandActions.swift @@ -1090,14 +1090,15 @@ final class MainContentCommandActions: ObservableObject { let tabId = tab.id guard let url = await chooseSaveURL(suggestedName) else { return false } do { - try await SQLFileService.writeFile(content: content, to: url) + try await SQLFileService.writeFile(content: content, to: url, encoding: .utf8) } catch { - Self.logger.error("Failed to save file: \(error.localizedDescription)") + Self.logger.error("Failed to save file: \(error.publicLogShape, privacy: .public)") + reportFileSaveFailures([Self.saveFailureMessage(for: error, fileName: url.lastPathComponent)]) return false } coordinator?.tabManager.mutate(tabId: tabId) { mutTab in mutTab.content.sourceFileURL = url - FileTabBaseline.recordWrite(of: content, to: url, in: &mutTab.content) + FileTabBaseline.recordWrite(of: content, to: url, as: .utf8, in: &mutTab.content) mutTab.title = url.deletingPathExtension().lastPathComponent } coordinator?.tabManager.markTabRenamed(tabId) @@ -1538,7 +1539,15 @@ final class MainContentCommandActions: ObservableObject { private func handleOpenSQLFiles(_ urls: [URL]) { Task { for url in urls { - try? await TabRouter.shared.route(.openSQLFile(url)) + do { + try await TabRouter.shared.route(.openSQLFile(url)) + } catch { + coordinator?.presentError( + String(localized: "Could Not Open File"), + error.localizedDescription, + closeAnchorWindow + ) + } } } } diff --git a/TablePro/Views/Shared/ObjectSourceView.swift b/TablePro/Views/Shared/ObjectSourceView.swift index 8e29bb9044..8f42b1f744 100644 --- a/TablePro/Views/Shared/ObjectSourceView.swift +++ b/TablePro/Views/Shared/ObjectSourceView.swift @@ -101,7 +101,7 @@ struct ObjectSourceView: View { private func export() async { guard let url = await SQLFileService.showSavePanel(suggestedName: exportFileName) else { return } do { - try await SQLFileService.writeFile(content: source, to: url) + try await SQLFileService.writeFile(content: source, to: url, encoding: .utf8) } catch { exportError = error.localizedDescription } diff --git a/TablePro/Views/Sidebar/LinkedFavoriteRowView.swift b/TablePro/Views/Sidebar/LinkedFavoriteRowView.swift index 021950634d..1c9f8cb11c 100644 --- a/TablePro/Views/Sidebar/LinkedFavoriteRowView.swift +++ b/TablePro/Views/Sidebar/LinkedFavoriteRowView.swift @@ -22,11 +22,16 @@ internal struct LinkedFavoriteRowView: View { Spacer() - if !favorite.isUTF8 { + if favorite.encodingCannotRepresentEveryCharacter { Image(systemName: "exclamationmark.triangle.fill") .font(.caption) .selectionAwareTint(.yellow) - .help(String(format: String(localized: "Non-UTF-8 file (%@). Saving may change the encoding."), favorite.encodingName)) + .help(String( + format: String( + localized: "Encoded as %@. Saving keeps this encoding, and a character it can't represent stops the save." + ), + favorite.encodingDisplayName + )) .accessibilityHidden(true) } @@ -56,7 +61,7 @@ internal struct LinkedFavoriteRowView: View { private var accessibilityDescription: String { var desc = favorite.name + ", " + String(localized: "linked file") if !favorite.isUTF8 { - desc += ", " + String(format: String(localized: "encoding: %@"), favorite.encodingName) + desc += ", " + String(format: String(localized: "encoding: %@"), favorite.encodingDisplayName) } if let keyword = favorite.keyword, !keyword.isEmpty { desc += ", " + String(format: String(localized: "keyword: %@"), keyword) diff --git a/TableProTests/Core/Diff/StructureDefinitionDiffPresentationTests.swift b/TableProTests/Core/Diff/StructureDefinitionDiffPresentationTests.swift new file mode 100644 index 0000000000..232568e2cc --- /dev/null +++ b/TableProTests/Core/Diff/StructureDefinitionDiffPresentationTests.swift @@ -0,0 +1,82 @@ +// +// StructureDefinitionDiffPresentationTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@Suite("StructureDefinitionDiffPresentation") +struct StructureDefinitionDiffPresentationTests { + @Test("the target is the before side and the source is the after side") + func targetIsBeforeAndSourceIsAfter() { + let presentation = StructureDefinitionDiffPresentation(input: StructureDefinitionDiffInput( + sourceLines: ["CREATE TABLE t (", " id INT,", " name TEXT", ")"], + targetLines: ["CREATE TABLE t (", " id BIGINT", ")"] + )) + + #expect(presentation.pairs == [ + DiffPair(before: "CREATE TABLE t (", after: "CREATE TABLE t (", kind: .unchanged), + DiffPair(before: " id BIGINT", after: " id INT,", kind: .changed), + DiffPair(before: nil, after: " name TEXT", kind: .added), + DiffPair(before: ")", after: ")", kind: .unchanged) + ]) + } + + @Test("a line only the target has is removed") + func targetOnlyLineIsRemoved() { + let presentation = StructureDefinitionDiffPresentation(input: StructureDefinitionDiffInput( + sourceLines: ["a", "c"], + targetLines: ["a", "b", "c"] + )) + + #expect(presentation.pairs == [ + DiffPair(before: "a", after: "a", kind: .unchanged), + DiffPair(before: "b", after: nil, kind: .removed), + DiffPair(before: "c", after: "c", kind: .unchanged) + ]) + } + + @Test("identical definitions pair every line as unchanged") + func identicalDefinitionsAreUnchanged() { + let lines = ["CREATE INDEX idx ON t (a)", "WHERE a > 0"] + let presentation = StructureDefinitionDiffPresentation(input: StructureDefinitionDiffInput( + sourceLines: lines, + targetLines: lines + )) + + #expect(presentation.pairs.allSatisfy { $0.kind == .unchanged }) + #expect(presentation.pairs.map(\.before) == lines) + #expect(presentation.pairs.map(\.after) == lines) + } + + @Test("two empty definitions have nothing to pair") + func emptyDefinitionsHaveNoPairs() { + let presentation = StructureDefinitionDiffPresentation(input: StructureDefinitionDiffInput( + sourceLines: [], + targetLines: [] + )) + + #expect(presentation.pairs.isEmpty) + } + + @Test("loading builds the same presentation as computing it in place") + func loadMatchesTheInPlacePresentation() async { + let input = StructureDefinitionDiffInput(sourceLines: ["a", "b"], targetLines: ["a", "c"]) + + let loaded = await StructureDefinitionDiffPresentation.load(input) + + #expect(loaded == StructureDefinitionDiffPresentation(input: input)) + } + + @Test("a presentation is current only for the definitions it was computed from") + func presentationIsCurrentOnlyForItsInput() { + let input = StructureDefinitionDiffInput(sourceLines: ["a"], targetLines: ["b"]) + let presentation = StructureDefinitionDiffPresentation(input: input) + + #expect(presentation.isCurrent(for: input)) + #expect(!presentation.isCurrent(for: StructureDefinitionDiffInput(sourceLines: ["a"], targetLines: ["c"]))) + #expect(!presentation.isCurrent(for: StructureDefinitionDiffInput(sourceLines: ["b"], targetLines: ["a"]))) + } +} diff --git a/TableProTests/Core/Git/GitIntegrationTests.swift b/TableProTests/Core/Git/GitIntegrationTests.swift index 050db63c03..6e50ba23bc 100644 --- a/TableProTests/Core/Git/GitIntegrationTests.swift +++ b/TableProTests/Core/Git/GitIntegrationTests.swift @@ -265,4 +265,49 @@ struct GitIntegrationTests { await #expect(throws: VersionHistoryError.fileChangedBeforeWriting) { try await plan.apply() } #expect(try repo.read("q.sql") == "edited\n") } + + @Test("A file whose encoding macOS recorded reads its past versions in that encoding and keeps it through a restore") + func restoreKeepsTheRecordedEncoding() async throws { + let repo = try ScratchRepository(client: client) + defer { repo.remove() } + let fixture = EncodedSQLFileFixture.shiftJISByAttribute + let url = repo.url("japanese.sql") + try await repo.git("init", "-q") + try fixture.write(fixture.original, to: url) + try await repo.git("add", "-A") + try await repo.git("commit", "-qm", "first") + try fixture.write(fixture.edited, to: url) + try await repo.git("commit", "-qam", "second") + let provider = LinkedFileVersionHistoryProvider(fileURL: url) + let first = try #require(try await provider.loadHistory().entries.last) + + #expect(try await provider.content(of: first.reference) == fixture.original) + + try await provider.prepareRestore(first.reference).apply() + + #expect(try Data(contentsOf: url) == fixture.bytes(of: fixture.original)) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == fixture.attributeValue) + let reloaded = try #require(FileTextLoader.load(url)) + #expect(reloaded.encoding == .shiftJIS) + #expect(reloaded.content == fixture.original) + } + + @Test("Discarding changes to a file whose encoding macOS recorded keeps that encoding") + func discardKeepsTheRecordedEncoding() async throws { + let repo = try ScratchRepository(client: client) + defer { repo.remove() } + let fixture = EncodedSQLFileFixture.windowsCyrillicByAttribute + let url = repo.url("cyrillic.sql") + try await repo.git("init", "-q") + try fixture.write(fixture.original, to: url) + try await repo.git("add", "-A") + try await repo.git("commit", "-qm", "first") + try fixture.write(fixture.edited, to: url) + + try await LinkedFileVersionHistoryProvider(fileURL: url).prepareDiscard().apply() + + #expect(try Data(contentsOf: url) == fixture.bytes(of: fixture.original)) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == fixture.attributeValue) + #expect(FileTextLoader.load(url)?.content == fixture.original) + } } diff --git a/TableProTests/Core/Git/GitOutputParserTests.swift b/TableProTests/Core/Git/GitOutputParserTests.swift index 5d289fe95d..5991315b47 100644 --- a/TableProTests/Core/Git/GitOutputParserTests.swift +++ b/TableProTests/Core/Git/GitOutputParserTests.swift @@ -307,4 +307,20 @@ struct FileTextLoaderDecodeTests { #expect(FileTextLoader.decode(Data([0x00, 0x00, 0xFE, 0xFF]) + utf32) == text) #expect(FileTextLoader.decode(Data()) == "") } + + @Test("A blob with no mark reads in the declared encoding, and a mark still outranks it") + func declaredEncoding() throws { + let text = "SELECT '\u{65E5}\u{672C}';" + let shiftJIS = try #require(text.data(using: .shiftJIS)) + let utf16 = try #require(text.data(using: .utf16BigEndian)) + #expect(FileTextLoader.decode(shiftJIS, declaredEncoding: .shiftJIS) == text) + #expect(FileTextLoader.decode(shiftJIS) != text) + #expect(FileTextLoader.decode(Data([0xFE, 0xFF]) + utf16, declaredEncoding: .shiftJIS) == text) + } + + @Test("An older version saved as UTF-8 reads as UTF-8 under a working copy recorded as Latin-1") + func utf8BlobOutranksASingleByteDeclaration() { + let text = "SELECT 'café';" + #expect(FileTextLoader.decode(Data(text.utf8), declaredEncoding: .isoLatin1) == text) + } } diff --git a/TableProTests/Core/Operations/AppSettingsCategoryParityTests.swift b/TableProTests/Core/Operations/AppSettingsCategoryParityTests.swift index b6f5a3d469..32ea661b4a 100644 --- a/TableProTests/Core/Operations/AppSettingsCategoryParityTests.swift +++ b/TableProTests/Core/Operations/AppSettingsCategoryParityTests.swift @@ -16,7 +16,7 @@ import Testing @Suite("AppSettingsCategory parity") struct AppSettingsCategoryParityTests { private static let managerSource = sourceFile("TablePro/Core/Storage/AppSettingsManager.swift") - private static let syncSource = sourceFile("TablePro/Core/Sync/SyncCoordinator.swift") + private static let syncSource = sourceFile("TablePro/Core/Sync/Extensions/SyncCoordinator+Settings.swift") @Test("Every synced category can be encoded and decoded") func syncedCategoriesRoundTrip() throws { diff --git a/TableProTests/Core/Services/Infrastructure/SQLFileOpeningTests.swift b/TableProTests/Core/Services/Infrastructure/SQLFileOpeningTests.swift new file mode 100644 index 0000000000..a7ca371932 --- /dev/null +++ b/TableProTests/Core/Services/Infrastructure/SQLFileOpeningTests.swift @@ -0,0 +1,70 @@ +// +// SQLFileOpeningTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@MainActor @Suite("Opening a SQL file from Finder or File > Open") +struct SQLFileOpeningTests { + private func makeFolder() throws -> URL { + let folder = FileManager.default.temporaryDirectory + .appendingPathComponent("sql-file-opening-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + return folder + } + + @Test("A file that is not UTF-8 opens with its text, its stamp and the encoding it was read in") + func opensANonUTF8FileWithItsEncoding() async throws { + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("japanese.sql") + let fixture = EncodedSQLFileFixture.shiftJISByAttribute + try fixture.write(fixture.original, to: url) + let connectionId = UUID() + + let payload = try await TabRouter.sqlFileTabPayload(for: url, connectionId: connectionId) + + #expect(payload.connectionId == connectionId) + #expect(payload.tabType == .query) + #expect(payload.sourceFileURL == url) + #expect(payload.initialQuery == fixture.original) + #expect(payload.sourceFileStamp == FileStamp.read(url)) + #expect(payload.sourceFileEncoding?.encoding == .shiftJIS) + #expect(payload.sourceFileEncoding?.attribute != nil) + } + + @Test("A file that cannot be read throws the read error instead of opening nothing") + func unreadableFileThrows() async throws { + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("locked.sql") + try Data("SELECT 1;\n".utf8).write(to: url) + try FileManager.default.setAttributes([.posixPermissions: 0o000], ofItemAtPath: url.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) } + + do { + _ = try await TabRouter.sqlFileTabPayload(for: url, connectionId: UUID()) + Issue.record("An unreadable file produced a tab") + } catch { + #expect((error as? CocoaError)?.code == .fileReadNoPermission) + #expect(error.localizedDescription.contains("locked.sql")) + } + } + + @Test("A file that is gone throws the read error") + func missingFileThrows() async throws { + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("gone.sql") + + do { + _ = try await TabRouter.sqlFileTabPayload(for: url, connectionId: UUID()) + Issue.record("A missing file produced a tab") + } catch { + #expect((error as? CocoaError)?.code == .fileReadNoSuchFile) + } + } +} diff --git a/TableProTests/Core/Services/PersistedTabRoundTripTests.swift b/TableProTests/Core/Services/PersistedTabRoundTripTests.swift index ebed65762d..f6892c5e43 100644 --- a/TableProTests/Core/Services/PersistedTabRoundTripTests.swift +++ b/TableProTests/Core/Services/PersistedTabRoundTripTests.swift @@ -533,7 +533,7 @@ struct PersistedTabRoundTripTests { kind: .trigger, name: "audit", database: "shop", schema: "public", table: "orders" ) ) - let json = String(decoding: try JSONEncoder().encode(tab), as: UTF8.self) + let json = try #require(String(data: try JSONEncoder().encode(tab), encoding: .utf8)) #expect(!json.contains("CREATE")) #expect(json.contains("orders")) } diff --git a/TableProTests/Core/Services/SQL/LinkedSQLFavoriteWriterTests.swift b/TableProTests/Core/Services/SQL/LinkedSQLFavoriteWriterTests.swift index 73e47a7aaa..0df7ea3566 100644 --- a/TableProTests/Core/Services/SQL/LinkedSQLFavoriteWriterTests.swift +++ b/TableProTests/Core/Services/SQL/LinkedSQLFavoriteWriterTests.swift @@ -250,4 +250,51 @@ struct LinkedSQLFavoriteWriterTests { let written = try Data(contentsOf: url) #expect(written == nameLine + Data("-- @keyword: revenue\nSELECT 1;\n".utf8)) } + + @Test( + "Editing a file's metadata keeps its encoding, byte order mark and encoding attribute", + arguments: EncodedSQLFileFixture.allCases + ) + func writeMetadataKeepsTheFileEncoding(fixture: EncodedSQLFileFixture) throws { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("linked-favorite-\(UUID().uuidString).sql") + defer { try? FileManager.default.removeItem(at: url) } + try fixture.write("-- @name: Old\n" + fixture.original, to: url) + + try LinkedSQLFavoriteWriter.writeMetadata( + Metadata(name: "New", keyword: "kw", description: nil), + to: url + ) + + let expected = try #require(fixture.bytes(of: "-- @name: New\n-- @keyword: kw\n" + fixture.original)) + #expect(try Data(contentsOf: url) == expected) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == fixture.attributeValue) + let reloaded = try #require(FileTextLoader.load(url)) + #expect(reloaded.encoding == fixture.reportedEncoding) + #expect(SQLFrontmatter.parse(reloaded.content) == Metadata(name: "New", keyword: "kw", description: nil)) + } + + @Test("Metadata the file's encoding cannot hold is refused with that encoding and the file is left alone") + func writeMetadataRefusesUnrepresentableText() throws { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("linked-favorite-\(UUID().uuidString).sql") + defer { try? FileManager.default.removeItem(at: url) } + let fixture = EncodedSQLFileFixture.windowsCyrillicByAttribute + try fixture.write("-- @name: Old\n" + fixture.original, to: url) + let before = try Data(contentsOf: url) + + do { + try LinkedSQLFavoriteWriter.writeMetadata( + Metadata(name: "\u{1F600}", keyword: nil, description: nil), + to: url + ) + Issue.record("A name the file's encoding cannot hold was written") + } catch LinkedSQLFavoriteWriter.WriteError.encodingMismatch(let encoding) { + #expect(encoding.encoding == .windowsCP1251) + #expect(encoding.displayName == String.localizedName(of: .windowsCP1251)) + } + + #expect(try Data(contentsOf: url) == before) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == fixture.attributeValue) + } } diff --git a/TableProTests/Core/Storage/FavoriteTablesStorageTests.swift b/TableProTests/Core/Storage/FavoriteTablesStorageTests.swift index ec3ed51e03..fe5b9649c7 100644 --- a/TableProTests/Core/Storage/FavoriteTablesStorageTests.swift +++ b/TableProTests/Core/Storage/FavoriteTablesStorageTests.swift @@ -4,6 +4,7 @@ import TableProSyncTransport import Testing @Suite("FavoriteTablesStorage") +@MainActor struct FavoriteTablesStorageTests { private func makeStorage() throws -> (FavoriteTablesStorage, SyncMetadataStorage) { let favoritesSuite = "FavoriteTablesStorageTests.favorites.\(UUID().uuidString)" diff --git a/TableProTests/Core/Storage/GroupStorageTests.swift b/TableProTests/Core/Storage/GroupStorageTests.swift index efc06fb1cf..2f404bff76 100644 --- a/TableProTests/Core/Storage/GroupStorageTests.swift +++ b/TableProTests/Core/Storage/GroupStorageTests.swift @@ -4,10 +4,10 @@ // import Combine -import TableProPluginKit @testable import TablePro -import XCTest +import TableProPluginKit import TableProSyncTransport +import XCTest @MainActor final class GroupStorageTests: XCTestCase { @@ -27,9 +27,9 @@ final class GroupStorageTests: XCTestCase { try await super.setUp() let unique = UUID().uuidString suiteName = "com.TablePro.tests.GroupStorage.\(unique)" - defaults = UserDefaults(suiteName: suiteName)! + defaults = try XCTUnwrap(UserDefaults(suiteName: suiteName)) syncSuiteName = "com.TablePro.tests.Sync.\(unique)" - syncDefaults = UserDefaults(suiteName: syncSuiteName)! + syncDefaults = try XCTUnwrap(UserDefaults(suiteName: syncSuiteName)) let metadata = SyncMetadataStorage(userDefaults: syncDefaults) tracker = SyncChangeTracker(metadataStorage: metadata) connectionFileURL = FileManager.default.temporaryDirectory @@ -359,8 +359,6 @@ final class GroupStorageTests: XCTestCase { XCTAssertEqual(storage.loadGroups().map(\.id), [b.id]) } - /// saveGroups marks every group dirty and the push uploads every dirty group, so writing a - /// record that changed nothing re-uploads the whole list to the device it came from. func testApplyingAnUnchangedRemoteGroupWritesNothing() throws { let group = ConnectionGroup(name: "Shared", color: .green) try storage.addGroup(group) diff --git a/TableProTests/Core/Storage/SQLFavoriteStorageOpenTests.swift b/TableProTests/Core/Storage/SQLFavoriteStorageOpenTests.swift new file mode 100644 index 0000000000..c8625bb37b --- /dev/null +++ b/TableProTests/Core/Storage/SQLFavoriteStorageOpenTests.swift @@ -0,0 +1,70 @@ +import Foundation +@testable import TablePro +import Testing + +@Suite("SQL favorite storage open and sync reads") +struct SQLFavoriteStorageOpenTests { + private let scratch: URL + + init() throws { + scratch = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("sql-favorite-open-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: scratch, withIntermediateDirectories: true) + } + + @Test("A store whose first open failed opens on the next access once the path is usable") + func reopensAfterAFailedOpen() async throws { + let blocker = scratch.appendingPathComponent("favorites-directory") + #expect(FileManager.default.createFile(atPath: blocker.path, contents: Data())) + let storage = SQLFavoriteStorage( + databaseURL: blocker.appendingPathComponent("sql_favorites.db"), + removeDatabaseOnDeinit: true + ) + let favorite = SQLFavorite(name: "Revenue", query: "SELECT 1") + + let addedWhileBlocked = await storage.addFavorite(favorite) + #expect(!addedWhileBlocked) + + try FileManager.default.removeItem(at: blocker) + + #expect(await storage.addFavorite(favorite)) + #expect(await storage.fetchFavorite(id: favorite.id)?.name == "Revenue") + } + + @Test("An empty store reads as no saved queries rather than as unreadable") + func emptyStoreReadsAsEmpty() async { + let storage = SQLFavoriteStorage( + databaseURL: scratch.appendingPathComponent("sql_favorites.db"), + removeDatabaseOnDeinit: true + ) + + #expect(await storage.readAllFavorites()?.isEmpty == true) + #expect(await storage.readAllFolders()?.isEmpty == true) + } + + @Test("A store read reports every saved query and folder") + func readsEverySavedQueryAndFolder() async { + let storage = SQLFavoriteStorage( + databaseURL: scratch.appendingPathComponent("sql_favorites.db"), + removeDatabaseOnDeinit: true + ) + let folder = SQLFavoriteFolder(name: "Reports") + let favorite = SQLFavorite(name: "Revenue", query: "SELECT 1", folderId: folder.id) + #expect(await storage.addFolder(folder)) + #expect(await storage.addFavorite(favorite)) + + #expect(await storage.readAllFavorites()?.map(\.id) == [favorite.id]) + #expect(await storage.readAllFolders()?.map(\.id) == [folder.id]) + } + + @Test("A file that is not a database reads as unreadable, not as an empty store") + func notADatabaseReadsAsUnreadable() async throws { + let url = scratch.appendingPathComponent("sql_favorites.db") + try Data(repeating: 0x2A, count: 4_096).write(to: url) + let storage = SQLFavoriteStorage(databaseURL: url, removeDatabaseOnDeinit: true) + + #expect(await storage.readAllFavorites() == nil) + #expect(await storage.readAllFolders() == nil) + } +} diff --git a/TableProTests/Core/Storage/SyncDirtyMarkingTests.swift b/TableProTests/Core/Storage/SyncDirtyMarkingTests.swift new file mode 100644 index 0000000000..5893f1ef6a --- /dev/null +++ b/TableProTests/Core/Storage/SyncDirtyMarkingTests.swift @@ -0,0 +1,173 @@ +import Foundation +import TableProSyncTransport +import Testing + +@testable import TablePro + +@Suite("Per-record sync marks") +@MainActor +struct SyncDirtyMarkingTests { + private let unique = UUID().uuidString + private let metadata: SyncMetadataStorage + private let tracker: SyncChangeTracker + private let keychain = InMemoryKeychain() + private let directory: URL + private let connections: ConnectionStorage + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("dirty-marking-\(unique)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + metadata = SyncMetadataStorage( + userDefaults: try #require(UserDefaults(suiteName: "com.TablePro.tests.DirtyMarking.sync.\(unique)")) + ) + tracker = SyncChangeTracker(metadataStorage: metadata) + connections = ConnectionStorage( + fileURL: directory.appendingPathComponent("connections.json"), + userDefaults: try #require(UserDefaults(suiteName: "com.TablePro.tests.DirtyMarking.conn.\(unique)")), + syncTracker: tracker, + keychain: keychain, + integrity: ConnectionStoreIntegrity(keySource: StoredIntegrityKeySource(store: keychain)) + ) + } + + private func defaults(_ name: String) throws -> UserDefaults { + try #require(UserDefaults(suiteName: "com.TablePro.tests.DirtyMarking.\(name).\(unique)")) + } + + @Test("Saving the tag library marks the tag that changed and no other") + func tagSaveMarksOnlyTheChangedTag() throws { + let storage = TagStorage(userDefaults: try defaults("tags"), syncTracker: tracker, appEvents: AppEvents()) + let renamed = ConnectionTag(name: "staging") + let untouched = ConnectionTag(name: "qa") + try storage.addTag(renamed) + try storage.addTag(untouched) + metadata.clearDirty(type: .tag) + + let edited = storage.loadTags().map { tag -> ConnectionTag in + guard tag.id == renamed.id else { return tag } + var changed = tag + changed.color = .purple + return changed + } + #expect(storage.saveTags(edited)) + + #expect(metadata.dirtyIds(for: .tag) == [renamed.id.uuidString]) + } + + @Test("Adding a tag marks the new tag alone") + func addingATagMarksOnlyTheNewTag() throws { + let storage = TagStorage(userDefaults: try defaults("tags-add"), syncTracker: tracker, appEvents: AppEvents()) + try storage.addTag(ConnectionTag(name: "staging")) + metadata.clearDirty(type: .tag) + let added = ConnectionTag(name: "qa") + + try storage.addTag(added) + + #expect(metadata.dirtyIds(for: .tag) == [added.id.uuidString]) + } + + @Test("Deleting a tag tombstones it and marks none of the others") + func deletingATagMarksNothingElse() throws { + let storage = TagStorage(userDefaults: try defaults("tags-delete"), syncTracker: tracker, appEvents: AppEvents()) + let doomed = ConnectionTag(name: "staging") + try storage.addTag(doomed) + try storage.addTag(ConnectionTag(name: "qa")) + metadata.clearDirty(type: .tag) + + storage.deleteTag(doomed) + + #expect(metadata.dirtyIds(for: .tag).isEmpty) + #expect(metadata.tombstones(for: .tag).map(\.id) == [doomed.id.uuidString]) + } + + @Test("Renaming a group marks that group alone") + func groupRenameMarksOnlyThatGroup() throws { + let storage = GroupStorage( + userDefaults: try defaults("groups"), + syncTracker: tracker, + connectionStorage: connections, + appEvents: AppEvents() + ) + let renamed = ConnectionGroup(name: "Production") + try storage.addGroup(renamed) + try storage.addGroup(ConnectionGroup(name: "Staging")) + metadata.clearDirty(type: .group) + + try storage.mutateGroup(id: renamed.id) { $0.name = "Prod" } + + #expect(metadata.dirtyIds(for: .group) == [renamed.id.uuidString]) + } + + @Test("Moving a group marks it for its new parent without marking its old siblings") + func groupMoveMarksTheMovedGroup() throws { + let storage = GroupStorage( + userDefaults: try defaults("groups-move"), + syncTracker: tracker, + connectionStorage: connections, + appEvents: AppEvents() + ) + let parent = ConnectionGroup(name: "Clients") + let moved = ConnectionGroup(name: "Acme") + let sibling = ConnectionGroup(name: "Internal") + try storage.addGroup(parent) + try storage.addGroup(moved) + try storage.addGroup(sibling) + metadata.clearDirty(type: .group) + + try storage.moveGroups([moved.id], toParent: parent.id, before: nil) + + #expect(metadata.dirtyIds(for: .group) == [moved.id.uuidString]) + } + + @Test("Editing one SSH profile marks that profile alone") + func sshProfileUpdateMarksOnlyThatProfile() throws { + let storage = SSHProfileStorage( + userDefaults: try defaults("ssh"), + keychain: keychain, + syncTracker: tracker, + connectionStorage: connections + ) + var edited = SSHProfile(name: "Bastion", host: "bastion.example.com", username: "deploy") + #expect(storage.addProfile(edited)) + #expect(storage.addProfile(SSHProfile(name: "Jump", host: "jump.example.com", username: "deploy"))) + metadata.clearDirty(type: .sshProfile) + + edited.host = "bastion2.example.com" + #expect(storage.updateProfile(edited)) + + #expect(metadata.dirtyIds(for: .sshProfile) == [edited.id.uuidString]) + } + + @Test("A remote removal of a table favorite this Mac no longer holds still drops its mark") + func remoteRemovalOfAMissingFavoriteDropsTheMark() throws { + let tables = FavoriteTablesStorage(userDefaults: try defaults("tables"), syncTracker: tracker) + let tableId = String(repeating: "a", count: 64) + tracker.markDirty(.tableFavorite, id: tableId) + + tables.removeFavoriteWithoutSync(id: tableId) + + #expect(metadata.dirtyIds(for: .tableFavorite).isEmpty) + } + + @Test("Editing one credential profile marks that profile alone") + func credentialProfileUpdateMarksOnlyThatProfile() throws { + let storage = CredentialProfileStorage( + fileURL: directory.appendingPathComponent("credentialProfiles.json"), + keychain: keychain, + syncTracker: tracker, + connectionStorage: connections, + integrity: ConnectionStoreIntegrity(keySource: StoredIntegrityKeySource(store: keychain)) + ) + #expect(storage.addProfile(CredentialProfile(name: "Reader", username: "reader"))) + #expect(storage.addProfile(CredentialProfile(name: "Writer", username: "writer"))) + metadata.clearDirty(type: .credentialProfile) + var edited = try #require(storage.loadProfiles().first { $0.name == "Reader" }) + + edited.username = "reporting" + #expect(storage.updateProfile(edited)) + + #expect(metadata.dirtyIds(for: .credentialProfile) == [edited.id.uuidString]) + } +} diff --git a/TableProTests/Core/Storage/TagStorageTests.swift b/TableProTests/Core/Storage/TagStorageTests.swift index 180d25eeb9..b59b826d63 100644 --- a/TableProTests/Core/Storage/TagStorageTests.swift +++ b/TableProTests/Core/Storage/TagStorageTests.swift @@ -113,8 +113,6 @@ final class TagStorageTests: XCTestCase { // MARK: - Remote Apply - /// saveTags marks every tag dirty and the push uploads every dirty tag, so writing a record - /// that changed nothing re-uploads the whole library to the device it came from. func testApplyingAnUnchangedRemoteTagWritesNothing() throws { let tag = customTag try storage.addTag(tag) diff --git a/TableProTests/Core/Sync/SyncChangeTrackerTests.swift b/TableProTests/Core/Sync/SyncChangeTrackerTests.swift index 660b6829cb..ccc21b84b6 100644 --- a/TableProTests/Core/Sync/SyncChangeTrackerTests.swift +++ b/TableProTests/Core/Sync/SyncChangeTrackerTests.swift @@ -105,4 +105,92 @@ struct SyncChangeTrackerTests { #expect(tracker.dirtyRecords(for: .connection) == ["x"]) #expect(tracker.dirtyRecords(for: .group) == ["y"]) } + + @Test("A record nobody touched after the snapshot clears against it") + func untouchedRecordClearsAgainstTheSnapshot() { + tracker.markDirty(.tag, id: "a") + let snapshot = tracker.editSnapshot() + + let cleared = tracker.clearDirty(SyncRecordIdentity(type: .tag, id: "a"), unlessEditedSince: snapshot) + + #expect(cleared) + #expect(tracker.dirtyRecords(for: .tag).isEmpty) + } + + @Test("A record edited after the snapshot stays dirty through the clear") + func editAfterTheSnapshotKeepsTheMark() { + tracker.markDirty(.tag, id: "a") + let snapshot = tracker.editSnapshot() + tracker.markDirty(.tag, id: "a") + + let cleared = tracker.clearDirty(SyncRecordIdentity(type: .tag, id: "a"), unlessEditedSince: snapshot) + + #expect(!cleared) + #expect(tracker.dirtyRecords(for: .tag) == ["a"]) + } + + @Test("A record cleared after its push is not an edit until it is marked again") + func clearedRecordIsNotAnEdit() { + let identity = SyncRecordIdentity(type: .tag, id: "a") + tracker.markDirty(.tag, id: "a") + let snapshot = tracker.editSnapshot() + + tracker.clearDirty(.tag, id: "a") + + #expect(!tracker.hasEdit(identity, since: snapshot)) + + tracker.markDirty(.tag, id: "a") + + #expect(tracker.hasEdit(identity, since: snapshot)) + } + + @Test("A mark discarded by a remote delete is not an edit") + func discardedRecordIsNotAnEdit() { + tracker.markDirty(.tag, id: "a") + let snapshot = tracker.editSnapshot() + + tracker.discardDirty(.tag, ids: ["a"]) + + #expect(!tracker.hasEdit(SyncRecordIdentity(type: .tag, id: "a"), since: snapshot)) + } + + @Test("A mark left over from an earlier launch clears unless it is edited again") + func markFromAnEarlierLaunchClears() { + metadata.markDirty("a", type: .tag) + let snapshot = tracker.editSnapshot() + + #expect(!tracker.hasEdit(SyncRecordIdentity(type: .tag, id: "a"), since: snapshot)) + + tracker.markDirty(.tag, id: "a") + + #expect(tracker.hasEdit(SyncRecordIdentity(type: .tag, id: "a"), since: snapshot)) + } + + @Test("A record first dirtied after the snapshot counts as edited") + func recordOutsideTheSnapshotCountsAsEdited() { + let snapshot = tracker.editSnapshot() + tracker.markDirty(.tag, id: "late") + + #expect(tracker.hasEdit(SyncRecordIdentity(type: .tag, id: "late"), since: snapshot)) + } + + @Test("A delete after the snapshot counts as an edit") + func deleteAfterTheSnapshotCountsAsAnEdit() { + tracker.markDirty(.group, id: "a") + let snapshot = tracker.editSnapshot() + tracker.markDeleted(.group, id: "a") + + #expect(tracker.hasEdit(SyncRecordIdentity(type: .group, id: "a"), since: snapshot)) + } + + @Test("A write made while a pull applies is not an edit") + func suppressedWriteIsNotAnEdit() { + tracker.markDirty(.tag, id: "a") + let snapshot = tracker.editSnapshot() + tracker.isSuppressed = true + tracker.markDirty(.tag, id: "a") + tracker.isSuppressed = false + + #expect(!tracker.hasEdit(SyncRecordIdentity(type: .tag, id: "a"), since: snapshot)) + } } diff --git a/TableProTests/Core/Sync/SyncCoordinatorEchoTests.swift b/TableProTests/Core/Sync/SyncCoordinatorEchoTests.swift new file mode 100644 index 0000000000..889315207a --- /dev/null +++ b/TableProTests/Core/Sync/SyncCoordinatorEchoTests.swift @@ -0,0 +1,555 @@ +import CloudKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +@MainActor +@Suite("Sync coordinator push and pull cycle") +struct SyncCoordinatorEchoTests { + private static let zoneID = CKRecordZone.ID( + zoneName: CloudKitSyncEngine.zoneName, + ownerName: CKCurrentUserDefaultName + ) + + private let unique = UUID().uuidString + private let keychain = InMemoryKeychain() + private let directory: URL + private let defaults: UserDefaults + private let metadata: SyncMetadataStorage + private let tracker: SyncChangeTracker + private let recordCache: SyncRecordCache + private let connections: ConnectionStorage + private let groups: GroupStorage + private let tags: TagStorage + private let favoriteDatabases: FavoriteDatabasesStorage + private let columnLayouts: FileColumnLayoutPersister + private let favorites: SQLFavoriteManager + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("sync-echo-\(unique)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defaults = try #require(UserDefaults(suiteName: "com.TablePro.tests.SyncEcho.\(unique)")) + metadata = SyncMetadataStorage( + userDefaults: try #require(UserDefaults(suiteName: "com.TablePro.tests.SyncEcho.sync.\(unique)")) + ) + tracker = SyncChangeTracker(metadataStorage: metadata) + recordCache = SyncRecordCache( + directory: directory.appendingPathComponent("SyncRecordCache", isDirectory: true), + defaults: nil + ) + let connectionStore = ConnectionStorage( + fileURL: directory.appendingPathComponent("connections.json"), + userDefaults: defaults, + syncTracker: tracker, + keychain: keychain, + integrity: ConnectionStoreIntegrity(keySource: StoredIntegrityKeySource(store: keychain)) + ) + connections = connectionStore + groups = GroupStorage( + userDefaults: defaults, + syncTracker: tracker, + connectionStorage: connectionStore, + appEvents: AppEvents() + ) + tags = TagStorage(userDefaults: defaults, syncTracker: tracker, appEvents: AppEvents()) + favoriteDatabases = FavoriteDatabasesStorage(defaults: defaults, syncTracker: tracker) + columnLayouts = FileColumnLayoutPersister( + storageDirectory: directory.appendingPathComponent("ColumnLayout", isDirectory: true), + defaults: defaults, + syncTracker: tracker + ) + favorites = SQLFavoriteManager( + storage: SQLFavoriteStorage( + databaseURL: directory.appendingPathComponent("sql_favorites.db"), + removeDatabaseOnDeinit: true + ), + syncTracker: tracker + ) + } + + @Test("A tag edited while its push is in flight keeps the edit, and a sibling's remote change still lands") + func tagEditedDuringPushSurvivesItsEcho() async throws { + let edited = ConnectionTag(name: "staging", color: .orange) + let sibling = ConnectionTag(name: "qa", color: .blue) + try tags.addTag(edited) + try tags.addTag(sibling) + var remoteSibling = sibling + remoteSibling.name = "quality" + let remoteSiblingRecord = SyncRecordMapper.toCKRecord(remoteSibling, in: Self.zoneID) + let tagStorage = tags + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + duringPush: { + let renamed = tagStorage.loadTags().map { tag -> ConnectionTag in + guard tag.id == edited.id else { return tag } + var changed = tag + changed.name = "stage" + return changed + } + tagStorage.saveTags(renamed) + }, + pulled: { pushed in + PullResult( + changedRecords: pushed.filter { $0.recordID != remoteSiblingRecord.recordID } + [remoteSiblingRecord], + deletedRecordIDs: [], + newToken: nil + ) + } + ) + + let failure = await makeCoordinator(transport: transport).runSyncCycle() + + #expect(failure == nil) + #expect(tags.tag(for: edited.id)?.name == "stage") + #expect(tracker.dirtyRecords(for: .tag).contains(edited.id.uuidString)) + #expect(tags.tag(for: sibling.id)?.name == "quality") + #expect(!tracker.dirtyRecords(for: .tag).contains(sibling.id.uuidString)) + } + + @Test("A saved query edited while its push is in flight keeps the edit and stays dirty") + func savedQueryEditedDuringPushSurvivesItsEcho() async throws { + let original = SQLFavorite(name: "Revenue", query: "SELECT 1") + #expect(await favorites.addFavorite(original)) + var edited = original + edited.query = "SELECT 2" + let manager = favorites + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + duringPush: { _ = await manager.updateFavorite(edited) }, + pulled: { pushed in PullResult(changedRecords: pushed, deletedRecordIDs: [], newToken: nil) } + ) + + let failure = await makeCoordinator(transport: transport).runSyncCycle() + + #expect(failure == nil) + #expect(await favorites.fetchFavorite(id: original.id)?.query == "SELECT 2") + #expect(tracker.dirtyRecords(for: .favorite).contains(original.id.uuidString)) + } + + @Test("A record pushed and left alone is clean after the cycle") + func untouchedPushedRecordIsClean() async throws { + let tag = ConnectionTag(name: "staging") + try tags.addTag(tag) + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + pulled: { pushed in PullResult(changedRecords: pushed, deletedRecordIDs: [], newToken: nil) } + ) + + let failure = await makeCoordinator(transport: transport).runSyncCycle() + + #expect(failure == nil) + #expect(tracker.dirtyRecords(for: .tag).isEmpty) + #expect(tags.tag(for: tag.id)?.name == "staging") + } + + @Test("A pull with no push before it applies over a dirty tag as it always has") + func pullWithoutPushApplies() async throws { + let tag = ConnectionTag(name: "staging") + try tags.addTag(tag) + var remote = tag + remote.name = "stage" + + let acknowledged = await makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + .applyPullResult( + PullResult( + changedRecords: [SyncRecordMapper.toCKRecord(remote, in: Self.zoneID)], + deletedRecordIDs: [], + newToken: nil + ) + ) + + #expect(acknowledged) + #expect(tags.tag(for: tag.id)?.name == "stage") + } + + @Test("A dirty tag the push could not save still takes the pull that follows") + func rejectedRecordStillTakesPulls() async throws { + let tag = ConnectionTag(name: "staging") + try tags.addTag(tag) + var remote = tag + remote.name = "stage" + let remoteRecord = SyncRecordMapper.toCKRecord(remote, in: Self.zoneID) + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + rejecting: [remoteRecord.recordID], + pulled: { _ in PullResult(changedRecords: [remoteRecord], deletedRecordIDs: [], newToken: nil) } + ) + + let failure = await makeCoordinator(transport: transport).runSyncCycle() + + #expect(failure != nil) + #expect(tags.tag(for: tag.id)?.name == "stage") + #expect(tracker.dirtyRecords(for: .tag).contains(tag.id.uuidString)) + } + + @Test("A local-only connection's database favorite, never pushed, still takes a pull") + func neverPushedDatabaseFavoriteTakesPulls() async throws { + var connection = TestFixtures.makeConnection() + connection.localOnly = true + connections.addConnection(connection) + favoriteDatabases.setFavorite(database: "orders", environment: .development, connectionId: connection.id) + let remote = FavoriteDatabaseEntry(connectionId: connection.id, database: "orders", environment: .production) + let remoteRecord = SyncRecordMapper.toCKRecord(favoriteDatabase: remote, in: Self.zoneID) + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + pulled: { _ in PullResult(changedRecords: [remoteRecord], deletedRecordIDs: [], newToken: nil) } + ) + + let failure = await makeCoordinator(transport: transport).runSyncCycle() + + #expect(failure == nil) + #expect(await transport.pushedRecords.isEmpty) + #expect(favoriteDatabases.favorites(for: connection.id).map(\.environment) == [.production]) + } + + @Test("A tag and a group deleted elsewhere take their local marks with them") + func remoteDeletionsDiscardMarks() async throws { + let tag = ConnectionTag(name: "staging") + try tags.addTag(tag) + let group = ConnectionGroup(name: "Clients") + try groups.addGroup(group) + let deletions = [ + SyncRecordMapper.recordID(type: .tag, id: tag.id.uuidString, in: Self.zoneID), + SyncRecordMapper.recordID(type: .group, id: group.id.uuidString, in: Self.zoneID) + ] + + let acknowledged = await makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + .applyPullResult(PullResult(changedRecords: [], deletedRecordIDs: deletions, newToken: nil)) + + #expect(acknowledged) + #expect(tags.tag(for: tag.id) == nil) + #expect(groups.group(for: group.id) == nil) + #expect(!tracker.dirtyRecords(for: .tag).contains(tag.id.uuidString)) + #expect(!tracker.dirtyRecords(for: .group).contains(group.id.uuidString)) + #expect(metadata.tombstones(for: .tag).isEmpty) + #expect(metadata.tombstones(for: .group).isEmpty) + } + + @Test("A connection edited during its push merges its echo and keeps both edits") + func connectionEchoMergesBothEdits() async throws { + var connection = TestFixtures.makeConnection(name: "Primary") + connection.port = 5_432 + connections.addConnection(connection) + let connectionId = connection.id + var remote = connection + remote.port = 6_543 + let remoteRecord = SyncRecordMapper.toCKRecord(remote, in: Self.zoneID) + let store = connections + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + duringPush: { _ = store.mutateConnections(ids: [connectionId]) { $0.name = "Renamed" } }, + pulled: { _ in PullResult(changedRecords: [remoteRecord], deletedRecordIDs: [], newToken: nil) } + ) + + let failure = await makeCoordinator(transport: transport).runSyncCycle() + + let merged = try #require(connections.loadConnection(id: connectionId)) + #expect(failure == nil) + #expect(merged.name == "Renamed") + #expect(merged.port == 6_543) + #expect(tracker.dirtyRecords(for: .connection).contains(connectionId.uuidString)) + } + + @Test("A record held back as this Mac's own echo is not cached as the server's copy") + func withheldEchoIsNotCached() async throws { + let tag = ConnectionTag(name: "staging") + try tags.addTag(tag) + var serverCopy = tag + serverCopy.name = "server" + let serverRecord = SyncRecordMapper.toCKRecord(serverCopy, in: Self.zoneID) + let tagStorage = tags + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + duringPush: { Self.rename(tag.id, to: "stage", in: tagStorage) }, + pulled: { _ in PullResult(changedRecords: [serverRecord], deletedRecordIDs: [], newToken: nil) } + ) + + let failure = await makeCoordinator(transport: transport).runSyncCycle() + + let cached = try #require(recordCache.record(for: serverRecord.recordID)) + #expect(failure == nil) + #expect(tags.tag(for: tag.id)?.name == "stage") + #expect(SyncRecordMapper.toTag(cached)?.name == "staging") + } + + @Test("A notification pull that arrives mid-push waits for the cycle and applies no echo") + func notificationDuringPushIsDeferred() async throws { + let tag = ConnectionTag(name: "staging") + try tags.addTag(tag) + let tagStorage = tags + let box = CoordinatorBox() + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + duringPush: { + Self.rename(tag.id, to: "stage", in: tagStorage) + await box.coordinator?.pullForRemoteNotification() + }, + pulled: { pushed in PullResult(changedRecords: pushed, deletedRecordIDs: [], newToken: nil) } + ) + let coordinator = makeCoordinator(transport: transport) + box.coordinator = coordinator + + let failure = await coordinator.runSyncCycle() + + #expect(failure == nil) + #expect(tags.tag(for: tag.id)?.name == "stage") + #expect(tracker.dirtyRecords(for: .tag).contains(tag.id.uuidString)) + #expect(await transport.pullCount == 2) + } + + @Test("A push cut short after saving some records still holds back their echoes and clears the rest") + func interruptedPushStillGuardsEchoes() async throws { + let tag = ConnectionTag(name: "staging") + try tags.addTag(tag) + let tagStorage = tags + let transport = ScriptedSyncTransport( + zoneID: Self.zoneID, + interruption: CKError(.networkFailure), + duringPush: { Self.rename(tag.id, to: "stage", in: tagStorage) }, + pulled: { pushed in PullResult(changedRecords: pushed, deletedRecordIDs: [], newToken: nil) } + ) + + let failure = await makeCoordinator(transport: transport).runSyncCycle() + + #expect(failure == .networkUnavailable) + #expect(tags.tag(for: tag.id)?.name == "stage") + #expect(tracker.dirtyRecords(for: .tag) == [tag.id.uuidString]) + } + + @Test("A dirty mark on a local-only connection is dropped rather than carried forever") + func localOnlyConnectionMarkIsDropped() async { + var connection = TestFixtures.makeConnection() + connection.localOnly = true + connections.addConnection(connection) + tracker.markDirty(.connection, id: connection.id.uuidString) + let transport = ScriptedSyncTransport(zoneID: Self.zoneID) + + let failure = await makeCoordinator(transport: transport).runSyncCycle() + + #expect(failure == nil) + #expect(await transport.pushedRecords.isEmpty) + #expect(!tracker.dirtyRecords(for: .connection).contains(connection.id.uuidString)) + } + + @Test("Marks with no record behind them are dropped once their stores read cleanly") + func unreachableMarksAreDropped() async { + let missingTag = UUID().uuidString + let missingGroup = UUID().uuidString + let missingConnection = UUID().uuidString + let missingQuery = UUID().uuidString + tracker.markDirty(.tag, id: missingTag) + tracker.markDirty(.group, id: missingGroup) + tracker.markDirty(.connection, id: missingConnection) + tracker.markDirty(.favorite, id: missingQuery) + + let failure = await makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)).runSyncCycle() + + #expect(failure == nil) + #expect(!tracker.dirtyRecords(for: .tag).contains(missingTag)) + #expect(!tracker.dirtyRecords(for: .group).contains(missingGroup)) + #expect(!tracker.dirtyRecords(for: .connection).contains(missingConnection)) + #expect(!tracker.dirtyRecords(for: .favorite).contains(missingQuery)) + } + + @Test("A saved query mark survives a store that cannot be read") + func unreadableStoreKeepsTheMark() async throws { + let url = directory.appendingPathComponent("not-a-database.db") + try Data(repeating: 0x2A, count: 4_096).write(to: url) + let broken = SQLFavoriteManager( + storage: SQLFavoriteStorage(databaseURL: url, removeDatabaseOnDeinit: true), + syncTracker: tracker + ) + let id = UUID().uuidString + tracker.markDirty(.favorite, id: id) + + _ = await makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID), favorites: broken) + .runSyncCycle() + + #expect(tracker.dirtyRecords(for: .favorite).contains(id)) + } + + @Test("A saved query edited while a pull applies keeps the edit and its mark, whichever runs first") + func localEditRacingARemoteApplyIsKept() async throws { + let original = SQLFavorite(name: "Revenue", query: "SELECT 1") + #expect(await favorites.addFavorite(original)) + let echoGuard = SyncEchoGuard( + snapshot: tracker.editSnapshot(), + savedRecords: [ + SyncRecordMapper.recordID(type: .favorite, id: original.id.uuidString, in: Self.zoneID): + SyncRecordIdentity(type: .favorite, id: original.id.uuidString) + ] + ) + var edited = original + edited.query = "SELECT 2" + + let manager = favorites + async let edit = manager.updateFavorite(edited) + async let apply = manager.applyRemote(RemoteSQLFavoriteBatch(favorites: [original]), echoGuard: echoGuard) + let (editSaved, applied) = await (edit, apply) + + #expect(editSaved) + #expect(applied != .failed) + #expect(await favorites.fetchFavorite(id: original.id)?.query == "SELECT 2") + #expect(tracker.dirtyRecords(for: .favorite).contains(original.id.uuidString)) + } + + @Test("A pull's copy of a saved query edited since the push is withheld") + func remoteApplyAfterALocalEditIsWithheld() async throws { + let original = SQLFavorite(name: "Revenue", query: "SELECT 1") + #expect(await favorites.addFavorite(original)) + let echoGuard = SyncEchoGuard( + snapshot: tracker.editSnapshot(), + savedRecords: [ + SyncRecordMapper.recordID(type: .favorite, id: original.id.uuidString, in: Self.zoneID): + SyncRecordIdentity(type: .favorite, id: original.id.uuidString) + ] + ) + var edited = original + edited.query = "SELECT 2" + #expect(await favorites.updateFavorite(edited)) + + let outcome = await favorites.applyRemote(RemoteSQLFavoriteBatch(favorites: [original]), echoGuard: echoGuard) + + #expect(outcome == .skipped) + #expect(await favorites.fetchFavorite(id: original.id)?.query == "SELECT 2") + #expect(tracker.dirtyRecords(for: .favorite).contains(original.id.uuidString)) + } + + private static func rename(_ id: UUID, to name: String, in storage: TagStorage) { + let renamed = storage.loadTags().map { tag -> ConnectionTag in + guard tag.id == id else { return tag } + var changed = tag + changed.name = name + return changed + } + storage.saveTags(renamed) + } + + private func makeCoordinator( + transport: ScriptedSyncTransport, + favorites: SQLFavoriteManager? = nil + ) -> SyncCoordinator { + let live = AppServices.live + let services = AppServices( + appEvents: AppEvents(), + appSettings: live.appSettings, + appSettingsStorage: AppSettingsStorage(userDefaults: defaults), + connectionStorage: connections, + databaseManager: live.databaseManager, + pluginManager: live.pluginManager, + schemaService: live.schemaService, + schemaRefreshService: live.schemaRefreshService, + schemaProviderRegistry: live.schemaProviderRegistry, + catalogChangeService: live.catalogChangeService, + sqlFavoriteManager: favorites ?? self.favorites, + favoriteTablesStorage: FavoriteTablesStorage(userDefaults: defaults, syncTracker: tracker), + favoriteDatabasesStorage: favoriteDatabases, + aiChatStorage: live.aiChatStorage, + aiKeyStorage: live.aiKeyStorage, + groupStorage: groups, + tagStorage: tags, + sshProfileStorage: SSHProfileStorage( + userDefaults: defaults, + keychain: keychain, + syncTracker: tracker, + connectionStorage: connections + ), + credentialProfileStorage: CredentialProfileStorage( + fileURL: directory.appendingPathComponent("credentialProfiles.json"), + keychain: keychain, + syncTracker: tracker, + connectionStorage: connections, + integrity: ConnectionStoreIntegrity(keySource: StoredIntegrityKeySource(store: keychain)) + ), + licenseManager: live.licenseManager, + syncMetadataStorage: metadata, + favoritesExpansionState: live.favoritesExpansionState, + linkedFolderWatcher: live.linkedFolderWatcher, + queryHistoryManager: live.queryHistoryManager, + dateFormattingService: live.dateFormattingService, + copilotService: live.copilotService, + mcpServerManager: live.mcpServerManager, + syncTracker: tracker, + themeEngine: live.themeEngine, + welcomeRouter: live.welcomeRouter + ) + return SyncCoordinator( + services: services, + recordCache: recordCache, + transport: transport, + columnLayouts: columnLayouts + ) + } +} + +@MainActor +private final class CoordinatorBox { + var coordinator: SyncCoordinator? +} + +private actor ScriptedSyncTransport: SyncTransport { + let currentZoneID: CKRecordZone.ID + private let rejectedRecordIDs: Set + private let interruption: (any Error)? + private let duringPush: @MainActor @Sendable () async -> Void + private let pulled: @Sendable ([CKRecord]) -> PullResult + private(set) var pushedRecords: [CKRecord] = [] + private(set) var pullCount = 0 + + init( + zoneID: CKRecordZone.ID, + rejecting rejectedRecordIDs: Set = [], + interruption: (any Error)? = nil, + duringPush: @escaping @MainActor @Sendable () async -> Void = {}, + pulled: @escaping @Sendable ([CKRecord]) -> PullResult = { _ in + PullResult(changedRecords: [], deletedRecordIDs: [], newToken: nil) + } + ) { + self.currentZoneID = zoneID + self.rejectedRecordIDs = rejectedRecordIDs + self.interruption = interruption + self.duringPush = duringPush + self.pulled = pulled + } + + func accountStatus() async throws -> CKAccountStatus { + .available + } + + func currentAccountId() async throws -> String { + "tests" + } + + func ensureZoneExists() async throws {} + + func push(records: [CKRecord], deletions: [CKRecord.ID]) async throws -> PushOutcome { + pushedRecords.append(contentsOf: records) + await duringPush() + var outcome = PushOutcome() + for record in records { + guard rejectedRecordIDs.contains(record.recordID) else { + outcome.recordSave(record) + continue + } + outcome.recordFailure( + SyncItemFailure(code: .serverRejectedRequest, serverRecord: nil, clientRecord: record, message: "Rejected"), + for: record.recordID + ) + } + for recordID in deletions { + outcome.recordDeletion(recordID) + } + if let interruption { + throw SyncPushInterruption(completed: outcome, cause: interruption) + } + return outcome + } + + func pull(since token: CKServerChangeToken?) async throws -> PullResult { + pullCount += 1 + return pulled(pushedRecords) + } +} diff --git a/TableProTests/Core/Sync/SyncScopeTests.swift b/TableProTests/Core/Sync/SyncScopeTests.swift index 65e4e1656f..52467aa5bc 100644 --- a/TableProTests/Core/Sync/SyncScopeTests.swift +++ b/TableProTests/Core/Sync/SyncScopeTests.swift @@ -5,10 +5,11 @@ import Foundation @testable import TablePro -import Testing import TableProSyncTransport +import Testing @Suite("Sync scope") +@MainActor struct SyncScopeTests { @Test("Every current record type is declared synced") func allCurrentTypesSync() { diff --git a/TableProTests/Core/Utilities/FileTextLoaderTests.swift b/TableProTests/Core/Utilities/FileTextLoaderTests.swift index 7d84a7e1b2..630c471687 100644 --- a/TableProTests/Core/Utilities/FileTextLoaderTests.swift +++ b/TableProTests/Core/Utilities/FileTextLoaderTests.swift @@ -159,4 +159,88 @@ struct FileTextLoaderTests { #expect(header == nil) } + + private func withFixture(_ fixture: EncodedSQLFileFixture, _ body: (URL) -> T) throws -> T { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("FileTextLoaderTests-\(UUID().uuidString).sql") + try fixture.write(fixture.original, to: url) + defer { try? FileManager.default.removeItem(at: url) } + return body(url) + } + + @Test( + "Loading records the encoding, the byte order mark, and the attribute that named the encoding", + arguments: EncodedSQLFileFixture.allCases + ) + func recordsHowTheFileWasRead(fixture: EncodedSQLFileFixture) throws { + let expectedMark = ByteOrderMark.leading(try #require(fixture.bytes(of: ""))) + + let loaded = try #require(try withFixture(fixture) { FileTextLoader.load($0) }) + + #expect(loaded.content == fixture.original) + #expect(loaded.encoding == fixture.reportedEncoding) + #expect(loaded.textEncoding.byteOrderMark == expectedMark) + #expect(loaded.textEncoding.attribute.flatMap { String(data: $0.value, encoding: .utf8) } == fixture.attributeValue) + } + + @Test( + "The header is read the way the whole file is, the encoding attribute included", + arguments: EncodedSQLFileFixture.allCases + ) + func headerAgreesWithTheWholeFile(fixture: EncodedSQLFileFixture) throws { + let loaded = try withFixture(fixture) { url in + (header: FileTextLoader.loadHeader(url), whole: FileTextLoader.load(url)) + } + + let header = try #require(loaded.header) + let whole = try #require(loaded.whole) + #expect(header.textEncoding == whole.textEncoding) + #expect(header.content == whole.content) + } + + @Test("A UTF-8 byte order mark outranks a text encoding attribute and is recorded") + func utf8ByteOrderMarkOutranksTheEncodingAttribute() throws { + let text = "SELECT 'Caf\u{E9}';\n" + let bytes = Data([0xEF, 0xBB, 0xBF]) + Data(text.utf8) + let attribute = Array("MACINTOSH;0".utf8) + + let loaded = try withFile(bytes) { url in + _ = setxattr(url.path, "com.apple.TextEncoding", attribute, attribute.count, 0, 0) + return FileTextLoader.load(url) + } + + let result = try #require(loaded) + #expect(result.content == text) + #expect(result.textEncoding == FileTextEncoding(encoding: .utf8, byteOrderMark: .utf8)) + #expect(result.textEncoding.attribute == nil) + } + + @Test("A text encoding attribute that cannot decode the file is not kept") + func dropsAnAttributeThatDidNotDecodeTheFile() throws { + let bytes = Data([0x53, 0x81, 0x20, 0xFF, 0x0A]) + let attribute = Array("SHIFT_JIS;2561".utf8) + + let loaded = try withFile(bytes) { url in + _ = setxattr(url.path, "com.apple.TextEncoding", attribute, attribute.count, 0, 0) + return FileTextLoader.load(url) + } + + let result = try #require(loaded) + #expect(result.encoding == .isoLatin1) + #expect(result.textEncoding.attribute == nil) + } + + @Test("A file recorded as ASCII is read as UTF-8, and the ASCII record is not kept") + func readsAnASCIIRecordAsUTF8() throws { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("FileTextLoaderTests-\(UUID().uuidString).sql") + defer { try? FileManager.default.removeItem(at: url) } + try "SELECT 1;\n".write(to: url, atomically: true, encoding: .ascii) + + let loaded = try #require(FileTextLoader.load(url)) + let header = try #require(FileTextLoader.loadHeader(url)) + + #expect(loaded.textEncoding == .utf8) + #expect(header.textEncoding == .utf8) + } } diff --git a/TableProTests/Core/Utilities/FileTextWriterTests.swift b/TableProTests/Core/Utilities/FileTextWriterTests.swift new file mode 100644 index 0000000000..c68a7458ac --- /dev/null +++ b/TableProTests/Core/Utilities/FileTextWriterTests.swift @@ -0,0 +1,158 @@ +// +// FileTextWriterTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@Suite("File text writer") +struct FileTextWriterTests { + private struct RefusedAttribute: Error {} + + private func makeFolder() throws -> URL { + let folder = FileManager.default.temporaryDirectory + .appendingPathComponent("file-text-writer-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + return folder + } + + private func permissions(of url: URL) throws -> Int { + let attributes = try FileManager.default.attributesOfItem(atPath: url.path) + return try #require(attributes[.posixPermissions] as? Int) + } + + @Test("Replacing a file keeps its permissions") + func keepsPermissions() throws { + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("report.sql") + try Data("SELECT 1;\n".utf8).write(to: url) + try FileManager.default.setAttributes([.posixPermissions: 0o640], ofItemAtPath: url.path) + + try FileTextWriter.write("SELECT 2;\n", to: url, as: .utf8) + + #expect(try Data(contentsOf: url) == Data("SELECT 2;\n".utf8)) + #expect(try permissions(of: url) == 0o640) + } + + @Test("An encoding attribute that cannot be set leaves the original file and nothing else behind") + func attributeFailureLeavesTheOriginal() throws { + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("japanese.sql") + let fixture = EncodedSQLFileFixture.shiftJISByAttribute + try fixture.write(fixture.original, to: url) + let before = try Data(contentsOf: url) + let loaded = try #require(FileTextLoader.load(url)) + let replacement = try #require(fixture.bytes(of: fixture.edited)) + + #expect(throws: (any Error).self) { + try FileTextWriter.replaceContents( + of: url, + with: replacement, + attribute: loaded.textEncoding.attribute, + applyingAttribute: { _, _ in throw RefusedAttribute() } + ) + } + + #expect(try Data(contentsOf: url) == before) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == fixture.attributeValue) + #expect(try FileManager.default.contentsOfDirectory(atPath: folder.path) == ["japanese.sql"]) + } + + @Test("A read-only file that carries an encoding attribute still saves, read-only and tagged") + func readOnlyTaggedFileSaves() throws { + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("japanese.sql") + let fixture = EncodedSQLFileFixture.shiftJISByAttribute + try fixture.write(fixture.original, to: url) + try FileManager.default.setAttributes([.posixPermissions: 0o444], ofItemAtPath: url.path) + let loaded = try #require(FileTextLoader.load(url)) + + try FileTextWriter.write(fixture.edited, to: url, as: loaded.textEncoding) + + #expect(try Data(contentsOf: url) == fixture.bytes(of: fixture.edited)) + #expect(try permissions(of: url) == 0o444) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == fixture.attributeValue) + } + + @Test("A locked file is reported as a permission failure") + func lockedFileIsAPermissionFailure() throws { + let folder = try makeFolder() + let url = folder.appendingPathComponent("report.sql") + try Data("SELECT 1;\n".utf8).write(to: url) + try FileManager.default.setAttributes([.immutable: true], ofItemAtPath: url.path) + defer { + try? FileManager.default.setAttributes([.immutable: false], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: folder) + } + + do { + try FileTextWriter.write("SELECT 2;\n", to: url, as: .utf8) + Issue.record("A locked file took the save") + } catch { + #expect((error as? CocoaError)?.code == .fileWriteNoPermission) + } + + #expect(try Data(contentsOf: url) == Data("SELECT 1;\n".utf8)) + } + + @Test("A file reached through a symbolic link is written through the link, which stays a link") + func writesThroughASymbolicLink() throws { + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let target = folder.appendingPathComponent("target.sql") + let link = folder.appendingPathComponent("link.sql") + try Data("SELECT 1;\n".utf8).write(to: target) + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: target) + + try FileTextWriter.write("SELECT 2;\n", to: link, as: .utf8) + + let linkType = try FileManager.default.attributesOfItem(atPath: link.path)[.type] as? FileAttributeType + #expect(linkType == .typeSymbolicLink) + #expect(try Data(contentsOf: target) == Data("SELECT 2;\n".utf8)) + } + + @Test("A folder that refuses the write is reported against the file being saved") + func refusedWriteNamesTheFile() throws { + let folder = try makeFolder() + let url = folder.appendingPathComponent("report.sql") + try Data("SELECT 1;\n".utf8).write(to: url) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + try? FileManager.default.removeItem(at: folder) + } + + do { + try FileTextWriter.write("SELECT 2;\n", to: url, as: .utf8) + Issue.record("A folder without write permission took the save") + } catch { + #expect((error as? CocoaError)?.code == .fileWriteNoPermission) + #expect(error.localizedDescription.contains("report.sql")) + } + + #expect(try Data(contentsOf: url) == Data("SELECT 1;\n".utf8)) + } + + @Test("Bytes written over a file, as a version restore does, keep its encoding attribute") + func writtenBytesKeepTheEncodingAttribute() async throws { + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("japanese.sql") + let fixture = EncodedSQLFileFixture.shiftJISByAttribute + try fixture.write(fixture.edited, to: url) + let restored = try #require(fixture.bytes(of: fixture.original)) + + try await SQLFileService.writeData(restored, to: url) + + #expect(try Data(contentsOf: url) == restored) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == fixture.attributeValue) + let reloaded = try #require(FileTextLoader.load(url)) + #expect(reloaded.encoding == .shiftJIS) + #expect(reloaded.content == fixture.original) + } +} diff --git a/TableProTests/Core/Utilities/MainActorSerialQueueTests.swift b/TableProTests/Core/Utilities/MainActorSerialQueueTests.swift new file mode 100644 index 0000000000..1c7ccb5fee --- /dev/null +++ b/TableProTests/Core/Utilities/MainActorSerialQueueTests.swift @@ -0,0 +1,58 @@ +import Foundation +@testable import TablePro +import Testing + +@Suite("Main actor serial queue") +@MainActor +struct MainActorSerialQueueTests { + @MainActor + private final class EventLog { + var events: [String] = [] + } + + @Test("An operation queued behind a suspended one starts only after it finishes") + func queuedOperationWaitsForTheRunningOne() async { + let queue = MainActorSerialQueue() + let log = EventLog() + let (started, signalStarted) = AsyncStream.makeStream(of: Void.self) + let (gate, openGate) = AsyncStream.makeStream(of: Void.self) + let (queued, signalQueued) = AsyncStream.makeStream(of: Void.self) + + let first = Task { @MainActor in + await queue.run { + log.events.append("first started") + signalStarted.yield() + for await _ in gate { break } + log.events.append("first finished") + } + } + for await _ in started { break } + + let second = Task { @MainActor in + signalQueued.yield() + await queue.run { + log.events.append("second") + } + } + for await _ in queued { break } + + #expect(log.events == ["first started"]) + + openGate.yield() + await first.value + await second.value + + #expect(log.events == ["first started", "first finished", "second"]) + } + + @Test("Each operation hands back its own result") + func operationsReturnTheirResults() async { + let queue = MainActorSerialQueue() + + async let first = queue.run { 1 } + async let second = queue.run { "two" } + + #expect(await first == 1) + #expect(await second == "two") + } +} diff --git a/TableProTests/Core/Utilities/Text/ByteOrderMarkTests.swift b/TableProTests/Core/Utilities/Text/ByteOrderMarkTests.swift index 6cf6c447ed..779a6f99a7 100644 --- a/TableProTests/Core/Utilities/Text/ByteOrderMarkTests.swift +++ b/TableProTests/Core/Utilities/Text/ByteOrderMarkTests.swift @@ -15,7 +15,8 @@ struct ByteOrderMarkTests { ([0xFF, 0xFE, 0x00, 0x00], .utf32LittleEndian), ([0x00, 0x00, 0xFE, 0xFF], .utf32BigEndian), ([0xFF, 0xFE], .utf16LittleEndian), - ([0xFE, 0xFF], .utf16BigEndian) + ([0xFE, 0xFF], .utf16BigEndian), + ([0xEF, 0xBB, 0xBF], .utf8) ] for (bytes, mark) in marks { @@ -34,7 +35,7 @@ struct ByteOrderMarkTests { #expect(mark == .utf32LittleEndian) #expect(mark?.encoding == .utf32) #expect(mark?.byteOrderedEncoding == .utf32LittleEndian) - #expect(mark?.codeUnitLength == 4) + #expect(mark?.byteOrderedEncoding.codeUnitLength == 4) } @Test("A declared encoding only takes the marks it can carry") @@ -50,12 +51,14 @@ struct ByteOrderMarkTests { #expect(ByteOrderMark.leading(Data([0xFE, 0xFF, 0x00, 0x61]), allowedBy: .utf32) == nil) } - @Test("A UTF-8 mark is left to Foundation, which drops it itself") - func leavesTheUTF8MarkAlone() { + @Test("A UTF-8 mark is recognised so a save can write it back, and decoded text never carries it") + func recognisesTheUTF8Mark() { let bytes = Data([0xEF, 0xBB, 0xBF]) + Data("SELECT 1;".utf8) - #expect(ByteOrderMark.leading(bytes) == nil) - #expect(ByteOrderMark.leading(bytes, allowedBy: .utf8) == nil) + #expect(ByteOrderMark.leading(bytes) == .utf8) + #expect(ByteOrderMark.leading(bytes, allowedBy: .utf8) == .utf8) + #expect(ByteOrderMark.leading(bytes, allowedBy: .isoLatin1) == nil) + #expect(ByteOrderMark.utf8.byteOrderedEncoding == .utf8) #expect(String(data: bytes, encoding: .utf8) == "SELECT 1;") } diff --git a/TableProTests/Core/Utilities/TextPrefixDecoderTests.swift b/TableProTests/Core/Utilities/TextPrefixDecoderTests.swift index 3af15e7dce..1ad30f5e8e 100644 --- a/TableProTests/Core/Utilities/TextPrefixDecoderTests.swift +++ b/TableProTests/Core/Utilities/TextPrefixDecoderTests.swift @@ -178,4 +178,40 @@ struct TextPrefixDecoderTests { func returnsNothingForNoBytes() { #expect(TextPrefixDecoder.decode(Data(), prefixLength: 4_096) == nil) } + + @Test("A declared encoding reads text with no mark, and a Shift JIS character across the limit survives") + func readsTheDeclaredEncoding() throws { + var bytes = Data(repeating: UInt8(ascii: "a"), count: Self.prefixLength - 1) + bytes.append(try #require("\u{65E5}\u{672C}".data(using: .shiftJIS))) + + let decoded = try #require( + TextPrefixDecoder.decode(bytes, prefixLength: Self.prefixLength, declaredEncoding: .shiftJIS) + ) + + #expect(decoded.encoding == .shiftJIS) + #expect(decoded.byteOrderMark == nil) + #expect(decoded.content.hasSuffix("\u{65E5}")) + } + + @Test("A byte order mark outranks a declared encoding and is reported") + func byteOrderMarkOutranksTheDeclaredEncoding() throws { + let bytes = Data([0xFE, 0xFF]) + (try #require("SELECT 1;".data(using: .utf16BigEndian))) + + let decoded = try #require(TextPrefixDecoder.decode(bytes, prefixLength: 4_096, declaredEncoding: .macOSRoman)) + + #expect(decoded.encoding == .utf16) + #expect(decoded.byteOrderMark == .utf16BigEndian) + #expect(decoded.content == "SELECT 1;") + } + + @Test("Text that falls back to ISO Latin-1 reports no mark, since the mark's bytes are now part of the text") + func latin1FallbackReportsNoMark() throws { + let bytes = Data([0xEF, 0xBB, 0xBF, 0x53, 0xFF]) + + let decoded = try #require(TextPrefixDecoder.decode(bytes, prefixLength: 4_096)) + + #expect(decoded.encoding == .isoLatin1) + #expect(decoded.byteOrderMark == nil) + #expect(decoded.content == String(data: bytes, encoding: .isoLatin1)) + } } diff --git a/TableProTests/Helpers/EncodedSQLFileFixture.swift b/TableProTests/Helpers/EncodedSQLFileFixture.swift new file mode 100644 index 0000000000..57d60b18f1 --- /dev/null +++ b/TableProTests/Helpers/EncodedSQLFileFixture.swift @@ -0,0 +1,117 @@ +// +// EncodedSQLFileFixture.swift +// TableProTests +// + +import Foundation + +internal enum EncodedSQLFileFixture: String, CaseIterable, Sendable { + case utf8 + case utf8WithByteOrderMark + case utf16LittleEndianWithByteOrderMark + case utf16BigEndianWithByteOrderMark + case utf32LittleEndianWithByteOrderMark + case utf32BigEndianWithByteOrderMark + case shiftJISByAttribute + case windowsCyrillicByAttribute + case macRomanByAttribute + case utf16ByAttributeWithoutByteOrderMark + case latin1Fallback + + private static let attributeName = "com.apple.TextEncoding" + private static let unicodeOriginal = "SELECT '\u{65E5}\u{672C} Caf\u{E9}';\n" + private static let unicodeEdited = "SELECT '\u{65E5}\u{672C} Caf\u{E9} \u{1F600}';\n" + + var original: String { + switch self { + case .shiftJISByAttribute: return "SELECT '\u{65E5}\u{672C}';\n" + case .windowsCyrillicByAttribute: return "SELECT '\u{0416}\u{0438}';\n" + case .macRomanByAttribute: return "SELECT 'Caf\u{E9} \u{2022}';\n" + case .latin1Fallback: return "SELECT 'Caf\u{E9}';\n" + default: return Self.unicodeOriginal + } + } + + var edited: String { + switch self { + case .shiftJISByAttribute: return "SELECT '\u{65E5}\u{672C}\u{8A9E}';\n" + case .windowsCyrillicByAttribute: return "SELECT '\u{0416}\u{0438}\u{0437}\u{043D}\u{044C}';\n" + case .macRomanByAttribute: return "SELECT 'Caf\u{E9} \u{2022} \u{2206}';\n" + case .latin1Fallback: return "SELECT 'Caf\u{E9} cr\u{E8}me';\n" + default: return Self.unicodeEdited + } + } + + var reportedEncoding: String.Encoding { + switch self { + case .utf8, .utf8WithByteOrderMark: return .utf8 + case .utf16LittleEndianWithByteOrderMark, .utf16BigEndianWithByteOrderMark: return .utf16 + case .utf32LittleEndianWithByteOrderMark, .utf32BigEndianWithByteOrderMark: return .utf32 + case .shiftJISByAttribute: return .shiftJIS + case .windowsCyrillicByAttribute: return .windowsCP1251 + case .macRomanByAttribute: return .macOSRoman + case .utf16ByAttributeWithoutByteOrderMark: return .utf16 + case .latin1Fallback: return .isoLatin1 + } + } + + var attributeValue: String? { + switch self { + case .shiftJISByAttribute: return "cp932;1056" + case .windowsCyrillicByAttribute: return "windows-1251;1282" + case .macRomanByAttribute: return "MACINTOSH;0" + case .utf16ByAttributeWithoutByteOrderMark: return "utf-16;256" + default: return nil + } + } + + func bytes(of text: String) -> Data? { + guard let body = text.data(using: bodyEncoding) else { return nil } + return Data(byteOrderMark) + body + } + + func write(_ text: String, to url: URL) throws { + guard let bytes = bytes(of: text) else { + throw CocoaError(.fileWriteInapplicableStringEncoding) + } + try bytes.write(to: url) + guard let attributeValue else { return } + let value = Array(attributeValue.utf8) + guard setxattr(url.path, Self.attributeName, value, value.count, 0, 0) == 0 else { + throw POSIXError(.EIO) + } + } + + static func attributeValue(of url: URL) -> String? { + let size = getxattr(url.path, attributeName, nil, 0, 0, 0) + guard size > 0 else { return nil } + var value = [UInt8](repeating: 0, count: size) + guard getxattr(url.path, attributeName, &value, size, 0, 0) == size else { return nil } + return String(bytes: value, encoding: .utf8) + } + + private var byteOrderMark: [UInt8] { + switch self { + case .utf8WithByteOrderMark: return [0xEF, 0xBB, 0xBF] + case .utf16LittleEndianWithByteOrderMark: return [0xFF, 0xFE] + case .utf16BigEndianWithByteOrderMark: return [0xFE, 0xFF] + case .utf32LittleEndianWithByteOrderMark: return [0xFF, 0xFE, 0x00, 0x00] + case .utf32BigEndianWithByteOrderMark: return [0x00, 0x00, 0xFE, 0xFF] + default: return [] + } + } + + private var bodyEncoding: String.Encoding { + switch self { + case .utf8, .utf8WithByteOrderMark: return .utf8 + case .utf16LittleEndianWithByteOrderMark: return .utf16LittleEndian + case .utf16BigEndianWithByteOrderMark, .utf16ByAttributeWithoutByteOrderMark: return .utf16BigEndian + case .utf32LittleEndianWithByteOrderMark: return .utf32LittleEndian + case .utf32BigEndianWithByteOrderMark: return .utf32BigEndian + case .shiftJISByAttribute: return .shiftJIS + case .windowsCyrillicByAttribute: return .windowsCP1251 + case .macRomanByAttribute: return .macOSRoman + case .latin1Fallback: return .isoLatin1 + } + } +} diff --git a/TableProTests/Models/FileTabBaselineTests.swift b/TableProTests/Models/FileTabBaselineTests.swift index 7e4b393676..87d8182144 100644 --- a/TableProTests/Models/FileTabBaselineTests.swift +++ b/TableProTests/Models/FileTabBaselineTests.swift @@ -102,4 +102,35 @@ struct FileTabBaselineTests { #expect(loaded.content == "SELECT 1") #expect(loaded.stamp != nil) } + + @Test("A rebuilt tab learns how its file is encoded, so a save writes it back the same way") + func hydratesTheEncoding() throws { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("baseline-\(UUID().uuidString).sql") + defer { try? FileManager.default.removeItem(at: url) } + let fixture = EncodedSQLFileFixture.utf16BigEndianWithByteOrderMark + try fixture.write(fixture.original, to: url) + var tab = fileTab(query: fixture.original, url: url) + + FileTabBaseline.hydrate(&tab) + + #expect(tab.content.sourceFileEncoding == FileTextEncoding(encoding: .utf16, byteOrderMark: .utf16BigEndian)) + #expect(tab.content.isFileDirty == false) + } + + @Test("A Save As write replaces the encoding the tab was opened with") + func recordingAWriteReplacesTheEncoding() throws { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("baseline-\(UUID().uuidString).sql") + defer { try? FileManager.default.removeItem(at: url) } + let fixture = EncodedSQLFileFixture.utf16BigEndianWithByteOrderMark + try fixture.write(fixture.original, to: url) + var tab = fileTab(query: fixture.original, url: url) + FileTabBaseline.hydrate(&tab) + #expect(tab.content.sourceFileEncoding?.byteOrderMark == .utf16BigEndian) + + FileTabBaseline.recordWrite(of: fixture.original, to: url, as: .utf8, in: &tab.content) + + #expect(tab.content.sourceFileEncoding == .utf8) + } } diff --git a/TableProTests/Models/FileTabExternalChangeTests.swift b/TableProTests/Models/FileTabExternalChangeTests.swift index f9cd130f39..c0e6fcc93c 100644 --- a/TableProTests/Models/FileTabExternalChangeTests.swift +++ b/TableProTests/Models/FileTabExternalChangeTests.swift @@ -143,7 +143,7 @@ struct FileTabExternalChangeTests { var tab = hydratedTab(for: url) try "SELECT 1 -- saved".write(to: url, atomically: true, encoding: .utf8) - FileTabBaseline.recordWrite(of: "SELECT 1 -- saved", to: url, in: &tab.content) + FileTabBaseline.recordWrite(of: "SELECT 1 -- saved", to: url, as: .utf8, in: &tab.content) #expect(tab.content.savedFileContent == "SELECT 1 -- saved") #expect(FileTabBaseline.diskChange(in: tab.content) == nil) @@ -185,4 +185,31 @@ struct FileTabExternalChangeTests { defer { state.coordinator.teardown() } #expect(state.tabManager.tabs.first?.content.savedFileStamp == loaded.stamp) } + + @Test("The file's encoding travels with the payload that opens the tab") + func payloadCarriesTheEncoding() throws { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("external-change-\(UUID().uuidString).sql") + defer { try? FileManager.default.removeItem(at: url) } + let fixture = EncodedSQLFileFixture.shiftJISByAttribute + try fixture.write(fixture.original, to: url) + let loaded = try #require(FileTextLoader.load(url)) + #expect(loaded.textEncoding.attribute != nil) + let connection = TestFixtures.makeConnection() + let payload = EditorTabPayload( + connectionId: connection.id, + tabType: .query, + initialQuery: loaded.content, + sourceFileURL: url, + sourceFileStamp: loaded.stamp, + sourceFileEncoding: loaded.textEncoding + ) + + let decoded = try JSONDecoder().decode(EditorTabPayload.self, from: JSONEncoder().encode(payload)) + #expect(decoded.sourceFileEncoding == loaded.textEncoding) + + let state = SessionStateFactory.create(connection: connection, payload: payload) + defer { state.coordinator.teardown() } + #expect(state.tabManager.tabs.first?.content.sourceFileEncoding == loaded.textEncoding) + } } diff --git a/TableProTests/Models/LinkedSQLFavoriteEncodingTests.swift b/TableProTests/Models/LinkedSQLFavoriteEncodingTests.swift new file mode 100644 index 0000000000..834a0132f4 --- /dev/null +++ b/TableProTests/Models/LinkedSQLFavoriteEncodingTests.swift @@ -0,0 +1,49 @@ +// +// LinkedSQLFavoriteEncodingTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@Suite("Linked SQL favorite encoding") +struct LinkedSQLFavoriteEncodingTests { + private func favorite(encodedAs encodingName: String) -> LinkedSQLFavorite { + LinkedSQLFavorite( + folderId: UUID(), + fileURL: URL(fileURLWithPath: "/tmp/report.sql"), + relativePath: "report.sql", + name: "report", + mtime: Date(), + fileSize: 8, + encodingName: encodingName + ) + } + + @Test("Only an encoding that cannot store every character is flagged") + func flagsOnlyEncodingsThatLackCharacters() { + let cases: [(encodingName: String, isFlagged: Bool)] = [ + ("utf-8", false), + ("utf-16", false), + ("utf-16be", false), + ("utf-32", false), + ("gb18030", false), + ("iso-8859-1", true), + ("cp932", true), + ("windows-1251", true), + ("macintosh", true) + ] + for testCase in cases { + let flagged = favorite(encodedAs: testCase.encodingName).encodingCannotRepresentEveryCharacter + #expect(flagged == testCase.isFlagged, "\(testCase.encodingName)") + } + } + + @Test("The encoding is named the way macOS names it, and an unknown name is shown as stored") + func namesTheEncoding() { + #expect(favorite(encodedAs: "cp932").encodingDisplayName == String.localizedName(of: .shiftJIS)) + #expect(favorite(encodedAs: "no-such-charset").encodingDisplayName == "no-such-charset") + #expect(favorite(encodedAs: "no-such-charset").encodingCannotRepresentEveryCharacter) + } +} diff --git a/TableProTests/Models/Query/TabQueryContentEqualityTests.swift b/TableProTests/Models/Query/TabQueryContentEqualityTests.swift index 4e63205212..d9b11833ff 100644 --- a/TableProTests/Models/Query/TabQueryContentEqualityTests.swift +++ b/TableProTests/Models/Query/TabQueryContentEqualityTests.swift @@ -60,6 +60,18 @@ struct TabQueryContentEqualityTests { #expect(a != b) } + @Test("sourceFileEncoding participates in equality") + func sourceFileEncodingEquality() { + var a = TabQueryContent(query: "Q") + var b = TabQueryContent(query: "Q") + a.sourceFileEncoding = .utf8 + #expect(a != b) + b.sourceFileEncoding = .utf8 + #expect(a == b) + b.sourceFileEncoding = FileTextEncoding(encoding: .utf16, byteOrderMark: .utf16BigEndian) + #expect(a != b) + } + @Test("Value semantics: mutating a copy does not change the original") func valueSemantics() { let a = TabQueryContent(query: "original") diff --git a/TableProTests/Views/Main/SourceFileDiskChangeHandlingTests.swift b/TableProTests/Views/Main/SourceFileDiskChangeHandlingTests.swift index d77e948e4a..b5c79c9d04 100644 --- a/TableProTests/Views/Main/SourceFileDiskChangeHandlingTests.swift +++ b/TableProTests/Views/Main/SourceFileDiskChangeHandlingTests.swift @@ -222,7 +222,9 @@ struct SourceFileDiskChangeHandlingTests { let monitor = SourceFileDiskChangeMonitor(tabManager: tabManager) { urls in await MainActor.run { try? "SELECT 2".write(to: url, atomically: true, encoding: .utf8) - tabManager.mutate(tabId: id) { FileTabBaseline.recordWrite(of: "SELECT 2", to: url, in: &$0.content) } + tabManager.mutate(tabId: id) { + FileTabBaseline.recordWrite(of: "SELECT 2", to: url, as: .utf8, in: &$0.content) + } } return urls.map { _ in nil } } @@ -254,7 +256,7 @@ struct SourceFileDiskChangeHandlingTests { [.modificationDate: Date(timeIntervalSinceNow: -3_600)], ofItemAtPath: url.path ) - let saved = await harness.actions.saveFile(of: restoredTab, to: url) + let saved = await harness.actions.saveFiles([(tab: restoredTab, url: url)]).contains(restoredTab.id) #expect(!saved) #expect(try String(contentsOf: url, encoding: .utf8) == "SELECT 0") @@ -293,7 +295,7 @@ struct SourceFileDiskChangeHandlingTests { let dirty = try #require(tab(id, in: harness.coordinator)) try FileManager.default.removeItem(at: url) - let saved = await harness.actions.saveFile(of: dirty, to: url) + let saved = await harness.actions.saveFiles([(tab: dirty, url: url)]).contains(dirty.id) #expect(!saved) #expect(!FileManager.default.fileExists(atPath: url.path)) @@ -310,7 +312,7 @@ struct SourceFileDiskChangeHandlingTests { let id = try openFileTab(at: url, editedTo: "SELECT 2", in: harness.coordinator) let dirty = try #require(tab(id, in: harness.coordinator)) - let saved = await harness.actions.saveFile(of: dirty, to: url) + let saved = await harness.actions.saveFiles([(tab: dirty, url: url)]).contains(dirty.id) #expect(saved) #expect(try String(contentsOf: url, encoding: .utf8) == "SELECT 2") diff --git a/TableProTests/Views/Main/SourceFileEncodingSaveTests.swift b/TableProTests/Views/Main/SourceFileEncodingSaveTests.swift new file mode 100644 index 0000000000..ffbece7107 --- /dev/null +++ b/TableProTests/Views/Main/SourceFileEncodingSaveTests.swift @@ -0,0 +1,320 @@ +// +// SourceFileEncodingSaveTests.swift +// TableProTests +// + +import AppKit +import Foundation +import SwiftUI +@testable import TablePro +import Testing + +@MainActor @Suite("Source file encoding on save") +struct SourceFileEncodingSaveTests { + private final class ReportedErrors { + var entries: [(title: String, message: String)] = [] + } + + private struct Harness { + let coordinator: MainContentCoordinator + let actions: MainContentCommandActions + let reported: ReportedErrors + } + + private static let unrepresentable = "SELECT '\u{1F600}';\n" + + private func makeHarness() -> Harness { + let connection = TestFixtures.makeConnection(database: "db_a") + let coordinator = SessionStateFactory.create(connection: connection, payload: nil).coordinator + + var selectedTables: Set = [] + var pendingTruncates: Set = [] + var pendingDeletes: Set = [] + var tableOperationOptions: [DatabaseTreeTableRef: TableOperationOptions] = [:] + + let actions = MainContentCommandActions( + coordinator: coordinator, + connection: connection, + selectionState: coordinator.selectionState, + selectedTables: Binding(get: { selectedTables }, set: { selectedTables = $0 }), + pendingTruncates: Binding(get: { pendingTruncates }, set: { pendingTruncates = $0 }), + pendingDeletes: Binding(get: { pendingDeletes }, set: { pendingDeletes = $0 }), + tableOperationOptions: Binding( + get: { tableOperationOptions }, + set: { tableOperationOptions = $0 } + ), + trailingPaneState: TrailingPaneState() + ) + actions.chooseSaveURL = { _ in + Issue.record("Save As was not expected") + return nil + } + let reported = ReportedErrors() + coordinator.presentError = { title, message, _ in reported.entries.append((title, message)) } + return Harness(coordinator: coordinator, actions: actions, reported: reported) + } + + private func makeFolder() throws -> URL { + let folder = FileManager.default.temporaryDirectory + .appendingPathComponent("source-file-encoding-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + return folder + } + + private func openFileTab( + _ fixture: EncodedSQLFileFixture, + at url: URL, + editedTo query: String, + in coordinator: MainContentCoordinator + ) throws -> UUID { + try fixture.write(fixture.original, to: url) + let loaded = try #require(FileTextLoader.load(url)) + #expect(loaded.content == fixture.original) + coordinator.tabManager.addTab( + initialQuery: loaded.content, + sourceFileURL: url, + sourceFileStamp: loaded.stamp, + sourceFileEncoding: loaded.textEncoding + ) + let id = try #require(coordinator.tabManager.selectedTabId) + coordinator.tabManager.mutate(tabId: id) { $0.content.query = query } + return id + } + + private func tab(_ id: UUID, in coordinator: MainContentCoordinator) -> QueryTab? { + coordinator.tabManager.tabs.first { $0.id == id } + } + + private func expectSaved( + _ fixture: EncodedSQLFileFixture, + at url: URL, + tab id: UUID, + in harness: Harness + ) throws { + let expected = try #require(fixture.bytes(of: fixture.edited)) + #expect(try Data(contentsOf: url) == expected, "\(fixture)") + #expect(EncodedSQLFileFixture.attributeValue(of: url) == fixture.attributeValue, "\(fixture)") + let reloaded = try #require(FileTextLoader.load(url)) + #expect(reloaded.content == fixture.edited, "\(fixture)") + #expect(reloaded.encoding == fixture.reportedEncoding, "\(fixture)") + #expect(tab(id, in: harness.coordinator)?.content.isFileDirty == false, "\(fixture)") + #expect(harness.reported.entries.isEmpty, "\(fixture)") + } + + @Test( + "Cmd+S writes a file back in the encoding it was read in", + arguments: EncodedSQLFileFixture.allCases + ) + func commandSaveKeepsTheEncoding(fixture: EncodedSQLFileFixture) async throws { + let harness = makeHarness() + defer { harness.coordinator.teardown() } + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("report.sql") + let id = try openFileTab(fixture, at: url, editedTo: fixture.edited, in: harness.coordinator) + + let saved = await harness.actions.saveSelectedFileAwaiting() + + #expect(saved) + try expectSaved(fixture, at: url, tab: id, in: harness) + } + + @Test( + "A batch save writes each file back in the encoding it was read in", + arguments: EncodedSQLFileFixture.allCases + ) + func batchSaveKeepsTheEncoding(fixture: EncodedSQLFileFixture) async throws { + let harness = makeHarness() + defer { harness.coordinator.teardown() } + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("report.sql") + let id = try openFileTab(fixture, at: url, editedTo: fixture.edited, in: harness.coordinator) + let dirty = try #require(tab(id, in: harness.coordinator)) + + let saved = await harness.actions.saveFiles([(tab: dirty, url: url)]).contains(id) + + #expect(saved) + try expectSaved(fixture, at: url, tab: id, in: harness) + } + + @Test("A second save keeps the encoding the first one wrote") + func repeatedSavesKeepTheEncoding() async throws { + let harness = makeHarness() + defer { harness.coordinator.teardown() } + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("report.sql") + let fixture = EncodedSQLFileFixture.shiftJISByAttribute + let id = try openFileTab(fixture, at: url, editedTo: fixture.original + "--", in: harness.coordinator) + + #expect(await harness.actions.saveSelectedFileAwaiting()) + harness.coordinator.tabManager.mutate(tabId: id) { $0.content.query = fixture.edited } + #expect(await harness.actions.saveSelectedFileAwaiting()) + + try expectSaved(fixture, at: url, tab: id, in: harness) + } + + @Test("A tab that never learned its file's encoding keeps the one on disk") + func unknownEncodingKeepsTheFilesOwn() async throws { + let harness = makeHarness() + defer { harness.coordinator.teardown() } + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("report.sql") + let fixture = EncodedSQLFileFixture.windowsCyrillicByAttribute + try fixture.write(fixture.original, to: url) + let loaded = try #require(FileTextLoader.load(url)) + harness.coordinator.tabManager.addTab( + initialQuery: loaded.content, + sourceFileURL: url, + sourceFileStamp: loaded.stamp + ) + let id = try #require(harness.coordinator.tabManager.selectedTabId) + #expect(tab(id, in: harness.coordinator)?.content.sourceFileEncoding == nil) + harness.coordinator.tabManager.mutate(tabId: id) { $0.content.query = fixture.edited } + + #expect(await harness.actions.saveSelectedFileAwaiting()) + + try expectSaved(fixture, at: url, tab: id, in: harness) + } + + @Test("Cmd+S refuses text the file's encoding cannot hold, names the file and why, and leaves it alone") + func commandSaveRefusesUnrepresentableText() async throws { + let harness = makeHarness() + defer { harness.coordinator.teardown() } + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("latin.sql") + let fixture = EncodedSQLFileFixture.latin1Fallback + let id = try openFileTab(fixture, at: url, editedTo: Self.unrepresentable, in: harness.coordinator) + let before = try Data(contentsOf: url) + + let saved = await harness.actions.saveSelectedFileAwaiting() + + #expect(!saved) + #expect(try Data(contentsOf: url) == before) + #expect(tab(id, in: harness.coordinator)?.content.query == Self.unrepresentable) + #expect(tab(id, in: harness.coordinator)?.content.isFileDirty == true) + #expect(harness.reported.entries.map(\.title) == [String(localized: "Couldn't Save File")]) + let message = try #require(harness.reported.entries.first?.message) + #expect(message.contains("latin.sql")) + #expect(message.contains(String.localizedName(of: .isoLatin1))) + } + + @Test("Save on close refuses text the file's encoding cannot hold and keeps the tab open") + func saveOnCloseRefusesUnrepresentableText() async throws { + let harness = makeHarness() + defer { harness.coordinator.teardown() } + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("japanese.sql") + let fixture = EncodedSQLFileFixture.shiftJISByAttribute + let id = try openFileTab(fixture, at: url, editedTo: Self.unrepresentable, in: harness.coordinator) + let before = try Data(contentsOf: url) + + let mayClose = await harness.actions.saveSelectedTabWork() + + #expect(!mayClose) + #expect(try Data(contentsOf: url) == before) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == fixture.attributeValue) + #expect(tab(id, in: harness.coordinator)?.content.query == Self.unrepresentable) + let message = try #require(harness.reported.entries.first?.message) + #expect(message.contains("japanese.sql")) + #expect(message.contains(String.localizedName(of: .shiftJIS))) + } + + @Test("Save on a batch close refuses each file its encoding cannot hold in one alert and closes only what saved") + func batchCloseRefusesUnrepresentableText() async throws { + let harness = makeHarness() + defer { harness.coordinator.teardown() } + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let latinURL = folder.appendingPathComponent("latin.sql") + let cyrillicURL = folder.appendingPathComponent("cyrillic.sql") + let savedURL = folder.appendingPathComponent("saved.sql") + let savedFixture = EncodedSQLFileFixture.utf16BigEndianWithByteOrderMark + let latinID = try openFileTab( + .latin1Fallback, at: latinURL, editedTo: Self.unrepresentable, in: harness.coordinator + ) + let cyrillicID = try openFileTab( + .windowsCyrillicByAttribute, at: cyrillicURL, editedTo: Self.unrepresentable, in: harness.coordinator + ) + let savedID = try openFileTab(savedFixture, at: savedURL, editedTo: savedFixture.edited, in: harness.coordinator) + harness.coordinator.tabManager.addTab(initialQuery: "SELECT 1") + let victimIDs = [latinID, cyrillicID, savedID] + #expect(harness.coordinator.tabManager.selectedTabId.map { !victimIDs.contains($0) } == true) + let victims = victimIDs.compactMap { tab($0, in: harness.coordinator) } + let latinBefore = try Data(contentsOf: latinURL) + let cyrillicBefore = try Data(contentsOf: cyrillicURL) + harness.actions.confirmSaveChanges = { _, _ in .save } + + let outcome = await harness.actions.resolveUnsavedWork(in: victims) + + #expect(outcome == .close([savedID])) + #expect(try Data(contentsOf: latinURL) == latinBefore) + #expect(try Data(contentsOf: cyrillicURL) == cyrillicBefore) + #expect(EncodedSQLFileFixture.attributeValue(of: cyrillicURL) == "windows-1251;1282") + #expect(try Data(contentsOf: savedURL) == savedFixture.bytes(of: savedFixture.edited)) + #expect(harness.reported.entries.map(\.title) == [String(localized: "Couldn't Save Files")]) + let message = try #require(harness.reported.entries.first?.message) + #expect(message.contains("latin.sql")) + #expect(message.contains("cyrillic.sql")) + #expect(!message.contains("saved.sql")) + #expect(tab(latinID, in: harness.coordinator)?.content.isFileDirty == true) + #expect(tab(cyrillicID, in: harness.coordinator)?.content.isFileDirty == true) + } + + @Test("A file recorded as ASCII saves any text as UTF-8 and loses the ASCII record") + func asciiFileSavesAsUTF8() async throws { + let harness = makeHarness() + defer { harness.coordinator.teardown() } + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("ascii.sql") + try "SELECT 1;\n".write(to: url, atomically: true, encoding: .ascii) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == "us-ascii;1536") + let loaded = try #require(FileTextLoader.load(url)) + harness.coordinator.tabManager.addTab( + initialQuery: loaded.content, + sourceFileURL: url, + sourceFileStamp: loaded.stamp, + sourceFileEncoding: loaded.textEncoding + ) + let id = try #require(harness.coordinator.tabManager.selectedTabId) + let edited = "SELECT 'Caf\u{E9} \u{1F600}';\n" + harness.coordinator.tabManager.mutate(tabId: id) { $0.content.query = edited } + + #expect(await harness.actions.saveSelectedFileAwaiting()) + + #expect(try Data(contentsOf: url) == Data(edited.utf8)) + #expect(EncodedSQLFileFixture.attributeValue(of: url) == nil) + #expect(FileTextLoader.load(url)?.encoding == .utf8) + #expect(harness.reported.entries.isEmpty) + } + + @Test("Save As writes UTF-8 and the tab saves as UTF-8 from then on") + func saveAsConvertsToUTF8() async throws { + let harness = makeHarness() + defer { harness.coordinator.teardown() } + let folder = try makeFolder() + defer { try? FileManager.default.removeItem(at: folder) } + let url = folder.appendingPathComponent("japanese.sql") + let copy = folder.appendingPathComponent("japanese-utf8.sql") + let fixture = EncodedSQLFileFixture.shiftJISByAttribute + let id = try openFileTab(fixture, at: url, editedTo: Self.unrepresentable, in: harness.coordinator) + harness.actions.chooseSaveURL = { _ in copy } + + #expect(await harness.actions.saveFileAsAwaiting()) + + #expect(try Data(contentsOf: copy) == Data(Self.unrepresentable.utf8)) + #expect(EncodedSQLFileFixture.attributeValue(of: copy) == nil) + #expect(tab(id, in: harness.coordinator)?.content.sourceFileEncoding == .utf8) + #expect(harness.reported.entries.isEmpty) + + let appended = Self.unrepresentable + "SELECT 2;\n" + harness.coordinator.tabManager.mutate(tabId: id) { $0.content.query = appended } + #expect(await harness.actions.saveSelectedFileAwaiting()) + #expect(try Data(contentsOf: copy) == Data(appended.utf8)) + } +} diff --git a/docs/features/favorites.mdx b/docs/features/favorites.mdx index 2b0126fad7..9c775fcdbe 100644 --- a/docs/features/favorites.mdx +++ b/docs/features/favorites.mdx @@ -129,7 +129,7 @@ Click a linked file to open it as an ordinary editor tab. `Cmd+S` writes back in A file changed on disk gets the same banner and diff sheet as any other [SQL file](/features/sql-files#when-the-file-changes-underneath-you). Files added and removed outside TablePro reach the sidebar a second later. -A non-UTF-8 file carries a yellow warning triangle. Saving keeps its encoding, and a character it cannot represent, an emoji in an ISO Latin-1 file, fails the save with an error rather than dropping the character. +A file in an encoding that cannot store every character, such as ISO Latin-1 or Shift JIS, carries a yellow warning triangle. Saving keeps its encoding, and a character it cannot represent, an emoji in an ISO Latin-1 file, fails the save with an error rather than dropping the character. ### Frontmatter diff --git a/docs/features/sql-files.mdx b/docs/features/sql-files.mdx index 54d6583422..52b2fee742 100644 --- a/docs/features/sql-files.mdx +++ b/docs/features/sql-files.mdx @@ -11,6 +11,8 @@ Three ways in: double-click in Finder, **File > Open File…** (`Cmd+O`), or dra Each file opens in a new tab. Opening one that is already open focuses its existing tab instead of making a second, and files opened before any connection exists wait in a queue until you connect. +The encoding comes from the file: its byte order mark first, then the encoding a Mac app recorded for it when saving, then UTF-8. Bytes that fit none of those open as ISO Latin-1. + The title bar carries the filename, and the close button gets the standard macOS unsaved-changes dot. ## Saving @@ -23,6 +25,14 @@ When a tab has both unsaved file changes and pending data grid edits, `Cmd+S` sa Closing a query tab keeps its SQL either way. Reopen it with `Cmd+Shift+T` or **File > Reopen Closed Tab**. See [Tabs](/features/tabs). +### File encoding + +A save writes the file in the encoding it was opened in. A byte order mark stays, and UTF-16 and UTF-32 keep their byte order. A file known only by the encoding a Mac app recorded for it keeps that record. A file recorded as ASCII saves as UTF-8. + +If the text holds a character the encoding cannot store, such as an emoji in an ISO Latin-1 file, the save stops. An alert names the file and its encoding, the file on disk is left as it was, and the tab stays open with your text. This holds for `Cmd+S`, for **Save** when closing a tab or a window, and for closing several tabs at once. + +Save As always writes UTF-8, which is how to convert a file. The tab then points at the new file and keeps saving it as UTF-8. + ## When the file changes underneath you An open file is compared with the copy on disk whenever its window becomes active, and whenever a [linked SQL folder](/features/favorites#linked-sql-folders) sees a change. Any difference counts: a `git pull`, an edit in another editor, or an older copy restored over it.