diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95f06eb4..b4c898b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,7 +187,12 @@ jobs: # Posts, and on later pushes updates, a single comment showing overall coverage and the # coverage of the files this PR actually changed. Both reports are passed together so the # numbers reflect the unit and on-device suites combined. + # + # Skipped for a pull request from a fork: its GITHUB_TOKEN is read-only whatever the + # workflow asks for, so the comment call would fail with a 403 and turn the whole job red for + # a reason unrelated to the change. - name: Comment coverage + if: github.event.pull_request.head.repo.full_name == github.repository uses: madrapps/jacoco-report@v1.7.1 with: paths: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d052508..3afefd17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # CHANGELOG -## [2.6.50] - 04.09.2026 +## [2.6.51] - 05.09.2026 **New** diff --git a/app/build.gradle b/app/build.gradle index d9d502d8..4ccc7e06 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -21,7 +21,7 @@ apply from: "$projectDir/gradle/libs-task.gradle" apply from: "$projectDir/gradle/changelog-task.gradle" -def appVersionCode = 50 +def appVersionCode = 51 def appVersionName = "2.6.${appVersionCode}" def gitCommitHashProvider = providers.exec { diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java index 6f1d88db..31f2ffc2 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java @@ -294,10 +294,11 @@ public void applySnapshot_repointsEditorBlocksAtTheFilesThisDeviceWrote() throws } @Test - public void applySnapshot_leavesEditorBlocksAloneWhenTheyDoNotLineUp() throws Exception { + public void applySnapshot_rewritesOnlyTheBlocksThatNameAKnownAttachment() throws Exception { byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); int noteId = seedNoteWithAttachment("photo.png", bytes); - // Two blocks, one attachment: the positional mapping cannot be trusted. + // Two blocks, one attachment: a block naming a file the column does not know is left + // exactly as it was, and the one that does is repointed by identity, not by position. String twoBlocks = "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_" + noteId @@ -311,8 +312,874 @@ public void applySnapshot_leavesEditorBlocksAloneWhenTheyDoNotLineUp() throws Ex store.applySnapshot(store.readSnapshot(), Collections.emptyList()); + List urls = + com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks.fileUrls( + db.noteDao().getNoteSync(noteId).getValueJson()); + assertThat(urls).hasSize(2); + File rendered = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.resolve( + context, urls.get(0)); + assertThat(rendered).isNotNull(); + assertThat(readAll(new java.io.FileInputStream(rendered))).isEqualTo(bytes); + assertThat(urls.get(1)).isEqualTo("editorjs://attachments/note_" + noteId + "/other.png"); + } + + @Test + public void applySnapshot_leavesLegacyBlocksAloneWhenTheyDoNotLineUp() throws Exception { + // A bundle from an older client names the sender's files in its blocks; for those the + // only mapping is positional, and two blocks for one attachment cannot be trusted. // Rewriting on a guess could point a block at the wrong file; leaving it is recoverable. - assertThat(db.noteDao().getNoteSync(noteId).getValueJson()).isEqualTo(twoBlocks); + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + int noteId = seedNoteWithAttachment("photo.png", bytes); + SyncRecord built = onlyNote(store.readSnapshot()); + String legacyBlocks = + "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_99/photo.png\"}}}," + + "{\"type\":\"image\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_99/other.png\"}}}]"; + JsonObject legacyPayload = built.getPayload(); + legacyPayload.addProperty("f", legacyBlocks); + SyncRecord legacy = + SyncRecord.live( + SyncRecord.Type.NOTE, + built.getId(), + built.getUpdatedAt().plusSeconds(1), + legacyPayload); + + store.applySnapshot( + new SyncSnapshot(Collections.singletonList(legacy)), Collections.emptyList()); + + assertThat(db.noteDao().getNoteSync(noteId).getValueJson()).isEqualTo(legacyBlocks); + } + + // ------------------------------------------------- a note that has crossed devices + + @Test + public void aNoteReceivedFromAnotherDeviceHashesIdenticallyWhenRebuiltThere() throws Exception { + // Device A: a rich note whose block names A's own file. A throwaway note first, so A's + // row id differs from the one B will assign. + seedNote("Placeholder", "x", null, "22222222-2222-4222-8222-222222222222"); + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + // The display name has no extension, so the MIME type can only come from the sender's + // file name — which B never sees. + int noteId = seedNoteWithAttachment("1700000000000_123.png", "photo", bytes); + Note seeded = db.noteDao().getNoteSync(noteId); + seeded.setValueJson( + "[{\"id\":\"blk1\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, "1700000000000_123.png") + + "\",\"name\":\"photo\"}}}]"); + db.noteDao().addNote(seeded); + SyncRecord fromA = + store.readSnapshot() + .find(SyncRecord.Type.NOTE, "11111111-1111-4111-8111-111111111111"); + assertThat(fromA).isNotNull(); + + // On the wire the block names the attachment, not A's row id and file. + String wireBlocks = fromA.getPayload().get("f").getAsString(); + assertThat(wireBlocks).doesNotContain("note_"); + assertThat(wireBlocks).contains("mynotes-sync://attachment/"); + + // Device B: its own database, its own row ids; the blob arrives through the sync cache. + AppDatabase dbB = + Room.inMemoryDatabaseBuilder(context, AppDatabase.class) + .allowMainThreadQueries() + .build(); + try { + RoomSyncStore storeB = new RoomSyncStore(context, dbB, mock(PreferenceHelper.class)); + storeB.writeAttachment(sha256(bytes), bytes.length, new ByteArrayInputStream(bytes)); + storeB.applySnapshot( + new SyncSnapshot(Collections.singletonList(fromA)), Collections.emptyList()); + + SyncRecord rebuiltOnB = onlyNote(storeB.readSnapshot()); + + // The same version, not a conflict against itself: restoring rewrote B's blocks and + // column to B's files, and every such note used to hash differently at the same + // timestamp and republish a bundle on every sync forever. + assertThat(rebuiltOnB.getCanonicalPayloadHash()) + .isEqualTo(fromA.getCanonicalPayloadHash()); + long localIdOnB = + dbB.syncMetadataDao() + .getByStableId(SyncMetadata.RECORD_TYPE_NOTE, fromA.getId()) + .localId; + String blockUrl = + com.pasich + .mynotes + .extendedEditor + .attach + .EditorAttachmentBlocks + .fileUrls(dbB.noteDao().getNoteSync((int) localIdOnB).getValueJson()) + .get(0); + File rendered = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.resolve( + context, blockUrl); + assertThat(rendered).isNotNull(); + assertThat(readAll(new java.io.FileInputStream(rendered))).isEqualTo(bytes); + } finally { + dbB.close(); + } + } + + // ------------------------------------------------- notes published by 2.6.50 + + @Test + public void aNoteSyncedBy2650HashesIdenticallyAfterTheUpgrade() throws Exception { + // The device that published the note under 2.6.50 upgrades. Its unchanged note, rebuilt + // by the upgraded store, has to be the same version as the one on Drive, or the merge + // reports a conflict against the note itself — and, when the old version won, again on + // every sync until the user edited the note. + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + int noteId = seedNoteWithAttachment("1700000000000_123.png", "photo.png", bytes); + String localUrl = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, "1700000000000_123.png"); + String blocks = + "[{\"id\":\"blk1\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + localUrl + + "\",\"name\":\"photo.png\"}}}]"; + Note seeded = db.noteDao().getNoteSync(noteId); + seeded.setValueJson(blocks); + db.noteDao().addNote(seeded); + SyncRecord onDrive = + published2650(seeded, localUrl, "photo.png", sha256(bytes), bytes.length); + + SyncRecord rebuilt = onlyNote(store.readSnapshot()); + com.pasich.mynotes.data.sync.SyncMergeResult merge = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(rebuilt)), + new SyncSnapshot(Collections.singletonList(onDrive))); + + assertThat(rebuilt.getCanonicalPayloadHash()).isEqualTo(onDrive.getCanonicalPayloadHash()); + assertThat(merge.getConflicts()).isEmpty(); + } + + @Test + public void aNoteReceivedUnder2650ConflictsOnceAfterTheUpgradeAndThenSyncsCleanly() + throws Exception { + // The other device: it received the note under 2.6.50, whose restore wrote the sender's + // hash-less id into its column and its own file name into the blocks. Its rebuilt + // version cannot equal the upgraded remote one, so one conflict is expected; what must + // not happen is the same conflict on every sync afterwards. + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + String senderUrl = "editorjs://attachments/note_77/1700000000000_123.png"; + String stableId = "11111111-1111-4111-8111-111111111111"; + String legacyId = + java.util + .UUID + .nameUUIDFromBytes( + (stableId + "\n0\n" + senderUrl + "\nphoto.png") + .getBytes(StandardCharsets.UTF_8)) + .toString(); + int noteId = seedNote("Shopping", "Milk", null); + File folder = new File(context.getFilesDir(), "attachments/note_" + noteId); + assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); + try (FileOutputStream out = new FileOutputStream(new File(folder, legacyId + "-" + hash))) { + out.write(bytes); + } + String receiverUrl = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, legacyId + "-" + hash); + Note received = db.noteDao().getNoteSync(noteId); + received.setAttachments( + "[{\"url\":\"" + + receiverUrl + + "\",\"name\":\"photo.png\",\"id\":\"" + + legacyId + + "\"}]"); + received.setValueJson( + "[{\"id\":\"blk1\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + receiverUrl + + "\",\"name\":\"photo.png\"}}}]"); + db.noteDao().addNote(received); + Note sender = new Note().create("Shopping", "Milk", 1_000L, ""); + sender.setValueJson( + "[{\"id\":\"blk1\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + senderUrl + + "\",\"name\":\"photo.png\"}}}]"); + SyncRecord onDrive = published2650(sender, senderUrl, "photo.png", hash, bytes.length); + + SyncRecord rebuilt = onlyNote(store.readSnapshot()); + com.pasich.mynotes.data.sync.SyncMergeResult first = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(rebuilt)), + new SyncSnapshot(Collections.singletonList(onDrive))); + assertThat(first.getConflicts()).hasSize(1); + store.applySnapshot(first.getMergedSnapshot(), first.getConflicts()); + + SyncRecord afterApply = onlyNote(store.readSnapshot()); + com.pasich.mynotes.data.sync.SyncMergeResult second = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(afterApply)), + first.getMergedSnapshot()); + + assertThat(second.getConflicts()).isEmpty(); + assertThat(afterApply.getCanonicalPayloadHash()) + .isEqualTo(first.getMergedSnapshot().getRecords().get(0).getCanonicalPayloadHash()); + } + + /** + * The record 2.6.50 published for {@code note}, read back through today's decoder: blocks + * naming the sender's own file, the id derived without the content hash, and the MIME type + * 2.6.50 detected from the display name. Built by hand because that code is gone. + */ + private static SyncRecord published2650( + Note note, String blockUrl, String displayName, String hash, long size) + throws IOException { + String stableId = "11111111-1111-4111-8111-111111111111"; + String legacyId = + java.util + .UUID + .nameUUIDFromBytes( + (stableId + "\n0\n" + blockUrl + "\n" + displayName) + .getBytes(StandardCharsets.UTF_8)) + .toString(); + JsonObject payload = new com.google.gson.Gson().toJsonTree(note).getAsJsonObject(); + payload.remove("a"); + payload.remove("h"); + JsonObject entry = new JsonObject(); + entry.addProperty("id", legacyId); + entry.addProperty("sha256", hash); + entry.addProperty("mimeType", "image/png"); + entry.addProperty("size", size); + entry.addProperty("path", "attachments/" + hash); + entry.addProperty("displayName", displayName); + com.google.gson.JsonArray manifest = new com.google.gson.JsonArray(); + manifest.add(entry); + payload.add("attachmentsManifest", manifest); + com.google.gson.JsonArray hashes = new com.google.gson.JsonArray(); + hashes.add(hash); + payload.add("attachmentHashes", hashes); + JsonObject names = new JsonObject(); + names.addProperty(legacyId, displayName); + payload.add("attachmentNames", names); + SyncRecord asPublished = + SyncRecord.live( + SyncRecord.Type.NOTE, + stableId, + java.time.Instant.ofEpochMilli(1_000L), + payload); + // Through the bundle codec, which is where an old payload meets today's decoder. + com.pasich.mynotes.data.sync.SyncBundleCodec codec = + new com.pasich.mynotes.data.sync.SyncBundleCodec(); + byte[] bundle = + codec.encode( + new SyncSnapshot(Collections.singletonList(asPublished)), + java.time.Instant.now()); + return codec.decode(new ByteArrayInputStream(bundle)) + .getSnapshot() + .find(SyncRecord.Type.NOTE, stableId); + } + + // ------------------------------------------------- conflicts and records that moved on + // ------------------------------------------------- conflicts and records that moved on + + @Test + public void applySnapshot_doesNotStoreAConflictForARecordSkippedAsStale() throws Exception { + int noteId = seedNote("Edited during the sync", "body", null); + SyncRecord local = onlyNote(store.readSnapshot()); + // The user edits while the sync is on Drive. + db.syncMetadataDao().touch(SyncMetadata.RECORD_TYPE_NOTE, noteId, 5_000L); + JsonObject remotePayload = local.getPayload(); + remotePayload.addProperty("b", "Remote title"); + SyncRecord remote = + SyncRecord.live( + SyncRecord.Type.NOTE, + local.getId(), + java.time.Instant.ofEpochMilli(2_000L), + remotePayload); + com.pasich.mynotes.data.sync.SyncMergeResult merge = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(local)), + new SyncSnapshot(Collections.singletonList(remote))); + assertThat(merge.getConflicts()).hasSize(1); + + store.applySnapshot(merge.getMergedSnapshot(), merge.getConflicts()); + + // The apply rightly skipped the note; the conflict it would have stored offered two + // versions older than the edit, and resolving it wrote one of them over the edit. + assertThat(store.getConflicts()).isEmpty(); + assertThat(db.noteDao().getNoteSync(noteId).getTitle()).isEqualTo("Edited during the sync"); + } + + @Test + public void applySnapshot_retiresAnOpenConflictWhoseWinnerIsNoLongerTheLiveVersion() + throws Exception { + int noteId = seedNote("Current", "body", null); + SyncRecord local = onlyNote(store.readSnapshot()); + db.syncConflictDao() + .insertIgnoringDuplicates( + Collections.singletonList( + noteConflictRow(local.getId(), "old-winner", 900L, 800L))); + JsonObject remotePayload = local.getPayload(); + remotePayload.addProperty("b", "Remote title"); + SyncRecord remote = + SyncRecord.live( + SyncRecord.Type.NOTE, + local.getId(), + java.time.Instant.ofEpochMilli(2_000L), + remotePayload); + com.pasich.mynotes.data.sync.SyncMergeResult merge = + new com.pasich.mynotes.data.sync.SyncMerger() + .merge( + new SyncSnapshot(Collections.singletonList(local)), + new SyncSnapshot(Collections.singletonList(remote))); + + store.applySnapshot(merge.getMergedSnapshot(), merge.getConflicts()); + + // The old row pre-selected a winner the record has moved past; one tap on it reverted + // the live version. Only the conflict against the current version remains. + List unresolved = + store.getUnresolvedConflicts(); + assertThat(unresolved).hasSize(1); + assertThat(unresolved.get(0).winnerVersionId).isEqualTo(remote.getCanonicalPayloadHash()); + assertThat(noteId).isGreaterThan(0); + } + + @Test + public void resolveConflict_dropsAConflictTheRecordHasMovedPastInsteadOfApplyingIt() + throws Exception { + int noteId = seedNote("Newest", "body", null); + db.syncMetadataDao().touch(SyncMetadata.RECORD_TYPE_NOTE, noteId, 5_000L); + db.syncConflictDao() + .insertIgnoringDuplicates( + Collections.singletonList( + noteConflictRow( + "11111111-1111-4111-8111-111111111111", + "stale-winner", + 3_000L, + 2_000L))); + long conflictId = db.syncConflictDao().getAll().get(0).id; + + store.resolveConflict(conflictId, SyncResolution.KEEP_WINNER); + + // Both offered versions are older than what the user has now; applying either would + // have overwritten the edit with a version that was never offered against it. + assertThat(db.noteDao().getNoteSync(noteId).getTitle()).isEqualTo("Newest"); + assertThat(db.syncConflictDao().getById(conflictId)).isNull(); + assertThat(db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_NOTE, noteId).updatedAt) + .isEqualTo(5_000L); + } + + /** A stored note conflict whose winner is titled after its version id. */ + private com.pasich.mynotes.data.database.entities.SyncConflictEntity noteConflictRow( + String stableId, String winnerVersionId, long winnerUpdatedAt, long loserUpdatedAt) { + String winner = + "{\"type\":\"note\",\"id\":\"" + + stableId + + "\",\"updatedAt\":\"" + + java.time.Instant.ofEpochMilli(winnerUpdatedAt) + + "\",\"deletedAt\":null,\"payload\":{\"b\":\"" + + winnerVersionId + + "\",\"c\":\"body\"}}"; + String loser = + "{\"type\":\"note\",\"id\":\"" + + stableId + + "\",\"updatedAt\":\"" + + java.time.Instant.ofEpochMilli(loserUpdatedAt) + + "\",\"deletedAt\":null,\"payload\":{\"b\":\"loser\",\"c\":\"body\"}}"; + return new com.pasich.mynotes.data.database.entities.SyncConflictEntity( + SyncMetadata.RECORD_TYPE_NOTE, + stableId, + "pair-" + winnerVersionId, + "REMOTE", + "LOCAL", + winnerVersionId, + "loser-" + winnerVersionId, + winner, + loser, + winnerUpdatedAt, + loserUpdatedAt, + false, + false, + "PENDING", + false, + 1L, + 0L); + } + + // ------------------------------------------------- settings edited during a sync + + @Test + public void applySnapshot_leavesSettingsChangedDuringTheSyncAlone() throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + preferencesStore.readState(); + adapter.current.set(preferencesWithTheme(1)); + preferencesStore.buildSnapshot(); + // The user flips a setting while the sync is on Drive; the settings screens write the + // preferences directly and nothing touches the sync record for them. + adapter.current.set(preferencesWithTheme(2)); + db.syncMetadataDao().setVersion(SyncMetadata.RECORD_TYPE_PREFERENCES, 0, 1_000L, null); + SyncRecord remote = + SyncRecord.live( + SyncRecord.Type.PREFERENCES, + "00000000-0000-4000-8000-000000000000", + java.time.Instant.ofEpochMilli(2_000L), + new com.google.gson.Gson() + .toJsonTree(preferencesWithTheme(3)) + .getAsJsonObject()); + + preferencesStore.applySnapshot( + new SyncSnapshot(Collections.singletonList(remote)), Collections.emptyList()); + + // Not committed, and the baseline not rewritten to hide it: the merged version was chosen + // against settings that no longer exist. + assertThat(adapter.committed.get()).isNull(); + assertThat(adapter.current.get().getThemeValue()).isEqualTo(2); + // The next build sees the edit and publishes it. + preferencesStore.buildSnapshot(); + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "00000000-0000-4000-8000-000000000000") + .updatedAt) + .isGreaterThan(2_000L); + } + + @Test + public void applySnapshot_leavesSettingsChangedInsideTheApplyTransactionAlone() + throws Exception { + // The guard used to run once before the transaction. Applying many notes takes seconds; + // a setting toggled in that window was still overwritten and its digest recorded as the + // baseline, so the next build saw nothing to publish. + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore preferencesStore = + new RoomSyncStore( + context, + db, + adapter.helper, + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage::resolve, + file -> sha256(readAll(new java.io.FileInputStream(file))), + record -> { + if (record.getType() == SyncRecord.Type.NOTE) { + // The user flips a setting while the notes are being applied. + adapter.current.set(preferencesWithTheme(2)); + } + }); + preferencesStore.readState(); + adapter.current.set(preferencesWithTheme(1)); + int noteId = seedNote("Applied first", "body", null); + SyncRecord note = onlyNote(preferencesStore.readSnapshot()); + db.syncMetadataDao().setVersion(SyncMetadata.RECORD_TYPE_PREFERENCES, 0, 1_000L, null); + JsonObject changed = note.getPayload(); + changed.addProperty("b", "Remote title"); + SyncRecord remoteNote = + SyncRecord.live( + SyncRecord.Type.NOTE, + note.getId(), + java.time.Instant.ofEpochMilli(2_000L), + changed); + SyncRecord remotePreferences = + SyncRecord.live( + SyncRecord.Type.PREFERENCES, + "00000000-0000-4000-8000-000000000000", + java.time.Instant.ofEpochMilli(2_000L), + new com.google.gson.Gson() + .toJsonTree(preferencesWithTheme(3)) + .getAsJsonObject()); + + preferencesStore.applySnapshot( + new SyncSnapshot(java.util.Arrays.asList(remoteNote, remotePreferences)), + Collections.emptyList(), + SyncState.success("google-drive", java.time.Instant.now(), 0)); + + assertThat(db.noteDao().getNoteSync(noteId).getTitle()).isEqualTo("Remote title"); + assertThat(adapter.committed.get()).isNull(); + assertThat(adapter.current.get().getThemeValue()).isEqualTo(2); + // Not recorded as holding the remote version, and the final state still lands. + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "00000000-0000-4000-8000-000000000000") + .updatedAt) + .isEqualTo(1_000L); + assertThat(preferencesStore.readState().getStatus()).isEqualTo(SyncState.Status.SUCCESS); + preferencesStore.buildSnapshot(); + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "00000000-0000-4000-8000-000000000000") + .updatedAt) + .isGreaterThan(2_000L); + } + + @Test + public void applySnapshot_skipsAnUnusablePreferencesPayloadInsteadOfFailingEverySync() + throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + preferencesStore.readState(); + int noteId = seedNote("Applied anyway", "body", null); + SyncRecord note = onlyNote(preferencesStore.readSnapshot()); + JsonObject changed = note.getPayload(); + changed.addProperty("b", "Remote title"); + SyncRecord remoteNote = + SyncRecord.live( + SyncRecord.Type.NOTE, + note.getId(), + java.time.Instant.ofEpochMilli(2_000L), + changed); + // A payload with no "g": nothing this app ever wrote, but one such record on Drive used + // to stop every device from syncing anything until someone changed a setting locally. + SyncRecord invalidPreferences = + SyncRecord.live( + SyncRecord.Type.PREFERENCES, + "00000000-0000-4000-8000-000000000000", + java.time.Instant.ofEpochMilli(2_000L), + new JsonObject()); + + preferencesStore.applySnapshot( + new SyncSnapshot(java.util.Arrays.asList(remoteNote, invalidPreferences)), + Collections.emptyList()); + + assertThat(db.noteDao().getNoteSync(noteId).getTitle()).isEqualTo("Remote title"); + assertThat(adapter.committed.get()).isNull(); + } + + @Test + public void clearAfterDisconnect_dropsThePendingPreferencesJournal() throws Exception { + db.syncPendingPreferencesDao() + .upsert( + new com.pasich.mynotes.data.database.entities.SyncPendingPreferencesEntity( + 1, "{}", "target", "baseline", 0L, false, 0L, "")); + + store.clearAfterDisconnect(); + + // Left behind, a fresh store replayed the disconnected account's settings onto the + // device at its next seeding. + assertThat(db.syncPendingPreferencesDao().get()).isNull(); + assertThat(db.syncPendingPreferencesDao().getIncludingQuarantined()).isNull(); + } + + // ------------------------------------------------- records deleted here, edited elsewhere + + @Test + public void applySnapshot_bringsBackATaskDeletedHereAndEditedElsewhere() throws Exception { + com.pasich.mynotes.data.model.Task task = new com.pasich.mynotes.data.model.Task("Call", 0); + int taskId = (int) db.taskDao().insertTask(task); + String stableId = "33333333-3333-4333-8333-333333333333"; + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TASK, taskId, stableId, 1_000L, null)); + db.taskDao().deleteById(taskId); + db.syncMetadataDao().markDeleted(SyncMetadata.RECORD_TYPE_TASK, taskId, 1_500L); + JsonObject edited = new com.google.gson.Gson().toJsonTree(task).getAsJsonObject(); + edited.addProperty("title", "Call back"); + SyncMetadata.stripDeviceLocalFields(SyncMetadata.RECORD_TYPE_TASK, edited); + + store.applySnapshot( + new SyncSnapshot( + Collections.singletonList( + SyncRecord.live( + SyncRecord.Type.TASK, + stableId, + java.time.Instant.ofEpochMilli(3_000L), + edited))), + Collections.emptyList()); + + // @Update on the deleted row was a silent no-op while the tombstone was cleared anyway, + // so the task never came back here and the remote edit was re-applied to nothing forever. + com.pasich.mynotes.data.model.Task revived = db.taskDao().getTaskSync(taskId); + assertThat(revived).isNotNull(); + assertThat(revived.getTitle()).isEqualTo("Call back"); + assertThat(db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TASK, taskId).deletedAt) + .isNull(); + } + + // ------------------------------------------------- tags created by name on two devices + + @Test + public void applySnapshot_reconcilesATagCreatedUnderTheSameNameOnAnotherDevice() + throws Exception { + com.pasich.mynotes.data.model.Tag local = + new com.pasich.mynotes.data.model.Tag().create("Work"); + long localId = db.tagsDao().addTag(local); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + localId, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + 1_000L, + null)); + + store.applySnapshot( + new SyncSnapshot( + Collections.singletonList( + remoteTag("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "Work"))), + Collections.emptyList()); + + // One "Work", under the identity both devices will settle on; the other identity is + // tombstoned so the next sync retires it everywhere instead of leaving two rows. + assertThat(tagsNamed("Work")).isEqualTo(1); + SyncMetadataEntity winner = + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_TAG, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"); + assertThat(winner).isNotNull(); + assertThat(winner.deletedAt).isNull(); + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_TAG, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb") + .deletedAt) + .isNotNull(); + } + + @Test + public void applySnapshot_keepsTheLocalTagWhenItHoldsTheWinningIdentity() throws Exception { + com.pasich.mynotes.data.model.Tag local = + new com.pasich.mynotes.data.model.Tag().create("Work"); + long localId = db.tagsDao().addTag(local); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + localId, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + 1_000L, + null)); + + store.applySnapshot( + new SyncSnapshot( + Collections.singletonList( + remoteTag("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", "Work"))), + Collections.emptyList()); + + assertThat(tagsNamed("Work")).isEqualTo(1); + assertThat(db.tagsDao().getTagSync(localId)).isNotNull(); + assertThat( + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_TAG, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")) + .isNull(); + } + + @Test + public void applySnapshot_doesNotReviveADeletedTagBesideItsSameNamedSuccessor() + throws Exception { + // Device A deleted "Work" and created a new "Work"; device B edited the old one later. + // Reviving the old row put a second "Work" beside the new one. The local identity holds + // the smaller id, so the old one is tombstoned afresh and retires everywhere next sync. + long newId = db.tagsDao().addTag(new com.pasich.mynotes.data.model.Tag().create("Work")); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + newId, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + 2_000L, + null)); + long oldId = db.tagsDao().addTag(new com.pasich.mynotes.data.model.Tag().create("Work")); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + oldId, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + 1_000L, + null)); + db.tagsDao().deleteById(oldId); + db.syncMetadataDao().markDeleted(SyncMetadata.RECORD_TYPE_TAG, oldId, 1_500L); + SyncRecord editedElsewhere = + SyncRecord.live( + SyncRecord.Type.TAG, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + java.time.Instant.ofEpochMilli(3_000L), + remoteTag("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", "Work").getPayload()); + + store.applySnapshot( + new SyncSnapshot(Collections.singletonList(editedElsewhere)), + Collections.emptyList()); + + assertThat(tagsNamed("Work")).isEqualTo(1); + assertThat(db.tagsDao().getTagSync(newId)).isNotNull(); + SyncMetadataEntity old = db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, oldId); + assertThat(old.deletedAt).isNotNull(); + assertThat(old.updatedAt).isGreaterThan(3_000L); + } + + @Test + public void applySnapshot_revivesADeletedTagAndRetiresItsSameNamedSuccessorWhenItWins() + throws Exception { + long newId = db.tagsDao().addTag(new com.pasich.mynotes.data.model.Tag().create("Work")); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + newId, + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + 2_000L, + null)); + long oldId = db.tagsDao().addTag(new com.pasich.mynotes.data.model.Tag().create("Work")); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_TAG, + oldId, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + 1_000L, + null)); + db.tagsDao().deleteById(oldId); + db.syncMetadataDao().markDeleted(SyncMetadata.RECORD_TYPE_TAG, oldId, 1_500L); + SyncRecord editedElsewhere = + SyncRecord.live( + SyncRecord.Type.TAG, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + java.time.Instant.ofEpochMilli(3_000L), + remoteTag("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "Work").getPayload()); + + store.applySnapshot( + new SyncSnapshot(Collections.singletonList(editedElsewhere)), + Collections.emptyList()); + + assertThat(tagsNamed("Work")).isEqualTo(1); + assertThat(db.tagsDao().getTagSync(oldId)).isNotNull(); + assertThat(db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, oldId).deletedAt) + .isNull(); + assertThat(db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, newId).deletedAt) + .isNotNull(); + } + + private static SyncRecord remoteTag(String stableId, String name) { + JsonObject payload = new JsonObject(); + payload.addProperty("b", name); + payload.addProperty("c", 0); + payload.addProperty("d", 0); + payload.addProperty("e", -1); + return SyncRecord.live( + SyncRecord.Type.TAG, stableId, java.time.Instant.ofEpochMilli(1_000L), payload); + } + + private int tagsNamed(String name) { + int count = 0; + for (com.pasich.mynotes.data.model.Tag tag : db.tagsDao().getTags().blockingFirst()) { + if (name.equals(tag.getNameTag())) count++; + } + return count; + } + + // ------------------------------------------------- blobs a fresh store has to find + + @Test + public void hasAttachment_findsABlobInANoteFolderWithoutHavingBuiltASnapshot() + throws Exception { + byte[] bytes = "owned bytes".getBytes(StandardCharsets.UTF_8); + seedNoteWithAttachment("owned.png", bytes); + + // The Backup screen's store, resolving a conflict the worker's store found: it has never + // built a snapshot, and the only index of note-folder files used to be built there. + RoomSyncStore fresh = new RoomSyncStore(context, db, mock(PreferenceHelper.class)); + + assertThat(fresh.hasAttachment(sha256(bytes))).isTrue(); + } + + // ------------------------------------------------- attachments whose file is gone + + @Test + public void buildSnapshot_describesAMissingFileFromWhatTheColumnRemembersAndRestoresIt() + throws Exception { + byte[] bytes = "restored later".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + String logicalId = "7d444840-9dc0-11d1-b245-5ffdce74fad2"; + int noteId = seedNote("Repairable", "body", null); + // The column a sync wrote: the file it names is gone, but the id, hash, size and type + // are all there — enough to publish the note and to fetch the bytes back. + Note note = db.noteDao().getNoteSync(noteId); + note.setAttachments( + "[{\"url\":\"" + + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, logicalId + "-" + hash) + + "\",\"name\":\"photo.png\",\"id\":\"" + + logicalId + + "\",\"sha256\":\"" + + hash + + "\",\"size\":" + + bytes.length + + ",\"mimeType\":\"image/png\"}]"); + db.noteDao().addNote(note); + // The blob is in the sync cache, as SyncService puts it after downloading from Drive. + store.writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + + SnapshotBuildResult result = store.buildSnapshot(); + + // Every sync used to end in MISSING_ATTACHMENT here, with no way back but deleting the + // block by hand. + assertThat(result.isPublishable()).isTrue(); + JsonObject payload = onlyNote(result.requireSnapshot()).getPayload(); + assertThat(payload.getAsJsonArray("attachmentHashes").get(0).getAsString()).isEqualTo(hash); + assertThat( + payload.getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .get("mimeType") + .getAsString()) + .isEqualTo("image/png"); + + store.applySnapshot(result.requireSnapshot(), Collections.emptyList()); + + File repaired = resolveFirstAttachment(db.noteDao().getNoteSync(noteId).getAttachments()); + assertThat(repaired.isFile()).isTrue(); + assertThat(readAll(new java.io.FileInputStream(repaired))).isEqualTo(bytes); + } + + @Test + public void buildSnapshot_stillNamesTheNoteWhenARememberedBlobIsNowhereToBeFound() + throws Exception { + // Published from the remembered hash and size, the note is fine as long as the cache or + // Drive holds the bytes. When neither does — a different account, say — the service has + // to be able to name the note, not fail forever on a hash. + byte[] bytes = "gone everywhere".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + String logicalId = "7d444840-9dc0-11d1-b245-5ffdce74fad2"; + int noteId = seedNote("Shopping list", "body", null); + Note note = db.noteDao().getNoteSync(noteId); + note.setAttachments( + "[{\"url\":\"" + + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, logicalId + "-" + hash) + + "\",\"name\":\"photo.png\",\"id\":\"" + + logicalId + + "\",\"sha256\":\"" + + hash + + "\",\"size\":" + + bytes.length + + ",\"mimeType\":\"image/png\"}]"); + db.noteDao().addNote(note); + + SnapshotBuildResult result = store.buildSnapshot(); + + assertThat(result.isPublishable()).isTrue(); + assertThat(store.hasAttachment(hash)).isFalse(); + SnapshotProblem problem = store.describeMissingAttachment(hash); + assertThat(problem).isNotNull(); + assertThat(problem.getKind()).isEqualTo(SnapshotProblem.Kind.MISSING_ATTACHMENT); + assertThat(problem.getLabel()).isEqualTo("Shopping list"); + } + + @Test + public void buildSnapshot_namesTheNoteWhoseAttachmentCannotBeFound() { + int noteId = seedNote("Shopping list", "body", null); + Note seeded = db.noteDao().getNoteSync(noteId); + seeded.setAttachments("[" + attachmentJson(noteId, "gone.png") + "]"); + db.noteDao().addNote(seeded); + + SnapshotBuildResult.SnapshotBuildException error = assertSnapshotBuildFails(store); + + // The account screen shows this string; "MISSING_ATTACHMENT" alone left the user to + // guess which note to open. + assertThat(error).hasMessageThat().contains("note \"Shopping list\""); + assertThat(error.getProblems().get(0).getLabel()).isEqualTo("Shopping list"); } /** Resolves the first entry of an attachments JSON the way the app's consumers do. */ @@ -460,17 +1327,17 @@ record -> { // ---- helpers ---- private int seedNote(String title, String value, String attachmentsJson) { + return seedNote(title, value, attachmentsJson, "11111111-1111-4111-8111-111111111111"); + } + + private int seedNote(String title, String value, String attachmentsJson, String stableId) { Note note = new Note().create(title, value, 1_000L, ""); note.setAttachments(attachmentsJson); int id = db.noteDao().addNote(note).intValue(); db.syncMetadataDao() .insertIfAbsent( new SyncMetadataEntity( - SyncMetadata.RECORD_TYPE_NOTE, - id, - "11111111-1111-4111-8111-111111111111", - 1_000L, - null)); + SyncMetadata.RECORD_TYPE_NOTE, id, stableId, 1_000L, null)); return id; } @@ -775,6 +1642,11 @@ private static com.pasich.mynotes.utils.backup.models.PreferencesBackup preferen /** Writes a real file into the note's own attachment folder and links it from the note. */ private int seedNoteWithAttachment(String fileName, byte[] bytes) throws IOException { + return seedNoteWithAttachment(fileName, fileName, bytes); + } + + private int seedNoteWithAttachment(String fileName, String displayName, byte[] bytes) + throws IOException { int id = seedNote("With attachment", "body", null); File folder = new File(context.getFilesDir(), "attachments/note_" + id); assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); @@ -782,7 +1654,7 @@ private int seedNoteWithAttachment(String fileName, byte[] bytes) throws IOExcep out.write(bytes); } // Production shape: EditorJSInterface writes editorjs://attachments/note_/. - String json = "[" + attachmentJson(id, fileName) + "]"; + String json = "[" + attachmentJson(id, fileName, displayName) + "]"; Note note = db.noteDao().getNoteSync(id); note.setAttachments(json); db.noteDao().addNote(note); @@ -809,10 +1681,15 @@ private static SnapshotBuildResult.SnapshotBuildException assertSnapshotBuildFai /** The canonical reference the editor and sync restore both produce. */ private static String attachmentJson(int noteId, String name) { + return attachmentJson(noteId, name, name); + } + + private static String attachmentJson(int noteId, String fileName, String displayName) { return "{\"url\":\"" - + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor(noteId, name) + + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, fileName) + "\",\"name\":\"" - + name + + displayName + "\"}"; } diff --git a/app/src/main/java/com/pasich/mynotes/base/activity/BaseActivity.java b/app/src/main/java/com/pasich/mynotes/base/activity/BaseActivity.java index 7908e59c..de0cc9b4 100644 --- a/app/src/main/java/com/pasich/mynotes/base/activity/BaseActivity.java +++ b/app/src/main/java/com/pasich/mynotes/base/activity/BaseActivity.java @@ -31,7 +31,7 @@ public abstract class BaseActivity extends AppCompatActivity implements BaseView { - @Inject ThemePreferencesCache themePreferencesCache; + @Inject protected ThemePreferencesCache themePreferencesCache; @Override public void selectTheme() { diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/NoteDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/NoteDao.java index 46a721d1..ff0a6198 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/NoteDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/NoteDao.java @@ -29,6 +29,10 @@ public interface NoteDao { @Query("SELECT * FROM notes WHERE id = :id LIMIT 1") Note getNoteSync(int id); + /** One round trip for a whole restore batch; the caller keeps the list under the bind limit. */ + @Query("SELECT * FROM notes WHERE id IN (:ids)") + List getNotesByIdsSync(List ids); + @Query("DELETE FROM notes WHERE id = :id") void deleteById(int id); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java index 24983967..8c5fb51f 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java @@ -36,6 +36,21 @@ public interface SyncConflictDao { @Query("SELECT * FROM sync_conflicts WHERE id = :id LIMIT 1") SyncConflictEntity getById(long id); + @Query("DELETE FROM sync_conflicts WHERE id = :id") + void deleteById(long id); + + /** + * Retires every open conflict for a record whose winner is no longer the version being applied. + * + *

Such a row offers, pre-selected, a version the record has since moved past; applying it + * reverted the newer edit. The loser it carried is republished with the bundle and comes back + * against the current version. + */ + @Query( + "DELETE FROM sync_conflicts WHERE resolved = 0 AND recordType = :recordType " + + "AND stableId = :stableId AND winnerVersionId != :winnerVersionId") + void deleteSupersededUnresolved(String recordType, String stableId, String winnerVersionId); + @Query( "UPDATE sync_conflicts SET resolution = :resolution, resolved = 1, resolvedAt = :resolvedAt WHERE id = :id") void markResolved(long id, String resolution, long resolvedAt); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncMetadataDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncMetadataDao.java index 26821015..80ab8197 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncMetadataDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncMetadataDao.java @@ -28,6 +28,12 @@ public interface SyncMetadataDao { + "WHERE recordType = :recordType AND localId = :localId)") boolean exists(String recordType, long localId); + /** The subset of {@code localIds} that already has a row; the caller chunks the list. */ + @Query( + "SELECT localId FROM sync_metadata " + + "WHERE recordType = :recordType AND localId IN (:localIds)") + List getExistingLocalIds(String recordType, List localIds); + @Query( "SELECT * FROM sync_metadata WHERE recordType = :recordType AND stableId = :stableId LIMIT 1") SyncMetadataEntity getByStableId(String recordType, String stableId); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java index 8ee0ac99..9f19484f 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java @@ -20,6 +20,9 @@ public interface TagsDao { @Query("SELECT * FROM tags WHERE id = :id LIMIT 1") Tag getTagSync(long id); + @Query("SELECT * FROM tags WHERE id IN (:ids)") + List getTagsByIdsSync(List ids); + /** Tags are referenced by name from a note, so the name is their real identity. */ @Query("SELECT * FROM tags WHERE name = :name LIMIT 1") Tag getTagByNameSync(String name); diff --git a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java index d62f5f80..93c839ba 100644 --- a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java +++ b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java @@ -15,13 +15,28 @@ public class AppPreferencesHelper implements PreferenceHelper { private final ThemePreferencesCache themeCache; private final SafePreferences prefs; + private final java.util.concurrent.Executor mainThread; @Inject AppPreferencesHelper( AppPreferencesCache appCache, ThemePreferencesCache themeCache, SafePreferences prefs) { + this( + appCache, + themeCache, + prefs, + new android.os.Handler(android.os.Looper.getMainLooper())::post); + } + + /** Test seam: where the theme application is posted to. */ + AppPreferencesHelper( + AppPreferencesCache appCache, + ThemePreferencesCache themeCache, + SafePreferences prefs, + java.util.concurrent.Executor mainThread) { this.prefs = prefs; this.appCache = appCache; this.themeCache = themeCache; + this.mainThread = mainThread; this.appCache.initialize(); this.themeCache.initialize(); } @@ -88,10 +103,19 @@ public PreferencesBackup getListPreferences() { PreferencesConfig.ARGUMENT_DEFAULT_UI_SCALING_VALUE)); } - /** Persists all fields from a backup and refreshes the caches. */ + /** + * Persists all fields from a backup and refreshes the caches. + * + *

The restore path. The theme is deliberately not applied here: {@code + * AppCompatDelegate.setDefaultNightMode} recreates every started activity when the mode + * changes, and this runs at the start of a restore whose note and tag inserts are still in + * flight on the Backup screen — recreating it disposed those inserts and left the database half + * restored with no message. The stored mode takes effect at the next activity creation, and the + * screen applies it itself once the restore has finished. + */ @Override public void setListPreferences(PreferencesBackup preferences) { - commitListPreferences(preferences); + commitListPreferences(preferences, false); } /** @@ -103,10 +127,16 @@ public void setListPreferences(PreferencesBackup preferences) { * plus {@code commit()} makes the whole set atomic and tells the caller whether it is durable, * which is what lets {@code RoomSyncStore} decide when the journal may be dropped. * + *

The sync path: a theme arriving from another device is applied at once. + * * @return true when the values are durably stored, false when the write failed. */ @Override public boolean commitListPreferences(PreferencesBackup preferences) { + return commitListPreferences(preferences, true); + } + + private boolean commitListPreferences(PreferencesBackup preferences, boolean applyThemeNow) { if (preferences == null || !preferences.isCreated()) { return false; } @@ -137,12 +167,13 @@ public boolean commitListPreferences(PreferencesBackup preferences) { } appCache.refresh(); themeCache.refresh(); - // Refreshing the caches only reloads the values. Light/dark is owned by - // AppCompatDelegate, which has to be told, or a theme arriving from another device sat - // in storage until the next activity was created. Posted to the main thread because this - // runs on a background thread for both a sync apply and a backup restore. - new android.os.Handler(android.os.Looper.getMainLooper()) - .post(themeCache::applyCurrentThemeMode); + if (applyThemeNow) { + // Refreshing the caches only reloads the values. Light/dark is owned by + // AppCompatDelegate, which has to be told, or a theme arriving from another device + // sat in storage until the next activity was created. Posted to the main thread + // because a sync apply runs on a background thread. + mainThread.execute(themeCache::applyCurrentThemeMode); + } return true; } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentHashCache.java b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentHashCache.java new file mode 100644 index 00000000..cd418128 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentHashCache.java @@ -0,0 +1,149 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Remembers the hash of an attachment file until the file changes. + * + *

Every snapshot build used to read and digest every attachment in the library, and a sync + * builds at least once — twice when the first-sync estimate precedes it — so an idle sync of a + * library with gigabytes of attachments was dominated by re-hashing bytes nothing had touched. A + * file is keyed by its path, size and modification time, the same test a version control index + * uses; a file rewritten in place with identical size within the same millisecond is the one case + * this cannot see, and no writer in this app does that. + * + *

Persisted as JSON next to the download cache so the saving survives the store instance, which + * lives only as long as one sync. Loading, saving and every lookup is best effort: a lost or + * unreadable cache costs one full re-hash, never correctness. + */ +final class AttachmentHashCache { + + /** Produces the hash when the cache has no answer. */ + interface Hasher { + @NonNull + String sha256(@NonNull File file) throws IOException; + } + + private static final class Entry { + final long size; + final long modifiedAt; + final String sha256; + + Entry(long size, long modifiedAt, String sha256) { + this.size = size; + this.modifiedAt = modifiedAt; + this.sha256 = sha256; + } + } + + @Nullable private final File storage; + private final Map entries = new LinkedHashMap<>(); + private boolean loaded; + private boolean dirty; + + /** + * @param storage where the cache persists, or {@code null} to keep it in memory only. + */ + AttachmentHashCache(@Nullable File storage) { + this.storage = storage; + } + + /** The file's hash, from the cache when its size and modification time still match. */ + @NonNull + synchronized String sha256(@NonNull File file, @NonNull Hasher hasher) throws IOException { + load(); + String key = file.getAbsolutePath(); + long size = file.length(); + long modifiedAt = file.lastModified(); + Entry cached = entries.get(key); + if (cached != null && cached.size == size && cached.modifiedAt == modifiedAt) { + return cached.sha256; + } + String hash = hasher.sha256(file); + entries.put(key, new Entry(size, modifiedAt, hash)); + dirty = true; + return hash; + } + + /** Writes the cache if anything changed; a failure here is logged by nobody on purpose. */ + synchronized void flush() { + if (!dirty || storage == null) { + return; + } + JsonObject root = new JsonObject(); + for (Map.Entry entry : entries.entrySet()) { + JsonObject value = new JsonObject(); + value.addProperty("size", entry.getValue().size); + value.addProperty("modifiedAt", entry.getValue().modifiedAt); + value.addProperty("sha256", entry.getValue().sha256); + root.add(entry.getKey(), value); + } + File parent = storage.getParentFile(); + if (parent != null && !parent.isDirectory() && !parent.mkdirs()) { + return; + } + File temporary = new File(parent, storage.getName() + ".tmp"); + try { + Files.write(temporary.toPath(), root.toString().getBytes(StandardCharsets.UTF_8)); + Files.move( + temporary.toPath(), + storage.toPath(), + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.ATOMIC_MOVE); + dirty = false; + } catch (IOException | RuntimeException ignored) { + // The next build simply hashes again. + temporary.delete(); + } + } + + /** Forgets everything, in memory and on disk. */ + synchronized void clear() { + entries.clear(); + loaded = true; + dirty = false; + if (storage != null) { + storage.delete(); + } + } + + private void load() { + if (loaded) { + return; + } + loaded = true; + if (storage == null || !storage.isFile()) { + return; + } + try { + String json = new String(Files.readAllBytes(storage.toPath()), StandardCharsets.UTF_8); + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + for (Map.Entry entry : root.entrySet()) { + JsonObject value = entry.getValue().getAsJsonObject(); + String sha256 = value.get("sha256").getAsString(); + if (!sha256.matches("[0-9a-f]{64}")) { + continue; + } + entries.put( + entry.getKey(), + new Entry( + value.get("size").getAsLong(), + value.get("modifiedAt").getAsLong(), + sha256)); + } + } catch (IOException | RuntimeException unreadable) { + entries.clear(); + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentLogicalIds.java b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentLogicalIds.java new file mode 100644 index 00000000..ef9ab7ed --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentLogicalIds.java @@ -0,0 +1,47 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +/** + * How an attachment that the editor never gave an id receives one. + * + *

Existing editor data predates logical attachment ids, so the store derives one from the note's + * stable id, the attachment's position, its stored URL and its display name. Two releases spell + * this differently: 2.6.50 stopped there, and its ids can therefore describe two different blobs at + * once, which failed every publish for an account whose note had a replaced attachment. Later + * releases fold in the content hash. Both spellings live here because the bundle decoder has to + * recognise the old one to upgrade a 2.6.50 payload into the current shape. + */ +final class AttachmentLogicalIds { + + private AttachmentLogicalIds() {} + + /** The current derivation: one id per (note, position, URL, name, content). */ + @NonNull + static String derive( + @NonNull String noteStableId, + int index, + @NonNull String url, + @NonNull String displayName, + @NonNull String sha256) { + return nameUuid( + noteStableId + "\n" + index + "\n" + url + "\n" + displayName + "\n" + sha256); + } + + /** The 2.6.50 derivation, kept only so its payloads can be recognised. */ + @NonNull + static String deriveLegacy( + @NonNull String noteStableId, + int index, + @NonNull String url, + @NonNull String displayName) { + return nameUuid(noteStableId + "\n" + index + "\n" + url + "\n" + displayName); + } + + @NonNull + private static String nameUuid(@NonNull String source) { + return UUID.nameUUIDFromBytes(source.getBytes(StandardCharsets.UTF_8)).toString(); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentWireUrl.java b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentWireUrl.java new file mode 100644 index 00000000..ccb52b7a --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentWireUrl.java @@ -0,0 +1,44 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import java.util.UUID; + +/** + * The device-independent form of an attachment reference inside a note's editor blocks. + * + *

A block on the device that wrote it names {@code editorjs://attachments/note_7/photo.png}: a + * Room row id and a file name that exist nowhere else. Restoring on another device necessarily + * rewrites it, so the same note hashed differently on every device that held it, and the merge + * reported a conflict against itself on every sync. On the wire a block therefore names the + * attachment's logical id, which is the one identity every device agrees on; each store maps it to + * its own file on the way in and back on the way out. + * + *

The scheme is deliberately one nothing else parses: if a wire reference ever leaked into a + * stored note it would render as a missing file rather than be mistaken for a local path. + */ +final class AttachmentWireUrl { + + private static final String PREFIX = "mynotes-sync://attachment/"; + + private AttachmentWireUrl() {} + + @NonNull + static String forLogicalId(@NonNull String logicalId) { + return PREFIX + logicalId; + } + + /** The logical id a wire reference names, or {@code null} for any other URL. */ + @Nullable + static String logicalIdOf(@Nullable String url) { + if (url == null || !url.startsWith(PREFIX)) { + return null; + } + String id = url.substring(PREFIX.length()); + try { + return UUID.fromString(id).toString().equals(id) ? id : null; + } catch (IllegalArgumentException notAUuid) { + return null; + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index 204f910e..777a8bfa 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -4,6 +4,7 @@ import androidx.annotation.Nullable; import com.google.gson.Gson; import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -15,17 +16,17 @@ import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Clock; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -40,13 +41,41 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private static final String MIME_JSON = "application/json; charset=UTF-8"; private static final String MIME_ZIP = "application/zip"; private static final String MIME_BINARY = "application/octet-stream"; + private static final String PROPERTY_OWNER = "mynotesOwner"; + private static final String PROPERTY_BUNDLE = "mynotesBundle"; + private static final String PROPERTY_BUNDLE_PUBLISHED_AT = "mynotesBundlePublishedAt"; + private static final String PROPERTY_ATTACHMENT_SHA256 = "mynotesAttachmentSha256"; + + /** + * Set on a bundle the first time a read finds it outside the frontier. Its value is Drive's own + * time at that moment, taken from the {@code Date} header of the listing that found it, so the + * grace period is measured on Drive's clock at both ends: the mark here, the current time from + * the latest response. Drive also stamps the update with {@code modifiedTime}, which serves as + * the fallback when a marking device had no server time to record. + */ + private static final String PROPERTY_BUNDLE_SUPERSEDED = "mynotesBundleSuperseded"; + private static final int MAX_BUNDLE_RESPONSE_BYTES = 32 * 1024 * 1024; - private static final int MAX_ATTACHMENT_RESPONSE_BYTES = 100 * 1024 * 1024; + private static final long MAX_ATTACHMENT_RESPONSE_BYTES = + SyncBundleValidator.MAX_ATTACHMENT_BYTES; private static final int RESUMABLE_CHUNK_BYTES = 256 * 1024; private static final int HTTP_RESUME_INCOMPLETE = 308; private static final int MAX_STALLED_CHUNK_ATTEMPTS = 3; private static final int MAX_ERROR_DETAIL_BYTES = 1024; private static final int MAX_ERROR_DETAIL_CHARS = 200; + + /** + * How long a superseded bundle stays after it was first seen to be superseded. + * + *

A device that listed the folder just before the successor was published may still be + * fetching the old bundle; an hour outlives any read, including the six-hourly worker's, which + * WorkManager stops after ten minutes. Measured from the supersession mark, not from the + * bundle's creation: a head created days ago and superseded seconds ago is exactly the file + * another device is most likely to be reading right now. Both ends of the measurement are + * Drive's clock, see {@link #PROPERTY_BUNDLE_SUPERSEDED}. + */ + static final long BUNDLE_PRUNE_GRACE_MILLIS = 60L * 60L * 1000L; + private static final Gson GSON = new Gson(); private final String accessToken; @@ -56,17 +85,40 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private final SyncBundleCodec bundleCodec; private final DriveRequestExecutor requestExecutor; private final SyncMerger merger = new SyncMerger(); - private List lastReadFrontierBundleIds = Collections.emptyList(); + private final long maxAttachmentBytes; private String lastReadToken = ""; + /** Every bundle file the last read saw, so a publish can retire the ones it supersedes. */ + private List lastReadBundles = Collections.emptyList(); + + /** + * Drive's clock, as last reported in a response's {@code Date} header; zero until one arrives. + * + *

The one clock the prune grace is measured on. A phone running an hour fast would otherwise + * see every fresh supersession mark as an hour old and delete a bundle another device was still + * reading. + */ + private long lastDriveTimeMillis; + + /** + * The owned root folders, listed once per sync. + * + *

Every attachment question used to re-list them, so an account with N attachments issued + * several times N folder listings per sync before moving a byte — which is what surfaced as + * Drive rate-limit errors on larger libraries. A backend instance lives for one sync; a root + * created concurrently by another device is the duplicate-root case the read path merges on the + * next sync anyway. + */ + @Nullable private List folderIds; + /** - * Blobs already read and hashed during this sync, keyed by root, hash and expected size. + * Blobs already verified during this sync, keyed by candidate and hash, with their size. * *

One attachment used to be downloaded in full two or three times per sync: once by * hasAttachment, once by the service re-verifying it, and once more while materializing it in * the canonical root. The verification itself is the point, so it still happens — once. */ - private final Set verifiedAttachments = new HashSet<>(); + private final Map verifiedAttachments = new HashMap<>(); public GoogleDriveSyncBackend(@NonNull String accessToken) { this(accessToken, DEFAULT_API, DEFAULT_UPLOAD, Clock.systemUTC(), new SyncBundleCodec()); @@ -78,6 +130,17 @@ public GoogleDriveSyncBackend(@NonNull String accessToken) { @NonNull String uploadBase, @NonNull Clock clock, @NonNull SyncBundleCodec bundleCodec) { + this(accessToken, apiBase, uploadBase, clock, bundleCodec, MAX_ATTACHMENT_RESPONSE_BYTES); + } + + /** Test seam: a small attachment ceiling makes the oversize paths reachable in a test. */ + GoogleDriveSyncBackend( + @NonNull String accessToken, + @NonNull String apiBase, + @NonNull String uploadBase, + @NonNull Clock clock, + @NonNull SyncBundleCodec bundleCodec, + long maxAttachmentBytes) { if (accessToken.trim().isEmpty()) { throw new IllegalArgumentException("accessToken is empty"); } @@ -86,6 +149,7 @@ public GoogleDriveSyncBackend(@NonNull String accessToken) { this.uploadBase = uploadBase; this.clock = clock; this.bundleCodec = bundleCodec; + this.maxAttachmentBytes = maxAttachmentBytes; this.requestExecutor = new DriveRequestExecutor(); } @@ -95,17 +159,21 @@ public String getIdentifier() { return "google-drive"; } + /** The merged remote records alone; a convenience for tests, not part of the protocol. */ @NonNull - @Override - public synchronized SyncSnapshot readSnapshot() throws IOException { + synchronized SyncSnapshot readSnapshot() throws IOException { return readSnapshotResult().getSnapshot(); } + @NonNull @Override public synchronized RemoteSnapshot readSnapshotResult() throws IOException { - List folderIds = findFolderIds(); - if (folderIds.isEmpty()) { - lastReadFrontierBundleIds = Collections.emptyList(); + // A read begins a sync, so it sees the roots as they are now; the listing then serves + // every attachment question until the next read. + folderIds = listFolderIds(); + List roots = folderIds; + if (roots.isEmpty()) { + lastReadBundles = Collections.emptyList(); lastReadToken = UUID.randomUUID().toString(); return new RemoteSnapshot( SyncSnapshot.empty(), @@ -117,19 +185,25 @@ public synchronized RemoteSnapshot readSnapshotResult() throws IOException { } Map bundlesByLogicalId = new HashMap<>(); - Map bytesByLogicalId = new HashMap<>(); - for (String folderId : folderIds) { - for (String bundleId : findBundles(folderId)) { + // Only a digest per logical bundle is retained for the duplicate-copy check; holding + // every bundle's bytes for the whole read grew with the account's history. + Map digestByLogicalId = new HashMap<>(); + List bundleFiles = new ArrayList<>(); + for (String folderId : roots) { + for (BundleFile file : findBundles(folderId)) { byte[] bytes = requestBytes( "GET", - apiBase + "/files/" + bundleId + "?alt=media", + apiBase + "/files/" + file.fileId + "?alt=media", MAX_BUNDLE_RESPONSE_BYTES); SyncBundleCodec.DecodedBundle decoded = bundleCodec.decode(new ByteArrayInputStream(bytes)); - byte[] previousBytes = bytesByLogicalId.putIfAbsent(decoded.getBundleId(), bytes); - if (previousBytes != null) { - if (!java.util.Arrays.equals(previousBytes, bytes)) { + String digest = Sha256.of(bytes); + bundleFiles.add(file.withLogicalId(decoded.getBundleId())); + String previousDigest = + digestByLogicalId.putIfAbsent(decoded.getBundleId(), digest); + if (previousDigest != null) { + if (!previousDigest.equals(digest)) { throw new IOException( "Drive contains conflicting physical copies of one bundle"); } @@ -187,24 +261,62 @@ public synchronized RemoteSnapshot readSnapshotResult() throws IOException { SyncMergeResult.Source.REMOTE)); } - lastReadFrontierBundleIds = Collections.unmodifiableList(new ArrayList<>(frontier)); + markSupersededBundles(bundleFiles, frontier); + lastReadBundles = Collections.unmodifiableList(bundleFiles); lastReadToken = UUID.randomUUID().toString(); return new RemoteSnapshot( merged, conflicts, frontier, alternatives, resolvedAlternativeIds, lastReadToken); } - @Override - public synchronized void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IOException { - throw new IOException( - "A Drive publish requires the read context it was derived from; use publish()"); + /** + * Stamps every bundle that has left the frontier, once, so its grace period starts now. + * + *

Best effort: a bundle that cannot be marked is never pruned, which costs a download per + * sync and nothing else. + */ + private void markSupersededBundles( + @NonNull List bundles, @NonNull Collection frontier) { + for (BundleFile bundle : bundles) { + if (frontier.contains(bundle.logicalId) || bundle.supersededAtMillis != null) { + continue; + } + try { + JsonObject patch = new JsonObject(); + patch.add( + "appProperties", + appProperties( + PROPERTY_BUNDLE_SUPERSEDED, + lastDriveTimeMillis > 0L + ? Long.toString(lastDriveTimeMillis) + : "1")); + // HttpURLConnection has no PATCH; Google's APIs honour the override header. + HttpURLConnection connection = + open("POST", apiBase + "/files/" + bundle.fileId + "?fields=id"); + connection.setRequestProperty("X-HTTP-Method-Override", "PATCH"); + connection.setRequestProperty("Content-Type", MIME_JSON); + connection.setDoOutput(true); + try { + try (OutputStream output = connection.getOutputStream()) { + output.write(jsonBytes(patch)); + } + ensureSuccess(connection); + } finally { + connection.disconnect(); + } + } catch (IOException ignored) { + // Marked at the next read instead; the grace period simply starts later. + } + } } @Override public synchronized void publish(@NonNull SyncPublication publication) throws IOException { // Causal parents used to come from a mutable field, so a write with no preceding read // published a parentless root that permanently forked the DAG. The read that produced - // this publication has to be this backend's most recent one. - String token = publication.getReadContext().getReadToken(); + // this publication has to be this backend's most recent one, and it is that read — not a + // second copy of its frontier kept on this object — that names the parents. + RemoteSnapshot readContext = publication.getReadContext(); + String token = readContext.getReadToken(); if (token.isEmpty() || !token.equals(lastReadToken)) { throw new IOException( "Drive publish is not derived from this backend's latest remote read"); @@ -221,7 +333,7 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO bundleCodec.encode( snapshot, clock.instant(), - lastReadFrontierBundleIds, + readContext.getFrontierBundleIds(), publication.getUnresolvedAlternatives(), publication.getResolvedAlternativeIds()); // Every bundle is immutable. Drive offers no conditional update based on its version @@ -235,10 +347,47 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO // POST is deliberately not blindly retried. The server may have accepted the upload // before the client lost its response; rediscovering the unique name makes that // outcome successful without publishing a second logical bundle. - if (!hasBundleNamed(folderId, bundleName)) { + if (!mayHaveCommitted(uploadFailure) || !hasBundleNamed(folderId, bundleName)) { throw uploadFailure; } } + pruneSupersededBundles(readContext.getFrontierBundleIds()); + } + + /** + * Retires the bundles the one just published makes redundant. + * + *

Nothing ever deleted a bundle, so an account accumulated one full snapshot per changed + * sync, and every later sync downloaded, unzipped and validated all of them to compute a + * frontier of one or two heads. A bundle is a complete snapshot, so everything a superseded + * bundle held — records, tombstones, unresolved alternatives — lives on in its descendants, and + * the read path already tolerates a missing ancestor. A bundle goes only once it has been + * marked superseded for the whole grace period; the heads this publish descended from are not + * even marked yet, and are what a concurrent publisher is about to name as parents. + * + *

Best effort by design: a bundle that cannot be removed costs a download next time, never + * correctness. + */ + private void pruneSupersededBundles(@NonNull Collection frontierBundleIds) { + // Drive's clock against Drive's clock; the device clock is only the fallback when no + // response carried a Date header, which none of Google's do. + long now = lastDriveTimeMillis > 0L ? lastDriveTimeMillis : clock.millis(); + long cutoff = now - BUNDLE_PRUNE_GRACE_MILLIS; + for (BundleFile bundle : lastReadBundles) { + // Only a bundle that was already marked superseded when this sync read it, with the + // mark's Drive-side timestamp older than the grace, may go. A bundle marked during + // this very read, or one whose mark carries no usable time, stays. + if (frontierBundleIds.contains(bundle.logicalId) + || bundle.supersededAtMillis == null + || bundle.supersededAtMillis > cutoff) { + continue; + } + try { + deleteFile(bundle.fileId); + } catch (IOException ignored) { + // Still there next time; the read path copes with it either way. + } + } } /** @@ -251,36 +400,53 @@ public synchronized void publish(@NonNull SyncPublication publication) throws IO @Override public synchronized boolean hasAttachment(@NonNull String sha256) throws IOException { for (String folderId : findFolderIds()) { - if (findAttachment(folderId, sha256) != null) { + if (!listAttachmentCandidates(folderId, sha256).isEmpty()) { return true; } } return false; } + /** + * Opens one copy of the blob, choosing a verified candidate where one is known. + * + *

The stream is not verified here: every caller wraps it in its own verifier, and doing it + * here as well meant downloading the blob twice — once to check it and once to hand it over. + * Only when several unverified copies exist is one read ahead of time, so that a corrupt + * duplicate cannot be the one handed to the caller. + */ @Nullable @Override public synchronized InputStream readAttachment(@NonNull String sha256) throws IOException { - for (String folderId : findFolderIds()) { - String attachmentId = findVerifiedAttachment(folderId, sha256, null); - if (attachmentId == null) { - continue; + List roots = findFolderIds(); + Map> candidatesByRoot = new java.util.LinkedHashMap<>(); + int total = 0; + for (String folderId : roots) { + List candidates = listAttachmentCandidates(folderId, sha256); + candidatesByRoot.put(folderId, candidates); + total += candidates.size(); + for (AttachmentCandidate candidate : candidates) { + if (isVerifiedWithoutReading(candidate, sha256, null)) { + return openAttachment(candidate.id); + } } - // Streamed, not buffered: reading a 100 MB attachment into a byte[] (which the growing - // ByteArrayOutputStream first doubled, then copied) was the largest single allocation - // in - // the sync and an OutOfMemoryError on an ordinary phone. - HttpURLConnection connection = - requestExecutor.executeIdempotent( - () -> - openSuccessful( - "GET", - apiBase + "/files/" + attachmentId + "?alt=media")); - try { - return new ConnectionInputStream(connection, MAX_ATTACHMENT_RESPONSE_BYTES); - } catch (IOException failure) { - connection.disconnect(); - throw failure; + } + if (total == 1) { + // The only copy in the whole account: a corrupt one fails the caller's verifier + // exactly as it would fail one here, and there is nothing else to fall back to. The + // difference is one download instead of two. + for (List candidates : candidatesByRoot.values()) { + if (!candidates.isEmpty()) { + return openAttachment(candidates.get(0).id); + } + } + } + // Several unverified copies, across duplicate roots or within one: read ahead of time so + // a corrupt copy in one root cannot shadow the good copy in another. + for (String folderId : roots) { + String chosen = findVerifiedAttachment(folderId, sha256, null); + if (chosen != null) { + return openAttachment(chosen); } } return null; @@ -294,67 +460,77 @@ public synchronized void writeAttachment( if (findVerifiedAttachment(folderId, sha256, sizeBytes >= 0L ? sizeBytes : null) != null) { return; } - if (sizeBytes >= 0L) { - if (sizeBytes > MAX_ATTACHMENT_RESPONSE_BYTES) { - throw new IOException("Attachment exceeds the 100 MiB sync upload limit"); - } - uploadAttachmentOrConfirm(folderId, sha256, content, sizeBytes); + if (sizeBytes < 0L) { + // No declared size, so the content length cannot be computed up front. Rare: sizes + // come from the bundle manifest, which also supplies the hashes being uploaded. + byte[] buffered = + readBounded( + content, + maxAttachmentBytes, + "Attachment exceeds the 100 MiB sync upload limit"); + uploadAttachmentOrConfirm( + folderId, sha256, new ByteArrayInputStream(buffered), buffered.length); return; } - // No declared size, so the multipart content length cannot be computed up front. Rare: - // sizes come from the bundle manifest, which also supplies the hashes being uploaded. - uploadAttachmentOrConfirm( - folderId, sha256, readFullyLimited(content, MAX_ATTACHMENT_RESPONSE_BYTES)); + if (sizeBytes > maxAttachmentBytes) { + throw new IOException("Attachment exceeds the 100 MiB sync upload limit"); + } + uploadAttachmentOrConfirm(folderId, sha256, content, sizeBytes); } + @NonNull private List findFolderIds() throws IOException { + if (folderIds == null) { + folderIds = listFolderIds(); + } + return folderIds; + } + + @NonNull + private List listFolderIds() throws IOException { JsonArray folders = listFiles( "mimeType = '" + MIME_FOLDER + "' and trashed = false and " - + appPropertyClause("mynotesOwner", "1"), + + appPropertyClause(PROPERTY_OWNER, "1"), "files(id,name)"); List result = new ArrayList<>(folders.size()); for (int index = 0; index < folders.size(); index++) { result.add(folders.get(index).getAsJsonObject().get("id").getAsString()); } result.sort(Comparator.naturalOrder()); - return result; - } - - /** Deterministically selects one byte-identical content-addressed attachment duplicate. */ - @Nullable - private static String smallestId(@NonNull JsonArray files) { - String selected = null; - for (int index = 0; index < files.size(); index++) { - String id = files.get(index).getAsJsonObject().get("id").getAsString(); - if (selected == null || id.compareTo(selected) < 0) { - selected = id; - } - } - return selected; + return Collections.unmodifiableList(result); } @NonNull private String ensureCanonicalFolderId() throws IOException { - List folderIds = findFolderIds(); - if (!folderIds.isEmpty()) { - return folderIds.get(0); + List roots = findFolderIds(); + if (roots.isEmpty()) { + // The cached answer may predate another device's first sync; only a fresh listing + // may justify creating a root. + roots = listFolderIds(); + folderIds = roots; + } + if (!roots.isEmpty()) { + return roots.get(0); } JsonObject metadata = new JsonObject(); metadata.addProperty("name", FOLDER_NAME); metadata.addProperty("mimeType", MIME_FOLDER); - metadata.add("appProperties", appProperties("mynotesOwner", "1")); + metadata.add("appProperties", appProperties(PROPERTY_OWNER, "1")); try { - return uploadMetadata(metadata); + String created = uploadMetadata(metadata); + folderIds = Collections.singletonList(created); + return created; } catch (IOException createFailure) { // Folder POST can have committed before a lost response. Duplicate roots are a // supported read state; rediscovery avoids a blind retry creating another one. - folderIds = findFolderIds(); - if (!folderIds.isEmpty()) { - return folderIds.get(0); + roots = listFolderIds(); + folderIds = roots; + if (!roots.isEmpty()) { + return roots.get(0); } throw createFailure; } @@ -380,8 +556,8 @@ private void materializeAttachmentsInCanonicalRoot( if (source == null) { throw new IOException("Required attachment is unavailable in any Drive root"); } - try (VerifiedAttachmentInputStream input = - new VerifiedAttachmentInputStream(source, hash, attachment.getValue())) { + try (VerifyingInputStream input = + new VerifyingInputStream(source, hash, attachment.getValue())) { uploadAttachmentOrConfirm(canonicalRootId, hash, input, attachment.getValue()); input.verifyEndOfStream(); } @@ -411,7 +587,7 @@ private void ensureCanonicalAlternativeAttachments( } @NonNull - private static Map attachmentSizes(@NonNull Collection notes) + private Map attachmentSizes(@NonNull Collection notes) throws IOException { Map sizes = new HashMap<>(); for (SyncRecord record : notes) { @@ -426,7 +602,7 @@ private static Map attachmentSizes(@NonNull Collection } String hash = entry.get("sha256").getAsString(); long size = entry.get("size").getAsLong(); - if (size < 0L || size > MAX_ATTACHMENT_RESPONSE_BYTES) { + if (size < 0L || size > maxAttachmentBytes) { throw new IOException("Attachment size exceeds the sync limit"); } Long previous = sizes.putIfAbsent(hash, size); @@ -451,32 +627,62 @@ private static Map attachmentSizes(@NonNull Collection */ private static void validateBundleDag( @NonNull Map bundles) throws IOException { - Set visiting = new HashSet<>(); - Set visited = new HashSet<>(); - for (String bundleId : bundles.keySet()) { - validateAcyclic(bundleId, bundles, visiting, visited); + Map> parentsById = new HashMap<>(); + for (Map.Entry entry : bundles.entrySet()) { + parentsById.put(entry.getKey(), entry.getValue().getParentBundleIds()); } + validateAncestry(parentsById); } - private static void validateAcyclic( - @NonNull String bundleId, - @NonNull Map bundles, - @NonNull Set visiting, - @NonNull Set visited) + /** + * Rejects a cycle in the parent graph. + * + *

Iterative on purpose: the recursive walk went one frame deeper per ancestor, so a long + * linear history — exactly what an account that syncs after every edit accumulates — could + * overflow the worker's stack, and a {@code StackOverflowError} is not an {@code IOException} + * the sync knows how to report. + */ + static void validateAncestry(@NonNull Map> parentsById) throws IOException { - if (visited.contains(bundleId)) return; - SyncBundleCodec.DecodedBundle bundle = bundles.get(bundleId); - if (bundle == null) { - // An ancestor that is no longer stored. Nothing to walk and nothing to lose. - return; + Set visited = new HashSet<>(); + Set visiting = new HashSet<>(); + Deque stack = new ArrayDeque<>(); + for (String root : parentsById.keySet()) { + if (visited.contains(root)) { + continue; + } + visiting.add(root); + stack.push(new Frame(root, parentsById.get(root).iterator())); + while (!stack.isEmpty()) { + Frame frame = stack.peek(); + if (!frame.parents.hasNext()) { + stack.pop(); + visiting.remove(frame.bundleId); + visited.add(frame.bundleId); + continue; + } + String parent = frame.parents.next(); + Collection grandparents = parentsById.get(parent); + if (grandparents == null || visited.contains(parent)) { + // An ancestor that is no longer stored, or one already walked. + continue; + } + if (!visiting.add(parent)) { + throw new IOException("Drive bundle ancestry contains a cycle"); + } + stack.push(new Frame(parent, grandparents.iterator())); + } } - if (!visiting.add(bundleId)) - throw new IOException("Drive bundle ancestry contains a cycle"); - for (String parent : bundle.getParentBundleIds()) { - validateAcyclic(parent, bundles, visiting, visited); + } + + private static final class Frame { + private final String bundleId; + private final Iterator parents; + + private Frame(String bundleId, Iterator parents) { + this.bundleId = bundleId; + this.parents = parents; } - visiting.remove(bundleId); - visited.add(bundleId); } @NonNull @@ -494,68 +700,152 @@ private static List computeFrontier( return frontier; } + /** One physical bundle file in a root; the logical id is known once it has been decoded. */ + private static final class BundleFile { + private final String fileId; + @Nullable private final String logicalId; + + /** Drive's modifiedTime of the supersession mark, or null when unmarked or undated. */ + @Nullable private final Long supersededAtMillis; + + private BundleFile( + @NonNull String fileId, + @Nullable String logicalId, + @Nullable Long supersededAtMillis) { + this.fileId = fileId; + this.logicalId = logicalId; + this.supersededAtMillis = supersededAtMillis; + } + + @NonNull + private BundleFile withLogicalId(@NonNull String id) { + return new BundleFile(fileId, id, supersededAtMillis); + } + } + @NonNull - private List findBundles(@NonNull String folderId) throws IOException { + private List findBundles(@NonNull String folderId) throws IOException { JsonArray bundles = listFiles( - "'" - + folderId - + "' in parents and trashed = false and " - + appPropertyClause("mynotesBundle", "1"), - "files(id,name)"); - List result = new ArrayList<>(bundles.size()); + ownedFilesQuery(folderId, PROPERTY_BUNDLE, "1"), + "files(id,name,modifiedTime,appProperties)"); + List result = new ArrayList<>(bundles.size()); for (int index = 0; index < bundles.size(); index++) { - result.add(bundles.get(index).getAsJsonObject().get("id").getAsString()); - } - result.sort(Comparator.naturalOrder()); + JsonObject file = bundles.get(index).getAsJsonObject(); + result.add( + new BundleFile( + file.get("id").getAsString(), + null, + supersededAtOf( + file.getAsJsonObject("appProperties"), + optionalString(file, "modifiedTime")))); + } + result.sort(Comparator.comparing(file -> file.fileId)); return result; } + /** + * When the bundle was marked superseded, on Drive's clock: the server time the marking device + * recorded in the property, else Drive's {@code modifiedTime} of that update; null when + * unmarked or when neither is usable. + */ @Nullable - private String findAttachment(@NonNull String folderId, @NonNull String sha256) - throws IOException { + private static Long supersededAtOf( + @Nullable JsonObject appProperties, @Nullable String modifiedTime) { + if (appProperties == null || !appProperties.has(PROPERTY_BUNDLE_SUPERSEDED)) { + return null; + } + try { + long recorded = + Long.parseLong(appProperties.get(PROPERTY_BUNDLE_SUPERSEDED).getAsString()); + if (recorded > 1L) { + return recorded; + } + } catch (RuntimeException notATimestamp) { + // A marker without a time; Drive's own stamp of the update stands in. + } + return instantOf(modifiedTime); + } + + /** Drive reports times as RFC 3339; anything else counts as unknown. */ + @Nullable + private static Long instantOf(@Nullable String time) { + if (time == null) { + return null; + } + try { + return java.time.Instant.parse(time).toEpochMilli(); + } catch (RuntimeException malformed) { + return null; + } + } + + /** One Drive object indexed under a hash, with what Drive itself says about its bytes. */ + private static final class AttachmentCandidate { + private final String id; + @Nullable private final Long size; + @Nullable private final String sha256Checksum; + + private AttachmentCandidate( + @NonNull String id, @Nullable Long size, @Nullable String sha256Checksum) { + this.id = id; + this.size = size; + this.sha256Checksum = sha256Checksum; + } + } + + /** Every object in the root indexed under {@code sha256}, smallest id first. */ + @NonNull + private List listAttachmentCandidates( + @NonNull String folderId, @NonNull String sha256) throws IOException { JsonArray files = listFiles( - "'" - + folderId - + "' in parents and trashed = false and " - + appPropertyClause("mynotesAttachmentSha256", sha256), - "files(id,name)"); + ownedFilesQuery(folderId, PROPERTY_ATTACHMENT_SHA256, sha256), + "files(id,name,size,sha256Checksum)"); + List candidates = new ArrayList<>(files.size()); + for (int index = 0; index < files.size(); index++) { + JsonObject file = files.get(index).getAsJsonObject(); + candidates.add( + new AttachmentCandidate( + file.get("id").getAsString(), + optionalLong(file, "size"), + optionalString(file, "sha256Checksum"))); + } // Attachments are content-addressed, so duplicates uploaded by two devices racing on the // same hash are byte-identical and either one will do. Rejecting them used to break every // subsequent sync permanently. - return smallestId(files); + candidates.sort(Comparator.comparing(candidate -> candidate.id)); + return candidates; } /** - * An app property is only an index. Read and verify every candidate before it may satisfy a + * An app property is only an index. Every candidate is verified before it may satisfy a * content-addressed reference; corrupt candidates remain harmless Drive orphans. + * + *

Drive computes a checksum over the bytes it stores and reports it with the listing, so a + * candidate is usually verified without a download; only an object Drive has not checksummed is + * read. The old version read every candidate in full on every sync, which for a library of a + * few hundred megabytes meant re-downloading all of it on every six-hourly run. */ @Nullable private String findVerifiedAttachment( @NonNull String folderId, @NonNull String sha256, @Nullable Long expectedSize) throws IOException { - JsonArray files = - listFiles( - "'" - + folderId - + "' in parents and trashed = false and " - + appPropertyClause("mynotesAttachmentSha256", sha256), - "files(id,name)"); - List candidateIds = new ArrayList<>(files.size()); - for (int index = 0; index < files.size(); index++) { - candidateIds.add(files.get(index).getAsJsonObject().get("id").getAsString()); - } - candidateIds.sort(Comparator.naturalOrder()); - for (String candidateId : candidateIds) { - String cacheKey = candidateId + "\u0000" + sha256 + "\u0000" + expectedSize; - if (verifiedAttachments.contains(cacheKey)) { - return candidateId; + for (AttachmentCandidate candidate : listAttachmentCandidates(folderId, sha256)) { + if (isVerifiedWithoutReading(candidate, sha256, expectedSize)) { + return candidate.id; + } + if (candidate.sha256Checksum != null && !candidate.sha256Checksum.equals(sha256)) { + // Drive's own digest disagrees with the index; reading would only confirm it. + // Anything less definite — a matching digest without a usable size, or one the + // expected size disagrees with — is read and verified like an unlisted object, + // rather than counted as corrupt and re-uploaded on every sync. + continue; } - try (InputStream candidate = openAttachment(candidateId)) { - verifyAttachment(candidate, sha256, expectedSize); - verifiedAttachments.add(cacheKey); - return candidateId; + try (InputStream content = openAttachment(candidate.id)) { + long size = VerifyingInputStream.verify(content, sha256, expectedSize); + verifiedAttachments.put(memoKey(candidate.id, sha256), size); + return candidate.id; } catch (AttachmentIntegrityException corrupt) { // A second content-addressed duplicate may be valid. Never accept the property // alone and never delete this object during a correctness path. @@ -564,10 +854,31 @@ private String findVerifiedAttachment( return null; } + /** True when this sync, or Drive's own checksum, already vouches for the candidate's bytes. */ + private boolean isVerifiedWithoutReading( + @NonNull AttachmentCandidate candidate, + @NonNull String sha256, + @Nullable Long expectedSize) { + Long verifiedSize = verifiedAttachments.get(memoKey(candidate.id, sha256)); + if (verifiedSize == null + && sha256.equals(candidate.sha256Checksum) + && candidate.size != null + && candidate.size <= maxAttachmentBytes) { + verifiedSize = candidate.size; + verifiedAttachments.put(memoKey(candidate.id, sha256), verifiedSize); + } + return verifiedSize != null && (expectedSize == null || expectedSize.equals(verifiedSize)); + } + + @NonNull + private static String memoKey(@NonNull String candidateId, @NonNull String sha256) { + return candidateId + " " + sha256; + } + /** * True only when a remote blob exists and its actual bytes hash to {@code sha256}. * - *

Drive's {@code appProperties} index is a claim, not proof, so the bytes are read. The + *

Drive's {@code appProperties} index is a claim, not proof, so the bytes are checked. The * result is remembered for this sync so the caller does not have to download the blob again * purely to repeat the same check. */ @@ -584,58 +895,22 @@ public synchronized boolean hasVerifiedAttachment( @NonNull private InputStream openAttachment(@NonNull String attachmentId) throws IOException { + // Streamed, not buffered: reading a 100 MB attachment into a byte[] (which the growing + // ByteArrayOutputStream first doubled, then copied) was the largest single allocation in + // the sync and an OutOfMemoryError on an ordinary phone. HttpURLConnection connection = requestExecutor.executeIdempotent( () -> openSuccessful( "GET", apiBase + "/files/" + attachmentId + "?alt=media")); try { - return new ConnectionInputStream(connection, MAX_ATTACHMENT_RESPONSE_BYTES); + return new ConnectionInputStream(connection, maxAttachmentBytes); } catch (IOException failure) { connection.disconnect(); throw failure; } } - private static void verifyAttachment( - @NonNull InputStream input, @NonNull String expectedHash, @Nullable Long expectedSize) - throws IOException { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - long size = 0L; - byte[] buffer = new byte[8192]; - int read; - while ((read = input.read(buffer)) != -1) { - digest.update(buffer, 0, read); - size += read; - if (size > MAX_ATTACHMENT_RESPONSE_BYTES) { - throw new AttachmentIntegrityException("Attachment exceeds the sync size limit"); - } - } - String actual = toHex(digest.digest()); - if (!expectedHash.equals(actual)) { - throw new AttachmentIntegrityException( - "Attachment checksum does not match its declared hash"); - } - if (expectedSize != null && expectedSize.longValue() != size) { - throw new AttachmentIntegrityException( - "Attachment size does not match its declared size"); - } - } - - @NonNull - private static String toHex(@NonNull byte[] bytes) { - StringBuilder value = new StringBuilder(bytes.length * 2); - for (byte byteValue : bytes) { - value.append(String.format(Locale.US, "%02x", byteValue & 0xff)); - } - return value.toString(); - } - @NonNull private JsonArray listFiles(@NonNull String query, @NonNull String fields) throws IOException { JsonArray result = new JsonArray(); @@ -677,6 +952,20 @@ private String uploadMetadata(@NonNull JsonObject metadata) throws IOException { return created.get("id").getAsString(); } + /** Permanently removes one file this app created; bundles are complete, so nothing is lost. */ + private void deleteFile(@NonNull String fileId) throws IOException { + HttpURLConnection connection = open("DELETE", apiBase + "/files/" + fileId); + try { + int code = connection.getResponseCode(); + if (code == HttpURLConnection.HTTP_NOT_FOUND) { + return; + } + ensureSuccess(connection); + } finally { + connection.disconnect(); + } + } + private boolean hasBundleNamed(@NonNull String folderId, @NonNull String name) throws IOException { JsonArray bundles = @@ -686,11 +975,21 @@ private boolean hasBundleNamed(@NonNull String folderId, @NonNull String name) + "' in parents and trashed = false and name = '" + escapeQuery(name) + "' and " - + appPropertyClause("mynotesBundle", "1"), + + appPropertyClause(PROPERTY_BUNDLE, "1"), "files(id)"); return bundles.size() > 0; } + /** The one spelling of "our file, in this root, indexed under this property". */ + @NonNull + private static String ownedFilesQuery( + @NonNull String folderId, @NonNull String key, @NonNull String value) { + return "'" + + folderId + + "' in parents and trashed = false and " + + appPropertyClause(key, value); + } + private void uploadFile( @NonNull String folderId, @NonNull String name, @@ -729,7 +1028,7 @@ private void uploadEmptyAttachment( @NonNull InputStream content) throws IOException { if (content.read() != -1) { - throw new IOException("Attachment exceeds its declared size"); + throw new AttachmentIntegrityException("Attachment exceeds its declared size"); } uploadMultipart(folderId, name, mimeType, new ByteArrayInputStream(new byte[0]), 0L, false); } @@ -744,6 +1043,12 @@ private void uploadEmptyAttachment( * never terminated, and it replayed the buffer under offsets past the end of the file. Nothing * here is derived — the buffer window is recomputed from absolute offsets on every pass, so a * byte can only ever be sent under the one offset it occupies in the source. + * + *

Every request but the last carries a whole 256 KiB window, which is what Drive's resumable + * protocol requires. After a partial acknowledgement the window therefore slides: the + * unacknowledged tail moves to the front of the buffer and the window is refilled from the + * source. Sending only the tail — a short chunk that is not the last — was answered with a 400 + * that nothing retries. */ private void uploadResumableAttachment( @NonNull String folderId, @@ -762,31 +1067,36 @@ private void uploadResumableAttachment( while (acknowledgedExclusive < sizeBytes) { throwIfInterrupted(); - if (acknowledgedExclusive >= bufferStart + bufferLength) { - // Everything buffered is durable; read the next window from the source. + if (acknowledgedExclusive > bufferStart) { + // Everything before the acknowledged offset is durable; keep only the tail. + int consumed = (int) (acknowledgedExclusive - bufferStart); + if (consumed >= bufferLength) { + bufferLength = 0; + } else { + System.arraycopy(buffer, consumed, buffer, 0, bufferLength - consumed); + bufferLength -= consumed; + } bufferStart = acknowledgedExclusive; - bufferLength = - readChunk( - content, - buffer, - (int) Math.min(buffer.length, sizeBytes - bufferStart)); - if (bufferLength <= 0) { - throw new IOException("Attachment ended before its declared size"); + } + int wanted = (int) Math.min(buffer.length, sizeBytes - bufferStart); + if (bufferLength < wanted) { + bufferLength += readChunk(content, buffer, bufferLength, wanted - bufferLength); + if (bufferLength < wanted) { + // The source disagrees with its own manifest: an integrity failure, so it + // is never mistaken for a lost response worth confirming by discovery. + throw new AttachmentIntegrityException( + "Attachment ended before its declared size"); } } - int offsetInBuffer = (int) (acknowledgedExclusive - bufferStart); - int length = bufferLength - offsetInBuffer; - long chunkStart = acknowledgedExclusive; + int length = bufferLength; + long chunkStart = bufferStart; long chunkEndExclusive = chunkStart + length; // A chunk PUT is idempotent: it is addressed by an absolute Content-Range, so a // replay of the identical range either lands at the same offset or is already // committed. Retrying is therefore safe, and it keeps one transient 5xx between // chunks from discarding a large upload that is nearly complete. - final int retryOffset = offsetInBuffer; - final int retryLength = length; - final long retryStart = chunkStart; long reported = requestExecutor.executeIdempotent( () -> @@ -794,9 +1104,9 @@ private void uploadResumableAttachment( sessionUrl, mimeType, buffer, - retryOffset, - retryLength, - retryStart, + 0, + length, + chunkStart, sizeBytes)); if (reported < acknowledgedExclusive) { @@ -822,7 +1132,7 @@ private void uploadResumableAttachment( } if (content.read() != -1) { - throw new IOException("Attachment exceeds its declared size"); + throw new AttachmentIntegrityException("Attachment exceeds its declared size"); } } @@ -925,17 +1235,21 @@ private static long resumableAcknowledgedExclusive(@Nullable String range) throw } } - private static int readChunk(@NonNull InputStream input, @NonNull byte[] buffer, int maximum) + /** + * Fills {@code buffer} from {@code offset} with up to {@code count} bytes, or to end of input. + */ + private static int readChunk( + @NonNull InputStream input, @NonNull byte[] buffer, int offset, int count) throws IOException { - int offset = 0; - while (offset < maximum) { - int read = input.read(buffer, offset, maximum - offset); + int filled = 0; + while (filled < count) { + int read = input.read(buffer, offset + filled, count - filled); if (read == -1) { break; } - offset += read; + filled += read; } - return offset; + return filled; } private static void throwIfInterrupted() throws IOException { @@ -954,7 +1268,7 @@ private static JsonObject attachmentMetadata(@NonNull String folderId, @NonNull parents.add(folderId); metadata.add("parents", parents); JsonObject properties = new JsonObject(); - properties.addProperty("mynotesAttachmentSha256", sha256); + properties.addProperty(PROPERTY_ATTACHMENT_SHA256, sha256); metadata.add("appProperties", properties); return metadata; } @@ -965,47 +1279,53 @@ private void uploadAttachmentOrConfirm( @NonNull InputStream content, long sizeBytes) throws IOException { + if (sizeBytes < 0L) { + throw new IllegalArgumentException("An attachment upload needs a declared size"); + } try { - if (sizeBytes >= 0L) { - uploadStream(folderId, sha256, MIME_BINARY, content, sizeBytes); - } else { - uploadFile(folderId, sha256, MIME_BINARY, readFully(content), false); - } + uploadStream(folderId, sha256, MIME_BINARY, content, sizeBytes); } catch (IOException uploadFailure) { // Attachment identity is its SHA-256. A successful request whose response was lost is // confirmed by discovery, not repeated with an already-consumed stream. - if (!isAmbiguousTransportFailure(uploadFailure) + if (!mayHaveCommitted(uploadFailure) || findVerifiedAttachment(folderId, sha256, sizeBytes) == null) { throw uploadFailure; } } } - private void uploadAttachmentOrConfirm( - @NonNull String folderId, @NonNull String sha256, @NonNull byte[] content) - throws IOException { - try { - uploadFile(folderId, sha256, MIME_BINARY, content, false); - } catch (IOException uploadFailure) { - if (!isAmbiguousTransportFailure(uploadFailure) - || findVerifiedAttachment(folderId, sha256, (long) content.length) == null) { - throw uploadFailure; - } + /** + * Whether a failed create may nonetheless have been committed by Drive. + * + *

The one answer for bundle and attachment uploads alike; they used to classify differently, + * so a 429 on the bundle POST was followed by rediscovery while the same 429 on an attachment + * POST failed the sync outright. A response Drive definitely never acted on — a rejected + * request, a bad token, a blob that failed its own checksum — is a plain failure. Everything + * else, including a read timeout that arrived after the whole body was sent, is worth one + * listing to find out. + */ + static boolean mayHaveCommitted(@NonNull IOException failure) { + if (failure instanceof AttachmentIntegrityException) { + return false; } - } - - private static boolean isAmbiguousTransportFailure(@NonNull IOException failure) { - if (failure instanceof AttachmentIntegrityException - || failure instanceof java.io.InterruptedIOException) { + // A subclass of InterruptedIOException, so it has to be asked about first: a timeout + // waiting for the response is precisely the case where the upload may have landed. + if (failure instanceof java.net.SocketTimeoutException) { + return true; + } + if (failure instanceof java.io.InterruptedIOException) { return false; } if (failure instanceof DriveRequestExecutor.DriveHttpException) { - int status = ((DriveRequestExecutor.DriveHttpException) failure).statusCode; - return status >= 500 && status <= 599; + DriveRequestExecutor.DriveHttpException http = + (DriveRequestExecutor.DriveHttpException) failure; + int status = http.statusCode; + if (status >= 500 || status == 408 || status == 429) { + return true; + } + return status == 403 && http.isRateLimit(); } - return failure instanceof java.net.SocketException - || failure instanceof java.net.SocketTimeoutException - || failure instanceof java.net.ConnectException; + return true; } /** @@ -1033,10 +1353,10 @@ private void uploadMultipart( JsonObject appProperties = new JsonObject(); if (bundleFile) { - appProperties.addProperty("mynotesBundle", "1"); - appProperties.addProperty("mynotesBundlePublishedAt", Long.toString(clock.millis())); + appProperties.addProperty(PROPERTY_BUNDLE, "1"); + appProperties.addProperty(PROPERTY_BUNDLE_PUBLISHED_AT, Long.toString(clock.millis())); } else { - appProperties.addProperty("mynotesAttachmentSha256", name); + appProperties.addProperty(PROPERTY_ATTACHMENT_SHA256, name); } metadata.add("appProperties", appProperties); @@ -1135,9 +1455,19 @@ private JsonObject requestJsonIdempotent( @NonNull private JsonObject readJsonResponse(@NonNull HttpURLConnection connection) throws IOException { ensureSuccess(connection); + long serverTime = connection.getHeaderFieldDate("Date", 0L); + if (serverTime > 0L) { + lastDriveTimeMillis = serverTime; + } try (InputStream input = connection.getInputStream()) { return GSON.fromJson( - new String(readFully(input), StandardCharsets.UTF_8), JsonObject.class); + new String( + readBounded( + input, + MAX_BUNDLE_RESPONSE_BYTES, + "Drive response exceeds the sync size limit"), + StandardCharsets.UTF_8), + JsonObject.class); } } @@ -1154,16 +1484,7 @@ private byte[] requestBytesOnce(@NonNull String method, @NonNull String url, int try { ensureSuccess(connection); try (InputStream input = connection.getInputStream()) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - byte[] buffer = new byte[8192]; - int read; - while ((read = input.read(buffer)) != -1) { - if (output.size() > maxBytes - read) { - throw new IOException("Drive response exceeds the sync size limit"); - } - output.write(buffer, 0, read); - } - return output.toByteArray(); + return readBounded(input, maxBytes, "Drive response exceeds the sync size limit"); } } finally { connection.disconnect(); @@ -1237,34 +1558,21 @@ private static String readErrorDetail(@Nullable InputStream error) { } } + /** Reads to end of stream, refusing anything past {@code maxBytes}; never closes the input. */ @NonNull - private static byte[] readFully(@NonNull InputStream input) throws IOException { - try (InputStream stream = input; - ByteArrayOutputStream output = new ByteArrayOutputStream()) { - byte[] buffer = new byte[8192]; - int read; - while ((read = stream.read(buffer)) != -1) { - output.write(buffer, 0, read); - } - return output.toByteArray(); - } - } - - @NonNull - private static byte[] readFullyLimited(@NonNull InputStream input, int maxBytes) + private static byte[] readBounded( + @NonNull InputStream input, long maxBytes, @NonNull String limitMessage) throws IOException { - try (InputStream stream = input; - ByteArrayOutputStream output = new ByteArrayOutputStream()) { - byte[] buffer = new byte[8192]; - int read; - while ((read = stream.read(buffer)) != -1) { - if (output.size() > maxBytes - read) { - throw new IOException("Attachment exceeds the 100 MiB sync upload limit"); - } - output.write(buffer, 0, read); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + if (output.size() > maxBytes - read) { + throw new IOException(limitMessage); } - return output.toByteArray(); + output.write(buffer, 0, read); } + return output.toByteArray(); } @NonNull @@ -1293,11 +1601,34 @@ private static String escapeQuery(@NonNull String value) { return value.replace("\\", "\\\\").replace("'", "\\'"); } + @Nullable + private static String optionalString(@NonNull JsonObject object, @NonNull String field) { + JsonElement value = object.get(field); + return value == null || value.isJsonNull() || !value.isJsonPrimitive() + ? null + : value.getAsString(); + } + + @Nullable + private static Long optionalLong(@NonNull JsonObject object, @NonNull String field) { + String value = optionalString(object, field); + if (value == null) { + return null; + } + try { + return Long.parseLong(value); + } catch (NumberFormatException malformed) { + return null; + } + } + /** * A response body that stays attached to its connection until the reader is done. * *

Lets an attachment be piped straight from the socket to disk while still enforcing the - * response ceiling, and releases the connection on close. + * response ceiling, and releases the connection on close. Exceeding the ceiling is an integrity + * failure like any other: a candidate that grows past it is skipped in favour of the next copy + * rather than failing the sync, as a plain I/O error used to. */ private static final class ConnectionInputStream extends FilterInputStream { private final HttpURLConnection connection; @@ -1332,7 +1663,7 @@ public int read(byte[] buffer, int offset, int length) throws IOException { private void count(int read) throws IOException { byteCount += read; if (byteCount > maxBytes) { - throw new IOException("Drive response exceeds the sync size limit"); + throw new AttachmentIntegrityException("Attachment exceeds the sync size limit"); } } @@ -1345,68 +1676,4 @@ public void close() throws IOException { } } } - - /** Verifies an untrusted remote blob before it may support canonical bundle publication. */ - private static final class VerifiedAttachmentInputStream extends FilterInputStream { - private final String expectedHash; - private final long expectedSize; - private final MessageDigest digest; - private long size; - private boolean reachedEnd; - - private VerifiedAttachmentInputStream( - @NonNull InputStream source, @NonNull String expectedHash, long expectedSize) - throws IOException { - super(source); - this.expectedHash = expectedHash; - this.expectedSize = expectedSize; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - } - - @Override - public int read() throws IOException { - int value = super.read(); - if (value == -1) { - reachedEnd = true; - } else { - digest.update((byte) value); - size++; - } - return value; - } - - @Override - public int read(byte[] buffer, int offset, int length) throws IOException { - int read = super.read(buffer, offset, length); - if (read == -1) { - reachedEnd = true; - } else if (read > 0) { - digest.update(buffer, offset, read); - size += read; - } - return read; - } - - private void verifyEndOfStream() throws IOException { - if (!reachedEnd) { - throw new IOException("Attachment upload ended before the source was verified"); - } - if (size != expectedSize) { - throw new AttachmentIntegrityException( - "Attachment size does not match sync metadata"); - } - StringBuilder actualHash = new StringBuilder(64); - for (byte value : digest.digest()) { - actualHash.append(String.format(java.util.Locale.US, "%02x", value & 0xff)); - } - if (!expectedHash.equals(actualHash.toString())) { - throw new AttachmentIntegrityException( - "Attachment checksum does not match sync metadata"); - } - } - } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java index 00d83bac..9b4003b3 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java @@ -70,7 +70,11 @@ public Result doWork() { getApplicationContext(), dependencies.database(), dependencies.preferenceHelper())) - .sync(new GoogleDriveSyncBackend(authorization.getAccessToken())); + .sync( + new GoogleDriveSyncBackend(authorization.getAccessToken()), + // Re-checked under the sync lock: a disconnect during the + // token round trip must not be followed by a write. + () -> dependencies.preferenceHelper().isSyncEnabled()); if (state.getStatus() == SyncState.Status.SUCCESS) return Result.success(); return isRetryable(state.getErrorMessage()) ? Result.retry() : Result.failure(); } catch (Exception error) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/LegacyNotePayload.java b/app/src/main/java/com/pasich/mynotes/data/sync/LegacyNotePayload.java new file mode 100644 index 00000000..6facbaeb --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/LegacyNotePayload.java @@ -0,0 +1,103 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.pasich.mynotes.extendedEditor.attach.AttachmentUrl; +import com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks; +import java.util.ArrayList; +import java.util.List; + +/** + * Brings a note published by 2.6.50 into the shape the current store builds. + * + *

2.6.50 published a note's editor blocks with the writing device's own file URLs and derived + * attachment ids without the content hash. The current store publishes wire references and ids that + * include the hash, so an unchanged note read back from Drive after the upgrade hashed differently + * from the same note rebuilt locally at the same timestamp. The merge reported a conflict against + * itself — and, when the old version won the tiebreaker, applied it, rebuilt it in the new shape, + * found nothing to publish because the merged snapshot equalled the remote one, and raised the same + * conflict again on every sync until the user happened to edit the note. + * + *

The old shape carries everything the new derivation needs: the note's stable id, each block's + * URL, each entry's name and hash. A payload is upgraded only when every manifest id is provably + * the old derivation of exactly those inputs, so a payload written by any other rule is left alone. + * + *

Deliberately free of {@code android.*}: the equality it restores is what every sync depends + * on. + */ +final class LegacyNotePayload { + + private LegacyNotePayload() {} + + /** + * Rewrites {@code payload} in place when it is a 2.6.50-shaped note. + * + * @return true when the payload was upgraded. + */ + static boolean upgrade(@NonNull String noteStableId, @NonNull JsonObject payload) { + JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); + JsonElement blocks = payload.get("f"); + if (manifest == null + || manifest.size() == 0 + || blocks == null + || !blocks.isJsonPrimitive() + || !blocks.getAsJsonPrimitive().isString()) { + return false; + } + List urls = EditorAttachmentBlocks.fileUrls(blocks.getAsString()); + if (urls.size() != manifest.size()) { + return false; + } + List newIds = new ArrayList<>(manifest.size()); + for (int index = 0; index < manifest.size(); index++) { + JsonElement element = manifest.get(index); + if (!element.isJsonObject()) { + return false; + } + JsonObject entry = element.getAsJsonObject(); + String url = urls.get(index); + if (AttachmentUrl.parse(url) == null + || !isString(entry.get("id")) + || !isString(entry.get("sha256")) + || !isString(entry.get("displayName"))) { + return false; + } + String displayName = entry.get("displayName").getAsString(); + if (!entry.get("id") + .getAsString() + .equals( + AttachmentLogicalIds.deriveLegacy( + noteStableId, index, url, displayName))) { + return false; + } + newIds.add( + AttachmentLogicalIds.derive( + noteStableId, + index, + url, + displayName, + entry.get("sha256").getAsString())); + } + + JsonObject names = new JsonObject(); + for (int index = 0; index < manifest.size(); index++) { + JsonObject entry = manifest.get(index).getAsJsonObject(); + entry.addProperty("id", newIds.get(index)); + names.addProperty(newIds.get(index), entry.get("displayName").getAsString()); + } + payload.add("attachmentNames", names); + int[] position = {0}; + payload.addProperty( + "f", + EditorAttachmentBlocks.rewriteUrls( + blocks.getAsString(), + url -> AttachmentWireUrl.forLogicalId(newIds.get(position[0]++)))); + return true; + } + + private static boolean isString(JsonElement value) { + return value != null && value.isJsonPrimitive() && value.getAsJsonPrimitive().isString(); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecision.java b/app/src/main/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecision.java new file mode 100644 index 00000000..1b04355d --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecision.java @@ -0,0 +1,60 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import java.util.regex.Pattern; + +/** + * Decides whether the live preferences differ from the last value sync recorded. + * + *

SharedPreferences has no mutation hook, so a local settings change is only ever noticed by + * comparing a digest of the live values against the baseline stored at the last sync. The baseline + * format changed once: 2.6.49 stored the payload's raw JSON, later releases store its SHA-256. A + * device upgrading across that change still holds a JSON baseline, and comparing it against a + * digest can only ever say "changed" — which manufactured a local edit on every upgraded device and + * let its untouched settings outrank a genuine change made elsewhere. + * + *

Deliberately free of {@code android.*} so every branch runs under a plain JVM test. + */ +final class PreferencesBaselineDecision { + + enum Action { + /** Nothing changed since the last sync. */ + UNCHANGED, + /** + * The baseline predates the digest format but describes the live values; store the digest. + */ + MIGRATE_BASELINE, + /** The user changed a setting since the last sync; the record must be touched. */ + LOCAL_EDIT + } + + private static final Pattern DIGEST = Pattern.compile("[0-9a-f]{64}"); + + private PreferencesBaselineDecision() {} + + /** + * @param storedBaseline the baseline read from preferences, or {@code null} when none exists. + * @param liveDigest SHA-256 of the live preferences in the current format. + * @param legacyFingerprint the live preferences serialized the way 2.6.49 stored its baseline. + */ + @NonNull + static Action decide( + @Nullable String storedBaseline, + @NonNull String liveDigest, + @NonNull String legacyFingerprint) { + if (storedBaseline == null) { + // No baseline at all means sync has never seen these settings; treating them as a + // local edit is the conservative reading, because the alternative silently loses a + // fresh install's configuration to an older version already on Drive. + return Action.LOCAL_EDIT; + } + if (liveDigest.equals(storedBaseline)) { + return Action.UNCHANGED; + } + if (!DIGEST.matcher(storedBaseline).matches() && storedBaseline.equals(legacyFingerprint)) { + return Action.MIGRATE_BASELINE; + } + return Action.LOCAL_EDIT; + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index 0cd191a7..c5a95150 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -21,6 +21,8 @@ import com.pasich.mynotes.data.model.TaskCategory; import com.pasich.mynotes.data.preferences.PreferenceHelper; import com.pasich.mynotes.extendedEditor.attach.AttachmentStorage; +import com.pasich.mynotes.extendedEditor.attach.AttachmentUrl; +import com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks; import com.pasich.mynotes.extendedEditor.models.EditorAttachment; import com.pasich.mynotes.utils.backup.models.PreferencesBackup; import java.io.File; @@ -30,15 +32,15 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URLConnection; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -49,6 +51,9 @@ public final class RoomSyncStore implements SyncStore { private static final String LEGACY_STATE = "last_state"; private static final String PREFERENCES_HASH = "preferences_hash"; private static final String PREFERENCES_STABLE_ID = "00000000-0000-4000-8000-000000000000"; + private static final String ATTACHMENT_CACHE_DIR = "sync-attachments"; + private static final String HASH_CACHE_FILE = "hash-cache.json"; + private static final String SHA_256 = "[0-9a-f]{64}"; private final AppDatabase database; private final SharedPreferences preferences; private volatile boolean seeded; @@ -58,6 +63,7 @@ public final class RoomSyncStore implements SyncStore { private final AttachmentResolver attachmentResolver; private final AttachmentHasher attachmentHasher; private final TransactionFailureInjector transactionFailureInjector; + private final AttachmentHashCache hashCache; /** * Content hash to the note-folder file holding it, indexed while the snapshot is built so the @@ -65,6 +71,16 @@ public final class RoomSyncStore implements SyncStore { */ private final Map localAttachments = new ConcurrentHashMap<>(); + /** Whether every note folder has been indexed into {@link #localAttachments}. */ + private volatile boolean noteFoldersIndexed; + + /** + * Attachments the last build described from the column alone because their file is gone, keyed + * by hash, so the service can name the note when no endpoint holds the blob either. + */ + private final Map rememberedOnlyAttachments = + new ConcurrentHashMap<>(); + /** * Set when an apply actually changed the visible settings, so the screen can redraw. * @@ -84,7 +100,7 @@ public RoomSyncStore( database, preferenceHelper, AttachmentStorage::resolve, - RoomSyncStore::sha256, + Sha256::of, record -> {}); } @@ -122,6 +138,11 @@ public RoomSyncStore( this.transactionFailureInjector = transactionFailureInjector; this.preferences = context.getApplicationContext().getSharedPreferences(PREFS, Context.MODE_PRIVATE); + this.hashCache = + new AttachmentHashCache( + new File( + new File(this.context.getFilesDir(), ATTACHMENT_CACHE_DIR), + HASH_CACHE_FILE)); } /** @@ -165,28 +186,35 @@ public SnapshotBuildResult buildSnapshot() throws IOException { ensureSeeded(); List records = new ArrayList<>(); List problems = new ArrayList<>(); - for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { - if (metadata.deletedAt != null) { - records.add( - SyncRecord.tombstone( - SyncRecord.Type.fromWireValue(metadata.recordType), - metadata.stableId, - Instant.ofEpochMilli(metadata.updatedAt), - Instant.ofEpochMilli(metadata.deletedAt))); - continue; - } - JsonObject payload = payload(metadata, problems); - if (payload != null) { - SyncMetadataEntity current = - database.syncMetadataDao().get(metadata.recordType, metadata.localId); - records.add( - SyncRecord.live( - SyncRecord.Type.fromWireValue(metadata.recordType), - metadata.stableId, - Instant.ofEpochMilli( - current == null ? metadata.updatedAt : current.updatedAt), - payload)); + rememberedOnlyAttachments.clear(); + try { + for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { + if (metadata.deletedAt != null) { + records.add( + SyncRecord.tombstone( + SyncRecord.Type.fromWireValue(metadata.recordType), + metadata.stableId, + Instant.ofEpochMilli(metadata.updatedAt), + Instant.ofEpochMilli(metadata.deletedAt))); + continue; + } + JsonObject payload = payload(metadata, problems); + if (payload != null) { + SyncMetadataEntity current = + database.syncMetadataDao().get(metadata.recordType, metadata.localId); + records.add( + SyncRecord.live( + SyncRecord.Type.fromWireValue(metadata.recordType), + metadata.stableId, + Instant.ofEpochMilli( + current == null + ? metadata.updatedAt + : current.updatedAt), + payload)); + } } + } finally { + hashCache.flush(); } SyncSnapshot snapshot = new SyncSnapshot(records); return problems.isEmpty() @@ -220,12 +248,14 @@ private void applySnapshotInternal( stagedPreferences == null ? null : gson.toJson(stagedPreferences); String stagedPreferencesTarget = stagedPreferences == null ? "" : preferencesDigest(stagedPreferences); - String preferencesBaseline = stagedPreferences == null ? "" : livePreferencesDigest(); SyncRecord preferencesRecord = snapshot.find(SyncRecord.Type.PREFERENCES, PREFERENCES_STABLE_ID); long stagedPreferencesUpdatedAt = preferencesRecord == null ? 0L : preferencesRecord.getUpdatedAt().toEpochMilli(); boolean deferFinalState = stagedPreferences != null && finalState != null; + // The live digest at the moment the journal is written, decided inside the transaction + // below; the commit afterwards refuses to run if the settings moved again since. + String[] preferencesBaseline = {null}; try { database.runInTransaction( () -> { @@ -233,15 +263,16 @@ private void applySnapshotInternal( Map byStableId = new HashMap<>(); for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { - byStableId.put( - metadata.recordType + ":" + metadata.stableId, metadata); + byStableId.put(recordKey(metadata), metadata); } + // Records the merge decided about but this apply left untouched. A + // conflict for one of them names versions that no longer describe + // the local record, so it must not be stored for the user to apply. + Set skippedKeys = new HashSet<>(); + SyncMetadataEntity[] preferencesMetadata = {null}; for (SyncRecord record : snapshot.getRecords()) { - SyncMetadataEntity metadata = - byStableId.get( - record.getType().getWireValue() - + ":" - + record.getId()); + String key = recordKey(record); + SyncMetadataEntity metadata = byStableId.get(key); if (metadata == null && !record.isTombstone()) { long localId = insertRemoteRecord(record); if (localId >= 0) { @@ -259,14 +290,23 @@ private void applySnapshotInternal( continue; } if (metadata == null) continue; + if (record.getType() == SyncRecord.Type.PREFERENCES + && !record.isTombstone()) { + // Decided last, once every other record has been applied: + // the settings screens write SharedPreferences directly and + // nothing touches the sync record for them, so the + // stale-record guard below cannot see a setting changed while + // this transaction ran. Committing the merged version anyway + // overwrote such an edit and recorded its digest as the + // baseline, hiding the loss from the next build. + preferencesMetadata[0] = metadata; + continue; + } // The snapshot was built before Drive was read and every blob - // transferred, which - // can take minutes, and the six-hourly worker does it while the - // user is in the - // editor. If the record moved on locally since then, the merge - // chose between - // versions one of which no longer exists, so applying its result - // would silently + // transferred, which can take minutes, and the six-hourly worker + // does it while the user is in the editor. If the record moved on + // locally since then, the merge chose between versions one of + // which no longer exists, so applying its result would silently // drop the newer edit. Leave it; the next sync merges the real // current version. if (metadata.updatedAt > record.getUpdatedAt().toEpochMilli()) { @@ -275,6 +315,7 @@ private void applySnapshotInternal( "Skipping a stale sync result for " + metadata.recordType + "; it was edited during this sync"); + skippedKeys.add(key); continue; } if (record.isTombstone()) { @@ -288,7 +329,10 @@ private void applySnapshotInternal( transactionFailureInjector.afterRecordApplied(record); continue; } - applyPayload(metadata, record.getPayload()); + if (!applyPayload(metadata, record.getPayload())) { + skippedKeys.add(key); + continue; + } database.syncMetadataDao() .setVersion( metadata.recordType, @@ -297,20 +341,49 @@ private void applySnapshotInternal( null); transactionFailureInjector.afterRecordApplied(record); } - persistConflicts(conflicts); - if (stagedPreferencesJson != null) { - database.syncPendingPreferencesDao() - .upsert( - new SyncPendingPreferencesEntity( - 1, - stagedPreferencesJson, - stagedPreferencesTarget, - preferencesBaseline, - stagedPreferencesUpdatedAt, - false, - 0L, - "")); + if (preferencesMetadata[0] != null) { + String preferencesKey = recordKey(preferencesMetadata[0]); + String live = livePreferencesDigest(); + String baseline = preferences.getString(PREFERENCES_HASH, null); + boolean edited = baseline != null && !baseline.equals(live); + boolean stale = + preferencesMetadata[0].updatedAt + > stagedPreferencesUpdatedAt; + if (stagedPreferencesJson == null || edited || stale) { + // Unusable payload, or edited since the build: the version + // is not going to be committed, so it is not recorded as + // the one this device holds either. The next build sees the + // live values differ from the baseline and publishes them + // as the local edit they are. + if (edited) { + Log.w( + TAG, + "Skipping synchronized preferences; they were" + + " edited during this sync"); + } + skippedKeys.add(preferencesKey); + } else { + preferencesBaseline[0] = live; + database.syncMetadataDao() + .setVersion( + preferencesMetadata[0].recordType, + preferencesMetadata[0].localId, + stagedPreferencesUpdatedAt, + null); + database.syncPendingPreferencesDao() + .upsert( + new SyncPendingPreferencesEntity( + 1, + stagedPreferencesJson, + stagedPreferencesTarget, + live, + stagedPreferencesUpdatedAt, + false, + 0L, + "")); + } } + persistConflicts(conflicts, skippedKeys); if (finalState != null && !deferFinalState) { database.syncStateDao().upsert(toEntity(finalState)); } @@ -321,11 +394,16 @@ private void applySnapshotInternal( } catch (SyncRuntimeException error) { throw error.ioException; } - if (stagedPreferences != null) { - // The journal is only dropped once the adapter reports a durable commit; a failure - // here leaves it in place for recoverPendingPreferences and keeps the sync state - // retryable rather than claiming success. - commitPendingPreferences(stagedPreferences, stagedPreferencesTarget); + if (deferFinalState || preferencesBaseline[0] != null) { + if (preferencesBaseline[0] != null) { + // The journal is only dropped once the adapter reports a durable commit; a + // failure here leaves it in place for recoverPendingPreferences and keeps the + // sync state retryable rather than claiming success. A commit refused because + // the settings moved in the meantime is not a failure: the journal is dropped + // and the edit is published by the next build. + commitPendingPreferences( + stagedPreferences, stagedPreferencesTarget, preferencesBaseline[0]); + } database.runInTransaction( () -> { database.syncPendingPreferencesDao().clear(); @@ -336,6 +414,16 @@ private void applySnapshotInternal( pruneAttachmentCache(snapshot); } + @NonNull + private static String recordKey(@NonNull SyncMetadataEntity metadata) { + return metadata.recordType + ":" + metadata.stableId; + } + + @NonNull + private static String recordKey(@NonNull SyncRecord record) { + return record.getType().getWireValue() + ":" + record.getId(); + } + /** * Drops cached blobs nothing can still need. * @@ -346,7 +434,9 @@ private void applySnapshotInternal( * deleting the bytes would be unrecoverable. * *

Best effort by design: this is a space optimization, and correctness must not depend on it - * running, or on it finishing. + * running, or on it finishing. An unreadable conflict row throws out of the collection below, + * which lands here and skips the whole pass: a row that cannot be read must never authorize a + * deletion. */ private void pruneAttachmentCache(@NonNull SyncSnapshot applied) { try { @@ -355,14 +445,13 @@ private void pruneAttachmentCache(@NonNull SyncSnapshot applied) { collectConflictAttachmentHashes(conflict.winnerJson, required); collectConflictAttachmentHashes(conflict.loserJson, required); } - File dir = new File(context.getFilesDir(), "sync-attachments"); - File[] cached = dir.listFiles(); + File[] cached = attachmentCacheDir().listFiles(); if (cached == null) { return; } for (File file : cached) { String name = file.getName(); - if (!file.isFile() || !name.matches("[0-9a-f]{64}") || required.contains(name)) { + if (!file.isFile() || !name.matches(SHA_256) || required.contains(name)) { continue; } if (!file.delete()) { @@ -380,44 +469,43 @@ private void collectConflictAttachmentHashes( if (recordJson == null || recordJson.isEmpty()) { return; } - try { - JsonObject root = JsonParser.parseString(recordJson).getAsJsonObject(); - JsonObject payload = root.getAsJsonObject("payload"); - if (payload == null) { - return; - } - JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); - if (manifest != null) { - for (JsonElement element : manifest) { - if (!element.isJsonObject()) continue; - JsonObject entry = element.getAsJsonObject(); - if (entry.has("sha256")) into.add(entry.get("sha256").getAsString()); - } - } - JsonArray hashes = payload.getAsJsonArray("attachmentHashes"); - if (hashes != null) { - for (JsonElement element : hashes) into.add(element.getAsString()); + JsonObject root = JsonParser.parseString(recordJson).getAsJsonObject(); + JsonObject payload = root.getAsJsonObject("payload"); + if (payload == null) { + return; + } + JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); + if (manifest != null) { + for (JsonElement element : manifest) { + if (!element.isJsonObject()) continue; + JsonObject entry = element.getAsJsonObject(); + if (entry.has("sha256")) into.add(entry.get("sha256").getAsString()); } - } catch (RuntimeException unreadable) { - // An unreadable conflict row must never authorize a deletion, so fail closed by - // keeping everything: the caller only removes blobs nothing claimed. - throw unreadable; + } + JsonArray hashes = payload.getAsJsonArray("attachmentHashes"); + if (hashes != null) { + for (JsonElement element : hashes) into.add(element.getAsString()); } } + /** + * The preferences version this apply should commit, or {@code null} when there is none. + * + *

An unusable payload is skipped rather than thrown: it came from Drive, every device sees + * the same one, and throwing here failed every sync on every device until somebody happened to + * change a setting locally. Skipping leaves the local record's version untouched, so the local + * values keep winning the next merge and the bad version is replaced the moment any device + * publishes a real one. + */ @Nullable - private PreferencesBackup selectedPreferences(@NonNull SyncSnapshot snapshot) - throws IOException { + private PreferencesBackup selectedPreferences(@NonNull SyncSnapshot snapshot) { SyncRecord record = snapshot.find(SyncRecord.Type.PREFERENCES, PREFERENCES_STABLE_ID); if (record == null || record.isTombstone()) return null; try { - PreferencesBackup parsed = gson.fromJson(record.getPayload(), PreferencesBackup.class); - if (parsed == null || !parsed.isCreated()) { - throw new IOException("Sync preferences payload is invalid"); - } - return parsed; - } catch (RuntimeException error) { - throw new IOException("Sync preferences payload is invalid", error); + return requirePreferences(record.getPayload()); + } catch (IOException invalid) { + Log.w(TAG, "Skipping an unreadable synchronized preferences payload", invalid); + return null; } } @@ -488,7 +576,8 @@ private void recoverPendingPreferences() throws IOException { return; case REPLAY: default: - commitPendingPreferences(backup, target); + // Replay was decided because the live values still match the baseline. + commitPendingPreferences(backup, target, pending.baselineHash); finishJournal(pending); } } @@ -506,10 +595,20 @@ private void finishJournal(@NonNull SyncPendingPreferencesEntity pending) { * Applies one journaled preferences payload, failing loudly when it is not durable. * * @param expectedDigest digest the live preferences must show afterwards. + * @param baselineDigest digest the live preferences showed when the write was decided; if they + * no longer match, the user changed a setting in between and the write is refused. + * @return true when the values were written, false when the write was refused as superseded. */ - private void commitPendingPreferences( - @NonNull PreferencesBackup backup, @NonNull String expectedDigest) throws IOException { + private boolean commitPendingPreferences( + @NonNull PreferencesBackup backup, + @NonNull String expectedDigest, + @NonNull String baselineDigest) + throws IOException { String before = livePreferencesDigest(); + if (!before.equals(baselineDigest) && !before.equals(expectedDigest)) { + Log.w(TAG, "Refusing to commit synchronized preferences; they changed meanwhile"); + return false; + } boolean committed; try { committed = preferenceHelper.commitListPreferences(backup); @@ -525,6 +624,7 @@ private void commitPendingPreferences( // The digest doubles as the snapshot-build baseline, so recording it here keeps the next // build from treating a freshly received version as a local edit. preferences.edit().putString(PREFERENCES_HASH, expectedDigest).commit(); + return true; } /** @@ -539,16 +639,25 @@ private void commitPendingPreferences( private void noteLocalPreferenceEdit( @NonNull SyncMetadataEntity metadata, @Nullable PreferencesBackup live) { String digest = preferencesDigest(live); - String baseline = preferences.getString(PREFERENCES_HASH, null); - if (digest.equals(baseline)) { - return; + PreferencesBaselineDecision.Action action = + PreferencesBaselineDecision.decide( + preferences.getString(PREFERENCES_HASH, null), + digest, + legacyPreferencesFingerprint(live)); + switch (action) { + case UNCHANGED: + return; + case MIGRATE_BASELINE: + // 2.6.49 stored the payload JSON here. Same values, older spelling: rewrite it + // without touching the record, or every upgraded device claims a local edit. + preferences.edit().putString(PREFERENCES_HASH, digest).commit(); + return; + case LOCAL_EDIT: + default: + database.syncMetadataDao() + .touch(metadata.recordType, metadata.localId, System.currentTimeMillis()); + preferences.edit().putString(PREFERENCES_HASH, digest).commit(); } - // No baseline at all means sync has never seen these settings; treating them as a local - // edit is the conservative reading, because the alternative silently loses a fresh - // install's configuration to an older version already on Drive. - database.syncMetadataDao() - .touch(metadata.recordType, metadata.localId, System.currentTimeMillis()); - preferences.edit().putString(PREFERENCES_HASH, digest).commit(); } /** Digest of the preferences currently visible to the app. */ @@ -560,12 +669,13 @@ private String livePreferencesDigest() { /** Stable digest of one preferences payload, used for the journal and the build baseline. */ @NonNull private String preferencesDigest(@Nullable PreferencesBackup backup) { - String json = backup == null ? "" : gson.toJson(backup); - try { - return sha256(new java.io.ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))); - } catch (IOException impossible) { - throw new IllegalStateException("Could not digest preferences", impossible); - } + return Sha256.of(backup == null ? "" : gson.toJson(backup)); + } + + /** The baseline exactly as 2.6.49 wrote it: the payload object's own JSON. */ + @NonNull + private String legacyPreferencesFingerprint(@Nullable PreferencesBackup backup) { + return backup == null ? "" : gson.toJsonTree(backup).getAsJsonObject().toString(); } /** Reads a preferences payload, refusing anything that is not a usable settings snapshot. */ @@ -612,7 +722,8 @@ else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { } } if ("note".equals(metadata.recordType) - && !addAttachmentMetadata(result, metadata, snapshotProblems)) { + && !addAttachmentMetadata( + result, metadata, ((Note) value).getTitle(), snapshotProblems)) { return null; } // Runs last: the blocks above still need the local categoryId and attachment paths. @@ -620,7 +731,19 @@ else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { return result; } - private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) throws IOException { + /** + * Writes one live version over the local row. + * + * @return false when the version was deliberately not applied and the record's local version + * must therefore not be advanced to it. + */ + private boolean applyPayload(SyncMetadataEntity metadata, JsonObject payload) + throws IOException { + // A record that was deleted here and then edited on another device has no row left to + // update; @Update on a missing row is a silent no-op, and the tombstone was cleared + // regardless, so tasks, categories and tags marked live never came back. The REPLACE + // inserts put the row back under its own local id. + boolean revive = metadata.deletedAt != null; if ("note".equals(metadata.recordType)) { Note note = gson.fromJson(payload, Note.class); note.setId((int) metadata.localId); @@ -637,19 +760,70 @@ private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) throw payload.get("categoryStableId").getAsString()); if (category != null) task.setCategoryId((int) category.localId); } - database.taskDao().updateTask(task); + if (revive || database.taskDao().getTaskSync(task.getId()) == null) { + database.taskDao().insertTask(task); + } else { + database.taskDao().updateTask(task); + } } else if (SyncMetadata.RECORD_TYPE_CATEGORY.equals(metadata.recordType)) { TaskCategory category = gson.fromJson(payload, TaskCategory.class); category.setId((int) metadata.localId); - database.taskCategoryDao().updateCategory(category); + if (revive || database.taskCategoryDao().getCategorySync(category.getId()) == null) { + database.taskCategoryDao().insertCategory(category); + } else { + database.taskCategoryDao().updateCategory(category); + } } else if ("tag".equals(metadata.recordType)) { Tag tag = gson.fromJson(payload, Tag.class); tag.id = metadata.localId; + if (revive || database.tagsDao().getTagSync(tag.id) == null) { + return reviveTag(metadata, tag); + } database.tagsDao().updateTag(tag); } else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { // SharedPreferences is outside Room. applySnapshotInternal journals and commits this // payload only after the Room transaction succeeds. } + return true; + } + + /** + * Puts a deleted tag's row back, unless a tag of that name has since been created here. + * + *

A tag is identified by its name: a note stores the name and the table has no unique index + * on it, so putting the row back beside a same-named one shows the user the same tag twice. The + * same rule as {@link #insertRemoteTag} settles which identity survives: the smaller stable id. + * When the local tag holds it, the reviving identity is tombstoned afresh so that the + * tombstone, now newer than the remote edit, retires it everywhere at the next sync — and the + * record is reported as not applied, so its version is not advanced. + */ + private boolean reviveTag(@NonNull SyncMetadataEntity metadata, @NonNull Tag tag) { + String name = tag.getNameTag(); + Tag existing = + name == null || name.isEmpty() ? null : database.tagsDao().getTagByNameSync(name); + if (existing != null && existing.getId() != tag.id) { + SyncMetadataEntity existingMetadata = + database.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, existing.getId()); + if (existingMetadata != null + && existingMetadata.stableId.compareTo(metadata.stableId) < 0) { + database.syncMetadataDao() + .markDeleted( + SyncMetadata.RECORD_TYPE_TAG, + metadata.localId, + System.currentTimeMillis()); + return false; + } + database.tagsDao().deleteById(existing.getId()); + if (existingMetadata != null) { + database.syncMetadataDao() + .markDeleted( + SyncMetadata.RECORD_TYPE_TAG, + existing.getId(), + System.currentTimeMillis()); + } + } + database.tagsDao().addTag(tag); + return true; } private long insertRemoteRecord(SyncRecord record) throws IOException { @@ -690,13 +864,50 @@ private long insertRemoteRecord(SyncRecord record) throws IOException { return database.taskCategoryDao().insertCategory(category); } if (record.getType() == SyncRecord.Type.TAG) { - Tag tag = gson.fromJson(record.getPayload(), Tag.class); - tag.id = 0; - return database.tagsDao().addTag(tag); + return insertRemoteTag(record); } return -1; } + /** + * Inserts a tag another device created, unless this device already has it by name. + * + *

A note stores its tag by name and the table has no unique index on it, so a tag created as + * "Work" on two devices before their first sync arrived here as two stable ids and became two + * rows the user saw twice. The name is the tag's real identity; the two stable ids are + * reconciled deterministically so every device ends up with the same one: the smaller id wins, + * the loser's row is deleted and tombstoned, and the tombstone retires the other id on every + * device at the next sync. The device already holding the winning id simply keeps it. + * + * @return the local row bound to the record's stable id, or -1 when the record is skipped. + */ + private long insertRemoteTag(@NonNull SyncRecord record) { + Tag tag = gson.fromJson(record.getPayload(), Tag.class); + tag.id = 0; + String name = tag.getNameTag(); + Tag existing = + name == null || name.isEmpty() ? null : database.tagsDao().getTagByNameSync(name); + if (existing == null) { + return database.tagsDao().addTag(tag); + } + SyncMetadataEntity existingMetadata = + database.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_TAG, existing.getId()); + if (existingMetadata == null) { + // A row sync has never described; the remote identity becomes its identity. + return existing.getId(); + } + if (existingMetadata.stableId.compareTo(record.getId()) < 0) { + // This device holds the winning identity; the other one is retired by whichever + // device holds it once it sees ours. + return -1; + } + database.tagsDao().deleteById(existing.getId()); + database.syncMetadataDao() + .markDeleted( + SyncMetadata.RECORD_TYPE_TAG, existing.getId(), System.currentTimeMillis()); + return database.tagsDao().addTag(tag); + } + private void markDeleted(SyncMetadataEntity metadata) { if ("note".equals(metadata.recordType)) database.noteDao().deleteById((int) metadata.localId); @@ -726,28 +937,39 @@ public List getConflicts() { * *

Record identity in {@code sync_metadata} is deliberately kept: it is local, and discarding * it would make the whole library look brand new to the next account. What goes is the sync - * status, the conflict queue and the blobs downloaded from the disconnected account's Drive. + * status, the conflict queue, the preferences journal and the blobs downloaded from the + * disconnected account's Drive. The journal matters: left behind, a fresh store replayed the + * old account's settings onto the device at its next seeding. * *

Clearing the status also repairs a dead end: the Backup screen decided whether to ask for * first-sync consent from {@code lastSuccessfulSyncAt}, which survived a sign-out, while {@code * SyncCoordinator} gated the sync on a preference the sign-out reset. The dialog was skipped * and the sync refused, with no way to reach the consent again. + * + *

Waits for a sync in flight rather than racing it: the six-hourly worker holds the sync + * lock for minutes, and clearing under it left the worker writing the old account's state and + * conflicts back after the wipe, into a cache directory that had just been deleted. */ public void clearAfterDisconnect() { - database.runInTransaction( + SyncService.runWhileNoSyncRuns( () -> { - database.syncStateDao().clear(); - database.syncConflictDao().clearAll(); + database.runInTransaction( + () -> { + database.syncStateDao().clear(); + database.syncConflictDao().clearAll(); + database.syncPendingPreferencesDao().clear(); + }); + preferences.edit().remove(PREFERENCES_HASH).remove(LEGACY_STATE).apply(); + localAttachments.clear(); + noteFoldersIndexed = false; + hashCache.clear(); + deleteAttachmentCache(); }); - preferences.edit().remove(PREFERENCES_HASH).remove(LEGACY_STATE).apply(); - localAttachments.clear(); - deleteAttachmentCache(); } /** Removes the download cache only; the notes' own attachment folders are untouched. */ private void deleteAttachmentCache() { - File dir = new File(context.getFilesDir(), "sync-attachments"); - File[] cached = dir.listFiles(); + File[] cached = attachmentCacheDir().listFiles(); if (cached == null) { return; } @@ -768,6 +990,17 @@ public void resolveConflict(long conflictId, @NonNull SyncResolution resolution) SyncConflictEntity pending = database.syncConflictDao().getById(conflictId); if (pending == null || pending.resolved) return; + if (isSuperseded(pending)) { + // The record moved on after this conflict was recorded — an edit here, or a newer + // version applied from another device — so both stored versions are older than what + // the user now has. Applying either would overwrite the newer edit with a version + // that was never offered against it. The alternative still travels with the bundle + // and comes back as a fresh conflict against the current version at the next sync. + Log.w(TAG, "Dropping a conflict that the record's newer version has superseded"); + database.runInTransaction(() -> database.syncConflictDao().deleteById(conflictId)); + return; + } + // Resolution is a user-visible mutation. Verify and pin the selected version before its // conflict row can be marked resolved; a missing blob must leave both the note and the // conflict untouched, including when the winner happens to already be visible in Room. @@ -801,6 +1034,14 @@ public void resolveConflict(long conflictId, @NonNull SyncResolution resolution) } } + /** True when the local record is newer than both versions the conflict offers. */ + private boolean isSuperseded(@NonNull SyncConflictEntity conflict) { + SyncMetadataEntity metadata = + database.syncMetadataDao().getByStableId(conflict.recordType, conflict.stableId); + return metadata != null + && metadata.updatedAt > Math.max(conflict.winnerUpdatedAt, conflict.loserUpdatedAt); + } + /** * Applies a chosen preferences version through the same journal a snapshot apply uses. * @@ -840,8 +1081,11 @@ private void resolvePreferencesConflict( }); // Throws when the write is not durable, leaving the journal in place and the conflict - // unresolved so the user can try again. - commitPendingPreferences(chosen, target); + // unresolved so the user can try again. A setting changed while the choice was being + // applied is treated the same way; recovery then discards the journal as stale. + if (!commitPendingPreferences(chosen, target, baseline)) { + throw new IOException("Settings changed while the conflict was being resolved"); + } try { finalizeResolvedPreferencesConflict(conflictId, resolution.name()); @@ -901,12 +1145,27 @@ private void pinResolvedConflictAttachments(@NonNull SyncRecord selected) throws } } - private void persistConflicts(@NonNull List conflicts) { + /** + * Stores the conflicts this apply produced and retires the ones it makes meaningless. + * + * @param skippedKeys records this apply left untouched; a conflict for one of them offers + * versions that no longer describe the local record and is not stored. + */ + private void persistConflicts( + @NonNull List conflicts, @NonNull Set skippedKeys) { if (conflicts.isEmpty()) return; long createdAt = System.currentTimeMillis(); List rows = new ArrayList<>(conflicts.size()); + Map> winnersByRecord = new HashMap<>(); for (SyncMergeResult.Conflict conflict : conflicts) { + String key = conflict.getType().getWireValue() + ":" + conflict.getId(); + if (skippedKeys.contains(key)) { + continue; + } + winnersByRecord + .computeIfAbsent(key, ignored -> new LinkedHashSet<>()) + .add(conflict.getWinnerVersionId()); rows.add( new SyncConflictEntity( conflict.getType().getWireValue(), @@ -927,25 +1186,38 @@ private void persistConflicts(@NonNull List conflicts) createdAt, 0L)); } + if (rows.isEmpty()) return; + // Every conflict for a record names the version this sync applies as its winner. An + // older unresolved row for the same record therefore offers a winner that is no longer + // the live version, and applying it — which "keep the version the merge selected" did + // with one tap — reverted the user's newer edit. Its alternative is not lost: it still + // travels with the bundle and is among the rows stored here, now against the current + // version. + for (SyncMergeResult.Conflict conflict : conflicts) { + String key = conflict.getType().getWireValue() + ":" + conflict.getId(); + Set winners = winnersByRecord.remove(key); + if (winners == null || winners.size() != 1) { + continue; + } + database.syncConflictDao() + .deleteSupersededUnresolved( + conflict.getType().getWireValue(), + conflict.getId(), + winners.iterator().next()); + } database.syncConflictDao().insertIgnoringDuplicates(rows); } @NonNull private static String conflictVersionPairHash(@NonNull SyncMergeResult.Conflict conflict) { - String source = + return Sha256.of( conflict.getType().getWireValue() + "\n" + conflict.getId() + "\n" + conflict.getWinner().canonicalSerializedPayload() + "\n" - + conflict.getLoser().canonicalSerializedPayload(); - try { - return sha256( - new java.io.ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); - } catch (IOException impossible) { - throw new IllegalStateException("Could not hash sync conflict identity", impossible); - } + + conflict.getLoser().canonicalSerializedPayload()); } /** True when {@code resolution} names the version the merge selected. */ @@ -1090,6 +1362,18 @@ public boolean hasAttachment(@NonNull String sha256) { return resolveLocalAttachment(sha256) != null; } + @Nullable + @Override + public SnapshotProblem describeMissingAttachment(@NonNull String sha256) { + return rememberedOnlyAttachments.get(sha256); + } + + @Override + public boolean hasDurableAttachment(@NonNull String sha256, long sizeBytes) { + File cached = attachmentFile(sha256); + return cached.isFile() && (sizeBytes < 0L || cached.length() == sizeBytes); + } + @NonNull @Override public InputStream readAttachment(@NonNull String sha256) throws IOException { @@ -1110,7 +1394,9 @@ public InputStream readAttachment(@NonNull String sha256) throws IOException { * this returns true, that failure was permanent for any account holding a single attachment. * *

The note folders are indexed while the snapshot is built rather than copied into the - * cache, so a large attachment set is not stored twice. + * cache, so a large attachment set is not stored twice. A store that never built a snapshot — + * the Backup screen's, resolving a conflict the worker's store found — indexes them on its + * first miss instead, so a version whose blob sits in a note folder is not reported missing. */ @Nullable private File resolveLocalAttachment(@NonNull String sha256) { @@ -1119,9 +1405,42 @@ private File resolveLocalAttachment(@NonNull String sha256) { return cached; } File owned = localAttachments.get(sha256); + if (owned != null && owned.isFile()) { + return owned; + } + if (noteFoldersIndexed) { + return null; + } + indexNoteFolders(); + owned = localAttachments.get(sha256); return owned != null && owned.isFile() ? owned : null; } + /** Hashes every file under every note folder into {@link #localAttachments}, once. */ + private synchronized void indexNoteFolders() { + if (noteFoldersIndexed) { + return; + } + File[] folders = AttachmentStorage.baseDirPath(context).listFiles(); + if (folders != null) { + for (File folder : folders) { + File[] files = folder.isDirectory() ? folder.listFiles() : null; + if (files == null) continue; + for (File file : files) { + if (!file.isFile()) continue; + try { + localAttachments.putIfAbsent( + hashCache.sha256(file, attachmentHasher::sha256), file); + } catch (IOException | RuntimeException unreadable) { + // A file that cannot be hashed cannot satisfy a reference either. + } + } + } + } + hashCache.flush(); + noteFoldersIndexed = true; + } + @Override public void writeAttachment( @NonNull String sha256, long sizeBytes, @NonNull InputStream content) @@ -1145,15 +1464,31 @@ public void writeAttachment( if (!temp.renameTo(target)) throw new IOException("Cannot store attachment"); } + private File attachmentCacheDir() { + return new File(context.getFilesDir(), ATTACHMENT_CACHE_DIR); + } + private File attachmentFile(String sha256) { - File dir = new File(context.getFilesDir(), "sync-attachments"); + File dir = attachmentCacheDir(); if (!dir.exists()) dir.mkdirs(); return new File(dir, sha256); } + /** + * Describes a note's attachments for the wire and puts its editor blocks into wire form. + * + *

The attachments column names files by this device's row id and file name; the wire + * describes each attachment by a logical id, its content hash and size. A file that is gone + * from disk is still describable when the column remembers its hash and size — which every + * attachment that ever came through a sync does — so such a note is published from that + * metadata and {@link SyncService} fetches the bytes from the cache or Drive; the following + * apply then puts the file back. Only an attachment nothing but this device ever knew about + * fails the build, and then the problem names the note. + */ private boolean addAttachmentMetadata( JsonObject payload, SyncMetadataEntity metadata, + @Nullable String noteTitle, @NonNull List snapshotProblems) { String json = payload.has("h") && !payload.get("h").isJsonNull() @@ -1165,43 +1500,39 @@ private boolean addAttachmentMetadata( attachments = JsonParser.parseString(json).getAsJsonArray(); } catch (RuntimeException error) { addSnapshotProblem( - snapshotProblems, SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, metadata); + snapshotProblems, + SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, + metadata, + noteTitle); return false; } JsonArray manifest = new JsonArray(); JsonArray hashes = new JsonArray(); JsonObject names = new JsonObject(); + Map logicalIdByUrl = new HashMap<>(); + Set logicalIdsInNote = new HashSet<>(); boolean complete = true; for (int attachmentIndex = 0; attachmentIndex < attachments.size(); attachmentIndex++) { JsonElement element = attachments.get(attachmentIndex); - if (!element.isJsonObject()) { - addSnapshotProblem( - snapshotProblems, - SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, - metadata); - complete = false; - continue; - } - EditorAttachment attachment; - try { - attachment = gson.fromJson(element, EditorAttachment.class); - } catch (RuntimeException error) { - addSnapshotProblem( - snapshotProblems, - SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, - metadata); - complete = false; - continue; + EditorAttachment attachment = null; + if (element.isJsonObject()) { + try { + attachment = gson.fromJson(element, EditorAttachment.class); + } catch (RuntimeException error) { + attachment = null; + } } if (attachment == null || attachment.url == null || attachment.url.trim().isEmpty()) { addSnapshotProblem( snapshotProblems, SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, - metadata); + metadata, + noteTitle); complete = false; continue; } + JsonObject column = element.getAsJsonObject(); File file; try { file = attachmentResolver.resolve(context, attachment); @@ -1209,85 +1540,177 @@ private boolean addAttachmentMetadata( addSnapshotProblem( snapshotProblems, SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, - metadata); - complete = false; - continue; - } - if (file == null || !file.isFile()) { - addSnapshotProblem( - snapshotProblems, SnapshotProblem.Kind.MISSING_ATTACHMENT, metadata); - complete = false; - continue; - } - if (!file.canRead()) { - addSnapshotProblem( - snapshotProblems, SnapshotProblem.Kind.UNREADABLE_ATTACHMENT, metadata); + metadata, + noteTitle); complete = false; continue; } + String rememberedHash = optionalString(column, "sha256"); + long rememberedSize = optionalLong(column, "size"); String hash; - try { - hash = attachmentHasher.sha256(file); - } catch (IOException error) { - addSnapshotProblem( - snapshotProblems, SnapshotProblem.Kind.ATTACHMENT_HASH_FAILED, metadata); - complete = false; - continue; + long size; + if (file == null || !file.isFile()) { + if (rememberedHash == null + || !rememberedHash.matches(SHA_256) + || rememberedSize < 0L) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.MISSING_ATTACHMENT, + metadata, + noteTitle); + complete = false; + continue; + } + Log.w( + TAG, + "An attachment file is missing; it will be restored from the sync cache" + + " or Drive"); + file = null; + hash = rememberedHash; + size = rememberedSize; + rememberedOnlyAttachments.put( + hash, + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + metadata.recordType, + metadata.stableId, + noteTitle)); + } else { + if (!file.canRead()) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.UNREADABLE_ATTACHMENT, + metadata, + noteTitle); + complete = false; + continue; + } + try { + hash = hashCache.sha256(file, attachmentHasher::sha256); + } catch (IOException error) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.ATTACHMENT_HASH_FAILED, + metadata, + noteTitle); + complete = false; + continue; + } + size = file.length(); + localAttachments.put(hash, file); } - localAttachments.put(hash, file); - String displayName = - attachment.name == null || attachment.name.trim().isEmpty() - ? file.getName() - : attachment.name.trim(); + String displayName = displayNameFor(attachment, file, hash); String logicalId = attachment.id; - if (!isCanonicalUuid(logicalId)) { + // A column entry may repeat an id — a duplicated block whose file was later replaced. + // The manifest is keyed by id, so the repeat is given its own, as if it had none. + if (!isCanonicalUuid(logicalId) || logicalIdsInNote.contains(logicalId)) { // Existing editor data predates logical attachment IDs. Deriving from the stable - // note, source URL and position keeps the migration deterministic while allowing - // equal-content references to remain distinct logical attachments. + // note, source URL, position and content keeps the migration deterministic while + // allowing equal-content references to remain distinct logical attachments. The + // content hash is part of it so that one id can never describe two different + // blobs: a bundle manifest is keyed by id, and two versions of a note disagreeing + // about an id's content used to fail every publish for the account. // // The check is canonical-UUID rather than a loose 36-character pattern: the // bundle manifest only accepts canonical lowercase UUIDs, so an uppercase or // otherwise non-canonical id used to pass here and then throw during encode, // failing every publish for the whole account while that note existed. logicalId = - UUID.nameUUIDFromBytes( - (metadata.stableId - + "\n" - + attachmentIndex - + "\n" - + attachment.url - + "\n" - + displayName) - .getBytes(StandardCharsets.UTF_8)) - .toString(); + AttachmentLogicalIds.derive( + metadata.stableId, + attachmentIndex, + attachment.url, + displayName, + hash); } + logicalIdsInNote.add(logicalId); hashes.add(hash); names.addProperty(logicalId, displayName); + logicalIdByUrl.put(comparableUrl(attachment.url), logicalId); + String rememberedMimeType = optionalString(column, "mimeType"); JsonObject manifestEntry = new JsonObject(); manifestEntry.addProperty("id", logicalId); manifestEntry.addProperty("sha256", hash); - manifestEntry.addProperty("mimeType", detectMimeType(file, attachment, displayName)); - manifestEntry.addProperty("size", file.length()); + manifestEntry.addProperty( + "mimeType", + rememberedMimeType != null + ? rememberedMimeType + : detectMimeType(file, attachment, displayName)); + manifestEntry.addProperty("size", size); manifestEntry.addProperty("path", "attachments/" + hash); manifestEntry.addProperty("displayName", displayName); manifest.add(manifestEntry); } if (!complete) return false; - if (manifest.size() == 0) { - // A note whose attachments column is "[]" — which is what the editor stores for a - // note that simply has none — used to get three empty arrays here, while a decoded - // remote record carries no attachment fields at all. The two shapes hashed - // differently, so every attachment-free note reported a conflict against itself on - // every sync and republished a bundle each time. - return true; - } + // Empty ones are dropped again by SyncRecord, the one place that rule lives. payload.add("attachmentsManifest", manifest); payload.add("attachmentHashes", hashes); payload.add("attachmentNames", names); + // The blocks name this device's files. On the wire they name the logical attachment, + // which every device agrees on; otherwise the same note hashed differently on every + // device that had ever restored it and conflicted with itself on every sync. + JsonElement valueJson = payload.get("f"); + if (valueJson != null && valueJson.isJsonPrimitive()) { + String local = valueJson.getAsString(); + String wire = + EditorAttachmentBlocks.rewriteUrls( + local, + url -> { + String logicalId = logicalIdByUrl.get(comparableUrl(url)); + return logicalId == null + ? null + : AttachmentWireUrl.forLogicalId(logicalId); + }); + if (wire != null && !wire.equals(local)) { + payload.addProperty("f", wire); + } + } return true; } + /** A URL in the one spelling both the legacy and the canonical scheme reduce to. */ + @NonNull + private static String comparableUrl(@NonNull String url) { + AttachmentUrl parsed = AttachmentUrl.parse(url); + return parsed == null ? url.trim() : parsed.canonical(); + } + + @NonNull + private static String displayNameFor( + @NonNull EditorAttachment attachment, @Nullable File file, @NonNull String hash) { + if (attachment.name != null && !attachment.name.trim().isEmpty()) { + return attachment.name.trim(); + } + if (file != null) { + return file.getName(); + } + AttachmentUrl parsed = AttachmentUrl.parse(attachment.url); + return parsed == null ? hash : parsed.getFileName(); + } + + @Nullable + private static String optionalString(@NonNull JsonObject object, @NonNull String field) { + JsonElement value = object.get(field); + if (value == null || !value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString()) { + return null; + } + String text = value.getAsString().trim(); + return text.isEmpty() ? null : text; + } + + private static long optionalLong(@NonNull JsonObject object, @NonNull String field) { + JsonElement value = object.get(field); + if (value == null || !value.isJsonPrimitive() || !value.getAsJsonPrimitive().isNumber()) { + return -1L; + } + try { + return value.getAsLong(); + } catch (RuntimeException notALong) { + return -1L; + } + } + /** True only for a lowercase canonical UUID, which is all the bundle manifest accepts. */ private static boolean isCanonicalUuid(@Nullable String value) { if (value == null) { @@ -1303,14 +1726,19 @@ private static boolean isCanonicalUuid(@Nullable String value) { private static void addSnapshotProblem( @NonNull List problems, @NonNull SnapshotProblem.Kind kind, - @NonNull SyncMetadataEntity metadata) { - problems.add(new SnapshotProblem(kind, metadata.recordType, metadata.stableId)); + @NonNull SyncMetadataEntity metadata, + @Nullable String label) { + problems.add(new SnapshotProblem(kind, metadata.recordType, metadata.stableId, label)); } /** * Materializes every attachment before changing the Room row. Targets use the immutable * logical-ID/content-ID pair rather than a display name, so a rollback can leave only harmless * new files and can never alter bytes addressed by the pre-transaction note. + * + *

The column written here remembers the logical id, hash, size and MIME type alongside the + * local file, which is what lets the next build describe the attachment identically to the + * device that sent it — and describe it at all should the file go missing. */ private void restoreAttachments(Note note, JsonObject payload) throws IOException { JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); @@ -1325,6 +1753,7 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio throw new IOException("Could not create attachment folder"); } JsonArray restored = new JsonArray(); + Map localUrlByLogicalId = new HashMap<>(); for (JsonElement element : manifest) { if (!element.isJsonObject()) { throw new IOException("Attachment manifest entry is invalid"); @@ -1338,7 +1767,7 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio if (entry.id == null || entry.sha256 == null || !entry.id.matches("[0-9a-fA-F-]{36}") - || !entry.sha256.matches("[0-9a-f]{64}") + || !entry.sha256.matches(SHA_256) || entry.size < 0L) { throw new IOException("Attachment manifest entry is invalid"); } @@ -1365,16 +1794,22 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio if (note.getId() <= 0) { throw new IOException("Cannot restore attachments for an unsaved note"); } - JsonObject attachment = new JsonObject(); // Canonical editorjs:// form, the only shape EditorAttachmentsWebViewClient serves. // Writing file:// here left every synced attachment unrenderable on the receiver. - attachment.addProperty("url", AttachmentStorage.urlFor(note.getId(), target.getName())); + String url = AttachmentStorage.urlFor(note.getId(), target.getName()); + JsonObject attachment = new JsonObject(); + attachment.addProperty("url", url); attachment.addProperty("name", displayName); attachment.addProperty("id", entry.id); + attachment.addProperty("sha256", entry.sha256); + attachment.addProperty("size", entry.size); + attachment.addProperty("mimeType", entry.mimeType); restored.add(attachment); + localUrlByLogicalId.put(entry.id, url); } note.setAttachments(gson.toJson(restored)); - note.setValueJson(rewriteEditorAttachmentUrls(note.getValueJson(), restored)); + note.setValueJson( + rewriteEditorAttachmentUrls(note.getValueJson(), restored, localUrlByLogicalId)); } /** @@ -1385,54 +1820,54 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio * note_/}. The column is what the file list reads; the blocks are what * the editor renders, so a received rich note showed its attachments as broken. * - *

The mapping is positional, which is exactly how the manifest was built: the sender's - * attachments column comes from {@code EditorJsonUtils} walking these same blocks in document - * order, and {@code addAttachmentMetadata} walks that column in the same order. If the two do - * not line up the JSON is returned untouched rather than guessed at — a note that renders the - * old broken URL is recoverable, one whose content was rewritten wrongly is not. + *

A block written by this release names its attachment's logical id, which maps directly. A + * bundle from an older client still names the sender's files, and for those the mapping is + * positional, which is exactly how such a manifest was built: the sender's attachments column + * came from {@code EditorJsonUtils} walking these same blocks in document order, and the + * manifest walked that column in the same order. If the two do not line up the JSON is returned + * untouched rather than guessed at — a note that renders the old broken URL is recoverable, one + * whose content was rewritten wrongly is not. */ @Nullable private String rewriteEditorAttachmentUrls( - @Nullable String valueJson, @NonNull JsonArray restored) { + @Nullable String valueJson, + @NonNull JsonArray restored, + @NonNull Map localUrlByLogicalId) { if (valueJson == null || valueJson.trim().isEmpty() || restored.size() == 0) { return valueJson; } - try { - JsonArray blocks = JsonParser.parseString(valueJson).getAsJsonArray(); - List files = new ArrayList<>(); - for (JsonElement element : blocks) { - if (!element.isJsonObject()) continue; - JsonObject block = element.getAsJsonObject(); - String type = - block.has("type") && block.get("type").isJsonPrimitive() - ? block.get("type").getAsString() - : ""; - if (!"attaches".equals(type) && !"image".equals(type)) continue; - JsonObject data = block.getAsJsonObject("data"); - if (data == null) continue; - JsonObject file = data.getAsJsonObject("file"); - if (file != null) files.add(file); - } - if (files.size() != restored.size()) { - Log.w(TAG, "Editor blocks do not match the restored attachments; leaving them"); - return valueJson; - } - for (int index = 0; index < files.size(); index++) { - JsonObject target = restored.get(index).getAsJsonObject(); - files.get(index).addProperty("url", target.get("url").getAsString()); - files.get(index).addProperty("name", target.get("name").getAsString()); + List urls = EditorAttachmentBlocks.fileUrls(valueJson); + boolean wireForm = false; + for (String url : urls) { + if (AttachmentWireUrl.logicalIdOf(url) != null) { + wireForm = true; + break; } - return gson.toJson(blocks); - } catch (RuntimeException malformed) { - Log.w(TAG, "Could not rewrite editor attachment URLs; leaving them untouched"); + } + if (wireForm) { + return EditorAttachmentBlocks.rewriteUrls( + valueJson, + url -> { + String logicalId = AttachmentWireUrl.logicalIdOf(url); + return logicalId == null ? null : localUrlByLogicalId.get(logicalId); + }); + } + if (urls.size() != restored.size()) { + Log.w(TAG, "Editor blocks do not match the restored attachments; leaving them"); return valueJson; } + int[] position = {0}; + return EditorAttachmentBlocks.rewriteUrls( + valueJson, + url -> restored.get(position[0]++).getAsJsonObject().get("url").getAsString()); } - private static boolean isVerifiedAttachmentFile( + private boolean isVerifiedAttachmentFile( @NonNull File file, @NonNull String expectedHash, long expectedSize) throws IOException { - return file.isFile() && file.length() == expectedSize && expectedHash.equals(sha256(file)); + return file.isFile() + && file.length() == expectedSize + && expectedHash.equals(hashCache.sha256(file, Sha256::of)); } private static void copyVerifiedAttachment( @@ -1443,39 +1878,22 @@ private static void copyVerifiedAttachment( throws IOException { File temporary = new File(target.getParentFile(), target.getName() + ".tmp-" + UUID.randomUUID()); - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (java.security.NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - long copied = 0L; - try (InputStream in = new FileInputStream(source); + try (VerifyingInputStream in = + new VerifyingInputStream( + new FileInputStream(source), expectedHash, expectedSize); OutputStream out = new FileOutputStream(temporary)) { byte[] buffer = new byte[8192]; int read; while ((read = in.read(buffer)) != -1) { out.write(buffer, 0, read); - digest.update(buffer, 0, read); - copied += read; - if (copied > expectedSize) { - throw new AttachmentIntegrityException("Attachment exceeds its declared size"); - } } + in.verifyEndOfStream(); } catch (IOException failure) { if (temporary.exists() && !temporary.delete()) { Log.w(TAG, "Could not remove failed staged attachment"); } throw failure; } - StringBuilder hash = new StringBuilder(64); - for (byte value : digest.digest()) hash.append(String.format("%02x", value & 0xff)); - String actual = hash.toString(); - if (copied != expectedSize || !expectedHash.equals(actual)) { - if (!temporary.delete()) Log.w(TAG, "Could not remove invalid staged attachment"); - throw new AttachmentIntegrityException( - "Attachment checksum does not match sync metadata"); - } if (!temporary.renameTo(target)) { if (!temporary.delete()) Log.w(TAG, "Could not remove uncommitted staged attachment"); throw new IOException("Could not finalize staged attachment"); @@ -1483,51 +1901,7 @@ private static void copyVerifiedAttachment( } private static boolean isSafeAttachmentName(@NonNull String name) { - String value = name.trim(); - if (value.isEmpty() || value.equals(".") || value.equals("..")) return false; - if (value.length() > 255 || value.contains("/") || value.contains("\\")) return false; - if (value.contains("..") || value.indexOf('\u0000') >= 0) return false; - for (int i = 0; i < value.length(); i++) { - if (Character.isISOControl(value.charAt(i))) return false; - } - return new File(value).getName().equals(value); - } - - private static String sha256(File file) throws IOException { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (java.security.NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - try (InputStream in = new FileInputStream(file)) { - byte[] buffer = new byte[8192]; - int read; - while ((read = in.read(buffer)) != -1) digest.update(buffer, 0, read); - } - StringBuilder hex = new StringBuilder(64); - for (byte value : digest.digest()) hex.append(String.format("%02x", value & 0xff)); - return hex.toString(); - } - - @NonNull - private static String sha256(@NonNull InputStream input) throws IOException { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (java.security.NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - try (InputStream in = input) { - byte[] buffer = new byte[8192]; - int read; - while ((read = in.read(buffer)) != -1) { - digest.update(buffer, 0, read); - } - } - StringBuilder hex = new StringBuilder(64); - for (byte value : digest.digest()) hex.append(String.format("%02x", value & 0xff)); - return hex.toString(); + return AttachmentUrl.isSafeSegment(name.trim()); } /** Resolves a serialized note attachment to its app-private file. */ @@ -1549,7 +1923,7 @@ public interface TransactionFailureInjector { @NonNull private static String detectMimeType( - @NonNull File file, EditorAttachment attachment, @NonNull String displayName) { + @Nullable File file, EditorAttachment attachment, @NonNull String displayName) { String mimeType = URLConnection.guessContentTypeFromName(displayName); if (mimeType != null && !mimeType.trim().isEmpty()) { return mimeType; @@ -1566,7 +1940,7 @@ private static String detectMimeType( } } - mimeType = URLConnection.guessContentTypeFromName(file.getName()); + mimeType = file == null ? null : URLConnection.guessContentTypeFromName(file.getName()); return mimeType == null || mimeType.trim().isEmpty() ? "application/octet-stream" : mimeType; diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/Sha256.java b/app/src/main/java/com/pasich/mynotes/data/sync/Sha256.java new file mode 100644 index 00000000..52b77eb1 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/Sha256.java @@ -0,0 +1,77 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * The one SHA-256 the sync code hashes with. + * + *

Five copies of the digest-and-hex loop used to live in the store, the validator, the service + * and the backend, three of them formatting through {@code String.format("%02x")} with the default + * locale. A blob hashed by one copy has to match a manifest written by another, so the hex encoding + * is a lookup table here and nowhere else. + */ +final class Sha256 { + + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + private Sha256() {} + + /** Lowercase hex, locale-independent by construction. */ + @NonNull + static String hex(@NonNull byte[] bytes) { + char[] out = new char[bytes.length * 2]; + for (int index = 0; index < bytes.length; index++) { + int value = bytes[index] & 0xff; + out[index * 2] = HEX[value >>> 4]; + out[index * 2 + 1] = HEX[value & 0xf]; + } + return new String(out); + } + + @NonNull + static MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException missing) { + // Mandatory on every Java platform; treating it as recoverable only hides a broken + // runtime behind a sync error. + throw new IllegalStateException("SHA-256 is unavailable", missing); + } + } + + @NonNull + static String of(@NonNull byte[] bytes) { + return hex(newDigest().digest(bytes)); + } + + @NonNull + static String of(@NonNull String value) { + return of(value.getBytes(StandardCharsets.UTF_8)); + } + + /** Reads {@code input} to its end without closing it. */ + @NonNull + static String of(@NonNull InputStream input) throws IOException { + MessageDigest digest = newDigest(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + digest.update(buffer, 0, read); + } + return hex(digest.digest()); + } + + @NonNull + static String of(@NonNull File file) throws IOException { + try (InputStream input = new FileInputStream(file)) { + return of(input); + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java index 566bb9d1..ddacf042 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java @@ -56,12 +56,24 @@ public SyncSnapshot requireSnapshot() throws IOException { return snapshot; } + /** + * The failure for a problem discovered after the build — a blob the build described from + * remembered metadata that no endpoint turned out to hold — worded like a build failure so the + * user reads the same "which note" message. + */ + @NonNull + public static SnapshotBuildException incompleteBecause(@NonNull SnapshotProblem problem) { + return new SnapshotBuildException(Collections.singletonList(problem)); + } + /** Typed, coarse error suitable for persisted sync state and telemetry. */ public static final class SnapshotBuildException extends IOException { @NonNull private final List problems; private SnapshotBuildException(@NonNull List problems) { - super("Local snapshot is incomplete: " + problems.get(0).getKind().name()); + // The first problem names its record: a user who reads this on the account screen + // has to know which note to open, not only that some attachment somewhere is gone. + super("Local snapshot is incomplete: " + problems.get(0).describe()); this.problems = Collections.unmodifiableList(new ArrayList<>(problems)); } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java index 6d8aeb9d..daba7b30 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java @@ -1,6 +1,7 @@ package com.pasich.mynotes.data.sync; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import java.util.Objects; /** A privacy-safe reason why a local snapshot cannot safely be published. */ @@ -13,15 +14,32 @@ public enum Kind { INVALID_ATTACHMENT_METADATA } + private static final int MAX_LABEL_LENGTH = 40; + @NonNull private final Kind kind; @NonNull private final String recordType; @NonNull private final String stableId; + @Nullable private final String label; public SnapshotProblem( @NonNull Kind kind, @NonNull String recordType, @NonNull String stableId) { + this(kind, recordType, stableId, null); + } + + /** + * @param label what the user calls the record — a note's title — so the failure names the note + * to fix instead of leaving them to guess. Stays on the device: it is shown on the account + * screen and kept in the local sync state, never published. + */ + public SnapshotProblem( + @NonNull Kind kind, + @NonNull String recordType, + @NonNull String stableId, + @Nullable String label) { this.kind = Objects.requireNonNull(kind, "kind"); this.recordType = Objects.requireNonNull(recordType, "recordType"); this.stableId = Objects.requireNonNull(stableId, "stableId"); + this.label = label == null || label.trim().isEmpty() ? null : truncate(label.trim()); } @NonNull @@ -38,4 +56,23 @@ public String getRecordType() { public String getStableId() { return stableId; } + + @Nullable + public String getLabel() { + return label; + } + + /** One line naming the failure and, when known, the record it is in. */ + @NonNull + public String describe() { + return label == null + ? kind.name() + : kind.name() + " in " + recordType + " \"" + label + "\""; + } + + private static String truncate(String value) { + return value.length() <= MAX_LABEL_LENGTH + ? value + : value.substring(0, MAX_LABEL_LENGTH) + "…"; + } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java index 74b04833..c8f6e14b 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java @@ -10,9 +10,14 @@ * *

Implementations may use Google Drive, WebDAV, or an on-device test store. Attachments are * addressed by their content SHA-256, so writing the same attachment more than once must be safe. - * {@link #writeSnapshot(SyncSnapshot)} is the commit point: implementations must not make a new + * {@link #publish(SyncPublication)} is the commit point: implementations must not make a new * manifest visible before the call succeeds. The service uploads every required attachment before * it calls this method. + * + *

There is deliberately no plain "read a snapshot" or "write a snapshot" pair. A publish has to + * quote the read it was derived from, or a backend that keeps causal history cannot tell a write + * with stale parents from a legitimate one; the Drive backend had to disable exactly such a + * default, and a future backend must not be able to inherit it by accident. */ public interface SyncBackend { @@ -21,36 +26,21 @@ public interface SyncBackend { String getIdentifier(); /** - * Reads the current remote manifest. + * Reads the remote causal frontier. * - *

Returns {@link SyncSnapshot#empty()} when this backend has not yet received a sync bundle. - */ - @NonNull - SyncSnapshot readSnapshot() throws IOException; - - /** - * Reads the remote causal frontier. Legacy adapters expose one snapshot and no remote - * conflicts; Drive overrides this so concurrent immutable bundle heads remain recoverable. + *

Returns an empty snapshot, with a read token the next publish can quote, when this backend + * has not yet received a sync bundle. */ @NonNull - default RemoteSnapshot readSnapshotResult() throws IOException { - return RemoteSnapshot.of(readSnapshot()); - } - - /** Publishes a complete remote snapshot. Implementations must not expose a partial snapshot. */ - void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IOException; + RemoteSnapshot readSnapshotResult() throws IOException; /** * Publishes a snapshot together with the unresolved conflict versions it must keep alive and * the read context it was derived from. * - *

Backends that keep no causal history fall back to the plain snapshot write; the Drive - * backend overrides this so a publish cannot use stale causal parents and cannot drop an - * unresolved alternative on the floor. + *

Implementations must refuse a publication whose read context is not their latest read. */ - default void publish(@NonNull SyncPublication publication) throws IOException { - writeSnapshot(publication.getSnapshot()); - } + void publish(@NonNull SyncPublication publication) throws IOException; /** Returns true when the immutable attachment blob already exists remotely. */ boolean hasAttachment(@NonNull String sha256) throws IOException; @@ -68,43 +58,26 @@ default boolean hasVerifiedAttachment(@NonNull String sha256, @Nullable Long exp if (content == null) { return false; } - java.security.MessageDigest digest; - try { - digest = java.security.MessageDigest.getInstance("SHA-256"); - } catch (java.security.NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } - long size = 0L; try (InputStream input = content) { - byte[] buffer = new byte[8192]; - int read; - while ((read = input.read(buffer)) != -1) { - digest.update(buffer, 0, read); - size += read; - } - } - StringBuilder actual = new StringBuilder(64); - for (byte value : digest.digest()) { - actual.append(String.format(java.util.Locale.US, "%02x", value & 0xff)); + VerifyingInputStream.verify(input, sha256, expectedSize); + return true; + } catch (AttachmentIntegrityException corrupt) { + return false; } - return sha256.equals(actual.toString()) && (expectedSize == null || expectedSize == size); } /** * Opens an attachment by its lowercase SHA-256 hash, or returns {@code null} when it is absent. - * The caller closes the returned stream. + * The caller closes the returned stream and is responsible for verifying it. */ @Nullable InputStream readAttachment(@NonNull String sha256) throws IOException; /** - * Stores a complete attachment under {@code sha256}. + * Stores one immutable blob, streaming it rather than holding it in memory. * *

The implementation must consume the stream before returning and must not expose a partial * file after an exception. - */ - /** - * Stores one immutable blob, streaming it rather than holding it in memory. * * @param sizeBytes the blob's declared size, or a negative value when it is unknown. A known * size lets an implementation avoid buffering the whole blob to compute a content length. diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 98e089c7..3920eef4 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -35,6 +35,17 @@ public final class SyncBundleCodec { public static final String ENTRY_RECORDS = "records.json"; public static final String BUNDLE_FORMAT = "mynotes-sync"; public static final int SCHEMA_VERSION = 1; + + /** + * Wire field mapping a re-keyed manifest id back to the record's own attachment id. + * + *

Unknown to 2.6.50, which reads the re-keyed id as the attachment's own; on such a client + * the alternative's attachment identity is lost until it upgrades. Accepted: the field only + * appears when one note's versions disagree about an id's content, and the alternative is still + * restorable there, under the re-keyed id. + */ + static final String FIELD_ATTACHMENT_ID_ALIASES = "attachmentIdAliases"; + private static final Pattern SHA_256 = Pattern.compile("[0-9a-f]{64}"); private static final Pattern MIME_TYPE = Pattern.compile("^[a-z0-9!#$&^_.+-]+/[a-z0-9!#$&^_.+-]+$"); @@ -82,15 +93,18 @@ public byte[] encode( @NonNull Collection unresolvedAlternatives, @NonNull Collection resolvedAlternativeIds) throws IOException { + List alternatives = dedupeAlternatives(unresolvedAlternatives); + AttachmentPlan attachmentPlan = planAttachments(snapshot, alternatives); JsonObject recordsRoot = new JsonObject(); - recordsRoot.add("notes", liveArray(snapshot, SyncRecord.Type.NOTE)); - recordsRoot.add("tasks", liveArray(snapshot, SyncRecord.Type.TASK)); - recordsRoot.add("tags", liveArray(snapshot, SyncRecord.Type.TAG)); - recordsRoot.add("categories", liveArray(snapshot, SyncRecord.Type.CATEGORY)); - recordsRoot.add("preferences", liveArray(snapshot, SyncRecord.Type.PREFERENCES)); + recordsRoot.add("notes", liveArray(snapshot, SyncRecord.Type.NOTE, attachmentPlan)); + recordsRoot.add("tasks", liveArray(snapshot, SyncRecord.Type.TASK, attachmentPlan)); + recordsRoot.add("tags", liveArray(snapshot, SyncRecord.Type.TAG, attachmentPlan)); + recordsRoot.add( + "categories", liveArray(snapshot, SyncRecord.Type.CATEGORY, attachmentPlan)); + recordsRoot.add( + "preferences", liveArray(snapshot, SyncRecord.Type.PREFERENCES, attachmentPlan)); recordsRoot.add("tombstones", tombstones(snapshot)); - List alternatives = dedupeAlternatives(unresolvedAlternatives); - recordsRoot.add("alternatives", alternativeArray(alternatives)); + recordsRoot.add("alternatives", alternativeArray(alternatives, attachmentPlan)); JsonArray resolved = new JsonArray(); for (String versionId : new java.util.TreeSet<>(resolvedAlternativeIds)) { if (!SHA_256.matcher(versionId).matches()) { @@ -100,7 +114,7 @@ public byte[] encode( } recordsRoot.add("resolvedAlternatives", resolved); - JsonArray attachments = collectAttachments(snapshot, alternatives); + JsonArray attachments = attachmentPlan.manifest(); byte[] recordBytes = GSON.toJson(recordsRoot).getBytes(StandardCharsets.UTF_8); if (recordBytes.length > SyncBundleValidator.MAX_RECORD_BYTES) { throw new IOException("Sync records exceed the schema-1 size limit"); @@ -192,31 +206,50 @@ private static void writeEntry(ZipOutputStream zip, String name, byte[] bytes) @NonNull private static JsonArray liveArray( - @NonNull SyncSnapshot snapshot, @NonNull SyncRecord.Type type) throws IOException { + @NonNull SyncSnapshot snapshot, + @NonNull SyncRecord.Type type, + @NonNull AttachmentPlan attachmentPlan) + throws IOException { JsonArray array = new JsonArray(); for (SyncRecord record : snapshot.getLiveRecords(type)) { JsonObject item = record.getPayload(); item.addProperty("id", record.getId()); item.addProperty("updatedAt", record.getUpdatedAt().toString()); if (type == SyncRecord.Type.NOTE) { - normalizeNoteAttachmentFields(item); + normalizeNoteAttachmentFields(item, attachmentPlan.wireIdsFor(record)); } array.add(item); } return array; } - private static void normalizeNoteAttachmentFields(JsonObject note) throws IOException { + /** + * Replaces the local attachment fields with the wire references. + * + * @param wireIds the manifest id each of this record's attachments travels under, by manifest + * position; identical to the logical id except for a re-keyed collision. + */ + private static void normalizeNoteAttachmentFields( + JsonObject note, @NonNull List wireIds) throws IOException { JsonArray manifestEntries = note.getAsJsonArray("attachmentsManifest"); JsonArray attachmentIds = new JsonArray(); JsonObject attachmentNames = new JsonObject(); + JsonObject aliases = new JsonObject(); if (manifestEntries != null) { - for (JsonElement element : manifestEntries) { + for (int index = 0; index < manifestEntries.size(); index++) { AttachmentManifestEntry attachment = - AttachmentManifestEntry.fromJson(element.getAsJsonObject()); - attachmentIds.add(attachment.id); + AttachmentManifestEntry.fromJson( + manifestEntries.get(index).getAsJsonObject()); + // By position, not by logical id: one record can carry the same id twice with + // different content, and a map keyed by id sent both references to the second + // blob, so every receiver lost the first one's bytes. + String wireId = index < wireIds.size() ? wireIds.get(index) : attachment.id; + attachmentIds.add(wireId); + if (!wireId.equals(attachment.id)) { + aliases.addProperty(wireId, attachment.id); + } if (attachment.displayName != null && !attachment.displayName.isEmpty()) { - attachmentNames.addProperty(attachment.id, attachment.displayName); + attachmentNames.addProperty(wireId, attachment.displayName); } } } @@ -227,12 +260,16 @@ private static void normalizeNoteAttachmentFields(JsonObject note) throws IOExce // empty, and a decoded record then hashed differently from the local one that produced // it — a conflict against itself on every sync. note.remove("attachmentNames"); + note.remove(FIELD_ATTACHMENT_ID_ALIASES); if (attachmentIds.size() > 0) { note.add("attachmentIds", attachmentIds); } if (attachmentNames.size() > 0) { note.add("attachmentNames", attachmentNames); } + if (aliases.size() > 0) { + note.add(FIELD_ATTACHMENT_ID_ALIASES, aliases); + } } /** @@ -258,7 +295,8 @@ private static List dedupeAlternatives( } @NonNull - private static JsonArray alternativeArray(@NonNull List alternatives) + private static JsonArray alternativeArray( + @NonNull List alternatives, @NonNull AttachmentPlan attachmentPlan) throws IOException { JsonArray array = new JsonArray(); for (SyncRecord alternative : alternatives) { @@ -270,7 +308,7 @@ private static JsonArray alternativeArray(@NonNull List alternatives if (alternative.isTombstone()) { item.addProperty("deletedAt", alternative.getDeletedAt().toString()); } else if (alternative.getType() == SyncRecord.Type.NOTE) { - normalizeNoteAttachmentFields(item); + normalizeNoteAttachmentFields(item, attachmentPlan.wireIdsFor(alternative)); } array.add(item); } @@ -291,12 +329,23 @@ private static JsonArray tombstones(@NonNull SyncSnapshot snapshot) { return array; } + /** + * Lays out the bundle's attachment manifest for every version it carries. + * + *

The manifest is keyed by logical attachment id and a note references its entries by that + * id, so one id can describe only one blob per bundle. A live note and one of its unresolved + * alternatives may nonetheless carry different content under one id — the file was replaced in + * place, or two devices derived the same id — and refusing to encode that used to fail every + * publish for the account, before the conflict could even be stored for the user to settle. The + * later version's entry now travels under a derived id and the record carries the mapping back, + * so the decoded version is byte-for-byte the one that was published and keeps the identity + * every device's resolution bookkeeping refers to. + */ @NonNull - private static JsonArray collectAttachments( + private static AttachmentPlan planAttachments( @NonNull SyncSnapshot snapshot, @NonNull List alternatives) throws IOException { - JsonArray attachments = new JsonArray(); - Map seenById = new LinkedHashMap<>(); + AttachmentPlan plan = new AttachmentPlan(); Map seenByHash = new LinkedHashMap<>(); List notes = new ArrayList<>(snapshot.getLiveRecords(SyncRecord.Type.NOTE)); // An unresolved alternative is only recoverable if its blobs are described here too. @@ -311,12 +360,29 @@ private static JsonArray collectAttachments( for (JsonElement element : manifestEntries) { AttachmentManifestEntry attachment = AttachmentManifestEntry.fromJson(element.getAsJsonObject()); - AttachmentManifestEntry sameId = seenById.putIfAbsent(attachment.id, attachment); + List recordWireIds = plan.wireIdsFor(record); + String wireId = attachment.id; + AttachmentManifestEntry sameId = plan.byWireId.get(wireId); if (sameId != null && !sameId.sameRemoteFile(attachment)) { - // The same logical attachment may appear in both a live note and one of its - // unresolved alternatives; only differing content is a contradiction. - throw new IOException("Two notes reference conflicting attachment metadata"); + wireId = aliasFor(attachment, 0); + sameId = plan.byWireId.get(wireId); + if (sameId != null && !sameId.sameRemoteFile(attachment)) { + throw new IOException( + "Two notes reference conflicting attachment metadata"); + } + } + // One record may repeat an entry — 2.6.50 published a duplicated block that way, + // and its receivers hold such manifests as conflict alternatives. Each repeat gets + // a wire id of its own, so a note never references one entry twice on the wire + // and every repeat maps back to the record's own id on the way in. + for (int occurrence = 1; recordWireIds.contains(wireId); occurrence++) { + wireId = aliasFor(attachment, occurrence); + sameId = plan.byWireId.get(wireId); + } + if (sameId == null) { + plan.byWireId.put(wireId, attachment.withId(wireId)); } + recordWireIds.add(wireId); AttachmentManifestEntry previous = seenByHash.putIfAbsent(attachment.sha256, attachment); if (previous != null && !previous.sameRemoteFile(attachment)) { @@ -327,10 +393,43 @@ private static JsonArray collectAttachments( if (seenByHash.size() > SyncBundleValidator.MAX_ATTACHMENT_COUNT) { throw new IOException("Sync bundle exceeds the schema-1 attachment limit"); } - for (AttachmentManifestEntry attachment : seenById.values()) { - attachments.add(attachment.toJson(false)); + return plan; + } + + /** Deterministic, so two devices publishing the same collision write the same bundle. */ + @NonNull + private static String aliasFor(@NonNull AttachmentManifestEntry attachment, int occurrence) { + String source = "attachment-alias\n" + attachment.id + "\n" + attachment.sha256; + if (occurrence > 0) { + source += "\n" + occurrence; + } + return UUID.nameUUIDFromBytes(source.getBytes(StandardCharsets.UTF_8)).toString(); + } + + /** The manifest entries by wire id, and each record's wire ids in manifest order. */ + private static final class AttachmentPlan { + private final Map byWireId = new LinkedHashMap<>(); + private final Map> wireIdsByRecord = + new java.util.IdentityHashMap<>(); + + @NonNull + List wireIdsFor(@NonNull SyncRecord record) { + List wireIds = wireIdsByRecord.get(record); + if (wireIds == null) { + wireIds = new ArrayList<>(); + wireIdsByRecord.put(record, wireIds); + } + return wireIds; + } + + @NonNull + JsonArray manifest() { + JsonArray attachments = new JsonArray(); + for (AttachmentManifestEntry attachment : byWireId.values()) { + attachments.add(attachment.toJson(false)); + } + return attachments; } - return attachments; } private static void parseLiveRecords( @@ -353,7 +452,7 @@ private static void parseLiveRecords( // Bundles written before the device-local fields were stripped still carry them. SyncMetadata.stripDeviceLocalFields(type.getWireValue(), payload); if (type == SyncRecord.Type.NOTE) { - hydrateNoteAttachments(payload, attachmentsById); + hydrateNoteAttachments(id, payload, attachmentsById); } String identity = type.getWireValue() + ":" + id; if (!identities.add(identity)) { @@ -363,12 +462,29 @@ private static void parseLiveRecords( } } + /** + * Rebuilds the local attachment fields from the wire references. + * + *

A payload written by 2.6.50 comes out in the current shape as well, so an unchanged note + * synced before the upgrade hashes exactly as the upgraded store rebuilds it; see {@link + * LegacyNotePayload}. + */ private static void hydrateNoteAttachments( - JsonObject payload, Map attachmentsById) + String noteStableId, + JsonObject payload, + Map attachmentsById) throws IOException { JsonArray attachmentIds = payload.getAsJsonArray("attachmentIds"); JsonObject attachmentNames = payload.getAsJsonObject("attachmentNames"); - if (attachmentIds == null) return; + JsonObject aliases = payload.getAsJsonObject(FIELD_ATTACHMENT_ID_ALIASES); + // Wire-only, all three: the local store never produces them, and leaving one behind made + // a decoded record hash differently from the identical local one. + payload.remove("attachmentIds"); + payload.remove(FIELD_ATTACHMENT_ID_ALIASES); + if (attachmentIds == null) { + payload.remove("attachmentNames"); + return; + } JsonArray attachmentHashes = new JsonArray(); JsonArray manifest = new JsonArray(); // Keyed by logical attachment UUID, exactly as the wire carries it and exactly as @@ -377,14 +493,22 @@ private static void hydrateNoteAttachments( // attachment reported a conflict against itself on every sync, forever. JsonObject namesById = new JsonObject(); for (JsonElement element : attachmentIds) { - String attachmentId = element.getAsString(); - AttachmentManifestEntry attachment = attachmentsById.get(attachmentId); + String wireId = element.getAsString(); + AttachmentManifestEntry attachment = attachmentsById.get(wireId); if (attachment == null) { throw new IOException("Note references an unknown attachment manifest entry"); } - JsonObject value = attachment.toJson(true); - if (attachmentNames != null && attachmentNames.has(attachmentId)) { - value.addProperty("displayName", attachmentNames.get(attachmentId).getAsString()); + // A re-keyed collision travels under a derived id; the record's own id is restored + // so the decoded version is the one that was published. + String attachmentId = + aliases != null && aliases.has(wireId) + ? aliases.get(wireId).getAsString() + : wireId; + JsonObject value = attachment.withId(attachmentId).toJson(true); + if (attachmentNames != null && attachmentNames.has(wireId)) { + // Trimmed here as everywhere else: the validator judged the trimmed name, and + // this is the copy the store hashes and shows. + value.addProperty("displayName", attachmentNames.get(wireId).getAsString().trim()); } manifest.add(value); attachmentHashes.add(attachment.sha256); @@ -397,9 +521,7 @@ private static void hydrateNoteAttachments( // The display name restoreAttachments actually uses travels on the manifest entry above; // this map exists only so the payload matches the one the local store builds. payload.add("attachmentNames", namesById); - // Wire-only: the local store never produces it, and leaving it behind made a decoded - // record hash differently from the identical local one. - payload.remove("attachmentIds"); + LegacyNotePayload.upgrade(noteStableId, payload); } @NonNull @@ -434,7 +556,7 @@ private static List parseAlternatives( payload.remove("deletedAt"); SyncMetadata.stripDeviceLocalFields(type.getWireValue(), payload); if (type == SyncRecord.Type.NOTE) { - hydrateNoteAttachments(payload, attachmentsById); + hydrateNoteAttachments(id, payload, attachmentsById); } alternatives.add(SyncRecord.live(type, id, updatedAt, payload)); } @@ -493,8 +615,8 @@ private static JsonArray requireArray(@NonNull JsonObject object, @NonNull Strin } @NonNull - private static String sha256(byte[] bytes) throws IOException { - return SyncBundleValidator.sha256(bytes); + private static String sha256(byte[] bytes) { + return Sha256.of(bytes); } public static final class DecodedBundle { @@ -601,11 +723,12 @@ public static AttachmentManifestEntry fromJson(@NonNull JsonObject value) if (!path.equals("attachments/" + sha256)) { throw new IOException("Sync bundle contains an invalid attachment path"); } + // Trimmed here, once, so every consumer sees the name the validator judged. String displayName = value.has("displayName") && !value.get("displayName").isJsonNull() && value.get("displayName").isJsonPrimitive() - ? value.get("displayName").getAsString() + ? value.get("displayName").getAsString().trim() : null; if (value.has("displayName") && !value.get("displayName").isJsonNull() @@ -635,5 +758,13 @@ JsonObject toJson(boolean includeDisplayName) { boolean sameRemoteFile(@NonNull AttachmentManifestEntry other) { return sha256.equals(other.sha256) && path.equals(other.path) && size == other.size; } + + /** The same blob under another logical id. */ + @NonNull + AttachmentManifestEntry withId(@NonNull String newId) { + return newId.equals(id) + ? this + : new AttachmentManifestEntry(newId, sha256, mimeType, size, path, displayName); + } } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java index 083ca977..21534111 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java @@ -10,13 +10,10 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.time.format.DateTimeParseException; import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -178,11 +175,21 @@ private static void validateAttachmentReferences( throw new IOException("Sync note exceeds the attachment limit"); } JsonObject attachmentNames = note.getAsJsonObject("attachmentNames"); + validateAttachmentIdAliases(note, attachmentsById); + JsonObject aliases = note.getAsJsonObject(SyncBundleCodec.FIELD_ATTACHMENT_ID_ALIASES); + Set withinNote = new LinkedHashSet<>(); for (JsonElement element : attachmentIds) { if (element == null || !element.isJsonPrimitive()) { throw new IOException("Sync note attachmentIds entry is invalid"); } String attachmentId = element.getAsString(); + if (!withinNote.add(attachmentId) && aliases != null && aliases.has(attachmentId)) { + // A plain id repeated is what 2.6.50 published for a duplicated block: two + // references, one blob, harmless, and it has to keep decoding. A re-keyed id + // repeated is something else: the record's two attachments were collapsed into + // one on the way out, and accepting it wrote one blob's bytes for both. + throw new IOException("Sync note collapses two attachments into one reference"); + } SyncBundleCodec.AttachmentManifestEntry attachment = attachmentsById.get(attachmentId); if (attachment == null) { throw new IOException("Note references an unknown attachment manifest entry"); @@ -199,6 +206,33 @@ private static void validateAttachmentReferences( } } + /** + * Checks the map that lets two versions of one note carry different content under one logical + * attachment id; see {@code SyncBundleCodec.collectAttachments}. + */ + private static void validateAttachmentIdAliases( + @NonNull JsonObject note, + @NonNull Map attachmentsById) + throws IOException { + JsonElement aliases = note.get(SyncBundleCodec.FIELD_ATTACHMENT_ID_ALIASES); + if (aliases == null || aliases.isJsonNull()) { + return; + } + if (!aliases.isJsonObject()) { + throw new IOException("Sync note attachmentIdAliases is invalid"); + } + for (Map.Entry alias : aliases.getAsJsonObject().entrySet()) { + validateUuid(alias.getKey()); + if (!alias.getValue().isJsonPrimitive()) { + throw new IOException("Sync note attachmentIdAliases entry is invalid"); + } + validateUuid(alias.getValue().getAsString()); + if (!attachmentsById.containsKey(alias.getKey())) { + throw new IOException("Note aliases an unknown attachment manifest entry"); + } + } + } + /** * Validates the unresolved conflict versions a bundle carries. * @@ -511,23 +545,18 @@ private static void validateJsonValue(@NonNull JsonElement value, int depth) } } + /** + * Accepts a display name only in the form every consumer will see it: trimmed, then held to the + * one path-segment rule the attachment code shares. + * + *

Three validators used to exist with three answers for a name with trailing whitespace; the + * day any path built a file name from the display name, the bundle validator's answer and the + * URL parser's would have disagreed and the cleaner would have refused the note. + */ static void validateDisplayName(@NonNull String name) throws IOException { - String value = name.trim(); - if (value.isEmpty() - || value.equals(".") - || value.equals("..") - || value.length() > 255 - || value.contains("..") - || value.contains("/") - || value.contains("\\") - || new java.io.File(value).getName().equals(value) == false) { + if (!com.pasich.mynotes.extendedEditor.attach.AttachmentUrl.isSafeSegment(name.trim())) { throw new IOException("Sync attachment name is not a safe file name"); } - for (int i = 0; i < value.length(); i++) { - if (Character.isISOControl(value.charAt(i))) { - throw new IOException("Sync attachment name contains a control character"); - } - } } static void validateUuid(@NonNull String value) throws IOException { @@ -541,17 +570,8 @@ static void validateUuid(@NonNull String value) throws IOException { } @NonNull - static String sha256(@NonNull byte[] bytes) throws IOException { - try { - byte[] digest = MessageDigest.getInstance("SHA-256").digest(bytes); - StringBuilder value = new StringBuilder(digest.length * 2); - for (byte byteValue : digest) { - value.append(String.format(Locale.US, "%02x", byteValue & 0xff)); - } - return value.toString(); - } catch (NoSuchAlgorithmException error) { - throw new IOException("SHA-256 is unavailable", error); - } + static String sha256(@NonNull byte[] bytes) { + return Sha256.of(bytes); } /** Caps compressed input before ZIP parsing to make bundle-size limits independent of Drive. */ diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java index d3bc990b..47f42e9e 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java @@ -45,11 +45,18 @@ interface StableIdGenerator { String nextStableId(); } - /** Repoints a restored note's attachments when its row id had to change. */ + /** + * Moves a restored note's attachments to where its final row id expects them. + * + * @return true when the note's attachment or block JSON was rewritten and must be stored. + */ interface AttachmentRelocation { - void relocate(@NonNull Note note, int previousId); + boolean relocate(@NonNull Note note, int previousId); } + /** Keeps every {@code IN (...)} clause under SQLite's bound-variable limit. */ + private static final int QUERY_CHUNK = 500; + private final TransactionExecutor transactionExecutor; private final NoteDao noteDao; private final TaskDao taskDao; @@ -93,7 +100,9 @@ public T call() { (note, previousId) -> { com.pasich.mynotes.extendedEditor.attach.NoteAttachmentRelocator.Result moved = com.pasich.mynotes.extendedEditor.attach.NoteAttachmentRelocator - .relocate( + .adoptStaged( + com.pasich.mynotes.extendedEditor.attach + .AttachmentStorage.restoreStagingDir(context), com.pasich.mynotes.extendedEditor.attach .AttachmentStorage.baseDirPath(context), previousId, @@ -104,6 +113,7 @@ public T call() { note.setAttachments(moved.attachmentsJson); note.setValueJson(moved.valueJson); } + return moved.changed; }); } @@ -127,7 +137,7 @@ public T call() { syncMetadataDao, timeProvider, stableIdGenerator, - (note, previousId) -> {}); + (note, previousId) -> false); } SyncMutationCoordinator( @@ -175,10 +185,16 @@ public void insertTags(List incoming) { SyncMetadata.RECORD_TYPE_TAG, extractTagIds(tags)); // Same REPLACE-insert collision as notes; see insertNotes. + Set taken = new LinkedHashSet<>(); + for (Tag existing : tagsByIds(extractTagIds(tags))) { + taken.add(existing.getId()); + } List keepingId = new ArrayList<>(); List reassigned = new ArrayList<>(); + Set claimed = new LinkedHashSet<>(); for (Tag tag : tags) { - if (tag.getId() > 0 && tagsDao.getTagSync(tag.getId()) != null) { + if (tag.getId() > 0 + && (taken.contains(tag.getId()) || !claimed.add(tag.getId()))) { tag.id = 0; reassigned.add(tag); } else { @@ -284,7 +300,12 @@ public void insertNotes(List incoming) { if (incoming == null || incoming.isEmpty()) return; transactionExecutor.run( () -> { - List notes = withoutNotesAlreadyPresent(incoming); + // One query for every incoming id rather than one per note per pass: a + // large restore ran four point lookups per note inside the transaction and + // sat on the restore dialog for tens of seconds doing nothing else. + Map existingById = notesByIds(extractNoteIds(incoming)); + List notes = withoutNotesAlreadyPresent(incoming, existingById); + adoptAttachmentsOfNotesAlreadyPresent(incoming, notes); if (notes.isEmpty()) return null; long timestamp = resolveBatchTimestamp( @@ -294,13 +315,18 @@ public void insertNotes(List incoming) { // their id first leaves the autoincrement counter past all of them, which is // what stops a reassigned note being handed an id a later note in the same // batch is about to claim: addNotes is a REPLACE insert, so that collision - // silently destroyed one of the two restored notes. + // silently destroyed one of the two restored notes. An id claimed twice + // within the batch itself is the same collision: the second note would + // REPLACE the first, so only the first may keep it. List keepingId = new ArrayList<>(); List reassigned = new ArrayList<>(); Map previousIds = new java.util.IdentityHashMap<>(); + Set claimed = new LinkedHashSet<>(); for (Note note : notes) { previousIds.put(note, note.getId()); - if (note.getId() > 0 && noteDao.getNoteSync(note.getId()) != null) { + if (note.getId() > 0 + && (existingById.containsKey(note.getId()) + || !claimed.add(note.getId()))) { note.setId(0); reassigned.add(note); } else { @@ -314,6 +340,35 @@ public void insertNotes(List incoming) { }); } + /** + * Gives a note this device already holds the files the archive carries for it. + * + *

A note skipped as already present may still be missing its files — a row restored from a + * JSON backup, or a library whose attachment folder was cleared — and the archive is the only + * place they are. Its staged files are adopted into its own folder like any other restored + * note's; the row is stored again only if a colliding file forced a reference to be rewritten. + */ + private void adoptAttachmentsOfNotesAlreadyPresent( + @NonNull List incoming, @NonNull List inserted) { + Set kept = java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<>()); + kept.addAll(inserted); + long now = timeProvider.now(); + for (Note note : incoming) { + if (kept.contains(note) || note.getId() <= 0) continue; + if (attachmentRelocation.relocate(note, note.getId())) { + noteDao.updateNoteContent( + note.getId(), + note.getTitle(), + note.getValue(), + note.getValueJson(), + note.getDate(), + note.getTag(), + note.getAttachments()); + touchRecord(SyncMetadata.RECORD_TYPE_NOTE, note.getId(), now); + } + } + } + /** Inserts one group and settles each note's final id, metadata and attachment folder. */ private void assignInsertedNoteIds( @NonNull List notes, long timestamp, @NonNull Map previousIds) { @@ -324,10 +379,9 @@ private void assignInsertedNoteIds( int previous = previousIds.get(note); int localId = resolveIntId(note.getId(), insertedIds[i]); note.setId(localId); - if (previous > 0 && previous != localId) { - // Its attachments were extracted under the old id and would otherwise share a - // folder with whichever note owns that id now. - attachmentRelocation.relocate(note, previous); + // Its attachments were staged under the id the archive knew; they move into the + // folder of the id it has now, whether or not the two differ. + if (previous > 0 && attachmentRelocation.relocate(note, previous)) { noteDao.updateNoteContent( localId, note.getTitle(), @@ -629,10 +683,11 @@ private long insertNoteInternal(@NonNull Note note, long timestamp) { * the existing one instead of overwriting it. */ @NonNull - private List withoutNotesAlreadyPresent(@NonNull List incoming) { + private static List withoutNotesAlreadyPresent( + @NonNull List incoming, @NonNull Map existingById) { List result = new ArrayList<>(incoming.size()); for (Note note : incoming) { - Note existing = note.getId() > 0 ? noteDao.getNoteSync(note.getId()) : null; + Note existing = note.getId() > 0 ? existingById.get(note.getId()) : null; if (existing == null || !isSameNoteContent(existing, note)) { result.add(note); } @@ -640,6 +695,44 @@ private List withoutNotesAlreadyPresent(@NonNull List incoming) { return result; } + /** The existing rows for these ids, fetched in bounded batches. */ + @NonNull + private Map notesByIds(@NonNull List ids) { + Map result = new java.util.HashMap<>(); + List positive = new ArrayList<>(); + for (Long id : ids) { + if (id != null && id > 0L) positive.add(id.intValue()); + } + for (List chunk : chunks(positive)) { + for (Note note : noteDao.getNotesByIdsSync(chunk)) { + result.put(note.getId(), note); + } + } + return result; + } + + @NonNull + private List tagsByIds(@NonNull List ids) { + List result = new ArrayList<>(); + List positive = new ArrayList<>(); + for (Long id : ids) { + if (id != null && id > 0L) positive.add(id); + } + for (List chunk : chunks(positive)) { + result.addAll(tagsDao.getTagsByIdsSync(chunk)); + } + return result; + } + + @NonNull + private static List> chunks(@NonNull List values) { + List> result = new ArrayList<>(); + for (int start = 0; start < values.size(); start += QUERY_CHUNK) { + result.add(values.subList(start, Math.min(values.size(), start + QUERY_CHUNK))); + } + return result; + } + /** True when two rows carry the same user-visible note. */ private static boolean isSameNoteContent(@NonNull Note existing, @NonNull Note incoming) { return equalText(existing.getTitle(), incoming.getTitle()) @@ -704,7 +797,7 @@ private void markDeletedRecord(@NonNull String recordType, long localId, long ti } private void ensureMetadataRow(@NonNull String recordType, long localId) { - if (syncMetadataDao.exists(recordType, localId)) return; + // INSERT OR IGNORE: a row that exists is left alone without a lookup first. syncMetadataDao.insertIfAbsent( new SyncMetadataEntity( recordType, localId, stableIdGenerator.nextStableId(), 0L, null)); @@ -725,11 +818,15 @@ private long resolveBatchTimestamp(@NonNull String recordType, @NonNull List localIds) { + List positive = new ArrayList<>(); for (Long localId : localIds) { - if (localId == null || localId <= 0L) continue; - if (!syncMetadataDao.exists(recordType, localId)) return true; + if (localId != null && localId > 0L) positive.add(localId); + } + Set known = new LinkedHashSet<>(); + for (List chunk : chunks(positive)) { + known.addAll(syncMetadataDao.getExistingLocalIds(recordType, chunk)); } - return false; + return !known.containsAll(positive); } private static List extractNoteIds(List notes) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncRecord.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncRecord.java index 9ecbfdb0..c0ed704e 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncRecord.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncRecord.java @@ -8,9 +8,6 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -80,7 +77,41 @@ private SyncRecord( throw new IllegalArgumentException("deletedAt must not be before updatedAt"); } this.deletedAt = deletedAt; - this.payload = Objects.requireNonNull(payload, "payload").deepCopy(); + this.payload = normalize(type, Objects.requireNonNull(payload, "payload").deepCopy()); + } + + /** + * Puts a payload into the one shape its version hash is taken from. + * + *

A note's attachment fields are present exactly when it has attachments. The local build, + * the bundle encoder and the bundle decoder each used to enforce that separately, and every + * time one of them drifted — an empty {@code attachmentNames} left on the wire, three empty + * arrays emitted for a note with none — a decoded record hashed differently from the identical + * local one and every affected note conflicted with itself on every sync. Every record passes + * through here, whichever side built it, so the three sites can no longer disagree. + */ + @NonNull + private static JsonObject normalize(@NonNull Type type, @NonNull JsonObject payload) { + if (type != Type.NOTE) { + return payload; + } + for (String field : new String[] {"attachmentsManifest", "attachmentHashes"}) { + JsonElement value = payload.get(field); + if (value != null + && (value.isJsonNull() + || !value.isJsonArray() + || value.getAsJsonArray().size() == 0)) { + payload.remove(field); + } + } + JsonElement names = payload.get("attachmentNames"); + if (names != null + && (names.isJsonNull() + || !names.isJsonObject() + || names.getAsJsonObject().size() == 0)) { + payload.remove("attachmentNames"); + } + return payload; } @NonNull @@ -171,18 +202,7 @@ private static void validateCanonicalUuid(String id) { @NonNull private static String sha256(String value) { - try { - byte[] digest = - MessageDigest.getInstance("SHA-256") - .digest(value.getBytes(StandardCharsets.UTF_8)); - StringBuilder hex = new StringBuilder(digest.length * 2); - for (byte byteValue : digest) { - hex.append(String.format("%02x", byteValue & 0xff)); - } - return hex.toString(); - } catch (NoSuchAlgorithmException exception) { - throw new IllegalStateException("SHA-256 is unavailable", exception); - } + return Sha256.of(value); } @NonNull diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java index 254a4ade..f53a3db6 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java @@ -5,11 +5,8 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Clock; import java.time.Instant; import java.util.Collection; @@ -63,9 +60,42 @@ public SyncService(@NonNull SyncStore store, @NonNull SyncMerger merger, @NonNul private static final long LOCK_WAIT_SECONDS = 5L; - /** Runs one serialized manual synchronization attempt and returns its durable final state. */ + /** + * Runs {@code action} while no sync is in flight, waiting for a running one to finish first. + * + *

For work that tears down what a sync writes — the disconnect wipe of state, conflicts and + * cached blobs. Done concurrently, that wipe raced the six-hourly worker: the worker's cache + * directory vanished under it and its final state and conflict rows landed after the wipe. + */ + public static void runWhileNoSyncRuns(@NonNull Runnable action) { + SYNC_LOCK.lock(); + try { + action.run(); + } finally { + SYNC_LOCK.unlock(); + } + } + + /** Runs one serialized synchronization attempt and returns its durable final state. */ @NonNull public SyncState sync(@NonNull SyncBackend backend) { + return sync(backend, () -> true); + } + + /** + * Runs one serialized synchronization attempt, unless {@code stillEnabled} says otherwise once + * the lock is held. + * + *

Disconnect turns sync off and then wipes the account's state under the lock. A worker that + * had already passed its own checks and was waiting for a token could still take the lock after + * that wipe and write the old account's state and conflicts back. The predicate is evaluated + * under the lock, after the wipe has either finished or not yet begun, so a disabled sync never + * gets to write anything. + */ + @NonNull + public SyncState sync( + @NonNull SyncBackend backend, + @NonNull java.util.function.BooleanSupplier stillEnabled) { boolean acquired = false; try { acquired = SYNC_LOCK.tryLock(LOCK_WAIT_SECONDS, TimeUnit.SECONDS); @@ -81,6 +111,13 @@ public SyncState sync(@NonNull SyncBackend backend) { "Another sync is already running; this attempt was temporarily skipped"); } try { + if (!stillEnabled.getAsBoolean()) { + // Deliberately not persisted: there is no account left to record it for. + return SyncState.error( + "google-drive", + safeReadState().getLastSuccessfulSyncAt(), + "Sync was turned off before this attempt could start"); + } return syncExclusively(backend); } finally { SYNC_LOCK.unlock(); @@ -152,7 +189,8 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { // The merged snapshot contains only the deterministic winner. A conflict row is not // durable unless the loser can later be restored as well, so preflight and pin each // version independently; SyncSnapshot deliberately forbids two versions of one ID. - synchronizeAttachments(backend, merged, expectedSizes); + java.util.Set synchronizedHashes = new java.util.HashSet<>(); + synchronizeAttachments(backend, merged, expectedSizes, synchronizedHashes); for (SyncMergeResult.Conflict conflict : allConflicts) { // Best effort. The merged snapshot's own blobs are mandatory and were just // transferred above; these are the extra copies that let a conflict be resolved @@ -161,8 +199,8 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { // syncing anything at all, including the devices that could never resolve it. // Resolution still verifies before it applies, so a version that cannot be // materialized simply cannot be chosen. - pinConflictVersionQuietly(backend, conflict.getWinner()); - pinConflictVersionQuietly(backend, conflict.getLoser()); + pinConflictVersionQuietly(backend, conflict.getWinner(), synchronizedHashes); + pinConflictVersionQuietly(backend, conflict.getLoser(), synchronizedHashes); } if (needsPublication( @@ -341,13 +379,27 @@ private static boolean snapshotsMatch( return true; } + /** + * Makes every blob {@code merged} references available, verified, at both endpoints. + * + * @param synchronizedHashes blobs already settled by an earlier call this sync, skipped here + * and extended with the ones settled now. A conflict version shares most of its blobs with + * the merged snapshot, and re-verifying each one meant re-hashing and re-downloading it + * once per version while the conflict stayed open. + */ private void synchronizeAttachments( - SyncBackend backend, SyncSnapshot merged, Map expectedSizes) + SyncBackend backend, + SyncSnapshot merged, + Map expectedSizes, + java.util.Set synchronizedHashes) throws IOException { Collection hashes = Objects.requireNonNull(store.getAttachmentHashes(merged), "hashes"); for (String hash : hashes) { validateHash(hash); + if (!synchronizedHashes.add(hash)) { + continue; + } if (store.hasAttachment(hash)) { Long expectedSize = expectedSizes.get(hash); // Index lookup only; the bytes are checked once, below. @@ -395,7 +447,7 @@ private void synchronizeAttachments( } else { InputStream remoteAttachment = backend.readAttachment(hash); if (remoteAttachment == null) { - throw new IOException("Required attachment is unavailable: " + hash); + throw unavailableAttachment(hash); } copyVerified( hash, expectedSizes.get(hash), remoteAttachment, store::writeAttachment); @@ -405,9 +457,11 @@ private void synchronizeAttachments( /** Pins a conflict version's blobs, logging rather than failing the whole sync. */ private void pinConflictVersionQuietly( - @NonNull SyncBackend backend, @NonNull SyncRecord record) { + @NonNull SyncBackend backend, + @NonNull SyncRecord record, + @NonNull java.util.Set synchronizedHashes) { try { - pinConflictVersion(backend, record); + pinConflictVersion(backend, record, synchronizedHashes); } catch (IOException unavailable) { Log.w( TAG, @@ -416,32 +470,58 @@ private void pinConflictVersionQuietly( } } - /** Pins required conflict blobs into the store's durable content-addressed cache. */ - private void pinConflictVersion(@NonNull SyncBackend backend, @NonNull SyncRecord record) + /** + * Pins required conflict blobs into the store's durable content-addressed cache. + * + *

A blob already in that cache is left alone: copying it onto itself rewrote hundreds of + * megabytes per sync for as long as a conflict on a large note stayed open. + */ + private void pinConflictVersion( + @NonNull SyncBackend backend, + @NonNull SyncRecord record, + @NonNull java.util.Set synchronizedHashes) throws IOException { if (record.isTombstone()) return; SyncSnapshot snapshot = new SyncSnapshot(java.util.Collections.singletonList(record)); Map expectedSizes = attachmentSizes(snapshot); - synchronizeAttachments(backend, snapshot, expectedSizes); + synchronizeAttachments(backend, snapshot, expectedSizes, synchronizedHashes); for (String hash : store.getAttachmentHashes(snapshot)) { Long expectedSize = expectedSizes.get(hash); + if (store.hasDurableAttachment(hash, expectedSize == null ? -1L : expectedSize)) { + continue; + } copyVerified(hash, expectedSize, store.readAttachment(hash), store::writeAttachment); } } + /** + * The failure for a blob neither endpoint holds. + * + *

When the store published the attachment from remembered metadata — its file gone, the + * bytes expected back from the remote — the message names the note, as a build failure would + * have; a hash alone left the user with a permanently failing sync and no way to find the note. + */ + @NonNull + private IOException unavailableAttachment(@NonNull String hash) { + SnapshotProblem problem = null; + try { + problem = store.describeMissingAttachment(hash); + } catch (RuntimeException ignored) { + // The description is a courtesy; the failure below stands without it. + } + return problem != null + ? SnapshotBuildResult.incompleteBecause(problem) + : new IOException("Required attachment is unavailable: " + hash); + } + private void verifyAttachment(String hash, Long expectedSize, InputStream source) throws IOException { if (source == null) { - throw new IOException("Required attachment is unavailable: " + hash); + throw unavailableAttachment(hash); } - try (InputStream input = source; - VerifyingInputStream verified = - new VerifyingInputStream(input, hash, expectedSize)) { - byte[] buffer = new byte[8192]; - while (verified.read(buffer) != -1) { - // Consume the complete blob before accepting an existing remote attachment. - } - verified.verifyEndOfStream(); + // Consume the complete blob before accepting an existing attachment. + try (InputStream input = source) { + VerifyingInputStream.verify(input, hash, expectedSize); } } @@ -538,126 +618,4 @@ private static void validateHash(String hash) throws IOException { private interface AttachmentWriter { void write(String hash, long sizeBytes, InputStream content) throws IOException; } - - /** Verifies the hash only after the receiving endpoint consumed every byte. */ - private static final class VerifyingInputStream extends FilterInputStream { - private final MessageDigest digest; - private final String expectedHash; - private final Long expectedSize; - private long byteCount; - private VerificationState verificationState = VerificationState.UNVERIFIED; - private AttachmentIntegrityException integrityFailure; - - private enum VerificationState { - UNVERIFIED, - VERIFIED, - FAILED - } - - VerifyingInputStream(InputStream input, String expectedHash, Long expectedSize) { - super(input); - this.expectedHash = expectedHash; - this.expectedSize = expectedSize; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException exception) { - throw new IllegalStateException("SHA-256 is unavailable", exception); - } - } - - @Override - public int read() throws IOException { - rethrowIntegrityFailure(); - int value = super.read(); - if (value >= 0) { - digest.update((byte) value); - byteCount++; - enforceSizeLimit(); - } else { - verifyEndOfStream(); - } - return value; - } - - @Override - public int read(byte[] buffer, int offset, int length) throws IOException { - rethrowIntegrityFailure(); - int read = super.read(buffer, offset, length); - if (read > 0) { - digest.update(buffer, offset, read); - byteCount += read; - enforceSizeLimit(); - } else if (read < 0) { - verifyEndOfStream(); - } - return read; - } - - private void enforceSizeLimit() throws IOException { - if (byteCount > SyncBundleValidator.MAX_ATTACHMENT_BYTES) { - failIntegrity("Attachment exceeds the sync size limit"); - } - } - - /** - * Checks the digest, draining anything the destination left behind first. - * - *

Reached from {@link #read} at end of stream, so a destination that streams straight to - * its final location still learns about a mismatch before it commits. - */ - void verifyEndOfStream() throws IOException { - if (verificationState == VerificationState.VERIFIED) { - return; - } - rethrowIntegrityFailure(); - try { - drainRemaining(); - String actualHash = toHex(digest.digest()); - if (!expectedHash.equals(actualHash)) { - failIntegrity("Attachment checksum does not match its declared hash"); - } - if (expectedSize != null && expectedSize.longValue() != byteCount) { - failIntegrity("Attachment size does not match its declared size"); - } - verificationState = VerificationState.VERIFIED; - } catch (AttachmentIntegrityException failure) { - integrityFailure = failure; - verificationState = VerificationState.FAILED; - throw failure; - } - } - - private void failIntegrity(String message) throws AttachmentIntegrityException { - AttachmentIntegrityException failure = new AttachmentIntegrityException(message); - integrityFailure = failure; - verificationState = VerificationState.FAILED; - throw failure; - } - - private void rethrowIntegrityFailure() throws AttachmentIntegrityException { - if (verificationState == VerificationState.FAILED && integrityFailure != null) { - throw integrityFailure; - } - } - - /** Reads through {@code super} so the digest covers bytes the destination skipped. */ - private void drainRemaining() throws IOException { - byte[] scratch = new byte[8192]; - int read; - while ((read = super.read(scratch, 0, scratch.length)) != -1) { - digest.update(scratch, 0, read); - byteCount += read; - enforceSizeLimit(); - } - } - - @NonNull - private static String toHex(byte[] bytes) { - StringBuilder result = new StringBuilder(bytes.length * 2); - for (byte value : bytes) { - result.append(String.format("%02x", value & 0xff)); - } - return result.toString(); - } - } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java index 8a0bd8db..71ecc652 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java @@ -1,6 +1,7 @@ package com.pasich.mynotes.data.sync; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import java.io.IOException; import java.io.InputStream; import java.util.Collection; @@ -71,18 +72,41 @@ default java.util.Set getResolvedAlternativeIds() throws IOException { /** True when the complete attachment is locally available. */ boolean hasAttachment(@NonNull String sha256) throws IOException; + /** + * Names the record a blob belongs to when the store published it without holding the bytes. + * + *

A store may describe an attachment from remembered metadata after its file has gone, + * expecting the bytes to come back from the remote. When no endpoint has them either, the + * failure has to name the note to fix, not a hash; a store that never does this answers null. + */ + @Nullable + default SnapshotProblem describeMissingAttachment(@NonNull String sha256) { + return null; + } + + /** + * True when the blob already sits in the store's own durable cache, as written by {@link + * #writeAttachment}, rather than only in a note's folder that a later edit may empty. + * + *

Lets the service pin a conflict version's blobs without copying a cached blob onto itself; + * a store without such a cache answers false and is simply written to again. + * + * @param sizeBytes the declared size, or a negative value to accept any. + */ + default boolean hasDurableAttachment(@NonNull String sha256, long sizeBytes) + throws IOException { + return false; + } + /** Opens one complete local attachment. The caller closes the returned stream. */ @NonNull InputStream readAttachment(@NonNull String sha256) throws IOException; /** - * Stores one complete attachment locally. + * Stores one immutable blob, streaming it rather than holding it in memory. * *

The implementation must consume the stream before returning and must not expose a partial * file after an exception. - */ - /** - * Stores one immutable blob, streaming it rather than holding it in memory. * * @param sizeBytes the blob's declared size, or a negative value when it is unknown. A known * size lets an implementation avoid buffering the whole blob to compute a content length. diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/VerifyingInputStream.java b/app/src/main/java/com/pasich/mynotes/data/sync/VerifyingInputStream.java new file mode 100644 index 00000000..bf9cebfa --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/VerifyingInputStream.java @@ -0,0 +1,147 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; + +/** + * Checks a content-addressed blob against its declared hash and size as the bytes go past. + * + *

This is the single verifier for every attachment transfer: local file to Drive, Drive to the + * local cache, cache to a note folder, and the Drive-to-Drive copy into the canonical root. Four + * separate implementations used to exist with four slightly different rules — one enforced the size + * ceiling, one did not, one returned false where the others threw — so a blob accepted on one path + * could be rejected on the next and leave a bundle pointing at bytes no device would serve. + * + *

Verification happens at end of stream, which is reached inside the destination's own read + * loop, so a destination that streams straight to its final location still learns about a mismatch + * before it commits. {@link #verifyEndOfStream()} drains whatever the destination left unread, so a + * consumer that stopped early cannot accept a blob it never finished checking. + */ +final class VerifyingInputStream extends FilterInputStream { + + private final MessageDigest digest = Sha256.newDigest(); + private final String expectedHash; + private final Long expectedSize; + private final long maxBytes; + private long byteCount; + private boolean verified; + private AttachmentIntegrityException integrityFailure; + + VerifyingInputStream( + @NonNull InputStream input, @NonNull String expectedHash, @Nullable Long expectedSize) { + this(input, expectedHash, expectedSize, SyncBundleValidator.MAX_ATTACHMENT_BYTES); + } + + VerifyingInputStream( + @NonNull InputStream input, + @NonNull String expectedHash, + @Nullable Long expectedSize, + long maxBytes) { + super(input); + this.expectedHash = expectedHash; + this.expectedSize = expectedSize; + this.maxBytes = maxBytes; + } + + /** + * Reads {@code input} to its end and verifies it; the caller closes the stream. + * + * @return the blob's size in bytes. + */ + static long verify( + @NonNull InputStream input, @NonNull String expectedHash, @Nullable Long expectedSize) + throws IOException { + VerifyingInputStream verifying = + new VerifyingInputStream(input, expectedHash, expectedSize); + verifying.verifyEndOfStream(); + return verifying.bytesRead(); + } + + /** Bytes seen so far; the blob's size once {@link #verifyEndOfStream()} has passed. */ + long bytesRead() { + return byteCount; + } + + @Override + public int read() throws IOException { + rethrowIntegrityFailure(); + int value = super.read(); + if (value >= 0) { + digest.update((byte) value); + byteCount++; + enforceSizeLimit(); + } else { + verifyEndOfStream(); + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + rethrowIntegrityFailure(); + int read = super.read(buffer, offset, length); + if (read > 0) { + digest.update(buffer, offset, read); + byteCount += read; + enforceSizeLimit(); + } else if (read < 0) { + verifyEndOfStream(); + } + return read; + } + + private void enforceSizeLimit() throws AttachmentIntegrityException { + if (byteCount > maxBytes) { + failIntegrity("Attachment exceeds the sync size limit"); + } + } + + /** + * Checks the digest, draining anything the destination left behind first. + * + *

Idempotent: the destination's read loop reaches end of stream and verifies, and the caller + * verifies again after the destination returns; the second call is a no-op. + */ + void verifyEndOfStream() throws IOException { + if (verified) { + return; + } + rethrowIntegrityFailure(); + drainRemaining(); + String actualHash = Sha256.hex(digest.digest()); + if (!expectedHash.equals(actualHash)) { + failIntegrity("Attachment checksum does not match its declared hash"); + } + if (expectedSize != null && expectedSize.longValue() != byteCount) { + failIntegrity("Attachment size does not match its declared size"); + } + verified = true; + } + + private void failIntegrity(String message) throws AttachmentIntegrityException { + AttachmentIntegrityException failure = new AttachmentIntegrityException(message); + integrityFailure = failure; + throw failure; + } + + private void rethrowIntegrityFailure() throws AttachmentIntegrityException { + if (integrityFailure != null) { + throw integrityFailure; + } + } + + /** Reads through {@code super} so the digest covers bytes the destination skipped. */ + private void drainRemaining() throws IOException { + byte[] scratch = new byte[8192]; + int read; + while ((read = super.read(scratch, 0, scratch.length)) != -1) { + digest.update(scratch, 0, read); + byteCount += read; + enforceSizeLimit(); + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java index ec814498..a34fb3a4 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java @@ -205,6 +205,19 @@ public static File baseDirPath(Context ctx) { return new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR); } + /** + * Where a backup's attachments are unpacked before any note has been inserted. + * + *

Extracting straight into {@link #baseDirPath} overwrote the files of whatever local note + * happened to share a row id with a note in the archive, before the restore had decided whether + * that note would even keep its id. Files wait here until the row exists, then move into the + * folder of the id the note actually received. Under the cache directory so a restore that + * never finishes leaves only something the system may reclaim. + */ + public static File restoreStagingDir(Context ctx) { + return new File(new File(ctx.getCacheDir(), "restore-staging"), ATTACHMENTS_BASE_DIR); + } + /** * Builds the canonical URL for a file this app just wrote into a note's folder. * diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java index 9a23ec65..734f4105 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java @@ -149,8 +149,14 @@ private static int indexOfAny(@NonNull String value, char first, char second) { return -1; } - /** True only for a name that is exactly one ordinary path segment. */ - static boolean isSafeSegment(@Nullable String name) { + /** + * True only for a name that is exactly one ordinary path segment. + * + *

The single rule for attachment names: the sync store and the bundle validator both defer + * to it, so a name one of them accepted can never be one the other refuses to build a path + * from. + */ + public static boolean isSafeSegment(@Nullable String name) { if (name == null) { return false; } diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocks.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocks.java new file mode 100644 index 00000000..8af4cd84 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocks.java @@ -0,0 +1,146 @@ +package com.pasich.mynotes.extendedEditor.attach; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.util.ArrayList; +import java.util.List; + +/** + * The one walk over an Editor.js document that knows where attachment references live. + * + *

A block carries its file either as {@code data.file} (the attaches and image tools) or as + * {@code data.files[]}; the block type is deliberately not consulted, because a reference is a + * reference whichever tool wrote it. Three separate walkers used to exist — one filtered by type, + * one ignored {@code files[]}, one did both — so the same note could relocate correctly after a ZIP + * restore and render broken links after a Drive restore. + * + *

Deliberately free of {@code android.*}: sync hashes the rewritten document, so the rewrite has + * to be provably deterministic under ordinary JVM tests. + */ +public final class EditorAttachmentBlocks { + + /** Maps one stored URL to its replacement, or returns {@code null} to leave it alone. */ + public interface UrlMapper { + @Nullable + String map(@NonNull String url); + } + + private EditorAttachmentBlocks() {} + + /** + * Rewrites every file URL the mapper has a replacement for. + * + * @return the rewritten document, or {@code valueJson} itself when nothing changed or the + * document could not be read. Returning the original string verbatim, rather than a + * re-serialization of it, keeps an untouched note byte-identical on every device. + */ + @Nullable + public static String rewriteUrls(@Nullable String valueJson, @NonNull UrlMapper mapper) { + if (valueJson == null || valueJson.trim().isEmpty()) { + return valueJson; + } + JsonArray blocks; + try { + blocks = JsonParser.parseString(valueJson).getAsJsonArray(); + } catch (RuntimeException unreadable) { + return valueJson; + } + boolean changed = false; + for (JsonObject file : fileObjects(blocks)) { + String url = file.get("url").getAsString(); + String replacement = mapper.map(url); + if (replacement != null && !replacement.equals(url)) { + file.addProperty("url", replacement); + changed = true; + } + } + // JsonElement.toString(), not Gson.toJson(): the latter HTML-escapes text, and both + // directions of a sync must serialize identically or the hashes disagree. + return changed ? blocks.toString() : valueJson; + } + + /** Every file URL in document order. */ + @NonNull + public static List fileUrls(@Nullable String valueJson) { + List urls = new ArrayList<>(); + if (valueJson == null || valueJson.trim().isEmpty()) { + return urls; + } + try { + for (JsonObject file : + fileObjects(JsonParser.parseString(valueJson).getAsJsonArray())) { + urls.add(file.get("url").getAsString()); + } + } catch (RuntimeException unreadable) { + return new ArrayList<>(); + } + return urls; + } + + /** The file object of the block with {@code blockId}, or {@code null}. */ + @Nullable + public static JsonObject findFile(@Nullable String valueJson, @Nullable String blockId) { + if (valueJson == null || valueJson.isEmpty() || blockId == null || blockId.isEmpty()) { + return null; + } + try { + for (JsonElement element : JsonParser.parseString(valueJson).getAsJsonArray()) { + if (!element.isJsonObject()) continue; + JsonObject block = element.getAsJsonObject(); + if (!block.has("id") || !blockId.equals(block.get("id").getAsString())) continue; + List files = fileObjectsOf(block); + return files.isEmpty() ? null : files.get(0); + } + } catch (RuntimeException unreadable) { + return null; + } + return null; + } + + @NonNull + private static List fileObjects(@NonNull JsonArray blocks) { + List files = new ArrayList<>(); + for (JsonElement element : blocks) { + if (element.isJsonObject()) { + files.addAll(fileObjectsOf(element.getAsJsonObject())); + } + } + return files; + } + + /** {@code data.file} first, then {@code data.files[]}, each only when it carries a URL. */ + @NonNull + private static List fileObjectsOf(@NonNull JsonObject block) { + List files = new ArrayList<>(); + JsonElement dataElement = block.get("data"); + if (dataElement == null || !dataElement.isJsonObject()) { + return files; + } + JsonObject data = dataElement.getAsJsonObject(); + JsonElement file = data.get("file"); + if (hasUrl(file)) { + files.add(file.getAsJsonObject()); + } + JsonElement list = data.get("files"); + if (list != null && list.isJsonArray()) { + for (JsonElement candidate : list.getAsJsonArray()) { + if (hasUrl(candidate)) { + files.add(candidate.getAsJsonObject()); + } + } + } + return files; + } + + private static boolean hasUrl(@Nullable JsonElement element) { + if (element == null || !element.isJsonObject()) { + return false; + } + JsonElement url = element.getAsJsonObject().get("url"); + return url != null && url.isJsonPrimitive() && url.getAsJsonPrimitive().isString(); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java index 463270b0..8e5eb6af 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java @@ -40,8 +40,15 @@ public static final class Result { private NoteAttachmentRelocator() {} + /** Turns one stored reference into its new URL, or {@code null} to leave it alone. */ + private interface ReferenceMover { + @Nullable + String move(@NonNull String url); + } + /** - * Repoints every reference from {@code previousNoteId} to {@code newNoteId}. + * Repoints every reference from {@code previousNoteId} to {@code newNoteId}, copying the files + * out of the old note's live folder. * * @param attachmentsRoot the app-private {@code attachments} directory. * @param attachmentsJson the note's attachments column. @@ -57,6 +64,72 @@ public static Result relocate( if (previousNoteId <= 0 || newNoteId <= 0 || previousNoteId == newNoteId) { return new Result(attachmentsJson, valueJson, false); } + return rewrite( + attachmentsJson, + valueJson, + url -> moveReference(attachmentsRoot, previousNoteId, newNoteId, url)); + } + + /** + * Moves a restored note's staged attachments into the folder of the id it actually received. + * + *

A backup's files are unpacked into a staging directory rather than the live folders, so + * this runs for every restored note, whether or not its id changed. A file already present + * under the same name in the destination is reused when it holds the same bytes and left alone + * otherwise — the staged copy then gets a fresh name — so a restore can never overwrite a file + * another note still shows. A reference with nothing staged falls back to the live-folder + * relocation above, which is what a note restored by an older release relied on. + * + * @param stagingRoot the staging directory that stands in for {@code attachments}. + * @param attachmentsRoot the app-private {@code attachments} directory. + */ + @NonNull + public static Result adoptStaged( + @NonNull File stagingRoot, + @NonNull File attachmentsRoot, + int previousNoteId, + int newNoteId, + @Nullable String attachmentsJson, + @Nullable String valueJson) { + if (previousNoteId <= 0 || newNoteId <= 0) { + return new Result(attachmentsJson, valueJson, false); + } + File stagedFolder = new File(stagingRoot, "note_" + previousNoteId); + Result result = + rewrite( + attachmentsJson, + valueJson, + url -> { + String adopted = + adoptReference( + stagedFolder, + attachmentsRoot, + previousNoteId, + newNoteId, + url); + if (adopted != null || previousNoteId == newNoteId) { + return adopted; + } + return moveReference(attachmentsRoot, previousNoteId, newNoteId, url); + }); + // Whatever the note did not reference was never going to be shown; it need not linger. + deleteRecursively(stagedFolder); + return result; + } + + /** Rewrites the column first and, only when it changed, the blocks that mirror it. */ + @NonNull + private static Result rewrite( + @Nullable String attachmentsJson, @Nullable String valueJson, ReferenceMover decide) { + // One answer per URL: the column and the blocks name the same files, and a mover that + // picked a fresh name on a collision gave each of them a different copy — the cleaner + // then deleted the one the column did not know and the block rendered a missing file. + java.util.Map> decided = new java.util.HashMap<>(); + ReferenceMover mover = + url -> + decided.computeIfAbsent( + url, key -> java.util.Optional.ofNullable(decide.move(key))) + .orElse(null); String movedAttachments = attachmentsJson; boolean changed = false; @@ -67,13 +140,10 @@ public static Result relocate( if (!element.isJsonObject()) continue; JsonObject entry = element.getAsJsonObject(); if (!entry.has("url") || !entry.get("url").isJsonPrimitive()) continue; - String rewritten = - moveReference( - attachmentsRoot, - previousNoteId, - newNoteId, - entry.get("url").getAsString()); - if (rewritten != null) { + String url = entry.get("url").getAsString(); + String rewritten = mover.move(url); + // A staged file adopted under the very same URL changes nothing worth storing. + if (rewritten != null && !rewritten.equals(url)) { entry.addProperty("url", rewritten); changed = true; } @@ -88,39 +158,112 @@ public static Result relocate( } String movedValueJson = valueJson; - if (changed && valueJson != null && !valueJson.trim().isEmpty()) { - try { - JsonArray blocks = JsonParser.parseString(valueJson).getAsJsonArray(); - boolean rewroteBlock = false; - for (JsonElement element : blocks) { - if (!element.isJsonObject()) continue; - JsonObject data = element.getAsJsonObject().getAsJsonObject("data"); - if (data == null) continue; - JsonObject file = data.getAsJsonObject("file"); - if (file == null || !file.has("url") || !file.get("url").isJsonPrimitive()) { - continue; - } - String rewritten = - moveReference( - attachmentsRoot, - previousNoteId, - newNoteId, - file.get("url").getAsString()); - if (rewritten != null) { - file.addProperty("url", rewritten); - rewroteBlock = true; - } + if (changed) { + // The blocks are what the editor renders; the shared walker leaves an unreadable + // document untouched rather than risk corrupting the note's content. + movedValueJson = EditorAttachmentBlocks.rewriteUrls(valueJson, mover::move); + } + + return new Result(movedAttachments, movedValueJson, changed); + } + + /** + * Copies one staged file into the new note's folder and returns its new URL. + * + * @return the rewritten URL, or {@code null} when the reference does not belong to the old note + * or nothing was staged for it. + */ + @Nullable + private static String adoptReference( + @NonNull File stagedFolder, + @NonNull File attachmentsRoot, + int previousNoteId, + int newNoteId, + @NonNull String url) { + AttachmentUrl parsed = AttachmentUrl.parse(url); + if (parsed == null || !parsed.getNoteFolder().equals("note_" + previousNoteId)) { + return null; + } + File source = new File(stagedFolder, parsed.getFileName()); + if (!source.isFile()) { + return null; + } + File targetFolder = new File(attachmentsRoot, "note_" + newNoteId); + try { + if (!targetFolder.isDirectory() && !targetFolder.mkdirs()) { + return null; + } + File target = new File(targetFolder, parsed.getFileName()); + if (target.exists() && !sameContent(source, target)) { + target = new File(targetFolder, uniqueName(parsed.getFileName())); + } + if (!target.exists()) { + Files.copy(source.toPath(), target.toPath()); + } + return AttachmentUrl.canonical(newNoteId, target.getName()); + } catch (IOException | SecurityException | IllegalArgumentException failure) { + return null; + } + } + + /** Streams both files; a restore has no attachment size ceiling to load them whole under. */ + private static boolean sameContent(@NonNull File first, @NonNull File second) + throws IOException { + if (first.length() != second.length()) { + return false; + } + try (java.io.InputStream left = + new java.io.BufferedInputStream(new java.io.FileInputStream(first)); + java.io.InputStream right = + new java.io.BufferedInputStream(new java.io.FileInputStream(second))) { + byte[] leftBuffer = new byte[8192]; + byte[] rightBuffer = new byte[8192]; + while (true) { + int leftRead = readFully(left, leftBuffer); + int rightRead = readFully(right, rightBuffer); + if (leftRead != rightRead + || !java.util.Arrays.equals( + java.util.Arrays.copyOf(leftBuffer, leftRead), + java.util.Arrays.copyOf(rightBuffer, rightRead))) { + return false; } - if (rewroteBlock) { - movedValueJson = blocks.toString(); + if (leftRead < leftBuffer.length) { + return true; } - } catch (RuntimeException unreadable) { - // Keep the blocks untouched rather than risk corrupting the note's content. - movedValueJson = valueJson; } } + } - return new Result(movedAttachments, movedValueJson, changed); + private static int readFully(@NonNull java.io.InputStream input, @NonNull byte[] buffer) + throws IOException { + int filled = 0; + while (filled < buffer.length) { + int read = input.read(buffer, filled, buffer.length - filled); + if (read == -1) break; + filled += read; + } + return filled; + } + + /** {@code name.ext} becomes {@code name-.ext}, still a safe single segment. */ + @NonNull + private static String uniqueName(@NonNull String fileName) { + String suffix = java.util.UUID.randomUUID().toString(); + int dot = fileName.lastIndexOf('.'); + if (dot <= 0) { + return fileName + "-" + suffix; + } + return fileName.substring(0, dot) + "-" + suffix + fileName.substring(dot); + } + + private static void deleteRecursively(@NonNull File file) { + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + file.delete(); } /** diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/utils/EditorJsonUtils.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/utils/EditorJsonUtils.java index 34007226..4f5d604f 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/utils/EditorJsonUtils.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/utils/EditorJsonUtils.java @@ -17,39 +17,16 @@ public class EditorJsonUtils { * data.file - data.files[] */ public static EditorAttachment findAttachmentByBlockId(Note note, String blockId) { + if (note == null) return null; try { - if (note == null || blockId == null || blockId.isEmpty()) return null; - - String json = note.getValueJson(); - if (json == null || json.isEmpty()) return null; - - JSONArray blocks = new JSONArray(json); - - for (int i = 0; i < blocks.length(); i++) { - JSONObject block = blocks.optJSONObject(i); - if (block == null) continue; - - if (!blockId.equals(block.optString("id"))) continue; - - JSONObject data = block.optJSONObject("data"); - if (data == null) return null; - - // data.file - JSONObject fileObj = data.optJSONObject("file"); - if (fileObj != null) { - return EditorAttachment.fromJsonObject(fileObj); - } - - // data.files[] - JSONArray filesArr = data.optJSONArray("files"); - if (filesArr != null && filesArr.length() > 0) { - JSONObject f = filesArr.optJSONObject(0); - if (f != null) return EditorAttachment.fromJsonObject(f); - } - - return null; - } - + // The same walk sync and restore use, so a block one of them can see is a block + // the editor can open. + com.google.gson.JsonObject file = + com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks.findFile( + note.getValueJson(), blockId); + return file == null + ? null + : EditorAttachment.fromJsonObject(new JSONObject(file.toString())); } catch (Exception e) { Log.e(TAG, "findAttachmentByBlockId() failed", e); } diff --git a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java index bb5f5b5a..a09ebbdd 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java +++ b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java @@ -145,7 +145,9 @@ public void resolveConflict( @Override public SyncState sync(@NonNull String accessToken) { return new SyncService(store) - .sync(new GoogleDriveSyncBackend(accessToken)); + .sync( + new GoogleDriveSyncBackend(accessToken), + preferenceHelper::isSyncEnabled); } @Override diff --git a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java index 39ed2c8c..225879d6 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java +++ b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java @@ -111,25 +111,7 @@ public class BackupActivity extends BaseActivity if (result.getResultCode() == Activity.RESULT_OK) { if (result.getData() != null && result.getData().getData() != null) { Uri uri = result.getData().getData(); - - BackupFileValidator.isValidBackupFile( - this, - uri, - new BackupFileValidator.BackupValidatorCallback() { - @Override - public void onValid(String fileName) { - presenter.readFileBackupLocal(uri); - } - - @Override - public void onInvalid(String errorMessage) { - onInfoSnack( - errorMessage, - null, - SnackBarInfo.Error, - Snackbar.LENGTH_LONG); - } - }); + validatePickedBackup(uri); } } }); @@ -180,6 +162,16 @@ public void onInvalid(String errorMessage) { private SyncCoordinatorFactory.Result syncSetup; @Nullable private AccountSyncFragment accountTab; private boolean syncRunning; + + /** + * Set once a received settings version has scheduled this screen's recreation. + * + *

The controls stay usable for the moment the sync result is shown, and anything started in + * that moment — a sync, a sign-out, a conflict dialog — was torn down with the old instance, + * its callback dropped and the new instance showing "ready" while the sync still ran. + */ + private boolean recreatePending; + private SyncCoordinator syncCoordinator; private final ExecutorService syncExecutor = Executors.newSingleThreadExecutor(); @Inject FirebaseGoogleAuth firebaseGoogleAuth; @@ -226,6 +218,46 @@ public void handleOnBackPressed() { }); } + /** + * Decides off the main thread whether the picked document is a backup. + * + *

When the name does not settle it the document is opened and inspected through the document + * provider, which on a wrong pick can mean inflating a large archive; on the main thread that + * froze the screen. The verdict is delivered back to the main thread. + */ + private void validatePickedBackup(@NonNull Uri uri) { + runInBackground( + () -> + BackupFileValidator.isValidBackupFile( + this, + uri, + new BackupFileValidator.BackupValidatorCallback() { + @Override + public void onValid(String fileName) { + runOnUiThread( + () -> { + if (!isFinishing() && !isDestroyed()) { + presenter.readFileBackupLocal(uri); + } + }); + } + + @Override + public void onInvalid(String errorMessage) { + runOnUiThread( + () -> { + if (!isFinishing() && !isDestroyed()) { + onInfoSnack( + errorMessage, + null, + SnackBarInfo.Error, + Snackbar.LENGTH_LONG); + } + }); + } + })); + } + /** The account tab draws itself from the state the next updateSyncUi() pushes. */ private void updateGoogleSignInButton() { updateSyncUi(); @@ -297,7 +329,7 @@ private void renderSyncUi( } private void startSync() { - if (syncCoordinator == null) return; + if (syncCoordinator == null || recreatePending) return; if (!syncCoordinator.getProfile().isSignedIn()) { onInfoSnack( R.string.google_sign_in_failed, null, SnackBarInfo.Error, Snackbar.LENGTH_LONG); @@ -420,7 +452,7 @@ private static String formatBytes(long bytes) { } private void onGoogleSignInClicked() { - if (syncCoordinator == null) return; + if (syncCoordinator == null || recreatePending) return; if (syncCoordinator.getProfile().isSignedIn()) { syncCoordinator.disconnect( new SyncCoordinator.Callback() { @@ -501,16 +533,48 @@ private void applyReceivedPreferences() { null, SnackBarInfo.Success, Snackbar.LENGTH_LONG); + // Nothing may start while the rebuild is pending; see recreatePending. + recreatePending = true; + if (accountTab != null) accountTab.setSyncing(true); binding.getRoot() .postDelayed( () -> { - if (!isFinishing() && !isDestroyed()) { + if (isFinishing() || isDestroyed() || syncRunning) { + return; + } + // The stored mode is pushed to AppCompat only here, once the sync + // result is in hand; pushing it from inside the apply recreated this + // screen mid-sync and dropped the result. A changed mode recreates + // the screen by itself; otherwise the rebuild is done here. + int before = + androidx.appcompat.app.AppCompatDelegate.getDefaultNightMode(); + themePreferencesCache.applyCurrentThemeMode(); + if (androidx.appcompat.app.AppCompatDelegate.getDefaultNightMode() + == before) { recreate(); } }, 1500L); } + /** + * Applies a restored theme once the restore has finished. + * + *

Applying it while the inserts were running recreated this screen and disposed them. + * Delayed so the result stays readable for a moment before the screen rebuilds, and only a mode + * that actually changed causes a rebuild at all. + */ + private void applyRestoredTheme() { + binding.getRoot() + .postDelayed( + () -> { + if (!isFinishing() && !isDestroyed()) { + themePreferencesCache.applyCurrentThemeMode(); + } + }, + 1500L); + } + private void finishSyncError(Exception error) { if (isFinishing() || isDestroyed()) { return; @@ -554,6 +618,7 @@ private void onBackgroundSyncToggled(boolean enabled) { } private void showNextConflictDialog() { + if (recreatePending) return; runInBackground( () -> { List unresolved = @@ -1035,12 +1100,11 @@ public void restoreFinish(int infoCode) { progressDialog.dismiss(); } switch (infoCode) { - case CloudErrors.OKAY_RESTORE -> - onInfoSnack( - R.string.restoreDataOkay, - null, - SnackBarInfo.Success, - Snackbar.LENGTH_LONG); + case CloudErrors.OKAY_RESTORE -> { + onInfoSnack( + R.string.restoreDataOkay, null, SnackBarInfo.Success, Snackbar.LENGTH_LONG); + applyRestoredTheme(); + } case CloudErrors.BACKUP_DESTROY -> onInfoSnack( R.string.restoreDataFall, diff --git a/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java b/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java index 74e8d6bd..72a1606b 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java +++ b/app/src/main/java/com/pasich/mynotes/utils/backup/local/BackupFileValidator.java @@ -5,11 +5,24 @@ import android.net.Uri; import android.provider.OpenableColumns; import android.util.Log; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import com.pasich.mynotes.R; +import com.pasich.mynotes.utils.constants.Backup; +import java.io.IOException; +import java.io.InputStream; +import java.util.Locale; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; /** - * Small helper class for validating selected backup files. Ensures that the file exists and has a - * supported extension. + * Decides whether a picked document is a backup this app can restore. + * + *

The decision used to rest on the display name's extension alone. Android's document picker + * appends " (2)" when a second backup is saved into a folder that already holds one, so the app + * refused its own freshly written file. The content decides now: an archive holding the backup JSON + * is a backup whatever it is called; the extension is only kept as the fast path for the legacy + * non-archive formats. */ public class BackupFileValidator { @@ -20,11 +33,29 @@ public class BackupFileValidator { private static final String EXT_MNBK = ".mnbkn"; /** - * Validate backup file based on its filename and extension. + * How much of a picked archive is inflated looking for the backup JSON before giving up. + * + *

This app writes the JSON as the first entry, so the intended case costs a few kilobytes. A + * wrong pick that happens to be a large ZIP container — an APK, a document — would otherwise be + * inflated entry by entry to its end. + */ + static final long MAX_INSPECTED_BYTES = 8L * 1024L * 1024L; + + /** Opens the picked document, so the content can be inspected. */ + public interface ContentOpener { + @Nullable + InputStream open() throws IOException; + } + + /** + * Validate a picked backup file. + * + *

Reads the document when its name does not settle the question, so call it off the main + * thread; the callback is invoked on the calling thread. * *

- If user cancels selection → return silently (no errors shown). - If filename cannot be - * determined → callback.onInvalid(...) - If extension unsupported → callback.onInvalid(...) - - * If valid → callback.onValid(filename) + * determined → callback.onInvalid(...) - If neither name nor content is a backup → + * callback.onInvalid(...) - If valid → callback.onValid(filename) */ public static void isValidBackupFile(Context ctx, Uri uri, BackupValidatorCallback callback) { @@ -37,12 +68,7 @@ public static void isValidBackupFile(Context ctx, Uri uri, BackupValidatorCallba return; } - String lower = name.toLowerCase(); - - boolean ok = - lower.endsWith(EXT_JSON) || lower.endsWith(EXT_ZIP) || lower.endsWith(EXT_MNBK); - - if (!ok) { + if (!isAcceptable(name, () -> ctx.getContentResolver().openInputStream(uri))) { callback.onInvalid(ctx.getString(R.string.file_wrong_format)); return; } @@ -50,6 +76,61 @@ public static void isValidBackupFile(Context ctx, Uri uri, BackupValidatorCallba callback.onValid(name); } + /** + * The decision itself, free of {@code android.*} so every branch runs under a plain JVM test. + * + * @param name the document's display name, which the picker may have decorated. + * @param content the document's bytes, consulted when the name alone does not settle it. + */ + static boolean isAcceptable(@NonNull String name, @NonNull ContentOpener content) { + String lower = name.toLowerCase(Locale.ROOT); + if (lower.endsWith(EXT_JSON) || lower.endsWith(EXT_ZIP) || lower.endsWith(EXT_MNBK)) { + return true; + } + try (InputStream input = content.open()) { + return input != null && isBackupArchive(input); + } catch (IOException | RuntimeException unreadable) { + Log.w(TAG, "Could not inspect the picked document", unreadable); + return false; + } + } + + /** True when the bytes are a ZIP archive holding the backup JSON entry. */ + static boolean isBackupArchive(@NonNull InputStream input) throws IOException { + return isBackupArchive(input, MAX_INSPECTED_BYTES); + } + + /** + * @param maxInflatedBytes how much entry data may be inflated before the archive is judged not + * to be a backup; bounds the cost of a wrong pick. + */ + static boolean isBackupArchive(@NonNull InputStream input, long maxInflatedBytes) + throws IOException { + try (ZipInputStream zip = new ZipInputStream(input)) { + byte[] scratch = new byte[8192]; + long inflated = 0L; + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + if (Backup.FILE_NAME_BACKUP.equals(entry.getName())) { + return true; + } + // Skipping an entry inflates it; count what that costs and stop rather than + // read a large container to its end. + int read; + while ((read = zip.read(scratch)) != -1) { + inflated += read; + if (inflated > maxInflatedBytes) { + return false; + } + } + } + } catch (RuntimeException notAnArchive) { + // A ZipInputStream over arbitrary bytes throws on a malformed entry header. + return false; + } + return false; + } + /** * Extract filename from content Uri via OpenableColumns. * diff --git a/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java b/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java index 3bc05517..2005ea08 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java +++ b/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java @@ -7,7 +7,11 @@ import android.content.Context; import android.net.Uri; import android.util.Log; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import com.google.gson.Gson; +import com.pasich.mynotes.extendedEditor.attach.AttachmentStorage; +import com.pasich.mynotes.extendedEditor.attach.AttachmentUrl; import com.pasich.mynotes.utils.backup.models.JsonBackup; import java.io.ByteArrayOutputStream; import java.io.File; @@ -16,6 +20,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -25,6 +30,19 @@ public class ZipBackupHelper { private static final String TAG = "ZipBackupHelper"; + /** + * The most an archive may unpack to, all entries together. + * + *

An archive is untrusted input, and ZIP compresses a run of zeros a thousandfold; without a + * ceiling a small file could fill the device. Generous for a real library, far below what a + * bomb wants. + */ + static final long MAX_TOTAL_UNCOMPRESSED_BYTES = 4L * 1024L * 1024L * 1024L; + + /** The only entry shape an archive may place a file under: one note folder, one file name. */ + private static final Pattern ATTACHMENT_ENTRY = + Pattern.compile(Pattern.quote(ATTACHMENTS_BASE_DIR) + "/(note_[1-9][0-9]*)/([^/]+)"); + /** Detect ZIP by magic header "PK" */ public static boolean isZip(byte[] data) { return data.length > 2 && data[0] == 0x50 && data[1] == 0x4B; @@ -76,79 +94,134 @@ public static File writeZipBackup(Context ctx, JsonBackup backup) throws Excepti return zipFile; } - /** Parse ZIP backup */ + /** + * Parses a ZIP backup, unpacking its attachments into the restore staging directory. + * + *

Nothing here touches a note's live attachment folder. The archive is untrusted input and + * the restore has not yet decided which row id each note will get, so files are staged and + * adopted per note once it is inserted; see {@code NoteAttachmentRelocator.adoptStaged}. + */ public static JsonBackup readZipBackup(Context ctx, Uri uri) throws Exception { - - JsonBackup backup = null; - + File staging = AttachmentStorage.restoreStagingDir(ctx); + // Whatever an earlier restore left behind belongs to that restore, not this one. + deleteRecursively(staging); try (InputStream is = ctx.getContentResolver().openInputStream(uri); ZipInputStream zis = new ZipInputStream(is)) { + JsonBackup backup = readZipBackup(zis, staging); + if (backup.isError()) { + deleteRecursively(staging); + } + return backup; + } + } - ZipEntry entry; - - while ((entry = zis.getNextEntry()) != null) { - - // ================== backup.json ================== - if (entry.getName().equals(FILE_NAME_BACKUP)) { - - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - byte[] tmp = new byte[4096]; - int n; + /** + * Filesystem-only core: reads the archive into {@code stagingRoot}, which stands in for the + * {@code attachments} directory an archive entry names. + */ + @NonNull + static JsonBackup readZipBackup(@NonNull ZipInputStream zis, @NonNull File stagingRoot) + throws IOException { + return readZipBackup(zis, stagingRoot, MAX_TOTAL_UNCOMPRESSED_BYTES); + } - while ((n = zis.read(tmp)) != -1) { - buffer.write(tmp, 0, n); + @NonNull + static JsonBackup readZipBackup( + @NonNull ZipInputStream zis, @NonNull File stagingRoot, long maxTotalBytes) + throws IOException { + JsonBackup backup = null; + long[] total = {0L}; + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entry.getName().equals(FILE_NAME_BACKUP)) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] tmp = new byte[4096]; + int n; + while ((n = zis.read(tmp)) != -1) { + if (!account(total, n, maxTotalBytes)) { + return new JsonBackup().error(); } - - String json = buffer.toString("UTF-8"); - backup = new Gson().fromJson(json, JsonBackup.class); + buffer.write(tmp, 0, n); } - - // ================== attachments/... ================== - else if (entry.getName().startsWith(ATTACHMENTS_BASE_DIR)) { - - // A backup file is untrusted input: it can be edited, or come from - // somewhere else entirely. "attachments/../../databases/notes" also starts - // with the prefix above, so without resolving the path first an archive - // could write anywhere the app can write. - File out = safeAttachmentTarget(ctx, entry.getName()); - if (out == null || entry.isDirectory()) { - Log.w(TAG, "Skipping a backup entry outside the attachment directory"); - zis.closeEntry(); - continue; - } - File parent = out.getParentFile(); - if (parent != null && !parent.exists() && !parent.mkdirs()) { - Log.w(TAG, "Could not create the attachment directory for a backup entry"); - zis.closeEntry(); - continue; - } - - try (FileOutputStream fos = new FileOutputStream(out)) { - byte[] data = new byte[4096]; - int n; - - while ((n = zis.read(data)) != -1) { - fos.write(data, 0, n); + try { + backup = new Gson().fromJson(buffer.toString("UTF-8"), JsonBackup.class); + } catch (RuntimeException malformed) { + Log.w(TAG, "The backup JSON could not be read", malformed); + backup = null; + } + } else if (entry.getName().startsWith(ATTACHMENTS_BASE_DIR)) { + // A backup file is untrusted input: it can be edited, or come from somewhere else + // entirely. Only "attachments/note_/" is a place a restored note can + // reference; anything else — a traversal, a nested path, a folder the cleaner + // would never look in — is skipped rather than written somewhere and forgotten. + File out = stagedAttachmentTarget(stagingRoot, entry.getName()); + if (out == null || entry.isDirectory()) { + Log.w(TAG, "Skipping a backup entry outside the attachment directory"); + zis.closeEntry(); + continue; + } + File parent = out.getParentFile(); + if (parent != null && !parent.exists() && !parent.mkdirs()) { + Log.w(TAG, "Could not create the attachment directory for a backup entry"); + zis.closeEntry(); + continue; + } + try (FileOutputStream fos = new FileOutputStream(out)) { + byte[] data = new byte[4096]; + int n; + while ((n = zis.read(data)) != -1) { + if (!account(total, n, maxTotalBytes)) { + Log.w(TAG, "The backup unpacks to more than the restore allows"); + return new JsonBackup().error(); } + fos.write(data, 0, n); } } - - zis.closeEntry(); } + zis.closeEntry(); } - return backup != null ? backup : new JsonBackup().error(); } + /** Adds to the running total; false once the ceiling is passed. */ + private static boolean account(long[] total, int read, long maxTotalBytes) { + total[0] += read; + return total[0] <= maxTotalBytes; + } + /** - * Resolves one archive entry inside the attachment directory, or {@code null} if it escapes. + * Resolves one archive entry inside the staging directory, or {@code null} if it does not name + * exactly one file in one note folder. * * @param entryName the raw name from the archive, which is attacker-controlled. */ - private static File safeAttachmentTarget(Context ctx, String entryName) throws IOException { - File root = new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR).getCanonicalFile(); - File resolved = new File(ctx.getFilesDir(), entryName).getCanonicalFile(); + @Nullable + static File stagedAttachmentTarget(@NonNull File stagingRoot, @NonNull String entryName) + throws IOException { + java.util.regex.Matcher matcher = ATTACHMENT_ENTRY.matcher(entryName); + if (!matcher.matches() || !AttachmentUrl.isSafeSegment(matcher.group(2))) { + return null; + } + File root = stagingRoot.getCanonicalFile(); + File resolved = + new File(new File(root, matcher.group(1)), matcher.group(2)).getCanonicalFile(); + // The pattern already forbids traversal; this is the second of two independent guards. String prefix = root.getPath() + File.separator; return resolved.getPath().startsWith(prefix) ? resolved : null; } + + private static void deleteRecursively(@Nullable File file) { + if (file == null || !file.exists()) { + return; + } + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + if (!file.delete()) { + Log.w(TAG, "Could not remove " + file.getName()); + } + } } diff --git a/app/src/test/java/com/pasich/mynotes/data/preferences/AppPreferencesHelperTest.java b/app/src/test/java/com/pasich/mynotes/data/preferences/AppPreferencesHelperTest.java new file mode 100644 index 00000000..baeae2e2 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/preferences/AppPreferencesHelperTest.java @@ -0,0 +1,55 @@ +package com.pasich.mynotes.data.preferences; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.pasich.mynotes.cache.AppPreferencesCache; +import com.pasich.mynotes.cache.ThemePreferencesCache; +import com.pasich.mynotes.utils.backup.models.PreferencesBackup; +import org.junit.Before; +import org.junit.Test; + +/** + * When a stored theme mode is pushed to AppCompat. + * + *

Pushing it recreates every started activity. That is wanted for a theme arriving with a sync + * and fatal during a backup restore, whose inserts are still running on the Backup screen: the + * recreation disposed them and left the database half restored with no message. + */ +public class AppPreferencesHelperTest { + + private ThemePreferencesCache themeCache; + private AppPreferencesHelper helper; + + @Before + public void setUp() { + AppPreferencesCache appCache = mock(AppPreferencesCache.class); + themeCache = mock(ThemePreferencesCache.class); + SafePreferences prefs = mock(SafePreferences.class); + when(prefs.commitAll(any())).thenReturn(true); + helper = new AppPreferencesHelper(appCache, themeCache, prefs, Runnable::run); + } + + @Test + public void aRestoreStoresTheThemeWithoutApplyingItMidWay() { + helper.setListPreferences(backup()); + + verify(themeCache).refresh(); + verify(themeCache, never()).applyCurrentThemeMode(); + } + + @Test + public void aSyncAppliesAReceivedThemeAtOnce() { + helper.commitListPreferences(backup()); + + verify(themeCache).applyCurrentThemeMode(); + } + + private static PreferencesBackup backup() { + return new PreferencesBackup( + 1, "sans", "date", 14, 11, false, 2, false, false, false, 1.0f); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentHashCacheTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentHashCacheTest.java new file mode 100644 index 00000000..ebe1a4d6 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentHashCacheTest.java @@ -0,0 +1,104 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Every snapshot build used to re-hash every attachment in the library; the cache is what makes an + * idle sync cheap. Correctness must never depend on it, so the tests also pin when it forgets. + */ +public class AttachmentHashCacheTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private final AtomicInteger hashed = new AtomicInteger(); + private final AttachmentHashCache.Hasher counting = + file -> { + hashed.incrementAndGet(); + return Sha256.of(file); + }; + + @Test + public void hashesAFileOnceWhileItIsUnchanged() throws Exception { + File file = write("photo.png", "bytes"); + AttachmentHashCache cache = new AttachmentHashCache(storage()); + + String first = cache.sha256(file, counting); + String second = cache.sha256(file, counting); + + assertThat(second).isEqualTo(first); + assertThat(first).isEqualTo(Sha256.of(file)); + assertThat(hashed.get()).isEqualTo(1); + } + + @Test + public void rehashesAFileWhoseContentChanged() throws Exception { + File file = write("photo.png", "bytes"); + AttachmentHashCache cache = new AttachmentHashCache(storage()); + cache.sha256(file, counting); + + Files.write(file.toPath(), "different length".getBytes(StandardCharsets.UTF_8)); + String rehashed = cache.sha256(file, counting); + + assertThat(rehashed).isEqualTo(Sha256.of(file)); + assertThat(hashed.get()).isEqualTo(2); + } + + @Test + public void survivesTheStoreInstanceThatBuiltIt() throws Exception { + // A store lives for one sync; without persistence the first build of every sync — and + // the estimate before the first one — hashed the whole library again. + File file = write("photo.png", "bytes"); + AttachmentHashCache first = new AttachmentHashCache(storage()); + first.sha256(file, counting); + first.flush(); + + AttachmentHashCache second = new AttachmentHashCache(storage()); + String hash = second.sha256(file, counting); + + assertThat(hash).isEqualTo(Sha256.of(file)); + assertThat(hashed.get()).isEqualTo(1); + } + + @Test + public void anUnreadableCacheFileCostsARehashNotAFailure() throws Exception { + File file = write("photo.png", "bytes"); + assertThat(storage().getParentFile().mkdirs()).isTrue(); + Files.write(storage().toPath(), "{not json".getBytes(StandardCharsets.UTF_8)); + AttachmentHashCache cache = new AttachmentHashCache(storage()); + + assertThat(cache.sha256(file, counting)).isEqualTo(Sha256.of(file)); + assertThat(hashed.get()).isEqualTo(1); + } + + @Test + public void clearForgetsEverything() throws Exception { + File file = write("photo.png", "bytes"); + AttachmentHashCache cache = new AttachmentHashCache(storage()); + cache.sha256(file, counting); + cache.flush(); + + cache.clear(); + + assertThat(storage().exists()).isFalse(); + cache.sha256(file, counting); + assertThat(hashed.get()).isEqualTo(2); + } + + private File storage() { + return new File(temporaryFolder.getRoot(), "sync-attachments/hash-cache.json"); + } + + private File write(String name, String content) throws Exception { + File file = new File(temporaryFolder.getRoot(), name); + Files.write(file.toPath(), content.getBytes(StandardCharsets.UTF_8)); + return file; + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentWireUrlTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentWireUrlTest.java new file mode 100644 index 00000000..7184b559 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/AttachmentWireUrlTest.java @@ -0,0 +1,31 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; + +public class AttachmentWireUrlTest { + + private static final String ID = "7d444840-9dc0-11d1-b245-5ffdce74fad2"; + + @Test + public void roundTripsALogicalId() { + String wire = AttachmentWireUrl.forLogicalId(ID); + + assertThat(AttachmentWireUrl.logicalIdOf(wire)).isEqualTo(ID); + } + + @Test + public void isNotMistakenForALocalReference() { + // A wire reference must never parse as a note-folder path, or a leaked one would be + // taken for a file this device owns. + assertThat( + com.pasich.mynotes.extendedEditor.attach.AttachmentUrl.parse( + AttachmentWireUrl.forLogicalId(ID))) + .isNull(); + assertThat(AttachmentWireUrl.logicalIdOf("editorjs://attachments/note_5/photo.png")) + .isNull(); + assertThat(AttachmentWireUrl.logicalIdOf("mynotes-sync://attachment/not-a-uuid")).isNull(); + assertThat(AttachmentWireUrl.logicalIdOf(null)).isNull(); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index 2d4b132f..b41155d2 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -538,10 +538,16 @@ public void publishingWithoutAPrecedingReadIsRefused() throws Exception { GoogleDriveSyncBackend backend = backend(); try { - backend.writeSnapshot(snapshot(NOTE_ID, null)); + // A context that no read of this backend produced: the token is empty. + backend.publish( + new SyncPublication( + snapshot(NOTE_ID, null), + Collections.emptyList(), + Collections.emptySet(), + RemoteSnapshot.of(snapshot(NOTE_ID, null)))); throw new AssertionError("Expected a publish with no read context to be refused"); } catch (IOException expected) { - assertThat(expected).hasMessageThat().contains("read context"); + assertThat(expected).hasMessageThat().contains("latest remote read"); } } @@ -589,6 +595,365 @@ public void aDeletedAncestorBundleDoesNotBreakSyncOrLoseAnAlternative() throws E .isEqualTo("written on A"); } + // ------------------------------------------------- transport failure classification + + @Test + public void mayHaveCommitted_treatsAReadTimeoutAsAmbiguous() { + // SocketTimeoutException extends InterruptedIOException, and the old classifier asked + // about the parent first, so a timeout waiting for the response of an upload that had + // already landed failed the sync instead of being confirmed by discovery. + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(new java.net.SocketTimeoutException())) + .isTrue(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(new java.io.InterruptedIOException())) + .isFalse(); + } + + @Test + public void mayHaveCommitted_agreesWithTheRetryPolicyAboutTransientStatuses() { + // One answer for bundle and attachment uploads: a 429 used to be rediscovered for the + // bundle POST and rethrown for the attachment POST. + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(429, ""))).isTrue(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(503, ""))).isTrue(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(403, "rateLimitExceeded"))) + .isTrue(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(403, "forbidden"))).isFalse(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(401, ""))).isFalse(); + assertThat(GoogleDriveSyncBackend.mayHaveCommitted(http(400, ""))).isFalse(); + assertThat( + GoogleDriveSyncBackend.mayHaveCommitted( + new AttachmentIntegrityException("checksum"))) + .isFalse(); + } + + private static IOException http(int status, String detail) { + return new DriveRequestExecutor.DriveHttpException(status, null, detail); + } + + // ------------------------------------------------- bundle history + + @Test + public void validateAncestry_walksALongLinearHistoryWithoutOverflowingTheStack() + throws Exception { + // An account that syncs after every edit builds exactly this shape. The recursive walk + // used one frame per ancestor and a StackOverflowError is not an IOException the sync + // knows how to report. + Map> parents = new LinkedHashMap<>(); + String previous = null; + for (int index = 0; index < 200_000; index++) { + String id = "bundle-" + index; + parents.put(id, previous == null ? Collections.emptyList() : List.of(previous)); + previous = id; + } + Throwable[] failure = new Throwable[1]; + Thread small = + new Thread( + null, + () -> { + try { + GoogleDriveSyncBackend.validateAncestry(parents); + } catch (Throwable error) { + failure[0] = error; + } + }, + "small-stack", + 256L * 1024L); + small.start(); + small.join(30_000L); + + assertThat(small.isAlive()).isFalse(); + assertThat(failure[0]).isNull(); + } + + @Test + public void validateAncestry_stillRejectsACycle() { + Map> parents = new LinkedHashMap<>(); + parents.put("a", List.of("b")); + parents.put("b", List.of("c")); + parents.put("c", List.of("a", "missing")); + + try { + GoogleDriveSyncBackend.validateAncestry(parents); + throw new AssertionError("Expected the cycle to be refused"); + } catch (IOException expected) { + assertThat(expected).hasMessageThat().contains("cycle"); + } + } + + @Test + public void publish_namesTheFrontierOfTheReadItQuotesAsParents() throws Exception { + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + server.seedOwnedBundleBytes(base); + GoogleDriveSyncBackend backend = backend(); + RemoteSnapshot context = backend.readSnapshotResult(); + + backend.publish( + new SyncPublication( + snapshotWithTitle("Next"), + Collections.emptyList(), + Collections.emptySet(), + context)); + + // The parents come from the quoted read, not from a second copy of its frontier kept on + // the backend that had to be kept in step by hand. + assertThat(context.getFrontierBundleIds()).containsExactly(baseId); + assertThat( + codec.decode(new ByteArrayInputStream(server.newestBundleBytes())) + .getParentBundleIds()) + .containsExactly(baseId); + } + + @Test + public void publish_retiresABundleOnlyOnceItHasBeenSupersededForTheWholeGrace() + throws Exception { + // Nothing ever deleted a bundle, so every sync downloaded and decoded the whole history + // to find one or two heads. A bundle is marked the first time a read finds it outside + // the frontier; Drive dates the mark, and the grace runs from there — not from the + // bundle's creation. A head created days ago and superseded seconds ago is exactly the + // file another device is most likely to be reading. + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + byte[] first = + codec.encode( + snapshotWithTitle("First"), CLOCK.instant(), Collections.singleton(baseId)); + String firstId = codec.decode(new ByteArrayInputStream(first)).getBundleId(); + byte[] second = + codec.encode( + snapshotWithTitle("Second"), + CLOCK.instant(), + Collections.singleton(firstId)); + server.seedOwnedBundleBytes(base); + server.seedOwnedBundleBytes(first); + server.seedOwnedBundleBytes(second); + // Created long ago; superseded only as far as this sync can tell. + server.ageBundles(3L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); + + publish(backend(), snapshotWithTitle("Third")); + + // Old by creation, but their supersession was only just recorded: nothing goes yet. + assertThat(server.deletedFileIds()).isEmpty(); + assertThat(server.supersededBundleCount()).isEqualTo(2); + assertThat(server.bundleCount()).isEqualTo(4); + + // Two hours on, by Drive's clock and this device's alike. + server.advanceClock(2L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); + GoogleDriveSyncBackend later = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + Clock.offset(CLOCK, java.time.Duration.ofHours(2)), + new SyncBundleCodec()); + publish(later, snapshotWithTitle("Fourth")); + + // base and first were marked two hours ago and go; second was superseded by Third and + // is only marked now; Third is the head this publish descends from. + assertThat(server.deletedFileIds()).hasSize(2); + assertThat(server.bundleCount()).isEqualTo(3); + assertThat( + backend() + .readSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID) + .getPayload() + .get("title") + .getAsString()) + .isEqualTo("Fourth"); + } + + @Test + public void publish_measuresTheGraceOnDrivesClockNotThePhones() throws Exception { + // A phone running two hours fast used to see every fresh mark as two hours old and + // delete a bundle another device was still reading; Drive's Date header is the clock now. + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + byte[] head = + codec.encode( + snapshotWithTitle("Head"), CLOCK.instant(), Collections.singleton(baseId)); + server.seedOwnedBundleBytes(base); + server.seedOwnedBundleBytes(head); + publish(backend(), snapshotWithTitle("Next")); + assertThat(server.supersededBundleCount()).isEqualTo(1); + + GoogleDriveSyncBackend fastPhone = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + Clock.offset(CLOCK, java.time.Duration.ofHours(2)), + new SyncBundleCodec()); + publish(fastPhone, snapshotWithTitle("Later")); + + // Drive's clock has not moved, so nothing has been superseded for the grace period. + assertThat(server.deletedFileIds()).isEmpty(); + assertThat(server.bundleCount()).isEqualTo(4); + } + + @Test + public void publish_neverPrunesABundleWhoseSupersessionCarriesNoUsableTime() throws Exception { + // A bare marker written without server time, and Drive reporting no modifiedTime for + // the file: nothing can be proven old enough, so nothing goes. + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + byte[] head = + codec.encode( + snapshotWithTitle("Head"), CLOCK.instant(), Collections.singleton(baseId)); + server.seedOwnedBundleBytes(base); + server.seedOwnedBundleBytes(head); + publish(backend(), snapshotWithTitle("Next")); + server.stripSupersessionTimes(); + server.withholdModifiedTime(); + server.advanceClock(2L * GoogleDriveSyncBackend.BUNDLE_PRUNE_GRACE_MILLIS); + + publish( + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + Clock.offset(CLOCK, java.time.Duration.ofHours(2)), + new SyncBundleCodec()), + snapshotWithTitle("Later")); + + assertThat(server.deletedFileIds()).isEmpty(); + assertThat(server.bundleCount()).isEqualTo(4); + } + + @Test + public void readAttachment_fallsBackToTheGoodCopyInAnotherRoot() throws Exception { + // Duplicate roots are a supported state. With Drive's checksums withheld the bytes must + // be read to tell the copies apart; handing over the first root's only candidate unread + // let a corrupt copy there shadow the good one in the other root on every sync. + byte[] good = "good bytes".getBytes(StandardCharsets.UTF_8); + String hash = sha256(good); + server.seedCorruptAttachment(hash, "corrupt".getBytes(StandardCharsets.UTF_8)); + server.registerAttachment(good); + server.seedOwnedBundle(snapshot(NOTE_ID, hash)); + server.withholdChecksums(); + assertThat(server.ownedFolderCount()).isEqualTo(2); + + try (java.io.InputStream restored = backend().readAttachment(hash)) { + assertThat(restored).isNotNull(); + assertThat(readAll(restored)).isEqualTo(good); + } + } + + // ------------------------------------------------- attachment transfer cost + + @Test + public void readAttachment_downloadsTheBlobOnce() throws Exception { + byte[] bytes = "photo".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + backend().writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + + try (java.io.InputStream stream = backend().readAttachment(hash)) { + assertThat(readAll(stream)).isEqualTo(bytes); + } + + // It used to be downloaded in full to pick a verified candidate and then downloaded + // again to hand over; every caller verifies the stream it receives anyway. + assertThat(server.mediaReadsOfAttachment(hash)).isEqualTo(1); + } + + @Test + public void hasVerifiedAttachment_trustsDrivesOwnChecksumWithoutDownloading() throws Exception { + byte[] bytes = "photo".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + backend().writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + byte[] wrong = "wrong bytes".getBytes(StandardCharsets.UTF_8); + String claimed = sha256("something else".getBytes(StandardCharsets.UTF_8)); + server.seedCorruptAttachment(claimed, wrong); + + GoogleDriveSyncBackend backend = backend(); + + // Every attachment in the account used to be re-downloaded on every sync just to answer + // this; Drive computes the digest over the stored bytes, so a mismatch shows in the + // listing as well. + assertThat(backend.hasVerifiedAttachment(hash, (long) bytes.length)).isTrue(); + assertThat(backend.hasVerifiedAttachment(claimed, (long) wrong.length)).isFalse(); + assertThat(server.mediaReadsOfAttachment(hash)).isEqualTo(0); + assertThat(server.mediaReadsOfAttachment(claimed)).isEqualTo(0); + } + + @Test + public void hasVerifiedAttachment_readsTheBlobWhenTheListingCannotConfirmItsSize() + throws Exception { + // Drive's digest matches but the listing carries no usable size. Treating that as corrupt + // reported a good blob absent, so the service uploaded a duplicate on every sync and then + // failed anyway when the duplicate listed the same way. + byte[] bytes = "photo".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + backend().writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + server.withholdSizes(); + GoogleDriveSyncBackend backend = backend(); + + assertThat(backend.hasVerifiedAttachment(hash, (long) bytes.length)).isTrue(); + + assertThat(server.mediaReadsOfAttachment(hash)).isEqualTo(1); + assertThat(server.ownedAttachmentCount(hash)).isEqualTo(1); + } + + @Test + public void oneSyncListsTheRootFoldersOnce() throws Exception { + byte[] bytes = "photo".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + backend().writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + GoogleDriveSyncBackend backend = backend(); + int before = server.folderListings(); + + backend.readSnapshotResult(); + backend.hasAttachment(hash); + backend.hasVerifiedAttachment(hash, (long) bytes.length); + try (java.io.InputStream stream = backend.readAttachment(hash)) { + readAll(stream); + } + + // Each of those used to list the roots again; with N attachments that was several times + // N listings per sync before a byte moved, which is what Drive rate-limited. + assertThat(server.folderListings() - before).isEqualTo(1); + } + + @Test + public void anOversizedCandidateIsSkippedRatherThanFailingTheSync() throws Exception { + // A Drive object larger than the ceiling and tagged with the expected hash: the ceiling + // used to throw a plain IOException from the stream, past the "skip a corrupt candidate" + // path, so one bad object failed every sync. Drive is asked to withhold its checksum so + // the bytes have to be read, which is what the ceiling guards. + byte[] good = "good".getBytes(StandardCharsets.UTF_8); + String hash = sha256(good); + server.seedCorruptAttachment(hash, new byte[64]); + server.withholdChecksums(); + GoogleDriveSyncBackend backend = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + CLOCK, + new SyncBundleCodec(), + 16L); + + assertThat(backend.hasVerifiedAttachment(hash, (long) good.length)).isFalse(); + backend.writeAttachment(hash, good.length, new ByteArrayInputStream(good)); + + assertThat(server.ownedAttachmentCount(hash)).isEqualTo(2); + try (java.io.InputStream restored = backend.readAttachment(hash)) { + assertThat(readAll(restored)).isEqualTo(good); + } + } + + @Test + public void writeAttachment_withAnUnknownSizeBuffersWithinTheCeilingAndUploads() + throws Exception { + byte[] bytes = "size unknown".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + + backend().writeAttachment(hash, -1L, new ByteArrayInputStream(bytes)); + + assertThat(server.attachmentContent(hash)).isEqualTo(bytes); + } + private static final java.time.Instant T10 = java.time.Instant.parse("2026-08-31T12:00:10Z"); private static final java.time.Instant T20 = java.time.Instant.parse("2026-08-31T12:00:20Z"); @@ -888,6 +1253,17 @@ private static final class FakeDriveServer implements AutoCloseable { java.util.Collections.synchronizedList(new ArrayList<>()); private volatile boolean running = true; private volatile CyclicBarrier emptyRootListingBarrier; + private final AtomicInteger folderListings = new AtomicInteger(); + private final Map mediaReads = new ConcurrentHashMap<>(); + private final List deletedFileIds = + java.util.Collections.synchronizedList(new ArrayList<>()); + private volatile boolean withholdChecksums; + private volatile boolean withholdSizes; + private volatile boolean withholdModifiedTime; + + /** Drive's clock, as the fake stamps files with it; starts at the tests' fixed CLOCK. */ + private volatile long serverNowMillis = CLOCK.millis(); + private SyncSnapshot updateBeforeNextPatch; private final AtomicInteger nextId = new AtomicInteger(1); @@ -953,6 +1329,37 @@ List rejectedChunkRanges() { return new ArrayList<>(rejectedChunkRanges); } + /** How many times the folder index was listed. */ + int folderListings() { + return folderListings.get(); + } + + /** How many times the bytes of the blob carrying {@code sha256} were downloaded. */ + int mediaReadsOfAttachment(String sha256) { + int total = 0; + for (DriveFile file : files.values()) { + if (sha256.equals(file.appProperties.get("mynotesAttachmentSha256"))) { + total += mediaReads.getOrDefault(file.id, 0); + } + } + return total; + } + + /** Ids of files the client deleted. */ + List deletedFileIds() { + return new ArrayList<>(deletedFileIds); + } + + /** Moves every stored bundle's Drive-side timestamps {@code millis} into the past. */ + void ageBundles(long millis) { + for (DriveFile file : files.values()) { + if ("1".equals(file.appProperties.get("mynotesBundle"))) { + file.createdAtMillis -= millis; + file.modifiedAtMillis -= millis; + } + } + } + /** Committed bytes of the attachment blob carrying {@code sha256}, or null. */ byte[] attachmentContent(String sha256) { for (DriveFile file : files.values()) { @@ -1047,6 +1454,62 @@ byte[] readBundleBytes() { return null; } + /** The most recently created bundle file's bytes. */ + byte[] newestBundleBytes() { + DriveFile newest = null; + for (DriveFile file : files.values()) { + if ("1".equals(file.appProperties.get("mynotesBundle")) + && (newest == null + || Integer.parseInt(file.id) > Integer.parseInt(newest.id))) { + newest = file; + } + } + return newest == null ? null : newest.content; + } + + /** Models objects Drive has not (yet) checksummed: listings carry no digest or size. */ + void withholdChecksums() { + withholdChecksums = true; + } + + /** Models a listing that reports no modification time for its files. */ + void withholdModifiedTime() { + withholdModifiedTime = true; + } + + /** Models objects whose listing carries a checksum but no usable size. */ + void withholdSizes() { + withholdSizes = true; + } + + /** Moves Drive's clock forward; later stamps are dated from the new time. */ + void advanceClock(long millis) { + serverNowMillis += millis; + } + + /** + * Turns every supersession mark into a bare marker, as a client without server time writes. + */ + void stripSupersessionTimes() { + for (DriveFile file : files.values()) { + if (file.appProperties.containsKey("mynotesBundleSuperseded")) { + file.appProperties.put("mynotesBundleSuperseded", "1"); + } + } + } + + /** Whether the client has marked the newest {@code count} bundles as superseded. */ + int supersededBundleCount() { + int count = 0; + for (DriveFile file : files.values()) { + if ("1".equals(file.appProperties.get("mynotesBundle")) + && file.appProperties.containsKey("mynotesBundleSuperseded")) { + count++; + } + } + return count; + } + String registerAttachment(byte[] bytes) throws Exception { String hash = sha256(bytes); seededAttachmentContent.put(hash, bytes); @@ -1144,7 +1607,7 @@ private Response dispatch(Request request) throws IOException { return Response.json(405, "{}"); } if (path.startsWith("/drive/v3/files/")) { - return handleFileRead(uri, path.substring("/drive/v3/files/".length())); + return handleFileRead(request, uri, path.substring("/drive/v3/files/".length())); } if ("/upload/drive/v3/files".equals(path) && "POST".equals(request.method)) { if ("resumable".equals(parseQuery(uri).get("uploadType"))) { @@ -1175,12 +1638,40 @@ && ownedFolderCount() == 0) { return Response.json(500, "{}"); } } + if (query != null + && query.contains("mimeType = 'application/vnd.google-apps.folder'")) { + folderListings.incrementAndGet(); + } JsonArray array = new JsonArray(); for (DriveFile file : files.values()) { if (matchesQuery(file, query)) { JsonObject value = new JsonObject(); value.addProperty("id", file.id); value.addProperty("name", file.name); + // Drive reports these for binary files; the fake computes them from the + // stored bytes exactly as Drive does, so a corrupt object is exposed by its + // real digest rather than by the property the uploader claimed. + if (!withholdChecksums) { + if (!withholdSizes) { + value.addProperty("size", String.valueOf(file.content.length)); + } + if (!"application/vnd.google-apps.folder".equals(file.mimeType)) { + value.addProperty("sha256Checksum", sha256Unchecked(file.content)); + } + } + JsonObject appProperties = new JsonObject(); + for (Map.Entry entry : file.appProperties.entrySet()) { + appProperties.addProperty(entry.getKey(), entry.getValue()); + } + value.add("appProperties", appProperties); + // Drive's own clock, RFC 3339, as the real listing reports it. + value.addProperty( + "createdTime", Instant.ofEpochMilli(file.createdAtMillis).toString()); + if (!withholdModifiedTime) { + value.addProperty( + "modifiedTime", + Instant.ofEpochMilli(file.modifiedAtMillis).toString()); + } array.add(value); } } @@ -1189,6 +1680,14 @@ && ownedFolderCount() == 0) { return Response.json(200, response.toString()); } + private static String sha256Unchecked(byte[] bytes) { + try { + return sha256(bytes); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + private Response handleCreateMetadata(byte[] body) throws IOException { JsonObject metadata = readJson(body); DriveFile file = @@ -1200,12 +1699,31 @@ private Response handleCreateMetadata(byte[] body) throws IOException { return Response.json(200, fileMetadata(file).toString(), file.eTag()); } - private Response handleFileRead(URI uri, String id) { + private Response handleFileRead(Request request, URI uri, String id) { DriveFile file = files.get(id); if (file == null) { return Response.json(404, "{}"); } + if ("DELETE".equals(request.method)) { + files.remove(id); + deletedFileIds.add(id); + return Response.json(204, ""); + } + if ("POST".equals(request.method) + && "PATCH".equals(request.headers.get("x-http-method-override"))) { + // files.update: merges appProperties and, like Drive, stamps modifiedTime. + JsonObject patch = readJson(request.body); + if (patch.has("appProperties")) { + for (Map.Entry entry : + patch.getAsJsonObject("appProperties").entrySet()) { + file.appProperties.put(entry.getKey(), entry.getValue().getAsString()); + } + } + file.modifiedAtMillis = serverNowMillis; + return Response.json(200, fileMetadata(file).toString(), file.eTag()); + } if ("media".equals(parseQuery(uri).get("alt"))) { + mediaReads.merge(id, 1, Integer::sum); return Response.binary(200, file.content, file.eTag()); } return Response.json(200, fileMetadata(file).toString(), file.eTag()); @@ -1293,6 +1811,13 @@ private Response handleResumableChunk(Request request, String sessionId) rejectedChunkRanges.add(range); return Response.json(400, "{}"); } + if (end + 1L < session.totalBytes && request.body.length % (256 * 1024) != 0) { + // Drive's rule: every chunk but the last is a multiple of 256 KiB. A partially + // acknowledged window used to be continued with just its tail, which Drive + // answers with a 400 nothing retries. + rejectedChunkRanges.add(range); + return Response.json(400, "{}"); + } if (script != null && script.forcedRangeInclusiveEnd != null) { Map headers = new LinkedHashMap<>(); @@ -1356,6 +1881,8 @@ private void applyMetadata(DriveFile file, JsonObject metadata) { private DriveFile createFile(String name, String mimeType, String parentId) { DriveFile file = new DriveFile(Integer.toString(nextId.getAndIncrement()), name, mimeType); + file.createdAtMillis = serverNowMillis; + file.modifiedAtMillis = serverNowMillis; if (parentId != null) { file.parents.add(parentId); } @@ -1521,13 +2048,19 @@ private static String readLine(BufferedInputStream input) throws IOException { return output.toString(StandardCharsets.ISO_8859_1.name()); } - private static void writeResponse(OutputStream output, Response response) - throws IOException { + private void writeResponse(OutputStream output, Response response) throws IOException { + long serverNow = serverNowMillis; StringBuilder headers = new StringBuilder(); headers.append("HTTP/1.1 ").append(response.code).append(" OK\r\n"); headers.append("Content-Length: ").append(response.body.length).append("\r\n"); headers.append("Connection: close\r\n"); headers.append("Content-Type: ").append(response.contentType).append("\r\n"); + // Drive's own clock, as every Google response reports it. + headers.append("Date: ") + .append( + java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME.format( + Instant.ofEpochMilli(serverNow).atZone(ZoneOffset.UTC))) + .append("\r\n"); for (Map.Entry header : response.headers.entrySet()) { headers.append(header.getKey()) .append(": ") @@ -1560,6 +2093,12 @@ public void close() { private static final class DriveFile { private final String id; + + /** Drive's own timestamps, stamped from the fake server's clock. */ + private long createdAtMillis = CLOCK.millis(); + + private long modifiedAtMillis = CLOCK.millis(); + private byte[] content = new byte[0]; private String name; private String mimeType; diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/LegacyNotePayloadTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/LegacyNotePayloadTest.java new file mode 100644 index 00000000..ccc8cc42 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/LegacyNotePayloadTest.java @@ -0,0 +1,136 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.pasich.mynotes.extendedEditor.attach.EditorAttachmentBlocks; +import java.time.Instant; +import org.junit.Test; + +/** + * A note published by 2.6.50 has to come out of the decoder in the shape the upgraded store builds + * for the same, unchanged note. Otherwise the two hash differently at the same timestamp and the + * merge reports a conflict against the note itself — on every sync, when the old version wins. + */ +public class LegacyNotePayloadTest { + + private static final String NOTE_ID = "550e8400-e29b-41d4-a716-446655440000"; + private static final String HASH = + "d6f1f3d5d8cf9b5a4a2469787998dc45eb59f401b93b1b4cde4998dc409ebdc8"; + private static final String LOCAL_URL = "editorjs://attachments/note_7/1700000000000_1.png"; + + @Test + public void upgradesA2650PayloadToTheCurrentDerivationAndWireForm() { + JsonObject legacy = legacyPayload(LOCAL_URL, "photo.png"); + String expectedId = AttachmentLogicalIds.derive(NOTE_ID, 0, LOCAL_URL, "photo.png", HASH); + + assertThat(LegacyNotePayload.upgrade(NOTE_ID, legacy)).isTrue(); + + JsonObject entry = legacy.getAsJsonArray("attachmentsManifest").get(0).getAsJsonObject(); + assertThat(entry.get("id").getAsString()).isEqualTo(expectedId); + assertThat(legacy.getAsJsonObject("attachmentNames").get(expectedId).getAsString()) + .isEqualTo("photo.png"); + assertThat(EditorAttachmentBlocks.fileUrls(legacy.get("f").getAsString())) + .containsExactly(AttachmentWireUrl.forLogicalId(expectedId)); + assertThat(legacy.getAsJsonArray("attachmentHashes").get(0).getAsString()).isEqualTo(HASH); + } + + @Test + public void theUpgradedPayloadHashesLikeTheCurrentBuildOfTheSameNote() { + // What RoomSyncStore builds today for the same unchanged note: the same derivation over + // the same inputs, the block already in wire form, names keyed by the new id. + String id = AttachmentLogicalIds.derive(NOTE_ID, 0, LOCAL_URL, "photo.png", HASH); + JsonObject current = legacyPayload(LOCAL_URL, "photo.png"); + current.getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .addProperty("id", id); + JsonObject names = new JsonObject(); + names.addProperty(id, "photo.png"); + current.add("attachmentNames", names); + current.addProperty( + "f", + EditorAttachmentBlocks.rewriteUrls( + current.get("f").getAsString(), url -> AttachmentWireUrl.forLogicalId(id))); + JsonObject legacy = legacyPayload(LOCAL_URL, "photo.png"); + + LegacyNotePayload.upgrade(NOTE_ID, legacy); + + Instant at = Instant.parse("2026-08-31T12:00:01Z"); + assertThat( + SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, at, legacy) + .getCanonicalPayloadHash()) + .isEqualTo( + SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, at, current) + .getCanonicalPayloadHash()); + } + + @Test + public void leavesAPayloadWhoseIdsWereNotDerivedTheOldWayAlone() { + // An id a receiving device restored, or one already derived with the hash: not provably + // the old shape, so it is not touched. + JsonObject payload = legacyPayload(LOCAL_URL, "photo.png"); + payload.getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .addProperty("id", "7d444840-9dc0-11d1-b245-5ffdce74fad2"); + String before = payload.toString(); + + assertThat(LegacyNotePayload.upgrade(NOTE_ID, payload)).isFalse(); + assertThat(payload.toString()).isEqualTo(before); + } + + @Test + public void leavesAPayloadWhoseBlocksDoNotLineUpWithItsManifestAlone() { + JsonObject payload = legacyPayload(LOCAL_URL, "photo.png"); + payload.addProperty( + "f", + "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + LOCAL_URL + + "\"}}},{\"type\":\"image\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_7/other.png\"}}}]"); + String before = payload.toString(); + + assertThat(LegacyNotePayload.upgrade(NOTE_ID, payload)).isFalse(); + assertThat(payload.toString()).isEqualTo(before); + } + + @Test + public void leavesAPayloadAlreadyInWireFormAlone() { + JsonObject payload = legacyPayload(AttachmentWireUrl.forLogicalId(NOTE_ID), "photo.png"); + + assertThat(LegacyNotePayload.upgrade(NOTE_ID, payload)).isFalse(); + } + + /** Exactly what 2.6.50's store built: local block URL, hash-less id, names keyed by it. */ + private static JsonObject legacyPayload(String blockUrl, String displayName) { + String legacyId = AttachmentLogicalIds.deriveLegacy(NOTE_ID, 0, blockUrl, displayName); + JsonObject payload = new JsonObject(); + payload.addProperty("b", "Shopping"); + payload.addProperty("c", "Milk"); + payload.addProperty( + "f", + "[{\"id\":\"blk\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + blockUrl + + "\",\"name\":\"" + + displayName + + "\"}}}]"); + JsonObject entry = new JsonObject(); + entry.addProperty("id", legacyId); + entry.addProperty("sha256", HASH); + entry.addProperty("mimeType", "image/png"); + entry.addProperty("size", 42L); + entry.addProperty("path", "attachments/" + HASH); + entry.addProperty("displayName", displayName); + JsonArray manifest = new JsonArray(); + manifest.add(entry); + payload.add("attachmentsManifest", manifest); + JsonArray hashes = new JsonArray(); + hashes.add(HASH); + payload.add("attachmentHashes", hashes); + JsonObject names = new JsonObject(); + names.addProperty(legacyId, displayName); + payload.add("attachmentNames", names); + return payload; + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecisionTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecisionTest.java new file mode 100644 index 00000000..78a38ceb --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/PreferencesBaselineDecisionTest.java @@ -0,0 +1,51 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; + +/** + * Whether the live settings count as a local edit. + * + *

The baseline changed format once — 2.6.49 stored the payload JSON, later releases its digest — + * and every device upgrading across that boundary claimed a local edit it never made, letting its + * untouched settings outrank a genuine change from another device. + */ +public class PreferencesBaselineDecisionTest { + + private static final String LEGACY_JSON = "{\"a\":1,\"b\":14,\"c\":11,\"g\":true}"; + private static final String DIGEST = Sha256.of(LEGACY_JSON); + + @Test + public void unchangedSettingsAreNotAnEdit() { + assertThat(PreferencesBaselineDecision.decide(DIGEST, DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.UNCHANGED); + } + + @Test + public void aLegacyBaselineDescribingTheLiveValuesIsMigratedNotTreatedAsAnEdit() { + assertThat(PreferencesBaselineDecision.decide(LEGACY_JSON, DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.MIGRATE_BASELINE); + } + + @Test + public void aLegacyBaselineForOtherValuesIsAnEdit() { + assertThat( + PreferencesBaselineDecision.decide( + "{\"a\":1,\"b\":14,\"c\":2,\"g\":true}", DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.LOCAL_EDIT); + } + + @Test + public void aDifferentDigestIsAnEdit() { + assertThat(PreferencesBaselineDecision.decide(Sha256.of("other"), DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.LOCAL_EDIT); + } + + @Test + public void noBaselineAtAllIsAnEdit() { + // Sync has never seen these settings; publishing them is the conservative reading. + assertThat(PreferencesBaselineDecision.decide(null, DIGEST, LEGACY_JSON)) + .isEqualTo(PreferencesBaselineDecision.Action.LOCAL_EDIT); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/Sha256Test.java b/app/src/test/java/com/pasich/mynotes/data/sync/Sha256Test.java new file mode 100644 index 00000000..177c60e2 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/Sha256Test.java @@ -0,0 +1,59 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Locale; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * The one digest every sync component hashes with. + * + *

A blob hashed by the store has to match a manifest written by the validator and a checksum + * compared by the backend, so the encoding is pinned rather than left to whichever of the former + * five copies a caller happened to reach. + */ +public class Sha256Test { + + private static final String EMPTY = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void everyOverloadAgreesOnTheKnownVector() throws Exception { + File file = temporaryFolder.newFile("empty"); + Files.write(file.toPath(), new byte[0]); + + assertThat(Sha256.of(new byte[0])).isEqualTo(EMPTY); + assertThat(Sha256.of("")).isEqualTo(EMPTY); + assertThat(Sha256.of(new ByteArrayInputStream(new byte[0]))).isEqualTo(EMPTY); + assertThat(Sha256.of(file)).isEqualTo(EMPTY); + assertThat(Sha256.of("abc".getBytes(StandardCharsets.UTF_8))) + .isEqualTo("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + } + + @Test + public void hexIsLowercaseAndZeroPadded() { + assertThat(Sha256.hex(new byte[] {0, (byte) 0xff, 0x10, 0x0a})).isEqualTo("00ff100a"); + } + + @Test + public void hexDoesNotDependOnTheDefaultLocale() { + // Three of the former copies formatted through String.format("%02x") with the default + // locale; a locale with its own digits would have made a locally hashed blob mismatch the + // manifest it was validated against. + Locale previous = Locale.getDefault(); + try { + Locale.setDefault(Locale.forLanguageTag("ar-EG")); + assertThat(Sha256.of("")).isEqualTo(EMPTY); + } finally { + Locale.setDefault(previous); + } + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SnapshotProblemTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SnapshotProblemTest.java new file mode 100644 index 00000000..4bf9b45b --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SnapshotProblemTest.java @@ -0,0 +1,59 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import java.util.Collections; +import org.junit.Test; + +public class SnapshotProblemTest { + + private static final String NOTE_ID = "550e8400-e29b-41d4-a716-446655440000"; + + @Test + public void theFailureNamesTheNoteItIsIn() throws java.io.IOException { + // "MISSING_ATTACHMENT" alone left the user to guess which of their notes to open; the + // account screen shows this string as the sync status. + SnapshotBuildResult result = + SnapshotBuildResult.incomplete( + SyncSnapshot.empty(), + Collections.singletonList( + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + SyncMetadata.RECORD_TYPE_NOTE, + NOTE_ID, + "Shopping list"))); + + try { + result.requireSnapshot(); + throw new AssertionError("Expected an incomplete snapshot to be refused"); + } catch (SnapshotBuildResult.SnapshotBuildException expected) { + assertThat(expected) + .hasMessageThat() + .isEqualTo( + "Local snapshot is incomplete: MISSING_ATTACHMENT in note" + + " \"Shopping list\""); + } + } + + @Test + public void aLongTitleIsCutShortAndABlankOneIsDropped() { + String title = "x".repeat(80); + + SnapshotProblem long_ = + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + SyncMetadata.RECORD_TYPE_NOTE, + NOTE_ID, + title); + SnapshotProblem blank = + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + SyncMetadata.RECORD_TYPE_NOTE, + NOTE_ID, + " "); + + assertThat(long_.getLabel()).hasLength(41); + assertThat(blank.getLabel()).isNull(); + assertThat(blank.describe()).isEqualTo("MISSING_ATTACHMENT"); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java index e157bcbb..adc0980a 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java @@ -303,6 +303,320 @@ public void encode_dropsEmptyAttachmentFieldsInsteadOfLeakingThemToTheWire() thr .doesNotContain("attachmentNames"); } + @Test + public void encode_carriesAnAlternativeWhoseAttachmentIdMapsToDifferentContent() + throws Exception { + // A live note and one of its unresolved alternatives can describe different bytes under + // one logical attachment id — the file was replaced in place, or two devices derived the + // same id. Refusing to encode that failed every publish for the account, before the + // conflict could even be stored for the user to settle. + String otherHash = "0000000000000000000000000000000000000000000000000000000000000001"; + // Both payloads in the shape RoomSyncStore builds, names included. + JsonObject livePayload = notePayload("Live", "image/png", 42L, "receipt.png"); + JsonObject names = new JsonObject(); + names.addProperty(ATTACHMENT_ID, "receipt.png"); + livePayload.add("attachmentNames", names.deepCopy()); + SyncRecord live = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:05Z"), + livePayload); + JsonObject alternativePayload = notePayload("Older", "image/png", 77L, "receipt.png"); + alternativePayload.add("attachmentNames", names.deepCopy()); + alternativePayload + .getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .addProperty("sha256", otherHash); + alternativePayload + .getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .addProperty("path", "attachments/" + otherHash); + alternativePayload + .getAsJsonArray("attachmentHashes") + .set(0, new com.google.gson.JsonPrimitive(otherHash)); + SyncRecord alternative = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:04Z"), + alternativePayload); + + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] bundle = + codec.encode( + new SyncSnapshot(java.util.Collections.singletonList(live)), + CREATED_AT, + java.util.Collections.emptyList(), + java.util.Collections.singletonList(alternative), + java.util.Collections.emptySet()); + SyncBundleCodec.DecodedBundle decoded = codec.decode(new ByteArrayInputStream(bundle)); + + // Both blobs are described, and the decoded alternative is the version that went in: + // same logical id, same content, same identity for the resolution bookkeeping. + assertThat(decoded.getAttachmentsByHash().keySet()).containsExactly(HASH, otherHash); + SyncRecord decodedAlternative = decoded.getAlternatives().get(0); + JsonObject entry = + decodedAlternative + .getPayload() + .getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject(); + assertThat(entry.get("id").getAsString()).isEqualTo(ATTACHMENT_ID); + assertThat(entry.get("sha256").getAsString()).isEqualTo(otherHash); + assertThat(decodedAlternative.getCanonicalPayloadHash()) + .isEqualTo(alternative.getCanonicalPayloadHash()); + assertThat( + decoded.getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID) + .getCanonicalPayloadHash()) + .isEqualTo(live.getCanonicalPayloadHash()); + } + + @Test + public void encode_keepsBothBlobsWhenOneNoteCarriesOneIdWithTwoContents() throws Exception { + // A duplicated block whose file was later replaced: one logical id, two hashes, in ONE + // record. Keying the re-keying by id sent both references to the second blob, so every + // receiver wrote its bytes for both and the first attachment's content was lost. + String otherHash = "0000000000000000000000000000000000000000000000000000000000000002"; + JsonObject payload = notePayload("Body", "image/png", 42L, "first.png"); + JsonObject second = + new SyncBundleCodec.AttachmentManifestEntry( + ATTACHMENT_ID, + otherHash, + "image/png", + 43L, + "attachments/" + otherHash, + "second.png") + .toJson(true); + payload.getAsJsonArray("attachmentsManifest").add(second); + payload.getAsJsonArray("attachmentHashes").add(otherHash); + JsonObject names = new JsonObject(); + names.addProperty(ATTACHMENT_ID, "second.png"); + payload.add("attachmentNames", names); + SyncRecord local = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:01Z"), + payload); + + SyncBundleCodec codec = new SyncBundleCodec(); + SyncBundleCodec.DecodedBundle decoded = + codec.decode( + new ByteArrayInputStream( + codec.encode( + new SyncSnapshot( + java.util.Collections.singletonList(local)), + CREATED_AT))); + + assertThat(decoded.getAttachmentsByHash().keySet()).containsExactly(HASH, otherHash); + SyncRecord note = decoded.getSnapshot().find(SyncRecord.Type.NOTE, NOTE_ID); + JsonArray manifest = note.getPayload().getAsJsonArray("attachmentsManifest"); + assertThat(manifest).hasSize(2); + assertThat(manifest.get(0).getAsJsonObject().get("sha256").getAsString()).isEqualTo(HASH); + assertThat(manifest.get(1).getAsJsonObject().get("sha256").getAsString()) + .isEqualTo(otherHash); + assertThat(manifest.get(0).getAsJsonObject().get("id").getAsString()) + .isEqualTo(ATTACHMENT_ID); + assertThat(manifest.get(1).getAsJsonObject().get("id").getAsString()) + .isEqualTo(ATTACHMENT_ID); + assertThat(note.getCanonicalPayloadHash()).isEqualTo(local.getCanonicalPayloadHash()); + } + + @Test + public void encode_republishesANoteThatRepeatsOneAttachmentWithoutCollapsingIt() + throws Exception { + // 2.6.50 receivers hold manifests with one entry repeated — a duplicated block whose file + // was not replaced — and republish them as conflict alternatives after the upgrade. + // Refusing the shape at encode failed every publish until the conflict was resolved; + // collapsing it changed the version's hash. Each repeat travels under its own wire id. + JsonObject payload = notePayload("Body", "image/png", 42L, "first.png"); + payload.getAsJsonArray("attachmentsManifest") + .add( + payload.getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .deepCopy()); + payload.getAsJsonArray("attachmentHashes").add(HASH); + JsonObject names = new JsonObject(); + names.addProperty(ATTACHMENT_ID, "first.png"); + payload.add("attachmentNames", names); + SyncRecord repeated = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:01Z"), + payload); + SyncRecord live = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:05Z"), + notePayload("Newer", "image/png", 42L, "first.png")); + + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] bundle = + codec.encode( + new SyncSnapshot(java.util.Collections.singletonList(live)), + CREATED_AT, + java.util.Collections.emptyList(), + java.util.Collections.singletonList(repeated), + java.util.Collections.emptySet()); + SyncBundleCodec.DecodedBundle decoded = codec.decode(new ByteArrayInputStream(bundle)); + + SyncRecord alternative = decoded.getAlternatives().get(0); + JsonArray manifest = alternative.getPayload().getAsJsonArray("attachmentsManifest"); + assertThat(manifest).hasSize(2); + assertThat(manifest.get(1).getAsJsonObject().get("id").getAsString()) + .isEqualTo(ATTACHMENT_ID); + assertThat(alternative.getCanonicalPayloadHash()) + .isEqualTo(repeated.getCanonicalPayloadHash()); + // Never the shape the validator has to refuse: one wire id, two references. + JsonObject records = + com.google.gson.JsonParser.parseString( + unzipToStrings(bundle).get(SyncBundleCodec.ENTRY_RECORDS)) + .getAsJsonObject(); + JsonArray wireIds = + records.getAsJsonArray("alternatives") + .get(0) + .getAsJsonObject() + .getAsJsonArray("attachmentIds"); + assertThat(wireIds.get(0).getAsString()).isNotEqualTo(wireIds.get(1).getAsString()); + } + + @Test + public void decode_acceptsA2650BundleThatReferencesOneAttachmentTwice() throws Exception { + // 2.6.50 published attachmentIds [X, X] for a duplicated block with unchanged content, and + // such bundles sit on Drive in closed-testing accounts. A read decodes every bundle in + // every root, so refusing this shape failed every sync forever with no way to publish a + // successor that would let the bundle be pruned. + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] valid = codec.encode(new SyncSnapshot(Arrays.asList(note("Milk"))), CREATED_AT); + JsonObject records = readRecords(valid); + JsonObject wireNote = records.getAsJsonArray("notes").get(0).getAsJsonObject(); + wireNote.getAsJsonArray("attachmentIds").add(ATTACHMENT_ID); + + SyncBundleCodec.DecodedBundle decoded = + codec.decode(new ByteArrayInputStream(rebuild(valid, records))); + + SyncRecord decodedNote = decoded.getSnapshot().find(SyncRecord.Type.NOTE, NOTE_ID); + JsonArray manifest = decodedNote.getPayload().getAsJsonArray("attachmentsManifest"); + assertThat(manifest).hasSize(2); + assertThat(manifest.get(1).getAsJsonObject().get("sha256").getAsString()).isEqualTo(HASH); + // And it round-trips as the same version, so it never conflicts with itself. + SyncRecord again = + codec.decode( + new ByteArrayInputStream( + codec.encode( + new SyncSnapshot( + java.util.Collections.singletonList( + decodedNote)), + CREATED_AT))) + .getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID); + assertThat(again.getCanonicalPayloadHash()) + .isEqualTo(decodedNote.getCanonicalPayloadHash()); + } + + @Test + public void decode_refusesANoteWhoseRepeatedReferenceIsARekeyedOne() throws Exception { + // The shape an encoder produced when it collapsed two different blobs under one alias: + // the same re-keyed id twice. Only one blob is described, so the receiver cannot restore + // the note as it was; refusing is the only safe answer. + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] valid = codec.encode(new SyncSnapshot(Arrays.asList(note("Milk"))), CREATED_AT); + JsonObject records = readRecords(valid); + JsonObject wireNote = records.getAsJsonArray("notes").get(0).getAsJsonObject(); + String alias = "550e8400-e29b-41d4-a716-446655440077"; + JsonArray ids = new JsonArray(); + ids.add(alias); + ids.add(alias); + wireNote.add("attachmentIds", ids); + JsonObject aliases = new JsonObject(); + aliases.addProperty(alias, ATTACHMENT_ID); + wireNote.add(SyncBundleCodec.FIELD_ATTACHMENT_ID_ALIASES, aliases); + JsonObject manifest = readManifest(valid); + manifest.getAsJsonArray("attachments").get(0).getAsJsonObject().addProperty("id", alias); + + try { + codec.decode(new ByteArrayInputStream(rebuild(manifest, records))); + throw new AssertionError("Expected the collapsed shape to be refused"); + } catch (IOException expected) { + assertThat(expected).hasMessageThat().contains("collapses two attachments"); + } + } + + private static JsonObject readRecords(byte[] bundle) throws IOException { + return com.google.gson.JsonParser.parseString( + unzipToStrings(bundle).get(SyncBundleCodec.ENTRY_RECORDS)) + .getAsJsonObject(); + } + + private static JsonObject readManifest(byte[] bundle) throws IOException { + return com.google.gson.JsonParser.parseString( + unzipToStrings(bundle).get(SyncBundleCodec.ENTRY_MANIFEST)) + .getAsJsonObject(); + } + + /** Re-zips a bundle around edited records, refreshing the manifest checksum and length. */ + private static byte[] rebuild(byte[] bundle, JsonObject records) throws IOException { + return rebuild(readManifest(bundle), records); + } + + private static byte[] rebuild(JsonObject manifest, JsonObject records) throws IOException { + byte[] recordBytes = records.toString().getBytes(StandardCharsets.UTF_8); + manifest.addProperty("recordsSha256", SyncBundleValidator.sha256(recordBytes)); + manifest.addProperty("recordsBytes", recordBytes.length); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (java.util.zip.ZipOutputStream zip = + new java.util.zip.ZipOutputStream(output, StandardCharsets.UTF_8)) { + zip.putNextEntry(new ZipEntry(SyncBundleCodec.ENTRY_MANIFEST)); + zip.write(manifest.toString().getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + zip.putNextEntry(new ZipEntry(SyncBundleCodec.ENTRY_RECORDS)); + zip.write(recordBytes); + zip.closeEntry(); + } + return output.toByteArray(); + } + + @Test + public void decode_trimsADisplayNameSoEveryConsumerSeesTheNameThatWasValidated() + throws Exception { + SyncBundleCodec codec = new SyncBundleCodec(); + SyncRecord local = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:01Z"), + notePayload("Body", "application/pdf", 12L, "report.pdf ")); + + SyncRecord decoded = + codec.decode( + new ByteArrayInputStream( + codec.encode( + new SyncSnapshot( + java.util.Collections.singletonList(local)), + CREATED_AT))) + .getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID); + + // The validator judged the trimmed name; a consumer building a file name from the + // untrimmed one would have been refused by the URL parser. + JsonObject entry = + decoded.getPayload().getAsJsonArray("attachmentsManifest").get(0).getAsJsonObject(); + assertThat(entry.get("displayName").getAsString()).isEqualTo("report.pdf"); + assertThat( + decoded.getPayload() + .getAsJsonObject("attachmentNames") + .get(ATTACHMENT_ID) + .getAsString()) + .isEqualTo("report.pdf"); + } + private static SyncRecord task(String title) { JsonObject payload = new JsonObject(); payload.addProperty("title", title); diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java index f49f99ef..84298e21 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java @@ -249,13 +249,13 @@ public String getIdentifier() { } @Override - public SyncSnapshot readSnapshot() { - return snapshot; + public RemoteSnapshot readSnapshotResult() { + return RemoteSnapshot.of(snapshot); } @Override - public void writeSnapshot(SyncSnapshot snapshot) { - this.snapshot = snapshot; + public void publish(SyncPublication publication) { + this.snapshot = publication.getSnapshot(); } @Override diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java index 876fb0ec..22de4a23 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java @@ -74,8 +74,9 @@ public void insertNotes_insertsIdKeepingNotesBeforeReassignedOnes() { taken.setId(1); Note free = new Note().create("Beta", "body", 20L, ""); free.setId(2); - when(noteDao.getNoteSync(1)).thenReturn(new Note().create("Occupant", "", 5L, "")); - when(noteDao.getNoteSync(2)).thenReturn(null); + Note occupant = new Note().create("Occupant", "", 5L, ""); + occupant.setId(1); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(occupant)); when(noteDao.addNotes(org.mockito.ArgumentMatchers.anyList())) .thenReturn(new long[] {2L}) .thenReturn(new long[] {3L}); @@ -119,7 +120,7 @@ public void insertNotes_skipsANoteThisDeviceAlreadyHasUnchanged() { existing.setId(5); Note fromBackup = new Note().create("Title", "Body", 10L, "work"); fromBackup.setId(5); - when(noteDao.getNoteSync(5)).thenReturn(existing); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(existing)); coordinator.insertNotes(new java.util.ArrayList<>(java.util.List.of(fromBackup))); @@ -134,7 +135,7 @@ public void insertNotes_keepsADifferentNoteThatHappensToShareARowId() { existing.setId(5); Note fromBackup = new Note().create("Other", "Other body", 20L, ""); fromBackup.setId(5); - when(noteDao.getNoteSync(5)).thenReturn(existing); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(existing)); when(noteDao.addNotes(org.mockito.ArgumentMatchers.anyList())).thenReturn(new long[] {77L}); coordinator.insertNotes(new java.util.ArrayList<>(java.util.List.of(fromBackup))); @@ -303,7 +304,7 @@ public void insertNotes_keepsBackupIdsWhenNothingOccupiesThem() { Note restored = new Note().create("One", "1", 1L, ""); restored.setId(7); notes.add(restored); - when(noteDao.getNoteSync(7)).thenReturn(null); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of()); when(noteDao.addNotes(anyList())).thenReturn(new long[] {7L}); coordinator.insertNotes(notes); @@ -324,7 +325,7 @@ public void insertNotes_doesNotOverwriteAnExistingNoteThatHoldsTheSameId() { notes.add(restored); Note occupant = new Note().create("Already here", "x", 2L, ""); occupant.setId(7); - when(noteDao.getNoteSync(7)).thenReturn(occupant); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(occupant)); when(noteDao.addNotes(anyList())).thenReturn(new long[] {42L}); coordinator.insertNotes(notes); @@ -388,6 +389,202 @@ public void renameTag_touchesTagAndAllNotesChangedByRename() { .isEqualTo(1_000L); } + @Test + public void insertNotes_looksUpEveryIncomingIdInOneBatchInsteadOfOnePerNote() { + // Restoring a few thousand notes ran four point queries per note inside the transaction + // and sat on the restore dialog for tens of seconds. One IN (...) query per batch, and + // no lookup at all before the INSERT OR IGNORE of the metadata row. + List notes = new ArrayList<>(); + for (int id = 1; id <= 3; id++) { + Note note = new Note().create("Note " + id, "body", id, ""); + note.setId(id); + notes.add(note); + } + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of()); + when(noteDao.addNotes(anyList())).thenReturn(new long[] {1L, 2L, 3L}); + + coordinator.insertNotes(notes); + + verify(noteDao, org.mockito.Mockito.times(1)).getNotesByIdsSync(anyList()); + verify(noteDao, never()).getNoteSync(org.mockito.ArgumentMatchers.anyInt()); + assertThat(syncMetadataDao.get(SyncMetadata.RECORD_TYPE_NOTE, 3L)).isNotNull(); + } + + @Test + public void insertNotes_keepsBothNotesWhenTheBackupItselfRepeatsAnId() { + // A backup made by concatenating two exports can carry one id twice. addNotes is a + // REPLACE insert, so the second note used to overwrite the first inside the same call + // — no error, no duplicate, one note gone. + Note first = new Note().create("First", "body", 10L, ""); + first.setId(12); + Note second = new Note().create("Second", "body", 20L, ""); + second.setId(12); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of()); + when(noteDao.addNotes(anyList())).thenReturn(new long[] {12L}).thenReturn(new long[] {13L}); + + coordinator.insertNotes(new ArrayList<>(List.of(first, second))); + + verify(noteDao, org.mockito.Mockito.times(2)).addNotes(anyList()); + assertThat(first.getId()).isEqualTo(12); + assertThat(second.getId()).isEqualTo(13); + } + + @Test + public void insertTags_keepsBothTagsWhenTheBackupItselfRepeatsAnId() { + Tag first = new Tag().create("work"); + first.id = 4; + Tag second = new Tag().create("home"); + second.id = 4; + when(tagsDao.getTagsByIdsSync(anyList())).thenReturn(List.of()); + when(tagsDao.addTags(anyList())).thenReturn(new long[] {4L}).thenReturn(new long[] {5L}); + + coordinator.insertTags(new ArrayList<>(List.of(first, second))); + + assertThat(first.getId()).isEqualTo(4L); + assertThat(second.getId()).isEqualTo(5L); + } + + @Test + public void insertNotes_adoptsStagedAttachmentsEvenWhenTheIdIsKept() { + // The archive's files wait in a staging directory until the row exists; a note that + // keeps its id still has to have them moved into its folder, so the relocation runs for + // every restored note and the content is stored again only when it rewrote something. + List relocatedFrom = new ArrayList<>(); + SyncMutationCoordinator staging = + new SyncMutationCoordinator( + new SyncMutationCoordinator.TransactionExecutor() { + @Override + public T run( + SyncMutationCoordinator.TransactionCallable callable) { + return callable.call(); + } + }, + noteDao, + taskDao, + tagsDao, + taskCategoryDao, + transactions, + syncMetadataDao, + new FixedTimeProvider(1_000L), + new QueueStableIdGenerator("stable-a"), + (note, previousId) -> { + relocatedFrom.add(previousId); + note.setAttachments("[moved]"); + return true; + }); + Note restored = new Note().create("Kept", "body", 1L, ""); + restored.setId(7); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of()); + when(noteDao.addNotes(anyList())).thenReturn(new long[] {7L}); + + staging.insertNotes(new ArrayList<>(List.of(restored))); + + assertThat(relocatedFrom).containsExactly(7); + verify(noteDao) + .updateNoteContent( + eq(7), + eq("Kept"), + eq("body"), + org.mockito.ArgumentMatchers.any(), + eq(1L), + eq(""), + eq("[moved]")); + } + + @Test + public void insertNotes_adoptsStagedFilesForANoteThisDeviceAlreadyHas() { + // A row can exist with its files gone — restored from a JSON backup, or a cleared + // attachment folder. The archive is the only place the files are, and skipping the note + // as already present used to skip its files too; the staged copies were then thrown away + // at the next restore, after a "restore OK". + List relocatedFrom = new ArrayList<>(); + SyncMutationCoordinator staging = + new SyncMutationCoordinator( + new SyncMutationCoordinator.TransactionExecutor() { + @Override + public T run( + SyncMutationCoordinator.TransactionCallable callable) { + return callable.call(); + } + }, + noteDao, + taskDao, + tagsDao, + taskCategoryDao, + transactions, + syncMetadataDao, + new FixedTimeProvider(1_000L), + new QueueStableIdGenerator("stable-a"), + (note, previousId) -> { + relocatedFrom.add(previousId); + return false; + }); + Note existing = new Note().create("Title", "Body", 10L, "work"); + existing.setId(5); + Note fromBackup = new Note().create("Title", "Body", 10L, "work"); + fromBackup.setId(5); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(existing)); + + staging.insertNotes(new ArrayList<>(List.of(fromBackup))); + + verify(noteDao, never()).addNotes(anyList()); + assertThat(relocatedFrom).containsExactly(5); + // Nothing was rewritten, so the row is left exactly as it was. + verify(noteDao, never()) + .updateNoteContent( + org.mockito.ArgumentMatchers.anyInt(), + org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.anyLong(), + org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.any()); + } + + @Test + public void insertNotes_storesAPresentNoteAgainWhenAdoptionRewroteAReference() { + SyncMutationCoordinator staging = + new SyncMutationCoordinator( + new SyncMutationCoordinator.TransactionExecutor() { + @Override + public T run( + SyncMutationCoordinator.TransactionCallable callable) { + return callable.call(); + } + }, + noteDao, + taskDao, + tagsDao, + taskCategoryDao, + transactions, + syncMetadataDao, + new FixedTimeProvider(1_000L), + new QueueStableIdGenerator("stable-a"), + (note, previousId) -> { + note.setAttachments("[renamed]"); + return true; + }); + Note existing = new Note().create("Title", "Body", 10L, "work"); + existing.setId(5); + Note fromBackup = new Note().create("Title", "Body", 10L, "work"); + fromBackup.setId(5); + when(noteDao.getNotesByIdsSync(anyList())).thenReturn(List.of(existing)); + + staging.insertNotes(new ArrayList<>(List.of(fromBackup))); + + verify(noteDao) + .updateNoteContent( + eq(5), + eq("Title"), + eq("Body"), + org.mockito.ArgumentMatchers.any(), + eq(10L), + eq("work"), + eq("[renamed]")); + // The row changed, so sync has to publish it. + assertThat(syncMetadataDao.get(SyncMetadata.RECORD_TYPE_NOTE, 5L)).isNotNull(); + } + private static final class FixedTimeProvider implements SyncMutationCoordinator.TimeProvider { private final long value; @@ -446,6 +643,15 @@ public boolean exists(String recordType, long localId) { return rows.containsKey(key(recordType, localId)); } + @Override + public List getExistingLocalIds(String recordType, List localIds) { + List existing = new ArrayList<>(); + for (Long localId : localIds) { + if (exists(recordType, localId)) existing.add(localId); + } + return existing; + } + @Override public SyncMetadataEntity getByStableId(String recordType, String stableId) { for (SyncMetadataEntity value : rows.values()) { diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncRecordTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncRecordTest.java new file mode 100644 index 00000000..99db5a2a --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncRecordTest.java @@ -0,0 +1,45 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.time.Instant; +import org.junit.Test; + +public class SyncRecordTest { + + private static final String NOTE_ID = "550e8400-e29b-41d4-a716-446655440000"; + private static final Instant AT = Instant.parse("2026-08-31T12:00:01Z"); + + @Test + public void aNoteWithEmptyAttachmentFieldsHashesLikeOneWithout() { + // The local build, the encoder and the decoder each enforced "absent when empty" on + // their own, and every time one drifted the same note hashed differently on the two + // sides and conflicted with itself on every sync. The record is the one place now. + JsonObject bare = new JsonObject(); + bare.addProperty("b", "Shopping"); + JsonObject withEmpties = bare.deepCopy(); + withEmpties.add("attachmentsManifest", new JsonArray()); + withEmpties.add("attachmentHashes", new JsonArray()); + withEmpties.add("attachmentNames", new JsonObject()); + + SyncRecord plain = SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, AT, bare); + SyncRecord normalized = SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, AT, withEmpties); + + assertThat(normalized.getCanonicalPayloadHash()).isEqualTo(plain.getCanonicalPayloadHash()); + assertThat(normalized.getPayload().has("attachmentNames")).isFalse(); + } + + @Test + public void populatedAttachmentFieldsAreKept() { + JsonObject payload = new JsonObject(); + JsonArray hashes = new JsonArray(); + hashes.add("d6f1f3d5d8cf9b5a4a2469787998dc45eb59f401b93b1b4cde4998dc409ebdc8"); + payload.add("attachmentHashes", hashes); + + SyncRecord record = SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, AT, payload); + + assertThat(record.getPayload().getAsJsonArray("attachmentHashes")).hasSize(1); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java index 7f7f40d5..a406d0b3 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java @@ -305,6 +305,107 @@ public void sync_replacesCorruptLocalAttachmentWithVerifiedRemoteBlob() throws E assertThat(backend.writeSnapshotCalls).isEqualTo(1); } + @Test + public void runWhileNoSyncRuns_waitsForTheSyncInFlightToFinish() throws Exception { + // Disconnect wipes state, conflicts and the blob cache. Done concurrently with the + // six-hourly worker it deleted the cache under the worker, which then wrote the old + // account's state and conflicts back after the wipe. + FakeStore store = new FakeStore(snapshot(note(TEN, "Local"))); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + backend.readStarted = new java.util.concurrent.CountDownLatch(1); + backend.readReleased = new java.util.concurrent.CountDownLatch(1); + List order = Collections.synchronizedList(new ArrayList<>()); + Thread syncing = + new Thread( + () -> { + new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + order.add("sync finished"); + }); + syncing.start(); + assertThat(backend.readStarted.await(5, java.util.concurrent.TimeUnit.SECONDS)).isTrue(); + + Thread clearing = + new Thread(() -> SyncService.runWhileNoSyncRuns(() -> order.add("cleared"))); + clearing.start(); + Thread.sleep(200L); + + // The sync still holds the lock, so the wipe has not run. + assertThat(order).isEmpty(); + backend.readReleased.countDown(); + syncing.join(5_000L); + clearing.join(5_000L); + assertThat(order).containsExactly("sync finished", "cleared").inOrder(); + assertThat(store.appliedSnapshot).isNotNull(); + } + + @Test + public void sync_pinsConflictBlobsWithoutTransferringOrVerifyingThemAgain() throws Exception { + byte[] bytes = "shared attachment".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + SyncRecord local = noteWithAttachment(TEN, "Local wording", hash, bytes.length); + SyncRecord remote = noteWithAttachment(TWENTY, "Remote wording", hash, bytes.length); + FakeStore store = new FakeStore(snapshot(local)); + store.attachmentHashes = Collections.singletonList(hash); + store.attachments.put(hash, bytes); + FakeBackend backend = new FakeBackend(snapshot(remote)); + backend.attachments.put(hash, bytes); + + SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); + assertThat(state.getConflictCount()).isEqualTo(1); + // The blob was verified once for the merged snapshot. Pinning the winner and the loser + // used to run the whole transfer again for each and copy the cached blob onto itself, + // so an open conflict on a large note cost hundreds of megabytes per sync. + assertThat(java.util.Collections.frequency(backend.events, "readAttachment")).isEqualTo(1); + assertThat(java.util.Collections.frequency(store.events, "writeAttachment")).isEqualTo(1); + } + + @Test + public void sync_refusesToStartOnceSyncWasTurnedOffEvenWithTheLockInHand() { + // Disconnect turns sync off and wipes the account's state under the lock; a worker that + // took the lock afterwards wrote the old account's state back. Nothing may be persisted. + FakeStore store = new FakeStore(snapshot(note(TEN, "Local"))); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + + SyncState state = + new SyncService(store, new SyncMerger(), CLOCK).sync(backend, () -> false); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.ERROR); + assertThat(state.getErrorMessage()).contains("turned off"); + assertThat(store.states).isEmpty(); + assertThat(store.applyCalls).isEqualTo(0); + assertThat(backend.events).isEmpty(); + } + + @Test + public void sync_namesTheNoteWhenABlobNoEndpointHoldsWasPublishedFromMemory() throws Exception { + // The store described the attachment from the hash and size its column remembered, + // expecting the bytes back from Drive. When Drive has nothing either — a different + // account, say — the failure has to name the note, as the build failure it replaced did, + // not leave the user with a hash and a sync that fails forever. + byte[] bytes = "gone everywhere".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + FakeStore store = + new FakeStore(snapshot(noteWithAttachment(TEN, "Body", hash, bytes.length))); + store.attachmentHashes = Collections.singletonList(hash); + store.rememberedOnly = + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + SyncMetadata.RECORD_TYPE_NOTE, + NOTE_ID, + "Shopping list"); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + + SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.ERROR); + assertThat(state.getErrorMessage()) + .isEqualTo( + "Local snapshot is incomplete: MISSING_ATTACHMENT in note \"Shopping list\""); + assertThat(backend.writeSnapshotCalls).isEqualTo(0); + } + private static SyncSnapshot snapshot(SyncRecord... records) { return new SyncSnapshot(Arrays.asList(records)); } @@ -402,6 +503,10 @@ private static final class FakeStore implements SyncStore { private List appliedConflicts = Collections.emptyList(); private SyncState state = SyncState.idle(); private final Map attachments = new HashMap<>(); + + /** Blobs written through writeAttachment: the durable cache RoomSyncStore keeps. */ + private final java.util.Set durable = new java.util.HashSet<>(); + private Collection attachmentHashes = Collections.emptyList(); private final List states = new ArrayList<>(); private final List events = new ArrayList<>(); @@ -458,6 +563,20 @@ public void writeAttachment(String sha256, long sizeBytes, InputStream content) throws IOException { events.add("writeAttachment"); attachments.put(sha256, readAll(content)); + durable.add(sha256); + } + + @Override + public boolean hasDurableAttachment(String sha256, long sizeBytes) { + return durable.contains(sha256); + } + + /** Set when the store described an attachment from remembered metadata only. */ + private SnapshotProblem rememberedOnly; + + @Override + public SnapshotProblem describeMissingAttachment(String sha256) { + return rememberedOnly; } @Override @@ -486,6 +605,11 @@ private static final class FakeBackend implements SyncBackend { private IOException readFailure; private int writeSnapshotCalls; + /** When set, the remote read announces itself and then waits to be released. */ + private java.util.concurrent.CountDownLatch readStarted; + + private java.util.concurrent.CountDownLatch readReleased; + FakeBackend(SyncSnapshot snapshot) { this.snapshot = snapshot; } @@ -496,19 +620,28 @@ public String getIdentifier() { } @Override - public SyncSnapshot readSnapshot() throws IOException { + public RemoteSnapshot readSnapshotResult() throws IOException { // Recorded so a test asserting "the remote was never read" actually proves it. events.add("readSnapshot"); if (readFailure != null) { throw readFailure; } - return snapshot; + if (readStarted != null) { + readStarted.countDown(); + try { + readReleased.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new java.io.InterruptedIOException(); + } + } + return RemoteSnapshot.of(snapshot); } @Override - public void writeSnapshot(SyncSnapshot snapshot) { + public void publish(SyncPublication publication) { events.add("writeSnapshot"); - this.snapshot = snapshot; + this.snapshot = publication.getSnapshot(); writeSnapshotCalls++; } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/VerifyingInputStreamTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/VerifyingInputStreamTest.java new file mode 100644 index 00000000..18165493 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/VerifyingInputStreamTest.java @@ -0,0 +1,103 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.junit.Test; + +/** + * The single verifier every attachment transfer goes through. + * + *

Four implementations used to disagree on the details pinned here — one had no size ceiling, + * one returned false where the others threw — so a blob accepted on one path could be refused on + * the next. + */ +public class VerifyingInputStreamTest { + + private static final byte[] BYTES = "attachment bytes".getBytes(StandardCharsets.UTF_8); + private static final String HASH = Sha256.of(BYTES); + + @Test + public void acceptsTheDeclaredBytesAndReportsTheirSize() throws Exception { + long size = + VerifyingInputStream.verify( + new ByteArrayInputStream(BYTES), HASH, (long) BYTES.length); + + assertThat(size).isEqualTo(BYTES.length); + } + + @Test + public void rejectsAChecksumMismatch() { + AttachmentIntegrityException failure = + expectIntegrityFailure(new ByteArrayInputStream(BYTES), Sha256.of("other"), null); + + assertThat(failure).hasMessageThat().contains("checksum"); + } + + @Test + public void rejectsASizeMismatch() { + AttachmentIntegrityException failure = + expectIntegrityFailure(new ByteArrayInputStream(BYTES), HASH, 3L); + + assertThat(failure).hasMessageThat().contains("size does not match"); + } + + @Test + public void rejectsABlobPastTheCeilingAsAnIntegrityFailure() throws Exception { + // A plain IOException here escaped the "skip a corrupt candidate" path in the Drive + // backend and failed the whole sync on one oversized object. + VerifyingInputStream verifying = + new VerifyingInputStream(new ByteArrayInputStream(BYTES), HASH, null, 4L); + try { + verifying.verifyEndOfStream(); + throw new AssertionError("Expected the ceiling to be enforced"); + } catch (AttachmentIntegrityException expected) { + assertThat(expected).hasMessageThat().contains("exceeds the sync size limit"); + } + } + + @Test + public void drainsWhatTheDestinationLeftUnreadBeforeJudging() throws Exception { + // A destination that stopped early must not be able to accept a blob it never finished + // checking: the tail is read here and the mismatch surfaces. + byte[] longer = "attachment bytes plus more".getBytes(StandardCharsets.UTF_8); + VerifyingInputStream verifying = + new VerifyingInputStream(new ByteArrayInputStream(longer), HASH, null); + byte[] buffer = new byte[BYTES.length]; + assertThat(verifying.read(buffer, 0, buffer.length)).isEqualTo(BYTES.length); + + try { + verifying.verifyEndOfStream(); + throw new AssertionError("Expected the drained tail to fail the checksum"); + } catch (AttachmentIntegrityException expected) { + assertThat(verifying.bytesRead()).isEqualTo(longer.length); + } + } + + @Test + public void verifyingTwiceIsANoOp() throws Exception { + VerifyingInputStream verifying = + new VerifyingInputStream( + new ByteArrayInputStream(BYTES), HASH, (long) BYTES.length); + while (verifying.read(new byte[8]) != -1) { + // Reaches end of stream, which verifies once inside the read. + } + + verifying.verifyEndOfStream(); + verifying.verifyEndOfStream(); + } + + private static AttachmentIntegrityException expectIntegrityFailure( + ByteArrayInputStream input, String hash, Long size) { + try { + VerifyingInputStream.verify(input, hash, size); + } catch (AttachmentIntegrityException expected) { + return expected; + } catch (IOException unexpected) { + throw new AssertionError(unexpected); + } + throw new AssertionError("Expected an integrity failure"); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocksTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocksTest.java new file mode 100644 index 00000000..e26908ff --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/EditorAttachmentBlocksTest.java @@ -0,0 +1,84 @@ +package com.pasich.mynotes.extendedEditor.attach; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gson.JsonObject; +import org.junit.Test; + +/** + * The one walk over an Editor.js document that finds attachment references. + * + *

Three walkers with three rule sets used to exist; the same note could relocate correctly after + * a ZIP restore and show broken links after a Drive restore depending on which one ran. + */ +public class EditorAttachmentBlocksTest { + + private static final String DOCUMENT = + "[{\"id\":\"p\",\"type\":\"paragraph\",\"data\":{\"text\":\"hello there\"}}," + + "{\"id\":\"a\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_5/one.pdf\",\"name\":\"one.pdf\"}}}," + + "{\"id\":\"g\",\"type\":\"gallery\",\"data\":{\"files\":[{\"url\":\"editorjs://attachments/note_5/two.png\"},{\"url\":\"editorjs://attachments/note_5/three.png\"}]}}," + + "{\"id\":\"v\",\"type\":\"video\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_5/four.mp4\"}}}]"; + + @Test + public void rewritesEveryFileUrlWhateverTheBlockType() { + String rewritten = + EditorAttachmentBlocks.rewriteUrls( + DOCUMENT, url -> url.replace("note_5", "note_9")); + + // The old sync walker filtered on the attaches and image types and never looked at + // files[]; the gallery and video blocks stayed pointing at the sending device's files. + assertThat(rewritten).doesNotContain("note_5"); + assertThat(EditorAttachmentBlocks.fileUrls(rewritten)) + .containsExactly( + "editorjs://attachments/note_9/one.pdf", + "editorjs://attachments/note_9/two.png", + "editorjs://attachments/note_9/three.png", + "editorjs://attachments/note_9/four.mp4") + .inOrder(); + // Text is carried verbatim: the paragraph's markup is not HTML-escaped on the way through. + assertThat(rewritten).contains("hello there"); + } + + @Test + public void returnsTheDocumentItselfWhenNothingChanges() { + // Identity, not equality: a note nothing touched must stay byte-identical on every + // device, and a re-serialization would already have been a different string. + assertThat(EditorAttachmentBlocks.rewriteUrls(DOCUMENT, url -> null)) + .isSameInstanceAs(DOCUMENT); + assertThat(EditorAttachmentBlocks.rewriteUrls(DOCUMENT, url -> url)) + .isSameInstanceAs(DOCUMENT); + } + + @Test + public void leavesAnUnreadableDocumentAlone() { + String broken = "[{\"type\":\"attaches\",\"data\":"; + + assertThat(EditorAttachmentBlocks.rewriteUrls(broken, url -> "x")).isSameInstanceAs(broken); + assertThat(EditorAttachmentBlocks.fileUrls(broken)).isEmpty(); + assertThat(EditorAttachmentBlocks.rewriteUrls(null, url -> "x")).isNull(); + } + + @Test + public void rewritingIsIdempotent() { + String once = EditorAttachmentBlocks.rewriteUrls(DOCUMENT, url -> url + "?v=2"); + String again = EditorAttachmentBlocks.rewriteUrls(once, url -> url); + + // Sync hashes the rewritten document on both devices, so re-parsing and re-serializing + // the output has to reproduce it exactly. + assertThat(again).isSameInstanceAs(once); + assertThat(EditorAttachmentBlocks.rewriteUrls(once, url -> url.replace("?v=2", "?v=2"))) + .isSameInstanceAs(once); + } + + @Test + public void findsTheFileOfABlockInEitherShape() { + JsonObject single = EditorAttachmentBlocks.findFile(DOCUMENT, "a"); + JsonObject fromList = EditorAttachmentBlocks.findFile(DOCUMENT, "g"); + + assertThat(single.get("name").getAsString()).isEqualTo("one.pdf"); + assertThat(fromList.get("url").getAsString()) + .isEqualTo("editorjs://attachments/note_5/two.png"); + assertThat(EditorAttachmentBlocks.findFile(DOCUMENT, "p")).isNull(); + assertThat(EditorAttachmentBlocks.findFile(DOCUMENT, "missing")).isNull(); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java index 7b51a176..3acd4de6 100644 --- a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java @@ -65,6 +65,130 @@ public void rewritesTheEditorBlocksThatCarryTheSameUrls() throws Exception { assertThat(result.valueJson).doesNotContain("note_5"); } + @Test + public void adoptsStagedFilesIntoTheFolderOfTheIdTheNoteKept() throws Exception { + File staging = temporaryFolder.newFolder("staging"); + File staged = stage(staging, 5, "photo.jpg", "from the archive"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 5, 5, attachmentsJson(5, "photo.jpg"), null); + + // Same id, same name: the reference is already right, only the bytes had to move. + assertThat(result.changed).isFalse(); + File adopted = new File(new File(root, "note_5"), "photo.jpg"); + assertThat(contentOf(adopted)).isEqualTo("from the archive"); + assertThat(staged.exists()).isFalse(); + } + + @Test + public void adoptsStagedFilesIntoTheFolderOfAReassignedId() throws Exception { + File staging = temporaryFolder.newFolder("staging"); + stage(staging, 5, "photo.jpg", "from the archive"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 5, 12, attachmentsJson(5, "photo.jpg"), null); + + assertThat(result.changed).isTrue(); + assertThat(result.attachmentsJson).contains("note_12"); + assertThat(contentOf(new File(new File(root, "note_12"), "photo.jpg"))) + .isEqualTo("from the archive"); + assertThat(new File(root, "note_5").exists()).isFalse(); + } + + @Test + public void neverOverwritesAFileAnotherNoteAlreadyShows() throws Exception { + // Extraction used to land straight in the live folder, so a foreign archive replaced the + // bytes of whichever local note shared the row id. The existing file stays; the staged + // one is adopted under a fresh name and the reference follows it. + File mine = seed(7, "photo.jpg", "mine"); + File staging = temporaryFolder.newFolder("staging"); + stage(staging, 7, "photo.jpg", "foreign"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 7, 7, attachmentsJson(7, "photo.jpg"), null); + + assertThat(contentOf(mine)).isEqualTo("mine"); + assertThat(result.changed).isTrue(); + assertThat(result.attachmentsJson).doesNotContain("note_7/photo.jpg"); + File[] files = new File(root, "note_7").listFiles(); + assertThat(files).hasLength(2); + String adoptedName = + files[0].getName().equals("photo.jpg") ? files[1].getName() : files[0].getName(); + assertThat(adoptedName).startsWith("photo-"); + assertThat(adoptedName).endsWith(".jpg"); + assertThat(result.attachmentsJson).contains("note_7/" + adoptedName); + } + + @Test + public void columnAndBlocksAgreeOnTheAdoptedNameWhenAFileCollides() throws Exception { + // The mover runs once for the column and once for the same URL in the blocks. Picking a + // fresh name on each call made two copies and pointed column and blocks at different + // files; the cleaner then deleted the one the column did not list. + seed(7, "photo.jpg", "mine"); + File staging = temporaryFolder.newFolder("staging"); + stage(staging, 7, "photo.jpg", "foreign"); + String blocks = + "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + AttachmentStorage.urlFor(7, "photo.jpg") + + "\",\"name\":\"photo.jpg\"}}}]"; + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 7, 7, attachmentsJson(7, "photo.jpg"), blocks); + + String columnUrl = + com.google.gson.JsonParser.parseString(result.attachmentsJson) + .getAsJsonArray() + .get(0) + .getAsJsonObject() + .get("url") + .getAsString(); + assertThat(EditorAttachmentBlocks.fileUrls(result.valueJson)).containsExactly(columnUrl); + // Exactly two files: the one that was there and the one adopted, no second copy. + assertThat(new File(root, "note_7").listFiles()).hasLength(2); + } + + @Test + public void reusesAnIdenticalFileInsteadOfDuplicatingIt() throws Exception { + seed(7, "photo.jpg", "same bytes"); + File staging = temporaryFolder.newFolder("staging"); + stage(staging, 7, "photo.jpg", "same bytes"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 7, 7, attachmentsJson(7, "photo.jpg"), null); + + assertThat(result.changed).isFalse(); + assertThat(new File(root, "note_7").listFiles()).hasLength(1); + } + + @Test + public void fallsBackToTheLiveFolderWhenNothingWasStaged() throws Exception { + // A note restored by a release that extracted into the live folders still relocates. + seed(5, "photo.jpg", "already live"); + File staging = temporaryFolder.newFolder("staging"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.adoptStaged( + staging, root, 5, 12, attachmentsJson(5, "photo.jpg"), null); + + assertThat(result.changed).isTrue(); + assertThat(contentOf(new File(new File(root, "note_12"), "photo.jpg"))) + .isEqualTo("already live"); + } + + private static File stage(File staging, int noteId, String name, String content) + throws Exception { + File folder = new File(staging, "note_" + noteId); + assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); + File file = new File(folder, name); + Files.write(file.toPath(), content.getBytes(StandardCharsets.UTF_8)); + return file; + } + @Test public void leavesReferencesThatBelongToAnotherNoteAlone() throws Exception { seed(7, "other.png", "not mine"); diff --git a/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java b/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java new file mode 100644 index 00000000..0df693b0 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/utils/backup/local/BackupFileValidatorTest.java @@ -0,0 +1,103 @@ +package com.pasich.mynotes.utils.backup.local; + +import static com.google.common.truth.Truth.assertThat; + +import com.pasich.mynotes.utils.constants.Backup; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.Test; + +/** + * Whether a picked document is accepted as a backup. + * + *

Android's document picker appends " (2)" to a second backup saved into the same folder, so + * judging by the extension alone made the app refuse its own file — reproduced on a Pixel 7a. + */ +public class BackupFileValidatorTest { + + @Test + public void acceptsABackupArchiveWhateverThePickerCalledIt() throws Exception { + byte[] archive = archiveWith(Backup.FILE_NAME_BACKUP); + + assertThat( + BackupFileValidator.isAcceptable( + "My_Notes_Backup.mnbkn (2)", + () -> new ByteArrayInputStream(archive))) + .isTrue(); + assertThat( + BackupFileValidator.isAcceptable( + "anything at all", () -> new ByteArrayInputStream(archive))) + .isTrue(); + } + + @Test + public void stillAcceptsTheKnownExtensionsWithoutOpeningTheDocument() { + // The legacy non-archive formats can only be judged by name. + assertThat( + BackupFileValidator.isAcceptable( + "old.json", + () -> { + throw new AssertionError("must not open"); + })) + .isTrue(); + assertThat(BackupFileValidator.isAcceptable("Backup.MNBKN", () -> null)).isTrue(); + } + + @Test + public void refusesADocumentThatIsNeitherByNameNorByContent() throws Exception { + byte[] otherArchive = archiveWith("notes.txt"); + + assertThat( + BackupFileValidator.isAcceptable( + "photo.jpg", + () -> + new ByteArrayInputStream( + "not a zip".getBytes(StandardCharsets.UTF_8)))) + .isFalse(); + assertThat( + BackupFileValidator.isAcceptable( + "other.zip (2)", () -> new ByteArrayInputStream(otherArchive))) + .isFalse(); + assertThat(BackupFileValidator.isAcceptable("missing (2)", () -> null)).isFalse(); + } + + @Test + public void givesUpOnALargeContainerInsteadOfInflatingItToTheEnd() throws Exception { + // A wrong pick that happens to be a big ZIP container was inflated entry by entry, on the + // main thread, looking for the JSON; the check now stops once it has inflated its budget. + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(bytes)) { + zip.putNextEntry(new ZipEntry("assets/big.bin")); + zip.write(new byte[512 * 1024]); + zip.closeEntry(); + zip.putNextEntry(new ZipEntry(Backup.FILE_NAME_BACKUP)); + zip.write("{}".getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + + assertThat( + BackupFileValidator.isBackupArchive( + new ByteArrayInputStream(bytes.toByteArray()), 64 * 1024)) + .isFalse(); + assertThat( + BackupFileValidator.isBackupArchive( + new ByteArrayInputStream(bytes.toByteArray()))) + .isTrue(); + } + + private static byte[] archiveWith(String entryName) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(bytes)) { + zip.putNextEntry(new ZipEntry("attachments/note_1/photo.jpg")); + zip.write(new byte[] {1, 2, 3}); + zip.closeEntry(); + zip.putNextEntry(new ZipEntry(entryName)); + zip.write("{}".getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + return bytes.toByteArray(); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java b/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java new file mode 100644 index 00000000..24c7e3e7 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelperTest.java @@ -0,0 +1,119 @@ +package com.pasich.mynotes.utils.backup.local; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gson.Gson; +import com.pasich.mynotes.utils.backup.models.JsonBackup; +import com.pasich.mynotes.utils.constants.Backup; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Reading a backup archive must not be able to change a note that already exists. + * + *

Attachments used to be extracted straight into the live {@code attachments/note_} folders, + * before the JSON was even validated and before the restore had decided whether that id belonged to + * some other note already on the device. + */ +public class ZipBackupHelperTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void extractsIntoTheStagingDirectoryAndNeverIntoTheLiveFolders() throws Exception { + File live = temporaryFolder.newFolder("files", "attachments", "note_7"); + File mine = new File(live, "photo.jpg"); + Files.write(mine.toPath(), "mine".getBytes(StandardCharsets.UTF_8)); + File staging = new File(temporaryFolder.getRoot(), "cache/restore-staging/attachments"); + Map entries = new LinkedHashMap<>(); + entries.put( + Backup.FILE_NAME_BACKUP, + new Gson().toJson(new JsonBackup()).getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/note_7/photo.jpg", "foreign".getBytes(StandardCharsets.UTF_8)); + + JsonBackup backup = ZipBackupHelper.readZipBackup(zip(entries), staging); + + assertThat(backup.isError()).isFalse(); + assertThat(new String(Files.readAllBytes(mine.toPath()), StandardCharsets.UTF_8)) + .isEqualTo("mine"); + File staged = new File(new File(staging, "note_7"), "photo.jpg"); + assertThat(new String(Files.readAllBytes(staged.toPath()), StandardCharsets.UTF_8)) + .isEqualTo("foreign"); + } + + @Test + public void skipsEveryEntryThatIsNotOneFileInOneNoteFolder() throws Exception { + // The archive is untrusted. "note_5/../note_6/x" lands in a folder the note's JSON never + // names, and "evil/x" or "note_5/sub/x" are places the cleaner never looks, so they would + // be written once and never reclaimed. + File staging = new File(temporaryFolder.getRoot(), "staging/attachments"); + Map entries = new LinkedHashMap<>(); + entries.put( + Backup.FILE_NAME_BACKUP, + new Gson().toJson(new JsonBackup()).getBytes(StandardCharsets.UTF_8)); + entries.put( + "attachments/note_5/../note_6/x.jpg", "traversal".getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/evil/x.jpg", "foreign folder".getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/note_5/sub/x.jpg", "nested".getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/../../databases/notes", "escape".getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/note_5/ok.jpg", "kept".getBytes(StandardCharsets.UTF_8)); + + JsonBackup backup = ZipBackupHelper.readZipBackup(zip(entries), staging); + + assertThat(backup.isError()).isFalse(); + assertThat(new File(staging, "note_5/ok.jpg").isFile()).isTrue(); + assertThat(new File(staging, "note_6/x.jpg").exists()).isFalse(); + assertThat(new File(staging, "evil").exists()).isFalse(); + assertThat(new File(staging, "note_5/sub").exists()).isFalse(); + assertThat(new File(temporaryFolder.getRoot(), "databases").exists()).isFalse(); + assertThat(new File(temporaryFolder.getRoot(), "staging/databases").exists()).isFalse(); + } + + @Test + public void anArchiveWithoutTheJsonIsAnError() throws Exception { + File staging = new File(temporaryFolder.getRoot(), "staging/attachments"); + Map entries = new LinkedHashMap<>(); + entries.put("attachments/note_5/ok.jpg", "kept".getBytes(StandardCharsets.UTF_8)); + + assertThat(ZipBackupHelper.readZipBackup(zip(entries), staging).isError()).isTrue(); + } + + @Test + public void refusesAnArchiveThatUnpacksPastTheCeiling() throws Exception { + // ZIP compresses a run of zeros a thousandfold; an archive is untrusted input and a + // small one could fill the device. + File staging = new File(temporaryFolder.getRoot(), "staging/attachments"); + Map entries = new LinkedHashMap<>(); + entries.put( + Backup.FILE_NAME_BACKUP, + new Gson().toJson(new JsonBackup()).getBytes(StandardCharsets.UTF_8)); + entries.put("attachments/note_5/zeros.bin", new byte[64 * 1024]); + + JsonBackup backup = ZipBackupHelper.readZipBackup(zip(entries), staging, 16 * 1024); + + assertThat(backup.isError()).isTrue(); + } + + private static ZipInputStream zip(Map entries) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(bytes)) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue()); + zip.closeEntry(); + } + } + return new ZipInputStream(new ByteArrayInputStream(bytes.toByteArray())); + } +}