From 3f96485c76a32ed3d46a88860d232a88e29a647a Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 11:24:26 +0300 Subject: [PATCH 01/16] fix: harden Google Drive sync --- .../pasich/mynotes/db/RoomSyncStoreTest.java | 270 +++++++++++++ .../pasich/mynotes/data/AppDataManager.java | 4 +- .../mynotes/data/database/AppDbHelper.java | 4 +- .../data/database/dao/SyncStateDao.java | 4 + .../data/database/helpers/DbNotesHelper.java | 2 +- .../data/sync/GoogleDriveSyncBackend.java | 374 ++++++++++++------ .../data/sync/GoogleDriveSyncWorker.java | 24 +- .../mynotes/data/sync/RoomSyncStore.java | 125 +++++- .../pasich/mynotes/data/sync/SyncBackend.java | 9 +- .../mynotes/data/sync/SyncBundleCodec.java | 10 + .../mynotes/data/sync/SyncMetadata.java | 32 ++ .../data/sync/SyncMutationCoordinator.java | 32 ++ .../pasich/mynotes/data/sync/SyncRollout.java | 49 +++ .../pasich/mynotes/data/sync/SyncService.java | 166 ++++++-- .../pasich/mynotes/data/sync/SyncStore.java | 9 +- .../mynotes/ui/presenter/MainPresenter.java | 2 +- .../dialogs/MoreNoteDialogPresenter.java | 3 +- .../mynotes/ui/sync/SyncCoordinator.java | 62 ++- .../ui/sync/SyncCoordinatorFactory.java | 32 +- .../ui/view/activity/BackupActivity.java | 145 ++++--- .../shareProcessors/SharedNoteCreator.java | 2 +- app/src/main/res/values-be/strings.xml | 1 + app/src/main/res/values-de/strings.xml | 1 + app/src/main/res/values-en-rGB/strings.xml | 1 + app/src/main/res/values-es/strings.xml | 1 + app/src/main/res/values-fr/strings.xml | 1 + app/src/main/res/values-it/strings.xml | 1 + app/src/main/res/values-kk/strings.xml | 1 + app/src/main/res/values-pl/strings.xml | 1 + app/src/main/res/values-ru/strings.xml | 1 + app/src/main/res/values-uk/strings.xml | 1 + app/src/main/res/values/strings.xml | 1 + .../data/sync/GoogleDriveSyncBackendTest.java | 6 +- .../data/sync/GoogleDriveSyncWorkerTest.java | 18 + .../data/sync/SyncBundleCodecTest.java | 86 ++++ .../data/sync/SyncConvergenceTest.java | 5 +- .../mynotes/data/sync/SyncMetadataTest.java | 37 ++ .../sync/SyncMutationCoordinatorTest.java | 59 +++ .../mynotes/data/sync/SyncRolloutTest.java | 46 +++ .../mynotes/data/sync/SyncServiceTest.java | 33 +- .../mynotes/ui/sync/SyncCoordinatorTest.java | 51 +++ 41 files changed, 1427 insertions(+), 285 deletions(-) create mode 100644 app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/SyncRollout.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/SyncRolloutTest.java diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java new file mode 100644 index 00000000..81d0c722 --- /dev/null +++ b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java @@ -0,0 +1,270 @@ +package com.pasich.mynotes.db; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; + +import android.content.Context; +import androidx.room.Room; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import com.google.gson.JsonObject; +import com.pasich.mynotes.data.database.AppDatabase; +import com.pasich.mynotes.data.database.entities.SyncMetadataEntity; +import com.pasich.mynotes.data.model.Note; +import com.pasich.mynotes.data.preferences.PreferenceHelper; +import com.pasich.mynotes.data.sync.RoomSyncStore; +import com.pasich.mynotes.data.sync.SyncMetadata; +import com.pasich.mynotes.data.sync.SyncRecord; +import com.pasich.mynotes.data.sync.SyncSnapshot; +import com.pasich.mynotes.data.sync.SyncState; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Collections; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Integration coverage for the sync store, which needs a real Room database and a real files + * directory. The pure protocol classes are unit-tested; everything here is the part that only + * behaves correctly against actual storage. + */ +@RunWith(AndroidJUnit4.class) +public class RoomSyncStoreTest { + + private Context context; + private AppDatabase db; + private RoomSyncStore store; + + @Before + public void setUp() { + context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + db = + Room.inMemoryDatabaseBuilder(context, AppDatabase.class) + .allowMainThreadQueries() + .build(); + store = new RoomSyncStore(context, db, mock(PreferenceHelper.class)); + deleteRecursively(new File(context.getFilesDir(), "sync-attachments")); + deleteRecursively(new File(context.getFilesDir(), "attachments")); + } + + @After + public void tearDown() { + db.close(); + deleteRecursively(new File(context.getFilesDir(), "sync-attachments")); + deleteRecursively(new File(context.getFilesDir(), "attachments")); + } + + @Test + public void readSnapshot_keepsLocalPrimaryKeysAndFilePathsOffTheWire() throws Exception { + int noteId = seedNote("Shopping", "Milk", null); + + SyncRecord record = onlyNote(store.readSnapshot()); + + JsonObject payload = record.getPayload(); + // "a" is Note.id and "h" is the attachments JSON of file:// paths. Both differ per device, + // so leaving them in makes two devices hash the same logical note differently and report a + // conflict on every sync forever. + assertThat(payload.has("a")).isFalse(); + assertThat(payload.has("h")).isFalse(); + assertThat(payload.get("b").getAsString()).isEqualTo("Shopping"); + assertThat(noteId).isGreaterThan(0); + } + + @Test + public void hasAttachment_findsABlobThatOnlyExistsInTheNotesOwnFolder() throws Exception { + byte[] bytes = "receipt bytes".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + int noteId = seedNoteWithAttachment("receipt.png", bytes); + + // The index is built while the snapshot is read, which is what every sync does first. + store.readSnapshot(); + + // Before this, only sync-attachments/ was consulted and nothing but the download path ever + // wrote there. On the device that owns the file the lookup failed, SyncService asked the + // backend for a blob nobody had uploaded, and every sync for that account aborted. + assertThat(store.hasAttachment(hash)).isTrue(); + try (InputStream in = store.readAttachment(hash)) { + assertThat(readAll(in)).isEqualTo(bytes); + } + assertThat(noteId).isGreaterThan(0); + } + + @Test + public void writeAttachment_leavesNothingBehindWhenTheStreamFails() { + String hash = sha256("whatever".getBytes(StandardCharsets.UTF_8)); + InputStream failing = + new InputStream() { + @Override + public int read() throws IOException { + throw new IOException("stream died"); + } + }; + + try { + store.writeAttachment(hash, 8L, failing); + throw new AssertionError("Expected the failing stream to propagate"); + } catch (IOException expected) { + // The blob is streamed to a temporary file and renamed only on a clean finish, so a + // half-written or checksum-mismatched blob must never appear under the hash's name. + File dir = new File(context.getFilesDir(), "sync-attachments"); + assertThat(new File(dir, hash).exists()).isFalse(); + assertThat(new File(dir, hash + ".tmp").exists()).isFalse(); + } + } + + @Test + public void applySnapshot_reapplyingTheLocalVersionKeepsItsAttachments() throws Exception { + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + int noteId = seedNoteWithAttachment("photo.png", bytes); + + // A sync applies the merged snapshot even when the local version won, so a note travels + // through applyPayload -> restoreAttachments unchanged. Resolving blobs from the download + // cache alone rewrote such a note with an empty attachment list and destroyed the files. + SyncSnapshot snapshot = store.readSnapshot(); + store.applySnapshot(snapshot, Collections.emptyList()); + + Note reloaded = db.noteDao().getNoteSync(noteId); + assertThat(reloaded.getAttachments()).isNotNull(); + assertThat(reloaded.getAttachments()).contains("photo.png"); + File restored = + new File( + new File(context.getFilesDir(), "attachments/note_" + noteId), "photo.png"); + assertThat(restored.isFile()).isTrue(); + assertThat(readAll(new java.io.FileInputStream(restored))).isEqualTo(bytes); + } + + @Test + public void clearAfterDisconnect_dropsStatusConflictsAndCachedBlobs() throws Exception { + store.writeState(SyncState.success("google-drive", java.time.Instant.now(), 0)); + byte[] bytes = "cached".getBytes(StandardCharsets.UTF_8); + store.writeAttachment(sha256(bytes), bytes.length, new ByteArrayInputStream(bytes)); + assertThat(new File(context.getFilesDir(), "sync-attachments").listFiles()).isNotEmpty(); + + store.clearAfterDisconnect(); + + // A stale lastSuccessfulSyncAt is what used to make a freshly connected account look + // already-synced, skipping the only dialog that could restore first-sync consent. + assertThat(store.readState().getLastSuccessfulSyncAt()).isNull(); + assertThat(store.getConflicts()).isEmpty(); + File[] cached = new File(context.getFilesDir(), "sync-attachments").listFiles(); + assertThat(cached == null || cached.length == 0).isTrue(); + } + + @Test + public void touch_advancesPastATimestampWrittenByAFasterDeviceClock() { + // Merging is last-write-wins on wall-clock time, but the SQL in SyncMetadataDao assigns + // max(now, stored + 1). A device whose clock runs behind therefore still outranks the + // version it just synced, instead of losing every edit silently. + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_NOTE, 1L, "stable-a", 5_000L, null)); + + db.syncMetadataDao().touch(SyncMetadata.RECORD_TYPE_NOTE, 1L, 1_000L); + + SyncMetadataEntity metadata = db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_NOTE, 1L); + assertThat(metadata.updatedAt).isEqualTo(5_001L); + assertThat(metadata.deletedAt).isNull(); + } + + @Test + public void touch_clearsATombstoneSoAReusedRowIsNotResurrectedAsDeleted() { + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_NOTE, 2L, "stable-b", 100L, 100L)); + + db.syncMetadataDao().touch(SyncMetadata.RECORD_TYPE_NOTE, 2L, 200L); + + SyncMetadataEntity metadata = db.syncMetadataDao().get(SyncMetadata.RECORD_TYPE_NOTE, 2L); + assertThat(metadata.deletedAt).isNull(); + assertThat(metadata.updatedAt).isEqualTo(200L); + } + + // ---- helpers ---- + + private int seedNote(String title, String value, String attachmentsJson) { + Note note = new Note().create(title, value, 1_000L, ""); + note.setAttachments(attachmentsJson); + int id = db.noteDao().addNote(note).intValue(); + db.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_NOTE, + id, + "11111111-1111-4111-8111-111111111111", + 1_000L, + null)); + return id; + } + + /** Writes a real file into the note's own attachment folder and links it from the note. */ + private int seedNoteWithAttachment(String fileName, byte[] bytes) throws IOException { + int id = seedNote("With attachment", "body", null); + File folder = new File(context.getFilesDir(), "attachments/note_" + id); + assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); + try (FileOutputStream out = new FileOutputStream(new File(folder, fileName))) { + out.write(bytes); + } + String json = + "[{\"url\":\"file://attachments/note_" + + id + + "/" + + fileName + + "\",\"name\":\"" + + fileName + + "\"}]"; + Note note = db.noteDao().getNoteSync(id); + note.setAttachments(json); + db.noteDao().addNote(note); + return id; + } + + private static SyncRecord onlyNote(SyncSnapshot snapshot) { + List notes = snapshot.getLiveRecords(SyncRecord.Type.NOTE); + assertThat(notes).hasSize(1); + return notes.get(0); + } + + private static byte[] readAll(InputStream input) throws IOException { + try (InputStream stream = input; + java.io.ByteArrayOutputStream output = new java.io.ByteArrayOutputStream()) { + byte[] buffer = new byte[4096]; + int read; + while ((read = stream.read(buffer)) != -1) { + output.write(buffer, 0, read); + } + return output.toByteArray(); + } + } + + private static String sha256(byte[] bytes) { + try { + StringBuilder hex = new StringBuilder(64); + for (byte value : MessageDigest.getInstance("SHA-256").digest(bytes)) { + hex.append(String.format("%02x", value & 0xff)); + } + return hex.toString(); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + + private static void deleteRecursively(File file) { + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + file.delete(); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java b/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java index 421dcb2c..bb28edf3 100644 --- a/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java +++ b/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java @@ -276,8 +276,8 @@ public Single getNoteForId(long idNote) { } @Override - public Single addNote(Note note, boolean copyNote) { - return dbHelper.addNote(note, copyNote); + public Single addNote(Note note) { + return dbHelper.addNote(note); } @Override diff --git a/app/src/main/java/com/pasich/mynotes/data/database/AppDbHelper.java b/app/src/main/java/com/pasich/mynotes/data/database/AppDbHelper.java index 6b6a5665..290fd33b 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/AppDbHelper.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/AppDbHelper.java @@ -157,12 +157,12 @@ public Single getNoteForId(long idNote) { } @Override - public Single addNote(Note note, boolean copyNote) { + public Single addNote(Note note) { return Single.fromCallable(() -> syncMutationCoordinator.insertNote(note)); } public Single copyNote(Note original) { - return addNote(original.duplicate(), true); + return addNote(original.duplicate()); } @Override diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncStateDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncStateDao.java index 0d9d94ac..a85db85c 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncStateDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncStateDao.java @@ -15,4 +15,8 @@ public interface SyncStateDao { @Insert(onConflict = OnConflictStrategy.REPLACE) void upsert(SyncStateEntity state); + + /** Forgets the stored status, so a freshly connected account starts from idle. */ + @Query("DELETE FROM sync_state") + void clear(); } diff --git a/app/src/main/java/com/pasich/mynotes/data/database/helpers/DbNotesHelper.java b/app/src/main/java/com/pasich/mynotes/data/database/helpers/DbNotesHelper.java index 5fb3350e..f41ec3f8 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/helpers/DbNotesHelper.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/helpers/DbNotesHelper.java @@ -23,7 +23,7 @@ public interface DbNotesHelper { Single getNoteForId(long idNote); - Single addNote(Note note, boolean copyNote); + Single addNote(Note note); Completable deleteNote(Note note); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index 9bc8be18..d9e8a191 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -7,6 +7,7 @@ import com.google.gson.JsonObject; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -32,6 +33,8 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private static final String MIME_BINARY = "application/octet-stream"; private static final int MAX_BUNDLE_RESPONSE_BYTES = 32 * 1024 * 1024; private static final int MAX_ATTACHMENT_RESPONSE_BYTES = 100 * 1024 * 1024; + private static final int MAX_ERROR_DETAIL_BYTES = 1024; + private static final int MAX_ERROR_DETAIL_CHARS = 200; private static final Gson GSON = new Gson(); private final String accessToken; @@ -41,6 +44,13 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private final SyncBundleCodec bundleCodec; private final SyncMerger merger = new SyncMerger(); + /** + * Bundles merged by this instance's {@link #readSnapshot()}, safe to delete once their content + * has been republished. One backend instance serves exactly one sync, so this can never name a + * bundle that arrived after the read. + */ + @Nullable private List supersededBundleIds; + public GoogleDriveSyncBackend(@NonNull String accessToken) { this(accessToken, DEFAULT_API, DEFAULT_UPLOAD, Clock.systemUTC(), new SyncBundleCodec()); } @@ -76,16 +86,19 @@ public synchronized SyncSnapshot readSnapshot() throws IOException { } SyncSnapshot merged = SyncSnapshot.empty(); - for (RemoteFileRef bundle : findBundles(folderId)) { + List readBundleIds = new ArrayList<>(); + for (String bundleId : findBundles(folderId)) { byte[] bytes = requestBytes( "GET", - apiBase + "/files/" + bundle.id + "?alt=media", + apiBase + "/files/" + bundleId + "?alt=media", MAX_BUNDLE_RESPONSE_BYTES); SyncSnapshot decoded = bundleCodec.decode(new ByteArrayInputStream(bytes)).getSnapshot(); merged = merger.merge(merged, decoded).getMergedSnapshot(); + readBundleIds.add(bundleId); } + supersededBundleIds = readBundleIds; return merged; } @@ -97,7 +110,36 @@ public synchronized void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IO // counter, so replacing one file leaves a race where another device can be overwritten. // Publishing a distinct file makes each successful upload independently durable; readers // merge the complete set deterministically. - uploadFile(folderId, nextBundleName(), MIME_ZIP, bundle, null, null, true); + uploadFile(folderId, nextBundleName(), MIME_ZIP, bundle, true); + discardSupersededBundles(); + } + + /** + * Removes the bundles whose content the just-published bundle already contains. + * + *

Without this, every sync that changed anything left one more full snapshot in Drive + * forever, and each later {@link #readSnapshot()} downloaded all of them. Cost grew without + * bound: a user syncing daily for a year would download 365 bundles per sync. + * + *

Only the IDs {@link #readSnapshot()} actually merged in this same sync are removed, so a + * bundle another device published in the meantime is never discarded unread. Deletion is + * best-effort: the new bundle is already durable, and a failure here only postpones cleanup. + */ + private void discardSupersededBundles() { + List superseded = supersededBundleIds; + supersededBundleIds = null; + if (superseded == null) { + return; + } + for (String bundleId : superseded) { + try { + HttpURLConnection connection = open("DELETE", apiBase + "/files/" + bundleId); + ensureSuccess(connection); + connection.disconnect(); + } catch (IOException ignored) { + // Another device may have collected it already. + } + } } @Override @@ -114,25 +156,34 @@ public synchronized InputStream readAttachment(@NonNull String sha256) throws IO return null; } - RemoteFileRef attachment = findAttachment(folderId, sha256); - if (attachment == null) { + String attachmentId = findAttachment(folderId, sha256); + if (attachmentId == null) { return null; } - return new ByteArrayInputStream( - requestBytes( - "GET", - apiBase + "/files/" + attachment.id + "?alt=media", - MAX_ATTACHMENT_RESPONSE_BYTES)); + // Streamed, not buffered: reading a 100 MB attachment into a byte[] (which the growing + // ByteArrayOutputStream first doubled, then copied) was the largest single allocation in + // the sync and an OutOfMemoryError on an ordinary phone. + HttpURLConnection connection = + open("GET", apiBase + "/files/" + attachmentId + "?alt=media"); + ensureSuccess(connection); + return new ConnectionInputStream(connection, MAX_ATTACHMENT_RESPONSE_BYTES); } @Override - public synchronized void writeAttachment(@NonNull String sha256, @NonNull InputStream content) + public synchronized void writeAttachment( + @NonNull String sha256, long sizeBytes, @NonNull InputStream content) throws IOException { String folderId = ensureFolderId(); if (findAttachment(folderId, sha256) != null) { return; } - uploadFile(folderId, sha256, MIME_BINARY, readFully(content), null, null, false); + if (sizeBytes >= 0L) { + uploadStream(folderId, sha256, MIME_BINARY, content, sizeBytes); + return; + } + // No declared size, so the multipart content length cannot be computed up front. Rare: + // sizes come from the bundle manifest, which also supplies the hashes being uploaded. + uploadFile(folderId, sha256, MIME_BINARY, readFully(content), false); } @Nullable @@ -144,12 +195,24 @@ private String findFolderId() throws IOException { + "' and trashed = false and " + appPropertyClause("mynotesOwner", "1"), "files(id,name)"); - if (folders.size() > 1) { - throw new IOException("Drive sync folder is duplicated"); + // Two devices whose first sync overlaps both run ensureFolderId and both create a folder. + // Throwing here made that permanent: every later sync on every device failed before it + // could do any work, and only manual cleanup in Drive recovered it. Converging on the + // lexicographically smallest ID instead makes all devices agree without coordination. + return smallestId(folders); + } + + /** Deterministic, coordination-free choice so every device selects the same file. */ + @Nullable + private static String smallestId(@NonNull JsonArray files) { + String selected = null; + for (int index = 0; index < files.size(); index++) { + String id = files.get(index).getAsJsonObject().get("id").getAsString(); + if (selected == null || id.compareTo(selected) < 0) { + selected = id; + } } - return folders.size() == 0 - ? null - : folders.get(0).getAsJsonObject().get("id").getAsString(); + return selected; } @NonNull @@ -163,11 +226,11 @@ private String ensureFolderId() throws IOException { metadata.addProperty("name", FOLDER_NAME); metadata.addProperty("mimeType", MIME_FOLDER); metadata.add("appProperties", appProperties("mynotesOwner", "1")); - return uploadMetadata(metadata).id; + return uploadMetadata(metadata); } - @Nullable - private List findBundles(@NonNull String folderId) throws IOException { + @NonNull + private List findBundles(@NonNull String folderId) throws IOException { JsonArray bundles = listFiles( "'" @@ -175,17 +238,16 @@ private List findBundles(@NonNull String folderId) throws IOExcep + "' in parents and trashed = false and " + appPropertyClause("mynotesBundle", "1"), "files(id,name)"); - List result = new ArrayList<>(bundles.size()); + List result = new ArrayList<>(bundles.size()); for (int index = 0; index < bundles.size(); index++) { - JsonObject item = bundles.get(index).getAsJsonObject(); - result.add(fetchFileRef(item.get("id").getAsString(), item.get("name").getAsString())); + result.add(bundles.get(index).getAsJsonObject().get("id").getAsString()); } - result.sort(Comparator.comparing(ref -> ref.id)); + result.sort(Comparator.naturalOrder()); return result; } @Nullable - private RemoteFileRef findAttachment(@NonNull String folderId, @NonNull String sha256) + private String findAttachment(@NonNull String folderId, @NonNull String sha256) throws IOException { JsonArray files = listFiles( @@ -194,14 +256,10 @@ private RemoteFileRef findAttachment(@NonNull String folderId, @NonNull String s + "' in parents and trashed = false and " + appPropertyClause("mynotesAttachmentSha256", sha256), "files(id,name)"); - if (files.size() == 0) { - return null; - } - if (files.size() > 1) { - throw new IOException("Drive attachment is duplicated: " + sha256); - } - JsonObject item = files.get(0).getAsJsonObject(); - return fetchFileRef(item.get("id").getAsString(), item.get("name").getAsString()); + // Attachments are content-addressed, so duplicates uploaded by two devices racing on the + // same hash are byte-identical and either one will do. Rejecting them used to break every + // subsequent sync permanently. + return smallestId(files); } @NonNull @@ -222,7 +280,7 @@ private JsonArray listFiles(@NonNull String query, @NonNull String fields) throw "&pageToken=" + URLEncoder.encode(nextPageToken, StandardCharsets.UTF_8.name()); } - JsonObject response = requestJson("GET", url, null, null, null); + JsonObject response = requestJson("GET", url, null, null); JsonArray files = response.getAsJsonArray("files"); if (files != null) { for (int index = 0; index < files.size(); index++) { @@ -238,37 +296,57 @@ private JsonArray listFiles(@NonNull String query, @NonNull String fields) throw } @NonNull - private RemoteFileRef uploadMetadata(@NonNull JsonObject metadata) throws IOException { + private String uploadMetadata(@NonNull JsonObject metadata) throws IOException { JsonObject created = requestJson( - "POST", - apiBase + "/files?fields=id,name", - MIME_JSON, - jsonBytes(metadata), - null); - String id = created.get("id").getAsString(); - String name = created.get("name").getAsString(); - return fetchFileRef(id, name); + "POST", apiBase + "/files?fields=id,name", MIME_JSON, jsonBytes(metadata)); + return created.get("id").getAsString(); } - @NonNull - private RemoteFileRef uploadFile( + private void uploadFile( @NonNull String folderId, @NonNull String name, @NonNull String mimeType, @NonNull byte[] data, - @Nullable String fileId, - @Nullable String ifMatch, + boolean bundleFile) + throws IOException { + uploadMultipart( + folderId, name, mimeType, new ByteArrayInputStream(data), data.length, bundleFile); + } + + /** Uploads an attachment of known length without ever holding it in memory. */ + private void uploadStream( + @NonNull String folderId, + @NonNull String name, + @NonNull String mimeType, + @NonNull InputStream content, + long sizeBytes) + throws IOException { + uploadMultipart(folderId, name, mimeType, content, sizeBytes, false); + } + + /** + * Writes one {@code multipart/related} upload straight to the socket. + * + *

The body length is computed from the declared size so {@link + * HttpURLConnection#setFixedLengthStreamingMode(long)} can be used. Without it {@code + * HttpURLConnection} buffers the entire request in memory to work out a Content-Length, which + * would put the whole attachment back on the heap and defeat the streaming read path. + */ + private void uploadMultipart( + @NonNull String folderId, + @NonNull String name, + @NonNull String mimeType, + @NonNull InputStream content, + long sizeBytes, boolean bundleFile) throws IOException { String boundary = "mynotes-" + System.nanoTime(); JsonObject metadata = new JsonObject(); metadata.addProperty("name", name); - if (fileId == null) { - JsonArray parents = new JsonArray(); - parents.add(folderId); - metadata.add("parents", parents); - } + JsonArray parents = new JsonArray(); + parents.add(folderId); + metadata.add("parents", parents); JsonObject appProperties = new JsonObject(); if (bundleFile) { @@ -279,23 +357,50 @@ private RemoteFileRef uploadFile( } metadata.add("appProperties", appProperties); - String path = - fileId == null - ? "?uploadType=multipart&fields=id,name" - : "/" + fileId + "?uploadType=multipart&fields=id,name"; - HttpURLConnection connection = open(fileId == null ? "POST" : "PATCH", uploadBase + path); + byte[] head = partHeader(boundary, MIME_JSON); + byte[] metadataBytes = jsonBytes(metadata); + byte[] separator = "\r\n".getBytes(StandardCharsets.UTF_8); + byte[] contentHeader = partHeader(boundary, mimeType); + byte[] closing = ("--" + boundary + "--\r\n").getBytes(StandardCharsets.UTF_8); + + HttpURLConnection connection = + open("POST", uploadBase + "?uploadType=multipart&fields=id,name"); connection.setRequestProperty("Content-Type", "multipart/related; boundary=" + boundary); - if (ifMatch != null) { - connection.setRequestProperty("If-Match", ifMatch); - } connection.setDoOutput(true); + connection.setFixedLengthStreamingMode( + (long) head.length + + metadataBytes.length + + separator.length + + contentHeader.length + + sizeBytes + + separator.length + + closing.length); + try (OutputStream out = connection.getOutputStream()) { - writePart(out, boundary, MIME_JSON, jsonBytes(metadata)); - writePart(out, boundary, mimeType, data); - out.write(("--" + boundary + "--\r\n").getBytes(StandardCharsets.UTF_8)); + out.write(head); + out.write(metadataBytes); + out.write(separator); + out.write(contentHeader); + copy(content, out); + out.write(separator); + out.write(closing); + } + readJsonResponse(connection); + } + + @NonNull + private static byte[] partHeader(@NonNull String boundary, @NonNull String mimeType) { + return ("--" + boundary + "\r\nContent-Type: " + mimeType + "\r\n\r\n") + .getBytes(StandardCharsets.UTF_8); + } + + private static void copy(@NonNull InputStream source, @NonNull OutputStream target) + throws IOException { + byte[] buffer = new byte[8192]; + int read; + while ((read = source.read(buffer)) != -1) { + target.write(buffer, 0, read); } - JsonObject response = readJsonResponse(connection); - return fetchFileRef(response.get("id").getAsString(), response.get("name").getAsString()); } @NonNull @@ -306,41 +411,17 @@ private static String nextBundleName() { + ".zip"; } - @NonNull - private RemoteFileRef fetchFileRef(@NonNull String id, @NonNull String fallbackName) - throws IOException { - HttpURLConnection connection = - open("GET", apiBase + "/files/" + id + "?fields=id,name,version,appProperties"); - JsonObject response = readJsonResponse(connection); - String name = - response.has("name") && !response.get("name").isJsonNull() - ? response.get("name").getAsString() - : fallbackName; - String version = - response.has("version") && !response.get("version").isJsonNull() - ? response.get("version").getAsString() - : null; - if (version == null || version.trim().isEmpty()) { - throw new IOException("Drive file metadata response is missing a version"); - } - return new RemoteFileRef(id, name, version); - } - @NonNull private JsonObject requestJson( @NonNull String method, @NonNull String url, @Nullable String contentType, - @Nullable byte[] body, - @Nullable String ifMatch) + @Nullable byte[] body) throws IOException { HttpURLConnection connection = open(method, url); if (contentType != null) { connection.setRequestProperty("Content-Type", contentType); } - if (ifMatch != null) { - connection.setRequestProperty("If-Match", ifMatch); - } if (body != null) { connection.setDoOutput(true); try (OutputStream output = connection.getOutputStream()) { @@ -393,17 +474,48 @@ private static void ensureSuccess(@NonNull HttpURLConnection connection) throws return; } - String detail = ""; - InputStream error = connection.getErrorStream(); - if (error != null) { - detail = new String(readFully(error), StandardCharsets.UTF_8); - } + String detail = readErrorDetail(connection.getErrorStream()); if (code == HttpURLConnection.HTTP_PRECON_FAILED) { throw new IOException("Drive snapshot changed since it was read"); } throw new IOException("Drive API HTTP " + code + (detail.isEmpty() ? "" : ": " + detail)); } + /** + * Reads a short, single-line excuse out of a Drive error response. + * + *

The whole body used to end up in this exception's message, which is shown in a Snackbar + * and persisted as {@code sync_state.errorMessage} — where the account screen then renders it + * as the sync status. A quota or permission response is a multi-line JSON document, so the + * status label became an unreadable blob that stayed until the next successful sync. + */ + @NonNull + private static String readErrorDetail(@Nullable InputStream error) { + if (error == null) { + return ""; + } + try (InputStream stream = error) { + byte[] buffer = new byte[MAX_ERROR_DETAIL_BYTES]; + int read = 0; + while (read < buffer.length) { + int count = stream.read(buffer, read, buffer.length - read); + if (count == -1) { + break; + } + read += count; + } + String detail = + new String(buffer, 0, read, StandardCharsets.UTF_8) + .replaceAll("\\s+", " ") + .trim(); + return detail.length() > MAX_ERROR_DETAIL_CHARS + ? detail.substring(0, MAX_ERROR_DETAIL_CHARS) + "…" + : detail; + } catch (IOException ignored) { + return ""; + } + } + @NonNull private static byte[] readFully(@NonNull InputStream input) throws IOException { try (InputStream stream = input; @@ -443,28 +555,56 @@ private static String escapeQuery(@NonNull String value) { return value.replace("\\", "\\\\").replace("'", "\\'"); } - private static void writePart( - @NonNull OutputStream out, - @NonNull String boundary, - @NonNull String mimeType, - byte[] data) - throws IOException { - out.write( - ("--" + boundary + "\r\nContent-Type: " + mimeType + "\r\n\r\n") - .getBytes(StandardCharsets.UTF_8)); - out.write(data); - out.write("\r\n".getBytes(StandardCharsets.UTF_8)); - } - - private static final class RemoteFileRef { - private final String id; - private final String name; - private final String eTag; - - private RemoteFileRef(@NonNull String id, @NonNull String name, @NonNull String eTag) { - this.id = id; - this.name = name; - this.eTag = eTag; + /** + * A response body that stays attached to its connection until the reader is done. + * + *

Lets an attachment be piped straight from the socket to disk while still enforcing the + * response ceiling, and releases the connection on close. + */ + private static final class ConnectionInputStream extends FilterInputStream { + private final HttpURLConnection connection; + private final long maxBytes; + private long byteCount; + + private ConnectionInputStream(@NonNull HttpURLConnection connection, long maxBytes) + throws IOException { + super(connection.getInputStream()); + this.connection = connection; + this.maxBytes = maxBytes; + } + + @Override + public int read() throws IOException { + int value = super.read(); + if (value >= 0) { + count(1); + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int read = super.read(buffer, offset, length); + if (read > 0) { + count(read); + } + return read; + } + + private void count(int read) throws IOException { + byteCount += read; + if (byteCount > maxBytes) { + throw new IOException("Drive response exceeds the sync size limit"); + } + } + + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + connection.disconnect(); + } } } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java index 593de964..2e80d6bb 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java @@ -37,6 +37,14 @@ public Result doWork() { FirebaseUser user = FirebaseAuth.getInstance(app).getCurrentUser(); if (user == null || user.getEmail() == null) return Result.success(); try { + SyncDependencies dependencies = + EntryPointAccessors.fromApplication( + getApplicationContext(), SyncDependencies.class); + // Checked before authorizing: asking Google Play services for a token only to discard + // it is a pointless network round trip on every scheduled run. + if (!isBackgroundSyncAllowed(dependencies.preferenceHelper())) { + return Result.success(); + } AuthorizationRequest request = new AuthorizationRequest.Builder() .setRequestedScopes(Collections.singletonList(DRIVE_FILE)) @@ -49,12 +57,6 @@ public Result doWork() { if (authorization.hasResolution() || authorization.getAccessToken() == null) { return Result.failure(); } - SyncDependencies dependencies = - EntryPointAccessors.fromApplication( - getApplicationContext(), SyncDependencies.class); - if (!isBackgroundSyncAllowed(dependencies.preferenceHelper())) { - return Result.success(); - } SyncState state = new SyncService( new RoomSyncStore( @@ -82,9 +84,17 @@ private static boolean isRetryable(String message) { || value.contains("temporar"); } + /** + * The rollout gate applies here too. + * + *

It used to live only on the manual "Sync now" path, so lowering the percentage to pull a + * bad release back would have left this six-hourly job running for every user who had already + * turned sync on — the very population a rollback needs to stop. + */ static boolean isBackgroundSyncAllowed(PreferenceHelper preferences) { return preferences.isSyncEnabled() && preferences.isBackgroundSyncEnabled() - && preferences.isFirstSyncConfirmed(); + && preferences.isFirstSyncConfirmed() + && SyncRollout.isWithinRollout(preferences); } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index 480e3fc1..375f945c 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -2,6 +2,7 @@ import android.content.Context; import android.content.SharedPreferences; +import android.util.Log; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import com.google.gson.Gson; @@ -38,9 +39,11 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; /** Room-backed sync store. Stable sync IDs remain separate from local integer primary keys. */ public final class RoomSyncStore implements SyncStore { + private static final String TAG = "RoomSyncStore"; private static final String PREFS = "sync_state"; private static final String LEGACY_STATE = "last_state"; private static final String PREFERENCES_HASH = "preferences_hash"; @@ -51,6 +54,12 @@ public final class RoomSyncStore implements SyncStore { private final Context context; private final Gson gson = new Gson(); + /** + * Content hash to the note-folder file holding it, indexed while the snapshot is built so the + * upload path can find blobs this device owns without duplicating them into the sync cache. + */ + private final Map localAttachments = new ConcurrentHashMap<>(); + public RoomSyncStore( @NonNull Context context, @NonNull AppDatabase database, @@ -221,6 +230,8 @@ else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { } } if ("note".equals(metadata.recordType)) addAttachmentMetadata(result); + // Runs last: the blocks above still need the local categoryId and attachment paths. + SyncMetadata.stripDeviceLocalFields(metadata.recordType, result); return result; } @@ -314,6 +325,43 @@ public List getConflicts() { return database.syncConflictDao().getAll(); } + /** + * Drops every trace of the account being disconnected. + * + *

Record identity in {@code sync_metadata} is deliberately kept: it is local, and discarding + * it would make the whole library look brand new to the next account. What goes is the sync + * status, the conflict queue and the blobs downloaded from the disconnected account's Drive. + * + *

Clearing the status also repairs a dead end: the Backup screen decided whether to ask for + * first-sync consent from {@code lastSuccessfulSyncAt}, which survived a sign-out, while {@code + * SyncCoordinator} gated the sync on a preference the sign-out reset. The dialog was skipped + * and the sync refused, with no way to reach the consent again. + */ + public void clearAfterDisconnect() { + database.runInTransaction( + () -> { + database.syncStateDao().clear(); + database.syncConflictDao().clearAll(); + }); + preferences.edit().remove(PREFERENCES_HASH).remove(LEGACY_STATE).apply(); + localAttachments.clear(); + deleteAttachmentCache(); + } + + /** Removes the download cache only; the notes' own attachment folders are untouched. */ + private void deleteAttachmentCache() { + File dir = new File(context.getFilesDir(), "sync-attachments"); + File[] cached = dir.listFiles(); + if (cached == null) { + return; + } + for (File file : cached) { + if (file.isFile() && !file.delete()) { + Log.w(TAG, "Could not remove cached attachment " + file.getName()); + } + } + } + public List getUnresolvedConflicts() { return database.syncConflictDao().getUnresolved(); } @@ -485,25 +533,60 @@ public Collection getAttachmentHashes(@NonNull SyncSnapshot snapshot) { @Override public boolean hasAttachment(@NonNull String sha256) { - return attachmentFile(sha256).isFile(); + return resolveLocalAttachment(sha256) != null; } @NonNull @Override public InputStream readAttachment(@NonNull String sha256) throws IOException { - return new FileInputStream(attachmentFile(sha256)); + File source = resolveLocalAttachment(sha256); + if (source == null) { + throw new java.io.FileNotFoundException("No local attachment for " + sha256); + } + return new FileInputStream(source); + } + + /** + * Finds a blob this device already holds, in the download cache or in a note's own folder. + * + *

Only the cache directory used to be consulted, and nothing but the download path ever + * wrote to it. On the device that owns an attachment the lookup therefore returned false, + * {@code SyncService} asked the backend for a blob nobody had uploaded yet, and the sync failed + * with "Required attachment is unavailable". Since the upload branch is reachable only when + * this returns true, that failure was permanent for any account holding a single attachment. + * + *

The note folders are indexed while the snapshot is built rather than copied into the + * cache, so a large attachment set is not stored twice. + */ + @Nullable + private File resolveLocalAttachment(@NonNull String sha256) { + File cached = attachmentFile(sha256); + if (cached.isFile()) { + return cached; + } + File owned = localAttachments.get(sha256); + return owned != null && owned.isFile() ? owned : null; } @Override - public void writeAttachment(@NonNull String sha256, @NonNull InputStream content) + public void writeAttachment( + @NonNull String sha256, long sizeBytes, @NonNull InputStream content) throws IOException { File target = attachmentFile(sha256); File temp = new File(target.getParentFile(), sha256 + ".tmp"); + // Written to a temporary file and only then renamed, so a stream that fails part-way — + // including a checksum mismatch, which SyncService raises at end of stream, inside this + // very loop — never leaves a half-written blob under the hash's name. try (InputStream in = content; FileOutputStream out = new FileOutputStream(temp)) { byte[] buffer = new byte[8192]; int read; while ((read = in.read(buffer)) != -1) out.write(buffer, 0, read); + } catch (IOException error) { + if (temp.exists() && !temp.delete()) { + Log.w(TAG, "Could not remove the partial attachment " + temp.getName()); + } + throw error; } if (!temp.renameTo(target)) throw new IOException("Cannot store attachment"); } @@ -530,6 +613,7 @@ private void addAttachmentMetadata(JsonObject payload) { File file = AttachmentStorage.resolve(context, attachment.url); if (file == null || !file.isFile()) continue; String hash = sha256(file); + localAttachments.put(hash, file); String displayName = attachment.name == null || attachment.name.trim().isEmpty() ? file.getName() @@ -552,7 +636,10 @@ private void addAttachmentMetadata(JsonObject payload) { payload.add("attachmentHashes", hashes); payload.add("attachmentNames", names); } - } catch (Exception ignored) { + } catch (Exception error) { + // Swallowing this silently used to drop a note's attachments from the bundle with no + // trace; the sync itself still succeeds without them. + Log.w(TAG, "Could not collect attachment metadata", error); } } @@ -565,18 +652,31 @@ private void restoreAttachments(Note note, JsonObject payload) { File folder = AttachmentStorage.noteFolder(context, note.getId()); for (JsonElement item : hashes) { String hash = item.getAsString(); - File source = attachmentFile(hash); - if (!source.isFile()) continue; + // Not just the download cache: when the local version of a note wins the merge it + // is re-applied through this same path, and its blobs live in the note's own + // folder. Resolving only the cache silently rewrote such a note with an empty + // attachment list, destroying files that were never in conflict. + File source = resolveLocalAttachment(hash); + if (source == null) continue; String name = names.has(hash) ? names.get(hash).getAsString() : hash; if (!isSafeAttachmentName(name)) { continue; } File target = new File(folder, name); - try (InputStream in = new FileInputStream(source); - OutputStream out = new FileOutputStream(target)) { - byte[] buffer = new byte[8192]; - int read; - while ((read = in.read(buffer)) != -1) out.write(buffer, 0, read); + if (!source.getAbsolutePath().equals(target.getAbsolutePath())) { + // Scoped per file: one unreadable blob must not discard the note's other + // attachments, which is what a single loop-wide catch used to do. Skipped + // entirely when the file is already in place, because opening it for writing + // would truncate the very file being read. + try (InputStream in = new FileInputStream(source); + OutputStream out = new FileOutputStream(target)) { + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) != -1) out.write(buffer, 0, read); + } catch (IOException error) { + Log.w(TAG, "Could not restore attachment " + hash, error); + continue; + } } JsonObject attachment = new JsonObject(); attachment.addProperty( @@ -585,7 +685,8 @@ private void restoreAttachments(Note note, JsonObject payload) { attachments.add(attachment); } note.setAttachments(gson.toJson(attachments)); - } catch (Exception ignored) { + } catch (RuntimeException error) { + Log.w(TAG, "Malformed attachment metadata for note " + note.getId(), error); } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java index 4c09d2fd..0d389e8d 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java @@ -47,5 +47,12 @@ public interface SyncBackend { *

The implementation must consume the stream before returning and must not expose a partial * file after an exception. */ - void writeAttachment(@NonNull String sha256, @NonNull InputStream content) throws IOException; + /** + * Stores one immutable blob, streaming it rather than holding it in memory. + * + * @param sizeBytes the blob's declared size, or a negative value when it is unknown. A known + * size lets an implementation avoid buffering the whole blob to compute a content length. + */ + void writeAttachment(@NonNull String sha256, long sizeBytes, @NonNull InputStream content) + throws IOException; } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 00bb93f9..e34f6d6a 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -214,6 +214,8 @@ private static void parseLiveRecords( JsonObject payload = item.deepCopy(); payload.remove("id"); payload.remove("updatedAt"); + // Bundles written before the device-local fields were stripped still carry them. + SyncMetadata.stripDeviceLocalFields(type.getWireValue(), payload); if (type == SyncRecord.Type.NOTE) { hydrateNoteAttachments(payload, attachmentsById); } @@ -233,6 +235,8 @@ private static void hydrateNoteAttachments( if (attachmentIds == null) return; JsonArray attachmentHashes = new JsonArray(); JsonArray manifest = new JsonArray(); + // The wire keys names by attachment UUID; the local store looks them up by content hash. + JsonObject namesByHash = new JsonObject(); for (JsonElement element : attachmentIds) { String attachmentId = element.getAsString(); AttachmentManifestEntry attachment = attachmentsById.get(attachmentId); @@ -245,9 +249,15 @@ private static void hydrateNoteAttachments( } manifest.add(value); attachmentHashes.add(attachment.sha256); + if (value.has("displayName") && !value.get("displayName").isJsonNull()) { + namesByHash.addProperty(attachment.sha256, value.get("displayName").getAsString()); + } } payload.add("attachmentsManifest", manifest); payload.add("attachmentHashes", attachmentHashes); + // Rekeyed by hash; leaving the UUID-keyed map made every restored file land on disk + // named after its bare SHA-256, with no extension. + payload.add("attachmentNames", namesByHash); } private static void parseTombstones( diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMetadata.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMetadata.java index a74337d3..f133508d 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMetadata.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMetadata.java @@ -19,6 +19,38 @@ public static String newStableId() { return UUID.randomUUID().toString().toLowerCase(Locale.ROOT); } + /** + * Removes every payload field that only means something on the device that wrote it. + * + *

Room primary keys and attachment {@code file://} paths differ per device, so leaving them + * in the payload makes two devices compute different canonical hashes for the same logical + * record. Because {@code applySnapshot} copies the record's {@code updatedAt} verbatim, the + * next merge sees equal timestamps, falls through to the hash tiebreaker, and reports a + * conflict for every note, tag, task and category on every sync forever. It also keeps {@code + * snapshotsMatch} permanently false, so each sync republishes a full bundle. + * + *

Identity travels in the record's stable ID and attachments travel in the bundle manifest, + * so nothing here is needed on the wire. Applied to decoded remote records as well, so bundles + * written by 2.6.48/2.6.49 normalize to the same shape instead of conflicting forever. + */ + public static void stripDeviceLocalFields( + String recordType, com.google.gson.JsonObject payload) { + if (payload == null) { + return; + } + if (RECORD_TYPE_NOTE.equals(recordType)) { + payload.remove("a"); // Note.id + payload.remove("h"); // Note.attachments: device-local file:// paths + } else if (RECORD_TYPE_TAG.equals(recordType)) { + payload.remove("a"); // Tag.id + } else if (RECORD_TYPE_TASK.equals(recordType)) { + payload.remove("id"); + payload.remove("categoryId"); // travels as categoryStableId + } else if (RECORD_TYPE_CATEGORY.equals(recordType)) { + payload.remove("id"); + } + } + /** Returns true only for the record types defined by sync schema version 1. */ public static boolean isSupportedRecordType(String recordType) { return RECORD_TYPE_NOTE.equals(recordType) diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java index a9256fa8..7bc2cdba 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java @@ -121,6 +121,7 @@ public void insertTags(List tags) { long timestamp = resolveBatchTimestamp( SyncMetadata.RECORD_TYPE_TAG, extractTagIds(tags)); + releaseTakenTagIds(tags); long[] insertedIds = tagsDao.addTags(tags); for (int i = 0; i < tags.size(); i++) { Tag tag = tags.get(i); @@ -216,6 +217,7 @@ public void insertNotes(List notes) { long timestamp = resolveBatchTimestamp( SyncMetadata.RECORD_TYPE_NOTE, extractNoteIds(notes)); + releaseTakenNoteIds(notes); long[] insertedIds = noteDao.addNotes(notes); for (int i = 0; i < notes.size(); i++) { Note note = notes.get(i); @@ -504,6 +506,36 @@ private long insertNoteInternal(@NonNull Note note, long timestamp) { return insertedId; } + /** + * Lets a restore keep its original IDs only where they are still free. + * + *

Backups carry the IDs the notes had when the backup was taken, and {@code addNotes} is a + * REPLACE insert. Restoring onto a device that already holds notes therefore destroyed every + * note whose ID happened to collide — silently, with no way back. Sync made that worse: the + * restored content inherited the destroyed note's stable ID through {@code ensureMetadataRow}, + * {@code touch()} cleared its tombstone, and the replacement propagated to every other device, + * overwriting the cloud copy too. + * + *

A colliding note is now inserted as a new row instead. Restoring onto an empty library — + * the ordinary case, and the one after a reinstall — still preserves every ID exactly. + */ + private void releaseTakenNoteIds(@NonNull List notes) { + for (Note note : notes) { + if (note.getId() > 0 && noteDao.getNoteSync(note.getId()) != null) { + note.setId(0); + } + } + } + + /** Same protection for a restored tag list; see {@link #releaseTakenNoteIds}. */ + private void releaseTakenTagIds(@NonNull List tags) { + for (Tag tag : tags) { + if (tag.getId() > 0 && tagsDao.getTagSync(tag.getId()) != null) { + tag.id = 0; + } + } + } + private void touchRecords(@NonNull String recordType, List localIds, long timestamp) { if (localIds == null || localIds.isEmpty()) return; for (Integer localId : localIds) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncRollout.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncRollout.java new file mode 100644 index 00000000..a0b9080e --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncRollout.java @@ -0,0 +1,49 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import com.pasich.mynotes.data.preferences.PreferenceHelper; +import java.security.SecureRandom; + +/** + * Staged-rollout gate for Google Drive sync. + * + *

The percentage and the bucket check used to live in {@code SyncCoordinator}, which only the + * manual "Sync now" button goes through. {@code GoogleDriveSyncWorker} never consulted them, so + * lowering the percentage to stop a bad release would have left the six-hourly background sync + * running for every user who had already enabled it — exactly the population a rollback needs to + * stop. Both paths now share this class. + */ +public final class SyncRollout { + + /** + * The v2.6.48 sync safety release completed its staged rollout; sync is available to all + * cohorts. Lower this to pull sync back from part of the population. + */ + public static final int CURRENT_PERCENT = 100; + + private static final SecureRandom RANDOM = new SecureRandom(); + + private SyncRollout() { + // no instance + } + + /** + * Returns this device's stable 1..100 cohort, assigning one on first use. + * + *

The bucket is drawn once and kept, so a device never moves between cohorts as the + * percentage changes. + */ + public static int ensureBucket(@NonNull PreferenceHelper preferences) { + int bucket = preferences.getSyncRolloutBucket(); + if (bucket < 1 || bucket > 100) { + bucket = RANDOM.nextInt(100) + 1; + preferences.setSyncRolloutBucket(bucket); + } + return bucket; + } + + /** True when this device's cohort is inside the current rollout. */ + public static boolean isWithinRollout(@NonNull PreferenceHelper preferences) { + return ensureBucket(preferences) <= CURRENT_PERCENT; + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java index 6e241192..a20e0da9 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java @@ -1,11 +1,10 @@ package com.pasich.mynotes.data.sync; +import android.util.Log; import androidx.annotation.NonNull; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; @@ -17,6 +16,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; /** @@ -29,7 +30,9 @@ */ public final class SyncService { + private static final String TAG = "SyncService"; private static final Pattern SHA_256 = Pattern.compile("[0-9a-f]{64}"); + private static final long MAX_TOLERATED_CLOCK_SKEW_MILLIS = 24L * 60L * 60L * 1000L; private final SyncStore store; private final SyncMerger merger; @@ -45,9 +48,44 @@ public SyncService(@NonNull SyncStore store, @NonNull SyncMerger merger, @NonNul this.clock = Objects.requireNonNull(clock, "clock"); } + /** + * Serializes every sync attempt in the process. + * + *

This method used to rely on {@code synchronized}, but a fresh {@code SyncService} is + * constructed for each attempt — once by the Backup screen and once by {@code + * GoogleDriveSyncWorker} — so the monitor was per-instance and guarded nothing. A manual sync + * and the six-hourly worker could interleave their Room writes and both publish a bundle. + */ + private static final ReentrantLock SYNC_LOCK = new ReentrantLock(); + + private static final long LOCK_WAIT_SECONDS = 5L; + /** Runs one serialized manual synchronization attempt and returns its durable final state. */ @NonNull - public synchronized SyncState sync(@NonNull SyncBackend backend) { + public SyncState sync(@NonNull SyncBackend backend) { + boolean acquired = false; + try { + acquired = SYNC_LOCK.tryLock(LOCK_WAIT_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + if (!acquired) { + // Deliberately not persisted: the sync that holds the lock owns the stored state. + // The wording keeps GoogleDriveSyncWorker.isRetryable() treating this as retryable. + return SyncState.error( + "google-drive", + safeReadState().getLastSuccessfulSyncAt(), + "Another sync is already running; this attempt was temporarily skipped"); + } + try { + return syncExclusively(backend); + } finally { + SYNC_LOCK.unlock(); + } + } + + @NonNull + private SyncState syncExclusively(@NonNull SyncBackend backend) { SyncState previousState = safeReadState(); String backendIdentifier = "unknown"; @@ -60,6 +98,7 @@ public synchronized SyncState sync(@NonNull SyncBackend backend) { backendIdentifier, startedAt, previousState.getLastSuccessfulSyncAt())); SyncSnapshot local = Objects.requireNonNull(store.readSnapshot(), "local snapshot"); SyncSnapshot remote = Objects.requireNonNull(backend.readSnapshot(), "remote snapshot"); + warnAboutClockSkew(remote); SyncMergeResult mergeResult = merger.merge(local, remote); SyncSnapshot merged = mergeResult.getMergedSnapshot(); @@ -84,6 +123,36 @@ public synchronized SyncState sync(@NonNull SyncBackend backend) { } } + /** + * Flags a device clock that disagrees badly with the rest of the account. + * + *

Merging is last-write-wins on wall-clock time. Per record that self-corrects: {@code + * SyncMetadataDao.touch} assigns {@code max(now, storedUpdatedAt + 1)}, so once a device has + * seen a newer remote version its own next edit outranks it however far behind its clock runs. + * What stays exposed is the first divergent edit to a record neither side has synced since, + * where the raw clocks decide and the slower device loses silently. A wrong device clock is + * therefore worth a line in the log when support has to explain a "lost" edit. + */ + private void warnAboutClockSkew(@NonNull SyncSnapshot remote) { + Instant newest = null; + for (SyncRecord record : remote.getRecords()) { + if (newest == null || record.getUpdatedAt().isAfter(newest)) { + newest = record.getUpdatedAt(); + } + } + if (newest == null) { + return; + } + long skewMillis = newest.toEpochMilli() - clock.millis(); + if (skewMillis > MAX_TOLERATED_CLOCK_SKEW_MILLIS) { + Log.w( + TAG, + "Remote records are " + + (skewMillis / 3_600_000L) + + "h ahead of this device's clock; merge order may be wrong"); + } + } + private static boolean snapshotsMatch( @NonNull SyncSnapshot first, @NonNull SyncSnapshot second) { Collection firstRecords = first.getRecords(); @@ -119,13 +188,18 @@ private void synchronizeAttachments( try { verifyAttachment(hash, expectedSizes.get(hash), store.readAttachment(hash)); } catch (IOException localError) { + // The local copy is missing or corrupt; repair it from the remote blob. copyVerified( hash, expectedSizes.get(hash), backend.readAttachment(hash), store::writeAttachment); } - verifyAttachment(hash, expectedSizes.get(hash), backend.readAttachment(hash)); + // A remote re-verification used to run here on every sync, downloading each + // attachment in full (up to 100 MB) purely to re-check a hash. Remote blobs are + // content-addressed and immutable, so the check could never fail for a reason + // the download itself would not already surface, and on the six-hourly worker + // it re-transferred the user's entire attachment set four times a day. } else { copyVerified( hash, @@ -160,6 +234,16 @@ private void verifyAttachment(String hash, Long expectedSize, InputStream source } } + /** + * Pipes one blob to the other endpoint, verifying it as the bytes go past. + * + *

This used to buffer the whole blob into a {@code ByteArrayOutputStream}, call {@code + * toByteArray()} and hand the destination a {@code ByteArrayInputStream}. With the 100 MB + * attachment ceiling that peaked at several hundred megabytes of heap for a single file — an + * {@code OutOfMemoryError} on any ordinary phone. Nothing is buffered now: {@link + * VerifyingInputStream} checks the digest at end of stream, which happens inside the + * destination's own read loop, so a corrupt blob still aborts the write before it is committed. + */ private void copyVerified( String expectedHash, Long expectedSize, @@ -167,20 +251,12 @@ private void copyVerified( AttachmentWriter destination) throws IOException { Objects.requireNonNull(source, "source"); - byte[] verifiedBytes; try (InputStream input = source; VerifyingInputStream verified = - new VerifyingInputStream(input, expectedHash, expectedSize); - ByteArrayOutputStream output = new ByteArrayOutputStream()) { - byte[] buffer = new byte[8192]; - int read; - while ((read = verified.read(buffer)) != -1) { - output.write(buffer, 0, read); - } + new VerifyingInputStream(input, expectedHash, expectedSize)) { + destination.write(expectedHash, expectedSize == null ? -1L : expectedSize, verified); verified.verifyEndOfStream(); - verifiedBytes = output.toByteArray(); } - destination.write(expectedHash, new ByteArrayInputStream(verifiedBytes)); } private static Map attachmentSizes(SyncSnapshot snapshot) throws IOException { @@ -247,7 +323,7 @@ private static void validateHash(String hash) throws IOException { } private interface AttachmentWriter { - void write(String hash, InputStream content) throws IOException; + void write(String hash, long sizeBytes, InputStream content) throws IOException; } /** Verifies the hash only after the receiving endpoint consumed every byte. */ @@ -275,9 +351,9 @@ public int read() throws IOException { if (value >= 0) { digest.update((byte) value); byteCount++; - if (byteCount > SyncBundleValidator.MAX_ATTACHMENT_BYTES) { - throw new IOException("Attachment exceeds the sync size limit"); - } + enforceSizeLimit(); + } else { + verifyEndOfStream(); } return value; } @@ -288,26 +364,50 @@ public int read(byte[] buffer, int offset, int length) throws IOException { if (read > 0) { digest.update(buffer, offset, read); byteCount += read; - if (byteCount > SyncBundleValidator.MAX_ATTACHMENT_BYTES) { - throw new IOException("Attachment exceeds the sync size limit"); - } + enforceSizeLimit(); + } else if (read < 0) { + verifyEndOfStream(); } return read; } + private void enforceSizeLimit() throws IOException { + if (byteCount > SyncBundleValidator.MAX_ATTACHMENT_BYTES) { + throw new IOException("Attachment exceeds the sync size limit"); + } + } + + /** + * Checks the digest, draining anything the destination left behind first. + * + *

Reached from {@link #read} at end of stream, so a destination that streams straight to + * its final location still learns about a mismatch before it commits. + */ void verifyEndOfStream() throws IOException { - if (!verified) { - while (read(new byte[8192]) != -1) { - // Drain an incorrectly implemented destination before declaring success. - } - String actualHash = toHex(digest.digest()); - if (!expectedHash.equals(actualHash)) { - throw new IOException("Attachment checksum does not match its declared hash"); - } - if (expectedSize != null && expectedSize.longValue() != byteCount) { - throw new IOException("Attachment size does not match its declared size"); - } - verified = true; + if (verified) { + return; + } + // Set before draining: drainRemaining reads through super, but a caller reaching this + // from read() must not be able to re-enter. + verified = true; + drainRemaining(); + String actualHash = toHex(digest.digest()); + if (!expectedHash.equals(actualHash)) { + throw new IOException("Attachment checksum does not match its declared hash"); + } + if (expectedSize != null && expectedSize.longValue() != byteCount) { + throw new IOException("Attachment size does not match its declared size"); + } + } + + /** Reads through {@code super} so the digest covers bytes the destination skipped. */ + private void drainRemaining() throws IOException { + byte[] scratch = new byte[8192]; + int read; + while ((read = super.read(scratch, 0, scratch.length)) != -1) { + digest.update(scratch, 0, read); + byteCount += read; + enforceSizeLimit(); } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java index 615206ec..f38a6d10 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java @@ -61,7 +61,14 @@ default void applySnapshot( *

The implementation must consume the stream before returning and must not expose a partial * file after an exception. */ - void writeAttachment(@NonNull String sha256, @NonNull InputStream content) throws IOException; + /** + * Stores one immutable blob, streaming it rather than holding it in memory. + * + * @param sizeBytes the blob's declared size, or a negative value when it is unknown. A known + * size lets an implementation avoid buffering the whole blob to compute a content length. + */ + void writeAttachment(@NonNull String sha256, long sizeBytes, @NonNull InputStream content) + throws IOException; /** Returns the last durable state, or {@link SyncState#idle()} before the first sync. */ @NonNull diff --git a/app/src/main/java/com/pasich/mynotes/ui/presenter/MainPresenter.java b/app/src/main/java/com/pasich/mynotes/ui/presenter/MainPresenter.java index e98de162..7d9edefa 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/presenter/MainPresenter.java +++ b/app/src/main/java/com/pasich/mynotes/ui/presenter/MainPresenter.java @@ -323,7 +323,7 @@ public void newNotesClick() { getCompositeDisposable() .add( getDataManager() - .addNote(newNote, false) + .addNote(newNote) .subscribeOn(getSchedulerProvider().io()) .observeOn(getSchedulerProvider().ui()) .subscribe( diff --git a/app/src/main/java/com/pasich/mynotes/ui/presenter/dialogs/MoreNoteDialogPresenter.java b/app/src/main/java/com/pasich/mynotes/ui/presenter/dialogs/MoreNoteDialogPresenter.java index fb3f635e..c53e5a0e 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/presenter/dialogs/MoreNoteDialogPresenter.java +++ b/app/src/main/java/com/pasich/mynotes/ui/presenter/dialogs/MoreNoteDialogPresenter.java @@ -116,8 +116,7 @@ public void copyNoteMainActivity() { mNote.getTitle() + " (copy)", mNote.getValue() + " ", new Date().getTime(), - mNote.getTag()), - true) + mNote.getTag())) .subscribeOn(getSchedulerProvider().io()) .subscribe( aLong -> getView().callableCopyNote(aLong), diff --git a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinator.java b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinator.java index 721d0ef9..5cf529b7 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinator.java +++ b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinator.java @@ -9,13 +9,13 @@ import com.pasich.mynotes.data.database.entities.SyncConflictEntity; import com.pasich.mynotes.data.preferences.PreferenceHelper; import com.pasich.mynotes.data.sync.SyncResolution; +import com.pasich.mynotes.data.sync.SyncRollout; import com.pasich.mynotes.data.sync.SyncState; import com.pasich.mynotes.utils.auth.FirebaseGoogleAuth; import com.pasich.mynotes.utils.auth.GoogleCredential; import com.pasich.mynotes.utils.auth.GoogleCredentialAuth; import com.pasich.mynotes.utils.auth.GoogleDriveAuthorization; import java.io.IOException; -import java.security.SecureRandom; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -45,6 +45,9 @@ void resolveConflict(long conflictId, @NonNull SyncResolution resolution) @NonNull SyncState sync(@NonNull String accessToken); + + /** Drops everything tied to the account being disconnected. */ + void clearAfterDisconnect(); } public interface BackgroundScheduler { @@ -95,14 +98,6 @@ public String getAvatarLabel() { private final Executor workerExecutor; private final Executor mainExecutor; - /** - * The v2.6.48 sync safety release completed its staged rollout; sync is now available to all - * cohorts. - */ - private static final int CURRENT_ROLLOUT_PERCENT = 100; - - private static final SecureRandom ROLLOUT_RANDOM = new SecureRandom(); - public SyncCoordinator( @NonNull PreferenceHelper preferenceHelper, @NonNull FirebaseGoogleAuth firebaseGoogleAuth, @@ -138,6 +133,18 @@ public boolean isBackgroundSyncEnabled() { return preferenceHelper.isBackgroundSyncEnabled(); } + /** + * Whether the user has consented to the first upload for the currently connected account. + * + *

The screen must decide whether to ask from this flag, not from a stored "last successful + * sync" timestamp: {@link #disconnect} clears the flag but the timestamp is durable, so the two + * disagreed after any sign-out and the consent dialog became unreachable while {@link #syncNow} + * kept refusing to run. + */ + public boolean isFirstSyncConfirmed() { + return preferenceHelper.isFirstSyncConfirmed(); + } + @NonNull public SyncState getLastState() { try { @@ -181,7 +188,7 @@ public void onSuccess(@NonNull GoogleCredential credential) { new FirebaseGoogleAuth.Callback() { @Override public void onSuccess(@NonNull FirebaseUser user) { - ensureRolloutBucket(); + SyncRollout.ensureBucket(preferenceHelper); preferenceHelper.setSyncEnabled(true); if (preferenceHelper.isBackgroundSyncEnabled() && preferenceHelper.isFirstSyncConfirmed()) { @@ -210,6 +217,21 @@ public void disconnect(@NonNull Callback callback) { preferenceHelper.setBackgroundSyncEnabled(false); preferenceHelper.setFirstSyncConfirmed(false); backgroundScheduler.disable(); + // The stored sync state, the conflict queue and the downloaded blob cache all describe the + // account being disconnected. Leaving them behind also left a lastSuccessfulSyncAt that + // made the next connection look like it had already synced. + // + // Off the main thread: this runs from a button tap and Room refuses main-thread access. + // A failure here must not take the sign-out down with it, so it is logged, not propagated. + runOnWorker( + callback, + () -> { + try { + conflictStore.clearAfterDisconnect(); + } catch (Exception error) { + Log.w(TAG, "Could not clear sync state after disconnect", error); + } + }); googleCredentialAuth.signOut( new GoogleCredentialAuth.SignOutCallback() { @Override @@ -235,8 +257,7 @@ public void syncNow(@NonNull Activity activity, @NonNull Callback cal new IllegalStateException("Confirm the first sync before continuing")); return; } - ensureRolloutBucket(); - if (preferenceHelper.getSyncRolloutBucket() > CURRENT_ROLLOUT_PERCENT) { + if (!SyncRollout.isWithinRollout(preferenceHelper)) { deliverError( callback, new IllegalStateException("Sync is not available in this rollout")); return; @@ -315,7 +336,15 @@ private void deliverError(@NonNull Callback callback, @NonNull Exception erro postToMain(() -> callback.onError(error)); } - /** Delivery to a destroyed screen is a no-op rather than a crash. */ + /** + * Hands the result to the main executor, which owns the decision to drop it. + * + *

Whether delivery to a destroyed screen is safe depends entirely on the executor that was + * injected — an {@code Activity::runOnUiThread} method reference posts to a handler and never + * rejects, so it would happily run the task against destroyed views. {@code + * SyncCoordinatorFactory} supplies a lifecycle-aware executor for that reason; the catch below + * only covers an executor that shuts down instead. + */ private void postToMain(@NonNull Runnable task) { try { mainExecutor.execute(task); @@ -323,11 +352,4 @@ private void postToMain(@NonNull Runnable task) { Log.w(TAG, "Sync result could not be delivered; the screen is gone", rejected); } } - - private void ensureRolloutBucket() { - int bucket = preferenceHelper.getSyncRolloutBucket(); - if (bucket < 1 || bucket > 100) { - preferenceHelper.setSyncRolloutBucket(ROLLOUT_RANDOM.nextInt(100) + 1); - } - } } diff --git a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java index 6e963699..ad80a018 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java +++ b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinatorFactory.java @@ -137,6 +137,11 @@ public SyncState sync(@NonNull String accessToken) { return new SyncService(store) .sync(new GoogleDriveSyncBackend(accessToken)); } + + @Override + public void clearAfterDisconnect() { + store.clearAfterDisconnect(); + } }, new SyncCoordinator.BackgroundScheduler() { @Override @@ -150,10 +155,35 @@ public void disable() { } }, backgroundExecutor, - activity::runOnUiThread); + mainExecutorFor(activity)); return new Result(coordinator, authorization, store); } + /** + * Main-thread delivery that really does drop work aimed at a screen that is gone. + * + *

{@code Activity::runOnUiThread} never throws {@link + * java.util.concurrent.RejectedExecutionException}: it posts to the activity's handler, and the + * task then runs against destroyed views. The rejection handling in {@link SyncCoordinator} + * therefore guarded nothing, and only BackupActivity's own {@code isDestroyed()} checks kept a + * late callback from crashing. The state is re-checked after the post as well, because the + * activity can be torn down while the task sits in the queue. + */ + @NonNull + private static Executor mainExecutorFor(@NonNull Activity activity) { + return command -> { + if (activity.isFinishing() || activity.isDestroyed()) { + return; + } + activity.runOnUiThread( + () -> { + if (!activity.isFinishing() && !activity.isDestroyed()) { + command.run(); + } + }); + }; + } + private static void enableBackgroundSync(Activity activity) { Constraints constraints = new Constraints.Builder() diff --git a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java index 0c756940..1177e552 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java +++ b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java @@ -20,12 +20,6 @@ import androidx.activity.result.contract.ActivityResultContracts; import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import androidx.work.BackoffPolicy; -import androidx.work.Constraints; -import androidx.work.ExistingPeriodicWorkPolicy; -import androidx.work.NetworkType; -import androidx.work.PeriodicWorkRequest; -import androidx.work.WorkManager; import com.google.android.material.dialog.MaterialAlertDialogBuilder; import com.google.android.material.snackbar.Snackbar; import com.google.android.material.tabs.TabLayoutMediator; @@ -41,6 +35,7 @@ import com.pasich.mynotes.data.preferences.PreferenceHelper; import com.pasich.mynotes.data.sync.RoomSyncStore; import com.pasich.mynotes.data.sync.SyncBundleCodec; +import com.pasich.mynotes.data.sync.SyncMetadata; import com.pasich.mynotes.data.sync.SyncResolution; import com.pasich.mynotes.data.sync.SyncSnapshot; import com.pasich.mynotes.data.sync.SyncState; @@ -74,7 +69,6 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import javax.inject.Inject; /** Activity for creating and restoring app data backups. */ @@ -83,7 +77,6 @@ public class BackupActivity extends BaseActivity implements BackupContract.view, AccountSyncFragment.Host { private static final String TAG = "BackupActivity"; - private static final String BACKGROUND_SYNC_WORK_NAME = "mynotes-drive-sync"; @Inject public BackupContract.presenter presenter; @Inject AppDatabase appDatabase; @@ -182,7 +175,6 @@ public void onInvalid(String errorMessage) { private SyncCoordinator syncCoordinator; private final ExecutorService syncExecutor = Executors.newSingleThreadExecutor(); @Inject FirebaseGoogleAuth firebaseGoogleAuth; - private boolean updatingSyncControls; @Override public void onRestoreSuccessFlag() { @@ -200,13 +192,9 @@ public void onCreate(Bundle savedInstanceState) { syncSetup = SyncCoordinatorFactory.create( this, appDatabase, preferenceHelper, firebaseGoogleAuth, syncExecutor); - if (syncSetup == null) { - // No Firebase configuration in this build; GoogleCredentialAuth rejects a blank client - // ID, so the sync controls are hidden instead of crashing the screen. - if (accountTab != null) { - accountTab.showUnavailable(); - } - } + // A build with no Firebase configuration gets a null setup; GoogleCredentialAuth rejects + // a blank client ID. The account tab hides its controls from onAccountTabAttached, which + // is the only point where the fragment actually exists. if (syncSetup != null) { syncCoordinator = syncSetup.getCoordinator(); roomSyncStore = syncSetup.getStore(); @@ -301,27 +289,22 @@ private void renderSyncUi( } private void startSync() { + if (syncCoordinator == null) return; if (!syncCoordinator.getProfile().isSignedIn()) { onInfoSnack( R.string.google_sign_in_failed, null, SnackBarInfo.Error, Snackbar.LENGTH_LONG); return; } - runInBackground( - () -> { - boolean neverSynced = - syncCoordinator.getLastState().getLastSuccessfulSyncAt() == null; - runOnUiThread( - () -> { - if (isFinishing() || isDestroyed()) { - return; - } - if (neverSynced) { - prepareFirstSyncConfirmation(); - } else { - runSync(); - } - }); - }); + // Asked from the same flag SyncCoordinator.syncNow() gates on. Deciding from the stored + // lastSuccessfulSyncAt instead let the two disagree after a sign-out: the timestamp is + // durable, the consent preference is not, so the dialog was skipped and every sync was + // then refused with no way left to give consent. + boolean needsConsent = !syncCoordinator.isFirstSyncConfirmed(); + if (needsConsent) { + prepareFirstSyncConfirmation(); + } else { + runSync(); + } } private void prepareFirstSyncConfirmation() { @@ -364,6 +347,12 @@ private void prepareFirstSyncConfirmation() { long estimatedBytes = bundle.length + attachmentBytes; runOnUiThread( () -> { + // Reading the snapshot hashes every attachment on disk, so + // seconds can pass here. Showing a dialog on a window that is + // already gone throws BadTokenException. + if (isFinishing() || isDestroyed()) { + return; + } syncRunning = false; if (accountTab != null) accountTab.setSyncing(false); new MaterialAlertDialogBuilder(this) @@ -423,6 +412,7 @@ private static String formatBytes(long bytes) { } private void onGoogleSignInClicked() { + if (syncCoordinator == null) return; if (syncCoordinator.getProfile().isSignedIn()) { syncCoordinator.disconnect( new SyncCoordinator.Callback() { @@ -481,30 +471,6 @@ private void finishSync(SyncState state) { } } - private void enableBackgroundSyncWork() { - Constraints constraints = - new Constraints.Builder() - .setRequiredNetworkType(NetworkType.UNMETERED) - .setRequiresBatteryNotLow(true) - .build(); - PeriodicWorkRequest request = - new PeriodicWorkRequest.Builder( - com.pasich.mynotes.data.sync.GoogleDriveSyncWorker.class, - 6, - TimeUnit.HOURS) - .setConstraints(constraints) - .setBackoffCriteria(BackoffPolicy.EXPONENTIAL, 30, TimeUnit.MINUTES) - .build(); - WorkManager.getInstance(getApplicationContext()) - .enqueueUniquePeriodicWork( - BACKGROUND_SYNC_WORK_NAME, ExistingPeriodicWorkPolicy.UPDATE, request); - } - - private void disableBackgroundSyncWork() { - WorkManager.getInstance(getApplicationContext()) - .cancelUniqueWork(BACKGROUND_SYNC_WORK_NAME); - } - private void finishSyncError(Exception error) { if (isFinishing() || isDestroyed()) { return; @@ -531,7 +497,7 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { } private void onBackgroundSyncToggled(boolean enabled) { - if (updatingSyncControls) return; + if (syncCoordinator == null) return; if (!syncCoordinator.getProfile().isSignedIn()) { updateSyncUi(); onInfoSnack( @@ -651,6 +617,7 @@ private String buildConflictMessage(@NonNull SyncConflictEntity conflict) { R.string.sync_conflict_version, getString(R.string.sync_conflict_local_label), describeConflictPayload( + conflict.recordType, conflict.winnerSource.equals("LOCAL") ? conflict.winnerJson : conflict.loserJson)) @@ -659,13 +626,17 @@ private String buildConflictMessage(@NonNull SyncConflictEntity conflict) { R.string.sync_conflict_version, getString(R.string.sync_conflict_drive_label), describeConflictPayload( + conflict.recordType, conflict.winnerSource.equals("REMOTE") ? conflict.winnerJson : conflict.loserJson)); } @NonNull - private String describeConflictPayload(@NonNull String recordJson) { + private String describeConflictPayload(@NonNull String recordType, @NonNull String recordJson) { + if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(recordType)) { + return getString(R.string.settings); + } try { JsonObject root = JsonParser.parseString(recordJson).getAsJsonObject(); JsonElement deletedAt = root.get("deletedAt"); @@ -674,23 +645,39 @@ private String describeConflictPayload(@NonNull String recordJson) { } JsonObject payload = root.getAsJsonObject("payload"); if (payload == null) return getString(R.string.sync_conflict_deleted); - if (payload.has("title") && !payload.get("title").isJsonNull()) { - String title = payload.get("title").getAsString(); - if (!title.trim().isEmpty()) return title.trim(); - } - if (payload.has("name") && !payload.get("name").isJsonNull()) { - String name = payload.get("name").getAsString(); - if (!name.trim().isEmpty()) return name.trim(); - } - if (payload.has("value") && !payload.get("value").isJsonNull()) { - String value = payload.get("value").getAsString().trim(); - if (!value.isEmpty()) { - return value.length() > 120 ? value.substring(0, 120) + "…" : value; - } + for (String key : conflictLabelKeys(recordType)) { + if (!payload.has(key) || payload.get(key).isJsonNull()) continue; + String value = payload.get(key).getAsString().trim(); + if (value.isEmpty()) continue; + return value.length() > 120 ? value.substring(0, 120) + "…" : value; } } catch (Exception ignored) { } - return getString(R.string.sync_status_ready); + return getString(R.string.sync_conflict_untitled); + } + + /** + * Payload keys that carry a human-readable label, most specific first. + * + *

Note and Tag are serialized through Gson's short field aliases, so probing "title" and + * "name" never matched them: every note and tag conflict showed the same placeholder for both + * the local and the Drive version, leaving no way to tell them apart before choosing one. + */ + @NonNull + private static String[] conflictLabelKeys(@NonNull String recordType) { + if (SyncMetadata.RECORD_TYPE_NOTE.equals(recordType)) { + return new String[] {"b", "c"}; // Note.title, Note.value + } + if (SyncMetadata.RECORD_TYPE_TAG.equals(recordType)) { + return new String[] {"b"}; // Tag.nameTag + } + if (SyncMetadata.RECORD_TYPE_TASK.equals(recordType)) { + return new String[] {"title", "description"}; + } + if (SyncMetadata.RECORD_TYPE_CATEGORY.equals(recordType)) { + return new String[] {"name"}; + } + return new String[0]; } private int unresolvedConflictCount(@NonNull List conflicts) { @@ -805,11 +792,15 @@ private boolean finishActivity() { @Override protected void onDestroy() { super.onDestroy(); - // onDestroy also runs on rotation, and a sync started here delivers its callback later. - // Killing the executor then made that callback throw RejectedExecutionException. - if (isFinishing()) { - syncExecutor.shutdown(); - } + // The executor belongs to this instance, so it has to die with it. Sparing it on rotation + // leaked both its live core thread and, through the queued tasks, this activity — once per + // rotation, for the lifetime of the process. + // + // shutdown(), not shutdownNow(): a sync already running is left to finish rather than + // interrupted mid-transaction, and the thread then exits on its own. New submissions are + // refused, which runInBackground already handles, and every delivery re-checks + // isFinishing()/isDestroyed() before touching a view. + syncExecutor.shutdown(); if (isDestroyed()) { presenter.detachView(); } diff --git a/app/src/main/java/com/pasich/mynotes/utils/shareProcessors/SharedNoteCreator.java b/app/src/main/java/com/pasich/mynotes/utils/shareProcessors/SharedNoteCreator.java index eea74738..398fd09e 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/shareProcessors/SharedNoteCreator.java +++ b/app/src/main/java/com/pasich/mynotes/utils/shareProcessors/SharedNoteCreator.java @@ -20,7 +20,7 @@ public SharedNoteCreator(DataManager dataManager) { public void create(String text, Callback callback) { disposables.add( dataManager - .addNote(new Note().create("", text, System.currentTimeMillis(), ""), false) + .addNote(new Note().create("", text, System.currentTimeMillis(), "")) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .subscribe(callback::onCreated, callback::onError)); diff --git a/app/src/main/res/values-be/strings.xml b/app/src/main/res/values-be/strings.xml index 8bf6d0ab..a976e605 100644 --- a/app/src/main/res/values-be/strings.xml +++ b/app/src/main/res/values-be/strings.xml @@ -429,6 +429,7 @@ Лакальная версія Версія з Google Drive Выдалена + Без назвы Наладзіць сінхранізацыю з Google Дыскам Падчас першай сінхранізацыі будзе аб’яднана запісаў: %1$d. Можа быць загружана каля %2$s. Лакальныя даныя застануцца даступнымі. Не цяпер diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index e83b892c..74f0b1d8 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -429,6 +429,7 @@ Lokale Version Google-Drive-Version Gelöscht + Ohne Titel Google-Drive-Synchronisierung einrichten Beim ersten Sync werden %1$d Einträge zusammengeführt und möglicherweise etwa %2$s hochgeladen. Deine lokalen Daten bleiben verfügbar. Nicht jetzt diff --git a/app/src/main/res/values-en-rGB/strings.xml b/app/src/main/res/values-en-rGB/strings.xml index 3d786752..0fe8d1ba 100644 --- a/app/src/main/res/values-en-rGB/strings.xml +++ b/app/src/main/res/values-en-rGB/strings.xml @@ -473,6 +473,7 @@ Local version Google Drive version Deleted + Untitled Set up Google Drive sync This first sync will merge %1$d records and may upload about %2$s. Your local data stays available while the merge completes. Not now diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 26c74021..923b32b2 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -430,6 +430,7 @@ Versión local Versión de Google Drive Eliminado + Sin título Configurar la sincronización con Google Drive La primera sincronización combinará %1$d registros y puede subir unos %2$s. Tus datos locales seguirán disponibles. Ahora no diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index d014797f..4c806ad2 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -425,6 +425,7 @@ Version locale Version Google Drive Supprimé + Sans titre Configurer la synchronisation Google Drive Cette première synchronisation fusionnera %1$d éléments et pourra envoyer environ %2$s. Vos données locales restent disponibles. Pas maintenant diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index b804dbb0..eedd5983 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -427,6 +427,7 @@ Versione locale Versione Google Drive Eliminato + Senza titolo Configura la sincronizzazione Google Drive La prima sincronizzazione unirà %1$d elementi e potrebbe caricare circa %2$s. I dati locali resteranno disponibili. Non ora diff --git a/app/src/main/res/values-kk/strings.xml b/app/src/main/res/values-kk/strings.xml index 0b7aeacf..3591ac93 100644 --- a/app/src/main/res/values-kk/strings.xml +++ b/app/src/main/res/values-kk/strings.xml @@ -426,6 +426,7 @@ Жергілікті нұсқа Google Drive нұсқасы Жойылды + Атауы жоқ Google Drive синхрондауды баптау Алғашқы синхрондау %1$d жазбаны біріктіреді және шамамен %2$s жүктеуі мүмкін. Жергілікті деректер қолжетімді болып қалады. Қазір емес diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index 9d4bfe28..0e6b9e1b 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -430,6 +430,7 @@ Wersja lokalna Wersja z Google Drive Usunięto + Bez tytułu Skonfiguruj synchronizację z Google Drive Pierwsza synchronizacja połączy %1$d wpisów i może wysłać około %2$s. Lokalne dane pozostaną dostępne. Nie teraz diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index e5eb6574..930589f5 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -434,6 +434,7 @@ Локальная версия Версия из Google Drive Удалено + Без названия Настроить синхронизацию с Google Диском При первой синхронизации будут объединены записи: %1$d. Возможно, будет загружено около %2$s. Локальные данные останутся доступны. Не сейчас diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index 380a3b67..78dcd080 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -471,6 +471,7 @@ Локальна версія Версія з Google Drive Видалено + Без назви diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 88546e12..12e64d5c 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -351,6 +351,7 @@ Local Drive Deleted version + Untitled Set up Google Drive sync This first sync will merge %1$d records and may upload about %2$s. Your local data stays available while the merge completes. Not now diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index 62bd7839..2351f597 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -64,8 +64,10 @@ public void writeSnapshot_createsOwnedFolderBundleAndAttachment() throws Excepti assertThat(backend.readSnapshot().getRecords()).isEmpty(); - backend.writeAttachment(hash, new ByteArrayInputStream(attachmentBytes)); - backend.writeAttachment(hash, new ByteArrayInputStream(attachmentBytes)); + backend.writeAttachment( + hash, attachmentBytes.length, new ByteArrayInputStream(attachmentBytes)); + backend.writeAttachment( + hash, attachmentBytes.length, new ByteArrayInputStream(attachmentBytes)); backend.writeSnapshot(snapshot(hash)); assertThat(server.ownedFolderCount()).isEqualTo(1); diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorkerTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorkerTest.java index 857a6807..b170463d 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorkerTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorkerTest.java @@ -2,6 +2,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.pasich.mynotes.data.preferences.PreferenceHelper; @@ -25,7 +26,24 @@ public void backgroundSyncAllowed_acceptsEnabledConfirmedSync() { when(preferences.isSyncEnabled()).thenReturn(true); when(preferences.isBackgroundSyncEnabled()).thenReturn(true); when(preferences.isFirstSyncConfirmed()).thenReturn(true); + when(preferences.getSyncRolloutBucket()).thenReturn(SyncRollout.CURRENT_PERCENT); assertThat(GoogleDriveSyncWorker.isBackgroundSyncAllowed(preferences)).isTrue(); } + + @Test + public void backgroundSyncAllowed_consultsTheRolloutGate() { + // The gate used to sit only on the manual "Sync now" path. Lowering the percentage to pull + // a bad release back would then have left this six-hourly job running for exactly the + // users a rollback needs to stop. + PreferenceHelper preferences = mock(PreferenceHelper.class); + when(preferences.isSyncEnabled()).thenReturn(true); + when(preferences.isBackgroundSyncEnabled()).thenReturn(true); + when(preferences.isFirstSyncConfirmed()).thenReturn(true); + when(preferences.getSyncRolloutBucket()).thenReturn(SyncRollout.CURRENT_PERCENT); + + GoogleDriveSyncWorker.isBackgroundSyncAllowed(preferences); + + verify(preferences).getSyncRolloutBucket(); + } } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java index ef122bfa..e3903718 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java @@ -98,6 +98,92 @@ public void encode_rejectsConflictingAttachmentMetadataForSameHash() { throw new AssertionError("Expected an IOException"); } + @Test + public void decode_keysAttachmentNamesByHashSoTheStoreCanResolveThem() throws Exception { + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] bundle = codec.encode(new SyncSnapshot(Arrays.asList(note("Milk"))), CREATED_AT); + + SyncRecord decoded = + codec.decode(new ByteArrayInputStream(bundle)) + .getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID); + + // RoomSyncStore.restoreAttachments looks names up by content hash. While the decoded map + // stayed keyed by attachment UUID it always missed, and every restored file landed on disk + // named after its bare SHA-256 with no extension. + JsonObject names = decoded.getPayload().getAsJsonObject("attachmentNames"); + assertThat(names.has(HASH)).isTrue(); + assertThat(names.get(HASH).getAsString()).isEqualTo("receipt.png"); + } + + @Test + public void decode_dropsDeviceLocalFieldsWrittenByOlderReleases() throws Exception { + SyncBundleCodec codec = new SyncBundleCodec(); + JsonObject legacy = new JsonObject(); + legacy.addProperty("title", "Buy milk"); + legacy.addProperty("isDone", false); + legacy.addProperty("id", 7); // Room primary key, meaningless on any other device + legacy.addProperty("categoryId", 3); + SyncRecord task = + SyncRecord.live( + SyncRecord.Type.TASK, + TASK_ID, + Instant.parse("2026-08-31T12:00:02Z"), + legacy); + + byte[] bundle = codec.encode(new SyncSnapshot(Arrays.asList(task)), CREATED_AT); + SyncRecord decoded = + codec.decode(new ByteArrayInputStream(bundle)) + .getSnapshot() + .find(SyncRecord.Type.TASK, TASK_ID); + + assertThat(decoded.getPayload().has("id")).isFalse(); + assertThat(decoded.getPayload().has("categoryId")).isFalse(); + assertThat(decoded.getPayload().get("title").getAsString()).isEqualTo("Buy milk"); + } + + @Test + public void decodedRecordMatchesLocalRecordThatNeverCarriedLocalKeys() throws Exception { + // Two devices hold the same logical task under different Room primary keys. Once the + // device-local fields are stripped on both sides the canonical hashes agree, so the + // equal-timestamp tiebreaker in SyncMerger no longer invents a conflict on every sync. + SyncBundleCodec codec = new SyncBundleCodec(); + JsonObject remotePayload = new JsonObject(); + remotePayload.addProperty("title", "Buy milk"); + remotePayload.addProperty("isDone", false); + remotePayload.addProperty("id", 7); + Instant updatedAt = Instant.parse("2026-08-31T12:00:02Z"); + byte[] bundle = + codec.encode( + new SyncSnapshot( + Arrays.asList( + SyncRecord.live( + SyncRecord.Type.TASK, + TASK_ID, + updatedAt, + remotePayload))), + CREATED_AT); + SyncRecord decoded = + codec.decode(new ByteArrayInputStream(bundle)) + .getSnapshot() + .find(SyncRecord.Type.TASK, TASK_ID); + + JsonObject localPayload = new JsonObject(); + localPayload.addProperty("title", "Buy milk"); + localPayload.addProperty("isDone", false); + localPayload.addProperty("id", 12); + SyncMetadata.stripDeviceLocalFields(SyncMetadata.RECORD_TYPE_TASK, localPayload); + SyncRecord local = SyncRecord.live(SyncRecord.Type.TASK, TASK_ID, updatedAt, localPayload); + + assertThat(decoded.getCanonicalPayloadHash()).isEqualTo(local.getCanonicalPayloadHash()); + assertThat(new SyncMerger().merge(snapshotOf(local), snapshotOf(decoded)).getConflicts()) + .isEmpty(); + } + + private static SyncSnapshot snapshotOf(SyncRecord record) { + return new SyncSnapshot(Arrays.asList(record)); + } + private static SyncRecord note(String body) { return SyncRecord.live( SyncRecord.Type.NOTE, diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java index 89113404..f49f99ef 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncConvergenceTest.java @@ -225,7 +225,7 @@ public InputStream readAttachment(String sha256) { } @Override - public void writeAttachment(String sha256, InputStream content) {} + public void writeAttachment(String sha256, long sizeBytes, InputStream content) {} @Override public SyncState readState() { @@ -270,7 +270,8 @@ public InputStream readAttachment(String sha256) { } @Override - public void writeAttachment(String sha256, InputStream content) throws IOException { + public void writeAttachment(String sha256, long sizeBytes, InputStream content) + throws IOException { attachments.put(sha256, new byte[0]); content.close(); } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMetadataTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMetadataTest.java index 0738f6d3..5f0bd6c2 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMetadataTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMetadataTest.java @@ -2,6 +2,7 @@ import static com.google.common.truth.Truth.assertThat; +import com.google.gson.JsonObject; import org.junit.Test; public class SyncMetadataTest { @@ -21,6 +22,42 @@ public void nextUpdatedAt_advancesWhenClockMovesBackwards() { assertThat(SyncMetadata.nextUpdatedAt(100L, 99L)).isEqualTo(101L); } + @Test + public void stripDeviceLocalFields_removesRoomKeysAndAttachmentPaths() { + JsonObject note = new JsonObject(); + note.addProperty("a", 5); // Note.id + note.addProperty("b", "Shopping"); // Note.title + note.addProperty("h", "[{\"url\":\"file://attachments/note_5/x.png\"}]"); + SyncMetadata.stripDeviceLocalFields(SyncMetadata.RECORD_TYPE_NOTE, note); + assertThat(note.has("a")).isFalse(); + assertThat(note.has("h")).isFalse(); + assertThat(note.get("b").getAsString()).isEqualTo("Shopping"); + + JsonObject task = new JsonObject(); + task.addProperty("id", 7); + task.addProperty("categoryId", 3); + task.addProperty("title", "Buy milk"); + SyncMetadata.stripDeviceLocalFields(SyncMetadata.RECORD_TYPE_TASK, task); + assertThat(task.has("id")).isFalse(); + assertThat(task.has("categoryId")).isFalse(); + assertThat(task.get("title").getAsString()).isEqualTo("Buy milk"); + } + + @Test + public void stripDeviceLocalFields_leavesPreferencesUntouched() { + // PreferencesBackup is serialized with the same short Gson aliases as Note, so "a" is the + // format count and "h" is a real setting. Stripping by key without checking the record + // type would silently drop two of the user's settings from every sync. + JsonObject preferences = new JsonObject(); + preferences.addProperty("a", 2); + preferences.addProperty("h", true); + + SyncMetadata.stripDeviceLocalFields(SyncMetadata.RECORD_TYPE_PREFERENCES, preferences); + + assertThat(preferences.get("a").getAsInt()).isEqualTo(2); + assertThat(preferences.get("h").getAsBoolean()).isTrue(); + } + @Test public void supportedRecordTypes_matchSyncSchema() { assertThat(SyncMetadata.isSupportedRecordType(SyncMetadata.RECORD_TYPE_NOTE)).isTrue(); diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java index 0cad01ec..48f4b263 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java @@ -169,6 +169,65 @@ public void insertNotes_usesSingleImportTimestampForWholeBatch() { assertThat(notes.get(1).getId()).isEqualTo(102); } + @Test + public void updateNote_onADeviceWithABackwardsClockStillOutranksWhatItSynced() { + // Merging is last-write-wins on wall-clock time, which reads like "the device whose clock + // runs slow always loses". It does not: applySnapshot copies the winner's timestamp into + // local metadata, and touch() then assigns max(now, stored + 1). So an edit made after + // seeing a newer remote version wins even when this device's clock is hours behind. + // This device's clock reads 1_000; the record it synced carries 5_000 from a device whose + // clock runs ahead. + long remoteTimestamp = 5_000L; + syncMetadataDao.insertIfAbsent( + new SyncMetadataEntity( + SyncMetadata.RECORD_TYPE_NOTE, 1L, "stable-a", remoteTimestamp, null)); + + Note note = new Note().create("Edited here", "text", 1L, ""); + note.setId(1); + coordinator.updateNoteContent(note); + + SyncMetadataEntity metadata = syncMetadataDao.get(SyncMetadata.RECORD_TYPE_NOTE, 1L); + assertThat(metadata.updatedAt).isGreaterThan(remoteTimestamp); + } + + @Test + public void insertNotes_keepsBackupIdsWhenNothingOccupiesThem() { + // The ordinary restore, and the one after a reinstall: an empty library, so every note + // keeps the ID it had when the backup was taken. + List notes = new ArrayList<>(); + Note restored = new Note().create("One", "1", 1L, ""); + restored.setId(7); + notes.add(restored); + when(noteDao.getNoteSync(7)).thenReturn(null); + when(noteDao.addNotes(anyList())).thenReturn(new long[] {7L}); + + coordinator.insertNotes(notes); + + assertThat(notes.get(0).getId()).isEqualTo(7); + assertThat(syncMetadataDao.get(SyncMetadata.RECORD_TYPE_NOTE, 7L)).isNotNull(); + } + + @Test + public void insertNotes_doesNotOverwriteAnExistingNoteThatHoldsTheSameId() { + // addNotes is a REPLACE insert and backups carry their original IDs, so restoring onto a + // device that already has notes used to destroy every colliding one — and hand its stable + // ID to the replacement, propagating the loss to every other device. The restored note is + // inserted as a new row instead; nothing existing is touched. + List notes = new ArrayList<>(); + Note restored = new Note().create("Restored", "r", 1L, ""); + restored.setId(7); + notes.add(restored); + Note occupant = new Note().create("Already here", "x", 2L, ""); + occupant.setId(7); + when(noteDao.getNoteSync(7)).thenReturn(occupant); + when(noteDao.addNotes(anyList())).thenReturn(new long[] {42L}); + + coordinator.insertNotes(notes); + + assertThat(notes.get(0).getId()).isEqualTo(42); + assertThat(syncMetadataDao.get(SyncMetadata.RECORD_TYPE_NOTE, 42L)).isNotNull(); + } + @Test public void deleteTask_keepsTombstoneAndMarksDeletionTimestamp() { syncMetadataDao.insertIfAbsent( diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncRolloutTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncRolloutTest.java new file mode 100644 index 00000000..cf5e1d68 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncRolloutTest.java @@ -0,0 +1,46 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.pasich.mynotes.data.preferences.PreferenceHelper; +import org.junit.Test; + +public class SyncRolloutTest { + + @Test + public void ensureBucket_keepsAnAlreadyAssignedCohort() { + PreferenceHelper preferences = mock(PreferenceHelper.class); + when(preferences.getSyncRolloutBucket()).thenReturn(37); + + assertThat(SyncRollout.ensureBucket(preferences)).isEqualTo(37); + } + + @Test + public void ensureBucket_assignsAValidCohortWhenStoredValueIsOutOfRange() { + for (int stored : new int[] {-1, 0, 101}) { + PreferenceHelper preferences = mock(PreferenceHelper.class); + when(preferences.getSyncRolloutBucket()).thenReturn(stored); + + int bucket = SyncRollout.ensureBucket(preferences); + + assertThat(bucket).isAtLeast(1); + assertThat(bucket).isAtMost(100); + } + } + + @Test + public void isWithinRollout_followsTheCurrentPercentage() { + PreferenceHelper inside = mock(PreferenceHelper.class); + when(inside.getSyncRolloutBucket()).thenReturn(SyncRollout.CURRENT_PERCENT); + assertThat(SyncRollout.isWithinRollout(inside)).isTrue(); + + // Only meaningful once the percentage is dialled back below 100 for a rollback. + if (SyncRollout.CURRENT_PERCENT < 100) { + PreferenceHelper outside = mock(PreferenceHelper.class); + when(outside.getSyncRolloutBucket()).thenReturn(SyncRollout.CURRENT_PERCENT + 1); + assertThat(SyncRollout.isWithinRollout(outside)).isFalse(); + } + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java index 11061045..ae5dd9eb 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java @@ -125,6 +125,27 @@ public void sync_skipsUploadingAttachmentWhenRemoteBlobAlreadyExists() throws Ex SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); + // Neither endpoint transfers the blob: the local copy is verified from disk and the + // remote one is content-addressed and immutable, so re-downloading it on every sync only + // cost the user bandwidth. + assertThat(backend.events).containsExactly("writeSnapshot"); + } + + @Test + public void sync_repairsCorruptLocalAttachmentFromRemote() throws Exception { + SyncRecord local = note(TEN, "Local with corrupt attachment"); + byte[] bytes = "local attachment".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + FakeStore store = new FakeStore(snapshot(local)); + store.attachmentHashes = Collections.singletonList(hash); + store.attachments.put(hash, "corrupted".getBytes(StandardCharsets.UTF_8)); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + backend.attachments.put(hash, bytes); + + SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); + assertThat(store.attachments.get(hash)).isEqualTo(bytes); assertThat(backend.events).containsExactly("readAttachment", "writeSnapshot").inOrder(); } @@ -146,7 +167,11 @@ public void sync_invalidAttachmentDoesNotPublishOrApplySnapshot() { assertThat(state.getErrorMessage()).contains("checksum"); assertThat(backend.writeSnapshotCalls).isEqualTo(0); assertThat(store.applyCalls).isEqualTo(0); - assertThat(store.events).isEmpty(); + // The blob is streamed rather than buffered, so the write is entered before the digest can + // be checked — the mismatch surfaces at end of stream, inside the destination's own read + // loop. What still must hold is that nothing was committed: RoomSyncStore writes to a + // temporary file and only renames it once the stream completed cleanly. + assertThat(store.attachments).isEmpty(); } @Test @@ -312,7 +337,8 @@ public InputStream readAttachment(String sha256) throws IOException { } @Override - public void writeAttachment(String sha256, InputStream content) throws IOException { + public void writeAttachment(String sha256, long sizeBytes, InputStream content) + throws IOException { events.add("writeAttachment"); attachments.put(sha256, readAll(content)); } @@ -380,7 +406,8 @@ public InputStream readAttachment(String sha256) { } @Override - public void writeAttachment(String sha256, InputStream content) throws IOException { + public void writeAttachment(String sha256, long sizeBytes, InputStream content) + throws IOException { events.add("writeAttachment"); attachments.put(sha256, readAll(content)); } diff --git a/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java index be1512e1..a4991ad5 100644 --- a/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java @@ -189,6 +189,49 @@ public void syncNow_requiresFirstSyncConfirmationBeforeAuthorizing() { Mockito.verifyNoInteractions(authorization); } + @Test + public void disconnect_clearsConsentAndStoredStateTogether() { + // The screen asks for first-sync consent when isFirstSyncConfirmed() is false, and + // syncNow() refuses while it is false. Both must flip together on a sign-out, and the + // durable state has to go with them: leaving a lastSuccessfulSyncAt behind is what used to + // make the next connection look already-synced, skipping a dialog that alone could restore + // the consent flag. Manual and background sync were then both dead until app data reset. + FakePreferenceHelper preferences = new FakePreferenceHelper(); + preferences.firstSyncConfirmed = true; + preferences.syncEnabled = true; + preferences.backgroundEnabled = true; + FakeConflictStore store = new FakeConflictStore(); + store.state = SyncState.success("google-drive", Instant.parse("2026-09-01T12:00:00Z"), 0); + GoogleCredentialAuth credentialAuth = mock(GoogleCredentialAuth.class); + Mockito.doAnswer( + invocation -> { + GoogleCredentialAuth.SignOutCallback callback = + invocation.getArgument(0); + callback.onSuccess(); + return null; + }) + .when(credentialAuth) + .signOut(Mockito.any(GoogleCredentialAuth.SignOutCallback.class)); + FakeScheduler scheduler = new FakeScheduler(); + SyncCoordinator coordinator = + new SyncCoordinator( + preferences, + firebaseAuth(mock(FirebaseUser.class)), + credentialAuth, + mock(GoogleDriveAuthorization.class), + store, + scheduler, + directExecutor, + directExecutor); + + coordinator.disconnect(new CapturingCallback<>()); + + assertThat(coordinator.isFirstSyncConfirmed()).isFalse(); + assertThat(store.clearCalls).isEqualTo(1); + assertThat(coordinator.getLastState().getLastSuccessfulSyncAt()).isNull(); + assertThat(scheduler.disableCalls).isAtLeast(1); + } + @Test public void syncNow_allowsUsersInTheHighestRolloutBucket() { FakePreferenceHelper preferences = new FakePreferenceHelper(); @@ -340,6 +383,7 @@ private static final class FakeConflictStore implements SyncCoordinator.Conflict private final List conflicts = new ArrayList<>(); private final List resolutions = new ArrayList<>(); private String lastToken; + private int clearCalls; @NonNull @Override @@ -367,6 +411,13 @@ public SyncState sync(@NonNull String accessToken) { lastToken = accessToken; return state; } + + @Override + public void clearAfterDisconnect() { + clearCalls++; + state = SyncState.idle(); + conflicts.clear(); + } } private static final class CapturingCallback implements SyncCoordinator.Callback { From b9ffa7827a5b47f4ec9bb676bde2c2f7cededb73 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 13:13:56 +0300 Subject: [PATCH 02/16] fix: harden Drive sync transport --- .github/workflows/ci-cd.yml | 3 + .github/workflows/ci.yml | 32 +- .gitignore | 2 + app/build.gradle | 2 +- .../pasich/mynotes/db/RoomSyncStoreTest.java | 170 +++++ app/src/main/AndroidManifest.xml | 10 +- .../pasich/mynotes/data/AppDataManager.java | 10 - .../preferences/AppPreferencesHelper.java | 12 - .../data/preferences/PreferenceHelper.java | 4 - .../data/sync/DriveRequestExecutor.java | 165 +++++ .../data/sync/GoogleDriveSyncBackend.java | 629 ++++++++++++++---- .../data/sync/GoogleDriveSyncWorker.java | 10 +- .../mynotes/data/sync/RoomSyncStore.java | 237 +++++-- .../data/sync/SnapshotBuildResult.java | 73 ++ .../mynotes/data/sync/SnapshotProblem.java | 41 ++ .../data/sync/SyncBundleValidator.java | 98 ++- .../pasich/mynotes/data/sync/SyncRollout.java | 49 -- .../pasich/mynotes/data/sync/SyncService.java | 15 +- .../pasich/mynotes/data/sync/SyncStore.java | 9 + .../attach/AttachmentStorage.java | 14 +- .../mynotes/ui/sync/SyncCoordinator.java | 7 - .../constants/settings/PreferencesConfig.java | 2 - .../data/sync/DriveRequestExecutorTest.java | 95 +++ .../data/sync/GoogleDriveSyncBackendTest.java | 339 +++++++++- .../data/sync/GoogleDriveSyncWorkerTest.java | 13 +- .../data/sync/SyncBundleValidatorTest.java | 51 ++ .../mynotes/data/sync/SyncRolloutTest.java | 46 -- .../mynotes/data/sync/SyncServiceTest.java | 59 +- .../mynotes/ui/sync/SyncCoordinatorTest.java | 18 +- 29 files changed, 1850 insertions(+), 365 deletions(-) create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/DriveRequestExecutor.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java delete mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/SyncRollout.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/DriveRequestExecutorTest.java delete mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/SyncRolloutTest.java diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index b3017487..11d248b0 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -73,6 +73,9 @@ jobs: - name: Decode keystore run: echo "${{ secrets.KEYSTORE_BASE64 }}" | base64 -d > /tmp/release.jks + - name: Run release safety gate + run: ./gradlew test lint assembleRelease --no-daemon --stacktrace + - name: Build signed release APK & AAB env: KEYSTORE_PATH: /tmp/release.jks diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 150af5c9..8f788d4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,8 +88,11 @@ jobs: - name: Build debug run: ./gradlew :app:assembleDebug --no-daemon --stacktrace + - name: Compile release + run: ./gradlew :app:assembleRelease --no-daemon --stacktrace + - name: Run unit tests and generate coverage - run: ./gradlew :app:testDebugUnitTest :app:createDebugUnitTestCoverageReport --no-daemon --stacktrace + run: ./gradlew :app:test :app:createDebugUnitTestCoverageReport --no-daemon --stacktrace - name: Upload debug unit-test coverage if: always() @@ -104,3 +107,30 @@ jobs: - name: Check Java formatting run: ./gradlew :app:spotlessCheck --no-daemon --stacktrace + + instrumentation: + name: Android instrumentation and Room migrations + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '17' + + - name: Set up Android SDK + uses: android-actions/setup-android@v4 + with: + log-accepted-android-sdk-licenses: 'false' + + - name: Run sync integration and migration tests + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 35 + target: google_apis + arch: x86_64 + script: ./gradlew :app:connectedDebugAndroidTest --no-daemon --stacktrace diff --git a/.gitignore b/.gitignore index d30974a5..ce75fe6b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ gha-creds-*.json # Local design/plan notes, deliberately kept out of the repository docs/ +!docs/ +!docs/google-drive-sync-invariants.md diff --git a/app/build.gradle b/app/build.gradle index a504a954..74d86e45 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -128,7 +128,7 @@ android { } namespace = 'com.pasich.mynotes' lint { - abortOnError false + abortOnError true } testOptions { unitTests { diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java index 81d0c722..d50e3a9d 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java @@ -13,6 +13,8 @@ import com.pasich.mynotes.data.model.Note; import com.pasich.mynotes.data.preferences.PreferenceHelper; import com.pasich.mynotes.data.sync.RoomSyncStore; +import com.pasich.mynotes.data.sync.SnapshotBuildResult; +import com.pasich.mynotes.data.sync.SnapshotProblem; import com.pasich.mynotes.data.sync.SyncMetadata; import com.pasich.mynotes.data.sync.SyncRecord; import com.pasich.mynotes.data.sync.SyncSnapshot; @@ -97,6 +99,109 @@ public void hasAttachment_findsABlobThatOnlyExistsInTheNotesOwnFolder() throws E assertThat(noteId).isGreaterThan(0); } + @Test + public void readSnapshot_failsClosedWhenAReferencedAttachmentIsMissing() { + int noteId = + seedNote("Missing attachment", "body", "[" + attachmentJson(1, "gone.png") + "]"); + String original = db.noteDao().getNoteSync(noteId).getAttachments(); + + SnapshotBuildResult.SnapshotBuildException error = assertSnapshotBuildFails(store); + + assertThat(error.getProblems().get(0).getKind()) + .isEqualTo(SnapshotProblem.Kind.MISSING_ATTACHMENT); + assertThat(db.noteDao().getNoteSync(noteId).getAttachments()).isEqualTo(original); + } + + @Test + public void readSnapshot_failsClosedWhenAnAttachmentCannotBeRead() throws Exception { + int noteId = seedNoteWithAttachment("locked.png", "bytes".getBytes(StandardCharsets.UTF_8)); + File real = new File(context.getFilesDir(), "attachments/note_" + noteId + "/locked.png"); + File unreadable = + new File(real.getAbsolutePath()) { + @Override + public boolean canRead() { + return false; + } + }; + RoomSyncStore failingStore = + new RoomSyncStore( + context, + db, + mock(PreferenceHelper.class), + (ignoredContext, ignoredAttachment) -> unreadable, + file -> sha256(readAll(new java.io.FileInputStream(file)))); + + SnapshotBuildResult.SnapshotBuildException error = assertSnapshotBuildFails(failingStore); + + assertThat(error.getProblems().get(0).getKind()) + .isEqualTo(SnapshotProblem.Kind.UNREADABLE_ATTACHMENT); + } + + @Test + public void readSnapshot_failsClosedWhenAttachmentHashingFails() throws Exception { + int noteId = seedNoteWithAttachment("hash.png", "bytes".getBytes(StandardCharsets.UTF_8)); + String original = db.noteDao().getNoteSync(noteId).getAttachments(); + RoomSyncStore failingStore = + new RoomSyncStore( + context, + db, + mock(PreferenceHelper.class), + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage::resolve, + file -> { + throw new IOException("cannot hash"); + }); + + SnapshotBuildResult.SnapshotBuildException error = assertSnapshotBuildFails(failingStore); + + assertThat(error.getProblems().get(0).getKind()) + .isEqualTo(SnapshotProblem.Kind.ATTACHMENT_HASH_FAILED); + assertThat(db.noteDao().getNoteSync(noteId).getAttachments()).isEqualTo(original); + } + + @Test + public void readSnapshot_rejectsTheWholeSnapshotWhenOneOfFiveAttachmentsIsMissing() + throws Exception { + int noteId = seedNote("Five attachments", "body", null); + File folder = new File(context.getFilesDir(), "attachments/note_" + noteId); + assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); + StringBuilder json = new StringBuilder("["); + for (int index = 0; index < 5; index++) { + String name = "item-" + index + ".png"; + if (index > 0) json.append(','); + json.append(attachmentJson(noteId, name)); + if (index < 4) { + try (FileOutputStream out = new FileOutputStream(new File(folder, name))) { + out.write(("bytes-" + index).getBytes(StandardCharsets.UTF_8)); + } + } + } + json.append(']'); + Note note = db.noteDao().getNoteSync(noteId); + note.setAttachments(json.toString()); + db.noteDao().addNote(note); + + SnapshotBuildResult.SnapshotBuildException error = assertSnapshotBuildFails(store); + + assertThat(error.getProblems()).isNotEmpty(); + assertThat(error.getProblems().get(0).getKind()) + .isEqualTo(SnapshotProblem.Kind.MISSING_ATTACHMENT); + assertThat(db.noteDao().getNoteSync(noteId).getAttachments()).isEqualTo(json.toString()); + } + + @Test + public void readSnapshot_acceptsANoteWithNoAttachments() throws Exception { + seedNote("No attachments", "body", "[]"); + + SnapshotBuildResult result = store.buildSnapshot(); + + assertThat(result.isPublishable()).isTrue(); + assertThat( + onlyNote(result.requireSnapshot()) + .getPayload() + .getAsJsonArray("attachmentHashes")) + .isEmpty(); + } + @Test public void writeAttachment_leavesNothingBehindWhenTheStreamFails() { String hash = sha256("whatever".getBytes(StandardCharsets.UTF_8)); @@ -189,6 +294,49 @@ public void touch_clearsATombstoneSoAReusedRowIsNotResurrectedAsDeleted() { assertThat(metadata.updatedAt).isEqualTo(200L); } + @Test + public void applySnapshot_rollsBackNotesMetadataAndStateWhenARecordMutationFails() + throws Exception { + int noteId = seedNote("Original", "body", null); + SyncRecord original = onlyNote(store.readSnapshot()); + JsonObject changedPayload = original.getPayload(); + changedPayload.addProperty("b", "Remote title"); + SyncSnapshot remote = + new SyncSnapshot( + Collections.singletonList( + SyncRecord.live( + SyncRecord.Type.NOTE, + original.getId(), + java.time.Instant.ofEpochMilli(2_000L), + changedPayload))); + RoomSyncStore failingStore = + new RoomSyncStore( + context, + db, + mock(PreferenceHelper.class), + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage::resolve, + file -> sha256(readAll(new java.io.FileInputStream(file))), + record -> { + throw new IllegalStateException("injected apply failure"); + }); + + try { + failingStore.applySnapshot( + remote, + Collections.emptyList(), + SyncState.success("google-drive", java.time.Instant.now(), 0)); + throw new AssertionError("Expected injected transaction failure"); + } catch (IllegalStateException expected) { + assertThat(expected).hasMessageThat().contains("injected apply failure"); + } + + assertThat(db.noteDao().getNoteSync(noteId).getTitle()).isEqualTo("Original"); + SyncMetadataEntity metadata = db.syncMetadataDao().get("note", noteId); + assertThat(metadata.updatedAt).isEqualTo(1_000L); + assertThat(store.readState().getStatus()).isEqualTo(SyncState.Status.IDLE); + assertThat(store.getConflicts()).isEmpty(); + } + // ---- helpers ---- private int seedNote(String title, String value, String attachmentsJson) { @@ -234,6 +382,28 @@ private static SyncRecord onlyNote(SyncSnapshot snapshot) { return notes.get(0); } + private static SnapshotBuildResult.SnapshotBuildException assertSnapshotBuildFails( + RoomSyncStore store) { + try { + store.readSnapshot(); + throw new AssertionError("Expected a local snapshot build failure"); + } catch (SnapshotBuildResult.SnapshotBuildException expected) { + return expected; + } catch (IOException unexpected) { + throw new AssertionError(unexpected); + } + } + + private static String attachmentJson(int noteId, String name) { + return "{\"url\":\"file://attachments/note_" + + noteId + + "/" + + name + + "\",\"name\":\"" + + name + + "\"}"; + } + private static byte[] readAll(InputStream input) throws IOException { try (InputStream stream = input; java.io.ByteArrayOutputStream output = new java.io.ByteArrayOutputStream()) { diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 2f0053ee..3705607a 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -41,7 +41,7 @@ android:exported="false" /> + android:exported="false" /> @@ -92,7 +92,7 @@ + android:exported="false"> @@ -104,11 +104,11 @@ android:exported="false" /> - \ No newline at end of file + diff --git a/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java b/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java index bb28edf3..8c7e2010 100644 --- a/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java +++ b/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java @@ -124,16 +124,6 @@ public void setFirstSyncConfirmed(boolean confirmed) { preferencesHelper.setFirstSyncConfirmed(confirmed); } - @Override - public int getSyncRolloutBucket() { - return preferencesHelper.getSyncRolloutBucket(); - } - - @Override - public void setSyncRolloutBucket(int bucket) { - preferencesHelper.setSyncRolloutBucket(bucket); - } - @Override public String getTypeFaceNoteActivity() { return preferencesHelper.getTypeFaceNoteActivity(); diff --git a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java index 19e93940..6cebac0a 100644 --- a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java +++ b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java @@ -187,16 +187,4 @@ public boolean isFirstSyncConfirmed() { public void setFirstSyncConfirmed(boolean confirmed) { prefs.putBoolean(PreferencesConfig.ARGUMENT_PREFERENCE_SYNC_FIRST_CONFIRMED, confirmed); } - - @Override - public int getSyncRolloutBucket() { - return prefs.getInt( - PreferencesConfig.ARGUMENT_PREFERENCE_SYNC_ROLLOUT_BUCKET, - PreferencesConfig.ARGUMENT_DEFAULT_SYNC_ROLLOUT_BUCKET); - } - - @Override - public void setSyncRolloutBucket(int bucket) { - prefs.putInt(PreferencesConfig.ARGUMENT_PREFERENCE_SYNC_ROLLOUT_BUCKET, bucket); - } } diff --git a/app/src/main/java/com/pasich/mynotes/data/preferences/PreferenceHelper.java b/app/src/main/java/com/pasich/mynotes/data/preferences/PreferenceHelper.java index b0e053f6..6665b753 100644 --- a/app/src/main/java/com/pasich/mynotes/data/preferences/PreferenceHelper.java +++ b/app/src/main/java/com/pasich/mynotes/data/preferences/PreferenceHelper.java @@ -37,8 +37,4 @@ public interface PreferenceHelper { boolean isFirstSyncConfirmed(); void setFirstSyncConfirmed(boolean confirmed); - - int getSyncRolloutBucket(); - - void setSyncRolloutBucket(int bucket); } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/DriveRequestExecutor.java b/app/src/main/java/com/pasich/mynotes/data/sync/DriveRequestExecutor.java new file mode 100644 index 00000000..ae8e76f7 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/DriveRequestExecutor.java @@ -0,0 +1,165 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.ConnectException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; +import java.security.SecureRandom; +import java.time.Clock; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +/** Shared retry policy for idempotent Google Drive requests. */ +final class DriveRequestExecutor { + + static final int MAX_ATTEMPTS = 4; + private static final long INITIAL_BACKOFF_MS = 250L; + private static final long MAX_BACKOFF_MS = 4_000L; + + interface Request { + T execute() throws IOException; + } + + interface Sleeper { + void sleep(long durationMs) throws InterruptedException; + } + + interface Jitter { + long nextLong(long upperExclusive); + } + + private final Clock clock; + private final Sleeper sleeper; + private final Jitter jitter; + + DriveRequestExecutor() { + this( + Clock.systemUTC(), + Thread::sleep, + upperExclusive -> new SecureRandom().nextInt((int) upperExclusive)); + } + + DriveRequestExecutor(@NonNull Clock clock, @NonNull Sleeper sleeper, @NonNull Jitter jitter) { + this.clock = clock; + this.sleeper = sleeper; + this.jitter = jitter; + } + + /** + * Executes only requests whose repeated execution cannot overwrite or duplicate logical data. + * Create/upload requests deliberately use post-failure discovery instead of this method. + */ + T executeIdempotent(@NonNull Request request) throws IOException { + IOException lastFailure = null; + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + throwIfInterrupted(); + try { + return request.execute(); + } catch (IOException failure) { + lastFailure = failure; + if (attempt == MAX_ATTEMPTS || !isRetryable(failure)) { + throw failure; + } + sleep(backoffDelayMs(attempt, retryAfterMs(failure))); + } + } + throw lastFailure == null ? new IOException("Drive request failed") : lastFailure; + } + + private void sleep(long delayMs) throws IOException { + try { + sleeper.sleep(delayMs); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + InterruptedIOException interrupted = + new InterruptedIOException("Drive request interrupted"); + interrupted.initCause(error); + throw interrupted; + } + throwIfInterrupted(); + } + + private static void throwIfInterrupted() throws InterruptedIOException { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedIOException("Drive request interrupted"); + } + } + + private long backoffDelayMs(int attempt, long retryAfterMs) { + if (retryAfterMs >= 0L) { + return Math.min(retryAfterMs, MAX_BACKOFF_MS); + } + long exponential = Math.min(MAX_BACKOFF_MS, INITIAL_BACKOFF_MS << (attempt - 1)); + return exponential / 2L + jitter.nextLong(exponential / 2L + 1L); + } + + private long retryAfterMs(@NonNull IOException failure) { + if (!(failure instanceof DriveHttpException)) { + return -1L; + } + return ((DriveHttpException) failure).retryAfterMs(clock.millis()); + } + + static boolean isRetryable(@NonNull IOException failure) { + if (failure instanceof InterruptedIOException) { + return !Thread.currentThread().isInterrupted() + && !(failure instanceof SocketTimeoutException + && Thread.currentThread().isInterrupted()); + } + if (failure instanceof DriveHttpException) { + int status = ((DriveHttpException) failure).statusCode; + return status == 429 + || status == 500 + || status == 502 + || status == 503 + || status == 504 + || (status == 403 && ((DriveHttpException) failure).isRateLimit()); + } + return failure instanceof ConnectException + || failure instanceof SocketException + || failure instanceof UnknownHostException; + } + + static final class DriveHttpException extends IOException { + final int statusCode; + @Nullable final String retryAfter; + @NonNull final String detail; + + DriveHttpException(int statusCode, @Nullable String retryAfter, @NonNull String detail) { + super("Drive API HTTP " + statusCode + (detail.isEmpty() ? "" : ": " + detail)); + this.statusCode = statusCode; + this.retryAfter = retryAfter; + this.detail = detail; + } + + boolean isRateLimit() { + String normalized = detail.toLowerCase(); + return normalized.contains("ratelimit") || normalized.contains("rate limit"); + } + + long retryAfterMs(long nowMs) { + if (retryAfter == null || retryAfter.trim().isEmpty()) { + return -1L; + } + String value = retryAfter.trim(); + try { + return Math.max(0L, Math.multiplyExact(Long.parseLong(value), 1_000L)); + } catch (NumberFormatException | ArithmeticException ignored) { + try { + return Math.max( + 0L, + ZonedDateTime.parse(value, DateTimeFormatter.RFC_1123_DATE_TIME) + .toInstant() + .toEpochMilli() + - nowMs); + } catch (RuntimeException malformedDate) { + return -1L; + } + } + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index d9e8a191..fd22ada3 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -15,10 +15,14 @@ import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Clock; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; /** Google Drive REST backend for the provider-independent sync protocol. */ @@ -33,6 +37,8 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private static final String MIME_BINARY = "application/octet-stream"; private static final int MAX_BUNDLE_RESPONSE_BYTES = 32 * 1024 * 1024; private static final int MAX_ATTACHMENT_RESPONSE_BYTES = 100 * 1024 * 1024; + private static final int RESUMABLE_CHUNK_BYTES = 256 * 1024; + private static final int HTTP_RESUME_INCOMPLETE = 308; private static final int MAX_ERROR_DETAIL_BYTES = 1024; private static final int MAX_ERROR_DETAIL_CHARS = 200; private static final Gson GSON = new Gson(); @@ -42,15 +48,9 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private final String uploadBase; private final Clock clock; private final SyncBundleCodec bundleCodec; + private final DriveRequestExecutor requestExecutor; private final SyncMerger merger = new SyncMerger(); - /** - * Bundles merged by this instance's {@link #readSnapshot()}, safe to delete once their content - * has been republished. One backend instance serves exactly one sync, so this can never name a - * bundle that arrived after the read. - */ - @Nullable private List supersededBundleIds; - public GoogleDriveSyncBackend(@NonNull String accessToken) { this(accessToken, DEFAULT_API, DEFAULT_UPLOAD, Clock.systemUTC(), new SyncBundleCodec()); } @@ -69,6 +69,7 @@ public GoogleDriveSyncBackend(@NonNull String accessToken) { this.uploadBase = uploadBase; this.clock = clock; this.bundleCodec = bundleCodec; + this.requestExecutor = new DriveRequestExecutor(); } @NonNull @@ -80,114 +81,113 @@ public String getIdentifier() { @NonNull @Override public synchronized SyncSnapshot readSnapshot() throws IOException { - String folderId = findFolderId(); - if (folderId == null) { + List folderIds = findFolderIds(); + if (folderIds.isEmpty()) { return SyncSnapshot.empty(); } SyncSnapshot merged = SyncSnapshot.empty(); - List readBundleIds = new ArrayList<>(); - for (String bundleId : findBundles(folderId)) { - byte[] bytes = - requestBytes( - "GET", - apiBase + "/files/" + bundleId + "?alt=media", - MAX_BUNDLE_RESPONSE_BYTES); - SyncSnapshot decoded = - bundleCodec.decode(new ByteArrayInputStream(bytes)).getSnapshot(); - merged = merger.merge(merged, decoded).getMergedSnapshot(); - readBundleIds.add(bundleId); - } - supersededBundleIds = readBundleIds; + for (String folderId : folderIds) { + for (String bundleId : findBundles(folderId)) { + byte[] bytes = + requestBytes( + "GET", + apiBase + "/files/" + bundleId + "?alt=media", + MAX_BUNDLE_RESPONSE_BYTES); + SyncSnapshot decoded = + bundleCodec.decode(new ByteArrayInputStream(bytes)).getSnapshot(); + merged = merger.merge(merged, decoded).getMergedSnapshot(); + } + } return merged; } @Override public synchronized void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IOException { - String folderId = ensureFolderId(); + String folderId = ensureCanonicalFolderId(); + // A first-sync race can leave valid bundles and immutable blobs in two owned folders. + // The read path always merges all roots. Before canonical publication, materialize every + // referenced blob in the canonical root as well, so no future cleanup decision can make + // the canonical bundle point at an object that exists only in a duplicate root. + ensureCanonicalAttachments(folderId, snapshot); byte[] bundle = bundleCodec.encode(snapshot, clock.instant()); // Every bundle is immutable. Drive offers no conditional update based on its version // counter, so replacing one file leaves a race where another device can be overwritten. // Publishing a distinct file makes each successful upload independently durable; readers // merge the complete set deterministically. - uploadFile(folderId, nextBundleName(), MIME_ZIP, bundle, true); - discardSupersededBundles(); - } - - /** - * Removes the bundles whose content the just-published bundle already contains. - * - *

Without this, every sync that changed anything left one more full snapshot in Drive - * forever, and each later {@link #readSnapshot()} downloaded all of them. Cost grew without - * bound: a user syncing daily for a year would download 365 bundles per sync. - * - *

Only the IDs {@link #readSnapshot()} actually merged in this same sync are removed, so a - * bundle another device published in the meantime is never discarded unread. Deletion is - * best-effort: the new bundle is already durable, and a failure here only postpones cleanup. - */ - private void discardSupersededBundles() { - List superseded = supersededBundleIds; - supersededBundleIds = null; - if (superseded == null) { - return; - } - for (String bundleId : superseded) { - try { - HttpURLConnection connection = open("DELETE", apiBase + "/files/" + bundleId); - ensureSuccess(connection); - connection.disconnect(); - } catch (IOException ignored) { - // Another device may have collected it already. + String bundleName = nextBundleName(); + try { + uploadFile(folderId, bundleName, MIME_ZIP, bundle, true); + } catch (IOException uploadFailure) { + // POST is deliberately not blindly retried. The server may have accepted the upload + // before the client lost its response; rediscovering the unique name makes that + // outcome successful without publishing a second logical bundle. + if (!hasBundleNamed(folderId, bundleName)) { + throw uploadFailure; } } } @Override public synchronized boolean hasAttachment(@NonNull String sha256) throws IOException { - String folderId = findFolderId(); - return folderId != null && findAttachment(folderId, sha256) != null; + for (String folderId : findFolderIds()) { + if (findAttachment(folderId, sha256) != null) { + return true; + } + } + return false; } @Nullable @Override public synchronized InputStream readAttachment(@NonNull String sha256) throws IOException { - String folderId = findFolderId(); - if (folderId == null) { - return null; - } - - String attachmentId = findAttachment(folderId, sha256); - if (attachmentId == null) { - return null; + for (String folderId : findFolderIds()) { + String attachmentId = findAttachment(folderId, sha256); + if (attachmentId == null) { + continue; + } + // Streamed, not buffered: reading a 100 MB attachment into a byte[] (which the growing + // ByteArrayOutputStream first doubled, then copied) was the largest single allocation + // in + // the sync and an OutOfMemoryError on an ordinary phone. + HttpURLConnection connection = + requestExecutor.executeIdempotent( + () -> + openSuccessful( + "GET", + apiBase + "/files/" + attachmentId + "?alt=media")); + try { + return new ConnectionInputStream(connection, MAX_ATTACHMENT_RESPONSE_BYTES); + } catch (IOException failure) { + connection.disconnect(); + throw failure; + } } - // Streamed, not buffered: reading a 100 MB attachment into a byte[] (which the growing - // ByteArrayOutputStream first doubled, then copied) was the largest single allocation in - // the sync and an OutOfMemoryError on an ordinary phone. - HttpURLConnection connection = - open("GET", apiBase + "/files/" + attachmentId + "?alt=media"); - ensureSuccess(connection); - return new ConnectionInputStream(connection, MAX_ATTACHMENT_RESPONSE_BYTES); + return null; } @Override public synchronized void writeAttachment( @NonNull String sha256, long sizeBytes, @NonNull InputStream content) throws IOException { - String folderId = ensureFolderId(); + String folderId = ensureCanonicalFolderId(); if (findAttachment(folderId, sha256) != null) { return; } if (sizeBytes >= 0L) { - uploadStream(folderId, sha256, MIME_BINARY, content, sizeBytes); + if (sizeBytes > MAX_ATTACHMENT_RESPONSE_BYTES) { + throw new IOException("Attachment exceeds the 100 MiB sync upload limit"); + } + uploadAttachmentOrConfirm(folderId, sha256, content, sizeBytes); return; } // No declared size, so the multipart content length cannot be computed up front. Rare: // sizes come from the bundle manifest, which also supplies the hashes being uploaded. - uploadFile(folderId, sha256, MIME_BINARY, readFully(content), false); + uploadAttachmentOrConfirm( + folderId, sha256, readFullyLimited(content, MAX_ATTACHMENT_RESPONSE_BYTES)); } - @Nullable - private String findFolderId() throws IOException { + private List findFolderIds() throws IOException { JsonArray folders = listFiles( "mimeType = '" @@ -195,14 +195,15 @@ private String findFolderId() throws IOException { + "' and trashed = false and " + appPropertyClause("mynotesOwner", "1"), "files(id,name)"); - // Two devices whose first sync overlaps both run ensureFolderId and both create a folder. - // Throwing here made that permanent: every later sync on every device failed before it - // could do any work, and only manual cleanup in Drive recovered it. Converging on the - // lexicographically smallest ID instead makes all devices agree without coordination. - return smallestId(folders); + List result = new ArrayList<>(folders.size()); + for (int index = 0; index < folders.size(); index++) { + result.add(folders.get(index).getAsJsonObject().get("id").getAsString()); + } + result.sort(Comparator.naturalOrder()); + return result; } - /** Deterministic, coordination-free choice so every device selects the same file. */ + /** Deterministically selects one byte-identical content-addressed attachment duplicate. */ @Nullable private static String smallestId(@NonNull JsonArray files) { String selected = null; @@ -216,17 +217,78 @@ private static String smallestId(@NonNull JsonArray files) { } @NonNull - private String ensureFolderId() throws IOException { - String folderId = findFolderId(); - if (folderId != null) { - return folderId; + private String ensureCanonicalFolderId() throws IOException { + List folderIds = findFolderIds(); + if (!folderIds.isEmpty()) { + return folderIds.get(0); } JsonObject metadata = new JsonObject(); metadata.addProperty("name", FOLDER_NAME); metadata.addProperty("mimeType", MIME_FOLDER); metadata.add("appProperties", appProperties("mynotesOwner", "1")); - return uploadMetadata(metadata); + try { + return uploadMetadata(metadata); + } catch (IOException createFailure) { + // Folder POST can have committed before a lost response. Duplicate roots are a + // supported read state; rediscovery avoids a blind retry creating another one. + folderIds = findFolderIds(); + if (!folderIds.isEmpty()) { + return folderIds.get(0); + } + throw createFailure; + } + } + + private void ensureCanonicalAttachments( + @NonNull String canonicalRootId, @NonNull SyncSnapshot snapshot) throws IOException { + Map sizes = attachmentSizes(snapshot); + if (sizes.isEmpty()) { + return; + } + for (Map.Entry attachment : sizes.entrySet()) { + String hash = attachment.getKey(); + if (findAttachment(canonicalRootId, hash) != null) { + continue; + } + InputStream source = readAttachment(hash); + if (source == null) { + throw new IOException("Required attachment is unavailable in any Drive root"); + } + try (VerifiedAttachmentInputStream input = + new VerifiedAttachmentInputStream(source, hash, attachment.getValue())) { + uploadAttachmentOrConfirm(canonicalRootId, hash, input, attachment.getValue()); + input.verifyEndOfStream(); + } + } + } + + @NonNull + private static Map attachmentSizes(@NonNull SyncSnapshot snapshot) + throws IOException { + Map sizes = new HashMap<>(); + for (SyncRecord record : snapshot.getLiveRecords(SyncRecord.Type.NOTE)) { + JsonArray manifest = record.getPayload().getAsJsonArray("attachmentsManifest"); + if (manifest == null) { + continue; + } + for (int index = 0; index < manifest.size(); index++) { + JsonObject entry = manifest.get(index).getAsJsonObject(); + if (!entry.has("sha256") || !entry.has("size")) { + throw new IOException("Attachment metadata is incomplete"); + } + String hash = entry.get("sha256").getAsString(); + long size = entry.get("size").getAsLong(); + if (size < 0L || size > MAX_ATTACHMENT_RESPONSE_BYTES) { + throw new IOException("Attachment size exceeds the sync limit"); + } + Long previous = sizes.putIfAbsent(hash, size); + if (previous != null && previous.longValue() != size) { + throw new IOException("Attachment metadata has conflicting sizes"); + } + } + } + return sizes; } @NonNull @@ -280,7 +342,7 @@ private JsonArray listFiles(@NonNull String query, @NonNull String fields) throw "&pageToken=" + URLEncoder.encode(nextPageToken, StandardCharsets.UTF_8.name()); } - JsonObject response = requestJson("GET", url, null, null); + JsonObject response = requestJsonIdempotent("GET", url, null, null); JsonArray files = response.getAsJsonArray("files"); if (files != null) { for (int index = 0; index < files.size(); index++) { @@ -303,6 +365,20 @@ private String uploadMetadata(@NonNull JsonObject metadata) throws IOException { return created.get("id").getAsString(); } + private boolean hasBundleNamed(@NonNull String folderId, @NonNull String name) + throws IOException { + JsonArray bundles = + listFiles( + "'" + + folderId + + "' in parents and trashed = false and name = '" + + escapeQuery(name) + + "' and " + + appPropertyClause("mynotesBundle", "1"), + "files(id)"); + return bundles.size() > 0; + } + private void uploadFile( @NonNull String folderId, @NonNull String name, @@ -322,7 +398,209 @@ private void uploadStream( @NonNull InputStream content, long sizeBytes) throws IOException { - uploadMultipart(folderId, name, mimeType, content, sizeBytes, false); + uploadResumableAttachment(folderId, name, mimeType, content, sizeBytes); + } + + /** + * Uploads a bounded attachment in resumable chunks. Only one chunk is retained in heap, so a + * dropped connection can be probed and the unacknowledged chunk replayed without re-reading the + * source stream. + */ + private void uploadResumableAttachment( + @NonNull String folderId, + @NonNull String sha256, + @NonNull String mimeType, + @NonNull InputStream content, + long sizeBytes) + throws IOException { + String sessionUrl = + initiateResumableAttachmentUpload(folderId, sha256, mimeType, sizeBytes); + byte[] chunk = new byte[RESUMABLE_CHUNK_BYTES]; + long offset = 0L; + while (offset < sizeBytes) { + throwIfInterrupted(); + int chunkSize = + readChunk(content, chunk, (int) Math.min(chunk.length, sizeBytes - offset)); + if (chunkSize <= 0) { + throw new IOException("Attachment ended before its declared size"); + } + long acknowledged = + uploadChunk(sessionUrl, mimeType, chunk, chunkSize, offset, sizeBytes); + if (acknowledged < offset - 1L || acknowledged >= offset + chunkSize) { + throw new IOException( + "Drive resumable upload returned an invalid acknowledged range"); + } + if (acknowledged < offset + chunkSize - 1L) { + // The server received only a prefix. The unread suffix remains in this one chunk; + // replay it rather than advancing the source stream. + int consumed = (int) (acknowledged - offset + 1L); + System.arraycopy(chunk, consumed, chunk, 0, chunkSize - consumed); + int remaining = chunkSize - consumed; + while (remaining > 0) { + acknowledged = + uploadChunk( + sessionUrl, + mimeType, + chunk, + remaining, + acknowledged + 1L, + sizeBytes); + if (acknowledged < offset + chunkSize - 1L) { + int newlyConsumed = (int) (acknowledged - offset - consumed + 1L); + System.arraycopy(chunk, newlyConsumed, chunk, 0, remaining - newlyConsumed); + remaining -= newlyConsumed; + consumed += newlyConsumed; + } + } + } + offset += chunkSize; + } + if (content.read() != -1) { + throw new IOException("Attachment exceeds its declared size"); + } + } + + @NonNull + private String initiateResumableAttachmentUpload( + @NonNull String folderId, + @NonNull String sha256, + @NonNull String mimeType, + long sizeBytes) + throws IOException { + JsonObject metadata = attachmentMetadata(folderId, sha256); + HttpURLConnection connection = + open("POST", uploadBase + "?uploadType=resumable&fields=id,name"); + connection.setRequestProperty("Content-Type", MIME_JSON); + connection.setRequestProperty("X-Upload-Content-Type", mimeType); + connection.setRequestProperty("X-Upload-Content-Length", Long.toString(sizeBytes)); + connection.setDoOutput(true); + byte[] body = jsonBytes(metadata); + connection.setFixedLengthStreamingMode(body.length); + try { + try (OutputStream output = connection.getOutputStream()) { + output.write(body); + } + ensureSuccess(connection); + String location = connection.getHeaderField("Location"); + if (location == null || location.trim().isEmpty()) { + throw new IOException("Drive did not return a resumable upload session"); + } + return location; + } finally { + connection.disconnect(); + } + } + + private long uploadChunk( + @NonNull String sessionUrl, + @NonNull String mimeType, + @NonNull byte[] chunk, + int chunkSize, + long start, + long total) + throws IOException { + HttpURLConnection connection = open("PUT", sessionUrl); + connection.setRequestProperty("Content-Type", mimeType); + connection.setRequestProperty( + "Content-Range", "bytes " + start + "-" + (start + chunkSize - 1L) + "/" + total); + connection.setDoOutput(true); + connection.setFixedLengthStreamingMode(chunkSize); + try { + try (OutputStream output = connection.getOutputStream()) { + output.write(chunk, 0, chunkSize); + } + int status = connection.getResponseCode(); + if (status >= 200 && status < 300) { + return total - 1L; + } + if (status == HTTP_RESUME_INCOMPLETE) { + String range = connection.getHeaderField("Range"); + return resumableRangeEnd(range); + } + String detail = readErrorDetail(connection.getErrorStream()); + throw new DriveRequestExecutor.DriveHttpException( + status, connection.getHeaderField("Retry-After"), detail); + } finally { + connection.disconnect(); + } + } + + private static long resumableRangeEnd(@Nullable String range) throws IOException { + if (range == null || !range.startsWith("bytes=0-")) { + return -1L; + } + try { + return Long.parseLong(range.substring("bytes=0-".length())); + } catch (NumberFormatException error) { + throw new IOException("Drive returned an invalid resumable upload range", error); + } + } + + private static int readChunk(@NonNull InputStream input, @NonNull byte[] buffer, int maximum) + throws IOException { + int offset = 0; + while (offset < maximum) { + int read = input.read(buffer, offset, maximum - offset); + if (read == -1) { + break; + } + offset += read; + } + return offset; + } + + private static void throwIfInterrupted() throws IOException { + if (Thread.currentThread().isInterrupted()) { + java.io.InterruptedIOException interrupted = + new java.io.InterruptedIOException("Drive resumable upload interrupted"); + throw interrupted; + } + } + + @NonNull + private static JsonObject attachmentMetadata(@NonNull String folderId, @NonNull String sha256) { + JsonObject metadata = new JsonObject(); + metadata.addProperty("name", sha256); + JsonArray parents = new JsonArray(); + parents.add(folderId); + metadata.add("parents", parents); + JsonObject properties = new JsonObject(); + properties.addProperty("mynotesAttachmentSha256", sha256); + metadata.add("appProperties", properties); + return metadata; + } + + private void uploadAttachmentOrConfirm( + @NonNull String folderId, + @NonNull String sha256, + @NonNull InputStream content, + long sizeBytes) + throws IOException { + try { + if (sizeBytes >= 0L) { + uploadStream(folderId, sha256, MIME_BINARY, content, sizeBytes); + } else { + uploadFile(folderId, sha256, MIME_BINARY, readFully(content), false); + } + } catch (IOException uploadFailure) { + // Attachment identity is its SHA-256. A successful request whose response was lost is + // confirmed by discovery, not repeated with an already-consumed stream. + if (findAttachment(folderId, sha256) == null) { + throw uploadFailure; + } + } + } + + private void uploadAttachmentOrConfirm( + @NonNull String folderId, @NonNull String sha256, @NonNull byte[] content) + throws IOException { + try { + uploadFile(folderId, sha256, MIME_BINARY, content, false); + } catch (IOException uploadFailure) { + if (findAttachment(folderId, sha256) == null) { + throw uploadFailure; + } + } } /** @@ -376,16 +654,20 @@ private void uploadMultipart( + separator.length + closing.length); - try (OutputStream out = connection.getOutputStream()) { - out.write(head); - out.write(metadataBytes); - out.write(separator); - out.write(contentHeader); - copy(content, out); - out.write(separator); - out.write(closing); + try { + try (OutputStream out = connection.getOutputStream()) { + out.write(head); + out.write(metadataBytes); + out.write(separator); + out.write(contentHeader); + copy(content, out); + out.write(separator); + out.write(closing); + } + readJsonResponse(connection); + } finally { + connection.disconnect(); } - readJsonResponse(connection); } @NonNull @@ -419,16 +701,30 @@ private JsonObject requestJson( @Nullable byte[] body) throws IOException { HttpURLConnection connection = open(method, url); - if (contentType != null) { - connection.setRequestProperty("Content-Type", contentType); - } - if (body != null) { - connection.setDoOutput(true); - try (OutputStream output = connection.getOutputStream()) { - output.write(body); + try { + if (contentType != null) { + connection.setRequestProperty("Content-Type", contentType); } + if (body != null) { + connection.setDoOutput(true); + try (OutputStream output = connection.getOutputStream()) { + output.write(body); + } + } + return readJsonResponse(connection); + } finally { + connection.disconnect(); } - return readJsonResponse(connection); + } + + @NonNull + private JsonObject requestJsonIdempotent( + @NonNull String method, + @NonNull String url, + @Nullable String contentType, + @Nullable byte[] body) + throws IOException { + return requestExecutor.executeIdempotent(() -> requestJson(method, url, contentType, body)); } @NonNull @@ -443,19 +739,29 @@ private JsonObject readJsonResponse(@NonNull HttpURLConnection connection) throw @NonNull private byte[] requestBytes(@NonNull String method, @NonNull String url, int maxBytes) throws IOException { + return requestExecutor.executeIdempotent(() -> requestBytesOnce(method, url, maxBytes)); + } + + @NonNull + private byte[] requestBytesOnce(@NonNull String method, @NonNull String url, int maxBytes) + throws IOException { HttpURLConnection connection = open(method, url); - ensureSuccess(connection); - try (InputStream input = connection.getInputStream()) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - byte[] buffer = new byte[8192]; - int read; - while ((read = input.read(buffer)) != -1) { - if (output.size() > maxBytes - read) { - throw new IOException("Drive response exceeds the sync size limit"); + try { + ensureSuccess(connection); + try (InputStream input = connection.getInputStream()) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + if (output.size() > maxBytes - read) { + throw new IOException("Drive response exceeds the sync size limit"); + } + output.write(buffer, 0, read); } - output.write(buffer, 0, read); + return output.toByteArray(); } - return output.toByteArray(); + } finally { + connection.disconnect(); } } @@ -468,6 +774,18 @@ private HttpURLConnection open(@NonNull String method, @NonNull String url) thro return connection; } + private HttpURLConnection openSuccessful(@NonNull String method, @NonNull String url) + throws IOException { + HttpURLConnection connection = open(method, url); + try { + ensureSuccess(connection); + return connection; + } catch (IOException failure) { + connection.disconnect(); + throw failure; + } + } + private static void ensureSuccess(@NonNull HttpURLConnection connection) throws IOException { int code = connection.getResponseCode(); if (code >= 200 && code < 300) { @@ -475,10 +793,8 @@ private static void ensureSuccess(@NonNull HttpURLConnection connection) throws } String detail = readErrorDetail(connection.getErrorStream()); - if (code == HttpURLConnection.HTTP_PRECON_FAILED) { - throw new IOException("Drive snapshot changed since it was read"); - } - throw new IOException("Drive API HTTP " + code + (detail.isEmpty() ? "" : ": " + detail)); + throw new DriveRequestExecutor.DriveHttpException( + code, connection.getHeaderField("Retry-After"), detail); } /** @@ -529,6 +845,23 @@ private static byte[] readFully(@NonNull InputStream input) throws IOException { } } + @NonNull + private static byte[] readFullyLimited(@NonNull InputStream input, int maxBytes) + throws IOException { + try (InputStream stream = input; + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + byte[] buffer = new byte[8192]; + int read; + while ((read = stream.read(buffer)) != -1) { + if (output.size() > maxBytes - read) { + throw new IOException("Attachment exceeds the 100 MiB sync upload limit"); + } + output.write(buffer, 0, read); + } + return output.toByteArray(); + } + } + @NonNull private static byte[] jsonBytes(@NonNull JsonObject object) { return GSON.toJson(object).getBytes(StandardCharsets.UTF_8); @@ -607,4 +940,66 @@ public void close() throws IOException { } } } + + /** Verifies an untrusted remote blob before it may support canonical bundle publication. */ + private static final class VerifiedAttachmentInputStream extends FilterInputStream { + private final String expectedHash; + private final long expectedSize; + private final MessageDigest digest; + private long size; + private boolean reachedEnd; + + private VerifiedAttachmentInputStream( + @NonNull InputStream source, @NonNull String expectedHash, long expectedSize) + throws IOException { + super(source); + this.expectedHash = expectedHash; + this.expectedSize = expectedSize; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException error) { + throw new IOException("SHA-256 is unavailable", error); + } + } + + @Override + public int read() throws IOException { + int value = super.read(); + if (value == -1) { + reachedEnd = true; + } else { + digest.update((byte) value); + size++; + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int read = super.read(buffer, offset, length); + if (read == -1) { + reachedEnd = true; + } else if (read > 0) { + digest.update(buffer, offset, read); + size += read; + } + return read; + } + + private void verifyEndOfStream() throws IOException { + if (!reachedEnd) { + throw new IOException("Attachment upload ended before the source was verified"); + } + if (size != expectedSize) { + throw new IOException("Attachment size does not match sync metadata"); + } + StringBuilder actualHash = new StringBuilder(64); + for (byte value : digest.digest()) { + actualHash.append(String.format(java.util.Locale.US, "%02x", value & 0xff)); + } + if (!expectedHash.equals(actualHash.toString())) { + throw new IOException("Attachment checksum does not match sync metadata"); + } + } + } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java index 2e80d6bb..13c2f812 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorker.java @@ -85,16 +85,12 @@ private static boolean isRetryable(String message) { } /** - * The rollout gate applies here too. - * - *

It used to live only on the manual "Sync now" path, so lowering the percentage to pull a - * bad release back would have left this six-hourly job running for every user who had already - * turned sync on — the very population a rollback needs to stop. + * Backup is available to every user who explicitly enables it; no remote rollout gate is + * consulted here. */ static boolean isBackgroundSyncAllowed(PreferenceHelper preferences) { return preferences.isSyncEnabled() && preferences.isBackgroundSyncEnabled() - && preferences.isFirstSyncConfirmed() - && SyncRollout.isWithinRollout(preferences); + && preferences.isFirstSyncConfirmed(); } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index 375f945c..52c47287 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -53,6 +53,9 @@ public final class RoomSyncStore implements SyncStore { private final PreferenceHelper preferenceHelper; private final Context context; private final Gson gson = new Gson(); + private final AttachmentResolver attachmentResolver; + private final AttachmentHasher attachmentHasher; + private final TransactionFailureInjector transactionFailureInjector; /** * Content hash to the note-folder file holding it, indexed while the snapshot is built so the @@ -64,9 +67,47 @@ public RoomSyncStore( @NonNull Context context, @NonNull AppDatabase database, @NonNull PreferenceHelper preferenceHelper) { + this( + context, + database, + preferenceHelper, + AttachmentStorage::resolve, + RoomSyncStore::sha256, + record -> {}); + } + + /** + * Test seam for storage failures that must prevent a publish rather than drop an attachment. + */ + public RoomSyncStore( + @NonNull Context context, + @NonNull AppDatabase database, + @NonNull PreferenceHelper preferenceHelper, + @NonNull AttachmentResolver attachmentResolver, + @NonNull AttachmentHasher attachmentHasher) { + this( + context, + database, + preferenceHelper, + attachmentResolver, + attachmentHasher, + record -> {}); + } + + /** Test seam used to prove that Room rolls back a partially applied remote snapshot. */ + public RoomSyncStore( + @NonNull Context context, + @NonNull AppDatabase database, + @NonNull PreferenceHelper preferenceHelper, + @NonNull AttachmentResolver attachmentResolver, + @NonNull AttachmentHasher attachmentHasher, + @NonNull TransactionFailureInjector transactionFailureInjector) { this.database = database; this.preferenceHelper = preferenceHelper; this.context = context.getApplicationContext(); + this.attachmentResolver = attachmentResolver; + this.attachmentHasher = attachmentHasher; + this.transactionFailureInjector = transactionFailureInjector; this.preferences = context.getApplicationContext().getSharedPreferences(PREFS, Context.MODE_PRIVATE); } @@ -96,8 +137,21 @@ private void ensureSeeded() { @NonNull @Override public SyncSnapshot readSnapshot() throws IOException { + return buildSnapshot().requireSnapshot(); + } + + /** + * Builds a local snapshot without ever treating an unresolved attachment as absent. + * + *

Returning an incomplete result leaves the database and the note attachment JSON exactly as + * they were. {@link SyncService} refuses to publish such a result before it talks to Drive. + */ + @NonNull + @Override + public SnapshotBuildResult buildSnapshot() throws IOException { ensureSeeded(); List records = new ArrayList<>(); + List problems = new ArrayList<>(); for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { if (metadata.deletedAt != null) { records.add( @@ -108,7 +162,7 @@ public SyncSnapshot readSnapshot() throws IOException { Instant.ofEpochMilli(metadata.deletedAt))); continue; } - JsonObject payload = payload(metadata); + JsonObject payload = payload(metadata, problems); if (payload != null) { SyncMetadataEntity current = database.syncMetadataDao().get(metadata.recordType, metadata.localId); @@ -121,7 +175,10 @@ public SyncSnapshot readSnapshot() throws IOException { payload)); } } - return new SyncSnapshot(records); + SyncSnapshot snapshot = new SyncSnapshot(records); + return problems.isEmpty() + ? SnapshotBuildResult.publishable(snapshot) + : SnapshotBuildResult.incomplete(snapshot, problems); } @Override @@ -167,6 +224,7 @@ private void applySnapshotInternal( record.getUpdatedAt().toEpochMilli(), null)); } + transactionFailureInjector.afterRecordApplied(record); continue; } if (metadata == null) continue; @@ -178,6 +236,7 @@ private void applySnapshotInternal( metadata.localId, record.getUpdatedAt().toEpochMilli(), record.getDeletedAt().toEpochMilli()); + transactionFailureInjector.afterRecordApplied(record); continue; } applyPayload(metadata, record.getPayload()); @@ -187,6 +246,7 @@ private void applySnapshotInternal( metadata.localId, record.getUpdatedAt().toEpochMilli(), null); + transactionFailureInjector.afterRecordApplied(record); } persistConflicts(conflicts); if (finalState != null) { @@ -196,7 +256,8 @@ private void applySnapshotInternal( } @Nullable - private JsonObject payload(SyncMetadataEntity metadata) { + private JsonObject payload( + SyncMetadataEntity metadata, @NonNull List snapshotProblems) { Object value = null; if ("note".equals(metadata.recordType)) value = database.noteDao().getNoteSync((int) metadata.localId); @@ -229,7 +290,10 @@ else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { result.addProperty("categoryStableId", categoryMetadata.stableId); } } - if ("note".equals(metadata.recordType)) addAttachmentMetadata(result); + if ("note".equals(metadata.recordType) + && !addAttachmentMetadata(result, metadata, snapshotProblems)) { + return null; + } // Runs last: the blocks above still need the local categoryId and attachment paths. SyncMetadata.stripDeviceLocalFields(metadata.recordType, result); return result; @@ -597,50 +661,117 @@ private File attachmentFile(String sha256) { return new File(dir, sha256); } - private void addAttachmentMetadata(JsonObject payload) { + private boolean addAttachmentMetadata( + JsonObject payload, + SyncMetadataEntity metadata, + @NonNull List snapshotProblems) { String json = payload.has("h") && !payload.get("h").isJsonNull() ? payload.get("h").getAsString() : null; - if (json == null || json.trim().isEmpty()) return; + if (json == null || json.trim().isEmpty()) return true; + JsonArray attachments; try { - JsonArray attachments = JsonParser.parseString(json).getAsJsonArray(); - JsonArray manifest = new JsonArray(); - JsonArray hashes = new JsonArray(); - JsonObject names = new JsonObject(); - for (JsonElement element : attachments) { - EditorAttachment attachment = gson.fromJson(element, EditorAttachment.class); - File file = AttachmentStorage.resolve(context, attachment.url); - if (file == null || !file.isFile()) continue; - String hash = sha256(file); - localAttachments.put(hash, file); - String displayName = - attachment.name == null || attachment.name.trim().isEmpty() - ? file.getName() - : attachment.name.trim(); - hashes.add(hash); - names.addProperty(hash, displayName); - - JsonObject manifestEntry = new JsonObject(); - manifestEntry.addProperty("id", stableAttachmentId(hash)); - manifestEntry.addProperty("sha256", hash); - manifestEntry.addProperty( - "mimeType", detectMimeType(file, attachment, displayName)); - manifestEntry.addProperty("size", file.length()); - manifestEntry.addProperty("path", "attachments/" + hash); - manifestEntry.addProperty("displayName", displayName); - manifest.add(manifestEntry); + attachments = JsonParser.parseString(json).getAsJsonArray(); + } catch (RuntimeException error) { + addSnapshotProblem( + snapshotProblems, SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, metadata); + return false; + } + + JsonArray manifest = new JsonArray(); + JsonArray hashes = new JsonArray(); + JsonObject names = new JsonObject(); + boolean complete = true; + for (JsonElement element : attachments) { + if (!element.isJsonObject()) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, + metadata); + complete = false; + continue; } - if (!hashes.isEmpty()) { - payload.add("attachmentsManifest", manifest); - payload.add("attachmentHashes", hashes); - payload.add("attachmentNames", names); + EditorAttachment attachment; + try { + attachment = gson.fromJson(element, EditorAttachment.class); + } catch (RuntimeException error) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, + metadata); + complete = false; + continue; + } + if (attachment == null || attachment.url == null || attachment.url.trim().isEmpty()) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, + metadata); + complete = false; + continue; + } + File file; + try { + file = attachmentResolver.resolve(context, attachment); + } catch (RuntimeException error) { + addSnapshotProblem( + snapshotProblems, + SnapshotProblem.Kind.INVALID_ATTACHMENT_METADATA, + metadata); + complete = false; + continue; + } + if (file == null || !file.isFile()) { + addSnapshotProblem( + snapshotProblems, SnapshotProblem.Kind.MISSING_ATTACHMENT, metadata); + complete = false; + continue; + } + if (!file.canRead()) { + addSnapshotProblem( + snapshotProblems, SnapshotProblem.Kind.UNREADABLE_ATTACHMENT, metadata); + complete = false; + continue; + } + String hash; + try { + hash = attachmentHasher.sha256(file); + } catch (IOException error) { + addSnapshotProblem( + snapshotProblems, SnapshotProblem.Kind.ATTACHMENT_HASH_FAILED, metadata); + complete = false; + continue; } - } catch (Exception error) { - // Swallowing this silently used to drop a note's attachments from the bundle with no - // trace; the sync itself still succeeds without them. - Log.w(TAG, "Could not collect attachment metadata", error); + localAttachments.put(hash, file); + String displayName = + attachment.name == null || attachment.name.trim().isEmpty() + ? file.getName() + : attachment.name.trim(); + hashes.add(hash); + names.addProperty(hash, displayName); + + JsonObject manifestEntry = new JsonObject(); + manifestEntry.addProperty("id", stableAttachmentId(hash)); + manifestEntry.addProperty("sha256", hash); + manifestEntry.addProperty("mimeType", detectMimeType(file, attachment, displayName)); + manifestEntry.addProperty("size", file.length()); + manifestEntry.addProperty("path", "attachments/" + hash); + manifestEntry.addProperty("displayName", displayName); + manifest.add(manifestEntry); } + if (!complete) return false; + payload.add("attachmentsManifest", manifest); + payload.add("attachmentHashes", hashes); + payload.add("attachmentNames", names); + return true; + } + + private static void addSnapshotProblem( + @NonNull List problems, + @NonNull SnapshotProblem.Kind kind, + @NonNull SyncMetadataEntity metadata) { + problems.add(new SnapshotProblem(kind, metadata.recordType, metadata.stableId)); } private void restoreAttachments(Note note, JsonObject payload) { @@ -701,8 +832,13 @@ private static boolean isSafeAttachmentName(@NonNull String name) { return new File(value).getName().equals(value); } - private static String sha256(File file) throws Exception { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); + private static String sha256(File file) throws IOException { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (java.security.NoSuchAlgorithmException error) { + throw new IOException("SHA-256 is unavailable", error); + } try (InputStream in = new FileInputStream(file)) { byte[] buffer = new byte[8192]; int read; @@ -713,6 +849,23 @@ private static String sha256(File file) throws Exception { return hex.toString(); } + /** Resolves a serialized note attachment to its app-private file. */ + public interface AttachmentResolver { + @Nullable + File resolve(@NonNull Context context, @NonNull EditorAttachment attachment); + } + + /** Hashes an attachment after it has passed basic filesystem checks. */ + public interface AttachmentHasher { + @NonNull + String sha256(@NonNull File file) throws IOException; + } + + /** Throws from tests after a Room mutation but before the enclosing transaction commits. */ + public interface TransactionFailureInjector { + void afterRecordApplied(@NonNull SyncRecord record); + } + @NonNull private static String stableAttachmentId(@NonNull String hash) { return UUID.nameUUIDFromBytes(hash.getBytes(StandardCharsets.UTF_8)).toString(); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java new file mode 100644 index 00000000..566bb9d1 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotBuildResult.java @@ -0,0 +1,73 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Result of building a local sync snapshot. + * + *

An unpublishable result is deliberately not convertible into a {@link SyncSnapshot}. This + * prevents a caller from accidentally publishing a note after attachment collection failed. + */ +public final class SnapshotBuildResult { + + @NonNull private final SyncSnapshot snapshot; + @NonNull private final List problems; + + private SnapshotBuildResult( + @NonNull SyncSnapshot snapshot, @NonNull List problems) { + this.snapshot = Objects.requireNonNull(snapshot, "snapshot"); + this.problems = Collections.unmodifiableList(new ArrayList<>(problems)); + } + + @NonNull + public static SnapshotBuildResult publishable(@NonNull SyncSnapshot snapshot) { + return new SnapshotBuildResult(snapshot, Collections.emptyList()); + } + + @NonNull + public static SnapshotBuildResult incomplete( + @NonNull SyncSnapshot snapshot, @NonNull List problems) { + if (problems.isEmpty()) { + throw new IllegalArgumentException("An incomplete snapshot requires a problem"); + } + return new SnapshotBuildResult(snapshot, problems); + } + + public boolean isPublishable() { + return problems.isEmpty(); + } + + @NonNull + public List getProblems() { + return problems; + } + + /** Returns the snapshot only when all local attachment references were verified. */ + @NonNull + public SyncSnapshot requireSnapshot() throws IOException { + if (!isPublishable()) { + throw new SnapshotBuildException(problems); + } + return snapshot; + } + + /** Typed, coarse error suitable for persisted sync state and telemetry. */ + public static final class SnapshotBuildException extends IOException { + @NonNull private final List problems; + + private SnapshotBuildException(@NonNull List problems) { + super("Local snapshot is incomplete: " + problems.get(0).getKind().name()); + this.problems = Collections.unmodifiableList(new ArrayList<>(problems)); + } + + @NonNull + public List getProblems() { + return problems; + } + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java new file mode 100644 index 00000000..6d8aeb9d --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SnapshotProblem.java @@ -0,0 +1,41 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import java.util.Objects; + +/** A privacy-safe reason why a local snapshot cannot safely be published. */ +public final class SnapshotProblem { + + public enum Kind { + MISSING_ATTACHMENT, + UNREADABLE_ATTACHMENT, + ATTACHMENT_HASH_FAILED, + INVALID_ATTACHMENT_METADATA + } + + @NonNull private final Kind kind; + @NonNull private final String recordType; + @NonNull private final String stableId; + + public SnapshotProblem( + @NonNull Kind kind, @NonNull String recordType, @NonNull String stableId) { + this.kind = Objects.requireNonNull(kind, "kind"); + this.recordType = Objects.requireNonNull(recordType, "recordType"); + this.stableId = Objects.requireNonNull(stableId, "stableId"); + } + + @NonNull + public Kind getKind() { + return kind; + } + + @NonNull + public String getRecordType() { + return recordType; + } + + @NonNull + public String getStableId() { + return stableId; + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java index 53cf5451..6b554176 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java @@ -6,6 +6,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -25,14 +26,19 @@ /** Validates schema-1 sync bundles before any remote data is exposed to the app. */ public final class SyncBundleValidator { + static final long MAX_COMPRESSED_BUNDLE_BYTES = 32L * 1024L * 1024L; static final long MAX_RECORD_BYTES = 25L * 1024L * 1024L; + static final long MAX_RECORD_PAYLOAD_BYTES = 4L * 1024L * 1024L; static final long MAX_RECORD_COUNT = 10_000L; static final long MAX_ATTACHMENT_COUNT = 10_000L; + static final long MAX_ATTACHMENTS_PER_NOTE = 1_000L; static final long MAX_ATTACHMENT_BYTES = 100L * 1024L * 1024L; static final long MAX_TOTAL_ATTACHMENT_BYTES = 500L * 1024L * 1024L; static final long MAX_TOTAL_UNCOMPRESSED_BYTES = 1024L * 1024L * 1024L; static final long MAX_COMPRESSION_RATIO = 100L; private static final long MAX_MANIFEST_BYTES = 2L * 1024L * 1024L; + private static final int MAX_METADATA_STRING_CHARS = 1_048_576; + private static final int MAX_JSON_DEPTH = 64; private static final Pattern SHA_256 = Pattern.compile("[0-9a-f]{64}"); @NonNull @@ -147,6 +153,7 @@ private static long validateLiveRecords( String id = requireString(item, "id"); validateUuid(id); parseInstant(requireString(item, "updatedAt"), "updatedAt"); + validatePayloadLimits(item); if (type == SyncRecord.Type.NOTE) { validateAttachmentReferences(item, attachmentsById, referencedAttachmentIds); } @@ -166,6 +173,9 @@ private static void validateAttachmentReferences( if (attachmentIds == null) { return; } + if (attachmentIds.size() > MAX_ATTACHMENTS_PER_NOTE) { + throw new IOException("Sync note exceeds the attachment limit"); + } JsonObject attachmentNames = note.getAsJsonObject("attachmentNames"); for (JsonElement element : attachmentIds) { if (element == null || !element.isJsonPrimitive()) { @@ -252,7 +262,10 @@ private static BundleEntries readEntries(@NonNull InputStream input) throws IOEx byte[] records = null; long totalUncompressedBytes = 0L; Set names = new LinkedHashSet<>(); - try (ZipInputStream zip = new ZipInputStream(input, StandardCharsets.UTF_8)) { + try (ZipInputStream zip = + new ZipInputStream( + new BoundedInputStream(input, MAX_COMPRESSED_BUNDLE_BYTES), + StandardCharsets.UTF_8)) { ZipEntry entry; while ((entry = zip.getNextEntry()) != null) { String name = entry.getName(); @@ -341,10 +354,17 @@ private static JsonArray requireArray(@NonNull JsonObject object, @NonNull Strin static String requireString(@NonNull JsonObject object, @NonNull String field) throws IOException { JsonElement value = object.get(field); - if (value == null || value.isJsonNull() || !value.isJsonPrimitive()) { + if (value == null + || value.isJsonNull() + || !value.isJsonPrimitive() + || !value.getAsJsonPrimitive().isString()) { throw new IOException("Sync JSON field " + field + " is missing or invalid"); } - return value.getAsString(); + String result = value.getAsString(); + if (result.length() > MAX_METADATA_STRING_CHARS) { + throw new IOException("Sync JSON field " + field + " exceeds the string limit"); + } + return result; } private static void requireString( @@ -367,6 +387,42 @@ static long requireLong(@NonNull JsonObject object, @NonNull String field) throw } } + private static void validatePayloadLimits(@NonNull JsonObject record) throws IOException { + long serializedBytes = record.toString().getBytes(StandardCharsets.UTF_8).length; + if (serializedBytes > MAX_RECORD_PAYLOAD_BYTES) { + throw new IOException("Sync record exceeds the payload size limit"); + } + validateJsonValue(record, 0); + } + + private static void validateJsonValue(@NonNull JsonElement value, int depth) + throws IOException { + if (depth > MAX_JSON_DEPTH) { + throw new IOException("Sync JSON exceeds the nesting limit"); + } + if (value.isJsonPrimitive()) { + if (value.getAsJsonPrimitive().isString() + && value.getAsString().length() > MAX_METADATA_STRING_CHARS) { + throw new IOException("Sync JSON string exceeds the size limit"); + } + return; + } + if (value.isJsonArray()) { + for (JsonElement element : value.getAsJsonArray()) { + validateJsonValue(element, depth + 1); + } + return; + } + if (value.isJsonObject()) { + for (Map.Entry entry : value.getAsJsonObject().entrySet()) { + if (entry.getKey().length() > MAX_METADATA_STRING_CHARS) { + throw new IOException("Sync JSON field name exceeds the size limit"); + } + validateJsonValue(entry.getValue(), depth + 1); + } + } + } + static void validateDisplayName(@NonNull String name) throws IOException { String value = name.trim(); if (value.isEmpty() @@ -410,6 +466,42 @@ static String sha256(@NonNull byte[] bytes) throws IOException { } } + /** Caps compressed input before ZIP parsing to make bundle-size limits independent of Drive. */ + private static final class BoundedInputStream extends FilterInputStream { + private final long maxBytes; + private long bytesRead; + + private BoundedInputStream(@NonNull InputStream input, long maxBytes) { + super(input); + this.maxBytes = maxBytes; + } + + @Override + public int read() throws IOException { + int value = super.read(); + if (value != -1) { + count(1); + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int read = super.read(buffer, offset, length); + if (read > 0) { + count(read); + } + return read; + } + + private void count(long read) throws IOException { + bytesRead += read; + if (bytesRead > maxBytes) { + throw new IOException("Sync bundle exceeds the compressed size limit"); + } + } + } + private static final class BundleEntries { private final byte[] manifestBytes; private final byte[] recordBytes; diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncRollout.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncRollout.java deleted file mode 100644 index a0b9080e..00000000 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncRollout.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.pasich.mynotes.data.sync; - -import androidx.annotation.NonNull; -import com.pasich.mynotes.data.preferences.PreferenceHelper; -import java.security.SecureRandom; - -/** - * Staged-rollout gate for Google Drive sync. - * - *

The percentage and the bucket check used to live in {@code SyncCoordinator}, which only the - * manual "Sync now" button goes through. {@code GoogleDriveSyncWorker} never consulted them, so - * lowering the percentage to stop a bad release would have left the six-hourly background sync - * running for every user who had already enabled it — exactly the population a rollback needs to - * stop. Both paths now share this class. - */ -public final class SyncRollout { - - /** - * The v2.6.48 sync safety release completed its staged rollout; sync is available to all - * cohorts. Lower this to pull sync back from part of the population. - */ - public static final int CURRENT_PERCENT = 100; - - private static final SecureRandom RANDOM = new SecureRandom(); - - private SyncRollout() { - // no instance - } - - /** - * Returns this device's stable 1..100 cohort, assigning one on first use. - * - *

The bucket is drawn once and kept, so a device never moves between cohorts as the - * percentage changes. - */ - public static int ensureBucket(@NonNull PreferenceHelper preferences) { - int bucket = preferences.getSyncRolloutBucket(); - if (bucket < 1 || bucket > 100) { - bucket = RANDOM.nextInt(100) + 1; - preferences.setSyncRolloutBucket(bucket); - } - return bucket; - } - - /** True when this device's cohort is inside the current rollout. */ - public static boolean isWithinRollout(@NonNull PreferenceHelper preferences) { - return ensureBucket(preferences) <= CURRENT_PERCENT; - } -} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java index a20e0da9..28787b1b 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java @@ -96,7 +96,12 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { persistState( SyncState.syncing( backendIdentifier, startedAt, previousState.getLastSuccessfulSyncAt())); - SyncSnapshot local = Objects.requireNonNull(store.readSnapshot(), "local snapshot"); + SnapshotBuildResult localBuild = + Objects.requireNonNull(store.buildSnapshot(), "local snapshot build"); + // Do this before reading Drive or transferring blobs. Publishing a snapshot that + // merely skipped an unresolved local attachment turns a local storage fault into + // permanent remote data loss on the next successful sync from another device. + SyncSnapshot local = localBuild.requireSnapshot(); SyncSnapshot remote = Objects.requireNonNull(backend.readSnapshot(), "remote snapshot"); warnAboutClockSkew(remote); SyncMergeResult mergeResult = merger.merge(local, remote); @@ -195,11 +200,9 @@ private void synchronizeAttachments( backend.readAttachment(hash), store::writeAttachment); } - // A remote re-verification used to run here on every sync, downloading each - // attachment in full (up to 100 MB) purely to re-check a hash. Remote blobs are - // content-addressed and immutable, so the check could never fail for a reason - // the download itself would not already surface, and on the six-hourly worker - // it re-transferred the user's entire attachment set four times a day. + // Drive is untrusted. A matching appProperty is only a claim, so verify the + // actual remote bytes before a bundle can make that blob durable state. + verifyAttachment(hash, expectedSizes.get(hash), backend.readAttachment(hash)); } else { copyVerified( hash, diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java index f38a6d10..a0951d69 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java @@ -20,6 +20,15 @@ public interface SyncStore { @NonNull SyncSnapshot readSnapshot() throws IOException; + /** + * Builds a local snapshot together with any integrity problems that make publication unsafe. + * Implementations that cannot identify such problems retain the legacy snapshot boundary. + */ + @NonNull + default SnapshotBuildResult buildSnapshot() throws IOException { + return SnapshotBuildResult.publishable(readSnapshot()); + } + /** * Applies the merged snapshot and conflict report atomically. * diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java index 30714259..2b701589 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java @@ -197,14 +197,24 @@ public static File resolve(Context ctx, EditorAttachment att) { public static File resolve(Context ctx, String url) { try { Uri uri = Uri.parse(url); + if (!"file".equals(uri.getScheme()) || !ATTACHMENTS_BASE_DIR.equals(uri.getAuthority())) { + return null; + } List seg = uri.getPathSegments(); - if (seg.size() < 2) return null; + if (seg.size() != 2 || !seg.get(0).matches("note_[1-9][0-9]*")) return null; String folder = seg.get(0); String name = seg.get(1); + if (name.isEmpty() || name.indexOf('/') >= 0 || name.indexOf('\\') >= 0) return null; + for (int index = 0; index < name.length(); index++) { + if (Character.isISOControl(name.charAt(index))) return null; + } - return new File(new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR), folder + "/" + name); + File root = new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR).getCanonicalFile(); + File resolved = new File(new File(root, folder), name).getCanonicalFile(); + String rootPath = root.getPath() + File.separator; + return resolved.getPath().startsWith(rootPath) ? resolved : null; } catch (Exception e) { return null; diff --git a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinator.java b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinator.java index 5cf529b7..f4dfc4c3 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinator.java +++ b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncCoordinator.java @@ -9,7 +9,6 @@ import com.pasich.mynotes.data.database.entities.SyncConflictEntity; import com.pasich.mynotes.data.preferences.PreferenceHelper; import com.pasich.mynotes.data.sync.SyncResolution; -import com.pasich.mynotes.data.sync.SyncRollout; import com.pasich.mynotes.data.sync.SyncState; import com.pasich.mynotes.utils.auth.FirebaseGoogleAuth; import com.pasich.mynotes.utils.auth.GoogleCredential; @@ -188,7 +187,6 @@ public void onSuccess(@NonNull GoogleCredential credential) { new FirebaseGoogleAuth.Callback() { @Override public void onSuccess(@NonNull FirebaseUser user) { - SyncRollout.ensureBucket(preferenceHelper); preferenceHelper.setSyncEnabled(true); if (preferenceHelper.isBackgroundSyncEnabled() && preferenceHelper.isFirstSyncConfirmed()) { @@ -257,11 +255,6 @@ public void syncNow(@NonNull Activity activity, @NonNull Callback cal new IllegalStateException("Confirm the first sync before continuing")); return; } - if (!SyncRollout.isWithinRollout(preferenceHelper)) { - deliverError( - callback, new IllegalStateException("Sync is not available in this rollout")); - return; - } googleDriveAuthorization.authorize( activity, new GoogleDriveAuthorization.Callback() { diff --git a/app/src/main/java/com/pasich/mynotes/utils/constants/settings/PreferencesConfig.java b/app/src/main/java/com/pasich/mynotes/utils/constants/settings/PreferencesConfig.java index 18360480..d1184d9d 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/constants/settings/PreferencesConfig.java +++ b/app/src/main/java/com/pasich/mynotes/utils/constants/settings/PreferencesConfig.java @@ -68,9 +68,7 @@ public static int normalizeNoteTextSize(int size) { public static final String ARGUMENT_PREFERENCE_SYNC_BACKGROUND_ENABLED = "sync_background_enabled"; public static final String ARGUMENT_PREFERENCE_SYNC_FIRST_CONFIRMED = "sync_first_confirmed"; - public static final String ARGUMENT_PREFERENCE_SYNC_ROLLOUT_BUCKET = "sync_rollout_bucket"; public static final boolean ARGUMENT_DEFAULT_SYNC_ENABLED = false; public static final boolean ARGUMENT_DEFAULT_SYNC_BACKGROUND_ENABLED = false; public static final boolean ARGUMENT_DEFAULT_SYNC_FIRST_CONFIRMED = false; - public static final int ARGUMENT_DEFAULT_SYNC_ROLLOUT_BUCKET = -1; } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/DriveRequestExecutorTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/DriveRequestExecutorTest.java new file mode 100644 index 00000000..94a3e5e9 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/DriveRequestExecutorTest.java @@ -0,0 +1,95 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Test; + +public class DriveRequestExecutorTest { + + @Test + public void retriesTransientHttpFailuresAndHonorsRetryAfter() throws Exception { + List delays = new ArrayList<>(); + DriveRequestExecutor executor = + executor(delays, Clock.fixed(Instant.ofEpochMilli(1_000L), ZoneOffset.UTC)); + AtomicInteger attempts = new AtomicInteger(); + + String result = + executor.executeIdempotent( + () -> { + if (attempts.getAndIncrement() == 0) { + throw new DriveRequestExecutor.DriveHttpException( + 429, "2", "rateLimitExceeded"); + } + return "ok"; + }); + + assertThat(result).isEqualTo("ok"); + assertThat(attempts.get()).isEqualTo(2); + assertThat(delays).containsExactly(2_000L); + } + + @Test + public void retriesConnectionFailuresButNotAuthenticationOrPermanentForbidden() { + assertThat(DriveRequestExecutor.isRetryable(new SocketTimeoutException())).isTrue(); + assertThat( + DriveRequestExecutor.isRetryable( + new DriveRequestExecutor.DriveHttpException(500, null, ""))) + .isTrue(); + assertThat( + DriveRequestExecutor.isRetryable( + new DriveRequestExecutor.DriveHttpException( + 403, null, "rateLimitExceeded"))) + .isTrue(); + assertThat( + DriveRequestExecutor.isRetryable( + new DriveRequestExecutor.DriveHttpException(401, null, ""))) + .isFalse(); + assertThat( + DriveRequestExecutor.isRetryable( + new DriveRequestExecutor.DriveHttpException( + 403, null, "forbidden"))) + .isFalse(); + } + + @Test + public void doesNotRetryPermanentFailure() { + DriveRequestExecutor executor = executor(new ArrayList<>(), Clock.systemUTC()); + AtomicInteger attempts = new AtomicInteger(); + + assertThrows( + IOException.class, + () -> + executor.executeIdempotent( + () -> { + attempts.incrementAndGet(); + throw new DriveRequestExecutor.DriveHttpException( + 401, null, "unauthorized"); + })); + + assertThat(attempts.get()).isEqualTo(1); + } + + @Test + public void interruptionIsPropagatedWithoutAnotherAttempt() { + DriveRequestExecutor executor = executor(new ArrayList<>(), Clock.systemUTC()); + Thread.currentThread().interrupt(); + try { + assertThrows(IOException.class, () -> executor.executeIdempotent(() -> "never")); + } finally { + Thread.interrupted(); + } + } + + private static DriveRequestExecutor executor(List delays, Clock clock) { + return new DriveRequestExecutor(clock, delays::add, upperExclusive -> 0L); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index 2351f597..fceb9586 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -26,6 +26,10 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.junit.After; @@ -83,6 +87,69 @@ public void writeSnapshot_createsOwnedFolderBundleAndAttachment() throws Excepti assertThat(remoteSnapshot.find(SyncRecord.Type.NOTE, NOTE_ID)).isNotNull(); } + @Test + public void writeAttachment_resumesAcrossMultipleDriveChunksWithoutBufferingTheFile() + throws Exception { + GoogleDriveSyncBackend backend = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + CLOCK, + new SyncBundleCodec()); + byte[] bytes = new byte[600 * 1024]; + for (int index = 0; index < bytes.length; index++) { + bytes[index] = (byte) (index % 251); + } + String hash = sha256(bytes); + + backend.writeAttachment(hash, bytes.length, new ByteArrayInputStream(bytes)); + + assertThat(server.ownedAttachmentCount(hash)).isEqualTo(1); + assertThat(server.readAttachment(hash)).isEqualTo(bytes); + } + + @Test + public void concurrentFirstSync_createsDuplicateRootsThenConvergesWithoutLosingEitherNote() + throws Exception { + server.pauseTheNextTwoEmptyRootListings(); + GoogleDriveSyncBackend first = backend(); + GoogleDriveSyncBackend second = backend(); + SyncSnapshot firstSnapshot = snapshot(NOTE_ID, null); + SyncSnapshot secondSnapshot = snapshot(SECOND_NOTE_ID, null); + Thread firstThread = new Thread(() -> writeUnchecked(first, firstSnapshot)); + Thread secondThread = new Thread(() -> writeUnchecked(second, secondSnapshot)); + + firstThread.start(); + secondThread.start(); + firstThread.join(5_000L); + secondThread.join(5_000L); + assertThat(firstThread.isAlive()).isFalse(); + assertThat(secondThread.isAlive()).isFalse(); + assertThat(server.ownedFolderCount()).isEqualTo(2); + + SyncSnapshot reconciled = backend().readSnapshot(); + + assertThat(reconciled.find(SyncRecord.Type.NOTE, NOTE_ID)).isNotNull(); + assertThat(reconciled.find(SyncRecord.Type.NOTE, SECOND_NOTE_ID)).isNotNull(); + first.writeSnapshot(reconciled); + assertThat(second.readSnapshot().find(SyncRecord.Type.NOTE, NOTE_ID)).isNotNull(); + assertThat(second.readSnapshot().find(SyncRecord.Type.NOTE, SECOND_NOTE_ID)).isNotNull(); + } + + private GoogleDriveSyncBackend backend() { + return new GoogleDriveSyncBackend( + "token", server.apiBase(), server.uploadBase(), CLOCK, new SyncBundleCodec()); + } + + private static void writeUnchecked(GoogleDriveSyncBackend backend, SyncSnapshot snapshot) { + try { + backend.writeSnapshot(snapshot); + } catch (IOException error) { + throw new AssertionError(error); + } + } + @Test public void readSnapshot_ignoresUnownedFiles() throws Exception { server.seedUnownedBundle( @@ -98,6 +165,61 @@ public void readSnapshot_ignoresUnownedFiles() throws Exception { assertThat(backend.readSnapshot().getRecords()).isEmpty(); } + @Test + public void readSnapshot_mergesEveryOwnedRootAfterAFirstSyncRace() throws Exception { + String firstHash = server.registerAttachment("first".getBytes(StandardCharsets.UTF_8)); + String secondHash = server.registerAttachment("other".getBytes(StandardCharsets.UTF_8)); + // These are the durable results of two devices that both listed Drive before either + // created its root folder. Choosing only the lowest folder ID would lose note B forever. + server.seedOwnedBundle(snapshot(NOTE_ID, firstHash)); + server.seedOwnedBundle(snapshot(SECOND_NOTE_ID, secondHash)); + GoogleDriveSyncBackend backend = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + CLOCK, + new SyncBundleCodec()); + + SyncSnapshot merged = backend.readSnapshot(); + + assertThat(server.ownedFolderCount()).isEqualTo(2); + assertThat(merged.find(SyncRecord.Type.NOTE, NOTE_ID)).isNotNull(); + assertThat(merged.find(SyncRecord.Type.NOTE, SECOND_NOTE_ID)).isNotNull(); + } + + @Test + public void writeSnapshot_copiesDuplicateRootAttachmentsIntoTheCanonicalRoot() + throws Exception { + String firstHash = server.registerAttachment("first".getBytes(StandardCharsets.UTF_8)); + String secondHash = server.registerAttachment("other".getBytes(StandardCharsets.UTF_8)); + server.seedOwnedBundle(snapshot(NOTE_ID, firstHash)); + server.seedOwnedBundle(snapshot(SECOND_NOTE_ID, secondHash)); + GoogleDriveSyncBackend backend = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + CLOCK, + new SyncBundleCodec()); + + SyncSnapshot merged = backend.readSnapshot(); + backend.writeSnapshot(merged); + + assertThat(server.ownedAttachmentCountInCanonicalRoot(firstHash)).isEqualTo(1); + assertThat(server.ownedAttachmentCountInCanonicalRoot(secondHash)).isEqualTo(1); + SyncSnapshot reread = + new GoogleDriveSyncBackend( + "token", + server.apiBase(), + server.uploadBase(), + CLOCK, + new SyncBundleCodec()) + .readSnapshot(); + assertThat(reread.find(SyncRecord.Type.NOTE, NOTE_ID)).isNotNull(); + assertThat(reread.find(SyncRecord.Type.NOTE, SECOND_NOTE_ID)).isNotNull(); + } + @Test public void writeSnapshot_createsNewBundleWhenLegacyBundleChanges() throws Exception { String hash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; @@ -157,19 +279,21 @@ private static SyncSnapshot snapshot(String noteId, String hash) throws IOExcept note.addProperty("title", "Shopping"); note.addProperty("value", "Milk"); JsonArray hashes = new JsonArray(); - hashes.add(hash); + if (hash != null) hashes.add(hash); note.add("attachmentHashes", hashes); JsonArray manifest = new JsonArray(); - manifest.add( - new SyncBundleCodec.AttachmentManifestEntry( - UUID.nameUUIDFromBytes(hash.getBytes(StandardCharsets.UTF_8)) - .toString(), - hash, - "image/png", - 5L, - "attachments/" + hash, - "photo.png") - .toJson(true)); + if (hash != null) { + manifest.add( + new SyncBundleCodec.AttachmentManifestEntry( + UUID.nameUUIDFromBytes(hash.getBytes(StandardCharsets.UTF_8)) + .toString(), + hash, + "image/png", + 5L, + "attachments/" + hash, + "photo.png") + .toJson(true)); + } note.add("attachmentsManifest", manifest); return new SyncSnapshot( Collections.singletonList( @@ -196,10 +320,13 @@ private static final class FakeDriveServer implements AutoCloseable { private final ServerSocket serverSocket; private final Thread thread; - private final Map files = new LinkedHashMap<>(); + private final Map files = new ConcurrentHashMap<>(); + private final Map seededAttachmentContent = new LinkedHashMap<>(); + private final Map uploadSessions = new ConcurrentHashMap<>(); private volatile boolean running = true; + private volatile CyclicBarrier emptyRootListingBarrier; private SyncSnapshot updateBeforeNextPatch; - private int nextId = 1; + private final AtomicInteger nextId = new AtomicInteger(1); FakeDriveServer() throws IOException { serverSocket = new ServerSocket(0, 50, InetAddress.getByName("127.0.0.1")); @@ -209,7 +336,16 @@ private static final class FakeDriveServer implements AutoCloseable { while (running) { try { Socket socket = serverSocket.accept(); - handle(socket); + new Thread( + () -> { + try { + handle(socket); + } catch (IOException ignored) { + // Individual test connection + // failed. + } + }) + .start(); } catch (IOException ignored) { if (running) { // Keep the fake server lightweight for tests. @@ -229,6 +365,10 @@ String uploadBase() { return "http://127.0.0.1:" + serverSocket.getLocalPort() + "/upload/drive/v3/files"; } + void pauseTheNextTwoEmptyRootListings() { + emptyRootListingBarrier = new CyclicBarrier(2); + } + int ownedFolderCount() { int count = 0; for (DriveFile file : files.values()) { @@ -260,6 +400,25 @@ int ownedAttachmentCount(String hash) { return count; } + int ownedAttachmentCountInCanonicalRoot(String hash) { + String canonical = null; + for (DriveFile file : files.values()) { + if ("application/vnd.google-apps.folder".equals(file.mimeType) + && "1".equals(file.appProperties.get("mynotesOwner")) + && (canonical == null || file.id.compareTo(canonical) < 0)) { + canonical = file.id; + } + } + int count = 0; + for (DriveFile file : files.values()) { + if (hash.equals(file.appProperties.get("mynotesAttachmentSha256")) + && file.parents.contains(canonical)) { + count++; + } + } + return count; + } + byte[] readAttachment(String hash) { for (DriveFile file : files.values()) { if (hash.equals(file.appProperties.get("mynotesAttachmentSha256"))) { @@ -278,6 +437,12 @@ byte[] readBundleBytes() { return null; } + String registerAttachment(byte[] bytes) throws Exception { + String hash = sha256(bytes); + seededAttachmentContent.put(hash, bytes); + return hash; + } + void seedOwnedBundle(SyncSnapshot snapshot) throws IOException { DriveFile folder = createFile("MyNotes Sync", "application/vnd.google-apps.folder", null); @@ -285,6 +450,21 @@ void seedOwnedBundle(SyncSnapshot snapshot) throws IOException { DriveFile bundle = createFile("MyNotes.sync.v1.zip", "application/zip", folder.id); bundle.appProperties.put("mynotesBundle", "1"); bundle.content = new SyncBundleCodec().encode(snapshot, CLOCK.instant()); + for (SyncRecord record : snapshot.getLiveRecords(SyncRecord.Type.NOTE)) { + JsonArray manifest = record.getPayload().getAsJsonArray("attachmentsManifest"); + if (manifest == null) { + continue; + } + for (int index = 0; index < manifest.size(); index++) { + JsonObject attachment = manifest.get(index).getAsJsonObject(); + String hash = attachment.get("sha256").getAsString(); + DriveFile blob = createFile(hash, "application/octet-stream", folder.id); + blob.appProperties.put("mynotesAttachmentSha256", hash); + blob.content = + seededAttachmentContent.getOrDefault( + hash, new byte[attachment.get("size").getAsInt()]); + } + } } void seedUnownedBundle(SyncSnapshot snapshot) throws IOException { @@ -334,8 +514,14 @@ private Response dispatch(Request request) throws IOException { return handleFileRead(uri, path.substring("/drive/v3/files/".length())); } if ("/upload/drive/v3/files".equals(path) && "POST".equals(request.method)) { + if ("resumable".equals(parseQuery(uri).get("uploadType"))) { + return handleResumableInitiation(request); + } return handleUpload(request, null); } + if (path.startsWith("/resumable/") && "PUT".equals(request.method)) { + return handleResumableChunk(request, path.substring("/resumable/".length())); + } if (path.startsWith("/upload/drive/v3/files/")) { return handleUpload(request, path.substring("/upload/drive/v3/files/".length())); } @@ -344,6 +530,18 @@ private Response dispatch(Request request) throws IOException { private Response handleList(URI uri) { String query = parseQuery(uri).get("q"); + CyclicBarrier barrier = emptyRootListingBarrier; + if (barrier != null + && query != null + && query.contains("mynotesOwner") + && ownedFolderCount() == 0) { + try { + barrier.await(5L, TimeUnit.SECONDS); + emptyRootListingBarrier = null; + } catch (Exception error) { + return Response.json(500, "{}"); + } + } JsonArray array = new JsonArray(); for (DriveFile file : files.values()) { if (matchesQuery(file, query)) { @@ -409,6 +607,72 @@ private Response handleUpload(Request request, String fileId) throws IOException return Response.json(200, fileMetadata(file).toString(), file.eTag()); } + private Response handleResumableInitiation(Request request) throws IOException { + String length = request.headers.get("x-upload-content-length"); + if (length == null) { + return Response.json(400, "{}"); + } + String id = "session-" + uploadSessions.size(); + uploadSessions.put( + id, + new UploadSession( + readJson(request.body), + Long.parseLong(length), + request.headers.get("x-upload-content-type"))); + Map headers = new LinkedHashMap<>(); + headers.put( + "Location", + "http://127.0.0.1:" + serverSocket.getLocalPort() + "/resumable/" + id); + return Response.json(200, "{}", headers); + } + + private Response handleResumableChunk(Request request, String sessionId) + throws IOException { + UploadSession session = uploadSessions.get(sessionId); + if (session == null) { + return Response.json(404, "{}"); + } + String range = request.headers.get("content-range"); + if (range == null) { + return Response.json(400, "{}"); + } + if (range.startsWith("bytes */")) { + return resumableProgress(session); + } + Matcher matcher = Pattern.compile("bytes (\\d+)-(\\d+)/(\\d+)").matcher(range); + if (!matcher.matches() || Long.parseLong(matcher.group(3)) != session.totalBytes) { + return Response.json(400, "{}"); + } + long start = Long.parseLong(matcher.group(1)); + long end = Long.parseLong(matcher.group(2)); + if (start != session.data.size() || end - start + 1L != request.body.length) { + return Response.json(400, "{}"); + } + session.data.write(request.body); + if (session.data.size() < session.totalBytes) { + return resumableProgress(session); + } + DriveFile file = + createFile( + session.metadata.get("name").getAsString(), + session.mimeType == null + ? "application/octet-stream" + : session.mimeType, + firstParent(session.metadata)); + file.content = session.data.toByteArray(); + applyMetadata(file, session.metadata); + uploadSessions.remove(sessionId); + return Response.json(200, fileMetadata(file).toString(), file.eTag()); + } + + private static Response resumableProgress(UploadSession session) { + Map headers = new LinkedHashMap<>(); + if (session.data.size() > 0) { + headers.put("Range", "bytes=0-" + (session.data.size() - 1)); + } + return Response.json(308, "", headers); + } + private void applyMetadata(DriveFile file, JsonObject metadata) { if (metadata.has("name")) { file.name = metadata.get("name").getAsString(); @@ -434,7 +698,8 @@ private void applyMetadata(DriveFile file, JsonObject metadata) { } private DriveFile createFile(String name, String mimeType, String parentId) { - DriveFile file = new DriveFile(Integer.toString(nextId++), name, mimeType); + DriveFile file = + new DriveFile(Integer.toString(nextId.getAndIncrement()), name, mimeType); if (parentId != null) { file.parents.add(parentId); } @@ -607,6 +872,12 @@ private static void writeResponse(OutputStream output, Response response) headers.append("Content-Length: ").append(response.body.length).append("\r\n"); headers.append("Connection: close\r\n"); headers.append("Content-Type: ").append(response.contentType).append("\r\n"); + for (Map.Entry header : response.headers.entrySet()) { + headers.append(header.getKey()) + .append(": ") + .append(header.getValue()) + .append("\r\n"); + } // Deliberately no ETag header. Drive API v3 dropped the ETags that v2 sent; a fake // that returns one lets code depending on the header pass its tests and fail against // the real API, which is exactly what happened before. @@ -670,25 +941,55 @@ private static final class Response { private final String contentType; private final byte[] body; private final String eTag; + private final Map headers; - private Response(int code, String contentType, byte[] body, String eTag) { + private Response( + int code, + String contentType, + byte[] body, + String eTag, + Map headers) { this.code = code; this.contentType = contentType; this.body = body; this.eTag = eTag; + this.headers = headers; } private static Response json(int code, String body) { - return json(code, body, null); + return json(code, body, (String) null); } private static Response json(int code, String body, String eTag) { return new Response( - code, "application/json", body.getBytes(StandardCharsets.UTF_8), eTag); + code, + "application/json", + body.getBytes(StandardCharsets.UTF_8), + eTag, + new LinkedHashMap<>()); + } + + private static Response json(int code, String body, Map headers) { + return new Response( + code, "application/json", body.getBytes(StandardCharsets.UTF_8), null, headers); } private static Response binary(int code, byte[] body, String eTag) { - return new Response(code, "application/octet-stream", body, eTag); + return new Response( + code, "application/octet-stream", body, eTag, new LinkedHashMap<>()); + } + } + + private static final class UploadSession { + private final JsonObject metadata; + private final long totalBytes; + private final String mimeType; + private final ByteArrayOutputStream data = new ByteArrayOutputStream(); + + private UploadSession(JsonObject metadata, long totalBytes, String mimeType) { + this.metadata = metadata; + this.totalBytes = totalBytes; + this.mimeType = mimeType; } } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorkerTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorkerTest.java index b170463d..a4781a8e 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorkerTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncWorkerTest.java @@ -2,7 +2,6 @@ import static com.google.common.truth.Truth.assertThat; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.pasich.mynotes.data.preferences.PreferenceHelper; @@ -26,24 +25,16 @@ public void backgroundSyncAllowed_acceptsEnabledConfirmedSync() { when(preferences.isSyncEnabled()).thenReturn(true); when(preferences.isBackgroundSyncEnabled()).thenReturn(true); when(preferences.isFirstSyncConfirmed()).thenReturn(true); - when(preferences.getSyncRolloutBucket()).thenReturn(SyncRollout.CURRENT_PERCENT); assertThat(GoogleDriveSyncWorker.isBackgroundSyncAllowed(preferences)).isTrue(); } @Test - public void backgroundSyncAllowed_consultsTheRolloutGate() { - // The gate used to sit only on the manual "Sync now" path. Lowering the percentage to pull - // a bad release back would then have left this six-hourly job running for exactly the - // users a rollback needs to stop. + public void backgroundSyncAllowed_doesNotRequireRemoteConfiguration() { PreferenceHelper preferences = mock(PreferenceHelper.class); when(preferences.isSyncEnabled()).thenReturn(true); when(preferences.isBackgroundSyncEnabled()).thenReturn(true); when(preferences.isFirstSyncConfirmed()).thenReturn(true); - when(preferences.getSyncRolloutBucket()).thenReturn(SyncRollout.CURRENT_PERCENT); - - GoogleDriveSyncWorker.isBackgroundSyncAllowed(preferences); - - verify(preferences).getSyncRolloutBucket(); + assertThat(GoogleDriveSyncWorker.isBackgroundSyncAllowed(preferences)).isTrue(); } } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleValidatorTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleValidatorTest.java index e24c1dc6..4574ee07 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleValidatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleValidatorTest.java @@ -11,6 +11,8 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Collections; +import java.util.Random; +import java.util.UUID; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.junit.Test; @@ -122,6 +124,55 @@ public void validate_rejectsZipTraversalEntries() throws Exception { throw new AssertionError("Expected an IOException"); } + @Test + public void validate_rejectsNoteWithTooManyAttachmentReferences() throws Exception { + byte[] valid = codec.encode(snapshot(), Instant.parse("2026-08-31T12:00:00Z")); + JsonObject records = readJsonEntry(valid, SyncBundleCodec.ENTRY_RECORDS); + JsonArray attachmentIds = + records.getAsJsonArray("notes") + .get(0) + .getAsJsonObject() + .getAsJsonArray("attachmentIds"); + for (int index = 1; index <= SyncBundleValidator.MAX_ATTACHMENTS_PER_NOTE; index++) { + attachmentIds.add(UUID.randomUUID().toString()); + } + + try { + validator.validate( + new ByteArrayInputStream( + rewriteEntry(valid, SyncBundleCodec.ENTRY_RECORDS, records))); + } catch (IOException error) { + assertThat(error).hasMessageThat().contains("note exceeds the attachment limit"); + return; + } + throw new AssertionError("Expected an IOException"); + } + + @Test + public void validate_rejectsOversizedRecordPayload() throws Exception { + byte[] valid = codec.encode(snapshot(), Instant.parse("2026-08-31T12:00:00Z")); + JsonObject records = readJsonEntry(valid, SyncBundleCodec.ENTRY_RECORDS); + StringBuilder oversized = new StringBuilder(); + Random random = new Random(0L); + for (int index = 0; index <= SyncBundleValidator.MAX_RECORD_PAYLOAD_BYTES; index++) { + oversized.append((char) ('a' + random.nextInt(26))); + } + records.getAsJsonArray("notes") + .get(0) + .getAsJsonObject() + .addProperty("value", oversized.toString()); + + try { + validator.validate( + new ByteArrayInputStream( + rewriteEntry(valid, SyncBundleCodec.ENTRY_RECORDS, records))); + } catch (IOException error) { + assertThat(error).hasMessageThat().contains("payload size limit"); + return; + } + throw new AssertionError("Expected an IOException"); + } + private SyncSnapshot snapshot() throws IOException { JsonObject payload = new JsonObject(); payload.addProperty("title", "Shopping"); diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncRolloutTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncRolloutTest.java deleted file mode 100644 index cf5e1d68..00000000 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncRolloutTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.pasich.mynotes.data.sync; - -import static com.google.common.truth.Truth.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.pasich.mynotes.data.preferences.PreferenceHelper; -import org.junit.Test; - -public class SyncRolloutTest { - - @Test - public void ensureBucket_keepsAnAlreadyAssignedCohort() { - PreferenceHelper preferences = mock(PreferenceHelper.class); - when(preferences.getSyncRolloutBucket()).thenReturn(37); - - assertThat(SyncRollout.ensureBucket(preferences)).isEqualTo(37); - } - - @Test - public void ensureBucket_assignsAValidCohortWhenStoredValueIsOutOfRange() { - for (int stored : new int[] {-1, 0, 101}) { - PreferenceHelper preferences = mock(PreferenceHelper.class); - when(preferences.getSyncRolloutBucket()).thenReturn(stored); - - int bucket = SyncRollout.ensureBucket(preferences); - - assertThat(bucket).isAtLeast(1); - assertThat(bucket).isAtMost(100); - } - } - - @Test - public void isWithinRollout_followsTheCurrentPercentage() { - PreferenceHelper inside = mock(PreferenceHelper.class); - when(inside.getSyncRolloutBucket()).thenReturn(SyncRollout.CURRENT_PERCENT); - assertThat(SyncRollout.isWithinRollout(inside)).isTrue(); - - // Only meaningful once the percentage is dialled back below 100 for a rollback. - if (SyncRollout.CURRENT_PERCENT < 100) { - PreferenceHelper outside = mock(PreferenceHelper.class); - when(outside.getSyncRolloutBucket()).thenReturn(SyncRollout.CURRENT_PERCENT + 1); - assertThat(SyncRollout.isWithinRollout(outside)).isFalse(); - } - } -} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java index ae5dd9eb..e185d1cb 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java @@ -76,6 +76,28 @@ public void sync_matchingRemoteSnapshotDoesNotPublishAgain() { assertThat(store.appliedSnapshot.getRecords()).containsExactly(note); } + @Test + public void sync_doesNotReadOrPublishRemoteDataWhenLocalSnapshotIsIncomplete() { + FakeStore store = new FakeStore(snapshot(note(TEN, "Local note"))); + store.snapshotBuildResult = + SnapshotBuildResult.incomplete( + store.snapshot, + Collections.singletonList( + new SnapshotProblem( + SnapshotProblem.Kind.MISSING_ATTACHMENT, + SyncMetadata.RECORD_TYPE_NOTE, + NOTE_ID))); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + + SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.ERROR); + assertThat(state.getErrorMessage()).contains("MISSING_ATTACHMENT"); + assertThat(backend.events).isEmpty(); + assertThat(backend.writeSnapshotCalls).isEqualTo(0); + assertThat(store.applyCalls).isEqualTo(0); + } + @Test public void sync_downloadsRequiredRemoteAttachmentBeforeApplyingSnapshot() throws Exception { SyncRecord remote = note(TEN, "Remote with attachment"); @@ -125,10 +147,9 @@ public void sync_skipsUploadingAttachmentWhenRemoteBlobAlreadyExists() throws Ex SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); - // Neither endpoint transfers the blob: the local copy is verified from disk and the - // remote one is content-addressed and immutable, so re-downloading it on every sync only - // cost the user bandwidth. - assertThat(backend.events).containsExactly("writeSnapshot"); + // Drive is untrusted even for a content-addressed object, so remote bytes are verified + // before publication. + assertThat(backend.events).containsExactly("readAttachment", "writeSnapshot"); } @Test @@ -146,7 +167,27 @@ public void sync_repairsCorruptLocalAttachmentFromRemote() throws Exception { assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); assertThat(store.attachments.get(hash)).isEqualTo(bytes); - assertThat(backend.events).containsExactly("readAttachment", "writeSnapshot").inOrder(); + assertThat(backend.events) + .containsExactly("readAttachment", "readAttachment", "writeSnapshot") + .inOrder(); + } + + @Test + public void sync_corruptRemoteAttachmentWithValidLocalCopyDoesNotPublish() throws Exception { + byte[] bytes = "local attachment".getBytes(StandardCharsets.UTF_8); + String hash = sha256(bytes); + FakeStore store = new FakeStore(snapshot(note(TEN, "Local"))); + store.attachmentHashes = Collections.singletonList(hash); + store.attachments.put(hash, bytes); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + backend.attachments.put(hash, "corrupt remote".getBytes(StandardCharsets.UTF_8)); + + SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.ERROR); + assertThat(state.getErrorMessage()).contains("checksum"); + assertThat(backend.writeSnapshotCalls).isEqualTo(0); + assertThat(store.applyCalls).isEqualTo(0); } @Test @@ -288,6 +329,7 @@ private static String sha256(byte[] bytes) throws Exception { private static final class FakeStore implements SyncStore { private SyncSnapshot snapshot; + private SnapshotBuildResult snapshotBuildResult; private SyncSnapshot appliedSnapshot; private List appliedConflicts = Collections.emptyList(); private SyncState state = SyncState.idle(); @@ -308,6 +350,13 @@ public SyncSnapshot readSnapshot() { return snapshot; } + @Override + public SnapshotBuildResult buildSnapshot() { + return snapshotBuildResult == null + ? SnapshotBuildResult.publishable(snapshot) + : snapshotBuildResult; + } + @Override public void applySnapshot(SyncSnapshot snapshot, List conflicts) { events.add("applySnapshot"); diff --git a/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java index a4991ad5..31ffec10 100644 --- a/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java @@ -233,10 +233,9 @@ public void disconnect_clearsConsentAndStoredStateTogether() { } @Test - public void syncNow_allowsUsersInTheHighestRolloutBucket() { + public void syncNow_allowsAnExplicitlyEnabledUser() { FakePreferenceHelper preferences = new FakePreferenceHelper(); preferences.firstSyncConfirmed = true; - preferences.rolloutBucket = 100; GoogleDriveAuthorization authorization = mock(GoogleDriveAuthorization.class); Mockito.doAnswer( invocation -> { @@ -270,10 +269,9 @@ public void syncNow_allowsUsersInTheHighestRolloutBucket() { } @Test - public void syncNow_repairsAnInvalidStoredRolloutBucketBeforeSyncing() { + public void syncNow_doesNotRequireAFeatureRollout() { FakePreferenceHelper preferences = new FakePreferenceHelper(); preferences.firstSyncConfirmed = true; - preferences.rolloutBucket = 0; GoogleDriveAuthorization authorization = mock(GoogleDriveAuthorization.class); Mockito.doAnswer( invocation -> { @@ -301,7 +299,6 @@ public void syncNow_repairsAnInvalidStoredRolloutBucketBeforeSyncing() { CapturingCallback callback = new CapturingCallback<>(); coordinator.syncNow(mock(Activity.class), callback); - assertThat(preferences.rolloutBucket >= 1 && preferences.rolloutBucket <= 100).isTrue(); assertThat(store.lastToken).isEqualTo("access-token"); assertThat(callback.error).isNull(); } @@ -439,7 +436,6 @@ private static final class FakePreferenceHelper implements PreferenceHelper { private boolean syncEnabled; private boolean backgroundEnabled; private boolean firstSyncConfirmed; - private int rolloutBucket = 1; @Override public int getFormatCount() { @@ -518,15 +514,5 @@ public boolean isFirstSyncConfirmed() { public void setFirstSyncConfirmed(boolean confirmed) { firstSyncConfirmed = confirmed; } - - @Override - public int getSyncRolloutBucket() { - return rolloutBucket; - } - - @Override - public void setSyncRolloutBucket(int bucket) { - rolloutBucket = bucket; - } } } From 93be3675eeaa52aeb2ee1c0f0aaac1694738d423 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 14:10:44 +0300 Subject: [PATCH 03/16] fix: preserve sync attachments and conflicts --- .../18.json | 501 ++++++++++++++++++ .../com/pasich/mynotes/db/MigrationTest.java | 32 ++ .../mynotes/data/database/AppDatabase.java | 21 + .../data/database/dao/SyncConflictDao.java | 5 +- .../database/entities/SyncConflictEntity.java | 6 +- .../sync/AttachmentIntegrityException.java | 21 + .../data/sync/GoogleDriveSyncBackend.java | 118 ++++- .../mynotes/data/sync/RoomSyncStore.java | 274 ++++++++-- .../mynotes/data/sync/SyncBundleCodec.java | 10 +- .../data/sync/SyncBundleValidator.java | 10 +- .../pasich/mynotes/data/sync/SyncService.java | 76 ++- .../pasich/mynotes/di/ApplicationModule.java | 3 +- .../models/EditorAttachment.java | 2 + .../utils/constants/DatabaseConstants.java | 2 +- .../data/sync/GoogleDriveSyncBackendTest.java | 46 +- .../data/sync/SyncBundleCodecTest.java | 42 ++ .../mynotes/ui/sync/SyncCoordinatorTest.java | 1 + 17 files changed, 1072 insertions(+), 98 deletions(-) create mode 100644 app/schemas/com.pasich.mynotes.data.database.AppDatabase/18.json create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/AttachmentIntegrityException.java diff --git a/app/schemas/com.pasich.mynotes.data.database.AppDatabase/18.json b/app/schemas/com.pasich.mynotes.data.database.AppDatabase/18.json new file mode 100644 index 00000000..7b68d98c --- /dev/null +++ b/app/schemas/com.pasich.mynotes.data.database.AppDatabase/18.json @@ -0,0 +1,501 @@ +{ + "formatVersion": 1, + "database": { + "version": 18, + "identityHash": "bd62f992291fa89b478f7f4f9a6ed5fb", + "entities": [ + { + "tableName": "tags", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `visibility` INTEGER NOT NULL, `systemAction` INTEGER NOT NULL, `position` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "nameTag", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "visibility", + "columnName": "visibility", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "systemAction", + "columnName": "systemAction", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "notes", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `value` TEXT, `date` INTEGER NOT NULL, `tag` TEXT, `valueJson` TEXT, `hasRichContent` INTEGER NOT NULL, `attachments` TEXT, `isTrash` INTEGER NOT NULL, `reminderTime` INTEGER, `isPinned` INTEGER NOT NULL, `reminderRepeat` TEXT NOT NULL, `reminderIntervalMinutes` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT" + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT" + }, + { + "fieldPath": "date", + "columnName": "date", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "tag", + "columnName": "tag", + "affinity": "TEXT" + }, + { + "fieldPath": "valueJson", + "columnName": "valueJson", + "affinity": "TEXT" + }, + { + "fieldPath": "hasRichContent", + "columnName": "hasRichContent", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "attachments", + "columnName": "attachments", + "affinity": "TEXT" + }, + { + "fieldPath": "isTrash", + "columnName": "isTrash", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reminderTime", + "columnName": "reminderTime", + "affinity": "INTEGER" + }, + { + "fieldPath": "isPinned", + "columnName": "isPinned", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reminderRepeat", + "columnName": "reminderRepeat", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "reminderIntervalMinutes", + "columnName": "reminderIntervalMinutes", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "tasks", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT NOT NULL, `description` TEXT, `isDone` INTEGER NOT NULL DEFAULT 0, `categoryId` INTEGER NOT NULL DEFAULT 0, `createdAt` INTEGER NOT NULL DEFAULT 0, `position` INTEGER NOT NULL DEFAULT 0, `reminderTime` INTEGER, `reminderIntervalMinutes` INTEGER NOT NULL DEFAULT 0)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT" + }, + { + "fieldPath": "isDone", + "columnName": "isDone", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "categoryId", + "columnName": "categoryId", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "reminderTime", + "columnName": "reminderTime", + "affinity": "INTEGER" + }, + { + "fieldPath": "reminderIntervalMinutes", + "columnName": "reminderIntervalMinutes", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "task_categories", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `colorHex` TEXT NOT NULL DEFAULT '#6750A4', `position` INTEGER NOT NULL DEFAULT 0)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "colorHex", + "columnName": "colorHex", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'#6750A4'" + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "sync_metadata", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`recordType` TEXT NOT NULL, `localId` INTEGER NOT NULL, `stableId` TEXT NOT NULL, `updatedAt` INTEGER NOT NULL, `deletedAt` INTEGER, PRIMARY KEY(`recordType`, `localId`))", + "fields": [ + { + "fieldPath": "recordType", + "columnName": "recordType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "localId", + "columnName": "localId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "stableId", + "columnName": "stableId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "updatedAt", + "columnName": "updatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deletedAt", + "columnName": "deletedAt", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "recordType", + "localId" + ] + }, + "indices": [ + { + "name": "index_sync_metadata_recordType_stableId", + "unique": true, + "columnNames": [ + "recordType", + "stableId" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_sync_metadata_recordType_stableId` ON `${TABLE_NAME}` (`recordType`, `stableId`)" + }, + { + "name": "index_sync_metadata_updatedAt", + "unique": false, + "columnNames": [ + "updatedAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_metadata_updatedAt` ON `${TABLE_NAME}` (`updatedAt`)" + }, + { + "name": "index_sync_metadata_deletedAt", + "unique": false, + "columnNames": [ + "deletedAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_metadata_deletedAt` ON `${TABLE_NAME}` (`deletedAt`)" + } + ] + }, + { + "tableName": "sync_conflicts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `recordType` TEXT NOT NULL, `stableId` TEXT NOT NULL, `versionPairHash` TEXT NOT NULL, `winnerSource` TEXT NOT NULL, `winnerJson` TEXT NOT NULL, `loserJson` TEXT NOT NULL, `winnerUpdatedAt` INTEGER NOT NULL, `loserUpdatedAt` INTEGER NOT NULL, `winnerTombstone` INTEGER NOT NULL, `loserTombstone` INTEGER NOT NULL, `resolution` TEXT NOT NULL, `resolved` INTEGER NOT NULL, `createdAt` INTEGER NOT NULL, `resolvedAt` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "recordType", + "columnName": "recordType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stableId", + "columnName": "stableId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionPairHash", + "columnName": "versionPairHash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerSource", + "columnName": "winnerSource", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerJson", + "columnName": "winnerJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "loserJson", + "columnName": "loserJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerUpdatedAt", + "columnName": "winnerUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "loserUpdatedAt", + "columnName": "loserUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "winnerTombstone", + "columnName": "winnerTombstone", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "loserTombstone", + "columnName": "loserTombstone", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "resolution", + "columnName": "resolution", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "resolved", + "columnName": "resolved", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "resolvedAt", + "columnName": "resolvedAt", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_sync_conflicts_recordType_stableId_versionPairHash", + "unique": true, + "columnNames": [ + "recordType", + "stableId", + "versionPairHash" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_sync_conflicts_recordType_stableId_versionPairHash` ON `${TABLE_NAME}` (`recordType`, `stableId`, `versionPairHash`)" + }, + { + "name": "index_sync_conflicts_resolved", + "unique": false, + "columnNames": [ + "resolved" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_conflicts_resolved` ON `${TABLE_NAME}` (`resolved`)" + }, + { + "name": "index_sync_conflicts_createdAt", + "unique": false, + "columnNames": [ + "createdAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_conflicts_createdAt` ON `${TABLE_NAME}` (`createdAt`)" + } + ] + }, + { + "tableName": "sync_state", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `status` TEXT NOT NULL, `backendIdentifier` TEXT, `lastSuccessfulSyncAt` INTEGER, `attemptStartedAt` INTEGER, `errorMessage` TEXT, `conflictCount` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "backendIdentifier", + "columnName": "backendIdentifier", + "affinity": "TEXT" + }, + { + "fieldPath": "lastSuccessfulSyncAt", + "columnName": "lastSuccessfulSyncAt", + "affinity": "INTEGER" + }, + { + "fieldPath": "attemptStartedAt", + "columnName": "attemptStartedAt", + "affinity": "INTEGER" + }, + { + "fieldPath": "errorMessage", + "columnName": "errorMessage", + "affinity": "TEXT" + }, + { + "fieldPath": "conflictCount", + "columnName": "conflictCount", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'bd62f992291fa89b478f7f4f9a6ed5fb')" + ] + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java index ffbfdb5c..5ff176c3 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java @@ -98,6 +98,38 @@ public void migrate16to17_createsSyncStateTable() throws IOException { } } + @Test + public void migrate17to18_preservesExistingConflictAndAllowsVersionPairsToCoexist() + throws IOException { + SupportSQLiteDatabase db = helper.createDatabase(TEST_DB, 17); + db.execSQL( + "INSERT INTO sync_conflicts " + + "(recordType, stableId, winnerSource, winnerJson, loserJson, winnerUpdatedAt, " + + "loserUpdatedAt, winnerTombstone, loserTombstone, resolution, resolved, createdAt, resolvedAt) " + + "VALUES ('note', 'stable', 'LOCAL', '{}', '{}', 1, 1, 0, 0, 'PENDING', 0, 1, 0)"); + db.close(); + + SupportSQLiteDatabase migrated = + helper.runMigrationsAndValidate(TEST_DB, 18, true, AppDatabase.MIGRATION_17_18); + try { + migrated.execSQL( + "INSERT INTO sync_conflicts " + + "(recordType, stableId, versionPairHash, winnerSource, winnerJson, loserJson, " + + "winnerUpdatedAt, loserUpdatedAt, winnerTombstone, loserTombstone, resolution, " + + "resolved, createdAt, resolvedAt) " + + "VALUES ('note', 'stable', 'new-pair', 'REMOTE', '{}', '{}', 2, 2, 0, 0, " + + "'PENDING', 0, 2, 0)"); + try (android.database.Cursor cursor = + migrated.query( + "SELECT COUNT(*) FROM sync_conflicts WHERE recordType = 'note' AND stableId = 'stable'")) { + assertThat(cursor.moveToFirst()).isTrue(); + assertThat(cursor.getInt(0)).isEqualTo(2); + } + } finally { + migrated.close(); + } + } + @Test public void migrate14to15_backfillsCategoryMetadata() throws IOException { SupportSQLiteDatabase db = helper.createDatabase(TEST_DB, 14); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java b/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java index 1b3aab7f..89fe3aaa 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java @@ -137,6 +137,27 @@ public void migrate(@NonNull SupportSQLiteDatabase database) { } }; + /** Preserves every unresolved version pair instead of replacing conflicts by logical record. */ + public static final Migration MIGRATION_17_18 = + new Migration(17, 18) { + @Override + public void migrate(@NonNull SupportSQLiteDatabase database) { + database.execSQL( + "ALTER TABLE `sync_conflicts` ADD COLUMN `versionPairHash` TEXT NOT NULL DEFAULT ''"); + // Version 17 could contain at most one row per logical record. Give each + // legacy row a durable unique identity without trying to hash untrusted JSON + // in SQLite during a migration. + database.execSQL( + "UPDATE `sync_conflicts` SET `versionPairHash` = 'legacy-' || `id` " + + "WHERE `versionPairHash` = ''"); + database.execSQL("DROP INDEX IF EXISTS `index_sync_conflicts_recordType_stableId`"); + database.execSQL( + "CREATE UNIQUE INDEX IF NOT EXISTS " + + "`index_sync_conflicts_recordType_stableId_versionPairHash` " + + "ON `sync_conflicts` (`recordType`, `stableId`, `versionPairHash`)"); + } + }; + private static void insertMetadataForExistingRecords( SupportSQLiteDatabase database, String recordType, diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java index 8f80162b..3825c293 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java @@ -10,8 +10,9 @@ @Dao public interface SyncConflictDao { - @Insert(onConflict = OnConflictStrategy.REPLACE) - void replaceAll(List conflicts); + /** Exact repeated observations are harmless; distinct version pairs must coexist. */ + @Insert(onConflict = OnConflictStrategy.IGNORE) + void insertIgnoringDuplicates(List conflicts); @Query("DELETE FROM sync_conflicts") void clearAll(); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncConflictEntity.java b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncConflictEntity.java index c6f6acb2..7b55d51d 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncConflictEntity.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncConflictEntity.java @@ -10,7 +10,7 @@ tableName = "sync_conflicts", indices = { @Index( - value = {"recordType", "stableId"}, + value = {"recordType", "stableId", "versionPairHash"}, unique = true), @Index(value = {"resolved"}), @Index(value = {"createdAt"}) @@ -22,6 +22,8 @@ public class SyncConflictEntity { @NonNull public String recordType; @NonNull public String stableId; + /** Stable digest of the exact winner/loser version pair; never use the mutable row id. */ + @NonNull public String versionPairHash; @NonNull public String winnerSource; @NonNull public String winnerJson; @NonNull public String loserJson; @@ -37,6 +39,7 @@ public class SyncConflictEntity { public SyncConflictEntity( @NonNull String recordType, @NonNull String stableId, + @NonNull String versionPairHash, @NonNull String winnerSource, @NonNull String winnerJson, @NonNull String loserJson, @@ -50,6 +53,7 @@ public SyncConflictEntity( long resolvedAt) { this.recordType = recordType; this.stableId = stableId; + this.versionPairHash = versionPairHash; this.winnerSource = winnerSource; this.winnerJson = winnerJson; this.loserJson = loserJson; diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentIntegrityException.java b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentIntegrityException.java new file mode 100644 index 00000000..5c0b514e --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentIntegrityException.java @@ -0,0 +1,21 @@ +package com.pasich.mynotes.data.sync; + +import java.io.IOException; + +/** + * Indicates that bytes did not satisfy the immutable attachment contract. + * + *

This is deliberately distinct from a transport failure. An object discovered after a lost + * HTTP response can only confirm an ambiguous request; it can never turn a hash or size mismatch + * into success. + */ +public final class AttachmentIntegrityException extends IOException { + + public AttachmentIntegrityException(String message) { + super(message); + } + + public AttachmentIntegrityException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index fd22ada3..38993b6c 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -22,6 +22,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.UUID; @@ -131,7 +132,7 @@ public synchronized void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IO @Override public synchronized boolean hasAttachment(@NonNull String sha256) throws IOException { for (String folderId : findFolderIds()) { - if (findAttachment(folderId, sha256) != null) { + if (findVerifiedAttachment(folderId, sha256, null) != null) { return true; } } @@ -142,7 +143,7 @@ public synchronized boolean hasAttachment(@NonNull String sha256) throws IOExcep @Override public synchronized InputStream readAttachment(@NonNull String sha256) throws IOException { for (String folderId : findFolderIds()) { - String attachmentId = findAttachment(folderId, sha256); + String attachmentId = findVerifiedAttachment(folderId, sha256, null); if (attachmentId == null) { continue; } @@ -171,7 +172,7 @@ public synchronized void writeAttachment( @NonNull String sha256, long sizeBytes, @NonNull InputStream content) throws IOException { String folderId = ensureCanonicalFolderId(); - if (findAttachment(folderId, sha256) != null) { + if (findVerifiedAttachment(folderId, sha256, sizeBytes >= 0L ? sizeBytes : null) != null) { return; } if (sizeBytes >= 0L) { @@ -248,7 +249,7 @@ private void ensureCanonicalAttachments( } for (Map.Entry attachment : sizes.entrySet()) { String hash = attachment.getKey(); - if (findAttachment(canonicalRootId, hash) != null) { + if (findVerifiedAttachment(canonicalRootId, hash, attachment.getValue()) != null) { continue; } InputStream source = readAttachment(hash); @@ -324,6 +325,91 @@ private String findAttachment(@NonNull String folderId, @NonNull String sha256) return smallestId(files); } + /** + * An app property is only an index. Read and verify every candidate before it may satisfy a + * content-addressed reference; corrupt candidates remain harmless Drive orphans. + */ + @Nullable + private String findVerifiedAttachment( + @NonNull String folderId, @NonNull String sha256, @Nullable Long expectedSize) + throws IOException { + JsonArray files = + listFiles( + "'" + + folderId + + "' in parents and trashed = false and " + + appPropertyClause("mynotesAttachmentSha256", sha256), + "files(id,name)"); + List candidateIds = new ArrayList<>(files.size()); + for (int index = 0; index < files.size(); index++) { + candidateIds.add(files.get(index).getAsJsonObject().get("id").getAsString()); + } + candidateIds.sort(Comparator.naturalOrder()); + for (String candidateId : candidateIds) { + try (InputStream candidate = openAttachment(candidateId)) { + verifyAttachment(candidate, sha256, expectedSize); + return candidateId; + } catch (AttachmentIntegrityException corrupt) { + // A second content-addressed duplicate may be valid. Never accept the property + // alone and never delete this object during a correctness path. + } + } + return null; + } + + @NonNull + private InputStream openAttachment(@NonNull String attachmentId) throws IOException { + HttpURLConnection connection = + requestExecutor.executeIdempotent( + () -> + openSuccessful( + "GET", apiBase + "/files/" + attachmentId + "?alt=media")); + try { + return new ConnectionInputStream(connection, MAX_ATTACHMENT_RESPONSE_BYTES); + } catch (IOException failure) { + connection.disconnect(); + throw failure; + } + } + + private static void verifyAttachment( + @NonNull InputStream input, @NonNull String expectedHash, @Nullable Long expectedSize) + throws IOException { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException error) { + throw new IOException("SHA-256 is unavailable", error); + } + long size = 0L; + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + digest.update(buffer, 0, read); + size += read; + if (size > MAX_ATTACHMENT_RESPONSE_BYTES) { + throw new AttachmentIntegrityException("Attachment exceeds the sync size limit"); + } + } + String actual = toHex(digest.digest()); + if (!expectedHash.equals(actual)) { + throw new AttachmentIntegrityException( + "Attachment checksum does not match its declared hash"); + } + if (expectedSize != null && expectedSize.longValue() != size) { + throw new AttachmentIntegrityException("Attachment size does not match its declared size"); + } + } + + @NonNull + private static String toHex(@NonNull byte[] bytes) { + StringBuilder value = new StringBuilder(bytes.length * 2); + for (byte byteValue : bytes) { + value.append(String.format(Locale.US, "%02x", byteValue & 0xff)); + } + return value.toString(); + } + @NonNull private JsonArray listFiles(@NonNull String query, @NonNull String fields) throws IOException { JsonArray result = new JsonArray(); @@ -585,7 +671,8 @@ private void uploadAttachmentOrConfirm( } catch (IOException uploadFailure) { // Attachment identity is its SHA-256. A successful request whose response was lost is // confirmed by discovery, not repeated with an already-consumed stream. - if (findAttachment(folderId, sha256) == null) { + if (!isAmbiguousTransportFailure(uploadFailure) + || findVerifiedAttachment(folderId, sha256, sizeBytes) == null) { throw uploadFailure; } } @@ -597,12 +684,27 @@ private void uploadAttachmentOrConfirm( try { uploadFile(folderId, sha256, MIME_BINARY, content, false); } catch (IOException uploadFailure) { - if (findAttachment(folderId, sha256) == null) { + if (!isAmbiguousTransportFailure(uploadFailure) + || findVerifiedAttachment(folderId, sha256, (long) content.length) == null) { throw uploadFailure; } } } + private static boolean isAmbiguousTransportFailure(@NonNull IOException failure) { + if (failure instanceof AttachmentIntegrityException + || failure instanceof java.io.InterruptedIOException) { + return false; + } + if (failure instanceof DriveRequestExecutor.DriveHttpException) { + int status = ((DriveRequestExecutor.DriveHttpException) failure).statusCode; + return status >= 500 && status <= 599; + } + return failure instanceof java.net.SocketException + || failure instanceof java.net.SocketTimeoutException + || failure instanceof java.net.ConnectException; + } + /** * Writes one {@code multipart/related} upload straight to the socket. * @@ -991,14 +1093,14 @@ private void verifyEndOfStream() throws IOException { throw new IOException("Attachment upload ended before the source was verified"); } if (size != expectedSize) { - throw new IOException("Attachment size does not match sync metadata"); + throw new AttachmentIntegrityException("Attachment size does not match sync metadata"); } StringBuilder actualHash = new StringBuilder(64); for (byte value : digest.digest()) { actualHash.append(String.format(java.util.Locale.US, "%02x", value & 0xff)); } if (!expectedHash.equals(actualHash.toString())) { - throw new IOException("Attachment checksum does not match sync metadata"); + throw new AttachmentIntegrityException("Attachment checksum does not match sync metadata"); } } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index 52c47287..aed34621 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -202,8 +202,10 @@ private void applySnapshotInternal( @NonNull List conflicts, @Nullable SyncState finalState) throws IOException { - database.runInTransaction( - () -> { + try { + database.runInTransaction( + () -> { + try { Map byStableId = new HashMap<>(); for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { byStableId.put(metadata.recordType + ":" + metadata.stableId, metadata); @@ -252,7 +254,13 @@ private void applySnapshotInternal( if (finalState != null) { database.syncStateDao().upsert(toEntity(finalState)); } - }); + } catch (IOException error) { + throw new SyncRuntimeException(error); + } + }); + } catch (SyncRuntimeException error) { + throw error.ioException; + } } @Nullable @@ -299,7 +307,7 @@ else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { return result; } - private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) { + private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) throws IOException { if ("note".equals(metadata.recordType)) { Note note = gson.fromJson(payload, Note.class); note.setId((int) metadata.localId); @@ -330,7 +338,7 @@ private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) { } } - private long insertRemoteRecord(SyncRecord record) { + private long insertRemoteRecord(SyncRecord record) throws IOException { if (record.getType() == SyncRecord.Type.NOTE) { Note note = gson.fromJson(record.getPayload(), Note.class); note.setId(0); @@ -433,6 +441,12 @@ public List getUnresolvedConflicts() { public void resolveConflict(long conflictId, @NonNull SyncResolution resolution) throws IOException { if (resolution == SyncResolution.PENDING) return; + SyncConflictEntity pending = database.syncConflictDao().getById(conflictId); + if (pending == null || pending.resolved) return; + // Resolution is a user-visible mutation. Verify and pin the selected version before its + // conflict row can be marked resolved; a missing blob must leave both the note and conflict + // untouched, including when the winner happens to already be visible in Room. + pinResolvedConflictAttachments(selectRecordForResolution(pending, resolution)); try { database.runInTransaction( () -> { @@ -461,6 +475,27 @@ public void resolveConflict(long conflictId, @NonNull SyncResolution resolution) } } + private void pinResolvedConflictAttachments(@NonNull SyncRecord selected) throws IOException { + if (selected.isTombstone() || selected.getType() != SyncRecord.Type.NOTE) return; + JsonArray manifest = selected.getPayload().getAsJsonArray("attachmentsManifest"); + if (manifest == null) return; + for (JsonElement element : manifest) { + if (!element.isJsonObject()) { + throw new IOException("Attachment manifest entry is invalid"); + } + SyncBundleCodec.AttachmentManifestEntry entry = + SyncBundleCodec.AttachmentManifestEntry.fromJson(element.getAsJsonObject()); + File source = resolveLocalAttachment(entry.sha256); + if (source == null || !isVerifiedAttachmentFile(source, entry.sha256, entry.size)) { + throw new IOException("Required conflict attachment is unavailable: " + entry.sha256); + } + File cache = attachmentFile(entry.sha256); + if (!isVerifiedAttachmentFile(cache, entry.sha256, entry.size)) { + copyVerifiedAttachment(source, cache, entry.sha256, entry.size); + } + } + } + private void persistConflicts(@NonNull List conflicts) { if (conflicts.isEmpty()) return; @@ -471,6 +506,7 @@ private void persistConflicts(@NonNull List conflicts) new SyncConflictEntity( conflict.getType().getWireValue(), conflict.getId(), + conflictVersionPairHash(conflict), conflict.getWinnerSource().name(), conflict.getWinner().canonicalSerializedPayload(), conflict.getLoser().canonicalSerializedPayload(), @@ -483,7 +519,27 @@ private void persistConflicts(@NonNull List conflicts) createdAt, 0L)); } - database.syncConflictDao().replaceAll(rows); + database.syncConflictDao().insertIgnoringDuplicates(rows); + } + + @NonNull + private static String conflictVersionPairHash(@NonNull SyncMergeResult.Conflict conflict) { + String source = + conflict.getType().getWireValue() + + "\n" + + conflict.getId() + + "\n" + + conflict.getWinnerSource().name() + + "\n" + + conflict.getWinner().canonicalSerializedPayload() + + "\n" + + conflict.getLoser().canonicalSerializedPayload(); + try { + return sha256( + new java.io.ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); + } catch (IOException impossible) { + throw new IllegalStateException("Could not hash sync conflict identity", impossible); + } } private void applyResolvedRecord( @@ -683,7 +739,8 @@ private boolean addAttachmentMetadata( JsonArray hashes = new JsonArray(); JsonObject names = new JsonObject(); boolean complete = true; - for (JsonElement element : attachments) { + for (int attachmentIndex = 0; attachmentIndex < attachments.size(); attachmentIndex++) { + JsonElement element = attachments.get(attachmentIndex); if (!element.isJsonObject()) { addSnapshotProblem( snapshotProblems, @@ -748,11 +805,28 @@ private boolean addAttachmentMetadata( attachment.name == null || attachment.name.trim().isEmpty() ? file.getName() : attachment.name.trim(); + String logicalId = attachment.id; + if (logicalId == null || !logicalId.matches("[0-9a-fA-F-]{36}")) { + // Existing editor data predates logical attachment IDs. Deriving from the stable + // note, source URL and position keeps the migration deterministic while allowing + // equal-content references to remain distinct logical attachments. + logicalId = + UUID.nameUUIDFromBytes( + (metadata.stableId + + "\n" + + attachmentIndex + + "\n" + + attachment.url + + "\n" + + displayName) + .getBytes(StandardCharsets.UTF_8)) + .toString(); + } hashes.add(hash); - names.addProperty(hash, displayName); + names.addProperty(logicalId, displayName); JsonObject manifestEntry = new JsonObject(); - manifestEntry.addProperty("id", stableAttachmentId(hash)); + manifestEntry.addProperty("id", logicalId); manifestEntry.addProperty("sha256", hash); manifestEntry.addProperty("mimeType", detectMimeType(file, attachment, displayName)); manifestEntry.addProperty("size", file.length()); @@ -774,50 +848,125 @@ private static void addSnapshotProblem( problems.add(new SnapshotProblem(kind, metadata.recordType, metadata.stableId)); } - private void restoreAttachments(Note note, JsonObject payload) { - if (!payload.has("attachmentHashes") || !payload.has("attachmentNames")) return; - try { - JsonArray hashes = payload.getAsJsonArray("attachmentHashes"); - JsonObject names = payload.getAsJsonObject("attachmentNames"); - JsonArray attachments = new JsonArray(); - File folder = AttachmentStorage.noteFolder(context, note.getId()); - for (JsonElement item : hashes) { - String hash = item.getAsString(); - // Not just the download cache: when the local version of a note wins the merge it - // is re-applied through this same path, and its blobs live in the note's own - // folder. Resolving only the cache silently rewrote such a note with an empty - // attachment list, destroying files that were never in conflict. - File source = resolveLocalAttachment(hash); - if (source == null) continue; - String name = names.has(hash) ? names.get(hash).getAsString() : hash; - if (!isSafeAttachmentName(name)) { - continue; + /** + * Materializes every attachment before changing the Room row. Targets use the immutable + * logical-ID/content-ID pair rather than a display name, so a rollback can leave only harmless + * new files and can never alter bytes addressed by the pre-transaction note. + */ + private void restoreAttachments(Note note, JsonObject payload) throws IOException { + JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); + if (manifest == null) { + if (payload.has("attachmentHashes")) { + throw new IOException("Attachment manifest is missing"); + } + return; + } + File folder = AttachmentStorage.noteFolder(context, note.getId()); + if (!folder.isDirectory() && !folder.mkdirs()) { + throw new IOException("Could not create attachment folder"); + } + JsonArray restored = new JsonArray(); + for (JsonElement element : manifest) { + if (!element.isJsonObject()) { + throw new IOException("Attachment manifest entry is invalid"); + } + SyncBundleCodec.AttachmentManifestEntry entry; + try { + entry = SyncBundleCodec.AttachmentManifestEntry.fromJson(element.getAsJsonObject()); + } catch (RuntimeException error) { + throw new IOException("Attachment manifest entry is invalid", error); + } + if (entry.id == null + || entry.sha256 == null + || !entry.id.matches("[0-9a-fA-F-]{36}") + || !entry.sha256.matches("[0-9a-f]{64}") + || entry.size < 0L) { + throw new IOException("Attachment manifest entry is invalid"); + } + String displayName = entry.displayName == null ? entry.id : entry.displayName; + if (!isSafeAttachmentName(displayName)) { + throw new IOException("Attachment display name is invalid"); + } + File source = resolveLocalAttachment(entry.sha256); + if (source == null || !source.isFile() || !source.canRead()) { + throw new IOException("Required attachment is unavailable: " + entry.sha256); + } + File target = new File(folder, entry.id + "-" + entry.sha256); + if (!isVerifiedAttachmentFile(target, entry.sha256, entry.size)) { + // A corrupted old target may still be referenced by the pre-sync note. Preserve it + // and use a fresh opaque immutable name for this candidate state instead. + if (target.exists()) { + target = + new File( + folder, + entry.id + + "-" + + entry.sha256 + + "-" + + UUID.randomUUID()); } - File target = new File(folder, name); - if (!source.getAbsolutePath().equals(target.getAbsolutePath())) { - // Scoped per file: one unreadable blob must not discard the note's other - // attachments, which is what a single loop-wide catch used to do. Skipped - // entirely when the file is already in place, because opening it for writing - // would truncate the very file being read. - try (InputStream in = new FileInputStream(source); - OutputStream out = new FileOutputStream(target)) { - byte[] buffer = new byte[8192]; - int read; - while ((read = in.read(buffer)) != -1) out.write(buffer, 0, read); - } catch (IOException error) { - Log.w(TAG, "Could not restore attachment " + hash, error); - continue; - } + copyVerifiedAttachment(source, target, entry.sha256, entry.size); + } + JsonObject attachment = new JsonObject(); + attachment.addProperty( + "url", "file://attachments/note_" + note.getId() + "/" + target.getName()); + attachment.addProperty("name", displayName); + attachment.addProperty("id", entry.id); + restored.add(attachment); + } + note.setAttachments(gson.toJson(restored)); + } + + private static boolean isVerifiedAttachmentFile( + @NonNull File file, @NonNull String expectedHash, long expectedSize) throws IOException { + return file.isFile() + && file.length() == expectedSize + && expectedHash.equals(sha256(file)); + } + + private static void copyVerifiedAttachment( + @NonNull File source, + @NonNull File target, + @NonNull String expectedHash, + long expectedSize) + throws IOException { + File temporary = + new File(target.getParentFile(), target.getName() + ".tmp-" + UUID.randomUUID()); + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (java.security.NoSuchAlgorithmException error) { + throw new IOException("SHA-256 is unavailable", error); + } + long copied = 0L; + try (InputStream in = new FileInputStream(source); + OutputStream out = new FileOutputStream(temporary)) { + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) != -1) { + out.write(buffer, 0, read); + digest.update(buffer, 0, read); + copied += read; + if (copied > expectedSize) { + throw new AttachmentIntegrityException("Attachment exceeds its declared size"); } - JsonObject attachment = new JsonObject(); - attachment.addProperty( - "url", "file://attachments/note_" + note.getId() + "/" + name); - attachment.addProperty("name", name); - attachments.add(attachment); } - note.setAttachments(gson.toJson(attachments)); - } catch (RuntimeException error) { - Log.w(TAG, "Malformed attachment metadata for note " + note.getId(), error); + } catch (IOException failure) { + if (temporary.exists() && !temporary.delete()) { + Log.w(TAG, "Could not remove failed staged attachment"); + } + throw failure; + } + StringBuilder hash = new StringBuilder(64); + for (byte value : digest.digest()) hash.append(String.format("%02x", value & 0xff)); + String actual = hash.toString(); + if (copied != expectedSize || !expectedHash.equals(actual)) { + if (!temporary.delete()) Log.w(TAG, "Could not remove invalid staged attachment"); + throw new AttachmentIntegrityException("Attachment checksum does not match sync metadata"); + } + if (!temporary.renameTo(target)) { + if (!temporary.delete()) Log.w(TAG, "Could not remove uncommitted staged attachment"); + throw new IOException("Could not finalize staged attachment"); } } @@ -849,6 +998,26 @@ private static String sha256(File file) throws IOException { return hex.toString(); } + @NonNull + private static String sha256(@NonNull InputStream input) throws IOException { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (java.security.NoSuchAlgorithmException error) { + throw new IOException("SHA-256 is unavailable", error); + } + try (InputStream in = input) { + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) != -1) { + digest.update(buffer, 0, read); + } + } + StringBuilder hex = new StringBuilder(64); + for (byte value : digest.digest()) hex.append(String.format("%02x", value & 0xff)); + return hex.toString(); + } + /** Resolves a serialized note attachment to its app-private file. */ public interface AttachmentResolver { @Nullable @@ -866,11 +1035,6 @@ public interface TransactionFailureInjector { void afterRecordApplied(@NonNull SyncRecord record); } - @NonNull - private static String stableAttachmentId(@NonNull String hash) { - return UUID.nameUUIDFromBytes(hash.getBytes(StandardCharsets.UTF_8)).toString(); - } - @NonNull private static String detectMimeType( @NonNull File file, EditorAttachment attachment, @NonNull String displayName) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index e34f6d6a..7cef530a 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -174,6 +174,7 @@ private static JsonArray tombstones(@NonNull SyncSnapshot snapshot) { @NonNull private static JsonArray collectAttachments(@NonNull SyncSnapshot snapshot) throws IOException { JsonArray attachments = new JsonArray(); + Map seenById = new LinkedHashMap<>(); Map seenByHash = new LinkedHashMap<>(); for (SyncRecord record : snapshot.getLiveRecords(SyncRecord.Type.NOTE)) { JsonArray manifestEntries = record.getPayload().getAsJsonArray("attachmentsManifest"); @@ -181,8 +182,10 @@ private static JsonArray collectAttachments(@NonNull SyncSnapshot snapshot) thro for (JsonElement element : manifestEntries) { AttachmentManifestEntry attachment = AttachmentManifestEntry.fromJson(element.getAsJsonObject()); - AttachmentManifestEntry previous = - seenByHash.putIfAbsent(attachment.sha256, attachment); + if (seenById.putIfAbsent(attachment.id, attachment) != null) { + throw new IOException("Two notes reference conflicting attachment metadata"); + } + AttachmentManifestEntry previous = seenByHash.putIfAbsent(attachment.sha256, attachment); if (previous != null && !previous.sameRemoteFile(attachment)) { throw new IOException("Two notes reference conflicting attachment metadata"); } @@ -191,7 +194,7 @@ private static JsonArray collectAttachments(@NonNull SyncSnapshot snapshot) thro if (seenByHash.size() > SyncBundleValidator.MAX_ATTACHMENT_COUNT) { throw new IOException("Sync bundle exceeds the schema-1 attachment limit"); } - for (AttachmentManifestEntry attachment : seenByHash.values()) { + for (AttachmentManifestEntry attachment : seenById.values()) { attachments.add(attachment.toJson(false)); } return attachments; @@ -398,7 +401,6 @@ JsonObject toJson(boolean includeDisplayName) { boolean sameRemoteFile(@NonNull AttachmentManifestEntry other) { return sha256.equals(other.sha256) && path.equals(other.path) - && mimeType.equals(other.mimeType) && size == other.size; } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java index 6b554176..ced3a7ee 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java @@ -53,7 +53,6 @@ public ValidatedBundle validate(@NonNull InputStream input) throws IOException { new LinkedHashMap<>(); Map attachmentsByHash = new LinkedHashMap<>(); - Set attachmentPaths = new LinkedHashSet<>(); long totalAttachmentBytes = 0L; JsonArray attachments = manifest.getAsJsonArray("attachments"); for (JsonElement element : attachments) { @@ -62,11 +61,10 @@ public ValidatedBundle validate(@NonNull InputStream input) throws IOException { if (attachmentsById.put(attachment.id, attachment) != null) { throw new IOException("Sync bundle contains duplicate attachment IDs"); } - if (attachmentsByHash.put(attachment.sha256, attachment) != null) { - throw new IOException("Sync bundle contains duplicate attachment hashes"); - } - if (!attachmentPaths.add(attachment.path)) { - throw new IOException("Sync bundle contains duplicate attachment paths"); + SyncBundleCodec.AttachmentManifestEntry previous = + attachmentsByHash.putIfAbsent(attachment.sha256, attachment); + if (previous != null && !previous.sameRemoteFile(attachment)) { + throw new IOException("Sync bundle contains conflicting attachment blob metadata"); } if (attachment.size > MAX_ATTACHMENT_BYTES) { throw new IOException("Sync bundle contains an oversized attachment"); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java index 28787b1b..446f3a9e 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java @@ -108,7 +108,14 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { SyncSnapshot merged = mergeResult.getMergedSnapshot(); Map expectedSizes = attachmentSizes(merged); + // The merged snapshot contains only the deterministic winner. A conflict row is not + // durable unless the loser can later be restored as well, so preflight and pin each + // version independently; SyncSnapshot deliberately forbids two versions of one ID. synchronizeAttachments(backend, merged, expectedSizes); + for (SyncMergeResult.Conflict conflict : mergeResult.getConflicts()) { + pinConflictVersion(backend, conflict.getWinner()); + pinConflictVersion(backend, conflict.getLoser()); + } if (!snapshotsMatch(merged, remote)) { backend.writeSnapshot(merged); } @@ -221,6 +228,24 @@ private void synchronizeAttachments( } } + /** Pins required conflict blobs into the store's durable content-addressed cache. */ + private void pinConflictVersion(@NonNull SyncBackend backend, @NonNull SyncRecord record) + throws IOException { + if (record.isTombstone()) return; + SyncSnapshot snapshot = new SyncSnapshot(java.util.Collections.singletonList(record)); + Map expectedSizes = attachmentSizes(snapshot); + synchronizeAttachments(backend, snapshot, expectedSizes); + for (String hash : store.getAttachmentHashes(snapshot)) { + Long expectedSize = expectedSizes.get(hash); + copyVerified( + hash, + expectedSize, + store.readAttachment(hash), + store::writeAttachment); + } + } + + private void verifyAttachment(String hash, Long expectedSize, InputStream source) throws IOException { if (source == null) { @@ -335,7 +360,14 @@ private static final class VerifyingInputStream extends FilterInputStream { private final String expectedHash; private final Long expectedSize; private long byteCount; - private boolean verified; + private VerificationState verificationState = VerificationState.UNVERIFIED; + private AttachmentIntegrityException integrityFailure; + + private enum VerificationState { + UNVERIFIED, + VERIFIED, + FAILED + } VerifyingInputStream(InputStream input, String expectedHash, Long expectedSize) { super(input); @@ -350,6 +382,7 @@ private static final class VerifyingInputStream extends FilterInputStream { @Override public int read() throws IOException { + rethrowIntegrityFailure(); int value = super.read(); if (value >= 0) { digest.update((byte) value); @@ -363,6 +396,7 @@ public int read() throws IOException { @Override public int read(byte[] buffer, int offset, int length) throws IOException { + rethrowIntegrityFailure(); int read = super.read(buffer, offset, length); if (read > 0) { digest.update(buffer, offset, read); @@ -376,7 +410,7 @@ public int read(byte[] buffer, int offset, int length) throws IOException { private void enforceSizeLimit() throws IOException { if (byteCount > SyncBundleValidator.MAX_ATTACHMENT_BYTES) { - throw new IOException("Attachment exceeds the sync size limit"); + failIntegrity("Attachment exceeds the sync size limit"); } } @@ -387,19 +421,37 @@ private void enforceSizeLimit() throws IOException { * its final location still learns about a mismatch before it commits. */ void verifyEndOfStream() throws IOException { - if (verified) { + if (verificationState == VerificationState.VERIFIED) { return; } - // Set before draining: drainRemaining reads through super, but a caller reaching this - // from read() must not be able to re-enter. - verified = true; - drainRemaining(); - String actualHash = toHex(digest.digest()); - if (!expectedHash.equals(actualHash)) { - throw new IOException("Attachment checksum does not match its declared hash"); + rethrowIntegrityFailure(); + try { + drainRemaining(); + String actualHash = toHex(digest.digest()); + if (!expectedHash.equals(actualHash)) { + failIntegrity("Attachment checksum does not match its declared hash"); + } + if (expectedSize != null && expectedSize.longValue() != byteCount) { + failIntegrity("Attachment size does not match its declared size"); + } + verificationState = VerificationState.VERIFIED; + } catch (AttachmentIntegrityException failure) { + integrityFailure = failure; + verificationState = VerificationState.FAILED; + throw failure; } - if (expectedSize != null && expectedSize.longValue() != byteCount) { - throw new IOException("Attachment size does not match its declared size"); + } + + private void failIntegrity(String message) throws AttachmentIntegrityException { + AttachmentIntegrityException failure = new AttachmentIntegrityException(message); + integrityFailure = failure; + verificationState = VerificationState.FAILED; + throw failure; + } + + private void rethrowIntegrityFailure() throws AttachmentIntegrityException { + if (verificationState == VerificationState.FAILED && integrityFailure != null) { + throw integrityFailure; } } diff --git a/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java b/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java index 16f6ac0f..de6a6792 100644 --- a/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java +++ b/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java @@ -73,7 +73,8 @@ AppDatabase providesAppDatabase(@ApplicationContext Context context) { AppDatabase.MIGRATION_13_14, AppDatabase.MIGRATION_14_15, AppDatabase.MIGRATION_15_16, - AppDatabase.MIGRATION_16_17) + AppDatabase.MIGRATION_16_17, + AppDatabase.MIGRATION_17_18) .build(); } diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/models/EditorAttachment.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/models/EditorAttachment.java index 728d0ffb..2fe6074e 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/models/EditorAttachment.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/models/EditorAttachment.java @@ -5,6 +5,8 @@ import org.json.JSONObject; public class EditorAttachment { + /** Immutable logical attachment identity; SHA-256 identifies only the shared blob bytes. */ + public String id; public String url; public String name; public String extension; diff --git a/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java b/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java index 89f964ee..5658c202 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java +++ b/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java @@ -3,5 +3,5 @@ public class DatabaseConstants { public static final String DB_NAME = "MyNotes.db"; - public static final int DB_VERSION = 17; + public static final int DB_VERSION = 18; } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index fceb9586..2547ef3e 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -109,6 +109,20 @@ public void writeAttachment_resumesAcrossMultipleDriveChunksWithoutBufferingTheF assertThat(server.readAttachment(hash)).isEqualTo(bytes); } + @Test + public void writeAttachment_doesNotTrustCorruptObjectTaggedWithExpectedHash() throws Exception { + byte[] expected = "verified attachment".getBytes(StandardCharsets.UTF_8); + String hash = sha256(expected); + server.seedCorruptAttachment(hash, "wrong bytes".getBytes(StandardCharsets.UTF_8)); + + backend().writeAttachment(hash, expected.length, new ByteArrayInputStream(expected)); + + assertThat(server.ownedAttachmentCount(hash)).isEqualTo(2); + try (java.io.InputStream restored = backend().readAttachment(hash)) { + assertThat(readAll(restored)).isEqualTo(expected); + } + } + @Test public void concurrentFirstSync_createsDuplicateRootsThenConvergesWithoutLosingEitherNote() throws Exception { @@ -222,8 +236,7 @@ public void writeSnapshot_copiesDuplicateRootAttachmentsIntoTheCanonicalRoot() @Test public void writeSnapshot_createsNewBundleWhenLegacyBundleChanges() throws Exception { - String hash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; - server.seedOwnedBundle(snapshot(hash)); + server.seedOwnedBundle(snapshot(NOTE_ID, null)); GoogleDriveSyncBackend backend = new GoogleDriveSyncBackend( "token", @@ -236,16 +249,14 @@ public void writeSnapshot_createsNewBundleWhenLegacyBundleChanges() throws Excep server.forceConcurrentBundleUpdate( snapshot("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc")); - backend.writeSnapshot(snapshot(hash)); + backend.writeSnapshot(snapshot(NOTE_ID, null)); assertThat(server.bundleCount()).isEqualTo(2); } @Test public void writeSnapshot_preservesUpdateThatArrivesBetweenReadAndPublish() throws Exception { - String firstHash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; - String secondHash = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; - server.seedOwnedBundle(snapshot(NOTE_ID, firstHash)); + server.seedOwnedBundle(snapshot(NOTE_ID, null)); GoogleDriveSyncBackend backend = new GoogleDriveSyncBackend( "token", @@ -255,8 +266,8 @@ public void writeSnapshot_preservesUpdateThatArrivesBetweenReadAndPublish() thro new SyncBundleCodec()); backend.readSnapshot(); - server.updateBundleImmediatelyBeforeNextUpload(snapshot(SECOND_NOTE_ID, secondHash)); - backend.writeSnapshot(snapshot(NOTE_ID, firstHash)); + server.updateBundleImmediatelyBeforeNextUpload(snapshot(SECOND_NOTE_ID, null)); + backend.writeSnapshot(snapshot(NOTE_ID, null)); SyncSnapshot remote = new GoogleDriveSyncBackend( @@ -313,6 +324,16 @@ private static String sha256(byte[] bytes) throws Exception { return value.toString(); } + private static byte[] readAll(java.io.InputStream input) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + output.write(buffer, 0, read); + } + return output.toByteArray(); + } + private static final class FakeDriveServer implements AutoCloseable { private static final Pattern PARENT_PATTERN = Pattern.compile("'([^']+)' in parents"); private static final Pattern APP_PROPERTY_PATTERN = @@ -443,6 +464,15 @@ String registerAttachment(byte[] bytes) throws Exception { return hash; } + void seedCorruptAttachment(String claimedHash, byte[] bytes) { + DriveFile folder = + createFile("MyNotes Sync", "application/vnd.google-apps.folder", null); + folder.appProperties.put("mynotesOwner", "1"); + DriveFile blob = createFile(claimedHash, "application/octet-stream", folder.id); + blob.appProperties.put("mynotesAttachmentSha256", claimedHash); + blob.content = bytes; + } + void seedOwnedBundle(SyncSnapshot snapshot) throws IOException { DriveFile folder = createFile("MyNotes Sync", "application/vnd.google-apps.folder", null); diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java index e3903718..bdcbbbb7 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java @@ -98,6 +98,48 @@ public void encode_rejectsConflictingAttachmentMetadataForSameHash() { throw new AssertionError("Expected an IOException"); } + @Test + public void encode_preservesTwoLogicalAttachmentsThatShareOneBlob() throws Exception { + SyncBundleCodec codec = new SyncBundleCodec(); + JsonObject first = notePayload("One", "image/png", 42L, "first.png"); + JsonObject second = notePayload("Two", "image/png", 42L, "second.png"); + second.getAsJsonArray("attachmentsManifest") + .get(0) + .getAsJsonObject() + .addProperty("id", "550e8400-e29b-41d4-a716-446655440099"); + SyncSnapshot decoded = + codec.decode( + new ByteArrayInputStream( + codec.encode( + new SyncSnapshot( + Arrays.asList( + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + CREATED_AT, + first), + SyncRecord.live( + SyncRecord.Type.NOTE, + "6ba7b812-9dad-11d1-80b4-00c04fd430c8", + CREATED_AT, + second))), + CREATED_AT))) + .getSnapshot(); + + assertThat( + decoded.find(SyncRecord.Type.NOTE, NOTE_ID) + .getPayload() + .getAsJsonArray("attachmentsManifest")) + .hasSize(1); + assertThat( + decoded.find( + SyncRecord.Type.NOTE, + "6ba7b812-9dad-11d1-80b4-00c04fd430c8") + .getPayload() + .getAsJsonArray("attachmentsManifest")) + .hasSize(1); + } + @Test public void decode_keysAttachmentNamesByHashSoTheStoreCanResolveThem() throws Exception { SyncBundleCodec codec = new SyncBundleCodec(); diff --git a/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java index 31ffec10..fb24e1b0 100644 --- a/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java @@ -311,6 +311,7 @@ public void resolveConflict_updatesStoreAndReturnsLatestConflicts() { new SyncConflictEntity( "note", "550e8400-e29b-41d4-a716-446655440000", + "test-version-pair", "LOCAL", "{}", "{}", From 4d905f8c83fb20d0573d87f2dd5c13e8d3814075 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 14:25:00 +0300 Subject: [PATCH 04/16] fix: journal preferences and track Drive heads --- .../19.json | 525 ++++++++++++++++++ .../mynotes/data/database/AppDatabase.java | 17 + .../dao/SyncPendingPreferencesDao.java | 20 + .../SyncPendingPreferencesEntity.java | 18 + .../data/sync/GoogleDriveSyncBackend.java | 88 ++- .../mynotes/data/sync/RemoteSnapshot.java | 31 ++ .../mynotes/data/sync/RoomSyncStore.java | 68 ++- .../pasich/mynotes/data/sync/SyncBackend.java | 9 + .../mynotes/data/sync/SyncBundleCodec.java | 43 +- .../data/sync/SyncBundleValidator.java | 18 + .../pasich/mynotes/data/sync/SyncService.java | 14 +- .../pasich/mynotes/di/ApplicationModule.java | 3 +- .../utils/constants/DatabaseConstants.java | 2 +- .../data/sync/GoogleDriveSyncBackendTest.java | 77 ++- 14 files changed, 909 insertions(+), 24 deletions(-) create mode 100644 app/schemas/com.pasich.mynotes.data.database.AppDatabase/19.json create mode 100644 app/src/main/java/com/pasich/mynotes/data/database/dao/SyncPendingPreferencesDao.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/RemoteSnapshot.java diff --git a/app/schemas/com.pasich.mynotes.data.database.AppDatabase/19.json b/app/schemas/com.pasich.mynotes.data.database.AppDatabase/19.json new file mode 100644 index 00000000..4c297067 --- /dev/null +++ b/app/schemas/com.pasich.mynotes.data.database.AppDatabase/19.json @@ -0,0 +1,525 @@ +{ + "formatVersion": 1, + "database": { + "version": 19, + "identityHash": "3e5a70b1d7e0a0d9730739cd71700469", + "entities": [ + { + "tableName": "tags", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `visibility` INTEGER NOT NULL, `systemAction` INTEGER NOT NULL, `position` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "nameTag", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "visibility", + "columnName": "visibility", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "systemAction", + "columnName": "systemAction", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "notes", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `value` TEXT, `date` INTEGER NOT NULL, `tag` TEXT, `valueJson` TEXT, `hasRichContent` INTEGER NOT NULL, `attachments` TEXT, `isTrash` INTEGER NOT NULL, `reminderTime` INTEGER, `isPinned` INTEGER NOT NULL, `reminderRepeat` TEXT NOT NULL, `reminderIntervalMinutes` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT" + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT" + }, + { + "fieldPath": "date", + "columnName": "date", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "tag", + "columnName": "tag", + "affinity": "TEXT" + }, + { + "fieldPath": "valueJson", + "columnName": "valueJson", + "affinity": "TEXT" + }, + { + "fieldPath": "hasRichContent", + "columnName": "hasRichContent", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "attachments", + "columnName": "attachments", + "affinity": "TEXT" + }, + { + "fieldPath": "isTrash", + "columnName": "isTrash", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reminderTime", + "columnName": "reminderTime", + "affinity": "INTEGER" + }, + { + "fieldPath": "isPinned", + "columnName": "isPinned", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reminderRepeat", + "columnName": "reminderRepeat", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "reminderIntervalMinutes", + "columnName": "reminderIntervalMinutes", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "tasks", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT NOT NULL, `description` TEXT, `isDone` INTEGER NOT NULL DEFAULT 0, `categoryId` INTEGER NOT NULL DEFAULT 0, `createdAt` INTEGER NOT NULL DEFAULT 0, `position` INTEGER NOT NULL DEFAULT 0, `reminderTime` INTEGER, `reminderIntervalMinutes` INTEGER NOT NULL DEFAULT 0)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT" + }, + { + "fieldPath": "isDone", + "columnName": "isDone", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "categoryId", + "columnName": "categoryId", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "reminderTime", + "columnName": "reminderTime", + "affinity": "INTEGER" + }, + { + "fieldPath": "reminderIntervalMinutes", + "columnName": "reminderIntervalMinutes", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "task_categories", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `colorHex` TEXT NOT NULL DEFAULT '#6750A4', `position` INTEGER NOT NULL DEFAULT 0)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "colorHex", + "columnName": "colorHex", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'#6750A4'" + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "sync_metadata", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`recordType` TEXT NOT NULL, `localId` INTEGER NOT NULL, `stableId` TEXT NOT NULL, `updatedAt` INTEGER NOT NULL, `deletedAt` INTEGER, PRIMARY KEY(`recordType`, `localId`))", + "fields": [ + { + "fieldPath": "recordType", + "columnName": "recordType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "localId", + "columnName": "localId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "stableId", + "columnName": "stableId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "updatedAt", + "columnName": "updatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deletedAt", + "columnName": "deletedAt", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "recordType", + "localId" + ] + }, + "indices": [ + { + "name": "index_sync_metadata_recordType_stableId", + "unique": true, + "columnNames": [ + "recordType", + "stableId" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_sync_metadata_recordType_stableId` ON `${TABLE_NAME}` (`recordType`, `stableId`)" + }, + { + "name": "index_sync_metadata_updatedAt", + "unique": false, + "columnNames": [ + "updatedAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_metadata_updatedAt` ON `${TABLE_NAME}` (`updatedAt`)" + }, + { + "name": "index_sync_metadata_deletedAt", + "unique": false, + "columnNames": [ + "deletedAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_metadata_deletedAt` ON `${TABLE_NAME}` (`deletedAt`)" + } + ] + }, + { + "tableName": "sync_pending_preferences", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `payloadJson` TEXT NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "payloadJson", + "columnName": "payloadJson", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "sync_conflicts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `recordType` TEXT NOT NULL, `stableId` TEXT NOT NULL, `versionPairHash` TEXT NOT NULL, `winnerSource` TEXT NOT NULL, `winnerJson` TEXT NOT NULL, `loserJson` TEXT NOT NULL, `winnerUpdatedAt` INTEGER NOT NULL, `loserUpdatedAt` INTEGER NOT NULL, `winnerTombstone` INTEGER NOT NULL, `loserTombstone` INTEGER NOT NULL, `resolution` TEXT NOT NULL, `resolved` INTEGER NOT NULL, `createdAt` INTEGER NOT NULL, `resolvedAt` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "recordType", + "columnName": "recordType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stableId", + "columnName": "stableId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionPairHash", + "columnName": "versionPairHash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerSource", + "columnName": "winnerSource", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerJson", + "columnName": "winnerJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "loserJson", + "columnName": "loserJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerUpdatedAt", + "columnName": "winnerUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "loserUpdatedAt", + "columnName": "loserUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "winnerTombstone", + "columnName": "winnerTombstone", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "loserTombstone", + "columnName": "loserTombstone", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "resolution", + "columnName": "resolution", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "resolved", + "columnName": "resolved", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "resolvedAt", + "columnName": "resolvedAt", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_sync_conflicts_recordType_stableId_versionPairHash", + "unique": true, + "columnNames": [ + "recordType", + "stableId", + "versionPairHash" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_sync_conflicts_recordType_stableId_versionPairHash` ON `${TABLE_NAME}` (`recordType`, `stableId`, `versionPairHash`)" + }, + { + "name": "index_sync_conflicts_resolved", + "unique": false, + "columnNames": [ + "resolved" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_conflicts_resolved` ON `${TABLE_NAME}` (`resolved`)" + }, + { + "name": "index_sync_conflicts_createdAt", + "unique": false, + "columnNames": [ + "createdAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_conflicts_createdAt` ON `${TABLE_NAME}` (`createdAt`)" + } + ] + }, + { + "tableName": "sync_state", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `status` TEXT NOT NULL, `backendIdentifier` TEXT, `lastSuccessfulSyncAt` INTEGER, `attemptStartedAt` INTEGER, `errorMessage` TEXT, `conflictCount` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "backendIdentifier", + "columnName": "backendIdentifier", + "affinity": "TEXT" + }, + { + "fieldPath": "lastSuccessfulSyncAt", + "columnName": "lastSuccessfulSyncAt", + "affinity": "INTEGER" + }, + { + "fieldPath": "attemptStartedAt", + "columnName": "attemptStartedAt", + "affinity": "INTEGER" + }, + { + "fieldPath": "errorMessage", + "columnName": "errorMessage", + "affinity": "TEXT" + }, + { + "fieldPath": "conflictCount", + "columnName": "conflictCount", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '3e5a70b1d7e0a0d9730739cd71700469')" + ] + } +} \ No newline at end of file diff --git a/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java b/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java index 89fe3aaa..7470772e 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java @@ -10,6 +10,7 @@ import com.pasich.mynotes.data.database.dao.NoteDao; import com.pasich.mynotes.data.database.dao.SyncConflictDao; import com.pasich.mynotes.data.database.dao.SyncMetadataDao; +import com.pasich.mynotes.data.database.dao.SyncPendingPreferencesDao; import com.pasich.mynotes.data.database.dao.SyncStateDao; import com.pasich.mynotes.data.database.dao.TagsDao; import com.pasich.mynotes.data.database.dao.TaskCategoryDao; @@ -17,6 +18,7 @@ import com.pasich.mynotes.data.database.dao.Transactions; import com.pasich.mynotes.data.database.entities.SyncConflictEntity; import com.pasich.mynotes.data.database.entities.SyncMetadataEntity; +import com.pasich.mynotes.data.database.entities.SyncPendingPreferencesEntity; import com.pasich.mynotes.data.database.entities.SyncStateEntity; import com.pasich.mynotes.data.model.Note; import com.pasich.mynotes.data.model.Tag; @@ -36,6 +38,7 @@ Task.class, TaskCategory.class, SyncMetadataEntity.class, + SyncPendingPreferencesEntity.class, SyncConflictEntity.class, SyncStateEntity.class }, @@ -158,6 +161,18 @@ public void migrate(@NonNull SupportSQLiteDatabase database) { } }; + /** Adds the Room journal that bridges snapshot transactions to SharedPreferences. */ + public static final Migration MIGRATION_18_19 = + new Migration(18, 19) { + @Override + public void migrate(@NonNull SupportSQLiteDatabase database) { + database.execSQL( + "CREATE TABLE IF NOT EXISTS `sync_pending_preferences` (" + + "`id` INTEGER NOT NULL, `payloadJson` TEXT NOT NULL, " + + "PRIMARY KEY(`id`))"); + } + }; + private static void insertMetadataForExistingRecords( SupportSQLiteDatabase database, String recordType, @@ -371,4 +386,6 @@ public static void setContext(Context context) { public abstract SyncConflictDao syncConflictDao(); public abstract SyncStateDao syncStateDao(); + + public abstract SyncPendingPreferencesDao syncPendingPreferencesDao(); } diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncPendingPreferencesDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncPendingPreferencesDao.java new file mode 100644 index 00000000..f57ce0bf --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncPendingPreferencesDao.java @@ -0,0 +1,20 @@ +package com.pasich.mynotes.data.database.dao; + +import androidx.room.Dao; +import androidx.room.Insert; +import androidx.room.OnConflictStrategy; +import androidx.room.Query; +import com.pasich.mynotes.data.database.entities.SyncPendingPreferencesEntity; + +@Dao +public interface SyncPendingPreferencesDao { + + @Query("SELECT * FROM sync_pending_preferences WHERE id = 1 LIMIT 1") + SyncPendingPreferencesEntity get(); + + @Insert(onConflict = OnConflictStrategy.REPLACE) + void upsert(SyncPendingPreferencesEntity pending); + + @Query("DELETE FROM sync_pending_preferences WHERE id = 1") + void clear(); +} diff --git a/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java new file mode 100644 index 00000000..30c0a404 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java @@ -0,0 +1,18 @@ +package com.pasich.mynotes.data.database.entities; + +import androidx.annotation.NonNull; +import androidx.room.Entity; +import androidx.room.PrimaryKey; + +/** Room journal for a preference adapter mutation that must follow a snapshot transaction. */ +@Entity(tableName = "sync_pending_preferences") +public final class SyncPendingPreferencesEntity { + + @PrimaryKey public int id; + @NonNull public String payloadJson; + + public SyncPendingPreferencesEntity(int id, @NonNull String payloadJson) { + this.id = id; + this.payloadJson = payloadJson; + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index 38993b6c..8354a045 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -19,11 +19,14 @@ import java.security.NoSuchAlgorithmException; import java.time.Clock; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.HashSet; import java.util.UUID; /** Google Drive REST backend for the provider-independent sync protocol. */ @@ -51,6 +54,7 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private final SyncBundleCodec bundleCodec; private final DriveRequestExecutor requestExecutor; private final SyncMerger merger = new SyncMerger(); + private List lastReadFrontierBundleIds = Collections.emptyList(); public GoogleDriveSyncBackend(@NonNull String accessToken) { this(accessToken, DEFAULT_API, DEFAULT_UPLOAD, Clock.systemUTC(), new SyncBundleCodec()); @@ -82,12 +86,19 @@ public String getIdentifier() { @NonNull @Override public synchronized SyncSnapshot readSnapshot() throws IOException { + return readSnapshotResult().getSnapshot(); + } + + @Override + public synchronized RemoteSnapshot readSnapshotResult() throws IOException { List folderIds = findFolderIds(); if (folderIds.isEmpty()) { - return SyncSnapshot.empty(); + lastReadFrontierBundleIds = Collections.emptyList(); + return RemoteSnapshot.of(SyncSnapshot.empty()); } - SyncSnapshot merged = SyncSnapshot.empty(); + Map bundlesByLogicalId = new HashMap<>(); + Map bytesByLogicalId = new HashMap<>(); for (String folderId : folderIds) { for (String bundleId : findBundles(folderId)) { byte[] bytes = @@ -95,12 +106,29 @@ public synchronized SyncSnapshot readSnapshot() throws IOException { "GET", apiBase + "/files/" + bundleId + "?alt=media", MAX_BUNDLE_RESPONSE_BYTES); - SyncSnapshot decoded = - bundleCodec.decode(new ByteArrayInputStream(bytes)).getSnapshot(); - merged = merger.merge(merged, decoded).getMergedSnapshot(); + SyncBundleCodec.DecodedBundle decoded = + bundleCodec.decode(new ByteArrayInputStream(bytes)); + byte[] previousBytes = bytesByLogicalId.putIfAbsent(decoded.getBundleId(), bytes); + if (previousBytes != null) { + if (!java.util.Arrays.equals(previousBytes, bytes)) { + throw new IOException("Drive contains conflicting physical copies of one bundle"); + } + continue; + } + bundlesByLogicalId.put(decoded.getBundleId(), decoded); } } - return merged; + validateBundleDag(bundlesByLogicalId); + List frontier = computeFrontier(bundlesByLogicalId); + SyncSnapshot merged = SyncSnapshot.empty(); + List conflicts = new ArrayList<>(); + for (String bundleId : frontier) { + SyncMergeResult result = merger.merge(merged, bundlesByLogicalId.get(bundleId).getSnapshot()); + merged = result.getMergedSnapshot(); + conflicts.addAll(result.getConflicts()); + } + lastReadFrontierBundleIds = Collections.unmodifiableList(new ArrayList<>(frontier)); + return new RemoteSnapshot(merged, conflicts, frontier); } @Override @@ -111,7 +139,7 @@ public synchronized void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IO // referenced blob in the canonical root as well, so no future cleanup decision can make // the canonical bundle point at an object that exists only in a duplicate root. ensureCanonicalAttachments(folderId, snapshot); - byte[] bundle = bundleCodec.encode(snapshot, clock.instant()); + byte[] bundle = bundleCodec.encode(snapshot, clock.instant(), lastReadFrontierBundleIds); // Every bundle is immutable. Drive offers no conditional update based on its version // counter, so replacing one file leaves a race where another device can be overwritten. // Publishing a distinct file makes each successful upload independently durable; readers @@ -292,6 +320,52 @@ private static Map attachmentSizes(@NonNull SyncSnapshot snapshot) return sizes; } + private static void validateBundleDag( + @NonNull Map bundles) throws IOException { + for (SyncBundleCodec.DecodedBundle bundle : bundles.values()) { + for (String parent : bundle.getParentBundleIds()) { + if (!bundles.containsKey(parent)) { + throw new IOException("Drive bundle references an unavailable ancestor"); + } + } + } + Set visiting = new HashSet<>(); + Set visited = new HashSet<>(); + for (String bundleId : bundles.keySet()) { + validateAcyclic(bundleId, bundles, visiting, visited); + } + } + + private static void validateAcyclic( + @NonNull String bundleId, + @NonNull Map bundles, + @NonNull Set visiting, + @NonNull Set visited) + throws IOException { + if (visited.contains(bundleId)) return; + if (!visiting.add(bundleId)) throw new IOException("Drive bundle ancestry contains a cycle"); + for (String parent : bundles.get(bundleId).getParentBundleIds()) { + validateAcyclic(parent, bundles, visiting, visited); + } + visiting.remove(bundleId); + visited.add(bundleId); + } + + @NonNull + private static List computeFrontier( + @NonNull Map bundles) { + Set ancestors = new HashSet<>(); + for (SyncBundleCodec.DecodedBundle bundle : bundles.values()) { + ancestors.addAll(bundle.getParentBundleIds()); + } + List frontier = new ArrayList<>(); + for (String bundleId : bundles.keySet()) { + if (!ancestors.contains(bundleId)) frontier.add(bundleId); + } + frontier.sort(Comparator.naturalOrder()); + return frontier; + } + @NonNull private List findBundles(@NonNull String folderId) throws IOException { JsonArray bundles = diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RemoteSnapshot.java b/app/src/main/java/com/pasich/mynotes/data/sync/RemoteSnapshot.java new file mode 100644 index 00000000..bceca2da --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RemoteSnapshot.java @@ -0,0 +1,31 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Immutable result of reading a remote causal frontier. */ +public final class RemoteSnapshot { + private final SyncSnapshot snapshot; + private final List conflicts; + private final List frontierBundleIds; + + public RemoteSnapshot( + @NonNull SyncSnapshot snapshot, + @NonNull List conflicts, + @NonNull List frontierBundleIds) { + this.snapshot = snapshot; + this.conflicts = Collections.unmodifiableList(new ArrayList<>(conflicts)); + this.frontierBundleIds = Collections.unmodifiableList(new ArrayList<>(frontierBundleIds)); + } + + @NonNull + public static RemoteSnapshot of(@NonNull SyncSnapshot snapshot) { + return new RemoteSnapshot(snapshot, Collections.emptyList(), Collections.emptyList()); + } + + @NonNull public SyncSnapshot getSnapshot() { return snapshot; } + @NonNull public List getConflicts() { return conflicts; } + @NonNull public List getFrontierBundleIds() { return frontierBundleIds; } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index aed34621..a1e81b98 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -13,6 +13,7 @@ import com.pasich.mynotes.data.database.AppDatabase; import com.pasich.mynotes.data.database.entities.SyncConflictEntity; import com.pasich.mynotes.data.database.entities.SyncMetadataEntity; +import com.pasich.mynotes.data.database.entities.SyncPendingPreferencesEntity; import com.pasich.mynotes.data.database.entities.SyncStateEntity; import com.pasich.mynotes.data.model.Note; import com.pasich.mynotes.data.model.Tag; @@ -119,7 +120,7 @@ public RoomSyncStore( * thread it happened to construct the store on; on the main thread Room throws. Seeding is now * deferred to the operations that already run in the background. */ - private void ensureSeeded() { + private void ensureSeeded() throws IOException { if (seeded) { return; } @@ -131,6 +132,7 @@ private void ensureSeeded() { "00000000-0000-4000-8000-000000000000", 0L, null)); + recoverPendingPreferences(); seeded = true; } @@ -202,6 +204,10 @@ private void applySnapshotInternal( @NonNull List conflicts, @Nullable SyncState finalState) throws IOException { + PreferencesBackup stagedPreferences = selectedPreferences(snapshot); + String stagedPreferencesJson = + stagedPreferences == null ? null : gson.toJson(stagedPreferences); + boolean deferFinalState = stagedPreferences != null && finalState != null; try { database.runInTransaction( () -> { @@ -251,7 +257,11 @@ private void applySnapshotInternal( transactionFailureInjector.afterRecordApplied(record); } persistConflicts(conflicts); - if (finalState != null) { + if (stagedPreferencesJson != null) { + database.syncPendingPreferencesDao() + .upsert(new SyncPendingPreferencesEntity(1, stagedPreferencesJson)); + } + if (finalState != null && !deferFinalState) { database.syncStateDao().upsert(toEntity(finalState)); } } catch (IOException error) { @@ -261,6 +271,55 @@ private void applySnapshotInternal( } catch (SyncRuntimeException error) { throw error.ioException; } + if (stagedPreferences != null) { + commitPendingPreferences(stagedPreferences); + database.runInTransaction( + () -> { + database.syncPendingPreferencesDao().clear(); + if (finalState != null) database.syncStateDao().upsert(toEntity(finalState)); + }); + } + } + + @Nullable + private PreferencesBackup selectedPreferences(@NonNull SyncSnapshot snapshot) throws IOException { + SyncRecord record = + snapshot.find( + SyncRecord.Type.PREFERENCES, + "00000000-0000-4000-8000-000000000000"); + if (record == null || record.isTombstone()) return null; + try { + PreferencesBackup parsed = gson.fromJson(record.getPayload(), PreferencesBackup.class); + if (parsed == null || !parsed.isCreated()) { + throw new IOException("Sync preferences payload is invalid"); + } + return parsed; + } catch (RuntimeException error) { + throw new IOException("Sync preferences payload is invalid", error); + } + } + + /** Completes a previously committed Room journal after process death or adapter failure. */ + private void recoverPendingPreferences() throws IOException { + SyncPendingPreferencesEntity pending = database.syncPendingPreferencesDao().get(); + if (pending == null) return; + PreferencesBackup backup; + try { + backup = gson.fromJson(pending.payloadJson, PreferencesBackup.class); + if (backup == null || !backup.isCreated()) throw new IOException("Pending preferences are invalid"); + } catch (RuntimeException error) { + throw new IOException("Pending preferences are invalid", error); + } + commitPendingPreferences(backup); + database.runInTransaction(() -> database.syncPendingPreferencesDao().clear()); + } + + private void commitPendingPreferences(@NonNull PreferencesBackup backup) throws IOException { + try { + preferenceHelper.setListPreferences(backup); + } catch (RuntimeException error) { + throw new IOException("Could not commit synchronized preferences", error); + } } @Nullable @@ -334,7 +393,8 @@ private void applyPayload(SyncMetadataEntity metadata, JsonObject payload) throw tag.id = metadata.localId; database.tagsDao().updateTag(tag); } else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { - preferenceHelper.setListPreferences(gson.fromJson(payload, PreferencesBackup.class)); + // SharedPreferences is outside Room. applySnapshotInternal journals and commits this + // payload only after the Room transaction succeeds. } } @@ -529,8 +589,6 @@ private static String conflictVersionPairHash(@NonNull SyncMergeResult.Conflict + "\n" + conflict.getId() + "\n" - + conflict.getWinnerSource().name() - + "\n" + conflict.getWinner().canonicalSerializedPayload() + "\n" + conflict.getLoser().canonicalSerializedPayload(); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java index 0d389e8d..e2cfb501 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java @@ -28,6 +28,15 @@ public interface SyncBackend { @NonNull SyncSnapshot readSnapshot() throws IOException; + /** + * Reads the remote causal frontier. Legacy adapters expose one snapshot and no remote + * conflicts; Drive overrides this so concurrent immutable bundle heads remain recoverable. + */ + @NonNull + default RemoteSnapshot readSnapshotResult() throws IOException { + return RemoteSnapshot.of(readSnapshot()); + } + /** Publishes a complete remote snapshot. Implementations must not expose a partial snapshot. */ void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IOException; diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 7cef530a..815d3acf 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -13,6 +13,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -43,6 +44,15 @@ public final class SyncBundleCodec { @NonNull public byte[] encode(@NonNull SyncSnapshot snapshot, @NonNull Instant createdAt) throws IOException { + return encode(snapshot, createdAt, Collections.emptyList()); + } + + @NonNull + public byte[] encode( + @NonNull SyncSnapshot snapshot, + @NonNull Instant createdAt, + @NonNull Collection parentBundleIds) + throws IOException { JsonObject recordsRoot = new JsonObject(); recordsRoot.add("notes", liveArray(snapshot, SyncRecord.Type.NOTE)); recordsRoot.add("tasks", liveArray(snapshot, SyncRecord.Type.TASK)); @@ -61,6 +71,16 @@ public byte[] encode(@NonNull SyncSnapshot snapshot, @NonNull Instant createdAt) manifest.addProperty("format", BUNDLE_FORMAT); manifest.addProperty("schemaVersion", SCHEMA_VERSION); manifest.addProperty("bundleId", UUID.randomUUID().toString()); + JsonArray parents = new JsonArray(); + LinkedHashSet uniqueParents = new LinkedHashSet<>(parentBundleIds); + if (uniqueParents.size() > SyncBundleValidator.MAX_PARENT_BUNDLE_COUNT) { + throw new IOException("Sync bundle exceeds the parent frontier limit"); + } + for (String parent : uniqueParents) { + SyncBundleValidator.validateUuid(parent); + parents.add(parent); + } + manifest.add("parentBundleIds", parents); manifest.addProperty("createdAt", createdAt.toString()); manifest.addProperty("recordsSha256", sha256(recordBytes)); manifest.addProperty("recordsBytes", recordBytes.length); @@ -105,7 +125,17 @@ public DecodedBundle decode(@NonNull InputStream input) throws IOException { identities, result); parseTombstones(records, identities, result); - return new DecodedBundle(new SyncSnapshot(result), validated.getAttachmentsByHash()); + JsonObject manifest = validated.getManifest(); + JsonArray parents = manifest.getAsJsonArray("parentBundleIds"); + List parentBundleIds = new ArrayList<>(); + if (parents != null) { + for (JsonElement parent : parents) parentBundleIds.add(parent.getAsString()); + } + return new DecodedBundle( + new SyncSnapshot(result), + validated.getAttachmentsByHash(), + manifest.get("bundleId").getAsString(), + parentBundleIds); } private static void writeEntry(ZipOutputStream zip, String name, byte[] bytes) @@ -301,12 +331,18 @@ private static String sha256(byte[] bytes) throws IOException { public static final class DecodedBundle { private final SyncSnapshot snapshot; private final Map attachmentsByHash; + private final String bundleId; + private final List parentBundleIds; DecodedBundle( @NonNull SyncSnapshot snapshot, - @NonNull Map attachmentsByHash) { + @NonNull Map attachmentsByHash, + @NonNull String bundleId, + @NonNull List parentBundleIds) { this.snapshot = snapshot; this.attachmentsByHash = attachmentsByHash; + this.bundleId = bundleId; + this.parentBundleIds = Collections.unmodifiableList(new ArrayList<>(parentBundleIds)); } @NonNull @@ -318,6 +354,9 @@ public SyncSnapshot getSnapshot() { public Map getAttachmentsByHash() { return attachmentsByHash; } + + @NonNull public String getBundleId() { return bundleId; } + @NonNull public List getParentBundleIds() { return parentBundleIds; } } public static final class AttachmentManifestEntry { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java index ced3a7ee..759d1ce4 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java @@ -32,6 +32,7 @@ public final class SyncBundleValidator { static final long MAX_RECORD_COUNT = 10_000L; static final long MAX_ATTACHMENT_COUNT = 10_000L; static final long MAX_ATTACHMENTS_PER_NOTE = 1_000L; + static final long MAX_PARENT_BUNDLE_COUNT = 1_000L; static final long MAX_ATTACHMENT_BYTES = 100L * 1024L * 1024L; static final long MAX_TOTAL_ATTACHMENT_BYTES = 500L * 1024L * 1024L; static final long MAX_TOTAL_UNCOMPRESSED_BYTES = 1024L * 1024L * 1024L; @@ -223,6 +224,23 @@ private static void validateManifest(@NonNull JsonObject manifest, @NonNull byte throw new IOException("Unsupported sync bundle schema version"); } validateUuid(requireString(manifest, "bundleId")); + JsonArray parents = manifest.getAsJsonArray("parentBundleIds"); + if (parents != null) { + if (parents.size() > MAX_PARENT_BUNDLE_COUNT) { + throw new IOException("Sync bundle exceeds the parent frontier limit"); + } + Set uniqueParents = new LinkedHashSet<>(); + for (JsonElement parent : parents) { + if (parent == null || !parent.isJsonPrimitive()) { + throw new IOException("Sync bundle parent ID is invalid"); + } + String value = parent.getAsString(); + validateUuid(value); + if (!uniqueParents.add(value)) { + throw new IOException("Sync bundle contains duplicate parent IDs"); + } + } + } parseInstant(requireString(manifest, "createdAt"), "createdAt"); String recordsSha = requireString(manifest, "recordsSha256"); if (!SHA_256.matcher(recordsSha).matches()) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java index 446f3a9e..1f4befba 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java @@ -102,17 +102,22 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { // merely skipped an unresolved local attachment turns a local storage fault into // permanent remote data loss on the next successful sync from another device. SyncSnapshot local = localBuild.requireSnapshot(); - SyncSnapshot remote = Objects.requireNonNull(backend.readSnapshot(), "remote snapshot"); + RemoteSnapshot remoteResult = + Objects.requireNonNull(backend.readSnapshotResult(), "remote snapshot"); + SyncSnapshot remote = remoteResult.getSnapshot(); warnAboutClockSkew(remote); SyncMergeResult mergeResult = merger.merge(local, remote); SyncSnapshot merged = mergeResult.getMergedSnapshot(); + java.util.List allConflicts = + new java.util.ArrayList<>(remoteResult.getConflicts()); + allConflicts.addAll(mergeResult.getConflicts()); Map expectedSizes = attachmentSizes(merged); // The merged snapshot contains only the deterministic winner. A conflict row is not // durable unless the loser can later be restored as well, so preflight and pin each // version independently; SyncSnapshot deliberately forbids two versions of one ID. synchronizeAttachments(backend, merged, expectedSizes); - for (SyncMergeResult.Conflict conflict : mergeResult.getConflicts()) { + for (SyncMergeResult.Conflict conflict : allConflicts) { pinConflictVersion(backend, conflict.getWinner()); pinConflictVersion(backend, conflict.getLoser()); } @@ -120,9 +125,8 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { backend.writeSnapshot(merged); } SyncState success = - SyncState.success( - backendIdentifier, clock.instant(), mergeResult.getConflicts().size()); - store.applySnapshot(merged, mergeResult.getConflicts(), success); + SyncState.success(backendIdentifier, clock.instant(), allConflicts.size()); + store.applySnapshot(merged, allConflicts, success); return success; } catch (Exception exception) { SyncState failure = diff --git a/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java b/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java index de6a6792..253c91a8 100644 --- a/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java +++ b/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java @@ -74,7 +74,8 @@ AppDatabase providesAppDatabase(@ApplicationContext Context context) { AppDatabase.MIGRATION_14_15, AppDatabase.MIGRATION_15_16, AppDatabase.MIGRATION_16_17, - AppDatabase.MIGRATION_17_18) + AppDatabase.MIGRATION_17_18, + AppDatabase.MIGRATION_18_19) .build(); } diff --git a/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java b/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java index 5658c202..3607f8c5 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java +++ b/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java @@ -3,5 +3,5 @@ public class DatabaseConstants { public static final String DB_NAME = "MyNotes.db"; - public static final int DB_VERSION = 18; + public static final int DB_VERSION = 19; } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index 2547ef3e..862187d3 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -21,6 +21,7 @@ import java.time.Instant; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -151,6 +152,54 @@ public void concurrentFirstSync_createsDuplicateRootsThenConvergesWithoutLosingE assertThat(second.readSnapshot().find(SyncRecord.Type.NOTE, SECOND_NOTE_ID)).isNotNull(); } + @Test + public void readSnapshotResult_preservesConflictBetweenConcurrentCausalHeads() throws Exception { + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + byte[] first = + codec.encode(snapshotWithTitle("First offline edit"), CLOCK.instant(), Collections.singleton(baseId)); + byte[] second = + codec.encode(snapshotWithTitle("Second offline edit"), CLOCK.instant(), Collections.singleton(baseId)); + server.seedOwnedBundleBytes(base); + server.seedOwnedBundleBytes(first); + server.seedOwnedBundleBytes(second); + + RemoteSnapshot remote = backend().readSnapshotResult(); + + assertThat(remote.getFrontierBundleIds()).hasSize(2); + assertThat(remote.getConflicts()).hasSize(1); + assertThat(remote.getConflicts().get(0).getLoser().getPayload().get("title").getAsString()) + .isAnyOf("First offline edit", "Second offline edit"); + } + + @Test + public void readSnapshotResult_descendantSupersedesSiblingHeadsWithoutRepeatingConflict() + throws Exception { + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); + String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); + byte[] first = codec.encode(snapshotWithTitle("First"), CLOCK.instant(), Collections.singleton(baseId)); + String firstId = codec.decode(new ByteArrayInputStream(first)).getBundleId(); + byte[] second = codec.encode(snapshotWithTitle("Second"), CLOCK.instant(), Collections.singleton(baseId)); + String secondId = codec.decode(new ByteArrayInputStream(second)).getBundleId(); + byte[] descendant = + codec.encode( + snapshotWithTitle("Resolved"), CLOCK.instant(), Arrays.asList(firstId, secondId)); + server.seedOwnedBundleBytes(base); + server.seedOwnedBundleBytes(first); + server.seedOwnedBundleBytes(second); + server.seedOwnedBundleBytes(descendant); + + RemoteSnapshot remote = backend().readSnapshotResult(); + + assertThat(remote.getFrontierBundleIds()).containsExactly( + codec.decode(new ByteArrayInputStream(descendant)).getBundleId()); + assertThat(remote.getConflicts()).isEmpty(); + assertThat(remote.getSnapshot().find(SyncRecord.Type.NOTE, NOTE_ID).getPayload().get("title").getAsString()) + .isEqualTo("Resolved"); + } + private GoogleDriveSyncBackend backend() { return new GoogleDriveSyncBackend( "token", server.apiBase(), server.uploadBase(), CLOCK, new SyncBundleCodec()); @@ -251,7 +300,7 @@ public void writeSnapshot_createsNewBundleWhenLegacyBundleChanges() throws Excep backend.writeSnapshot(snapshot(NOTE_ID, null)); - assertThat(server.bundleCount()).isEqualTo(2); + assertThat(server.bundleCount()).isEqualTo(3); } @Test @@ -285,6 +334,15 @@ private static SyncSnapshot snapshot(String hash) throws IOException { return snapshot(NOTE_ID, hash); } + private static SyncSnapshot snapshotWithTitle(String title) { + JsonObject note = new JsonObject(); + note.addProperty("title", title); + note.addProperty("value", "body"); + return new SyncSnapshot( + Collections.singletonList( + SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, CLOCK.instant(), note))); + } + private static SyncSnapshot snapshot(String noteId, String hash) throws IOException { JsonObject note = new JsonObject(); note.addProperty("title", "Shopping"); @@ -497,6 +555,15 @@ void seedOwnedBundle(SyncSnapshot snapshot) throws IOException { } } + void seedOwnedBundleBytes(byte[] bytes) { + DriveFile folder = + createFile("MyNotes Sync", "application/vnd.google-apps.folder", null); + folder.appProperties.put("mynotesOwner", "1"); + DriveFile bundle = createFile("MyNotes.sync.v1.zip", "application/zip", folder.id); + bundle.appProperties.put("mynotesBundle", "1"); + bundle.content = bytes; + } + void seedUnownedBundle(SyncSnapshot snapshot) throws IOException { DriveFile folder = createFile("Elsewhere", "application/vnd.google-apps.folder", null); DriveFile bundle = createFile("MyNotes.sync.v1.zip", "application/zip", folder.id); @@ -506,8 +573,12 @@ void seedUnownedBundle(SyncSnapshot snapshot) throws IOException { void forceConcurrentBundleUpdate(SyncSnapshot snapshot) throws IOException { for (DriveFile file : files.values()) { if ("1".equals(file.appProperties.get("mynotesBundle"))) { - file.content = new SyncBundleCodec().encode(snapshot, CLOCK.instant()); - file.version++; + // Bundles are immutable. Model another device's publication as a sibling, + // never as replacement of a durable history object. + String parent = file.parents.isEmpty() ? null : file.parents.get(0); + DriveFile sibling = createFile("MyNotes.sync.v1.zip", "application/zip", parent); + sibling.appProperties.put("mynotesBundle", "1"); + sibling.content = new SyncBundleCodec().encode(snapshot, CLOCK.instant()); return; } } From c45915d860df79a04280951cc517b32161421555 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 15:48:57 +0300 Subject: [PATCH 05/16] fix: close stop-ship data-loss paths in Drive sync Attachments outside sync: - AttachmentStorage.resolve accepted only file:// URLs while the editor writes editorjs://, so every reference failed to resolve. AttachmentCleaner then read the empty expected set as "everything here is an orphan" and deleted every attachment of a note on the next save. Parsing moves to AttachmentUrl, a pure type covered by JVM tests, which accepts the canonical editorjs:// form and legacy file:// input; cleanup now aborts rather than deleting whenever a reference cannot be resolved. Sync restore writes canonical URLs, so restored attachments render in the editor. Resumable upload: - The chunk replay loop never terminated once a partially acknowledged chunk was completed by a retry, and replayed its buffer at offsets past the end of the file. Rewritten around absolute offsets, rejecting acknowledgements that move backwards, exceed the declared size, cover bytes never sent, or stop making progress; chunk PUTs retry through DriveRequestExecutor. - A zero-byte attachment issued no PUT and reported success without creating anything, which then failed every later sync. It takes an explicit zero-length multipart path. Preferences: - Resolving a preferences conflict applied nothing yet still marked the conflict resolved, so the rejected value won the next sync. It now runs through the pending-preferences journal and marks the conflict resolved only after a durable commit. - setListPreferences issued eleven independent apply() calls and the journal was cleared before they were durable. One editor plus commit(), and the journal is cleared only on success. The journal carries target and baseline digests so replay can tell "already applied" from "still pending" from "the user has since changed these settings"; an unreadable payload is quarantined instead of disabling sync permanently. Conflicts: - Unresolved losing versions existed only in one device's local table, so publishing a merged descendant made them unreachable. Bundles now carry unresolved alternatives and the resolutions that retire them, and a device starting from an empty database recovers both the winner and the alternative. - Provenance is recorded per side and resolution addresses versions (KEEP_WINNER / KEEP_ALTERNATIVE) rather than endpoints, so a conflict between two Drive heads is no longer shown or applied as "this device". - Publishing requires the read context it was derived from instead of taking causal parents from a mutable field. Drive reads: - A missing ancestor bundle is tolerated rather than fatal, since every bundle is a complete checkpoint of the state its descendants inherit. - An attachment is read and verified once per sync instead of two or three times. Also prunes the sync-attachment cache of blobs nothing references, drops the unused ACTION_VIEW filter from the non-exported TrashActivity so the release lint gate passes, and stops .gitignore un-ignoring the whole docs tree. Schema 20 adds journal identity and quarantine plus per-side conflict provenance and version identities. --- .gitignore | 7 +- .../20.json | 567 ++++++++++++++++ .../com/pasich/mynotes/db/MigrationTest.java | 71 ++ .../pasich/mynotes/db/RoomSyncStoreTest.java | 247 ++++++- app/src/main/AndroidManifest.xml | 14 +- .../pasich/mynotes/data/AppDataManager.java | 5 + .../mynotes/data/database/AppDatabase.java | 45 +- .../data/database/dao/SyncConflictDao.java | 7 + .../dao/SyncPendingPreferencesDao.java | 17 +- .../database/entities/SyncConflictEntity.java | 19 + .../SyncPendingPreferencesEntity.java | 36 +- .../preferences/AppPreferencesHelper.java | 89 +-- .../data/preferences/PreferenceHelper.java | 7 + .../data/preferences/SafePreferences.java | 31 + .../sync/AttachmentIntegrityException.java | 6 +- .../data/sync/GoogleDriveSyncBackend.java | 371 +++++++++-- .../data/sync/PendingPreferencesDecision.java | 64 ++ .../mynotes/data/sync/RemoteSnapshot.java | 68 +- .../mynotes/data/sync/RoomSyncStore.java | 522 ++++++++++++--- .../pasich/mynotes/data/sync/SyncBackend.java | 47 ++ .../mynotes/data/sync/SyncBundleCodec.java | 206 +++++- .../data/sync/SyncBundleValidator.java | 72 ++ .../mynotes/data/sync/SyncMergeResult.java | 31 +- .../pasich/mynotes/data/sync/SyncMerger.java | 42 +- .../mynotes/data/sync/SyncPublication.java | 57 ++ .../mynotes/data/sync/SyncResolution.java | 27 + .../pasich/mynotes/data/sync/SyncService.java | 103 ++- .../pasich/mynotes/data/sync/SyncStore.java | 11 + .../pasich/mynotes/di/ApplicationModule.java | 3 +- .../attach/AttachmentCleaner.java | 142 ++-- .../attach/AttachmentStorage.java | 44 +- .../extendedEditor/attach/AttachmentUrl.java | 260 ++++++++ .../models/EditorAttachment.java | 1 + .../ui/view/activity/BackupActivity.java | 43 +- .../utils/constants/DatabaseConstants.java | 2 +- app/src/main/res/values-be/strings.xml | 3 + app/src/main/res/values-de/strings.xml | 3 + app/src/main/res/values-en-rGB/strings.xml | 3 + app/src/main/res/values-es/strings.xml | 3 + app/src/main/res/values-fr/strings.xml | 3 + app/src/main/res/values-it/strings.xml | 3 + app/src/main/res/values-kk/strings.xml | 3 + app/src/main/res/values-pl/strings.xml | 3 + app/src/main/res/values-ru/strings.xml | 3 + app/src/main/res/values-uk/strings.xml | 3 + app/src/main/res/values/strings.xml | 3 + .../data/sync/ConflictProvenanceTest.java | 142 ++++ .../data/sync/GoogleDriveSyncBackendTest.java | 629 +++++++++++++++++- .../sync/PendingPreferencesDecisionTest.java | 79 +++ .../data/sync/SyncBundleCodecTest.java | 4 +- .../mynotes/data/sync/SyncServiceTest.java | 63 +- .../attach/AttachmentCleanerTest.java | 170 +++++ .../attach/AttachmentUrlTest.java | 133 ++++ .../mynotes/ui/sync/SyncCoordinatorTest.java | 9 + 54 files changed, 4121 insertions(+), 425 deletions(-) create mode 100644 app/schemas/com.pasich.mynotes.data.database.AppDatabase/20.json create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/PendingPreferencesDecision.java create mode 100644 app/src/main/java/com/pasich/mynotes/data/sync/SyncPublication.java create mode 100644 app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/ConflictProvenanceTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/data/sync/PendingPreferencesDecisionTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleanerTest.java create mode 100644 app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrlTest.java diff --git a/.gitignore b/.gitignore index ce75fe6b..dff2e3c6 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,8 @@ app/src/main/res/raw/* app/google-services.json gha-creds-*.json -# Local design/plan notes, deliberately kept out of the repository -docs/ -!docs/ +# Local design/plan notes, deliberately kept out of the repository. +# Ignore the contents rather than the directory itself, so a single tracked file can be +# re-included without exposing everything else under docs/. +docs/* !docs/google-drive-sync-invariants.md diff --git a/app/schemas/com.pasich.mynotes.data.database.AppDatabase/20.json b/app/schemas/com.pasich.mynotes.data.database.AppDatabase/20.json new file mode 100644 index 00000000..bde6a888 --- /dev/null +++ b/app/schemas/com.pasich.mynotes.data.database.AppDatabase/20.json @@ -0,0 +1,567 @@ +{ + "formatVersion": 1, + "database": { + "version": 20, + "identityHash": "58cf4468c67f46bf2af8775a30fb0839", + "entities": [ + { + "tableName": "tags", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `visibility` INTEGER NOT NULL, `systemAction` INTEGER NOT NULL, `position` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "nameTag", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "visibility", + "columnName": "visibility", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "systemAction", + "columnName": "systemAction", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "notes", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `value` TEXT, `date` INTEGER NOT NULL, `tag` TEXT, `valueJson` TEXT, `hasRichContent` INTEGER NOT NULL, `attachments` TEXT, `isTrash` INTEGER NOT NULL, `reminderTime` INTEGER, `isPinned` INTEGER NOT NULL, `reminderRepeat` TEXT NOT NULL, `reminderIntervalMinutes` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT" + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT" + }, + { + "fieldPath": "date", + "columnName": "date", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "tag", + "columnName": "tag", + "affinity": "TEXT" + }, + { + "fieldPath": "valueJson", + "columnName": "valueJson", + "affinity": "TEXT" + }, + { + "fieldPath": "hasRichContent", + "columnName": "hasRichContent", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "attachments", + "columnName": "attachments", + "affinity": "TEXT" + }, + { + "fieldPath": "isTrash", + "columnName": "isTrash", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reminderTime", + "columnName": "reminderTime", + "affinity": "INTEGER" + }, + { + "fieldPath": "isPinned", + "columnName": "isPinned", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reminderRepeat", + "columnName": "reminderRepeat", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "reminderIntervalMinutes", + "columnName": "reminderIntervalMinutes", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "tasks", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT NOT NULL, `description` TEXT, `isDone` INTEGER NOT NULL DEFAULT 0, `categoryId` INTEGER NOT NULL DEFAULT 0, `createdAt` INTEGER NOT NULL DEFAULT 0, `position` INTEGER NOT NULL DEFAULT 0, `reminderTime` INTEGER, `reminderIntervalMinutes` INTEGER NOT NULL DEFAULT 0)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT" + }, + { + "fieldPath": "isDone", + "columnName": "isDone", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "categoryId", + "columnName": "categoryId", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "reminderTime", + "columnName": "reminderTime", + "affinity": "INTEGER" + }, + { + "fieldPath": "reminderIntervalMinutes", + "columnName": "reminderIntervalMinutes", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "task_categories", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `colorHex` TEXT NOT NULL DEFAULT '#6750A4', `position` INTEGER NOT NULL DEFAULT 0)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "colorHex", + "columnName": "colorHex", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'#6750A4'" + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "sync_metadata", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`recordType` TEXT NOT NULL, `localId` INTEGER NOT NULL, `stableId` TEXT NOT NULL, `updatedAt` INTEGER NOT NULL, `deletedAt` INTEGER, PRIMARY KEY(`recordType`, `localId`))", + "fields": [ + { + "fieldPath": "recordType", + "columnName": "recordType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "localId", + "columnName": "localId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "stableId", + "columnName": "stableId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "updatedAt", + "columnName": "updatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deletedAt", + "columnName": "deletedAt", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "recordType", + "localId" + ] + }, + "indices": [ + { + "name": "index_sync_metadata_recordType_stableId", + "unique": true, + "columnNames": [ + "recordType", + "stableId" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_sync_metadata_recordType_stableId` ON `${TABLE_NAME}` (`recordType`, `stableId`)" + }, + { + "name": "index_sync_metadata_updatedAt", + "unique": false, + "columnNames": [ + "updatedAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_metadata_updatedAt` ON `${TABLE_NAME}` (`updatedAt`)" + }, + { + "name": "index_sync_metadata_deletedAt", + "unique": false, + "columnNames": [ + "deletedAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_metadata_deletedAt` ON `${TABLE_NAME}` (`deletedAt`)" + } + ] + }, + { + "tableName": "sync_pending_preferences", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `payloadJson` TEXT NOT NULL, `targetHash` TEXT NOT NULL, `baselineHash` TEXT NOT NULL, `recordUpdatedAt` INTEGER NOT NULL, `quarantined` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "payloadJson", + "columnName": "payloadJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "targetHash", + "columnName": "targetHash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "baselineHash", + "columnName": "baselineHash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "recordUpdatedAt", + "columnName": "recordUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "quarantined", + "columnName": "quarantined", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "sync_conflicts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `recordType` TEXT NOT NULL, `stableId` TEXT NOT NULL, `versionPairHash` TEXT NOT NULL, `winnerSource` TEXT NOT NULL, `loserSource` TEXT NOT NULL, `winnerVersionId` TEXT NOT NULL, `loserVersionId` TEXT NOT NULL, `winnerJson` TEXT NOT NULL, `loserJson` TEXT NOT NULL, `winnerUpdatedAt` INTEGER NOT NULL, `loserUpdatedAt` INTEGER NOT NULL, `winnerTombstone` INTEGER NOT NULL, `loserTombstone` INTEGER NOT NULL, `resolution` TEXT NOT NULL, `resolved` INTEGER NOT NULL, `createdAt` INTEGER NOT NULL, `resolvedAt` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "recordType", + "columnName": "recordType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stableId", + "columnName": "stableId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionPairHash", + "columnName": "versionPairHash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerSource", + "columnName": "winnerSource", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "loserSource", + "columnName": "loserSource", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerVersionId", + "columnName": "winnerVersionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "loserVersionId", + "columnName": "loserVersionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerJson", + "columnName": "winnerJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "loserJson", + "columnName": "loserJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerUpdatedAt", + "columnName": "winnerUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "loserUpdatedAt", + "columnName": "loserUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "winnerTombstone", + "columnName": "winnerTombstone", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "loserTombstone", + "columnName": "loserTombstone", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "resolution", + "columnName": "resolution", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "resolved", + "columnName": "resolved", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "resolvedAt", + "columnName": "resolvedAt", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_sync_conflicts_recordType_stableId_versionPairHash", + "unique": true, + "columnNames": [ + "recordType", + "stableId", + "versionPairHash" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_sync_conflicts_recordType_stableId_versionPairHash` ON `${TABLE_NAME}` (`recordType`, `stableId`, `versionPairHash`)" + }, + { + "name": "index_sync_conflicts_resolved", + "unique": false, + "columnNames": [ + "resolved" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_conflicts_resolved` ON `${TABLE_NAME}` (`resolved`)" + }, + { + "name": "index_sync_conflicts_createdAt", + "unique": false, + "columnNames": [ + "createdAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_conflicts_createdAt` ON `${TABLE_NAME}` (`createdAt`)" + } + ] + }, + { + "tableName": "sync_state", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `status` TEXT NOT NULL, `backendIdentifier` TEXT, `lastSuccessfulSyncAt` INTEGER, `attemptStartedAt` INTEGER, `errorMessage` TEXT, `conflictCount` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "backendIdentifier", + "columnName": "backendIdentifier", + "affinity": "TEXT" + }, + { + "fieldPath": "lastSuccessfulSyncAt", + "columnName": "lastSuccessfulSyncAt", + "affinity": "INTEGER" + }, + { + "fieldPath": "attemptStartedAt", + "columnName": "attemptStartedAt", + "affinity": "INTEGER" + }, + { + "fieldPath": "errorMessage", + "columnName": "errorMessage", + "affinity": "TEXT" + }, + { + "fieldPath": "conflictCount", + "columnName": "conflictCount", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '58cf4468c67f46bf2af8775a30fb0839')" + ] + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java index 5ff176c3..e8363526 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java @@ -130,6 +130,77 @@ public void migrate17to18_preservesExistingConflictAndAllowsVersionPairsToCoexis } } + @Test + public void migrate18to19_createsThePendingPreferencesJournal() throws IOException { + SupportSQLiteDatabase db = helper.createDatabase(TEST_DB, 18); + db.close(); + + SupportSQLiteDatabase migrated = + helper.runMigrationsAndValidate(TEST_DB, 19, true, AppDatabase.MIGRATION_18_19); + try (android.database.Cursor cursor = + migrated.query( + "SELECT name FROM sqlite_master WHERE type = 'table' " + + "AND name = 'sync_pending_preferences'")) { + assertThat(cursor.moveToFirst()).isTrue(); + } finally { + migrated.close(); + } + } + + @Test + public void migrate19to20_addsJournalIdentityAndPerSideConflictProvenance() throws IOException { + SupportSQLiteDatabase db = helper.createDatabase(TEST_DB, 19); + db.execSQL( + "INSERT INTO sync_conflicts " + + "(recordType, stableId, versionPairHash, winnerSource, winnerJson, " + + "loserJson, winnerUpdatedAt, loserUpdatedAt, winnerTombstone, " + + "loserTombstone, resolution, resolved, createdAt, resolvedAt) " + + "VALUES ('note', 'stable', 'pair', 'LOCAL', '{}', '{}', 1, 1, 0, 0, " + + "'PENDING', 0, 1, 0)"); + db.close(); + + SupportSQLiteDatabase migrated = + helper.runMigrationsAndValidate(TEST_DB, 20, true, AppDatabase.MIGRATION_19_20); + try (android.database.Cursor cursor = + migrated.query( + "SELECT loserSource, winnerVersionId, loserVersionId FROM sync_conflicts")) { + assertThat(cursor.moveToFirst()).isTrue(); + // A row written before this column existed always had exactly one local side. + assertThat(cursor.getString(0)).isEqualTo("REMOTE"); + assertThat(cursor.getString(1)).isEmpty(); + assertThat(cursor.getString(2)).isEmpty(); + } finally { + migrated.close(); + } + } + + @Test + public void migrateFromTheLastReleasedVersion_reachesTheCurrentSchema() throws IOException { + // 17 is what 2.6.48 shipped; 18, 19 and 20 all land in the same release after it. + SupportSQLiteDatabase db = helper.createDatabase(TEST_DB, 17); + db.execSQL( + "INSERT INTO notes " + + "(id, title, value, date, tag, valueJson, hasRichContent, attachments, " + + "isTrash, reminderTime, isPinned, reminderRepeat, reminderIntervalMinutes) " + + "VALUES (7, 'Note', 'Body', 10, '', '', 0, '', 0, NULL, 0, 'NONE', 0)"); + db.close(); + + SupportSQLiteDatabase migrated = + helper.runMigrationsAndValidate( + TEST_DB, + 20, + true, + AppDatabase.MIGRATION_17_18, + AppDatabase.MIGRATION_18_19, + AppDatabase.MIGRATION_19_20); + try (android.database.Cursor cursor = migrated.query("SELECT COUNT(*) FROM notes")) { + assertThat(cursor.moveToFirst()).isTrue(); + assertThat(cursor.getInt(0)).isEqualTo(1); + } finally { + migrated.close(); + } + } + @Test public void migrate14to15_backfillsCategoryMetadata() throws IOException { SupportSQLiteDatabase db = helper.createDatabase(TEST_DB, 14); diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java index d50e3a9d..c9325374 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java @@ -17,6 +17,7 @@ import com.pasich.mynotes.data.sync.SnapshotProblem; import com.pasich.mynotes.data.sync.SyncMetadata; import com.pasich.mynotes.data.sync.SyncRecord; +import com.pasich.mynotes.data.sync.SyncResolution; import com.pasich.mynotes.data.sync.SyncSnapshot; import com.pasich.mynotes.data.sync.SyncState; import java.io.ByteArrayInputStream; @@ -354,6 +355,232 @@ private int seedNote(String title, String value, String attachmentsJson) { return id; } + @Test + public void readSnapshot_stillResolvesLegacyFileScheme() throws Exception { + int id = seedNote("Legacy", "body", null); + File folder = new File(context.getFilesDir(), "attachments/note_" + id); + assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); + try (FileOutputStream out = new FileOutputStream(new File(folder, "old.png"))) { + out.write("legacy bytes".getBytes(StandardCharsets.UTF_8)); + } + Note note = db.noteDao().getNoteSync(id); + note.setAttachments("[" + legacyAttachmentJson(id, "old.png") + "]"); + db.noteDao().addNote(note); + + SnapshotBuildResult result = store.buildSnapshot(); + + assertThat(result.isPublishable()).isTrue(); + } + + @Test + public void applySnapshot_dropsCachedBlobsNothingReferencesAndKeepsConflictBlobs() + throws Exception { + File cache = new File(context.getFilesDir(), "sync-attachments"); + assertThat(cache.mkdirs() || cache.isDirectory()).isTrue(); + String orphan = "1111111111111111111111111111111111111111111111111111111111111111"; + try (FileOutputStream out = new FileOutputStream(new File(cache, orphan))) { + out.write("nobody references this".getBytes(StandardCharsets.UTF_8)); + } + + store.applySnapshot(SyncSnapshot.empty(), Collections.emptyList()); + + assertThat(new File(cache, orphan).exists()).isFalse(); + } + + @Test + public void applySnapshot_keepsCachedBlobsAnUnresolvedConflictStillNeeds() throws Exception { + File cache = new File(context.getFilesDir(), "sync-attachments"); + assertThat(cache.mkdirs() || cache.isDirectory()).isTrue(); + String pinned = "2222222222222222222222222222222222222222222222222222222222222222"; + try (FileOutputStream out = new FileOutputStream(new File(cache, pinned))) { + out.write("needed by an unresolved conflict".getBytes(StandardCharsets.UTF_8)); + } + db.syncConflictDao() + .insertIgnoringDuplicates( + Collections.singletonList( + new com.pasich.mynotes.data.database.entities.SyncConflictEntity( + "note", + "550e8400-e29b-41d4-a716-446655440000", + "pair", + "LOCAL", + "REMOTE", + "winner-id", + "loser-id", + "{\"payload\":{\"attachmentHashes\":[\"" + pinned + "\"]}}", + "{\"payload\":{}}", + 1L, + 2L, + false, + false, + "PENDING", + false, + 3L, + 0L))); + + store.applySnapshot(SyncSnapshot.empty(), Collections.emptyList()); + + // Deleting this would make the losing version unrecoverable before the user has chosen. + assertThat(new File(cache, pinned).exists()).isTrue(); + } + + // ------------------------------------------------- preferences conflict resolution + + @Test + public void resolveConflict_appliesTheChosenPreferencesVersionDurably() throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + preferencesStore.readState(); + long conflictId = seedPreferencesConflict(9, 11); + + preferencesStore.resolveConflict(conflictId, SyncResolution.KEEP_DRIVE); + + assertThat(adapter.committed.get()).isNotNull(); + assertThat(adapter.committed.get().getThemeValue()).isEqualTo(11); + assertThat(db.syncConflictDao().getById(conflictId).resolved).isTrue(); + assertThat(db.syncPendingPreferencesDao().get()).isNull(); + } + + @Test + public void resolveConflict_leavesThePreferencesConflictOpenWhenTheCommitFails() + throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + adapter.succeeds.set(false); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + preferencesStore.readState(); + long conflictId = seedPreferencesConflict(9, 11); + + try { + preferencesStore.resolveConflict(conflictId, SyncResolution.KEEP_DRIVE); + throw new AssertionError("Expected a failed preferences commit to propagate"); + } catch (IOException expected) { + // Nothing may be claimed as resolved. + } + + assertThat(db.syncConflictDao().getById(conflictId).resolved).isFalse(); + // The journal survives so the next attempt can finish the job. + assertThat(db.syncPendingPreferencesDao().get()).isNotNull(); + // The record version must not move; otherwise the rejected value would win the next sync. + SyncMetadataEntity metadata = + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "00000000-0000-4000-8000-000000000000"); + assertThat(metadata.updatedAt).isEqualTo(0L); + } + + @Test + public void aRetryAfterAFailedCommit_completesTheResolution() throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + adapter.succeeds.set(false); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + preferencesStore.readState(); + long conflictId = seedPreferencesConflict(9, 11); + try { + preferencesStore.resolveConflict(conflictId, SyncResolution.KEEP_DRIVE); + } catch (IOException expected) { + // First attempt fails. + } + + adapter.succeeds.set(true); + preferencesStore.resolveConflict(conflictId, SyncResolution.KEEP_DRIVE); + + assertThat(adapter.committed.get().getThemeValue()).isEqualTo(11); + assertThat(db.syncConflictDao().getById(conflictId).resolved).isTrue(); + assertThat(db.syncPendingPreferencesDao().get()).isNull(); + } + + @Test + public void anUnreadableJournal_isQuarantinedInsteadOfDisablingSync() throws Exception { + db.syncPendingPreferencesDao() + .upsert( + new com.pasich.mynotes.data.database.entities.SyncPendingPreferencesEntity( + 1, "{not json", "target", "baseline", 0L, false)); + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + + // Must not throw: ensureSeeded gates both snapshot building and the status read. + SyncState state = preferencesStore.readState(); + + assertThat(state).isNotNull(); + assertThat(db.syncPendingPreferencesDao().get()).isNull(); + assertThat(db.syncPendingPreferencesDao().getIncludingQuarantined()).isNotNull(); + assertThat(adapter.committed.get()).isNull(); + } + + /** A preferences adapter whose durability can be turned off. */ + private static final class PreferencesAdapter { + private final PreferenceHelper helper = mock(PreferenceHelper.class); + private final java.util.concurrent.atomic.AtomicReference< + com.pasich.mynotes.utils.backup.models.PreferencesBackup> + current = + new java.util.concurrent.atomic.AtomicReference<>(preferencesWithTheme(1)); + private final java.util.concurrent.atomic.AtomicReference< + com.pasich.mynotes.utils.backup.models.PreferencesBackup> + committed = new java.util.concurrent.atomic.AtomicReference<>(); + private final java.util.concurrent.atomic.AtomicBoolean succeeds = + new java.util.concurrent.atomic.AtomicBoolean(true); + + PreferencesAdapter() { + org.mockito.Mockito.when(helper.getListPreferences()) + .thenAnswer(invocation -> current.get()); + org.mockito.Mockito.when( + helper.commitListPreferences(org.mockito.ArgumentMatchers.any())) + .thenAnswer( + invocation -> { + if (!succeeds.get()) { + return false; + } + com.pasich.mynotes.utils.backup.models.PreferencesBackup value = + invocation.getArgument(0); + committed.set(value); + current.set(value); + return true; + }); + } + } + + private long seedPreferencesConflict(int localTheme, int remoteTheme) { + String winner = preferencesRecordJson(remoteTheme, "2026-08-31T12:00:20Z"); + String loser = preferencesRecordJson(localTheme, "2026-08-31T12:00:10Z"); + db.syncConflictDao() + .insertIgnoringDuplicates( + Collections.singletonList( + new com.pasich.mynotes.data.database.entities.SyncConflictEntity( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "00000000-0000-4000-8000-000000000000", + "pair-hash", + "REMOTE", + "LOCAL", + "winner-version-id", + "loser-version-id", + winner, + loser, + 20L, + 10L, + false, + false, + "PENDING", + false, + 1L, + 0L))); + return db.syncConflictDao().getAll().get(0).id; + } + + private static String preferencesRecordJson(int themeValue, String updatedAt) { + return "{\"type\":\"preferences\",\"id\":\"00000000-0000-4000-8000-000000000000\"," + + "\"updatedAt\":\"" + + updatedAt + + "\",\"deletedAt\":null,\"payload\":" + + new com.google.gson.Gson().toJson(preferencesWithTheme(themeValue)) + + "}"; + } + + private static com.pasich.mynotes.utils.backup.models.PreferencesBackup preferencesWithTheme( + int themeValue) { + return new com.pasich.mynotes.utils.backup.models.PreferencesBackup( + 1, "sans", "date", 14, themeValue, false, 0, false, false, false, 1.0f); + } + /** Writes a real file into the note's own attachment folder and links it from the note. */ private int seedNoteWithAttachment(String fileName, byte[] bytes) throws IOException { int id = seedNote("With attachment", "body", null); @@ -362,14 +589,8 @@ private int seedNoteWithAttachment(String fileName, byte[] bytes) throws IOExcep try (FileOutputStream out = new FileOutputStream(new File(folder, fileName))) { out.write(bytes); } - String json = - "[{\"url\":\"file://attachments/note_" - + id - + "/" - + fileName - + "\",\"name\":\"" - + fileName - + "\"}]"; + // Production shape: EditorJSInterface writes editorjs://attachments/note_/. + String json = "[" + attachmentJson(id, fileName) + "]"; Note note = db.noteDao().getNoteSync(id); note.setAttachments(json); db.noteDao().addNote(note); @@ -394,7 +615,17 @@ private static SnapshotBuildResult.SnapshotBuildException assertSnapshotBuildFai } } + /** The canonical reference the editor and sync restore both produce. */ private static String attachmentJson(int noteId, String name) { + return "{\"url\":\"" + + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor(noteId, name) + + "\",\"name\":\"" + + name + + "\"}"; + } + + /** The pre-2.6.49 reference shape, kept readable for already-stored notes. */ + private static String legacyAttachmentJson(int noteId, String name) { return "{\"url\":\"file://attachments/note_" + noteId + "/" diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 3705607a..f68fd882 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -90,15 +90,15 @@ + - - - - - - + android:exported="false" /> diff --git a/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java b/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java index 8c7e2010..e5938bf0 100644 --- a/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java +++ b/app/src/main/java/com/pasich/mynotes/data/AppDataManager.java @@ -84,6 +84,11 @@ public void setListPreferences(PreferencesBackup preferences) { preferencesHelper.setListPreferences(preferences); } + @Override + public boolean commitListPreferences(PreferencesBackup preferences) { + return preferencesHelper.commitListPreferences(preferences); + } + @Override public String getLastKnownVersion() { return preferencesHelper.getLastKnownVersion(); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java b/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java index 7470772e..31b8b70f 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java @@ -153,7 +153,8 @@ public void migrate(@NonNull SupportSQLiteDatabase database) { database.execSQL( "UPDATE `sync_conflicts` SET `versionPairHash` = 'legacy-' || `id` " + "WHERE `versionPairHash` = ''"); - database.execSQL("DROP INDEX IF EXISTS `index_sync_conflicts_recordType_stableId`"); + database.execSQL( + "DROP INDEX IF EXISTS `index_sync_conflicts_recordType_stableId`"); database.execSQL( "CREATE UNIQUE INDEX IF NOT EXISTS " + "`index_sync_conflicts_recordType_stableId_versionPairHash` " @@ -173,6 +174,48 @@ public void migrate(@NonNull SupportSQLiteDatabase database) { } }; + /** + * Gives the pending-preferences journal enough identity to decide whether replay is still valid + * and a quarantine flag so an unreadable payload cannot disable sync forever, and gives each + * conflict side its own origin plus a deterministic version identity. + */ + public static final Migration MIGRATION_19_20 = + new Migration(19, 20) { + @Override + public void migrate(@NonNull SupportSQLiteDatabase database) { + database.execSQL( + "ALTER TABLE `sync_pending_preferences` " + + "ADD COLUMN `targetHash` TEXT NOT NULL DEFAULT ''"); + database.execSQL( + "ALTER TABLE `sync_pending_preferences` " + + "ADD COLUMN `baselineHash` TEXT NOT NULL DEFAULT ''"); + database.execSQL( + "ALTER TABLE `sync_pending_preferences` " + + "ADD COLUMN `recordUpdatedAt` INTEGER NOT NULL DEFAULT 0"); + database.execSQL( + "ALTER TABLE `sync_pending_preferences` " + + "ADD COLUMN `quarantined` INTEGER NOT NULL DEFAULT 0"); + + // Conflict provenance is per side, and each version carries a deterministic + // identity, so a resolution can name a version instead of an endpoint. + database.execSQL( + "ALTER TABLE `sync_conflicts` " + + "ADD COLUMN `loserSource` TEXT NOT NULL DEFAULT 'REMOTE'"); + database.execSQL( + "ALTER TABLE `sync_conflicts` " + + "ADD COLUMN `winnerVersionId` TEXT NOT NULL DEFAULT ''"); + database.execSQL( + "ALTER TABLE `sync_conflicts` " + + "ADD COLUMN `loserVersionId` TEXT NOT NULL DEFAULT ''"); + // Rows written before this column existed always had a local winner or a + // local loser, never two remote sides. + database.execSQL( + "UPDATE `sync_conflicts` SET `loserSource` = " + + "CASE WHEN `winnerSource` = 'LOCAL' THEN 'REMOTE' " + + "ELSE 'LOCAL' END"); + } + }; + private static void insertMetadataForExistingRecords( SupportSQLiteDatabase database, String recordType, diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java index 3825c293..24983967 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncConflictDao.java @@ -26,6 +26,13 @@ public interface SyncConflictDao { @Query("SELECT COUNT(*) FROM sync_conflicts WHERE resolved = 0") int getUnresolvedCount(); + /** Both sides of every settled conflict; neither version may be offered again. */ + @Query( + "SELECT winnerVersionId FROM sync_conflicts WHERE resolved = 1 AND winnerVersionId != ''" + + " UNION SELECT loserVersionId FROM sync_conflicts WHERE resolved = 1 AND" + + " loserVersionId != ''") + List getResolvedVersionIds(); + @Query("SELECT * FROM sync_conflicts WHERE id = :id LIMIT 1") SyncConflictEntity getById(long id); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncPendingPreferencesDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncPendingPreferencesDao.java index f57ce0bf..a026039d 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncPendingPreferencesDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/SyncPendingPreferencesDao.java @@ -9,12 +9,27 @@ @Dao public interface SyncPendingPreferencesDao { - @Query("SELECT * FROM sync_pending_preferences WHERE id = 1 LIMIT 1") + /** The journal awaiting replay. Quarantined rows are deliberately invisible here. */ + @Query("SELECT * FROM sync_pending_preferences WHERE id = 1 AND quarantined = 0 LIMIT 1") SyncPendingPreferencesEntity get(); + /** Includes quarantined rows; for diagnostics and tests only. */ + @Query("SELECT * FROM sync_pending_preferences WHERE id = 1 LIMIT 1") + SyncPendingPreferencesEntity getIncludingQuarantined(); + @Insert(onConflict = OnConflictStrategy.REPLACE) void upsert(SyncPendingPreferencesEntity pending); @Query("DELETE FROM sync_pending_preferences WHERE id = 1") void clear(); + + /** + * Sets a journal aside instead of deleting it. + * + *

An unreadable payload used to be thrown from {@code ensureSeeded}, which gates both + * snapshot building and the status read, so one bad row disabled sync permanently with no way + * to clear it. Quarantining keeps the bytes for support while letting sync run again. + */ + @Query("UPDATE sync_pending_preferences SET quarantined = 1 WHERE id = 1") + void quarantine(); } diff --git a/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncConflictEntity.java b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncConflictEntity.java index 7b55d51d..b2b3c394 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncConflictEntity.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncConflictEntity.java @@ -22,9 +22,22 @@ public class SyncConflictEntity { @NonNull public String recordType; @NonNull public String stableId; + /** Stable digest of the exact winner/loser version pair; never use the mutable row id. */ @NonNull public String versionPairHash; + + /** Origin of the winning version: LOCAL, REMOTE. */ @NonNull public String winnerSource; + + /** Origin of the losing version; both sides are REMOTE for a Drive-vs-Drive conflict. */ + @NonNull public String loserSource; + + /** Deterministic identity of the winning version, equal on every device. */ + @NonNull public String winnerVersionId; + + /** Deterministic identity of the losing version, equal on every device. */ + @NonNull public String loserVersionId; + @NonNull public String winnerJson; @NonNull public String loserJson; public long winnerUpdatedAt; @@ -41,6 +54,9 @@ public SyncConflictEntity( @NonNull String stableId, @NonNull String versionPairHash, @NonNull String winnerSource, + @NonNull String loserSource, + @NonNull String winnerVersionId, + @NonNull String loserVersionId, @NonNull String winnerJson, @NonNull String loserJson, long winnerUpdatedAt, @@ -55,6 +71,9 @@ public SyncConflictEntity( this.stableId = stableId; this.versionPairHash = versionPairHash; this.winnerSource = winnerSource; + this.loserSource = loserSource; + this.winnerVersionId = winnerVersionId; + this.loserVersionId = loserVersionId; this.winnerJson = winnerJson; this.loserJson = loserJson; this.winnerUpdatedAt = winnerUpdatedAt; diff --git a/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java index 30c0a404..99fbe140 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java @@ -4,15 +4,47 @@ import androidx.room.Entity; import androidx.room.PrimaryKey; -/** Room journal for a preference adapter mutation that must follow a snapshot transaction. */ +/** + * Room journal for a preference write that must follow a committed snapshot transaction. + * + *

SharedPreferences sits outside Room, so the two are bridged by writing this row inside the + * transaction and clearing it only once the adapter reports a durable commit. The two digests make + * the replay decidable rather than blind: {@code baselineHash} is what the live preferences looked + * like when the journal was written and {@code targetHash} is what they should look like + * afterwards, so recovery can tell "already applied" from "still pending" from "the user has since + * changed these settings themselves". + */ @Entity(tableName = "sync_pending_preferences") public final class SyncPendingPreferencesEntity { @PrimaryKey public int id; + @NonNull public String payloadJson; - public SyncPendingPreferencesEntity(int id, @NonNull String payloadJson) { + /** Digest of the preferences this journal is meant to produce. */ + @NonNull public String targetHash; + + /** Digest of the live preferences at the moment the journal was written. */ + @NonNull public String baselineHash; + + /** {@code updatedAt} of the sync record the payload came from. */ + public long recordUpdatedAt; + + /** Set when the payload could not be read; retained for support, skipped by recovery. */ + public boolean quarantined; + + public SyncPendingPreferencesEntity( + int id, + @NonNull String payloadJson, + @NonNull String targetHash, + @NonNull String baselineHash, + long recordUpdatedAt, + boolean quarantined) { this.id = id; this.payloadJson = payloadJson; + this.targetHash = targetHash; + this.baselineHash = baselineHash; + this.recordUpdatedAt = recordUpdatedAt; + this.quarantined = quarantined; } } diff --git a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java index 6cebac0a..01ad98e4 100644 --- a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java +++ b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java @@ -91,50 +91,53 @@ public PreferencesBackup getListPreferences() { /** Persists all fields from a backup and refreshes the caches. */ @Override public void setListPreferences(PreferencesBackup preferences) { - - if (preferences.isCreated()) { - - // OLD FIELDS - prefs.putInt( - PreferencesConfig.ARGUMENT_PREFERENCE_FORMAT, preferences.getFormatCount()); - - prefs.putString( - PreferencesConfig.ARGUMENT_PREFERENCE_TEXT_STYLE, - preferences.getTypeFaceNoteActivity()); - - prefs.putString(PreferencesConfig.ARGUMENT_PREFERENCE_SORT, preferences.getSortParam()); - - prefs.putInt( - PreferencesConfig.ARGUMENT_PREFERENCE_TEXT_SIZE, preferences.getSizeTextNote()); - - prefs.putInt(PreferencesConfig.ARGUMENT_PREFERENCE_THEME, preferences.getThemeValue()); - - prefs.putBoolean( - PreferencesConfig.ARGUMENT_PREFERENCE_DYNAMIC_COLOR, - preferences.isDynamicTheme()); - - prefs.putInt( - PreferencesConfig.ARGUMENT_PREFERENCE_THEME_MODE, preferences.getThemeMode()); - - prefs.putBoolean( - PreferencesConfig.ARGUMENT_PREFERENCE_IMAGEOPT, - preferences.isImageOptimizationEnabled()); - - prefs.putBoolean( - PreferencesConfig.ARGUMENT_PREFERENCE_SCREEN_PROTECTION, - preferences.isScreenProtection()); - - prefs.putBoolean( - PreferencesConfig.ARGUMENT_PREFERENCE_EXTENDED_EDITOR, - preferences.isExtendedEditor()); - - prefs.putFloat( - PreferencesConfig.ARGUMENT_PREFERENCE_UI_SCALING, preferences.getUiFontScale()); - - // Refresh caches - appCache.refresh(); - themeCache.refresh(); + commitListPreferences(preferences); + } + + /** + * Writes every backed-up preference as one durable edit. + * + *

This used to be eleven separate {@code apply()} calls. {@code apply()} is asynchronous and + * per key, so a process death part-way through left the user with a mixture of the old and the + * new settings, and left the sync journal that had "already committed" them cleared. One editor + * plus {@code commit()} makes the whole set atomic and tells the caller whether it is durable, + * which is what lets {@code RoomSyncStore} decide when the journal may be dropped. + * + * @return true when the values are durably stored, false when the write failed. + */ + @Override + public boolean commitListPreferences(PreferencesBackup preferences) { + if (preferences == null || !preferences.isCreated()) { + return false; + } + java.util.Map values = new java.util.LinkedHashMap<>(); + values.put(PreferencesConfig.ARGUMENT_PREFERENCE_FORMAT, preferences.getFormatCount()); + values.put( + PreferencesConfig.ARGUMENT_PREFERENCE_TEXT_STYLE, + preferences.getTypeFaceNoteActivity()); + values.put(PreferencesConfig.ARGUMENT_PREFERENCE_SORT, preferences.getSortParam()); + values.put(PreferencesConfig.ARGUMENT_PREFERENCE_TEXT_SIZE, preferences.getSizeTextNote()); + values.put(PreferencesConfig.ARGUMENT_PREFERENCE_THEME, preferences.getThemeValue()); + values.put( + PreferencesConfig.ARGUMENT_PREFERENCE_DYNAMIC_COLOR, preferences.isDynamicTheme()); + values.put(PreferencesConfig.ARGUMENT_PREFERENCE_THEME_MODE, preferences.getThemeMode()); + values.put( + PreferencesConfig.ARGUMENT_PREFERENCE_IMAGEOPT, + preferences.isImageOptimizationEnabled()); + values.put( + PreferencesConfig.ARGUMENT_PREFERENCE_SCREEN_PROTECTION, + preferences.isScreenProtection()); + values.put( + PreferencesConfig.ARGUMENT_PREFERENCE_EXTENDED_EDITOR, + preferences.isExtendedEditor()); + values.put(PreferencesConfig.ARGUMENT_PREFERENCE_UI_SCALING, preferences.getUiFontScale()); + + if (!prefs.commitAll(values)) { + return false; } + appCache.refresh(); + themeCache.refresh(); + return true; } @Override diff --git a/app/src/main/java/com/pasich/mynotes/data/preferences/PreferenceHelper.java b/app/src/main/java/com/pasich/mynotes/data/preferences/PreferenceHelper.java index 6665b753..3abc0439 100644 --- a/app/src/main/java/com/pasich/mynotes/data/preferences/PreferenceHelper.java +++ b/app/src/main/java/com/pasich/mynotes/data/preferences/PreferenceHelper.java @@ -22,6 +22,13 @@ public interface PreferenceHelper { void setListPreferences(PreferencesBackup preferences); + /** + * Writes every backed-up preference as one durable edit. + * + * @return true only when the whole set is durably stored. + */ + boolean commitListPreferences(PreferencesBackup preferences); + String getLastKnownVersion(); void setLastKnownVersion(String version); diff --git a/app/src/main/java/com/pasich/mynotes/data/preferences/SafePreferences.java b/app/src/main/java/com/pasich/mynotes/data/preferences/SafePreferences.java index cfde4136..c2b64dff 100644 --- a/app/src/main/java/com/pasich/mynotes/data/preferences/SafePreferences.java +++ b/app/src/main/java/com/pasich/mynotes/data/preferences/SafePreferences.java @@ -47,4 +47,35 @@ public void putBoolean(String key, boolean value) { public void putFloat(String key, float value) { prefs.edit().putFloat(key, value).apply(); } + + /** + * Writes several keys as one durable edit and reports whether it reached disk. + * + *

The per-key {@code putX} helpers above each call {@code apply()}, which is asynchronous + * and per-key: a caller writing eleven of them could be killed with some keys stored and others + * not, and a caller that then cleared a journal on the strength of those calls could lose the + * lot. {@code commit()} returns only once the write is durable, so a journal can be cleared on + * a {@code true} and kept on a {@code false}. + * + * @return true only when every value in {@code values} is durably stored. + */ + public boolean commitAll(java.util.Map values) { + SharedPreferences.Editor editor = prefs.edit(); + for (java.util.Map.Entry entry : values.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Integer) { + editor.putInt(entry.getKey(), (Integer) value); + } else if (value instanceof Boolean) { + editor.putBoolean(entry.getKey(), (Boolean) value); + } else if (value instanceof Float) { + editor.putFloat(entry.getKey(), (Float) value); + } else if (value instanceof String) { + editor.putString(entry.getKey(), (String) value); + } else { + throw new IllegalArgumentException( + "Unsupported preference type for " + entry.getKey()); + } + } + return editor.commit(); + } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentIntegrityException.java b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentIntegrityException.java index 5c0b514e..c942dce3 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentIntegrityException.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/AttachmentIntegrityException.java @@ -5,9 +5,9 @@ /** * Indicates that bytes did not satisfy the immutable attachment contract. * - *

This is deliberately distinct from a transport failure. An object discovered after a lost - * HTTP response can only confirm an ambiguous request; it can never turn a hash or size mismatch - * into success. + *

This is deliberately distinct from a transport failure. An object discovered after a lost HTTP + * response can only confirm an ambiguous request; it can never turn a hash or size mismatch into + * success. */ public final class AttachmentIntegrityException extends IOException { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java index 8354a045..204f910e 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackend.java @@ -19,14 +19,15 @@ import java.security.NoSuchAlgorithmException; import java.time.Clock; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.HashSet; import java.util.UUID; /** Google Drive REST backend for the provider-independent sync protocol. */ @@ -43,6 +44,7 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private static final int MAX_ATTACHMENT_RESPONSE_BYTES = 100 * 1024 * 1024; private static final int RESUMABLE_CHUNK_BYTES = 256 * 1024; private static final int HTTP_RESUME_INCOMPLETE = 308; + private static final int MAX_STALLED_CHUNK_ATTEMPTS = 3; private static final int MAX_ERROR_DETAIL_BYTES = 1024; private static final int MAX_ERROR_DETAIL_CHARS = 200; private static final Gson GSON = new Gson(); @@ -55,6 +57,16 @@ public final class GoogleDriveSyncBackend implements SyncBackend { private final DriveRequestExecutor requestExecutor; private final SyncMerger merger = new SyncMerger(); private List lastReadFrontierBundleIds = Collections.emptyList(); + private String lastReadToken = ""; + + /** + * Blobs already read and hashed during this sync, keyed by root, hash and expected size. + * + *

One attachment used to be downloaded in full two or three times per sync: once by + * hasAttachment, once by the service re-verifying it, and once more while materializing it in + * the canonical root. The verification itself is the point, so it still happens — once. + */ + private final Set verifiedAttachments = new HashSet<>(); public GoogleDriveSyncBackend(@NonNull String accessToken) { this(accessToken, DEFAULT_API, DEFAULT_UPLOAD, Clock.systemUTC(), new SyncBundleCodec()); @@ -94,7 +106,14 @@ public synchronized RemoteSnapshot readSnapshotResult() throws IOException { List folderIds = findFolderIds(); if (folderIds.isEmpty()) { lastReadFrontierBundleIds = Collections.emptyList(); - return RemoteSnapshot.of(SyncSnapshot.empty()); + lastReadToken = UUID.randomUUID().toString(); + return new RemoteSnapshot( + SyncSnapshot.empty(), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptySet(), + lastReadToken); } Map bundlesByLogicalId = new HashMap<>(); @@ -111,7 +130,8 @@ public synchronized RemoteSnapshot readSnapshotResult() throws IOException { byte[] previousBytes = bytesByLogicalId.putIfAbsent(decoded.getBundleId(), bytes); if (previousBytes != null) { if (!java.util.Arrays.equals(previousBytes, bytes)) { - throw new IOException("Drive contains conflicting physical copies of one bundle"); + throw new IOException( + "Drive contains conflicting physical copies of one bundle"); } continue; } @@ -123,23 +143,87 @@ public synchronized RemoteSnapshot readSnapshotResult() throws IOException { SyncSnapshot merged = SyncSnapshot.empty(); List conflicts = new ArrayList<>(); for (String bundleId : frontier) { - SyncMergeResult result = merger.merge(merged, bundlesByLogicalId.get(bundleId).getSnapshot()); + // Both sides are Drive bundle heads. Naming them explicitly stops the accumulator + // being reported to the user as "this device". + SyncMergeResult result = + merger.merge( + merged, + bundlesByLogicalId.get(bundleId).getSnapshot(), + SyncMergeResult.Source.REMOTE, + SyncMergeResult.Source.REMOTE); merged = result.getMergedSnapshot(); conflicts.addAll(result.getConflicts()); } + // Alternatives and the resolutions that retire them travel with the bundles, so a device + // that has never seen a conflict still discovers it and a device that resolved one still + // retires it everywhere. + Set resolvedAlternativeIds = new HashSet<>(); + for (String bundleId : frontier) { + resolvedAlternativeIds.addAll( + bundlesByLogicalId.get(bundleId).getResolvedAlternativeIds()); + } + Map alternativesByVersion = new java.util.LinkedHashMap<>(); + for (String bundleId : frontier) { + for (SyncRecord alternative : bundlesByLogicalId.get(bundleId).getAlternatives()) { + String versionId = alternative.getCanonicalPayloadHash(); + if (resolvedAlternativeIds.contains(versionId)) continue; + SyncRecord winner = merged.find(alternative.getType(), alternative.getId()); + if (winner == null || winner.getCanonicalPayloadHash().equals(versionId)) { + // Nothing to choose between: the alternative is the current value, or its + // record no longer exists at all. + continue; + } + alternativesByVersion.putIfAbsent(versionId, alternative); + } + } + List alternatives = new ArrayList<>(alternativesByVersion.values()); + for (SyncRecord alternative : alternatives) { + SyncRecord winner = merged.find(alternative.getType(), alternative.getId()); + conflicts.add( + new SyncMergeResult.Conflict( + winner, + alternative, + SyncMergeResult.Source.REMOTE, + SyncMergeResult.Source.REMOTE)); + } + lastReadFrontierBundleIds = Collections.unmodifiableList(new ArrayList<>(frontier)); - return new RemoteSnapshot(merged, conflicts, frontier); + lastReadToken = UUID.randomUUID().toString(); + return new RemoteSnapshot( + merged, conflicts, frontier, alternatives, resolvedAlternativeIds, lastReadToken); } @Override public synchronized void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IOException { + throw new IOException( + "A Drive publish requires the read context it was derived from; use publish()"); + } + + @Override + public synchronized void publish(@NonNull SyncPublication publication) throws IOException { + // Causal parents used to come from a mutable field, so a write with no preceding read + // published a parentless root that permanently forked the DAG. The read that produced + // this publication has to be this backend's most recent one. + String token = publication.getReadContext().getReadToken(); + if (token.isEmpty() || !token.equals(lastReadToken)) { + throw new IOException( + "Drive publish is not derived from this backend's latest remote read"); + } + SyncSnapshot snapshot = publication.getSnapshot(); String folderId = ensureCanonicalFolderId(); // A first-sync race can leave valid bundles and immutable blobs in two owned folders. // The read path always merges all roots. Before canonical publication, materialize every // referenced blob in the canonical root as well, so no future cleanup decision can make // the canonical bundle point at an object that exists only in a duplicate root. ensureCanonicalAttachments(folderId, snapshot); - byte[] bundle = bundleCodec.encode(snapshot, clock.instant(), lastReadFrontierBundleIds); + ensureCanonicalAlternativeAttachments(folderId, publication.getUnresolvedAlternatives()); + byte[] bundle = + bundleCodec.encode( + snapshot, + clock.instant(), + lastReadFrontierBundleIds, + publication.getUnresolvedAlternatives(), + publication.getResolvedAlternativeIds()); // Every bundle is immutable. Drive offers no conditional update based on its version // counter, so replacing one file leaves a race where another device can be overwritten. // Publishing a distinct file makes each successful upload independently durable; readers @@ -157,10 +241,17 @@ public synchronized void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IO } } + /** + * Whether any root indexes a blob under this hash, without reading it. + * + *

Deliberately an index lookup: this used to download and hash the whole blob, and its only + * caller then downloaded it a second time to verify it. Existence and verification are separate + * questions now, and {@link #hasVerifiedAttachment} answers the second one once. + */ @Override public synchronized boolean hasAttachment(@NonNull String sha256) throws IOException { for (String folderId : findFolderIds()) { - if (findVerifiedAttachment(folderId, sha256, null) != null) { + if (findAttachment(folderId, sha256) != null) { return true; } } @@ -271,7 +362,12 @@ private String ensureCanonicalFolderId() throws IOException { private void ensureCanonicalAttachments( @NonNull String canonicalRootId, @NonNull SyncSnapshot snapshot) throws IOException { - Map sizes = attachmentSizes(snapshot); + materializeAttachmentsInCanonicalRoot( + canonicalRootId, attachmentSizes(snapshot.getLiveRecords(SyncRecord.Type.NOTE))); + } + + private void materializeAttachmentsInCanonicalRoot( + @NonNull String canonicalRootId, @NonNull Map sizes) throws IOException { if (sizes.isEmpty()) { return; } @@ -292,11 +388,33 @@ private void ensureCanonicalAttachments( } } + /** + * Makes every blob an unresolved alternative needs available in the canonical root. + * + *

Works from a plain record list rather than a {@link SyncSnapshot}: one record can have + * several unresolved alternatives at once — three-way edits, or a second conflict on a note + * that already had one — and a snapshot deliberately refuses to hold two versions of one ID. + */ + private void ensureCanonicalAlternativeAttachments( + @NonNull String canonicalRootId, @NonNull List alternatives) + throws IOException { + List notes = new ArrayList<>(); + for (SyncRecord alternative : alternatives) { + if (!alternative.isTombstone() && alternative.getType() == SyncRecord.Type.NOTE) { + notes.add(alternative); + } + } + if (notes.isEmpty()) { + return; + } + materializeAttachmentsInCanonicalRoot(canonicalRootId, attachmentSizes(notes)); + } + @NonNull - private static Map attachmentSizes(@NonNull SyncSnapshot snapshot) + private static Map attachmentSizes(@NonNull Collection notes) throws IOException { Map sizes = new HashMap<>(); - for (SyncRecord record : snapshot.getLiveRecords(SyncRecord.Type.NOTE)) { + for (SyncRecord record : notes) { JsonArray manifest = record.getPayload().getAsJsonArray("attachmentsManifest"); if (manifest == null) { continue; @@ -320,15 +438,19 @@ private static Map attachmentSizes(@NonNull SyncSnapshot snapshot) return sizes; } + /** + * Checks the ancestry graph without requiring every historical bundle to still exist. + * + *

A missing ancestor used to be fatal, which inverted the rule that cleanup must never be + * needed for correctness: one bundle trashed by hand, or aged out of Drive's own trash, and + * sync failed forever with no way back. It is safe to tolerate because a bundle is a complete + * snapshot rather than a delta — every descendant already contains everything its ancestors + * held, including their unresolved alternatives — so an absent ancestor removes nothing from + * the state a head describes. It also cannot be a frontier head itself, since a head is a + * bundle no present bundle claims as a parent. + */ private static void validateBundleDag( @NonNull Map bundles) throws IOException { - for (SyncBundleCodec.DecodedBundle bundle : bundles.values()) { - for (String parent : bundle.getParentBundleIds()) { - if (!bundles.containsKey(parent)) { - throw new IOException("Drive bundle references an unavailable ancestor"); - } - } - } Set visiting = new HashSet<>(); Set visited = new HashSet<>(); for (String bundleId : bundles.keySet()) { @@ -343,8 +465,14 @@ private static void validateAcyclic( @NonNull Set visited) throws IOException { if (visited.contains(bundleId)) return; - if (!visiting.add(bundleId)) throw new IOException("Drive bundle ancestry contains a cycle"); - for (String parent : bundles.get(bundleId).getParentBundleIds()) { + SyncBundleCodec.DecodedBundle bundle = bundles.get(bundleId); + if (bundle == null) { + // An ancestor that is no longer stored. Nothing to walk and nothing to lose. + return; + } + if (!visiting.add(bundleId)) + throw new IOException("Drive bundle ancestry contains a cycle"); + for (String parent : bundle.getParentBundleIds()) { validateAcyclic(parent, bundles, visiting, visited); } visiting.remove(bundleId); @@ -420,8 +548,13 @@ private String findVerifiedAttachment( } candidateIds.sort(Comparator.naturalOrder()); for (String candidateId : candidateIds) { + String cacheKey = candidateId + "\u0000" + sha256 + "\u0000" + expectedSize; + if (verifiedAttachments.contains(cacheKey)) { + return candidateId; + } try (InputStream candidate = openAttachment(candidateId)) { verifyAttachment(candidate, sha256, expectedSize); + verifiedAttachments.add(cacheKey); return candidateId; } catch (AttachmentIntegrityException corrupt) { // A second content-addressed duplicate may be valid. Never accept the property @@ -431,6 +564,24 @@ private String findVerifiedAttachment( return null; } + /** + * True only when a remote blob exists and its actual bytes hash to {@code sha256}. + * + *

Drive's {@code appProperties} index is a claim, not proof, so the bytes are read. The + * result is remembered for this sync so the caller does not have to download the blob again + * purely to repeat the same check. + */ + @Override + public synchronized boolean hasVerifiedAttachment( + @NonNull String sha256, @Nullable Long expectedSize) throws IOException { + for (String folderId : findFolderIds()) { + if (findVerifiedAttachment(folderId, sha256, expectedSize) != null) { + return true; + } + } + return false; + } + @NonNull private InputStream openAttachment(@NonNull String attachmentId) throws IOException { HttpURLConnection connection = @@ -471,7 +622,8 @@ private static void verifyAttachment( "Attachment checksum does not match its declared hash"); } if (expectedSize != null && expectedSize.longValue() != size) { - throw new AttachmentIntegrityException("Attachment size does not match its declared size"); + throw new AttachmentIntegrityException( + "Attachment size does not match its declared size"); } } @@ -558,13 +710,40 @@ private void uploadStream( @NonNull InputStream content, long sizeBytes) throws IOException { + if (sizeBytes == 0L) { + // A resumable session has no chunk to send and therefore no way to finalize; the + // loop below would exit having created nothing while reporting success. An empty + // blob is valid user data, so it takes the multipart path, whose Content-Length is + // exact and whose empty body part Drive commits as a zero-byte file. + uploadEmptyAttachment(folderId, name, mimeType, content); + return; + } uploadResumableAttachment(folderId, name, mimeType, content, sizeBytes); } + /** Publishes a zero-length blob and proves the source really was empty. */ + private void uploadEmptyAttachment( + @NonNull String folderId, + @NonNull String name, + @NonNull String mimeType, + @NonNull InputStream content) + throws IOException { + if (content.read() != -1) { + throw new IOException("Attachment exceeds its declared size"); + } + uploadMultipart(folderId, name, mimeType, new ByteArrayInputStream(new byte[0]), 0L, false); + } + /** - * Uploads a bounded attachment in resumable chunks. Only one chunk is retained in heap, so a - * dropped connection can be probed and the unacknowledged chunk replayed without re-reading the - * source stream. + * Uploads a bounded attachment in resumable chunks. + * + *

Progress is tracked as one absolute count of bytes Drive has committed, {@code + * acknowledgedExclusive}, and every request starts at exactly that offset. An earlier version + * derived progress from a mutable {@code remaining} counter that could desynchronize from the + * absolute Drive offset: once a partially acknowledged chunk was completed by a retry the loop + * never terminated, and it replayed the buffer under offsets past the end of the file. Nothing + * here is derived — the buffer window is recomputed from absolute offsets on every pass, so a + * byte can only ever be sent under the one offset it occupies in the source. */ private void uploadResumableAttachment( @NonNull String folderId, @@ -575,46 +754,73 @@ private void uploadResumableAttachment( throws IOException { String sessionUrl = initiateResumableAttachmentUpload(folderId, sha256, mimeType, sizeBytes); - byte[] chunk = new byte[RESUMABLE_CHUNK_BYTES]; - long offset = 0L; - while (offset < sizeBytes) { + byte[] buffer = new byte[RESUMABLE_CHUNK_BYTES]; + long acknowledgedExclusive = 0L; + long bufferStart = 0L; + int bufferLength = 0; + int stalledAttempts = 0; + + while (acknowledgedExclusive < sizeBytes) { throwIfInterrupted(); - int chunkSize = - readChunk(content, chunk, (int) Math.min(chunk.length, sizeBytes - offset)); - if (chunkSize <= 0) { - throw new IOException("Attachment ended before its declared size"); + if (acknowledgedExclusive >= bufferStart + bufferLength) { + // Everything buffered is durable; read the next window from the source. + bufferStart = acknowledgedExclusive; + bufferLength = + readChunk( + content, + buffer, + (int) Math.min(buffer.length, sizeBytes - bufferStart)); + if (bufferLength <= 0) { + throw new IOException("Attachment ended before its declared size"); + } } - long acknowledged = - uploadChunk(sessionUrl, mimeType, chunk, chunkSize, offset, sizeBytes); - if (acknowledged < offset - 1L || acknowledged >= offset + chunkSize) { + + int offsetInBuffer = (int) (acknowledgedExclusive - bufferStart); + int length = bufferLength - offsetInBuffer; + long chunkStart = acknowledgedExclusive; + long chunkEndExclusive = chunkStart + length; + + // A chunk PUT is idempotent: it is addressed by an absolute Content-Range, so a + // replay of the identical range either lands at the same offset or is already + // committed. Retrying is therefore safe, and it keeps one transient 5xx between + // chunks from discarding a large upload that is nearly complete. + final int retryOffset = offsetInBuffer; + final int retryLength = length; + final long retryStart = chunkStart; + long reported = + requestExecutor.executeIdempotent( + () -> + uploadChunk( + sessionUrl, + mimeType, + buffer, + retryOffset, + retryLength, + retryStart, + sizeBytes)); + + if (reported < acknowledgedExclusive) { throw new IOException( - "Drive resumable upload returned an invalid acknowledged range"); + "Drive resumable upload moved its acknowledged range backwards"); } - if (acknowledged < offset + chunkSize - 1L) { - // The server received only a prefix. The unread suffix remains in this one chunk; - // replay it rather than advancing the source stream. - int consumed = (int) (acknowledged - offset + 1L); - System.arraycopy(chunk, consumed, chunk, 0, chunkSize - consumed); - int remaining = chunkSize - consumed; - while (remaining > 0) { - acknowledged = - uploadChunk( - sessionUrl, - mimeType, - chunk, - remaining, - acknowledged + 1L, - sizeBytes); - if (acknowledged < offset + chunkSize - 1L) { - int newlyConsumed = (int) (acknowledged - offset - consumed + 1L); - System.arraycopy(chunk, newlyConsumed, chunk, 0, remaining - newlyConsumed); - remaining -= newlyConsumed; - consumed += newlyConsumed; - } + if (reported > sizeBytes) { + throw new IOException("Drive acknowledged more bytes than the attachment declares"); + } + if (reported > chunkEndExclusive) { + throw new IOException("Drive acknowledged bytes that were never sent"); + } + if (reported == acknowledgedExclusive) { + // A 308 that commits nothing is tolerable once or twice; forever is the bug + // this loop exists to make impossible. + if (++stalledAttempts > MAX_STALLED_CHUNK_ATTEMPTS) { + throw new IOException("Drive resumable upload stopped making progress"); } + continue; } - offset += chunkSize; + stalledAttempts = 0; + acknowledgedExclusive = reported; } + if (content.read() != -1) { throw new IOException("Attachment exceeds its declared size"); } @@ -651,31 +857,40 @@ private String initiateResumableAttachmentUpload( } } + /** + * Sends one range and returns the absolute number of bytes Drive has committed afterwards. + * + *

Exclusive, not the inclusive index the {@code Range} header carries, so the caller never + * has to convert between the two conventions. + */ private long uploadChunk( @NonNull String sessionUrl, @NonNull String mimeType, - @NonNull byte[] chunk, - int chunkSize, + @NonNull byte[] buffer, + int offset, + int length, long start, long total) throws IOException { + if (length <= 0) { + throw new IOException("Drive resumable upload attempted an empty chunk"); + } HttpURLConnection connection = open("PUT", sessionUrl); connection.setRequestProperty("Content-Type", mimeType); connection.setRequestProperty( - "Content-Range", "bytes " + start + "-" + (start + chunkSize - 1L) + "/" + total); + "Content-Range", "bytes " + start + "-" + (start + length - 1L) + "/" + total); connection.setDoOutput(true); - connection.setFixedLengthStreamingMode(chunkSize); + connection.setFixedLengthStreamingMode(length); try { try (OutputStream output = connection.getOutputStream()) { - output.write(chunk, 0, chunkSize); + output.write(buffer, offset, length); } int status = connection.getResponseCode(); if (status >= 200 && status < 300) { - return total - 1L; + return total; } if (status == HTTP_RESUME_INCOMPLETE) { - String range = connection.getHeaderField("Range"); - return resumableRangeEnd(range); + return resumableAcknowledgedExclusive(connection.getHeaderField("Range")); } String detail = readErrorDetail(connection.getErrorStream()); throw new DriveRequestExecutor.DriveHttpException( @@ -685,12 +900,26 @@ private long uploadChunk( } } - private static long resumableRangeEnd(@Nullable String range) throws IOException { - if (range == null || !range.startsWith("bytes=0-")) { - return -1L; + /** + * Reads {@code Range: bytes=0-N} as an exclusive committed-byte count. + * + *

A 308 with no {@code Range} header means Drive holds nothing yet, which is zero — not a + * negative sentinel the caller then has to special-case at offset zero. + */ + private static long resumableAcknowledgedExclusive(@Nullable String range) throws IOException { + if (range == null || range.trim().isEmpty()) { + return 0L; + } + String value = range.trim(); + if (!value.startsWith("bytes=0-")) { + throw new IOException("Drive returned an unsupported resumable upload range: " + value); } try { - return Long.parseLong(range.substring("bytes=0-".length())); + long inclusiveEnd = Long.parseLong(value.substring("bytes=0-".length())); + if (inclusiveEnd < 0L) { + throw new IOException("Drive returned a negative resumable upload range"); + } + return inclusiveEnd + 1L; } catch (NumberFormatException error) { throw new IOException("Drive returned an invalid resumable upload range", error); } @@ -1167,14 +1396,16 @@ private void verifyEndOfStream() throws IOException { throw new IOException("Attachment upload ended before the source was verified"); } if (size != expectedSize) { - throw new AttachmentIntegrityException("Attachment size does not match sync metadata"); + throw new AttachmentIntegrityException( + "Attachment size does not match sync metadata"); } StringBuilder actualHash = new StringBuilder(64); for (byte value : digest.digest()) { actualHash.append(String.format(java.util.Locale.US, "%02x", value & 0xff)); } if (!expectedHash.equals(actualHash.toString())) { - throw new AttachmentIntegrityException("Attachment checksum does not match sync metadata"); + throw new AttachmentIntegrityException( + "Attachment checksum does not match sync metadata"); } } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/PendingPreferencesDecision.java b/app/src/main/java/com/pasich/mynotes/data/sync/PendingPreferencesDecision.java new file mode 100644 index 00000000..eda261a2 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/PendingPreferencesDecision.java @@ -0,0 +1,64 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +/** + * What to do with a preferences journal found at startup. + * + *

SharedPreferences is outside Room, so a snapshot apply writes its intent to a journal row + * inside the Room transaction and clears it only after a durable commit. Any of those steps can be + * interrupted, and the right response differs per case — replaying unconditionally would overwrite + * settings the user has since changed by hand. The decision is pure, and separate from Room, so + * every crash window is covered by ordinary unit tests rather than only on a device. + */ +public final class PendingPreferencesDecision { + + public enum Action { + /** No journal row; nothing to do. */ + NOTHING, + /** The payload cannot be read. Set it aside rather than fail sync forever. */ + QUARANTINE, + /** The live values already match the target: the write landed, only the clear was lost. */ + CLEAR_ALREADY_APPLIED, + /** The live values still match the baseline, so the payload is still the right answer. */ + REPLAY, + /** The user changed these settings after the journal was written; their choice wins. */ + DISCARD_STALE + } + + private PendingPreferencesDecision() {} + + /** + * Decides the fate of one journal row. + * + * @param payloadReadable whether the stored payload parsed into usable settings. + * @param targetHash digest the journal intends to produce; empty when unknown. + * @param baselineHash digest of the live settings when the journal was written; empty when + * unknown, which is treated as "cannot prove staleness" and therefore replayable. + * @param liveHash digest of the settings visible right now. + */ + @NonNull + public static Action decide( + boolean rowPresent, + boolean payloadReadable, + @Nullable String targetHash, + @Nullable String baselineHash, + @NonNull String liveHash) { + if (!rowPresent) { + return Action.NOTHING; + } + if (!payloadReadable) { + return Action.QUARANTINE; + } + if (targetHash != null && !targetHash.isEmpty() && targetHash.equals(liveHash)) { + return Action.CLEAR_ALREADY_APPLIED; + } + if (baselineHash == null || baselineHash.isEmpty()) { + // Written before the journal carried identity. Staleness cannot be proven, and the + // journal only exists because a sync meant to apply it, so replay is the safe read. + return Action.REPLAY; + } + return baselineHash.equals(liveHash) ? Action.REPLAY : Action.DISCARD_STALE; + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RemoteSnapshot.java b/app/src/main/java/com/pasich/mynotes/data/sync/RemoteSnapshot.java index bceca2da..ed60b410 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RemoteSnapshot.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RemoteSnapshot.java @@ -3,21 +3,52 @@ import androidx.annotation.NonNull; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; -/** Immutable result of reading a remote causal frontier. */ +/** + * Immutable result of reading a remote causal frontier. + * + *

Also the read context a publish must quote back. {@code writeSnapshot} used to take its causal + * parents from a mutable field on the backend, so a write with no preceding read silently published + * a parentless root that forked the DAG for good. The token here makes that mistake loud. + */ public final class RemoteSnapshot { private final SyncSnapshot snapshot; private final List conflicts; private final List frontierBundleIds; + private final List alternatives; + private final Set resolvedAlternativeIds; + private final String readToken; public RemoteSnapshot( @NonNull SyncSnapshot snapshot, @NonNull List conflicts, @NonNull List frontierBundleIds) { + this( + snapshot, + conflicts, + frontierBundleIds, + Collections.emptyList(), + Collections.emptySet(), + ""); + } + + public RemoteSnapshot( + @NonNull SyncSnapshot snapshot, + @NonNull List conflicts, + @NonNull List frontierBundleIds, + @NonNull List alternatives, + @NonNull Set resolvedAlternativeIds, + @NonNull String readToken) { this.snapshot = snapshot; this.conflicts = Collections.unmodifiableList(new ArrayList<>(conflicts)); this.frontierBundleIds = Collections.unmodifiableList(new ArrayList<>(frontierBundleIds)); + this.alternatives = Collections.unmodifiableList(new ArrayList<>(alternatives)); + this.resolvedAlternativeIds = + Collections.unmodifiableSet(new LinkedHashSet<>(resolvedAlternativeIds)); + this.readToken = readToken; } @NonNull @@ -25,7 +56,36 @@ public static RemoteSnapshot of(@NonNull SyncSnapshot snapshot) { return new RemoteSnapshot(snapshot, Collections.emptyList(), Collections.emptyList()); } - @NonNull public SyncSnapshot getSnapshot() { return snapshot; } - @NonNull public List getConflicts() { return conflicts; } - @NonNull public List getFrontierBundleIds() { return frontierBundleIds; } + @NonNull + public SyncSnapshot getSnapshot() { + return snapshot; + } + + @NonNull + public List getConflicts() { + return conflicts; + } + + @NonNull + public List getFrontierBundleIds() { + return frontierBundleIds; + } + + /** Losing versions the remote state still keeps recoverable. */ + @NonNull + public List getAlternatives() { + return alternatives; + } + + /** Version identities some device has recorded as explicitly resolved. */ + @NonNull + public Set getResolvedAlternativeIds() { + return resolvedAlternativeIds; + } + + /** Opaque proof that this read happened, quoted back by the matching publish. */ + @NonNull + public String getReadToken() { + return readToken; + } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index a1e81b98..0a8d25fb 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -48,6 +48,7 @@ public final class RoomSyncStore implements SyncStore { private static final String PREFS = "sync_state"; private static final String LEGACY_STATE = "last_state"; private static final String PREFERENCES_HASH = "preferences_hash"; + private static final String PREFERENCES_STABLE_ID = "00000000-0000-4000-8000-000000000000"; private final AppDatabase database; private final SharedPreferences preferences; private volatile boolean seeded; @@ -129,7 +130,7 @@ private void ensureSeeded() throws IOException { new SyncMetadataEntity( SyncMetadata.RECORD_TYPE_PREFERENCES, 0, - "00000000-0000-4000-8000-000000000000", + PREFERENCES_STABLE_ID, 0L, null)); recoverPendingPreferences(); @@ -207,63 +208,82 @@ private void applySnapshotInternal( PreferencesBackup stagedPreferences = selectedPreferences(snapshot); String stagedPreferencesJson = stagedPreferences == null ? null : gson.toJson(stagedPreferences); + String stagedPreferencesTarget = + stagedPreferences == null ? "" : preferencesDigest(stagedPreferences); + String preferencesBaseline = stagedPreferences == null ? "" : livePreferencesDigest(); + SyncRecord preferencesRecord = + snapshot.find(SyncRecord.Type.PREFERENCES, PREFERENCES_STABLE_ID); + long stagedPreferencesUpdatedAt = + preferencesRecord == null ? 0L : preferencesRecord.getUpdatedAt().toEpochMilli(); boolean deferFinalState = stagedPreferences != null && finalState != null; try { database.runInTransaction( () -> { try { - Map byStableId = new HashMap<>(); - for (SyncMetadataEntity metadata : database.syncMetadataDao().getAll()) { - byStableId.put(metadata.recordType + ":" + metadata.stableId, metadata); - } - for (SyncRecord record : snapshot.getRecords()) { - SyncMetadataEntity metadata = - byStableId.get( - record.getType().getWireValue() + ":" + record.getId()); - if (metadata == null && !record.isTombstone()) { - long localId = insertRemoteRecord(record); - if (localId >= 0) { + Map byStableId = new HashMap<>(); + for (SyncMetadataEntity metadata : + database.syncMetadataDao().getAll()) { + byStableId.put( + metadata.recordType + ":" + metadata.stableId, metadata); + } + for (SyncRecord record : snapshot.getRecords()) { + SyncMetadataEntity metadata = + byStableId.get( + record.getType().getWireValue() + + ":" + + record.getId()); + if (metadata == null && !record.isTombstone()) { + long localId = insertRemoteRecord(record); + if (localId >= 0) { + database.syncMetadataDao() + .insertIfAbsent( + new SyncMetadataEntity( + record.getType().getWireValue(), + localId, + record.getId(), + record.getUpdatedAt() + .toEpochMilli(), + null)); + } + transactionFailureInjector.afterRecordApplied(record); + continue; + } + if (metadata == null) continue; + if (record.isTombstone()) { + markDeleted(metadata); + database.syncMetadataDao() + .setVersion( + metadata.recordType, + metadata.localId, + record.getUpdatedAt().toEpochMilli(), + record.getDeletedAt().toEpochMilli()); + transactionFailureInjector.afterRecordApplied(record); + continue; + } + applyPayload(metadata, record.getPayload()); database.syncMetadataDao() - .insertIfAbsent( - new SyncMetadataEntity( - record.getType().getWireValue(), - localId, - record.getId(), - record.getUpdatedAt().toEpochMilli(), - null)); + .setVersion( + metadata.recordType, + metadata.localId, + record.getUpdatedAt().toEpochMilli(), + null); + transactionFailureInjector.afterRecordApplied(record); + } + persistConflicts(conflicts); + if (stagedPreferencesJson != null) { + database.syncPendingPreferencesDao() + .upsert( + new SyncPendingPreferencesEntity( + 1, + stagedPreferencesJson, + stagedPreferencesTarget, + preferencesBaseline, + stagedPreferencesUpdatedAt, + false)); + } + if (finalState != null && !deferFinalState) { + database.syncStateDao().upsert(toEntity(finalState)); } - transactionFailureInjector.afterRecordApplied(record); - continue; - } - if (metadata == null) continue; - if (record.isTombstone()) { - markDeleted(metadata); - database.syncMetadataDao() - .setVersion( - metadata.recordType, - metadata.localId, - record.getUpdatedAt().toEpochMilli(), - record.getDeletedAt().toEpochMilli()); - transactionFailureInjector.afterRecordApplied(record); - continue; - } - applyPayload(metadata, record.getPayload()); - database.syncMetadataDao() - .setVersion( - metadata.recordType, - metadata.localId, - record.getUpdatedAt().toEpochMilli(), - null); - transactionFailureInjector.afterRecordApplied(record); - } - persistConflicts(conflicts); - if (stagedPreferencesJson != null) { - database.syncPendingPreferencesDao() - .upsert(new SyncPendingPreferencesEntity(1, stagedPreferencesJson)); - } - if (finalState != null && !deferFinalState) { - database.syncStateDao().upsert(toEntity(finalState)); - } } catch (IOException error) { throw new SyncRuntimeException(error); } @@ -272,21 +292,93 @@ private void applySnapshotInternal( throw error.ioException; } if (stagedPreferences != null) { - commitPendingPreferences(stagedPreferences); + // The journal is only dropped once the adapter reports a durable commit; a failure + // here leaves it in place for recoverPendingPreferences and keeps the sync state + // retryable rather than claiming success. + commitPendingPreferences(stagedPreferences, stagedPreferencesTarget); database.runInTransaction( () -> { database.syncPendingPreferencesDao().clear(); - if (finalState != null) database.syncStateDao().upsert(toEntity(finalState)); + if (finalState != null) + database.syncStateDao().upsert(toEntity(finalState)); }); } + pruneAttachmentCache(snapshot); + } + + /** + * Drops cached blobs nothing can still need. + * + *

Runs only after the snapshot, its conflicts and any preference journal have all been + * committed, so "still needed" is answered from durable state rather than from work in + * progress. A blob survives if the applied snapshot references it or if any unresolved conflict + * does — a losing version the user has not chosen between yet is exactly the case where + * deleting the bytes would be unrecoverable. + * + *

Best effort by design: this is a space optimization, and correctness must not depend on it + * running, or on it finishing. + */ + private void pruneAttachmentCache(@NonNull SyncSnapshot applied) { + try { + LinkedHashSet required = new LinkedHashSet<>(getAttachmentHashes(applied)); + for (SyncConflictEntity conflict : database.syncConflictDao().getUnresolved()) { + collectConflictAttachmentHashes(conflict.winnerJson, required); + collectConflictAttachmentHashes(conflict.loserJson, required); + } + File dir = new File(context.getFilesDir(), "sync-attachments"); + File[] cached = dir.listFiles(); + if (cached == null) { + return; + } + for (File file : cached) { + String name = file.getName(); + if (!file.isFile() || !name.matches("[0-9a-f]{64}") || required.contains(name)) { + continue; + } + if (!file.delete()) { + Log.w(TAG, "Could not remove the unreferenced cached blob " + name); + } + } + } catch (RuntimeException error) { + Log.w(TAG, "Skipping attachment cache cleanup", error); + } + } + + /** Adds every content hash a stored conflict version references. */ + private void collectConflictAttachmentHashes( + @Nullable String recordJson, @NonNull LinkedHashSet into) { + if (recordJson == null || recordJson.isEmpty()) { + return; + } + try { + JsonObject root = JsonParser.parseString(recordJson).getAsJsonObject(); + JsonObject payload = root.getAsJsonObject("payload"); + if (payload == null) { + return; + } + JsonArray manifest = payload.getAsJsonArray("attachmentsManifest"); + if (manifest != null) { + for (JsonElement element : manifest) { + if (!element.isJsonObject()) continue; + JsonObject entry = element.getAsJsonObject(); + if (entry.has("sha256")) into.add(entry.get("sha256").getAsString()); + } + } + JsonArray hashes = payload.getAsJsonArray("attachmentHashes"); + if (hashes != null) { + for (JsonElement element : hashes) into.add(element.getAsString()); + } + } catch (RuntimeException unreadable) { + // An unreadable conflict row must never authorize a deletion, so fail closed by + // keeping everything: the caller only removes blobs nothing claimed. + throw unreadable; + } } @Nullable - private PreferencesBackup selectedPreferences(@NonNull SyncSnapshot snapshot) throws IOException { - SyncRecord record = - snapshot.find( - SyncRecord.Type.PREFERENCES, - "00000000-0000-4000-8000-000000000000"); + private PreferencesBackup selectedPreferences(@NonNull SyncSnapshot snapshot) + throws IOException { + SyncRecord record = snapshot.find(SyncRecord.Type.PREFERENCES, PREFERENCES_STABLE_ID); if (record == null || record.isTombstone()) return null; try { PreferencesBackup parsed = gson.fromJson(record.getPayload(), PreferencesBackup.class); @@ -299,26 +391,145 @@ private PreferencesBackup selectedPreferences(@NonNull SyncSnapshot snapshot) th } } - /** Completes a previously committed Room journal after process death or adapter failure. */ + /** + * Completes, discards or quarantines a journal left behind by an earlier attempt. + * + *

Three outcomes, decided from the two digests rather than applied blindly: + * + *

    + *
  • the live preferences already match the target — the write did land, clear the journal; + *
  • they still match the baseline — nothing has changed since, so replay is safe; + *
  • they match neither — the user has changed these settings since, and their newer choice + * outranks a stale remote payload, so the journal is dropped without being applied. + *
+ * + *

An unreadable payload is quarantined instead of thrown: this runs from {@code + * ensureSeeded}, which gates snapshot building and the status read alike, so throwing made one + * bad row disable sync permanently. + */ private void recoverPendingPreferences() throws IOException { SyncPendingPreferencesEntity pending = database.syncPendingPreferencesDao().get(); - if (pending == null) return; - PreferencesBackup backup; + + PreferencesBackup backup = null; + if (pending != null) { + try { + backup = gson.fromJson(pending.payloadJson, PreferencesBackup.class); + } catch (RuntimeException unreadable) { + backup = null; + } + if (backup != null && !backup.isCreated()) { + backup = null; + } + } + + PendingPreferencesDecision.Action action = + PendingPreferencesDecision.decide( + pending != null, + backup != null, + pending == null ? null : pending.targetHash, + pending == null ? null : pending.baselineHash, + livePreferencesDigest()); + + switch (action) { + case NOTHING: + return; + case QUARANTINE: + Log.w(TAG, "Quarantining an unreadable pending preferences journal"); + database.runInTransaction(() -> database.syncPendingPreferencesDao().quarantine()); + return; + case CLEAR_ALREADY_APPLIED: + database.runInTransaction(() -> database.syncPendingPreferencesDao().clear()); + return; + case DISCARD_STALE: + Log.w( + TAG, + "Discarding a stale pending preferences journal; local settings changed"); + database.runInTransaction(() -> database.syncPendingPreferencesDao().clear()); + return; + case REPLAY: + default: + String target = + pending.targetHash == null || pending.targetHash.isEmpty() + ? preferencesDigest(backup) + : pending.targetHash; + commitPendingPreferences(backup, target); + database.runInTransaction(() -> database.syncPendingPreferencesDao().clear()); + } + } + + /** + * Applies one journaled preferences payload, failing loudly when it is not durable. + * + * @param expectedDigest digest the live preferences must show afterwards. + */ + private void commitPendingPreferences( + @NonNull PreferencesBackup backup, @NonNull String expectedDigest) throws IOException { + boolean committed; try { - backup = gson.fromJson(pending.payloadJson, PreferencesBackup.class); - if (backup == null || !backup.isCreated()) throw new IOException("Pending preferences are invalid"); + committed = preferenceHelper.commitListPreferences(backup); } catch (RuntimeException error) { - throw new IOException("Pending preferences are invalid", error); + throw new IOException("Could not commit synchronized preferences", error); + } + if (!committed) { + throw new IOException("Could not commit synchronized preferences"); } - commitPendingPreferences(backup); - database.runInTransaction(() -> database.syncPendingPreferencesDao().clear()); + // The digest doubles as the snapshot-build baseline, so recording it here keeps the next + // build from treating a freshly received version as a local edit. + preferences.edit().putString(PREFERENCES_HASH, expectedDigest).commit(); } - private void commitPendingPreferences(@NonNull PreferencesBackup backup) throws IOException { + /** + * Records that the live preferences diverged from the last value sync knows about. + * + *

SharedPreferences has no mutation hook and the settings screens write it directly, so a + * local edit can only be noticed by comparing digests here. It now fires only for a genuine + * local change: the apply and conflict-resolution paths record the digest they committed, so a + * version received from another device is no longer mistaken for a local edit and cannot become + * artificially newer than the version it was received from. + */ + private void noteLocalPreferenceEdit( + @NonNull SyncMetadataEntity metadata, @Nullable PreferencesBackup live) { + String digest = preferencesDigest(live); + String baseline = preferences.getString(PREFERENCES_HASH, null); + if (digest.equals(baseline)) { + return; + } + // No baseline at all means sync has never seen these settings; treating them as a local + // edit is the conservative reading, because the alternative silently loses a fresh + // install's configuration to an older version already on Drive. + database.syncMetadataDao() + .touch(metadata.recordType, metadata.localId, System.currentTimeMillis()); + preferences.edit().putString(PREFERENCES_HASH, digest).commit(); + } + + /** Digest of the preferences currently visible to the app. */ + @NonNull + private String livePreferencesDigest() { + return preferencesDigest(preferenceHelper.getListPreferences()); + } + + /** Stable digest of one preferences payload, used for the journal and the build baseline. */ + @NonNull + private String preferencesDigest(@Nullable PreferencesBackup backup) { + String json = backup == null ? "" : gson.toJson(backup); try { - preferenceHelper.setListPreferences(backup); + return sha256(new java.io.ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))); + } catch (IOException impossible) { + throw new IllegalStateException("Could not digest preferences", impossible); + } + } + + /** Reads a preferences payload, refusing anything that is not a usable settings snapshot. */ + @NonNull + private PreferencesBackup requirePreferences(@NonNull JsonObject payload) throws IOException { + try { + PreferencesBackup parsed = gson.fromJson(payload, PreferencesBackup.class); + if (parsed == null || !parsed.isCreated()) { + throw new IOException("Sync preferences payload is invalid"); + } + return parsed; } catch (RuntimeException error) { - throw new IOException("Could not commit synchronized preferences", error); + throw new IOException("Sync preferences payload is invalid", error); } } @@ -340,13 +551,7 @@ else if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { if (value == null) return null; JsonObject result = gson.toJsonTree(value).getAsJsonObject(); if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(metadata.recordType)) { - String hash = result.toString(); - String previous = preferences.getString(PREFERENCES_HASH, null); - if (!hash.equals(previous)) { - database.syncMetadataDao() - .touch(metadata.recordType, metadata.localId, System.currentTimeMillis()); - preferences.edit().putString(PREFERENCES_HASH, hash).apply(); - } + noteLocalPreferenceEdit(metadata, (PreferencesBackup) value); } if ("task".equals(metadata.recordType)) { JsonElement category = result.get("categoryId"); @@ -503,10 +708,19 @@ public void resolveConflict(long conflictId, @NonNull SyncResolution resolution) if (resolution == SyncResolution.PENDING) return; SyncConflictEntity pending = database.syncConflictDao().getById(conflictId); if (pending == null || pending.resolved) return; + // Resolution is a user-visible mutation. Verify and pin the selected version before its - // conflict row can be marked resolved; a missing blob must leave both the note and conflict - // untouched, including when the winner happens to already be visible in Room. - pinResolvedConflictAttachments(selectRecordForResolution(pending, resolution)); + // conflict row can be marked resolved; a missing blob must leave both the note and the + // conflict untouched, including when the winner happens to already be visible in Room. + SyncRecord selected = selectRecordForResolution(pending, resolution); + pinResolvedConflictAttachments(selected); + + if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(pending.recordType) + && !selected.isTombstone()) { + resolvePreferencesConflict(conflictId, resolution, selected); + return; + } + try { database.runInTransaction( () -> { @@ -515,17 +729,10 @@ public void resolveConflict(long conflictId, @NonNull SyncResolution resolution) if (conflict == null || conflict.resolved) return; long resolvedAt = System.currentTimeMillis(); - boolean keepWinner = - (resolution == SyncResolution.KEEP_LOCAL - && "LOCAL".equals(conflict.winnerSource)) - || (resolution == SyncResolution.KEEP_DRIVE - && "REMOTE".equals(conflict.winnerSource)); - if (!keepWinner) { - try { - applyResolvedRecord(conflict, resolution, resolvedAt); - } catch (IOException error) { - throw new SyncRuntimeException(error); - } + try { + applyResolvedRecord(conflict, resolution, resolvedAt); + } catch (IOException error) { + throw new SyncRuntimeException(error); } database.syncConflictDao() .markResolved(conflictId, resolution.name(), resolvedAt); @@ -535,6 +742,73 @@ public void resolveConflict(long conflictId, @NonNull SyncResolution resolution) } } + /** + * Applies a chosen preferences version through the same journal a snapshot apply uses. + * + *

The old path called {@code applyPayload}, whose preferences branch is a no-op, then marked + * the conflict resolved and bumped the record's timestamp. Nothing was written, the conflict + * left the queue, and the untouched local values — now carrying the newest timestamp — + * overwrote the chosen version on every other device at the next sync. + * + *

The version bump is deliberately in the second phase. Bumping it before the adapter has + * committed would, on a failed write, publish the value the user rejected under a fresh + * timestamp; leaving it until after the commit means a failure changes nothing at all. + */ + private void resolvePreferencesConflict( + long conflictId, @NonNull SyncResolution resolution, @NonNull SyncRecord selected) + throws IOException { + PreferencesBackup chosen = requirePreferences(selected.getPayload()); + String target = preferencesDigest(chosen); + String baseline = livePreferencesDigest(); + String payloadJson = gson.toJson(chosen); + long recordUpdatedAt = selected.getUpdatedAt().toEpochMilli(); + + database.runInTransaction( + () -> { + SyncConflictEntity conflict = database.syncConflictDao().getById(conflictId); + if (conflict == null || conflict.resolved) return; + database.syncPendingPreferencesDao() + .upsert( + new SyncPendingPreferencesEntity( + 1, + payloadJson, + target, + baseline, + recordUpdatedAt, + false)); + }); + + // Throws when the write is not durable, leaving the journal in place and the conflict + // unresolved so the user can try again. + commitPendingPreferences(chosen, target); + + try { + database.runInTransaction( + () -> { + SyncConflictEntity conflict = + database.syncConflictDao().getById(conflictId); + if (conflict == null || conflict.resolved) return; + database.syncPendingPreferencesDao().clear(); + long resolvedAt = System.currentTimeMillis(); + SyncMetadataEntity metadata = + database.syncMetadataDao() + .getByStableId(conflict.recordType, conflict.stableId); + if (metadata != null) { + database.syncMetadataDao() + .setVersion( + conflict.recordType, + metadata.localId, + Math.max(resolvedAt, metadata.updatedAt + 1L), + null); + } + database.syncConflictDao() + .markResolved(conflictId, resolution.name(), resolvedAt); + }); + } catch (RuntimeException error) { + throw new IOException("Could not finalize the resolved preferences conflict", error); + } + } + private void pinResolvedConflictAttachments(@NonNull SyncRecord selected) throws IOException { if (selected.isTombstone() || selected.getType() != SyncRecord.Type.NOTE) return; JsonArray manifest = selected.getPayload().getAsJsonArray("attachmentsManifest"); @@ -547,7 +821,8 @@ private void pinResolvedConflictAttachments(@NonNull SyncRecord selected) throws SyncBundleCodec.AttachmentManifestEntry.fromJson(element.getAsJsonObject()); File source = resolveLocalAttachment(entry.sha256); if (source == null || !isVerifiedAttachmentFile(source, entry.sha256, entry.size)) { - throw new IOException("Required conflict attachment is unavailable: " + entry.sha256); + throw new IOException( + "Required conflict attachment is unavailable: " + entry.sha256); } File cache = attachmentFile(entry.sha256); if (!isVerifiedAttachmentFile(cache, entry.sha256, entry.size)) { @@ -568,6 +843,9 @@ private void persistConflicts(@NonNull List conflicts) conflict.getId(), conflictVersionPairHash(conflict), conflict.getWinnerSource().name(), + conflict.getLoserSource().name(), + conflict.getWinnerVersionId(), + conflict.getLoserVersionId(), conflict.getWinner().canonicalSerializedPayload(), conflict.getLoser().canonicalSerializedPayload(), conflict.getWinner().getUpdatedAt().toEpochMilli(), @@ -600,6 +878,23 @@ private static String conflictVersionPairHash(@NonNull SyncMergeResult.Conflict } } + /** True when {@code resolution} names the version the merge selected. */ + private static boolean keepsWinner( + @NonNull SyncConflictEntity conflict, @NonNull SyncResolution resolution) { + if (resolution == SyncResolution.KEEP_ALTERNATIVE) { + return false; + } + if (resolution == SyncResolution.KEEP_WINNER) { + return true; + } + String wanted = resolution == SyncResolution.KEEP_LOCAL ? "LOCAL" : "REMOTE"; + if (wanted.equals(conflict.winnerSource)) { + return true; + } + // Only select the alternative when it genuinely is the endpoint the user named. + return !wanted.equals(conflict.loserSource); + } + private void applyResolvedRecord( @NonNull SyncConflictEntity conflict, @NonNull SyncResolution resolution, @@ -653,15 +948,20 @@ private void applyResolvedRecord( .setVersion(conflict.recordType, metadata.localId, updatedAt, null); } + /** + * Picks the version the user chose, addressing it by position rather than by origin. + * + *

The deprecated endpoint-addressed values are still mapped, because a stored row may carry + * one, but they can no longer silently select the wrong side: when neither version came from + * the named endpoint — the Drive-vs-Drive case — the deterministic winner is kept rather than + * the alternative, which is what the old expression did by accident. + */ @NonNull private static SyncRecord selectRecordForResolution( @NonNull SyncConflictEntity conflict, @NonNull SyncResolution resolution) throws IOException { - boolean keepWinner = - (resolution == SyncResolution.KEEP_LOCAL && "LOCAL".equals(conflict.winnerSource)) - || (resolution == SyncResolution.KEEP_DRIVE - && "REMOTE".equals(conflict.winnerSource)); - String selectedJson = keepWinner ? conflict.winnerJson : conflict.loserJson; + String selectedJson = + keepsWinner(conflict, resolution) ? conflict.winnerJson : conflict.loserJson; JsonObject root = JsonParser.parseString(selectedJson).getAsJsonObject(); SyncRecord.Type type = SyncRecord.Type.fromWireValue(root.get("type").getAsString()); String id = root.get("id").getAsString(); @@ -684,6 +984,12 @@ private SyncRuntimeException(@NonNull IOException ioException) { } } + @NonNull + @Override + public java.util.Set getResolvedAlternativeIds() { + return new LinkedHashSet<>(database.syncConflictDao().getResolvedVersionIds()); + } + @NonNull @Override public Collection getAttachmentHashes(@NonNull SyncSnapshot snapshot) { @@ -957,17 +1263,17 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio target = new File( folder, - entry.id - + "-" - + entry.sha256 - + "-" - + UUID.randomUUID()); + entry.id + "-" + entry.sha256 + "-" + UUID.randomUUID()); } copyVerifiedAttachment(source, target, entry.sha256, entry.size); } + if (note.getId() <= 0) { + throw new IOException("Cannot restore attachments for an unsaved note"); + } JsonObject attachment = new JsonObject(); - attachment.addProperty( - "url", "file://attachments/note_" + note.getId() + "/" + target.getName()); + // Canonical editorjs:// form, the only shape EditorAttachmentsWebViewClient serves. + // Writing file:// here left every synced attachment unrenderable on the receiver. + attachment.addProperty("url", AttachmentStorage.urlFor(note.getId(), target.getName())); attachment.addProperty("name", displayName); attachment.addProperty("id", entry.id); restored.add(attachment); @@ -976,10 +1282,9 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio } private static boolean isVerifiedAttachmentFile( - @NonNull File file, @NonNull String expectedHash, long expectedSize) throws IOException { - return file.isFile() - && file.length() == expectedSize - && expectedHash.equals(sha256(file)); + @NonNull File file, @NonNull String expectedHash, long expectedSize) + throws IOException { + return file.isFile() && file.length() == expectedSize && expectedHash.equals(sha256(file)); } private static void copyVerifiedAttachment( @@ -1020,7 +1325,8 @@ private static void copyVerifiedAttachment( String actual = hash.toString(); if (copied != expectedSize || !expectedHash.equals(actual)) { if (!temporary.delete()) Log.w(TAG, "Could not remove invalid staged attachment"); - throw new AttachmentIntegrityException("Attachment checksum does not match sync metadata"); + throw new AttachmentIntegrityException( + "Attachment checksum does not match sync metadata"); } if (!temporary.renameTo(target)) { if (!temporary.delete()) Log.w(TAG, "Could not remove uncommitted staged attachment"); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java index e2cfb501..74b04833 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBackend.java @@ -40,9 +40,56 @@ default RemoteSnapshot readSnapshotResult() throws IOException { /** Publishes a complete remote snapshot. Implementations must not expose a partial snapshot. */ void writeSnapshot(@NonNull SyncSnapshot snapshot) throws IOException; + /** + * Publishes a snapshot together with the unresolved conflict versions it must keep alive and + * the read context it was derived from. + * + *

Backends that keep no causal history fall back to the plain snapshot write; the Drive + * backend overrides this so a publish cannot use stale causal parents and cannot drop an + * unresolved alternative on the floor. + */ + default void publish(@NonNull SyncPublication publication) throws IOException { + writeSnapshot(publication.getSnapshot()); + } + /** Returns true when the immutable attachment blob already exists remotely. */ boolean hasAttachment(@NonNull String sha256) throws IOException; + /** + * True only when the remote blob exists and its bytes really do hash to {@code sha256}. + * + *

Separate from {@link #hasAttachment} because a backend may index blobs by a claimed hash + * that has to be checked against the bytes before a bundle can depend on it. Implementations + * are expected to answer this at most once per blob per sync. + */ + default boolean hasVerifiedAttachment(@NonNull String sha256, @Nullable Long expectedSize) + throws IOException { + InputStream content = readAttachment(sha256); + if (content == null) { + return false; + } + java.security.MessageDigest digest; + try { + digest = java.security.MessageDigest.getInstance("SHA-256"); + } catch (java.security.NoSuchAlgorithmException error) { + throw new IOException("SHA-256 is unavailable", error); + } + long size = 0L; + try (InputStream input = content) { + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + digest.update(buffer, 0, read); + size += read; + } + } + StringBuilder actual = new StringBuilder(64); + for (byte value : digest.digest()) { + actual.append(String.format(java.util.Locale.US, "%02x", value & 0xff)); + } + return sha256.equals(actual.toString()) && (expectedSize == null || expectedSize == size); + } + /** * Opens an attachment by its lowercase SHA-256 hash, or returns {@code null} when it is absent. * The caller closes the returned stream. diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 815d3acf..37c0eb16 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -53,6 +53,35 @@ public byte[] encode( @NonNull Instant createdAt, @NonNull Collection parentBundleIds) throws IOException { + return encode( + snapshot, + createdAt, + parentBundleIds, + Collections.emptyList(), + Collections.emptySet()); + } + + /** + * Encodes one bundle, including the conflict versions that are still unresolved. + * + *

A merged descendant used to carry only the deterministic winner, so publishing it made + * every losing version unreachable: the bundles holding them stopped being frontier heads and + * nothing else referenced them. A device that had never seen the conflict could not discover + * it, and a device that had seen it held the only copy. Unresolved alternatives now travel in + * the bundle itself and are carried forward until some device records that the conflict was + * resolved, which makes them replicated durable state rather than one device's local queue. + * + * @param unresolvedAlternatives losing versions that must remain recoverable. + * @param resolvedAlternativeIds version identities a user has explicitly settled. + */ + @NonNull + public byte[] encode( + @NonNull SyncSnapshot snapshot, + @NonNull Instant createdAt, + @NonNull Collection parentBundleIds, + @NonNull Collection unresolvedAlternatives, + @NonNull Collection resolvedAlternativeIds) + throws IOException { JsonObject recordsRoot = new JsonObject(); recordsRoot.add("notes", liveArray(snapshot, SyncRecord.Type.NOTE)); recordsRoot.add("tasks", liveArray(snapshot, SyncRecord.Type.TASK)); @@ -60,8 +89,18 @@ public byte[] encode( recordsRoot.add("categories", liveArray(snapshot, SyncRecord.Type.CATEGORY)); recordsRoot.add("preferences", liveArray(snapshot, SyncRecord.Type.PREFERENCES)); recordsRoot.add("tombstones", tombstones(snapshot)); + List alternatives = dedupeAlternatives(unresolvedAlternatives); + recordsRoot.add("alternatives", alternativeArray(alternatives)); + JsonArray resolved = new JsonArray(); + for (String versionId : new java.util.TreeSet<>(resolvedAlternativeIds)) { + if (!SHA_256.matcher(versionId).matches()) { + throw new IOException("Sync bundle contains an invalid resolved version id"); + } + resolved.add(versionId); + } + recordsRoot.add("resolvedAlternatives", resolved); - JsonArray attachments = collectAttachments(snapshot); + JsonArray attachments = collectAttachments(snapshot, alternatives); byte[] recordBytes = GSON.toJson(recordsRoot).getBytes(StandardCharsets.UTF_8); if (recordBytes.length > SyncBundleValidator.MAX_RECORD_BYTES) { throw new IOException("Sync records exceed the schema-1 size limit"); @@ -125,6 +164,8 @@ public DecodedBundle decode(@NonNull InputStream input) throws IOException { identities, result); parseTombstones(records, identities, result); + List alternatives = parseAlternatives(records, attachmentsById); + java.util.Set resolvedAlternativeIds = parseResolvedAlternatives(records); JsonObject manifest = validated.getManifest(); JsonArray parents = manifest.getAsJsonArray("parentBundleIds"); List parentBundleIds = new ArrayList<>(); @@ -135,7 +176,9 @@ public DecodedBundle decode(@NonNull InputStream input) throws IOException { new SyncSnapshot(result), validated.getAttachmentsByHash(), manifest.get("bundleId").getAsString(), - parentBundleIds); + parentBundleIds, + alternatives, + resolvedAlternativeIds); } private static void writeEntry(ZipOutputStream zip, String name, byte[] bytes) @@ -187,6 +230,48 @@ private static void normalizeNoteAttachmentFields(JsonObject note) throws IOExce } } + /** + * Orders alternatives deterministically and drops exact duplicates. + * + *

Two devices publishing the same alternative must produce byte-identical bundles for the + * duplicate-copy check in the read path to keep working. + */ + @NonNull + private static List dedupeAlternatives( + @NonNull Collection alternatives) { + Map byVersion = new java.util.TreeMap<>(); + for (SyncRecord alternative : alternatives) { + byVersion.putIfAbsent( + alternative.getType().getWireValue() + + ":" + + alternative.getId() + + ":" + + alternative.getCanonicalPayloadHash(), + alternative); + } + return new ArrayList<>(byVersion.values()); + } + + @NonNull + private static JsonArray alternativeArray(@NonNull List alternatives) + throws IOException { + JsonArray array = new JsonArray(); + for (SyncRecord alternative : alternatives) { + JsonObject item = + alternative.isTombstone() ? new JsonObject() : alternative.getPayload(); + item.addProperty("type", alternative.getType().getWireValue()); + item.addProperty("id", alternative.getId()); + item.addProperty("updatedAt", alternative.getUpdatedAt().toString()); + if (alternative.isTombstone()) { + item.addProperty("deletedAt", alternative.getDeletedAt().toString()); + } else if (alternative.getType() == SyncRecord.Type.NOTE) { + normalizeNoteAttachmentFields(item); + } + array.add(item); + } + return array; + } + @NonNull private static JsonArray tombstones(@NonNull SyncSnapshot snapshot) { JsonArray array = new JsonArray(); @@ -202,20 +287,33 @@ private static JsonArray tombstones(@NonNull SyncSnapshot snapshot) { } @NonNull - private static JsonArray collectAttachments(@NonNull SyncSnapshot snapshot) throws IOException { + private static JsonArray collectAttachments( + @NonNull SyncSnapshot snapshot, @NonNull List alternatives) + throws IOException { JsonArray attachments = new JsonArray(); Map seenById = new LinkedHashMap<>(); Map seenByHash = new LinkedHashMap<>(); - for (SyncRecord record : snapshot.getLiveRecords(SyncRecord.Type.NOTE)) { + List notes = new ArrayList<>(snapshot.getLiveRecords(SyncRecord.Type.NOTE)); + // An unresolved alternative is only recoverable if its blobs are described here too. + for (SyncRecord alternative : alternatives) { + if (!alternative.isTombstone() && alternative.getType() == SyncRecord.Type.NOTE) { + notes.add(alternative); + } + } + for (SyncRecord record : notes) { JsonArray manifestEntries = record.getPayload().getAsJsonArray("attachmentsManifest"); if (manifestEntries == null) continue; for (JsonElement element : manifestEntries) { AttachmentManifestEntry attachment = AttachmentManifestEntry.fromJson(element.getAsJsonObject()); - if (seenById.putIfAbsent(attachment.id, attachment) != null) { + AttachmentManifestEntry sameId = seenById.putIfAbsent(attachment.id, attachment); + if (sameId != null && !sameId.sameRemoteFile(attachment)) { + // The same logical attachment may appear in both a live note and one of its + // unresolved alternatives; only differing content is a contradiction. throw new IOException("Two notes reference conflicting attachment metadata"); } - AttachmentManifestEntry previous = seenByHash.putIfAbsent(attachment.sha256, attachment); + AttachmentManifestEntry previous = + seenByHash.putIfAbsent(attachment.sha256, attachment); if (previous != null && !previous.sameRemoteFile(attachment)) { throw new IOException("Two notes reference conflicting attachment metadata"); } @@ -293,6 +391,66 @@ private static void hydrateNoteAttachments( payload.add("attachmentNames", namesByHash); } + @NonNull + private static List parseAlternatives( + @NonNull JsonObject records, + @NonNull Map attachmentsById) + throws IOException { + List alternatives = new ArrayList<>(); + JsonArray array = records.getAsJsonArray("alternatives"); + if (array == null) { + // Written by a client that predates durable alternatives. + return alternatives; + } + for (JsonElement element : array) { + JsonObject item = element.getAsJsonObject(); + SyncRecord.Type type = + SyncRecord.Type.fromWireValue(SyncBundleValidator.requireString(item, "type")); + String id = SyncBundleValidator.requireString(item, "id"); + SyncBundleValidator.validateUuid(id); + Instant updatedAt = Instant.parse(SyncBundleValidator.requireString(item, "updatedAt")); + JsonElement deletedAt = item.get("deletedAt"); + if (deletedAt != null && !deletedAt.isJsonNull()) { + alternatives.add( + SyncRecord.tombstone( + type, id, updatedAt, Instant.parse(deletedAt.getAsString()))); + continue; + } + JsonObject payload = item.deepCopy(); + payload.remove("type"); + payload.remove("id"); + payload.remove("updatedAt"); + payload.remove("deletedAt"); + SyncMetadata.stripDeviceLocalFields(type.getWireValue(), payload); + if (type == SyncRecord.Type.NOTE) { + hydrateNoteAttachments(payload, attachmentsById); + } + alternatives.add(SyncRecord.live(type, id, updatedAt, payload)); + } + return alternatives; + } + + @NonNull + private static java.util.Set parseResolvedAlternatives(@NonNull JsonObject records) + throws IOException { + java.util.Set resolved = new LinkedHashSet<>(); + JsonArray array = records.getAsJsonArray("resolvedAlternatives"); + if (array == null) { + return resolved; + } + for (JsonElement element : array) { + if (element == null || !element.isJsonPrimitive()) { + throw new IOException("Sync bundle contains an invalid resolved version id"); + } + String versionId = element.getAsString(); + if (!SHA_256.matcher(versionId).matches()) { + throw new IOException("Sync bundle contains an invalid resolved version id"); + } + resolved.add(versionId); + } + return resolved; + } + private static void parseTombstones( JsonObject records, LinkedHashSet identities, Collection output) throws IOException { @@ -333,16 +491,35 @@ public static final class DecodedBundle { private final Map attachmentsByHash; private final String bundleId; private final List parentBundleIds; + private final List alternatives; + private final java.util.Set resolvedAlternativeIds; DecodedBundle( @NonNull SyncSnapshot snapshot, @NonNull Map attachmentsByHash, @NonNull String bundleId, - @NonNull List parentBundleIds) { + @NonNull List parentBundleIds, + @NonNull List alternatives, + @NonNull java.util.Set resolvedAlternativeIds) { this.snapshot = snapshot; this.attachmentsByHash = attachmentsByHash; this.bundleId = bundleId; this.parentBundleIds = Collections.unmodifiableList(new ArrayList<>(parentBundleIds)); + this.alternatives = Collections.unmodifiableList(new ArrayList<>(alternatives)); + this.resolvedAlternativeIds = + Collections.unmodifiableSet(new LinkedHashSet<>(resolvedAlternativeIds)); + } + + /** Losing versions this bundle keeps recoverable. */ + @NonNull + public List getAlternatives() { + return alternatives; + } + + /** Version identities some device recorded as explicitly resolved. */ + @NonNull + public java.util.Set getResolvedAlternativeIds() { + return resolvedAlternativeIds; } @NonNull @@ -355,8 +532,15 @@ public Map getAttachmentsByHash() { return attachmentsByHash; } - @NonNull public String getBundleId() { return bundleId; } - @NonNull public List getParentBundleIds() { return parentBundleIds; } + @NonNull + public String getBundleId() { + return bundleId; + } + + @NonNull + public List getParentBundleIds() { + return parentBundleIds; + } } public static final class AttachmentManifestEntry { @@ -438,9 +622,7 @@ JsonObject toJson(boolean includeDisplayName) { } boolean sameRemoteFile(@NonNull AttachmentManifestEntry other) { - return sha256.equals(other.sha256) - && path.equals(other.path) - && size == other.size; + return sha256.equals(other.sha256) && path.equals(other.path) && size == other.size; } } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java index 759d1ce4..083ca977 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleValidator.java @@ -120,6 +120,8 @@ public ValidatedBundle validate(@NonNull InputStream input) throws IOException { attachmentsById, referencedAttachmentIds); recordCount += validateTombstones(records, recordIdentities); + recordCount += validateAlternatives(records, attachmentsById, referencedAttachmentIds); + validateResolvedAlternatives(records); if (recordCount > MAX_RECORD_COUNT) { throw new IOException("Sync bundle exceeds the schema-1 record limit"); } @@ -197,6 +199,76 @@ private static void validateAttachmentReferences( } } + /** + * Validates the unresolved conflict versions a bundle carries. + * + *

Deliberately not folded into the live-record identity set: an alternative is another + * version of a record the bundle already contains, so it shares that record's identity by + * design. What it must not do is reference attachment metadata the manifest lacks, or exceed + * the same payload limits as any other record. + */ + private static long validateAlternatives( + @NonNull JsonObject records, + @NonNull Map attachmentsById, + @NonNull Set referencedAttachmentIds) + throws IOException { + JsonArray array = records.getAsJsonArray("alternatives"); + if (array == null) { + return 0L; + } + if (array.size() > MAX_RECORD_COUNT) { + throw new IOException("Sync bundle exceeds the schema-1 record limit"); + } + Set versions = new LinkedHashSet<>(); + for (JsonElement element : array) { + JsonObject item = element.getAsJsonObject(); + SyncRecord.Type type = SyncRecord.Type.fromWireValue(requireString(item, "type")); + String id = requireString(item, "id"); + validateUuid(id); + Instant updatedAt = parseInstant(requireString(item, "updatedAt"), "updatedAt"); + JsonElement deletedAt = item.get("deletedAt"); + if (deletedAt != null && !deletedAt.isJsonNull()) { + Instant deleted = parseInstant(deletedAt.getAsString(), "deletedAt"); + if (deleted.isBefore(updatedAt)) { + throw new IOException( + "Sync alternative deletedAt must not be before updatedAt"); + } + } + validatePayloadLimits(item); + if (type == SyncRecord.Type.NOTE) { + validateAttachmentReferences(item, attachmentsById, referencedAttachmentIds); + } + if (!versions.add(type.getWireValue() + ":" + id + ":" + item.toString())) { + throw new IOException("Sync bundle contains duplicate conflict alternatives"); + } + } + return array.size(); + } + + private static void validateResolvedAlternatives(@NonNull JsonObject records) + throws IOException { + JsonArray array = records.getAsJsonArray("resolvedAlternatives"); + if (array == null) { + return; + } + if (array.size() > MAX_RECORD_COUNT) { + throw new IOException("Sync bundle exceeds the resolved-version limit"); + } + Set seen = new LinkedHashSet<>(); + for (JsonElement element : array) { + if (element == null || !element.isJsonPrimitive()) { + throw new IOException("Sync bundle contains an invalid resolved version id"); + } + String value = element.getAsString(); + if (!SHA_256.matcher(value).matches()) { + throw new IOException("Sync bundle contains an invalid resolved version id"); + } + if (!seen.add(value)) { + throw new IOException("Sync bundle contains duplicate resolved version ids"); + } + } + } + private static long validateTombstones( @NonNull JsonObject records, @NonNull Set identities) throws IOException { JsonArray array = requireArray(records, "tombstones"); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMergeResult.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMergeResult.java index 5cadef31..8d58866e 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMergeResult.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMergeResult.java @@ -9,7 +9,14 @@ /** Result of a deterministic snapshot merge, including every version that was not selected. */ public final class SyncMergeResult { - /** Indicates the endpoint from which a selected version came. */ + /** + * Where one version came from. + * + *

Recorded per side rather than only for the winner. A conflict between two remote bundle + * heads has no local side at all, and labelling one of them "local" because it happened to be + * the merge accumulator told the user something untrue and made {@code KEEP_LOCAL} apply a + * version that never existed on this device. + */ public enum Source { LOCAL, REMOTE @@ -22,14 +29,17 @@ public static final class Conflict { private final SyncRecord winner; private final SyncRecord loser; private final Source winnerSource; + private final Source loserSource; Conflict( @NonNull SyncRecord winner, @NonNull SyncRecord loser, - @NonNull Source winnerSource) { + @NonNull Source winnerSource, + @NonNull Source loserSource) { this.winner = Objects.requireNonNull(winner, "winner"); this.loser = Objects.requireNonNull(loser, "loser"); this.winnerSource = Objects.requireNonNull(winnerSource, "winnerSource"); + this.loserSource = Objects.requireNonNull(loserSource, "loserSource"); if (winner.getType() != loser.getType() || !winner.getId().equals(loser.getId())) { throw new IllegalArgumentException("A conflict must refer to one record identity"); } @@ -62,6 +72,23 @@ public Source getWinnerSource() { return winnerSource; } + @NonNull + public Source getLoserSource() { + return loserSource; + } + + /** Deterministic identity of the winning version, equal on every device. */ + @NonNull + public String getWinnerVersionId() { + return winner.getCanonicalPayloadHash(); + } + + /** Deterministic identity of the losing version, equal on every device. */ + @NonNull + public String getLoserVersionId() { + return loser.getCanonicalPayloadHash(); + } + public boolean isWinnerTombstone() { return winner.isTombstone(); } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMerger.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMerger.java index 11a3ba73..d639693c 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMerger.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMerger.java @@ -17,6 +17,22 @@ public final class SyncMerger { @NonNull public SyncMergeResult merge(@NonNull SyncSnapshot local, @NonNull SyncSnapshot remote) { + return merge(local, remote, SyncMergeResult.Source.LOCAL, SyncMergeResult.Source.REMOTE); + } + + /** + * Merges two snapshots whose origins are named explicitly. + * + *

Folding several remote bundle heads together is a merge between two remote versions, and + * the conflicts it reports have to say so; the two-argument overload above would otherwise + * label whichever bundle happened to be the accumulator as local. + */ + @NonNull + public SyncMergeResult merge( + @NonNull SyncSnapshot local, + @NonNull SyncSnapshot remote, + @NonNull SyncMergeResult.Source localSource, + @NonNull SyncMergeResult.Source remoteSource) { Objects.requireNonNull(local, "local"); Objects.requireNonNull(remote, "remote"); @@ -41,7 +57,14 @@ public SyncMergeResult merge(@NonNull SyncSnapshot local, @NonNull SyncSnapshot } else if (remoteRecord == null) { merged.put(key, localRecord); } else { - mergeVersions(localRecord, remoteRecord, merged, conflicts, key); + mergeVersions( + localRecord, + remoteRecord, + merged, + conflicts, + key, + localSource, + remoteSource); } } return new SyncMergeResult(new SyncSnapshot(merged.values()), conflicts); @@ -52,16 +75,18 @@ private void mergeVersions( SyncRecord remote, Map merged, ArrayList conflicts, - SyncSnapshot.RecordKey key) { + SyncSnapshot.RecordKey key, + SyncMergeResult.Source localSource, + SyncMergeResult.Source remoteSource) { int timestampComparison = local.getUpdatedAt().compareTo(remote.getUpdatedAt()); if (timestampComparison > 0) { merged.put(key, local); - addConflictWhenDifferent(local, remote, SyncMergeResult.Source.LOCAL, conflicts); + addConflictWhenDifferent(local, remote, localSource, remoteSource, conflicts); return; } if (timestampComparison < 0) { merged.put(key, remote); - addConflictWhenDifferent(remote, local, SyncMergeResult.Source.REMOTE, conflicts); + addConflictWhenDifferent(remote, local, remoteSource, localSource, conflicts); return; } @@ -71,12 +96,10 @@ private void mergeVersions( merged.put(key, local); } else if (localHash.compareTo(remoteHash) < 0) { merged.put(key, local); - conflicts.add( - new SyncMergeResult.Conflict(local, remote, SyncMergeResult.Source.LOCAL)); + conflicts.add(new SyncMergeResult.Conflict(local, remote, localSource, remoteSource)); } else { merged.put(key, remote); - conflicts.add( - new SyncMergeResult.Conflict(remote, local, SyncMergeResult.Source.REMOTE)); + conflicts.add(new SyncMergeResult.Conflict(remote, local, remoteSource, localSource)); } } @@ -84,9 +107,10 @@ private void addConflictWhenDifferent( SyncRecord winner, SyncRecord loser, SyncMergeResult.Source winnerSource, + SyncMergeResult.Source loserSource, ArrayList conflicts) { if (!winner.getCanonicalPayloadHash().equals(loser.getCanonicalPayloadHash())) { - conflicts.add(new SyncMergeResult.Conflict(winner, loser, winnerSource)); + conflicts.add(new SyncMergeResult.Conflict(winner, loser, winnerSource, loserSource)); } } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncPublication.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncPublication.java new file mode 100644 index 00000000..281abc6c --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncPublication.java @@ -0,0 +1,57 @@ +package com.pasich.mynotes.data.sync; + +import androidx.annotation.NonNull; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * Everything one publish must carry, including the read it was derived from. + * + *

Bundling the causal context with the content makes the ordering rule explicit rather than a + * convention: a backend can refuse a publish whose read context is missing or stale instead of + * quietly writing a bundle with the wrong parents. + */ +public final class SyncPublication { + + private final SyncSnapshot snapshot; + private final List unresolvedAlternatives; + private final Set resolvedAlternativeIds; + private final RemoteSnapshot readContext; + + public SyncPublication( + @NonNull SyncSnapshot snapshot, + @NonNull List unresolvedAlternatives, + @NonNull Set resolvedAlternativeIds, + @NonNull RemoteSnapshot readContext) { + this.snapshot = Objects.requireNonNull(snapshot, "snapshot"); + this.unresolvedAlternatives = + Collections.unmodifiableList(new ArrayList<>(unresolvedAlternatives)); + this.resolvedAlternativeIds = + Collections.unmodifiableSet(new LinkedHashSet<>(resolvedAlternativeIds)); + this.readContext = Objects.requireNonNull(readContext, "readContext"); + } + + @NonNull + public SyncSnapshot getSnapshot() { + return snapshot; + } + + @NonNull + public List getUnresolvedAlternatives() { + return unresolvedAlternatives; + } + + @NonNull + public Set getResolvedAlternativeIds() { + return resolvedAlternativeIds; + } + + @NonNull + public RemoteSnapshot getReadContext() { + return readContext; + } +} diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncResolution.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncResolution.java index c12b1a5c..37c82bce 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncResolution.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncResolution.java @@ -1,8 +1,30 @@ package com.pasich.mynotes.data.sync; +/** + * What the user chose for one conflict. + * + *

{@link #KEEP_WINNER} and {@link #KEEP_ALTERNATIVE} address the two versions by their place in + * the conflict rather than by where they came from. The older {@link #KEEP_LOCAL} and {@link + * #KEEP_DRIVE} assumed every conflict had exactly one local and one remote side, which is false for + * a conflict between two Drive bundle heads: whichever version happened to be the merge accumulator + * was labelled local, so "keep my device's version" applied something that had never been on the + * device. They are retained only so already-resolved rows still render. + */ public enum SyncResolution { PENDING, + /** Keep the version the deterministic merge selected. */ + KEEP_WINNER, + /** Keep the other version the merge set aside. */ + KEEP_ALTERNATIVE, + /** + * @deprecated provenance-sensitive; kept for reading historical rows. + */ + @Deprecated KEEP_LOCAL, + /** + * @deprecated provenance-sensitive; kept for reading historical rows. + */ + @Deprecated KEEP_DRIVE; public static SyncResolution fromStoredValue(String value) { @@ -13,4 +35,9 @@ public static SyncResolution fromStoredValue(String value) { return PENDING; } } + + /** True for a choice that names a version rather than an endpoint. */ + public boolean isVersionAddressed() { + return this == KEEP_WINNER || this == KEEP_ALTERNATIVE; + } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java index 1f4befba..e46cb4f0 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java @@ -108,9 +108,36 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { warnAboutClockSkew(remote); SyncMergeResult mergeResult = merger.merge(local, remote); SyncSnapshot merged = mergeResult.getMergedSnapshot(); - java.util.List allConflicts = - new java.util.ArrayList<>(remoteResult.getConflicts()); - allConflicts.addAll(mergeResult.getConflicts()); + + // A choice the user already made must never be offered again, wherever it was made. + java.util.Set settledVersionIds = + new java.util.LinkedHashSet<>(remoteResult.getResolvedAlternativeIds()); + settledVersionIds.addAll(store.getResolvedAlternativeIds()); + + java.util.List allConflicts = new java.util.ArrayList<>(); + for (SyncMergeResult.Conflict conflict : remoteResult.getConflicts()) { + if (!settledVersionIds.contains(conflict.getLoserVersionId())) { + allConflicts.add(conflict); + } + } + for (SyncMergeResult.Conflict conflict : mergeResult.getConflicts()) { + if (!settledVersionIds.contains(conflict.getLoserVersionId())) { + allConflicts.add(conflict); + } + } + + // Every still-open alternative is republished, so a merged descendant can never be + // the thing that makes a losing version unreachable. + Map alternatives = new java.util.LinkedHashMap<>(); + for (SyncMergeResult.Conflict conflict : allConflicts) { + alternatives.putIfAbsent(conflict.getLoserVersionId(), conflict.getLoser()); + } + for (SyncRecord carried : remoteResult.getAlternatives()) { + String versionId = carried.getCanonicalPayloadHash(); + if (!settledVersionIds.contains(versionId)) { + alternatives.putIfAbsent(versionId, carried); + } + } Map expectedSizes = attachmentSizes(merged); // The merged snapshot contains only the deterministic winner. A conflict row is not @@ -121,8 +148,15 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { pinConflictVersion(backend, conflict.getWinner()); pinConflictVersion(backend, conflict.getLoser()); } - if (!snapshotsMatch(merged, remote)) { - backend.writeSnapshot(merged); + + if (needsPublication( + merged, remote, alternatives.keySet(), settledVersionIds, remoteResult)) { + backend.publish( + new SyncPublication( + merged, + new java.util.ArrayList<>(alternatives.values()), + settledVersionIds, + remoteResult)); } SyncState success = SyncState.success(backendIdentifier, clock.instant(), allConflicts.size()); @@ -169,6 +203,30 @@ private void warnAboutClockSkew(@NonNull SyncSnapshot remote) { } } + /** + * Whether the remote state already says everything this sync would say. + * + *

Records alone are not enough: an unchanged record set with a newly discovered alternative, + * or with a conflict the user has just resolved, still has to be published or that information + * exists on one device only. + */ + private static boolean needsPublication( + @NonNull SyncSnapshot merged, + @NonNull SyncSnapshot remote, + @NonNull Collection alternativeVersionIds, + @NonNull java.util.Set settledVersionIds, + @NonNull RemoteSnapshot remoteResult) { + if (!snapshotsMatch(merged, remote)) { + return true; + } + java.util.Set publishedAlternatives = new java.util.LinkedHashSet<>(); + for (SyncRecord alternative : remoteResult.getAlternatives()) { + publishedAlternatives.add(alternative.getCanonicalPayloadHash()); + } + return !publishedAlternatives.equals(new java.util.LinkedHashSet<>(alternativeVersionIds)) + || !remoteResult.getResolvedAlternativeIds().equals(settledVersionIds); + } + private static boolean snapshotsMatch( @NonNull SyncSnapshot first, @NonNull SyncSnapshot second) { Collection firstRecords = first.getRecords(); @@ -200,24 +258,32 @@ private void synchronizeAttachments( for (String hash : hashes) { validateHash(hash); if (store.hasAttachment(hash)) { - if (backend.hasAttachment(hash)) { + Long expectedSize = expectedSizes.get(hash); + // Index lookup only; the bytes are checked once, below. + boolean remotePresent = backend.hasAttachment(hash); + if (remotePresent) { try { - verifyAttachment(hash, expectedSizes.get(hash), store.readAttachment(hash)); + verifyAttachment(hash, expectedSize, store.readAttachment(hash)); } catch (IOException localError) { - // The local copy is missing or corrupt; repair it from the remote blob. + // The local copy is missing or corrupt; repair it from the remote blob, + // which copyVerified refuses to accept unless it hashes correctly. copyVerified( hash, - expectedSizes.get(hash), + expectedSize, backend.readAttachment(hash), store::writeAttachment); } - // Drive is untrusted. A matching appProperty is only a claim, so verify the - // actual remote bytes before a bundle can make that blob durable state. - verifyAttachment(hash, expectedSizes.get(hash), backend.readAttachment(hash)); + // Drive is untrusted: a matching appProperty is only a claim. The blob is + // read and hashed exactly once per sync, and the result is remembered, so + // publishing it into the canonical root does not download it again. + if (!backend.hasVerifiedAttachment(hash, expectedSize)) { + throw new AttachmentIntegrityException( + "Attachment checksum does not match its declared hash"); + } } else { copyVerified( hash, - expectedSizes.get(hash), + expectedSize, store.readAttachment(hash), backend::writeAttachment); } @@ -241,15 +307,10 @@ private void pinConflictVersion(@NonNull SyncBackend backend, @NonNull SyncRecor synchronizeAttachments(backend, snapshot, expectedSizes); for (String hash : store.getAttachmentHashes(snapshot)) { Long expectedSize = expectedSizes.get(hash); - copyVerified( - hash, - expectedSize, - store.readAttachment(hash), - store::writeAttachment); + copyVerified(hash, expectedSize, store.readAttachment(hash), store::writeAttachment); } } - private void verifyAttachment(String hash, Long expectedSize, InputStream source) throws IOException { if (source == null) { @@ -282,7 +343,9 @@ private void copyVerified( InputStream source, AttachmentWriter destination) throws IOException { - Objects.requireNonNull(source, "source"); + if (source == null) { + throw new IOException("Required attachment is unavailable: " + expectedHash); + } try (InputStream input = source; VerifyingInputStream verified = new VerifyingInputStream(input, expectedHash, expectedSize)) { diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java index a0951d69..8a0bd8db 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncStore.java @@ -53,6 +53,17 @@ default void applySnapshot( writeState(finalState); } + /** + * Version identities the user has already settled, so they are never offered again. + * + *

Published with the bundle: a resolution has to retire an alternative on every device, not + * only on the one where the user made the choice. + */ + @NonNull + default java.util.Set getResolvedAlternativeIds() throws IOException { + return java.util.Collections.emptySet(); + } + /** Returns every attachment content hash referenced by {@code snapshot}. */ @NonNull Collection getAttachmentHashes(@NonNull SyncSnapshot snapshot) throws IOException; diff --git a/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java b/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java index 253c91a8..bdb2b843 100644 --- a/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java +++ b/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java @@ -75,7 +75,8 @@ AppDatabase providesAppDatabase(@ApplicationContext Context context) { AppDatabase.MIGRATION_15_16, AppDatabase.MIGRATION_16_17, AppDatabase.MIGRATION_17_18, - AppDatabase.MIGRATION_18_19) + AppDatabase.MIGRATION_18_19, + AppDatabase.MIGRATION_19_20) .build(); } diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleaner.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleaner.java index ffd6b331..e14cf309 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleaner.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleaner.java @@ -4,6 +4,8 @@ import android.content.Context; import android.util.Log; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import com.pasich.mynotes.BuildConfig; @@ -17,19 +19,33 @@ import java.util.Set; /** - * Utility class responsible for maintaining consistency of attachment files. + * Keeps a note's attachment folder consistent with its attachments JSON. * - *

This cleaner keeps the filesystem in sync with the note's attachments JSON: - Parses the - * note's attachment metadata. - Resolves actual file paths inside internal storage. - Deletes - * orphaned files that are no longer referenced by the JSON. + *

The one rule that matters here: a reference this class cannot parse is unknown, never + * absent. Treating an unresolvable reference as an orphan is what turned a URL-scheme + * mismatch into the deletion of every attachment a user owned, so an unresolved reference now + * aborts the whole pass and leaves the folder untouched. Leaving a genuine orphan on disk costs + * bytes; deleting a referenced file costs the file. * - *

Called after successful autosave or manual save of a note. + *

Called after a successful autosave or manual save of a note. */ public class AttachmentCleaner { private static final String TAG = "AttachmentCleaner"; private static final Gson gson = new Gson(); + /** Outcome of one cleanup pass; {@code ABORTED_*} guarantees nothing was deleted. */ + public enum Result { + /** Orphans were considered and any that existed were removed. */ + CLEANED, + /** No attachment folder for this note; nothing to do. */ + NO_FOLDER, + /** The attachments JSON could not be parsed. Nothing was deleted. */ + ABORTED_UNREADABLE_METADATA, + /** At least one reference could not be resolved safely. Nothing was deleted. */ + ABORTED_UNRESOLVED_REFERENCE + } + private static void d(String msg) { if (BuildConfig.DEBUG) Log.d(TAG, msg); } @@ -43,74 +59,76 @@ private static void e(String msg, Throwable t) { } /** - * Performs a cleanup of attachment files for a given note. - * - *

Logic: 1) Parses note.attachments JSON into EditorAttachment models. 2) Collects expected - * filenames referenced by the JSON. 3) Locates the actual attachment directory: - * /files/attachments/note_. 4) Deletes all files that are not referenced (orphans). - * - *

Notes: - Runs silently in production; detailed logs appear only in debug builds. - If the - * attachment folder does not exist, the method exits safely. - Never creates new directories — - * cleanup must not modify the FS structure. + * Removes files in {@code note_} that the note's JSON no longer references. * * @param ctx Application context. - * @param note Source note containing attachments metadata. + * @param note Source note carrying the attachments metadata. */ - public static void cleanup(Context ctx, Note note) { - - if (note == null) return; + public static Result cleanup(Context ctx, Note note) { + if (note == null || ctx == null) return Result.ABORTED_UNREADABLE_METADATA; + return cleanup( + new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR), + note.getId(), + note.getAttachments()); + } + /** + * Filesystem-only core, so the abort rules are exercised by ordinary JVM unit tests. + * + * @param attachmentsRoot the app-private attachment root. + * @param noteId the note whose folder is being cleaned. + * @param attachmentsJson the note's serialized attachment list. + */ + @NonNull + static Result cleanup( + @NonNull File attachmentsRoot, int noteId, @Nullable String attachmentsJson) { + List referenced; try { - d("Cleanup start"); - - String json = note.getAttachments(); Type type = new TypeToken>() {}.getType(); - List list = gson.fromJson(json, type); - if (list == null) list = new ArrayList<>(); - - d("Parsed attachments: " + list.size()); - - int noteId = note.getId(); - - // expected files - Set expected = new HashSet<>(); - for (EditorAttachment att : list) { - try { - File f = AttachmentStorage.resolve(ctx, att); - if (f != null) { - expected.add(f.getName()); - } else { - w("resolve null for url=" + att.url); - } - } catch (Exception ex) { - e("resolve error for " + att.url, ex); - } - } + referenced = gson.fromJson(attachmentsJson, type); + } catch (RuntimeException error) { + // Unparseable metadata says nothing about which files are still needed. + e("Attachments JSON is unreadable; skipping cleanup", error); + return Result.ABORTED_UNREADABLE_METADATA; + } + if (referenced == null) referenced = new ArrayList<>(); - // folder - File folder = - new File(new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR), "note_" + noteId); - if (!folder.exists() || !folder.isDirectory()) { - d("No folder → nothing to clean"); - return; + Set expected = new HashSet<>(); + for (EditorAttachment attachment : referenced) { + if (attachment == null) { + w("Null attachment entry; skipping cleanup"); + return Result.ABORTED_UNRESOLVED_REFERENCE; } - - File[] actualFiles = folder.listFiles(); - if (actualFiles == null) return; - - // delete orphans - for (File f : actualFiles) { - if (!expected.contains(f.getName())) { - boolean deleted = f.delete(); - w("Orphan deleted: " + f.getName() + " → " + deleted); - } + AttachmentUrl parsed = AttachmentUrl.parse(attachment.url); + if (parsed == null) { + w("Unresolvable attachment reference; skipping cleanup"); + return Result.ABORTED_UNRESOLVED_REFERENCE; } + if (parsed.resolveWithin(attachmentsRoot) == null) { + w("Attachment reference escapes the attachment root; skipping cleanup"); + return Result.ABORTED_UNRESOLVED_REFERENCE; + } + expected.add(parsed.getFileName()); + } - d("Cleanup complete"); + File folder = new File(attachmentsRoot, "note_" + noteId); + if (!folder.isDirectory()) { + d("No folder for note_" + noteId); + return Result.NO_FOLDER; + } + File[] actualFiles = folder.listFiles(); + if (actualFiles == null) { + // A directory that will not list is a filesystem fault, not an empty directory. + w("Attachment folder could not be listed; skipping cleanup"); + return Result.ABORTED_UNRESOLVED_REFERENCE; + } - } catch (Exception ex) { - e("cleanup failed", ex); + for (File candidate : actualFiles) { + if (!candidate.isFile() || expected.contains(candidate.getName())) continue; + boolean deleted = candidate.delete(); + w("Orphan deleted: " + candidate.getName() + " -> " + deleted); } + return Result.CLEANED; } public static void deleteAttachmentFolderByNoteId(Context ctx, long noteId) { @@ -119,7 +137,7 @@ public static void deleteAttachmentFolderByNoteId(Context ctx, long noteId) { File folder = new File(base, "note_" + noteId); if (!folder.exists()) { - d("Folder note_" + noteId + " → not found"); + d("Folder note_" + noteId + " -> not found"); return; } diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java index 2b701589..ec814498 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentStorage.java @@ -11,7 +11,6 @@ import com.pasich.mynotes.utils.file.ImageOptimizer; import java.io.File; import java.io.FileOutputStream; -import java.util.List; /** * Utility class for managing note attachments stored in the app's internal storage. Handles @@ -184,41 +183,36 @@ public static File read(Context ctx, EditorAttachment att) { /** * Resolves EditorAttachment.url → real File path inside internal storage. * - *

Expected URL format: file://attachments/note_/filename.ext + *

Canonical URL format: editorjs://attachments/note_/filename.ext + * + *

Legacy file://attachments/... references are still accepted. * * @param ctx app context * @param att attachment model * @return File instance or null on error */ public static File resolve(Context ctx, EditorAttachment att) { - return resolve(ctx, att.url); + return att == null ? null : resolve(ctx, att.url); } public static File resolve(Context ctx, String url) { - try { - Uri uri = Uri.parse(url); - if (!"file".equals(uri.getScheme()) || !ATTACHMENTS_BASE_DIR.equals(uri.getAuthority())) { - return null; - } - List seg = uri.getPathSegments(); - - if (seg.size() != 2 || !seg.get(0).matches("note_[1-9][0-9]*")) return null; - - String folder = seg.get(0); - String name = seg.get(1); - if (name.isEmpty() || name.indexOf('/') >= 0 || name.indexOf('\\') >= 0) return null; - for (int index = 0; index < name.length(); index++) { - if (Character.isISOControl(name.charAt(index))) return null; - } + AttachmentUrl parsed = AttachmentUrl.parse(url); + return parsed == null ? null : parsed.resolveWithin(baseDirPath(ctx)); + } - File root = new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR).getCanonicalFile(); - File resolved = new File(new File(root, folder), name).getCanonicalFile(); - String rootPath = root.getPath() + File.separator; - return resolved.getPath().startsWith(rootPath) ? resolved : null; + /** The app-private attachment root, without creating it. */ + public static File baseDirPath(Context ctx) { + return new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR); + } - } catch (Exception e) { - return null; - } + /** + * Builds the canonical URL for a file this app just wrote into a note's folder. + * + *

Every producer goes through here — the editor upload path and sync restore alike — so a + * reference can never be stored in a shape the WebView interceptor refuses to serve. + */ + public static String urlFor(int noteId, String fileName) { + return AttachmentUrl.canonical(noteId, fileName); } /** diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java new file mode 100644 index 00000000..9a23ec65 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrl.java @@ -0,0 +1,260 @@ +package com.pasich.mynotes.extendedEditor.attach; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * One parsed, validated reference to a note attachment. + * + *

The canonical form is {@code editorjs://attachments/note_<id>/<file>} — the shape + * {@code EditorJSInterface.uploadFile} writes and the only shape {@code + * EditorAttachmentsWebViewClient} serves. {@code file://attachments/...} is accepted as legacy + * input, because sync restore wrote that form for one release, but it is never produced: {@link + * #canonical(int, String)} is the single place a new attachment URL is built. + * + *

Deliberately free of {@code android.net.Uri}. Attachment bytes are deleted on the strength of + * this parse, so it has to be exercised by ordinary JVM unit tests rather than only on a device. + */ +public final class AttachmentUrl { + + /** Scheme the editor and the WebView interceptor agree on. */ + public static final String SCHEME = "editorjs"; + + /** Older scheme kept readable so previously stored references still resolve. */ + public static final String LEGACY_SCHEME = "file"; + + public static final String AUTHORITY = AttachmentStorage.ATTACHMENTS_BASE_DIR; + + private static final Pattern NOTE_FOLDER = Pattern.compile("note_[1-9][0-9]*"); + private static final int MAX_NAME_LENGTH = 255; + private static final char SEPARATOR = 0x5c; // backslash + + private static final String UNRESERVED = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~"; + + private final String noteFolder; + private final String fileName; + + private AttachmentUrl(@NonNull String noteFolder, @NonNull String fileName) { + this.noteFolder = noteFolder; + this.fileName = fileName; + } + + /** The {@code note_} directory this reference lives in. */ + @NonNull + public String getNoteFolder() { + return noteFolder; + } + + /** The decoded file name, guaranteed to be a single safe path segment. */ + @NonNull + public String getFileName() { + return fileName; + } + + /** + * Parses a stored attachment URL, or returns {@code null} when it is not a safe reference. + * + *

{@code null} means "this reference could not be understood". Callers must never read that + * as "this file is an orphan" — see {@link AttachmentCleaner}. + */ + @Nullable + public static AttachmentUrl parse(@Nullable String url) { + if (url == null) { + return null; + } + int schemeEnd = url.indexOf("://"); + if (schemeEnd <= 0) { + return null; + } + String scheme = url.substring(0, schemeEnd).toLowerCase(Locale.ROOT); + if (!SCHEME.equals(scheme) && !LEGACY_SCHEME.equals(scheme)) { + return null; + } + String remainder = url.substring(schemeEnd + 3); + // Strip anything after the path; a query or fragment has no meaning here. + int cut = indexOfAny(remainder, '?', '#'); + if (cut >= 0) { + remainder = remainder.substring(0, cut); + } + String prefix = AUTHORITY + "/"; + if (!remainder.startsWith(prefix)) { + return null; + } + String path = remainder.substring(prefix.length()); + int separator = path.indexOf('/'); + if (separator <= 0 || separator == path.length() - 1) { + return null; + } + String folder = decode(path.substring(0, separator)); + String name = decode(path.substring(separator + 1)); + if (folder == null || name == null) { + return null; + } + if (!NOTE_FOLDER.matcher(folder).matches() || !isSafeSegment(name)) { + return null; + } + return new AttachmentUrl(folder, name); + } + + /** Builds the canonical URL for a file inside a note's attachment folder. */ + @NonNull + public static String canonical(int noteId, @NonNull String fileName) { + if (noteId <= 0) { + throw new IllegalArgumentException("Attachment note id must be positive"); + } + if (!isSafeSegment(fileName)) { + throw new IllegalArgumentException("Attachment file name is not a safe path segment"); + } + return SCHEME + "://" + AUTHORITY + "/note_" + noteId + "/" + encode(fileName); + } + + /** Rebuilds this reference in canonical form, whichever scheme it was read from. */ + @NonNull + public String canonical() { + return SCHEME + "://" + AUTHORITY + "/" + noteFolder + "/" + encode(fileName); + } + + /** + * Resolves this reference against an attachment root, refusing anything that escapes it. + * + *

The segment checks above already forbid separators and {@code ..}, so this is the second + * of two independent guards rather than the only one. + */ + @Nullable + public File resolveWithin(@NonNull File attachmentsRoot) { + try { + File root = attachmentsRoot.getCanonicalFile(); + File resolved = new File(new File(root, noteFolder), fileName).getCanonicalFile(); + String rootPath = root.getPath() + File.separator; + return resolved.getPath().startsWith(rootPath) ? resolved : null; + } catch (IOException | SecurityException error) { + return null; + } + } + + private static int indexOfAny(@NonNull String value, char first, char second) { + for (int index = 0; index < value.length(); index++) { + char current = value.charAt(index); + if (current == first || current == second) { + return index; + } + } + return -1; + } + + /** True only for a name that is exactly one ordinary path segment. */ + static boolean isSafeSegment(@Nullable String name) { + if (name == null) { + return false; + } + String value = name.trim(); + if (!value.equals(name) || value.isEmpty() || value.length() > MAX_NAME_LENGTH) { + return false; + } + if (value.equals(".") || value.equals("..") || value.contains("..")) { + return false; + } + if (value.indexOf('/') >= 0 || value.indexOf(SEPARATOR) >= 0) { + return false; + } + for (int index = 0; index < value.length(); index++) { + if (Character.isISOControl(value.charAt(index))) { + return false; + } + } + return new File(value).getName().equals(value); + } + + /** + * Percent-decodes one path segment as UTF-8, or returns {@code null} when it is malformed. + * + *

Decoding happens before validation on purpose: {@code %2e%2e%2f} has to be rejected as the + * traversal it is, not accepted as an opaque name. + */ + @Nullable + private static String decode(@NonNull String segment) { + if (segment.indexOf('%') < 0) { + return segment; + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(segment.length()); + StringBuilder literal = new StringBuilder(); + for (int index = 0; index < segment.length(); ) { + char current = segment.charAt(index); + if (current != '%') { + literal.append(current); + index++; + continue; + } + flush(literal, bytes); + if (index + 2 >= segment.length()) { + return null; + } + int high = Character.digit(segment.charAt(index + 1), 16); + int low = Character.digit(segment.charAt(index + 2), 16); + if (high < 0 || low < 0) { + return null; + } + bytes.write((byte) ((high << 4) + low)); + index += 3; + } + flush(literal, bytes); + return new String(bytes.toByteArray(), StandardCharsets.UTF_8); + } + + /** Moves buffered literal characters into the byte stream as UTF-8. */ + private static void flush(@NonNull StringBuilder literal, @NonNull ByteArrayOutputStream out) { + if (literal.length() == 0) { + return; + } + byte[] encoded = literal.toString().getBytes(StandardCharsets.UTF_8); + out.write(encoded, 0, encoded.length); + literal.setLength(0); + } + + /** Percent-encodes everything outside the unreserved set, which every decoder agrees on. */ + @NonNull + private static String encode(@NonNull String segment) { + StringBuilder result = new StringBuilder(segment.length()); + for (byte value : segment.getBytes(StandardCharsets.UTF_8)) { + char current = (char) (value & 0xff); + if (UNRESERVED.indexOf(current) >= 0) { + result.append(current); + } else { + result.append('%') + .append(Character.toUpperCase(Character.forDigit((value >> 4) & 0xf, 16))) + .append(Character.toUpperCase(Character.forDigit(value & 0xf, 16))); + } + } + return result.toString(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof AttachmentUrl)) { + return false; + } + AttachmentUrl value = (AttachmentUrl) other; + return noteFolder.equals(value.noteFolder) && fileName.equals(value.fileName); + } + + @Override + public int hashCode() { + return noteFolder.hashCode() * 31 + fileName.hashCode(); + } + + @NonNull + @Override + public String toString() { + return canonical(); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/models/EditorAttachment.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/models/EditorAttachment.java index 2fe6074e..a764edce 100644 --- a/app/src/main/java/com/pasich/mynotes/extendedEditor/models/EditorAttachment.java +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/models/EditorAttachment.java @@ -7,6 +7,7 @@ public class EditorAttachment { /** Immutable logical attachment identity; SHA-256 identifies only the shared blob bytes. */ public String id; + public String url; public String name; public String extension; diff --git a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java index 1177e552..30652cde 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java +++ b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java @@ -538,11 +538,12 @@ private void showConflictDialog(List unresolved) { .setMessage(buildConflictMessage(conflict)) .setNegativeButton(R.string.sync_conflict_later, null) .setNeutralButton( - R.string.sync_conflict_keep_local, - (dialog, which) -> resolveConflict(conflict.id, SyncResolution.KEEP_LOCAL)) + R.string.sync_conflict_keep_winner, + (dialog, which) -> resolveConflict(conflict.id, SyncResolution.KEEP_WINNER)) .setPositiveButton( - R.string.sync_conflict_keep_drive, - (dialog, which) -> resolveConflict(conflict.id, SyncResolution.KEEP_DRIVE)) + R.string.sync_conflict_keep_alternative, + (dialog, which) -> + resolveConflict(conflict.id, SyncResolution.KEEP_ALTERNATIVE)) .show(); } @@ -615,21 +616,29 @@ private CharSequence formatLastSync(@NonNull SyncState state) { private String buildConflictMessage(@NonNull SyncConflictEntity conflict) { return getString( R.string.sync_conflict_version, - getString(R.string.sync_conflict_local_label), - describeConflictPayload( - conflict.recordType, - conflict.winnerSource.equals("LOCAL") - ? conflict.winnerJson - : conflict.loserJson)) - + "\n\n" + versionLabel(1, conflict.winnerSource), + describeConflictPayload(conflict.recordType, conflict.winnerJson)) + + "\n" + getString( R.string.sync_conflict_version, - getString(R.string.sync_conflict_drive_label), - describeConflictPayload( - conflict.recordType, - conflict.winnerSource.equals("REMOTE") - ? conflict.winnerJson - : conflict.loserJson)); + versionLabel(2, conflict.loserSource), + describeConflictPayload(conflict.recordType, conflict.loserJson)); + } + + /** + * Names one side of a conflict by its position and its true origin. + * + *

A conflict between two Drive bundle heads has no local side, so the two versions are + * numbered and each is labelled with where it actually came from. Calling an arbitrary remote + * version "this device" told the user something untrue about data they were about to discard. + */ + @NonNull + private String versionLabel(int position, @NonNull String source) { + int origin = + "LOCAL".equals(source) + ? R.string.sync_conflict_local_label + : R.string.sync_conflict_drive_label; + return getString(R.string.sync_conflict_version_label, position, getString(origin)); } @NonNull diff --git a/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java b/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java index 3607f8c5..12228cb7 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java +++ b/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java @@ -3,5 +3,5 @@ public class DatabaseConstants { public static final String DB_NAME = "MyNotes.db"; - public static final int DB_VERSION = 19; + public static final int DB_VERSION = 20; } diff --git a/app/src/main/res/values-be/strings.xml b/app/src/main/res/values-be/strings.xml index a976e605..16acab3e 100644 --- a/app/src/main/res/values-be/strings.xml +++ b/app/src/main/res/values-be/strings.xml @@ -423,6 +423,9 @@ Вырашыць канфлікты сінхранізацыі (%1$d засталося) Пакінуць лакальную версію Пакінуць версію з Google Drive + Keep version 1 + Keep version 2 + Version %1$d — %2$s Пазней Канфлікт сінхранізацыі вырашаны %1$s • %2$s diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 74f0b1d8..682b7652 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -423,6 +423,9 @@ Synchronisierungskonflikte lösen (%1$d verbleibend) Lokale Version behalten Google-Drive-Version behalten + Keep version 1 + Keep version 2 + Version %1$d — %2$s Später Synchronisierungskonflikt gelöst %1$s • %2$s diff --git a/app/src/main/res/values-en-rGB/strings.xml b/app/src/main/res/values-en-rGB/strings.xml index 0fe8d1ba..05f4fda1 100644 --- a/app/src/main/res/values-en-rGB/strings.xml +++ b/app/src/main/res/values-en-rGB/strings.xml @@ -467,6 +467,9 @@ Resolve sync conflicts (%1$d remaining) Keep local version Keep Google Drive version + Keep version 1 + Keep version 2 + Version %1$d — %2$s Later Sync conflict resolved %1$s • %2$s diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 923b32b2..9560908d 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -424,6 +424,9 @@ Resolver conflictos de sincronización (%1$d restantes) Mantener versión local Mantener versión de Google Drive + Keep version 1 + Keep version 2 + Version %1$d — %2$s Más tarde Conflicto de sincronización resuelto %1$s • %2$s diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index 4c806ad2..5e20f3b2 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -419,6 +419,9 @@ Résoudre les conflits de synchronisation (%1$d restants) Conserver la version locale Conserver la version Google Drive + Keep version 1 + Keep version 2 + Version %1$d — %2$s Plus tard Conflit de synchronisation résolu %1$s • %2$s diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index eedd5983..efef81f5 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -421,6 +421,9 @@ Risolvi i conflitti di sincronizzazione (%1$d rimanenti) Mantieni la versione locale Mantieni la versione Google Drive + Keep version 1 + Keep version 2 + Version %1$d — %2$s Più tardi Conflitto di sincronizzazione risolto %1$s • %2$s diff --git a/app/src/main/res/values-kk/strings.xml b/app/src/main/res/values-kk/strings.xml index 3591ac93..ddd3bcfc 100644 --- a/app/src/main/res/values-kk/strings.xml +++ b/app/src/main/res/values-kk/strings.xml @@ -420,6 +420,9 @@ Синхрондау қайшылықтарын шешу (%1$d қалды) Жергілікті нұсқаны сақтау Google Drive нұсқасын сақтау + Keep version 1 + Keep version 2 + Version %1$d — %2$s Кейінірек Синхрондау қайшылығы шешілді %1$s • %2$s diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index 0e6b9e1b..89ae82c2 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -424,6 +424,9 @@ Rozwiąż konflikty synchronizacji (%1$d pozostało) Zachowaj wersję lokalną Zachowaj wersję z Google Drive + Keep version 1 + Keep version 2 + Version %1$d — %2$s Później Konflikt synchronizacji rozwiązany %1$s • %2$s diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 930589f5..05eeecd3 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -428,6 +428,9 @@ Разрешить конфликты синхронизации (%1$d осталось) Оставить локальную версию Оставить версию из Google Drive + Keep version 1 + Keep version 2 + Version %1$d — %2$s Позже Конфликт синхронизации разрешён %1$s • %2$s diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index 78dcd080..e959553a 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -465,6 +465,9 @@ Розв’язати конфлікти синхронізації (%1$d залишилось) Залишити локальну версію Залишити версію з Google Drive + Keep version 1 + Keep version 2 + Version %1$d — %2$s Пізніше Конфлікт синхронізації розв’язано %1$s • %2$s diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 12e64d5c..925b622c 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -345,6 +345,9 @@ Resolve sync conflicts (%1$d remaining) Keep local Keep Drive + Keep version 1 + Keep version 2 + Version %1$d — %2$s Later Conflict resolved locally %1$s: %2$s diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/ConflictProvenanceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/ConflictProvenanceTest.java new file mode 100644 index 00000000..bdee9155 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/ConflictProvenanceTest.java @@ -0,0 +1,142 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gson.JsonObject; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import org.junit.Test; + +/** + * Where each side of a conflict actually came from. + * + *

The old model recorded one {@code Source} for the winner and inferred the loser's from it. + * That is wrong for a conflict between two Drive bundle heads, where neither side is local: the + * merge accumulator was reported as "this device", so the UI named a version the device had never + * held and {@code KEEP_LOCAL} applied it. + */ +public class ConflictProvenanceTest { + + private static final String NOTE = "550e8400-e29b-41d4-a716-446655440000"; + private static final Instant T10 = Instant.parse("2026-08-31T12:00:10Z"); + private static final Instant T20 = Instant.parse("2026-08-31T12:00:20Z"); + + @Test + public void localVersusDrive_namesOneSideLocalAndTheOtherRemote() { + SyncMergeResult result = + new SyncMerger() + .merge( + snapshot(note(T20, "from the phone")), + snapshot(note(T10, "from Drive"))); + + SyncMergeResult.Conflict conflict = only(result); + assertThat(conflict.getWinnerSource()).isEqualTo(SyncMergeResult.Source.LOCAL); + assertThat(conflict.getLoserSource()).isEqualTo(SyncMergeResult.Source.REMOTE); + } + + @Test + public void driveWinningOverLocal_stillNamesEachSideCorrectly() { + SyncMergeResult result = + new SyncMerger() + .merge( + snapshot(note(T10, "from the phone")), + snapshot(note(T20, "from Drive"))); + + SyncMergeResult.Conflict conflict = only(result); + assertThat(conflict.getWinnerSource()).isEqualTo(SyncMergeResult.Source.REMOTE); + assertThat(conflict.getLoserSource()).isEqualTo(SyncMergeResult.Source.LOCAL); + } + + @Test + public void remoteVersusRemote_neverClaimsAVersionCameFromThisDevice() { + SyncMergeResult result = + new SyncMerger() + .merge( + snapshot(note(T20, "bundle A")), + snapshot(note(T10, "bundle B")), + SyncMergeResult.Source.REMOTE, + SyncMergeResult.Source.REMOTE); + + SyncMergeResult.Conflict conflict = only(result); + assertThat(conflict.getWinnerSource()).isEqualTo(SyncMergeResult.Source.REMOTE); + assertThat(conflict.getLoserSource()).isEqualTo(SyncMergeResult.Source.REMOTE); + } + + @Test + public void aThreeWayMergeReportsEachPairWithItsOwnOrigins() { + // Two Drive heads folded together, then merged against local state. + SyncMerger merger = new SyncMerger(); + SyncMergeResult remoteFold = + merger.merge( + snapshot(note(T20, "bundle A")), + snapshot(note(T10, "bundle B")), + SyncMergeResult.Source.REMOTE, + SyncMergeResult.Source.REMOTE); + SyncMergeResult againstLocal = + merger.merge(snapshot(note(T10, "local edit")), remoteFold.getMergedSnapshot()); + + assertThat(only(remoteFold).getWinnerSource()).isEqualTo(SyncMergeResult.Source.REMOTE); + assertThat(only(remoteFold).getLoserSource()).isEqualTo(SyncMergeResult.Source.REMOTE); + assertThat(only(againstLocal).getWinnerSource()).isEqualTo(SyncMergeResult.Source.REMOTE); + assertThat(only(againstLocal).getLoserSource()).isEqualTo(SyncMergeResult.Source.LOCAL); + } + + @Test + public void twoConflictsForOneRecord_carryDistinctVersionIdentities() { + SyncMerger merger = new SyncMerger(); + SyncMergeResult remoteFold = + merger.merge( + snapshot(note(T20, "bundle A")), + snapshot(note(T10, "bundle B")), + SyncMergeResult.Source.REMOTE, + SyncMergeResult.Source.REMOTE); + SyncMergeResult againstLocal = + merger.merge(snapshot(note(T10, "local edit")), remoteFold.getMergedSnapshot()); + + List both = Arrays.asList(only(remoteFold), only(againstLocal)); + + assertThat(both.get(0).getId()).isEqualTo(both.get(1).getId()); + // Same record, genuinely different version pairs; identities must not collide. + assertThat(both.get(0).getLoserVersionId()).isNotEqualTo(both.get(1).getLoserVersionId()); + assertThat(both.get(0).getWinnerVersionId()).isEqualTo(both.get(1).getWinnerVersionId()); + } + + @Test + public void versionIdentityIsDeterministicAcrossDevices() { + SyncRecord one = note(T10, "same content"); + SyncRecord other = note(T10, "same content"); + + assertThat(one.getCanonicalPayloadHash()).isEqualTo(other.getCanonicalPayloadHash()); + assertThat(one.getCanonicalPayloadHash()) + .isNotEqualTo(note(T10, "different").getCanonicalPayloadHash()); + } + + @Test + public void resolutionValuesAddressVersionsRatherThanEndpoints() { + assertThat(SyncResolution.KEEP_WINNER.isVersionAddressed()).isTrue(); + assertThat(SyncResolution.KEEP_ALTERNATIVE.isVersionAddressed()).isTrue(); + assertThat(SyncResolution.KEEP_LOCAL.isVersionAddressed()).isFalse(); + assertThat(SyncResolution.KEEP_DRIVE.isVersionAddressed()).isFalse(); + // Historical rows still render. + assertThat(SyncResolution.fromStoredValue("KEEP_LOCAL")) + .isEqualTo(SyncResolution.KEEP_LOCAL); + assertThat(SyncResolution.fromStoredValue("NONSENSE")).isEqualTo(SyncResolution.PENDING); + } + + private static SyncMergeResult.Conflict only(SyncMergeResult result) { + assertThat(result.getConflicts()).hasSize(1); + return result.getConflicts().get(0); + } + + private static SyncSnapshot snapshot(SyncRecord record) { + return new SyncSnapshot(java.util.Collections.singletonList(record)); + } + + private static SyncRecord note(Instant updatedAt, String value) { + JsonObject payload = new JsonObject(); + payload.addProperty("title", "Shopping"); + payload.addProperty("value", value); + return SyncRecord.live(SyncRecord.Type.NOTE, NOTE, updatedAt, payload); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java index 862187d3..2d4b132f 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/GoogleDriveSyncBackendTest.java @@ -73,7 +73,7 @@ public void writeSnapshot_createsOwnedFolderBundleAndAttachment() throws Excepti hash, attachmentBytes.length, new ByteArrayInputStream(attachmentBytes)); backend.writeAttachment( hash, attachmentBytes.length, new ByteArrayInputStream(attachmentBytes)); - backend.writeSnapshot(snapshot(hash)); + publish(backend, snapshot(hash)); assertThat(server.ownedFolderCount()).isEqualTo(1); assertThat(server.bundleCount()).isEqualTo(1); @@ -127,13 +127,17 @@ public void writeAttachment_doesNotTrustCorruptObjectTaggedWithExpectedHash() th @Test public void concurrentFirstSync_createsDuplicateRootsThenConvergesWithoutLosingEitherNote() throws Exception { - server.pauseTheNextTwoEmptyRootListings(); GoogleDriveSyncBackend first = backend(); GoogleDriveSyncBackend second = backend(); + // Both devices read the empty account first, which is what makes the publishes concurrent. + RemoteSnapshot firstContext = first.readSnapshotResult(); + RemoteSnapshot secondContext = second.readSnapshotResult(); + server.pauseTheNextTwoEmptyRootListings(); SyncSnapshot firstSnapshot = snapshot(NOTE_ID, null); SyncSnapshot secondSnapshot = snapshot(SECOND_NOTE_ID, null); - Thread firstThread = new Thread(() -> writeUnchecked(first, firstSnapshot)); - Thread secondThread = new Thread(() -> writeUnchecked(second, secondSnapshot)); + Thread firstThread = new Thread(() -> publishUnchecked(first, firstSnapshot, firstContext)); + Thread secondThread = + new Thread(() -> publishUnchecked(second, secondSnapshot, secondContext)); firstThread.start(); secondThread.start(); @@ -147,20 +151,27 @@ public void concurrentFirstSync_createsDuplicateRootsThenConvergesWithoutLosingE assertThat(reconciled.find(SyncRecord.Type.NOTE, NOTE_ID)).isNotNull(); assertThat(reconciled.find(SyncRecord.Type.NOTE, SECOND_NOTE_ID)).isNotNull(); - first.writeSnapshot(reconciled); + publish(first, reconciled); assertThat(second.readSnapshot().find(SyncRecord.Type.NOTE, NOTE_ID)).isNotNull(); assertThat(second.readSnapshot().find(SyncRecord.Type.NOTE, SECOND_NOTE_ID)).isNotNull(); } @Test - public void readSnapshotResult_preservesConflictBetweenConcurrentCausalHeads() throws Exception { + public void readSnapshotResult_preservesConflictBetweenConcurrentCausalHeads() + throws Exception { SyncBundleCodec codec = new SyncBundleCodec(); byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); byte[] first = - codec.encode(snapshotWithTitle("First offline edit"), CLOCK.instant(), Collections.singleton(baseId)); + codec.encode( + snapshotWithTitle("First offline edit"), + CLOCK.instant(), + Collections.singleton(baseId)); byte[] second = - codec.encode(snapshotWithTitle("Second offline edit"), CLOCK.instant(), Collections.singleton(baseId)); + codec.encode( + snapshotWithTitle("Second offline edit"), + CLOCK.instant(), + Collections.singleton(baseId)); server.seedOwnedBundleBytes(base); server.seedOwnedBundleBytes(first); server.seedOwnedBundleBytes(second); @@ -179,13 +190,21 @@ public void readSnapshotResult_descendantSupersedesSiblingHeadsWithoutRepeatingC SyncBundleCodec codec = new SyncBundleCodec(); byte[] base = codec.encode(snapshotWithTitle("Base"), CLOCK.instant()); String baseId = codec.decode(new ByteArrayInputStream(base)).getBundleId(); - byte[] first = codec.encode(snapshotWithTitle("First"), CLOCK.instant(), Collections.singleton(baseId)); + byte[] first = + codec.encode( + snapshotWithTitle("First"), CLOCK.instant(), Collections.singleton(baseId)); String firstId = codec.decode(new ByteArrayInputStream(first)).getBundleId(); - byte[] second = codec.encode(snapshotWithTitle("Second"), CLOCK.instant(), Collections.singleton(baseId)); + byte[] second = + codec.encode( + snapshotWithTitle("Second"), + CLOCK.instant(), + Collections.singleton(baseId)); String secondId = codec.decode(new ByteArrayInputStream(second)).getBundleId(); byte[] descendant = codec.encode( - snapshotWithTitle("Resolved"), CLOCK.instant(), Arrays.asList(firstId, secondId)); + snapshotWithTitle("Resolved"), + CLOCK.instant(), + Arrays.asList(firstId, secondId)); server.seedOwnedBundleBytes(base); server.seedOwnedBundleBytes(first); server.seedOwnedBundleBytes(second); @@ -193,21 +212,482 @@ public void readSnapshotResult_descendantSupersedesSiblingHeadsWithoutRepeatingC RemoteSnapshot remote = backend().readSnapshotResult(); - assertThat(remote.getFrontierBundleIds()).containsExactly( - codec.decode(new ByteArrayInputStream(descendant)).getBundleId()); + assertThat(remote.getFrontierBundleIds()) + .containsExactly(codec.decode(new ByteArrayInputStream(descendant)).getBundleId()); assertThat(remote.getConflicts()).isEmpty(); - assertThat(remote.getSnapshot().find(SyncRecord.Type.NOTE, NOTE_ID).getPayload().get("title").getAsString()) + assertThat( + remote.getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID) + .getPayload() + .get("title") + .getAsString()) .isEqualTo("Resolved"); } + // ---------------------------------------------------------------- resumable uploads + + @Test + public void resumableUpload_completesWhenTheFirstChunkIsOnlyPartiallyAcknowledged() + throws Exception { + byte[] payload = payloadOfBytes(600 * 1024); + String hash = sha256(payload); + server.acceptOnlyNextChunkBytes(100_000); + + backend().writeAttachment(hash, payload.length, new ByteArrayInputStream(payload)); + + assertThat(server.attachmentContent(hash)).isEqualTo(payload); + assertThat(server.rejectedChunkRanges()).isEmpty(); + } + + @Test + public void resumableUpload_completesAcrossSeveralPartialAcknowledgements() throws Exception { + byte[] payload = payloadOfBytes(700 * 1024); + String hash = sha256(payload); + server.acceptOnlyNextChunkBytes(1); + server.acceptOnlyNextChunkBytes(50_000); + server.acceptOnlyNextChunkBytes(3); + server.acceptOnlyNextChunkBytes(200_000); + + backend().writeAttachment(hash, payload.length, new ByteArrayInputStream(payload)); + + assertThat(server.attachmentContent(hash)).isEqualTo(payload); + assertThat(server.rejectedChunkRanges()).isEmpty(); + } + + @Test + public void resumableUpload_completesOnExactChunkBoundaries() throws Exception { + byte[] payload = payloadOfBytes(512 * 1024); + String hash = sha256(payload); + + backend().writeAttachment(hash, payload.length, new ByteArrayInputStream(payload)); + + assertThat(server.attachmentContent(hash)).isEqualTo(payload); + assertThat(server.rejectedChunkRanges()).isEmpty(); + } + + @Test + public void resumableUpload_rejectsAnAcknowledgementThatMovesBackwards() throws Exception { + byte[] payload = payloadOfBytes(600 * 1024); + String hash = sha256(payload); + server.acceptOnlyNextChunkBytes(200_000); + server.reportNextChunkRangeEnd(1_000); + + IOException failure = assertUploadFails(hash, payload); + + assertThat(failure).hasMessageThat().contains("backwards"); + assertThat(server.attachmentContent(hash)).isNull(); + } + + @Test + public void resumableUpload_rejectsAnAcknowledgementBeyondTheDeclaredSize() throws Exception { + byte[] payload = payloadOfBytes(300 * 1024); + String hash = sha256(payload); + server.reportNextChunkRangeEnd(payload.length + 5_000L); + + IOException failure = assertUploadFails(hash, payload); + + assertThat(failure).hasMessageThat().contains("more bytes than the attachment declares"); + assertThat(server.attachmentContent(hash)).isNull(); + } + + @Test + public void resumableUpload_rejectsAnAcknowledgementOfBytesThatWereNeverSent() + throws Exception { + // Inside the declared size, but past the end of the 256 KiB range actually sent. + byte[] payload = payloadOfBytes(600 * 1024); + String hash = sha256(payload); + server.reportNextChunkRangeEnd(400_000L); + + IOException failure = assertUploadFails(hash, payload); + + assertThat(failure).hasMessageThat().contains("never sent"); + assertThat(server.attachmentContent(hash)).isNull(); + } + + @Test + public void resumableUpload_failsRatherThanSpinWhenDriveStopsMakingProgress() throws Exception { + byte[] payload = payloadOfBytes(300 * 1024); + String hash = sha256(payload); + // Every PUT answered with a 308 that commits nothing at all. + for (int index = 0; index < 6; index++) { + server.acceptOnlyNextChunkBytes(0); + } + + IOException failure = assertUploadFails(hash, payload); + + assertThat(failure).hasMessageThat().contains("stopped making progress"); + assertThat(server.attachmentContent(hash)).isNull(); + } + + @Test + public void resumableUpload_recoversFromATransientServerErrorBetweenChunks() throws Exception { + byte[] payload = payloadOfBytes(600 * 1024); + String hash = sha256(payload); + server.acceptOnlyNextChunkBytes(120_000); + server.failNextChunk(503); + + backend().writeAttachment(hash, payload.length, new ByteArrayInputStream(payload)); + + assertThat(server.attachmentContent(hash)).isEqualTo(payload); + assertThat(server.rejectedChunkRanges()).isEmpty(); + } + + @Test + public void resumableUpload_neverCommitsWrongBytesWhenTheConnectionDropsBetweenChunks() + throws Exception { + byte[] payload = payloadOfBytes(600 * 1024); + String hash = sha256(payload); + server.acceptOnlyNextChunkBytes(90_000); + server.dropNextChunkConnection(); + + try { + backend().writeAttachment(hash, payload.length, new ByteArrayInputStream(payload)); + } catch (IOException recoveredOrFailed) { + // Either outcome is acceptable here; a committed blob with wrong bytes is not. + } + + byte[] stored = server.attachmentContent(hash); + if (stored != null) { + assertThat(stored).isEqualTo(payload); + } + assertThat(server.rejectedChunkRanges()).isEmpty(); + } + + @Test + public void resumableUpload_abortsWhenTheThreadIsInterrupted() throws Exception { + byte[] payload = payloadOfBytes(600 * 1024); + String hash = sha256(payload); + + Thread.currentThread().interrupt(); + try { + backend().writeAttachment(hash, payload.length, new ByteArrayInputStream(payload)); + throw new AssertionError("Expected an interrupted upload to fail"); + } catch (IOException expected) { + assertThat(expected).isInstanceOf(java.io.InterruptedIOException.class); + } finally { + Thread.interrupted(); + } + + assertThat(server.attachmentContent(hash)).isNull(); + } + + @Test + public void resumableUpload_rejectsASourceShorterThanItsDeclaredSize() throws Exception { + byte[] payload = payloadOfBytes(600 * 1024); + String hash = sha256(payload); + byte[] truncated = Arrays.copyOf(payload, 300 * 1024); + + try { + backend().writeAttachment(hash, payload.length, new ByteArrayInputStream(truncated)); + throw new AssertionError("Expected a short source to fail"); + } catch (IOException expected) { + assertThat(expected).hasMessageThat().contains("ended before its declared size"); + } + + assertThat(server.attachmentContent(hash)).isNull(); + } + + @Test + public void resumableUpload_rejectsASourceLongerThanItsDeclaredSize() throws Exception { + byte[] declared = payloadOfBytes(300 * 1024); + byte[] actual = payloadOfBytes(400 * 1024); + String hash = sha256(declared); + + try { + backend().writeAttachment(hash, declared.length, new ByteArrayInputStream(actual)); + throw new AssertionError("Expected an oversized source to fail"); + } catch (IOException expected) { + assertThat(expected).hasMessageThat().contains("exceeds its declared size"); + } + } + + // ---------------------------------------------------------------- zero-byte attachments + + @Test + public void writeAttachment_publishesAZeroByteBlobThatIsReadableAgain() throws Exception { + byte[] empty = new byte[0]; + String hash = sha256(empty); + assertThat(hash) + .isEqualTo("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + + GoogleDriveSyncBackend backend = backend(); + backend.writeAttachment(hash, 0L, new ByteArrayInputStream(empty)); + + assertThat(server.attachmentContent(hash)).isEqualTo(empty); + assertThat(backend.hasAttachment(hash)).isTrue(); + try (java.io.InputStream restored = backend.readAttachment(hash)) { + assertThat(restored).isNotNull(); + assertThat(readAll(restored)).isEqualTo(empty); + } + } + + @Test + public void writeAttachment_rejectsANonEmptySourceDeclaredAsZeroBytes() throws Exception { + String hash = sha256(new byte[0]); + + try { + backend().writeAttachment(hash, 0L, new ByteArrayInputStream(new byte[] {1})); + throw new AssertionError("Expected a non-empty source declared as empty to fail"); + } catch (IOException expected) { + assertThat(expected).hasMessageThat().contains("exceeds its declared size"); + } + } + + private IOException assertUploadFails(String hash, byte[] payload) { + try { + backend().writeAttachment(hash, payload.length, new ByteArrayInputStream(payload)); + } catch (IOException failure) { + return failure; + } + throw new AssertionError("Expected the resumable upload to fail"); + } + + private static byte[] payloadOfBytes(int size) { + byte[] payload = new byte[size]; + for (int index = 0; index < size; index++) { + payload[index] = (byte) ((index * 31 + 7) & 0xff); + } + return payload; + } + + // ------------------------------------------------- durable unresolved conflicts + + @Test + public void aFreshDeviceStillDiscoversAnUnresolvedConflictAfterAMergedDescendant() + throws Exception { + // Device A publishes its version. + MemoryStore deviceA = new MemoryStore(note(NOTE_ID, T10, "written on A")); + assertThat(sync(deviceA).getStatus()).isEqualTo(SyncState.Status.SUCCESS); + + // Device B has its own concurrent edit of the same note, merges, and publishes the + // descendant. Before this change that descendant carried only the winner. + MemoryStore deviceB = new MemoryStore(note(NOTE_ID, T20, "written on B")); + assertThat(sync(deviceB).getStatus()).isEqualTo(SyncState.Status.SUCCESS); + assertThat(deviceB.conflicts).hasSize(1); + + // Device D is brand new: empty local database, no knowledge of either edit. + MemoryStore deviceD = new MemoryStore(); + assertThat(sync(deviceD).getStatus()).isEqualTo(SyncState.Status.SUCCESS); + + // The deterministic winner is visible... + SyncRecord winner = deviceD.snapshot.find(SyncRecord.Type.NOTE, NOTE_ID); + assertThat(winner).isNotNull(); + assertThat(winner.getPayload().get("value").getAsString()).isEqualTo("written on B"); + + // ...and the losing version is still recoverable, with identity enough to resolve it. + assertThat(deviceD.conflicts).hasSize(1); + SyncMergeResult.Conflict recovered = deviceD.conflicts.get(0); + assertThat(recovered.getLoser().getPayload().get("value").getAsString()) + .isEqualTo("written on A"); + assertThat(recovered.getLoserVersionId()).isNotEmpty(); + assertThat(recovered.getWinnerSource()).isEqualTo(SyncMergeResult.Source.REMOTE); + assertThat(recovered.getLoserSource()).isEqualTo(SyncMergeResult.Source.REMOTE); + } + + @Test + public void resolvingAConflictRetiresItForEveryOtherDevice() throws Exception { + MemoryStore deviceA = new MemoryStore(note(NOTE_ID, T10, "written on A")); + sync(deviceA); + MemoryStore deviceB = new MemoryStore(note(NOTE_ID, T20, "written on B")); + sync(deviceB); + assertThat(deviceB.conflicts).hasSize(1); + + // The user settles it on B, which records both versions as resolved. + deviceB.resolved.add(deviceB.conflicts.get(0).getWinnerVersionId()); + deviceB.resolved.add(deviceB.conflicts.get(0).getLoserVersionId()); + deviceB.conflicts.clear(); + sync(deviceB); + + MemoryStore deviceD = new MemoryStore(); + sync(deviceD); + + assertThat(deviceD.snapshot.find(SyncRecord.Type.NOTE, NOTE_ID)).isNotNull(); + assertThat(deviceD.conflicts).isEmpty(); + } + + @Test + public void anUnresolvedAlternativeSurvivesSeveralUnrelatedPublishes() throws Exception { + MemoryStore deviceA = new MemoryStore(note(NOTE_ID, T10, "written on A")); + sync(deviceA); + MemoryStore deviceB = new MemoryStore(note(NOTE_ID, T20, "written on B")); + sync(deviceB); + + // Three more publishes, each adding a note of its own so nothing else conflicts. + for (int round = 0; round < 3; round++) { + MemoryStore other = + new MemoryStore( + note( + "6ba7b810-9dad-11d1-80b4-00c04fd4300" + round, + T20.plusSeconds(round + 1), + "unrelated " + round)); + SyncState roundState = sync(other); + assertThat(roundState.getErrorMessage()).isNull(); + assertThat(roundState.getStatus()).isEqualTo(SyncState.Status.SUCCESS); + } + + MemoryStore deviceD = new MemoryStore(); + sync(deviceD); + + assertThat(deviceD.conflicts).hasSize(1); + assertThat(deviceD.conflicts.get(0).getLoser().getPayload().get("value").getAsString()) + .isEqualTo("written on A"); + } + + @Test + public void publishingWithoutAPrecedingReadIsRefused() throws Exception { + GoogleDriveSyncBackend backend = backend(); + + try { + backend.writeSnapshot(snapshot(NOTE_ID, null)); + throw new AssertionError("Expected a publish with no read context to be refused"); + } catch (IOException expected) { + assertThat(expected).hasMessageThat().contains("read context"); + } + } + + @Test + public void publishingWithAStaleReadContextIsRefused() throws Exception { + GoogleDriveSyncBackend backend = backend(); + RemoteSnapshot stale = backend.readSnapshotResult(); + // Something else reads through the same backend, so the earlier context is no longer + // the one describing remote state. + backend.readSnapshotResult(); + + try { + backend.publish( + new SyncPublication( + snapshot(NOTE_ID, null), + Collections.emptyList(), + Collections.emptySet(), + stale)); + throw new AssertionError("Expected a stale read context to be refused"); + } catch (IOException expected) { + assertThat(expected).hasMessageThat().contains("latest remote read"); + } + } + + @Test + public void aDeletedAncestorBundleDoesNotBreakSyncOrLoseAnAlternative() throws Exception { + MemoryStore deviceA = new MemoryStore(note(NOTE_ID, T10, "written on A")); + sync(deviceA); + MemoryStore deviceB = new MemoryStore(note(NOTE_ID, T20, "written on B")); + sync(deviceB); + assertThat(server.bundleCount()).isEqualTo(2); + + // The oldest bundle is now only an ancestor: its content lives on in the descendant. + assertThat(server.deleteOldestBundle()).isTrue(); + + MemoryStore deviceD = new MemoryStore(); + SyncState state = sync(deviceD); + + assertThat(state.getErrorMessage()).isNull(); + assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); + assertThat(deviceD.snapshot.find(SyncRecord.Type.NOTE, NOTE_ID)).isNotNull(); + // The losing version travels in the descendant, so removing the ancestor loses nothing. + assertThat(deviceD.conflicts).hasSize(1); + assertThat(deviceD.conflicts.get(0).getLoser().getPayload().get("value").getAsString()) + .isEqualTo("written on A"); + } + + private static final java.time.Instant T10 = java.time.Instant.parse("2026-08-31T12:00:10Z"); + private static final java.time.Instant T20 = java.time.Instant.parse("2026-08-31T12:00:20Z"); + + private SyncState sync(MemoryStore store) { + return new SyncService(store, new SyncMerger(), CLOCK).sync(backend()); + } + + private static SyncRecord note(String id, java.time.Instant updatedAt, String value) { + JsonObject payload = new JsonObject(); + payload.addProperty("title", "Shopping"); + payload.addProperty("value", value); + return SyncRecord.live(SyncRecord.Type.NOTE, id, updatedAt, payload); + } + + /** One device's durable state: its records, its conflict queue and its settled versions. */ + private static final class MemoryStore implements SyncStore { + private SyncSnapshot snapshot; + private final List conflicts = new ArrayList<>(); + private final java.util.Set resolved = new java.util.LinkedHashSet<>(); + private SyncState state = SyncState.idle(); + + MemoryStore(SyncRecord... records) { + snapshot = new SyncSnapshot(Arrays.asList(records)); + } + + @Override + public SyncSnapshot readSnapshot() { + return snapshot; + } + + @Override + public void applySnapshot(SyncSnapshot snapshot, List conflicts) { + this.snapshot = snapshot; + for (SyncMergeResult.Conflict conflict : conflicts) { + if (!resolved.contains(conflict.getLoserVersionId())) { + this.conflicts.add(conflict); + } + } + } + + @Override + public java.util.Set getResolvedAlternativeIds() { + return resolved; + } + + @Override + public java.util.Collection getAttachmentHashes(SyncSnapshot snapshot) { + return Collections.emptyList(); + } + + @Override + public boolean hasAttachment(String sha256) { + return false; + } + + @Override + public java.io.InputStream readAttachment(String sha256) { + return new ByteArrayInputStream(new byte[0]); + } + + @Override + public void writeAttachment(String sha256, long sizeBytes, java.io.InputStream content) {} + + @Override + public SyncState readState() { + return state; + } + + @Override + public void writeState(SyncState state) { + this.state = state; + } + } + + /** + * Publishes the way {@code SyncService} does: read first, then publish quoting that read. + * + *

{@code writeSnapshot} on its own is refused now, because taking causal parents from a + * mutable field let a write with no preceding read fork the bundle DAG permanently. + */ + private static void publish(GoogleDriveSyncBackend backend, SyncSnapshot snapshot) + throws IOException { + RemoteSnapshot context = backend.readSnapshotResult(); + backend.publish( + new SyncPublication( + snapshot, Collections.emptyList(), Collections.emptySet(), context)); + } + private GoogleDriveSyncBackend backend() { return new GoogleDriveSyncBackend( "token", server.apiBase(), server.uploadBase(), CLOCK, new SyncBundleCodec()); } - private static void writeUnchecked(GoogleDriveSyncBackend backend, SyncSnapshot snapshot) { + private static void publishUnchecked( + GoogleDriveSyncBackend backend, SyncSnapshot snapshot, RemoteSnapshot context) { try { - backend.writeSnapshot(snapshot); + backend.publish( + new SyncPublication( + snapshot, Collections.emptyList(), Collections.emptySet(), context)); } catch (IOException error) { throw new AssertionError(error); } @@ -267,7 +747,7 @@ public void writeSnapshot_copiesDuplicateRootAttachmentsIntoTheCanonicalRoot() new SyncBundleCodec()); SyncSnapshot merged = backend.readSnapshot(); - backend.writeSnapshot(merged); + publish(backend, merged); assertThat(server.ownedAttachmentCountInCanonicalRoot(firstHash)).isEqualTo(1); assertThat(server.ownedAttachmentCountInCanonicalRoot(secondHash)).isEqualTo(1); @@ -298,7 +778,7 @@ public void writeSnapshot_createsNewBundleWhenLegacyBundleChanges() throws Excep server.forceConcurrentBundleUpdate( snapshot("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc")); - backend.writeSnapshot(snapshot(NOTE_ID, null)); + publish(backend, snapshot(NOTE_ID, null)); assertThat(server.bundleCount()).isEqualTo(3); } @@ -316,7 +796,7 @@ public void writeSnapshot_preservesUpdateThatArrivesBetweenReadAndPublish() thro backend.readSnapshot(); server.updateBundleImmediatelyBeforeNextUpload(snapshot(SECOND_NOTE_ID, null)); - backend.writeSnapshot(snapshot(NOTE_ID, null)); + publish(backend, snapshot(NOTE_ID, null)); SyncSnapshot remote = new GoogleDriveSyncBackend( @@ -402,6 +882,10 @@ private static final class FakeDriveServer implements AutoCloseable { private final Map files = new ConcurrentHashMap<>(); private final Map seededAttachmentContent = new LinkedHashMap<>(); private final Map uploadSessions = new ConcurrentHashMap<>(); + private final java.util.Queue scriptedChunks = + new java.util.concurrent.ConcurrentLinkedQueue<>(); + private final List rejectedChunkRanges = + java.util.Collections.synchronizedList(new ArrayList<>()); private volatile boolean running = true; private volatile CyclicBarrier emptyRootListingBarrier; private SyncSnapshot updateBeforeNextPatch; @@ -444,6 +928,41 @@ String uploadBase() { return "http://127.0.0.1:" + serverSocket.getLocalPort() + "/upload/drive/v3/files"; } + /** Commits only the first {@code bytes} of the next chunk, then reports real progress. */ + void acceptOnlyNextChunkBytes(int bytes) { + scriptedChunks.add(ChunkScript.partial(bytes)); + } + + /** Answers the next chunk with an HTTP status and commits nothing. */ + void failNextChunk(int status) { + scriptedChunks.add(ChunkScript.status(status)); + } + + /** Closes the socket mid-chunk without writing a response. */ + void dropNextChunkConnection() { + scriptedChunks.add(ChunkScript.drop()); + } + + /** Answers the next chunk with a 308 carrying a fabricated acknowledged range. */ + void reportNextChunkRangeEnd(long inclusiveEnd) { + scriptedChunks.add(ChunkScript.forcedRange(inclusiveEnd)); + } + + /** Content-Range values the server refused because they did not continue the upload. */ + List rejectedChunkRanges() { + return new ArrayList<>(rejectedChunkRanges); + } + + /** Committed bytes of the attachment blob carrying {@code sha256}, or null. */ + byte[] attachmentContent(String sha256) { + for (DriveFile file : files.values()) { + if (sha256.equals(file.appProperties.get("mynotesAttachmentSha256"))) { + return file.content; + } + } + return null; + } + void pauseTheNextTwoEmptyRootListings() { emptyRootListingBarrier = new CyclicBarrier(2); } @@ -459,6 +978,18 @@ int ownedFolderCount() { return count; } + /** Removes one stored bundle, the way a user tidying Drive or its trash purge would. */ + boolean deleteOldestBundle() { + String oldest = null; + for (Map.Entry entry : files.entrySet()) { + if (!"1".equals(entry.getValue().appProperties.get("mynotesBundle"))) continue; + if (oldest == null || entry.getKey().compareTo(oldest) < 0) { + oldest = entry.getKey(); + } + } + return oldest != null && files.remove(oldest) != null; + } + int bundleCount() { int count = 0; for (DriveFile file : files.values()) { @@ -576,7 +1107,8 @@ void forceConcurrentBundleUpdate(SyncSnapshot snapshot) throws IOException { // Bundles are immutable. Model another device's publication as a sibling, // never as replacement of a durable history object. String parent = file.parents.isEmpty() ? null : file.parents.get(0); - DriveFile sibling = createFile("MyNotes.sync.v1.zip", "application/zip", parent); + DriveFile sibling = + createFile("MyNotes.sync.v1.zip", "application/zip", parent); sibling.appProperties.put("mynotesBundle", "1"); sibling.content = new SyncBundleCodec().encode(snapshot, CLOCK.instant()); return; @@ -733,6 +1265,14 @@ private Response handleResumableChunk(Request request, String sessionId) if (session == null) { return Response.json(404, "{}"); } + ChunkScript script = scriptedChunks.poll(); + if (script != null && script.dropConnection) { + throw new IOException("Fake Drive dropped the connection mid-chunk"); + } + if (script != null && script.status > 0) { + return Response.json(script.status, "{}"); + } + String range = request.headers.get("content-range"); if (range == null) { return Response.json(400, "{}"); @@ -742,14 +1282,29 @@ private Response handleResumableChunk(Request request, String sessionId) } Matcher matcher = Pattern.compile("bytes (\\d+)-(\\d+)/(\\d+)").matcher(range); if (!matcher.matches() || Long.parseLong(matcher.group(3)) != session.totalBytes) { + rejectedChunkRanges.add(range); return Response.json(400, "{}"); } long start = Long.parseLong(matcher.group(1)); long end = Long.parseLong(matcher.group(2)); if (start != session.data.size() || end - start + 1L != request.body.length) { + // The client tried to continue somewhere other than the first unacknowledged + // byte. Recorded so a test can assert this never happens. + rejectedChunkRanges.add(range); return Response.json(400, "{}"); } - session.data.write(request.body); + + if (script != null && script.forcedRangeInclusiveEnd != null) { + Map headers = new LinkedHashMap<>(); + headers.put("Range", "bytes=0-" + script.forcedRangeInclusiveEnd); + return Response.json(308, "", headers); + } + + int accepted = + script == null || script.acceptBytes < 0 + ? request.body.length + : Math.min(script.acceptBytes, request.body.length); + session.data.write(request.body, 0, accepted); if (session.data.size() < session.totalBytes) { return resumableProgress(session); } @@ -1081,6 +1636,38 @@ private static Response binary(int code, byte[] body, String eTag) { } } + /** One scripted response for the next resumable chunk PUT. */ + private static final class ChunkScript { + private final int acceptBytes; + private final int status; + private final Long forcedRangeInclusiveEnd; + private final boolean dropConnection; + + private ChunkScript( + int acceptBytes, int status, Long forcedRangeInclusiveEnd, boolean dropConnection) { + this.acceptBytes = acceptBytes; + this.status = status; + this.forcedRangeInclusiveEnd = forcedRangeInclusiveEnd; + this.dropConnection = dropConnection; + } + + static ChunkScript partial(int bytes) { + return new ChunkScript(bytes, 0, null, false); + } + + static ChunkScript status(int status) { + return new ChunkScript(-1, status, null, false); + } + + static ChunkScript forcedRange(long inclusiveEnd) { + return new ChunkScript(-1, 0, inclusiveEnd, false); + } + + static ChunkScript drop() { + return new ChunkScript(-1, 0, null, true); + } + } + private static final class UploadSession { private final JsonObject metadata; private final long totalBytes; diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/PendingPreferencesDecisionTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/PendingPreferencesDecisionTest.java new file mode 100644 index 00000000..a8531696 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/data/sync/PendingPreferencesDecisionTest.java @@ -0,0 +1,79 @@ +package com.pasich.mynotes.data.sync; + +import static com.google.common.truth.Truth.assertThat; + +import com.pasich.mynotes.data.sync.PendingPreferencesDecision.Action; +import org.junit.Test; + +/** + * Every crash window around the preferences journal. + * + *

The journal bridges a committed Room transaction to a SharedPreferences write that Room cannot + * roll back, so what happens after an interruption is decided here rather than by replaying + * blindly. + */ +public class PendingPreferencesDecisionTest { + + private static final String BASELINE = "baseline-digest"; + private static final String TARGET = "target-digest"; + + @Test + public void noJournal_doesNothing() { + assertThat(decide(false, true, TARGET, BASELINE, BASELINE)).isEqualTo(Action.NOTHING); + } + + @Test + public void crashBeforeThePreferencesWrite_replaysTheJournal() { + // Room committed, the adapter never ran: the live values are still the baseline. + assertThat(decide(true, true, TARGET, BASELINE, BASELINE)).isEqualTo(Action.REPLAY); + } + + @Test + public void crashAfterCommitButBeforeTheJournalClear_justClearsTheJournal() { + assertThat(decide(true, true, TARGET, BASELINE, TARGET)) + .isEqualTo(Action.CLEAR_ALREADY_APPLIED); + } + + @Test + public void aLocalEditAfterTheJournalWasWritten_discardsTheStaleJournal() { + // The user changed these settings themselves; a stale remote payload must not win. + assertThat(decide(true, true, TARGET, BASELINE, "edited-by-the-user")) + .isEqualTo(Action.DISCARD_STALE); + } + + @Test + public void anUnreadablePayload_isQuarantinedRatherThanFatal() { + assertThat(decide(true, false, TARGET, BASELINE, BASELINE)).isEqualTo(Action.QUARANTINE); + // Quarantine wins even when the digests would otherwise say "replay". + assertThat(decide(true, false, "", "", BASELINE)).isEqualTo(Action.QUARANTINE); + } + + @Test + public void aJournalWrittenBeforeIdentityExisted_isStillReplayed() { + assertThat(decide(true, true, "", "", "anything")).isEqualTo(Action.REPLAY); + } + + @Test + public void anAlreadyAppliedJournalWins_overAMissingBaseline() { + assertThat(decide(true, true, TARGET, "", TARGET)).isEqualTo(Action.CLEAR_ALREADY_APPLIED); + } + + @Test + public void retryingAFailedCommit_replaysWhileTheBaselineStillHolds() { + // First attempt: adapter reported failure, journal intact, values untouched. + assertThat(decide(true, true, TARGET, BASELINE, BASELINE)).isEqualTo(Action.REPLAY); + // Second attempt succeeded, so the following pass only has to clear the row. + assertThat(decide(true, true, TARGET, BASELINE, TARGET)) + .isEqualTo(Action.CLEAR_ALREADY_APPLIED); + } + + private static Action decide( + boolean rowPresent, + boolean payloadReadable, + String targetHash, + String baselineHash, + String liveHash) { + return PendingPreferencesDecision.decide( + rowPresent, payloadReadable, targetHash, baselineHash, liveHash); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java index bdcbbbb7..a333fb3e 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java @@ -132,9 +132,7 @@ public void encode_preservesTwoLogicalAttachmentsThatShareOneBlob() throws Excep .getAsJsonArray("attachmentsManifest")) .hasSize(1); assertThat( - decoded.find( - SyncRecord.Type.NOTE, - "6ba7b812-9dad-11d1-80b4-00c04fd430c8") + decoded.find(SyncRecord.Type.NOTE, "6ba7b812-9dad-11d1-80b4-00c04fd430c8") .getPayload() .getAsJsonArray("attachmentsManifest")) .hasSize(1); diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java index e185d1cb..63aa739b 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java @@ -147,8 +147,8 @@ public void sync_skipsUploadingAttachmentWhenRemoteBlobAlreadyExists() throws Ex SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); - // Drive is untrusted even for a content-addressed object, so remote bytes are verified - // before publication. + // Drive is untrusted even for a content-addressed object, so the remote bytes are still + // verified before publication — but exactly once, not once per question asked about them. assertThat(backend.events).containsExactly("readAttachment", "writeSnapshot"); } @@ -327,6 +327,65 @@ private static String sha256(byte[] bytes) throws Exception { return value.toString(); } + @Test + public void sync_convergesANoteWhoseAttachmentIsZeroBytes() throws Exception { + byte[] empty = new byte[0]; + String hash = sha256(empty); + SyncRecord local = noteWithAttachment(TEN, "Note with an empty file", hash, 0L); + + FakeStore uploader = new FakeStore(snapshot(local)); + uploader.attachmentHashes = Collections.singletonList(hash); + uploader.attachments.put(hash, empty); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + + SyncState published = new SyncService(uploader, new SyncMerger(), CLOCK).sync(backend); + + assertThat(published.getStatus()).isEqualTo(SyncState.Status.SUCCESS); + assertThat(backend.attachments).containsKey(hash); + assertThat(backend.attachments.get(hash)).isEqualTo(empty); + + // A second device starting empty must be able to pull the same blob back. + FakeStore downloader = new FakeStore(SyncSnapshot.empty()); + downloader.attachmentHashes = Collections.singletonList(hash); + + SyncState received = new SyncService(downloader, new SyncMerger(), CLOCK).sync(backend); + + assertThat(received.getStatus()).isEqualTo(SyncState.Status.SUCCESS); + assertThat(downloader.attachments.get(hash)).isEqualTo(empty); + assertThat(downloader.snapshot.getRecords()).containsExactly(local); + } + + @Test + public void sync_refusesAZeroByteAttachmentThatIsMissingEverywhere() throws Exception { + String hash = sha256(new byte[0]); + SyncRecord local = noteWithAttachment(TEN, "Note with an empty file", hash, 0L); + FakeStore store = new FakeStore(snapshot(local)); + store.attachmentHashes = Collections.singletonList(hash); + FakeBackend backend = new FakeBackend(SyncSnapshot.empty()); + + SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); + + assertThat(state.getStatus()).isEqualTo(SyncState.Status.ERROR); + assertThat(backend.writeSnapshotCalls).isEqualTo(0); + } + + private static SyncRecord noteWithAttachment( + java.time.Instant updatedAt, String value, String sha256, long size) { + com.google.gson.JsonObject payload = new com.google.gson.JsonObject(); + payload.addProperty("title", "Shopping"); + payload.addProperty("value", value); + com.google.gson.JsonObject entry = new com.google.gson.JsonObject(); + entry.addProperty("id", "8f1d1b2c-2f3a-4c5d-8e9f-0a1b2c3d4e5f"); + entry.addProperty("sha256", sha256); + entry.addProperty("mimeType", "application/octet-stream"); + entry.addProperty("size", size); + entry.addProperty("path", "attachments/" + sha256); + com.google.gson.JsonArray manifest = new com.google.gson.JsonArray(); + manifest.add(entry); + payload.add("attachmentsManifest", manifest); + return SyncRecord.live(SyncRecord.Type.NOTE, NOTE_ID, updatedAt, payload); + } + private static final class FakeStore implements SyncStore { private SyncSnapshot snapshot; private SnapshotBuildResult snapshotBuildResult; diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleanerTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleanerTest.java new file mode 100644 index 00000000..5e4d0b55 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleanerTest.java @@ -0,0 +1,170 @@ +package com.pasich.mynotes.extendedEditor.attach; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Destructive-cleanup rules. + * + *

Every fixture here uses the production {@code editorjs://attachments/...} shape. A suite built + * on synthetic {@code file://} URLs passed while the app deleted every attachment a user owned, so + * matching production exactly is the point of these tests rather than an incidental detail. + */ +public class AttachmentCleanerTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private File attachmentsRoot; + private File noteFolder; + + @Before + public void setUp() throws Exception { + attachmentsRoot = temporaryFolder.newFolder("attachments"); + noteFolder = new File(attachmentsRoot, "note_42"); + assertThat(noteFolder.mkdirs()).isTrue(); + } + + @Test + public void keepsEveryFileReferencedByProductionEditorUrls() throws Exception { + File first = write("1731000000000_882134.jpg", "first"); + File second = write("1731000000001_991245.pdf", "second"); + + AttachmentCleaner.Result result = + AttachmentCleaner.cleanup( + attachmentsRoot, + 42, + json( + "editorjs://attachments/note_42/1731000000000_882134.jpg", + "editorjs://attachments/note_42/1731000000001_991245.pdf")); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.CLEANED); + assertThat(first.exists()).isTrue(); + assertThat(second.exists()).isTrue(); + assertThat(contentOf(first)).isEqualTo("first"); + assertThat(contentOf(second)).isEqualTo("second"); + } + + @Test + public void deletesOnlyGenuineOrphans() throws Exception { + File referenced = write("1731000000000_882134.jpg", "keep"); + File orphan = write("1731000000009_000001.tmp", "drop"); + + AttachmentCleaner.Result result = + AttachmentCleaner.cleanup( + attachmentsRoot, + 42, + json("editorjs://attachments/note_42/1731000000000_882134.jpg")); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.CLEANED); + assertThat(referenced.exists()).isTrue(); + assertThat(orphan.exists()).isFalse(); + } + + @Test + public void abortsWithoutDeletingWhenOneReferenceCannotBeResolved() throws Exception { + File resolvable = write("1731000000000_882134.jpg", "keep"); + File unrelated = write("1731000000009_000001.tmp", "would-be-orphan"); + + AttachmentCleaner.Result result = + AttachmentCleaner.cleanup( + attachmentsRoot, + 42, + json( + "editorjs://attachments/note_42/1731000000000_882134.jpg", + "totally-broken-reference")); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.ABORTED_UNRESOLVED_REFERENCE); + assertThat(resolvable.exists()).isTrue(); + assertThat(unrelated.exists()).isTrue(); + } + + @Test + public void abortsOnATraversalReferenceWithoutDeletingAnything() throws Exception { + File kept = write("1731000000000_882134.jpg", "keep"); + + AttachmentCleaner.Result result = + AttachmentCleaner.cleanup( + attachmentsRoot, 42, json("editorjs://attachments/note_42/../../escape")); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.ABORTED_UNRESOLVED_REFERENCE); + assertThat(kept.exists()).isTrue(); + } + + @Test + public void abortsOnUnreadableMetadataWithoutDeletingAnything() throws Exception { + File kept = write("1731000000000_882134.jpg", "keep"); + + AttachmentCleaner.Result result = + AttachmentCleaner.cleanup(attachmentsRoot, 42, "{not valid json"); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.ABORTED_UNREADABLE_METADATA); + assertThat(kept.exists()).isTrue(); + } + + @Test + public void keepsSyncRestoredAttachmentsThatUseTheCanonicalUrlBuilder() throws Exception { + String restoredName = + "8f1d1b2c-2f3a-4c5d-8e9f-0a1b2c3d4e5f" + + "-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + File restored = write(restoredName, "restored bytes"); + + // Exactly the URL RoomSyncStore.restoreAttachments now stores. + String url = AttachmentStorage.urlFor(42, restoredName); + + AttachmentCleaner.Result result = AttachmentCleaner.cleanup(attachmentsRoot, 42, json(url)); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.CLEANED); + assertThat(restored.exists()).isTrue(); + assertThat(contentOf(restored)).isEqualTo("restored bytes"); + } + + @Test + public void aRestoredAttachmentIsReachableThroughTheSameParserTheWebViewUses() + throws Exception { + File restored = write("1731000000000_882134.jpg", "rendered bytes"); + String url = AttachmentStorage.urlFor(42, "1731000000000_882134.jpg"); + + AttachmentUrl parsed = AttachmentUrl.parse(url); + + assertThat(parsed).isNotNull(); + assertThat(url).startsWith("editorjs://attachments/"); + assertThat(parsed.resolveWithin(attachmentsRoot)).isEqualTo(restored.getCanonicalFile()); + assertThat(contentOf(parsed.resolveWithin(attachmentsRoot))).isEqualTo("rendered bytes"); + } + + @Test + public void treatsAnEmptyReferenceListAsAFullClean() throws Exception { + File orphan = write("1731000000009_000001.tmp", "drop"); + + AttachmentCleaner.Result result = AttachmentCleaner.cleanup(attachmentsRoot, 42, "[]"); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.CLEANED); + assertThat(orphan.exists()).isFalse(); + } + + private File write(String name, String content) throws Exception { + File file = new File(noteFolder, name); + Files.write(file.toPath(), content.getBytes(StandardCharsets.UTF_8)); + return file; + } + + private static String contentOf(File file) throws Exception { + return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + } + + private static String json(String... urls) { + StringBuilder result = new StringBuilder("["); + for (int index = 0; index < urls.length; index++) { + if (index > 0) result.append(','); + result.append("{\"url\":\"").append(urls[index]).append("\"}"); + } + return result.append(']').toString(); + } +} diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrlTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrlTest.java new file mode 100644 index 00000000..c6ebf550 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrlTest.java @@ -0,0 +1,133 @@ +package com.pasich.mynotes.extendedEditor.attach; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.File; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Parsing rules for stored attachment references. + * + *

Fixtures use the shape the editor actually writes ({@code editorjs://attachments/...}); the + * legacy {@code file://} form is covered separately rather than standing in for production. + */ +public class AttachmentUrlTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void parsesTheProductionEditorUrl() { + AttachmentUrl parsed = + AttachmentUrl.parse("editorjs://attachments/note_146/1731000000000_882134.jpg"); + + assertThat(parsed).isNotNull(); + assertThat(parsed.getNoteFolder()).isEqualTo("note_146"); + assertThat(parsed.getFileName()).isEqualTo("1731000000000_882134.jpg"); + } + + @Test + public void parsesTheLegacyFileUrlAndNormalizesItToTheCanonicalScheme() { + AttachmentUrl parsed = AttachmentUrl.parse("file://attachments/note_7/photo.png"); + + assertThat(parsed).isNotNull(); + assertThat(parsed.canonical()).isEqualTo("editorjs://attachments/note_7/photo.png"); + } + + @Test + public void parsesTheSyncRestoredFileNameShape() { + String name = + "8f1d1b2c-2f3a-4c5d-8e9f-0a1b2c3d4e5f" + + "-" + + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + AttachmentUrl parsed = AttachmentUrl.parse("editorjs://attachments/note_3/" + name); + + assertThat(parsed).isNotNull(); + assertThat(parsed.getFileName()).isEqualTo(name); + } + + @Test + public void decodesPercentEncodedNames() { + AttachmentUrl parsed = + AttachmentUrl.parse("editorjs://attachments/note_2/report%20final.pdf"); + + assertThat(parsed).isNotNull(); + assertThat(parsed.getFileName()).isEqualTo("report final.pdf"); + } + + @Test + public void roundTripsNonAsciiNamesThroughCanonicalForm() { + String canonical = AttachmentUrl.canonical(12, "звіт.pdf"); + AttachmentUrl parsed = AttachmentUrl.parse(canonical); + + assertThat(parsed).isNotNull(); + assertThat(parsed.getFileName()).isEqualTo("звіт.pdf"); + assertThat(canonical).doesNotContain("звіт"); + } + + @Test + public void rejectsTraversalInTheFileName() { + assertThat(AttachmentUrl.parse("editorjs://attachments/note_1/../../secret.txt")).isNull(); + assertThat(AttachmentUrl.parse("editorjs://attachments/note_1/..")).isNull(); + } + + @Test + public void rejectsPercentEncodedTraversal() { + assertThat(AttachmentUrl.parse("editorjs://attachments/note_1/%2e%2e%2fsecret.txt")) + .isNull(); + assertThat(AttachmentUrl.parse("editorjs://attachments/%2e%2e/note_1/x.png")).isNull(); + } + + @Test + public void rejectsForeignSchemesAuthoritiesAndShapes() { + assertThat(AttachmentUrl.parse("https://attachments/note_1/x.png")).isNull(); + assertThat(AttachmentUrl.parse("editorjs://elsewhere/note_1/x.png")).isNull(); + assertThat(AttachmentUrl.parse("editorjs://attachments/notes_1/x.png")).isNull(); + assertThat(AttachmentUrl.parse("editorjs://attachments/note_0/x.png")).isNull(); + assertThat(AttachmentUrl.parse("editorjs://attachments/note_1")).isNull(); + assertThat(AttachmentUrl.parse("editorjs://attachments/note_1/")).isNull(); + assertThat(AttachmentUrl.parse("editorjs://attachments/note_1/a/b.png")).isNull(); + assertThat(AttachmentUrl.parse("/data/data/pkg/files/attachments/note_1/x.png")).isNull(); + assertThat(AttachmentUrl.parse(null)).isNull(); + assertThat(AttachmentUrl.parse("")).isNull(); + } + + @Test + public void rejectsControlCharactersAndMalformedEscapes() { + assertThat(AttachmentUrl.parse("editorjs://attachments/note_1/a%00b.png")).isNull(); + assertThat(AttachmentUrl.parse("editorjs://attachments/note_1/a%zz.png")).isNull(); + assertThat(AttachmentUrl.parse("editorjs://attachments/note_1/a%2.png")).isNull(); + } + + @Test + public void resolvesInsideTheAttachmentRoot() throws Exception { + File root = temporaryFolder.newFolder("attachments"); + File noteFolder = new File(root, "note_5"); + assertThat(noteFolder.mkdirs()).isTrue(); + File file = new File(noteFolder, "photo.png"); + assertThat(file.createNewFile()).isTrue(); + + AttachmentUrl parsed = AttachmentUrl.parse("editorjs://attachments/note_5/photo.png"); + + assertThat(parsed).isNotNull(); + assertThat(parsed.resolveWithin(root)).isEqualTo(file.getCanonicalFile()); + } + + @Test + public void canonicalRefusesToBuildAnUnsafeReference() { + try { + AttachmentUrl.canonical(1, "../escape.png"); + throw new AssertionError("Expected an unsafe file name to be rejected"); + } catch (IllegalArgumentException expected) { + // The single URL producer must not be able to emit a traversal. + } + try { + AttachmentUrl.canonical(0, "photo.png"); + throw new AssertionError("Expected a non-positive note id to be rejected"); + } catch (IllegalArgumentException expected) { + // Note ids are SQLite row ids and start at 1. + } + } +} diff --git a/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java index fb24e1b0..c289aa99 100644 --- a/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncCoordinatorTest.java @@ -313,6 +313,9 @@ public void resolveConflict_updatesStoreAndReturnsLatestConflicts() { "550e8400-e29b-41d4-a716-446655440000", "test-version-pair", "LOCAL", + "REMOTE", + "winner-version-id", + "loser-version-id", "{}", "{}", 1L, @@ -478,6 +481,12 @@ public com.pasich.mynotes.utils.backup.models.PreferencesBackup getListPreferenc public void setListPreferences( com.pasich.mynotes.utils.backup.models.PreferencesBackup preferences) {} + @Override + public boolean commitListPreferences( + com.pasich.mynotes.utils.backup.models.PreferencesBackup preferences) { + return true; + } + @Override public String getLastKnownVersion() { return ""; From 9173af3329d4d53c8b6574fca881315e952cb61b Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 15:57:32 +0300 Subject: [PATCH 06/16] chore(editor): clear high-severity advisories in editor build dependencies npm audit --audit-level=high has been failing the editor CI job independently of any source change, on two advisories in build-time tooling: - browserslist <=4.28.6: unbounded memory growth and a prototype write via untrusted browserslist-stats.json (GHSA-c83g-rgw3-j3cx, GHSA-73wf-gq98-2v4g) - postcss-selector-parser: denial of service through uncontrolled AST recursion (GHSA-w9m9-85wc-3x92) Both are resolved by npm audit fix, which stays within the declared semver ranges: patch and minor bumps of browserslist, postcss-selector-parser and their transitive data packages. Neither ships in the editor bundle. Rebuilding the editor reproduces byte-identical assets, so nothing under app/src/main/assets/editor changes. --- notes_editor/package-lock.json | 74 ++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/notes_editor/package-lock.json b/notes_editor/package-lock.json index 19bea397..d6d02363 100644 --- a/notes_editor/package-lock.json +++ b/notes_editor/package-lock.json @@ -677,13 +677,16 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.8.31", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.31.tgz", - "integrity": "sha512-a28v2eWrrRWPpJSzxc+mKwm0ZtVx/G8SepdQZDArnXYU/XS+IF6mp8aB/4E+hH1tyGCoDo3KlUCdlSxGDsRkAw==", + "version": "2.11.21", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.21.tgz", + "integrity": "sha512-uh8vpY/1/YyFkunIDFH/12p7/7VdPKA1hejMVEbdkEaWnUz0Hesvx5EbiU6XxjyHZIOju+ZMbQJkRh+es3/spQ==", "dev": true, "license": "Apache-2.0", "bin": { - "baseline-browser-mapping": "dist/cli.js" + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" } }, "node_modules/boolbase": { @@ -694,9 +697,9 @@ "license": "ISC" }, "node_modules/browserslist": { - "version": "4.28.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", - "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", + "version": "4.28.8", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.8.tgz", + "integrity": "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA==", "dev": true, "funding": [ { @@ -714,11 +717,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.8.25", - "caniuse-lite": "^1.0.30001754", - "electron-to-chromium": "^1.5.249", - "node-releases": "^2.0.27", - "update-browserslist-db": "^1.1.4" + "baseline-browser-mapping": "^2.11.12", + "caniuse-lite": "^1.0.30001809", + "electron-to-chromium": "^1.5.402", + "node-releases": "^2.0.53", + "update-browserslist-db": "^1.3.0" }, "bin": { "browserslist": "cli.js" @@ -748,9 +751,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001757", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001757.tgz", - "integrity": "sha512-r0nnL/I28Zi/yjk1el6ilj27tKcdjLsNqAOZr0yVjWPrSQyHgKI2INaEWw21bAQSv2LXRt1XuCS/GomNpWOxsQ==", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -1088,9 +1091,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.260", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.260.tgz", - "integrity": "sha512-ov8rBoOBhVawpzdre+Cmz4FB+y66Eqrk6Gwqd8NGxuhv99GQ8XqMAr351KEkOt7gukXWDg6gJWEMKgL2RLMPtA==", + "version": "1.5.422", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.422.tgz", + "integrity": "sha512-UvA/32XqrLDdZSn7Jllo1AYNcWji/G0d5M0GTViE7KoGBiMunw3a34Sb2KO4ZZyrSEhqsxFoVhWWJshdyfKqJA==", "dev": true, "license": "ISC" }, @@ -1353,11 +1356,14 @@ } }, "node_modules/node-releases": { - "version": "2.0.27", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", - "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "version": "2.0.54", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz", + "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/normalize-url": { "version": "6.1.0", @@ -1781,9 +1787,9 @@ } }, "node_modules/postcss-modules-local-by-default/node_modules/postcss-selector-parser": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", - "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.6.tgz", + "integrity": "sha512-7qASPzhKF2l2KLboRZux8CCTRMdGiV08vWmyKzPz22qZ7ZjQBOeY7rNzNoCLSUiftJ7HUq0GERHmxw/t0dCdMw==", "dev": true, "license": "MIT", "dependencies": { @@ -1811,9 +1817,9 @@ } }, "node_modules/postcss-modules-scope/node_modules/postcss-selector-parser": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", - "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.6.tgz", + "integrity": "sha512-7qASPzhKF2l2KLboRZux8CCTRMdGiV08vWmyKzPz22qZ7ZjQBOeY7rNzNoCLSUiftJ7HUq0GERHmxw/t0dCdMw==", "dev": true, "license": "MIT", "dependencies": { @@ -2034,9 +2040,9 @@ } }, "node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -2399,9 +2405,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz", - "integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz", + "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==", "dev": true, "funding": [ { From 34ea5bb61cb40913a870b22470d4fd3bf30db3dc Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 17:21:09 +0300 Subject: [PATCH 07/16] fix(sync): correct conflict identity, staleness and recovery defects found in review An adversarial review of the sync stack surfaced defects that unit tests could not see because nothing exercised a Room-built record against its own decoded round trip. - attachmentNames was keyed by logical attachment id locally and by SHA-256 after decoding, and a decoded payload also kept the wire-only attachmentIds. A record therefore never hashed equal to itself across a round trip, so every note with an attachment reported a conflict against itself on every sync, forever, and republished a bundle each time. Both sides now produce the same shape, with a round-trip hash test pinning it. - applySnapshot wrote the merged result over records that had moved on locally while the sync was in flight. The snapshot is built before Drive is read and every blob transferred, and the six-hourly worker runs while the user is in the editor, so an edit made in that window was silently dropped. A record newer than the result being applied is now left alone for the next sync. - Conflicts reported by the backend named the winner of the remote fold, which is not necessarily the version the sync applies, so "keep the version the merge selected" could revert a record to a rejected version and republish it. Conflicts are re-pointed at the version actually applied. - A non-canonical logical attachment id passed the local check and then threw inside the manifest, failing every publish for the account while that note existed. - A corrupt remote blob with a valid local copy failed every sync permanently. Blobs are content-addressed and duplicates are tolerated, so the good local copy is published and the account repairs itself. - A missing blob behind an unresolved alternative stopped every device from syncing anything, including devices that could never resolve it. Pinning is best effort; resolution still verifies before it applies. - Recovery did not record the preferences digest on the already-applied path, letting an unchanged copy outrank a genuine edit from another device. - The settled-version set grew for the life of the account and, past the schema record limit, made every publish fail with no way out. It is now bounded. - A crash between the durable preference write and its bookkeeping left the user's choice applied but unversioned and the conflict pending, so the next sync could put the rejected version back. The journal carries the conflict it settles and recovery finishes the job. - Restored notes kept the sending device's URLs inside valueJson, so a received rich note showed its attachments as broken. The editor blocks are re-pointed positionally, and left untouched when they do not line up. Schema 21 carries the journal's conflict bookkeeping. It is a new version rather than an edit to 20, which has already been published on this branch and would fail Room's identity check on any device already running it. --- .../21.json | 579 ++++++++++++++++++ .../com/pasich/mynotes/db/MigrationTest.java | 30 +- .../pasich/mynotes/db/RoomSyncStoreTest.java | 125 +++- .../mynotes/data/database/AppDatabase.java | 21 + .../SyncPendingPreferencesEntity.java | 12 +- .../mynotes/data/sync/RoomSyncStore.java | 176 +++++- .../mynotes/data/sync/SyncBundleCodec.java | 18 +- .../pasich/mynotes/data/sync/SyncService.java | 164 ++++- .../pasich/mynotes/di/ApplicationModule.java | 3 +- .../utils/constants/DatabaseConstants.java | 2 +- .../data/sync/SyncBundleCodecTest.java | 46 +- .../data/sync/SyncBundleValidatorTest.java | 110 ++++ .../mynotes/data/sync/SyncServiceTest.java | 31 +- 13 files changed, 1235 insertions(+), 82 deletions(-) create mode 100644 app/schemas/com.pasich.mynotes.data.database.AppDatabase/21.json diff --git a/app/schemas/com.pasich.mynotes.data.database.AppDatabase/21.json b/app/schemas/com.pasich.mynotes.data.database.AppDatabase/21.json new file mode 100644 index 00000000..06d5cdcd --- /dev/null +++ b/app/schemas/com.pasich.mynotes.data.database.AppDatabase/21.json @@ -0,0 +1,579 @@ +{ + "formatVersion": 1, + "database": { + "version": 21, + "identityHash": "66e50d51d21f701748e066ac915f391c", + "entities": [ + { + "tableName": "tags", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `visibility` INTEGER NOT NULL, `systemAction` INTEGER NOT NULL, `position` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "nameTag", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "visibility", + "columnName": "visibility", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "systemAction", + "columnName": "systemAction", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "notes", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `value` TEXT, `date` INTEGER NOT NULL, `tag` TEXT, `valueJson` TEXT, `hasRichContent` INTEGER NOT NULL, `attachments` TEXT, `isTrash` INTEGER NOT NULL, `reminderTime` INTEGER, `isPinned` INTEGER NOT NULL, `reminderRepeat` TEXT NOT NULL, `reminderIntervalMinutes` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT" + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT" + }, + { + "fieldPath": "date", + "columnName": "date", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "tag", + "columnName": "tag", + "affinity": "TEXT" + }, + { + "fieldPath": "valueJson", + "columnName": "valueJson", + "affinity": "TEXT" + }, + { + "fieldPath": "hasRichContent", + "columnName": "hasRichContent", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "attachments", + "columnName": "attachments", + "affinity": "TEXT" + }, + { + "fieldPath": "isTrash", + "columnName": "isTrash", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reminderTime", + "columnName": "reminderTime", + "affinity": "INTEGER" + }, + { + "fieldPath": "isPinned", + "columnName": "isPinned", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reminderRepeat", + "columnName": "reminderRepeat", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "reminderIntervalMinutes", + "columnName": "reminderIntervalMinutes", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "tasks", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT NOT NULL, `description` TEXT, `isDone` INTEGER NOT NULL DEFAULT 0, `categoryId` INTEGER NOT NULL DEFAULT 0, `createdAt` INTEGER NOT NULL DEFAULT 0, `position` INTEGER NOT NULL DEFAULT 0, `reminderTime` INTEGER, `reminderIntervalMinutes` INTEGER NOT NULL DEFAULT 0)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT" + }, + { + "fieldPath": "isDone", + "columnName": "isDone", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "categoryId", + "columnName": "categoryId", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "reminderTime", + "columnName": "reminderTime", + "affinity": "INTEGER" + }, + { + "fieldPath": "reminderIntervalMinutes", + "columnName": "reminderIntervalMinutes", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "task_categories", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `colorHex` TEXT NOT NULL DEFAULT '#6750A4', `position` INTEGER NOT NULL DEFAULT 0)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "colorHex", + "columnName": "colorHex", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'#6750A4'" + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "sync_metadata", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`recordType` TEXT NOT NULL, `localId` INTEGER NOT NULL, `stableId` TEXT NOT NULL, `updatedAt` INTEGER NOT NULL, `deletedAt` INTEGER, PRIMARY KEY(`recordType`, `localId`))", + "fields": [ + { + "fieldPath": "recordType", + "columnName": "recordType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "localId", + "columnName": "localId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "stableId", + "columnName": "stableId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "updatedAt", + "columnName": "updatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deletedAt", + "columnName": "deletedAt", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "recordType", + "localId" + ] + }, + "indices": [ + { + "name": "index_sync_metadata_recordType_stableId", + "unique": true, + "columnNames": [ + "recordType", + "stableId" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_sync_metadata_recordType_stableId` ON `${TABLE_NAME}` (`recordType`, `stableId`)" + }, + { + "name": "index_sync_metadata_updatedAt", + "unique": false, + "columnNames": [ + "updatedAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_metadata_updatedAt` ON `${TABLE_NAME}` (`updatedAt`)" + }, + { + "name": "index_sync_metadata_deletedAt", + "unique": false, + "columnNames": [ + "deletedAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_metadata_deletedAt` ON `${TABLE_NAME}` (`deletedAt`)" + } + ] + }, + { + "tableName": "sync_pending_preferences", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `payloadJson` TEXT NOT NULL, `targetHash` TEXT NOT NULL, `baselineHash` TEXT NOT NULL, `recordUpdatedAt` INTEGER NOT NULL, `quarantined` INTEGER NOT NULL, `conflictId` INTEGER NOT NULL, `conflictResolution` TEXT NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "payloadJson", + "columnName": "payloadJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "targetHash", + "columnName": "targetHash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "baselineHash", + "columnName": "baselineHash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "recordUpdatedAt", + "columnName": "recordUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "quarantined", + "columnName": "quarantined", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "conflictId", + "columnName": "conflictId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "conflictResolution", + "columnName": "conflictResolution", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "sync_conflicts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `recordType` TEXT NOT NULL, `stableId` TEXT NOT NULL, `versionPairHash` TEXT NOT NULL, `winnerSource` TEXT NOT NULL, `loserSource` TEXT NOT NULL, `winnerVersionId` TEXT NOT NULL, `loserVersionId` TEXT NOT NULL, `winnerJson` TEXT NOT NULL, `loserJson` TEXT NOT NULL, `winnerUpdatedAt` INTEGER NOT NULL, `loserUpdatedAt` INTEGER NOT NULL, `winnerTombstone` INTEGER NOT NULL, `loserTombstone` INTEGER NOT NULL, `resolution` TEXT NOT NULL, `resolved` INTEGER NOT NULL, `createdAt` INTEGER NOT NULL, `resolvedAt` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "recordType", + "columnName": "recordType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stableId", + "columnName": "stableId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionPairHash", + "columnName": "versionPairHash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerSource", + "columnName": "winnerSource", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "loserSource", + "columnName": "loserSource", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerVersionId", + "columnName": "winnerVersionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "loserVersionId", + "columnName": "loserVersionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerJson", + "columnName": "winnerJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "loserJson", + "columnName": "loserJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "winnerUpdatedAt", + "columnName": "winnerUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "loserUpdatedAt", + "columnName": "loserUpdatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "winnerTombstone", + "columnName": "winnerTombstone", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "loserTombstone", + "columnName": "loserTombstone", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "resolution", + "columnName": "resolution", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "resolved", + "columnName": "resolved", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "resolvedAt", + "columnName": "resolvedAt", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_sync_conflicts_recordType_stableId_versionPairHash", + "unique": true, + "columnNames": [ + "recordType", + "stableId", + "versionPairHash" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_sync_conflicts_recordType_stableId_versionPairHash` ON `${TABLE_NAME}` (`recordType`, `stableId`, `versionPairHash`)" + }, + { + "name": "index_sync_conflicts_resolved", + "unique": false, + "columnNames": [ + "resolved" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_conflicts_resolved` ON `${TABLE_NAME}` (`resolved`)" + }, + { + "name": "index_sync_conflicts_createdAt", + "unique": false, + "columnNames": [ + "createdAt" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_sync_conflicts_createdAt` ON `${TABLE_NAME}` (`createdAt`)" + } + ] + }, + { + "tableName": "sync_state", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `status` TEXT NOT NULL, `backendIdentifier` TEXT, `lastSuccessfulSyncAt` INTEGER, `attemptStartedAt` INTEGER, `errorMessage` TEXT, `conflictCount` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "backendIdentifier", + "columnName": "backendIdentifier", + "affinity": "TEXT" + }, + { + "fieldPath": "lastSuccessfulSyncAt", + "columnName": "lastSuccessfulSyncAt", + "affinity": "INTEGER" + }, + { + "fieldPath": "attemptStartedAt", + "columnName": "attemptStartedAt", + "affinity": "INTEGER" + }, + { + "fieldPath": "errorMessage", + "columnName": "errorMessage", + "affinity": "TEXT" + }, + { + "fieldPath": "conflictCount", + "columnName": "conflictCount", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '66e50d51d21f701748e066ac915f391c')" + ] + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java index e8363526..4cecfcb2 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/MigrationTest.java @@ -174,6 +174,31 @@ public void migrate19to20_addsJournalIdentityAndPerSideConflictProvenance() thro } } + @Test + public void migrate20to21_addsTheConflictBookkeepingToTheJournal() throws IOException { + SupportSQLiteDatabase db = helper.createDatabase(TEST_DB, 20); + db.close(); + + SupportSQLiteDatabase migrated = + helper.runMigrationsAndValidate(TEST_DB, 21, true, AppDatabase.MIGRATION_20_21); + try { + migrated.execSQL( + "INSERT INTO sync_pending_preferences " + + "(id, payloadJson, targetHash, baselineHash, recordUpdatedAt, " + + "quarantined, conflictId, conflictResolution) " + + "VALUES (1, '{}', 't', 'b', 0, 0, 7, 'KEEP_WINNER')"); + try (android.database.Cursor cursor = + migrated.query( + "SELECT conflictId, conflictResolution FROM sync_pending_preferences")) { + assertThat(cursor.moveToFirst()).isTrue(); + assertThat(cursor.getLong(0)).isEqualTo(7L); + assertThat(cursor.getString(1)).isEqualTo("KEEP_WINNER"); + } + } finally { + migrated.close(); + } + } + @Test public void migrateFromTheLastReleasedVersion_reachesTheCurrentSchema() throws IOException { // 17 is what 2.6.48 shipped; 18, 19 and 20 all land in the same release after it. @@ -188,11 +213,12 @@ public void migrateFromTheLastReleasedVersion_reachesTheCurrentSchema() throws I SupportSQLiteDatabase migrated = helper.runMigrationsAndValidate( TEST_DB, - 20, + 21, true, AppDatabase.MIGRATION_17_18, AppDatabase.MIGRATION_18_19, - AppDatabase.MIGRATION_19_20); + AppDatabase.MIGRATION_19_20, + AppDatabase.MIGRATION_20_21); try (android.database.Cursor cursor = migrated.query("SELECT COUNT(*) FROM notes")) { assertThat(cursor.moveToFirst()).isTrue(); assertThat(cursor.getInt(0)).isEqualTo(1); diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java index c9325374..20887800 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java @@ -102,8 +102,12 @@ public void hasAttachment_findsABlobThatOnlyExistsInTheNotesOwnFolder() throws E @Test public void readSnapshot_failsClosedWhenAReferencedAttachmentIsMissing() { - int noteId = - seedNote("Missing attachment", "body", "[" + attachmentJson(1, "gone.png") + "]"); + int noteId = seedNote("Missing attachment", "body", null); + // The reference has to name this note's own folder, or the test would pass simply + // because nothing resolves rather than because the file is gone. + Note seeded = db.noteDao().getNoteSync(noteId); + seeded.setAttachments("[" + attachmentJson(noteId, "gone.png") + "]"); + db.noteDao().addNote(seeded); String original = db.noteDao().getNoteSync(noteId).getAttachments(); SnapshotBuildResult.SnapshotBuildException error = assertSnapshotBuildFails(store); @@ -240,6 +244,9 @@ public void applySnapshot_reapplyingTheLocalVersionKeepsItsAttachments() throws Note reloaded = db.noteDao().getNoteSync(noteId); assertThat(reloaded.getAttachments()).isNotNull(); assertThat(reloaded.getAttachments()).contains("photo.png"); + // The display name appearing in the JSON proves nothing about the stored reference, so + // resolve it the way the editor and the file list do. + assertThat(resolveFirstAttachment(reloaded.getAttachments()).isFile()).isTrue(); File restored = new File( new File(context.getFilesDir(), "attachments/note_" + noteId), "photo.png"); @@ -247,6 +254,77 @@ public void applySnapshot_reapplyingTheLocalVersionKeepsItsAttachments() throws assertThat(readAll(new java.io.FileInputStream(restored))).isEqualTo(bytes); } + @Test + public void applySnapshot_repointsEditorBlocksAtTheFilesThisDeviceWrote() throws Exception { + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + int noteId = seedNoteWithAttachment("photo.png", bytes); + String senderUrl = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.urlFor( + noteId, "photo.png"); + Note seeded = db.noteDao().getNoteSync(noteId); + seeded.setValueJson( + "[{\"id\":\"blk1\",\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + senderUrl + + "\",\"name\":\"photo.png\"}}}]"); + db.noteDao().addNote(seeded); + + store.applySnapshot(store.readSnapshot(), Collections.emptyList()); + + // The attachments column is what the file list reads; valueJson is what the editor + // renders. Rebuilding only the column left every received rich note showing a broken + // attachment, because the block still named the sending device's file. + Note reloaded = db.noteDao().getNoteSync(noteId); + String blockUrl = + com.google.gson.JsonParser.parseString(reloaded.getValueJson()) + .getAsJsonArray() + .get(0) + .getAsJsonObject() + .getAsJsonObject("data") + .getAsJsonObject("file") + .get("url") + .getAsString(); + File rendered = + com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.resolve( + context, blockUrl); + assertThat(rendered).isNotNull(); + assertThat(rendered.isFile()).isTrue(); + assertThat(readAll(new java.io.FileInputStream(rendered))).isEqualTo(bytes); + } + + @Test + public void applySnapshot_leavesEditorBlocksAloneWhenTheyDoNotLineUp() throws Exception { + byte[] bytes = "photo bytes".getBytes(StandardCharsets.UTF_8); + int noteId = seedNoteWithAttachment("photo.png", bytes); + // Two blocks, one attachment: the positional mapping cannot be trusted. + String twoBlocks = + "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_" + + noteId + + "/photo.png\"}}}," + + "{\"type\":\"image\",\"data\":{\"file\":{\"url\":\"editorjs://attachments/note_" + + noteId + + "/other.png\"}}}]"; + Note seeded = db.noteDao().getNoteSync(noteId); + seeded.setValueJson(twoBlocks); + db.noteDao().addNote(seeded); + + store.applySnapshot(store.readSnapshot(), Collections.emptyList()); + + // Rewriting on a guess could point a block at the wrong file; leaving it is recoverable. + assertThat(db.noteDao().getNoteSync(noteId).getValueJson()).isEqualTo(twoBlocks); + } + + /** Resolves the first entry of an attachments JSON the way the app's consumers do. */ + private File resolveFirstAttachment(String attachmentsJson) { + String url = + com.google.gson.JsonParser.parseString(attachmentsJson) + .getAsJsonArray() + .get(0) + .getAsJsonObject() + .get("url") + .getAsString(); + return com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.resolve(context, url); + } + @Test public void clearAfterDisconnect_dropsStatusConflictsAndCachedBlobs() throws Exception { store.writeState(SyncState.success("google-drive", java.time.Instant.now(), 0)); @@ -494,7 +572,7 @@ public void anUnreadableJournal_isQuarantinedInsteadOfDisablingSync() throws Exc db.syncPendingPreferencesDao() .upsert( new com.pasich.mynotes.data.database.entities.SyncPendingPreferencesEntity( - 1, "{not json", "target", "baseline", 0L, false)); + 1, "{not json", "target", "baseline", 0L, false, 0L, "")); PreferencesAdapter adapter = new PreferencesAdapter(); RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); @@ -507,6 +585,47 @@ public void anUnreadableJournal_isQuarantinedInsteadOfDisablingSync() throws Exc assertThat(adapter.committed.get()).isNull(); } + @Test + public void recoveryFinishesAConflictWhosePreferenceWriteLandedBeforeTheCrash() + throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore first = new RoomSyncStore(context, db, adapter.helper); + first.readState(); + long conflictId = seedPreferencesConflict(9, 11); + // Simulates a process death between the durable preference write and the bookkeeping: + // the journal is present and the live values already match its target. + adapter.helper.commitListPreferences(preferencesWithTheme(11)); + String chosenJson = new com.google.gson.Gson().toJson(preferencesWithTheme(11)); + // The target digest has to be the real one, or recovery reads the journal as stale and + // discards it instead of finishing what it started. + String target = sha256(chosenJson.getBytes(StandardCharsets.UTF_8)); + db.syncPendingPreferencesDao() + .upsert( + new com.pasich.mynotes.data.database.entities.SyncPendingPreferencesEntity( + 1, + chosenJson, + target, + "digest-before-the-write", + 0L, + false, + conflictId, + SyncResolution.KEEP_DRIVE.name())); + + // A fresh store seeds, which is where recovery runs. + new RoomSyncStore(context, db, adapter.helper).readState(); + + // Without this the choice stayed applied but unversioned and pending, so the next sync + // could put the rejected version back. + assertThat(db.syncPendingPreferencesDao().get()).isNull(); + assertThat(db.syncConflictDao().getById(conflictId).resolved).isTrue(); + SyncMetadataEntity metadata = + db.syncMetadataDao() + .getByStableId( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "00000000-0000-4000-8000-000000000000"); + assertThat(metadata.updatedAt).isGreaterThan(0L); + } + /** A preferences adapter whose durability can be turned off. */ private static final class PreferencesAdapter { private final PreferenceHelper helper = mock(PreferenceHelper.class); diff --git a/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java b/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java index 31b8b70f..2453f749 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/AppDatabase.java @@ -216,6 +216,27 @@ public void migrate(@NonNull SupportSQLiteDatabase database) { } }; + /** + * Lets recovery finish a conflict resolution whose preference write landed but whose + * bookkeeping did not, instead of leaving the user's choice applied yet unversioned and + * revertible by the next sync. + * + *

A separate version rather than an edit to 19→20: that schema has already been published on + * this branch, so a device running it would fail Room's identity check on next launch. + */ + public static final Migration MIGRATION_20_21 = + new Migration(20, 21) { + @Override + public void migrate(@NonNull SupportSQLiteDatabase database) { + database.execSQL( + "ALTER TABLE `sync_pending_preferences` " + + "ADD COLUMN `conflictId` INTEGER NOT NULL DEFAULT 0"); + database.execSQL( + "ALTER TABLE `sync_pending_preferences` " + + "ADD COLUMN `conflictResolution` TEXT NOT NULL DEFAULT ''"); + } + }; + private static void insertMetadataForExistingRecords( SupportSQLiteDatabase database, String recordType, diff --git a/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java index 99fbe140..0231f785 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/entities/SyncPendingPreferencesEntity.java @@ -33,18 +33,28 @@ public final class SyncPendingPreferencesEntity { /** Set when the payload could not be read; retained for support, skipped by recovery. */ public boolean quarantined; + /** The conflict this write settles, or 0 when it comes from an ordinary snapshot apply. */ + public long conflictId; + + /** The resolution to record once the write is durable; empty when there is no conflict. */ + @NonNull public String conflictResolution; + public SyncPendingPreferencesEntity( int id, @NonNull String payloadJson, @NonNull String targetHash, @NonNull String baselineHash, long recordUpdatedAt, - boolean quarantined) { + boolean quarantined, + long conflictId, + @NonNull String conflictResolution) { this.id = id; this.payloadJson = payloadJson; this.targetHash = targetHash; this.baselineHash = baselineHash; this.recordUpdatedAt = recordUpdatedAt; this.quarantined = quarantined; + this.conflictId = conflictId; + this.conflictResolution = conflictResolution; } } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index 0a8d25fb..16ca407d 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -249,6 +249,24 @@ private void applySnapshotInternal( continue; } if (metadata == null) continue; + // The snapshot was built before Drive was read and every blob + // transferred, which + // can take minutes, and the six-hourly worker does it while the + // user is in the + // editor. If the record moved on locally since then, the merge + // chose between + // versions one of which no longer exists, so applying its result + // would silently + // drop the newer edit. Leave it; the next sync merges the real + // current version. + if (metadata.updatedAt > record.getUpdatedAt().toEpochMilli()) { + Log.w( + TAG, + "Skipping a stale sync result for " + + metadata.recordType + + "; it was edited during this sync"); + continue; + } if (record.isTombstone()) { markDeleted(metadata); database.syncMetadataDao() @@ -279,7 +297,9 @@ private void applySnapshotInternal( stagedPreferencesTarget, preferencesBaseline, stagedPreferencesUpdatedAt, - false)); + false, + 0L, + "")); } if (finalState != null && !deferFinalState) { database.syncStateDao().upsert(toEntity(finalState)); @@ -430,6 +450,11 @@ private void recoverPendingPreferences() throws IOException { pending == null ? null : pending.baselineHash, livePreferencesDigest()); + String target = + pending == null || pending.targetHash == null || pending.targetHash.isEmpty() + ? preferencesDigest(backup) + : pending.targetHash; + switch (action) { case NOTHING: return; @@ -438,7 +463,12 @@ private void recoverPendingPreferences() throws IOException { database.runInTransaction(() -> database.syncPendingPreferencesDao().quarantine()); return; case CLEAR_ALREADY_APPLIED: - database.runInTransaction(() -> database.syncPendingPreferencesDao().clear()); + // The values landed but the digest may not have: commitPendingPreferences writes + // it after the adapter returns. Without it the next snapshot build reads the + // stale baseline, calls this a local edit and touches the record to now, which + // lets an unchanged copy outrank a genuine edit made on another device. + preferences.edit().putString(PREFERENCES_HASH, target).commit(); + finishJournal(pending); return; case DISCARD_STALE: Log.w( @@ -448,13 +478,18 @@ private void recoverPendingPreferences() throws IOException { return; case REPLAY: default: - String target = - pending.targetHash == null || pending.targetHash.isEmpty() - ? preferencesDigest(backup) - : pending.targetHash; commitPendingPreferences(backup, target); - database.runInTransaction(() -> database.syncPendingPreferencesDao().clear()); + finishJournal(pending); + } + } + + /** Clears the journal, completing the conflict bookkeeping when it names one. */ + private void finishJournal(@NonNull SyncPendingPreferencesEntity pending) { + if (pending.conflictId > 0) { + finalizeResolvedPreferencesConflict(pending.conflictId, pending.conflictResolution); + return; } + database.runInTransaction(() -> database.syncPendingPreferencesDao().clear()); } /** @@ -775,7 +810,9 @@ private void resolvePreferencesConflict( target, baseline, recordUpdatedAt, - false)); + false, + conflictId, + resolution.name())); }); // Throws when the write is not durable, leaving the journal in place and the conflict @@ -783,32 +820,41 @@ private void resolvePreferencesConflict( commitPendingPreferences(chosen, target); try { - database.runInTransaction( - () -> { - SyncConflictEntity conflict = - database.syncConflictDao().getById(conflictId); - if (conflict == null || conflict.resolved) return; - database.syncPendingPreferencesDao().clear(); - long resolvedAt = System.currentTimeMillis(); - SyncMetadataEntity metadata = - database.syncMetadataDao() - .getByStableId(conflict.recordType, conflict.stableId); - if (metadata != null) { - database.syncMetadataDao() - .setVersion( - conflict.recordType, - metadata.localId, - Math.max(resolvedAt, metadata.updatedAt + 1L), - null); - } - database.syncConflictDao() - .markResolved(conflictId, resolution.name(), resolvedAt); - }); + finalizeResolvedPreferencesConflict(conflictId, resolution.name()); } catch (RuntimeException error) { throw new IOException("Could not finalize the resolved preferences conflict", error); } } + /** + * Records that a preferences conflict is settled, once its value is durably applied. + * + *

Also reached from recovery: a crash between the adapter commit and this step used to leave + * the chosen value in place but unversioned and the conflict still pending, so the next sync + * could quietly put the rejected version back. + */ + private void finalizeResolvedPreferencesConflict(long conflictId, @NonNull String resolution) { + database.runInTransaction( + () -> { + SyncConflictEntity conflict = database.syncConflictDao().getById(conflictId); + database.syncPendingPreferencesDao().clear(); + if (conflict == null || conflict.resolved) return; + long resolvedAt = System.currentTimeMillis(); + SyncMetadataEntity metadata = + database.syncMetadataDao() + .getByStableId(conflict.recordType, conflict.stableId); + if (metadata != null) { + database.syncMetadataDao() + .setVersion( + conflict.recordType, + metadata.localId, + Math.max(resolvedAt, metadata.updatedAt + 1L), + null); + } + database.syncConflictDao().markResolved(conflictId, resolution, resolvedAt); + }); + } + private void pinResolvedConflictAttachments(@NonNull SyncRecord selected) throws IOException { if (selected.isTombstone() || selected.getType() != SyncRecord.Type.NOTE) return; JsonArray manifest = selected.getPayload().getAsJsonArray("attachmentsManifest"); @@ -1170,10 +1216,15 @@ private boolean addAttachmentMetadata( ? file.getName() : attachment.name.trim(); String logicalId = attachment.id; - if (logicalId == null || !logicalId.matches("[0-9a-fA-F-]{36}")) { + if (!isCanonicalUuid(logicalId)) { // Existing editor data predates logical attachment IDs. Deriving from the stable // note, source URL and position keeps the migration deterministic while allowing // equal-content references to remain distinct logical attachments. + // + // The check is canonical-UUID rather than a loose 36-character pattern: the + // bundle manifest only accepts canonical lowercase UUIDs, so an uppercase or + // otherwise non-canonical id used to pass here and then throw during encode, + // failing every publish for the whole account while that note existed. logicalId = UUID.nameUUIDFromBytes( (metadata.stableId @@ -1205,6 +1256,18 @@ private boolean addAttachmentMetadata( return true; } + /** True only for a lowercase canonical UUID, which is all the bundle manifest accepts. */ + private static boolean isCanonicalUuid(@Nullable String value) { + if (value == null) { + return false; + } + try { + return UUID.fromString(value).toString().equals(value); + } catch (IllegalArgumentException notAUuid) { + return false; + } + } + private static void addSnapshotProblem( @NonNull List problems, @NonNull SnapshotProblem.Kind kind, @@ -1279,6 +1342,59 @@ private void restoreAttachments(Note note, JsonObject payload) throws IOExceptio restored.add(attachment); } note.setAttachments(gson.toJson(restored)); + note.setValueJson(rewriteEditorAttachmentUrls(note.getValueJson(), restored)); + } + + /** + * Points the editor's own blocks at the files this device just wrote. + * + *

Restoring rebuilt the note's {@code attachments} column but left {@code valueJson} + * verbatim, so every attachment and image block still named the sending device's {@code + * note_/}. The column is what the file list reads; the blocks are what + * the editor renders, so a received rich note showed its attachments as broken. + * + *

The mapping is positional, which is exactly how the manifest was built: the sender's + * attachments column comes from {@code EditorJsonUtils} walking these same blocks in document + * order, and {@code addAttachmentMetadata} walks that column in the same order. If the two do + * not line up the JSON is returned untouched rather than guessed at — a note that renders the + * old broken URL is recoverable, one whose content was rewritten wrongly is not. + */ + @Nullable + private String rewriteEditorAttachmentUrls( + @Nullable String valueJson, @NonNull JsonArray restored) { + if (valueJson == null || valueJson.trim().isEmpty() || restored.size() == 0) { + return valueJson; + } + try { + JsonArray blocks = JsonParser.parseString(valueJson).getAsJsonArray(); + List files = new ArrayList<>(); + for (JsonElement element : blocks) { + if (!element.isJsonObject()) continue; + JsonObject block = element.getAsJsonObject(); + String type = + block.has("type") && block.get("type").isJsonPrimitive() + ? block.get("type").getAsString() + : ""; + if (!"attaches".equals(type) && !"image".equals(type)) continue; + JsonObject data = block.getAsJsonObject("data"); + if (data == null) continue; + JsonObject file = data.getAsJsonObject("file"); + if (file != null) files.add(file); + } + if (files.size() != restored.size()) { + Log.w(TAG, "Editor blocks do not match the restored attachments; leaving them"); + return valueJson; + } + for (int index = 0; index < files.size(); index++) { + JsonObject target = restored.get(index).getAsJsonObject(); + files.get(index).addProperty("url", target.get("url").getAsString()); + files.get(index).addProperty("name", target.get("name").getAsString()); + } + return gson.toJson(blocks); + } catch (RuntimeException malformed) { + Log.w(TAG, "Could not rewrite editor attachment URLs; leaving them untouched"); + return valueJson; + } } private static boolean isVerifiedAttachmentFile( diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 37c0eb16..925a3167 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -366,8 +366,11 @@ private static void hydrateNoteAttachments( if (attachmentIds == null) return; JsonArray attachmentHashes = new JsonArray(); JsonArray manifest = new JsonArray(); - // The wire keys names by attachment UUID; the local store looks them up by content hash. - JsonObject namesByHash = new JsonObject(); + // Keyed by logical attachment UUID, exactly as the wire carries it and exactly as + // RoomSyncStore builds it locally. Rekeying this map by content hash made a decoded + // record hash differently from the identical locally built one, so every note with an + // attachment reported a conflict against itself on every sync, forever. + JsonObject namesById = new JsonObject(); for (JsonElement element : attachmentIds) { String attachmentId = element.getAsString(); AttachmentManifestEntry attachment = attachmentsById.get(attachmentId); @@ -381,14 +384,17 @@ private static void hydrateNoteAttachments( manifest.add(value); attachmentHashes.add(attachment.sha256); if (value.has("displayName") && !value.get("displayName").isJsonNull()) { - namesByHash.addProperty(attachment.sha256, value.get("displayName").getAsString()); + namesById.addProperty(attachmentId, value.get("displayName").getAsString()); } } payload.add("attachmentsManifest", manifest); payload.add("attachmentHashes", attachmentHashes); - // Rekeyed by hash; leaving the UUID-keyed map made every restored file land on disk - // named after its bare SHA-256, with no extension. - payload.add("attachmentNames", namesByHash); + // The display name restoreAttachments actually uses travels on the manifest entry above; + // this map exists only so the payload matches the one the local store builds. + payload.add("attachmentNames", namesById); + // Wire-only: the local store never produces it, and leaving it behind made a decoded + // record hash differently from the identical local one. + payload.remove("attachmentIds"); } @NonNull diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java index e46cb4f0..254a4ade 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncService.java @@ -34,6 +34,9 @@ public final class SyncService { private static final Pattern SHA_256 = Pattern.compile("[0-9a-f]{64}"); private static final long MAX_TOLERATED_CLOCK_SKEW_MILLIS = 24L * 60L * 60L * 1000L; + /** Well under the schema record limit, so a bundle can always still be published. */ + private static final int MAX_PUBLISHED_SETTLED_IDS = 2_000; + private final SyncStore store; private final SyncMerger merger; private final Clock clock; @@ -111,8 +114,9 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { // A choice the user already made must never be offered again, wherever it was made. java.util.Set settledVersionIds = - new java.util.LinkedHashSet<>(remoteResult.getResolvedAlternativeIds()); - settledVersionIds.addAll(store.getResolvedAlternativeIds()); + new java.util.LinkedHashSet<>(store.getResolvedAlternativeIds()); + settledVersionIds.addAll(remoteResult.getResolvedAlternativeIds()); + settledVersionIds = capSettledVersionIds(settledVersionIds); java.util.List allConflicts = new java.util.ArrayList<>(); for (SyncMergeResult.Conflict conflict : remoteResult.getConflicts()) { @@ -126,6 +130,11 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { } } + // A conflict reported by the backend names the winner of the *remote* fold, which + // is not necessarily the version this sync ends up applying. Persisting it unchanged + // made "keep the version the merge selected" write a stale version over the live one. + allConflicts = realignWinners(allConflicts, merged, local); + // Every still-open alternative is republished, so a merged descendant can never be // the thing that makes a losing version unreachable. Map alternatives = new java.util.LinkedHashMap<>(); @@ -145,8 +154,15 @@ private SyncState syncExclusively(@NonNull SyncBackend backend) { // version independently; SyncSnapshot deliberately forbids two versions of one ID. synchronizeAttachments(backend, merged, expectedSizes); for (SyncMergeResult.Conflict conflict : allConflicts) { - pinConflictVersion(backend, conflict.getWinner()); - pinConflictVersion(backend, conflict.getLoser()); + // Best effort. The merged snapshot's own blobs are mandatory and were just + // transferred above; these are the extra copies that let a conflict be resolved + // later. An alternative whose bytes have gone from Drive is already beyond + // recovery, and failing here made that one missing blob stop every device from + // syncing anything at all, including the devices that could never resolve it. + // Resolution still verifies before it applies, so a version that cannot be + // materialized simply cannot be chosen. + pinConflictVersionQuietly(backend, conflict.getWinner()); + pinConflictVersionQuietly(backend, conflict.getLoser()); } if (needsPublication( @@ -203,6 +219,81 @@ private void warnAboutClockSkew(@NonNull SyncSnapshot remote) { } } + /** + * Bounds the set of settled versions a bundle carries. + * + *

Every resolution adds its two version identities and they were never dropped, so the array + * grew for the life of the account. Past the schema's record limit {@code encode} refuses the + * bundle and every publish fails permanently, with nothing the user can do about it. Trimming + * preserves the identities this device settled most recently; the worst case for a dropped one + * is that an already-settled conflict is offered again, which is recoverable, whereas a bundle + * that cannot be published is not. + */ + @NonNull + private static java.util.Set capSettledVersionIds( + @NonNull java.util.Set settled) { + if (settled.size() <= MAX_PUBLISHED_SETTLED_IDS) { + return settled; + } + Log.w( + TAG, + "Trimming " + + settled.size() + + " settled conflict versions to the publishable limit"); + java.util.Set trimmed = new java.util.LinkedHashSet<>(); + for (String versionId : settled) { + if (trimmed.size() >= MAX_PUBLISHED_SETTLED_IDS) { + break; + } + trimmed.add(versionId); + } + return trimmed; + } + + /** + * Re-points every conflict at the version this sync actually applies. + * + *

{@code KEEP_WINNER} promises the version the deterministic merge selected. The remote + * backend reports conflicts from folding Drive's heads together, before local state is + * considered, so its "winner" can be a version the final merge rejected. Left alone, choosing + * "keep winner" reverted the record to that rejected version and republished it everywhere. + * + *

A conflict whose winner and alternative collapse to the same version is dropped: there is + * nothing left for the user to choose between. + */ + @NonNull + private static java.util.List realignWinners( + @NonNull java.util.List conflicts, + @NonNull SyncSnapshot merged, + @NonNull SyncSnapshot local) { + java.util.List aligned = new java.util.ArrayList<>(); + for (SyncMergeResult.Conflict conflict : conflicts) { + SyncRecord winner = merged.find(conflict.getType(), conflict.getId()); + if (winner == null) { + aligned.add(conflict); + continue; + } + String winnerVersion = winner.getCanonicalPayloadHash(); + if (winnerVersion.equals(conflict.getLoserVersionId())) { + continue; + } + if (winnerVersion.equals(conflict.getWinnerVersionId())) { + aligned.add(conflict); + continue; + } + SyncRecord localRecord = local.find(conflict.getType(), conflict.getId()); + SyncMergeResult.Source winnerSource = + localRecord != null + && localRecord.getCanonicalPayloadHash().equals(winnerVersion) + ? SyncMergeResult.Source.LOCAL + : SyncMergeResult.Source.REMOTE; + aligned.add( + new SyncMergeResult.Conflict( + winner, conflict.getLoser(), winnerSource, conflict.getLoserSource())); + } + return aligned; + } + /** * Whether the remote state already says everything this sync would say. * @@ -261,31 +352,45 @@ private void synchronizeAttachments( Long expectedSize = expectedSizes.get(hash); // Index lookup only; the bytes are checked once, below. boolean remotePresent = backend.hasAttachment(hash); - if (remotePresent) { - try { - verifyAttachment(hash, expectedSize, store.readAttachment(hash)); - } catch (IOException localError) { - // The local copy is missing or corrupt; repair it from the remote blob, - // which copyVerified refuses to accept unless it hashes correctly. - copyVerified( - hash, - expectedSize, - backend.readAttachment(hash), - store::writeAttachment); - } - // Drive is untrusted: a matching appProperty is only a claim. The blob is - // read and hashed exactly once per sync, and the result is remembered, so - // publishing it into the canonical root does not download it again. - if (!backend.hasVerifiedAttachment(hash, expectedSize)) { - throw new AttachmentIntegrityException( - "Attachment checksum does not match its declared hash"); + try { + verifyAttachment(hash, expectedSize, store.readAttachment(hash)); + } catch (IOException localError) { + if (!remotePresent) { + throw localError; } - } else { + // The local copy is missing or corrupt; repair it from the remote blob, + // which copyVerified refuses to accept unless it hashes correctly. + copyVerified( + hash, + expectedSize, + backend.readAttachment(hash), + store::writeAttachment); + } + // Drive is untrusted: a matching appProperty is only a claim. The blob is read + // and hashed exactly once per sync, and the result is remembered, so publishing + // it into the canonical root does not download it again. + if (!remotePresent) { + copyVerified( + hash, + expectedSize, + store.readAttachment(hash), + backend::writeAttachment); + } else if (!backend.hasVerifiedAttachment(hash, expectedSize)) { + // Present but corrupt. Blobs are content-addressed and duplicates are + // tolerated — the reader picks a verified candidate — so publishing a fresh + // copy of the known-good local bytes repairs the account. Throwing here + // instead left every device failing every sync until someone deleted the bad + // object from Drive by hand. The replacement is confirmed before the bundle + // is allowed to depend on it. copyVerified( hash, expectedSize, store.readAttachment(hash), backend::writeAttachment); + if (!backend.hasVerifiedAttachment(hash, expectedSize)) { + throw new AttachmentIntegrityException( + "Attachment checksum does not match its declared hash"); + } } } else { InputStream remoteAttachment = backend.readAttachment(hash); @@ -298,6 +403,19 @@ private void synchronizeAttachments( } } + /** Pins a conflict version's blobs, logging rather than failing the whole sync. */ + private void pinConflictVersionQuietly( + @NonNull SyncBackend backend, @NonNull SyncRecord record) { + try { + pinConflictVersion(backend, record); + } catch (IOException unavailable) { + Log.w( + TAG, + "Could not pin a conflict version's attachments; it stays unresolvable: " + + safeErrorMessage(unavailable)); + } + } + /** Pins required conflict blobs into the store's durable content-addressed cache. */ private void pinConflictVersion(@NonNull SyncBackend backend, @NonNull SyncRecord record) throws IOException { diff --git a/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java b/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java index bdb2b843..16a4902a 100644 --- a/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java +++ b/app/src/main/java/com/pasich/mynotes/di/ApplicationModule.java @@ -76,7 +76,8 @@ AppDatabase providesAppDatabase(@ApplicationContext Context context) { AppDatabase.MIGRATION_16_17, AppDatabase.MIGRATION_17_18, AppDatabase.MIGRATION_18_19, - AppDatabase.MIGRATION_19_20) + AppDatabase.MIGRATION_19_20, + AppDatabase.MIGRATION_20_21) .build(); } diff --git a/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java b/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java index 12228cb7..455b0d00 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java +++ b/app/src/main/java/com/pasich/mynotes/utils/constants/DatabaseConstants.java @@ -3,5 +3,5 @@ public class DatabaseConstants { public static final String DB_NAME = "MyNotes.db"; - public static final int DB_VERSION = 20; + public static final int DB_VERSION = 21; } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java index a333fb3e..b4158fc6 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java @@ -148,12 +148,16 @@ public void decode_keysAttachmentNamesByHashSoTheStoreCanResolveThem() throws Ex .getSnapshot() .find(SyncRecord.Type.NOTE, NOTE_ID); - // RoomSyncStore.restoreAttachments looks names up by content hash. While the decoded map - // stayed keyed by attachment UUID it always missed, and every restored file landed on disk - // named after its bare SHA-256 with no extension. + // The name restoreAttachments actually reads is the one on the manifest entry. + JsonObject entry = + decoded.getPayload().getAsJsonArray("attachmentsManifest").get(0).getAsJsonObject(); + assertThat(entry.get("displayName").getAsString()).isEqualTo("receipt.png"); + + // The map itself stays keyed by logical attachment id, the same shape RoomSyncStore + // builds, so a decoded record hashes equal to the identical local one. JsonObject names = decoded.getPayload().getAsJsonObject("attachmentNames"); - assertThat(names.has(HASH)).isTrue(); - assertThat(names.get(HASH).getAsString()).isEqualTo("receipt.png"); + assertThat(names.has(HASH)).isFalse(); + assertThat(names.get(ATTACHMENT_ID).getAsString()).isEqualTo("receipt.png"); } @Test @@ -232,6 +236,38 @@ private static SyncRecord note(String body) { notePayload(body, "image/png", 42L, "receipt.png")); } + @Test + public void roundTrip_ofALocallyBuiltNoteWithAnAttachment_hashesIdentically() throws Exception { + // Exactly the payload shape RoomSyncStore.addAttachmentMetadata produces: a manifest, + // the hash list, and names keyed by logical attachment id. + JsonObject local = notePayload("Body", "image/png", 12L, "receipt.png"); + JsonObject names = new JsonObject(); + names.addProperty(ATTACHMENT_ID, "receipt.png"); + local.add("attachmentNames", names); + SyncRecord localRecord = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:01Z"), + local); + + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] bundle = + codec.encode( + new SyncSnapshot(java.util.Collections.singletonList(localRecord)), + Instant.parse("2026-08-31T12:00:00Z")); + SyncRecord decoded = + codec.decode(new ByteArrayInputStream(bundle)) + .getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID); + + // The invariant the merge engine depends on: a record that made the round trip is the + // same version as the one that went in. While these differed, every note with an + // attachment conflicted with itself on every sync and republished a bundle each time. + assertThat(decoded.getCanonicalPayloadHash()) + .isEqualTo(localRecord.getCanonicalPayloadHash()); + } + private static SyncRecord task(String title) { JsonObject payload = new JsonObject(); payload.addProperty("title", title); diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleValidatorTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleValidatorTest.java index 4574ee07..b4a22df4 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleValidatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleValidatorTest.java @@ -200,6 +200,116 @@ private SyncSnapshot snapshot() throws IOException { payload))); } + @Test + public void validate_acceptsABundleCarryingAnUnresolvedAlternative() throws Exception { + byte[] bundle = bundleWithAlternatives(alternative("A losing version"), null); + + SyncBundleValidator.ValidatedBundle validated = + validator.validate(new ByteArrayInputStream(bundle)); + + assertThat(validated.getRecords().getAsJsonArray("alternatives")).hasSize(1); + } + + @Test + public void validate_rejectsAnAlternativeWithAnInvalidRecordType() throws Exception { + JsonObject bad = alternative("Bad type"); + bad.addProperty("type", "not-a-record-type"); + + assertRejects(bundleWithAlternatives(bad, null), "Unsupported sync record type"); + } + + @Test + public void validate_rejectsAnAlternativeWithANonCanonicalId() throws Exception { + JsonObject bad = alternative("Bad id"); + bad.addProperty("id", "NOT-A-UUID"); + + assertRejects(bundleWithAlternatives(bad, null), "UUID"); + } + + @Test + public void validate_rejectsAnAlternativeDeletedBeforeItWasUpdated() throws Exception { + JsonObject bad = alternative("Impossible tombstone"); + bad.addProperty("updatedAt", "2026-08-31T12:00:10Z"); + bad.addProperty("deletedAt", "2026-08-31T12:00:00Z"); + + assertRejects(bundleWithAlternatives(bad, null), "deletedAt must not be before updatedAt"); + } + + @Test + public void validate_rejectsDuplicateAlternatives() throws Exception { + byte[] bundle = + bundleWithAlternatives( + alternative("Same version"), null, alternative("Same version")); + + assertRejects(bundle, "duplicate conflict alternatives"); + } + + @Test + public void validate_rejectsAResolvedVersionIdThatIsNotASha256() throws Exception { + assertRejects(bundleWithAlternatives(null, "not-a-digest"), "invalid resolved version id"); + } + + @Test + public void validate_rejectsDuplicateResolvedVersionIds() throws Exception { + JsonObject records = recordsOfAValidBundle(); + JsonArray resolved = new JsonArray(); + resolved.add(HASH); + resolved.add(HASH); + records.add("resolvedAlternatives", resolved); + + assertRejects(rebuild(records), "duplicate resolved version ids"); + } + + private void assertRejects(byte[] bundle, String expectedMessage) { + try { + validator.validate(new ByteArrayInputStream(bundle)); + throw new AssertionError("Expected the bundle to be rejected: " + expectedMessage); + } catch (IOException | RuntimeException error) { + assertThat(error).hasMessageThat().contains(expectedMessage); + } + } + + /** A minimal live-note alternative entry, in the shape the codec writes. */ + private static JsonObject alternative(String value) { + JsonObject item = new JsonObject(); + item.addProperty("type", "note"); + item.addProperty("id", NOTE_ID); + item.addProperty("updatedAt", "2026-08-31T12:00:00Z"); + item.addProperty("title", "Shopping"); + item.addProperty("value", value); + return item; + } + + private JsonObject recordsOfAValidBundle() throws IOException { + byte[] valid = codec.encode(snapshot(), Instant.parse("2026-08-31T12:00:00Z")); + return readJsonEntry(valid, SyncBundleCodec.ENTRY_RECORDS); + } + + private byte[] bundleWithAlternatives(JsonObject first, String resolvedId, JsonObject... more) + throws IOException { + JsonObject records = recordsOfAValidBundle(); + JsonArray alternatives = new JsonArray(); + if (first != null) alternatives.add(first); + for (JsonObject extra : more) alternatives.add(extra); + records.add("alternatives", alternatives); + if (resolvedId != null) { + JsonArray resolved = new JsonArray(); + resolved.add(resolvedId); + records.add("resolvedAlternatives", resolved); + } + return rebuild(records); + } + + /** Re-zips a bundle around edited records, refreshing the manifest checksum and length. */ + private byte[] rebuild(JsonObject records) throws IOException { + byte[] valid = codec.encode(snapshot(), Instant.parse("2026-08-31T12:00:00Z")); + byte[] recordBytes = records.toString().getBytes(StandardCharsets.UTF_8); + JsonObject manifest = readJsonEntry(valid, SyncBundleCodec.ENTRY_MANIFEST); + manifest.addProperty("recordsSha256", SyncBundleValidator.sha256(recordBytes)); + manifest.addProperty("recordsBytes", recordBytes.length); + return zip(manifest.toString(), records.toString()); + } + private static JsonObject readJsonEntry(byte[] bundle, String entryName) throws IOException { try (java.util.zip.ZipInputStream input = new java.util.zip.ZipInputStream(new ByteArrayInputStream(bundle))) { diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java index 63aa739b..7f7f40d5 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncServiceTest.java @@ -113,7 +113,7 @@ public void sync_downloadsRequiredRemoteAttachmentBeforeApplyingSnapshot() throw assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); assertThat(store.attachments.get(hash)).isEqualTo(bytes); assertThat(store.events).containsExactly("writeAttachment", "applySnapshot").inOrder(); - assertThat(backend.events).containsExactly("readAttachment"); + assertThat(backend.events).containsExactly("readSnapshot", "readAttachment").inOrder(); } @Test @@ -130,7 +130,9 @@ public void sync_uploadsAttachmentBeforePublishingSnapshot() throws Exception { assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); assertThat(backend.attachments.get(hash)).isEqualTo(bytes); - assertThat(backend.events).containsExactly("writeAttachment", "writeSnapshot").inOrder(); + assertThat(backend.events) + .containsExactly("readSnapshot", "writeAttachment", "writeSnapshot") + .inOrder(); } @Test @@ -149,7 +151,9 @@ public void sync_skipsUploadingAttachmentWhenRemoteBlobAlreadyExists() throws Ex assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); // Drive is untrusted even for a content-addressed object, so the remote bytes are still // verified before publication — but exactly once, not once per question asked about them. - assertThat(backend.events).containsExactly("readAttachment", "writeSnapshot"); + assertThat(backend.events) + .containsExactly("readSnapshot", "readAttachment", "writeSnapshot") + .inOrder(); } @Test @@ -168,12 +172,13 @@ public void sync_repairsCorruptLocalAttachmentFromRemote() throws Exception { assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); assertThat(store.attachments.get(hash)).isEqualTo(bytes); assertThat(backend.events) - .containsExactly("readAttachment", "readAttachment", "writeSnapshot") + .containsExactly( + "readSnapshot", "readAttachment", "readAttachment", "writeSnapshot") .inOrder(); } @Test - public void sync_corruptRemoteAttachmentWithValidLocalCopyDoesNotPublish() throws Exception { + public void sync_repairsACorruptRemoteAttachmentFromTheValidLocalCopy() throws Exception { byte[] bytes = "local attachment".getBytes(StandardCharsets.UTF_8); String hash = sha256(bytes); FakeStore store = new FakeStore(snapshot(note(TEN, "Local"))); @@ -184,10 +189,12 @@ public void sync_corruptRemoteAttachmentWithValidLocalCopyDoesNotPublish() throw SyncState state = new SyncService(store, new SyncMerger(), CLOCK).sync(backend); - assertThat(state.getStatus()).isEqualTo(SyncState.Status.ERROR); - assertThat(state.getErrorMessage()).contains("checksum"); - assertThat(backend.writeSnapshotCalls).isEqualTo(0); - assertThat(store.applyCalls).isEqualTo(0); + // Content-addressed blobs tolerate duplicates and the reader picks a verified candidate, + // so a corrupt remote object is repaired from the good local copy. Failing instead left + // every device stuck on every sync until the bad object was deleted from Drive by hand. + assertThat(state.getStatus()).isEqualTo(SyncState.Status.SUCCESS); + assertThat(backend.attachments.get(hash)).isEqualTo(bytes); + assertThat(store.attachments.get(hash)).isEqualTo(bytes); } @Test @@ -275,7 +282,9 @@ public void sync_oversizedAttachmentMetadataDoesNotUploadOrPublish() throws Exce assertThat(state.getStatus()).isEqualTo(SyncState.Status.ERROR); assertThat(state.getErrorMessage()).contains("size exceeds"); - assertThat(backend.events).isEmpty(); + // The remote is read before the manifest is inspected; nothing may be transferred or + // published after the oversized entry is found. + assertThat(backend.events).containsExactly("readSnapshot"); assertThat(backend.writeSnapshotCalls).isEqualTo(0); } @@ -488,6 +497,8 @@ public String getIdentifier() { @Override public SyncSnapshot readSnapshot() throws IOException { + // Recorded so a test asserting "the remote was never read" actually proves it. + events.add("readSnapshot"); if (readFailure != null) { throw readFailure; } From e2af22cb12ae07c239a7c61f24faf49a3e5f0bb2 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 17:21:28 +0300 Subject: [PATCH 08/16] fix(backup): stop restore duplicating, colliding and escaping its own directory A whole-app review against v2.6.46, the last version users actually run, found these in the local backup path rather than in sync. - Restore extracted archive entries with new File(filesDir, entry.getName()) and only checked that the name started with "attachments". A backup file is untrusted input, and "attachments/../../databases/notes" satisfies that, so an edited archive could write anywhere the app can. Entries are resolved and must land inside the attachment directory. - Restore inserts rather than replaces, so a backup from another device cannot destroy an unrelated note that happens to share a row id. The cost was that restoring a backup onto the library it came from duplicated every note and tag, while the dialog promises duplicates will be ignored. A row whose id is taken by an identical note is now skipped, so re-restoring is a no-op again, and a genuinely different note under that id is still kept alongside rather than overwriting. Tags are matched by name, which is how a note references them and where a duplicate row is indistinguishable to the user. - When a colliding note was inserted under a new id its attachments stayed in the folder named after the old one, so two notes shared a directory and saving the older one deleted the restored note's files as orphans. The files are copied into the new note's folder and both the attachments column and the editor blocks are re-pointed. Copied rather than moved, so a failure part-way leaves the pre-restore state intact. - commitAll threw on a null string value, where the previous per-key write removed the key. Because restore chains the preference write ahead of the notes and tags, a backup carrying an explicit null aborted the entire restore and left nothing written. - Gson maps the editor and backup models by field name, but only data.model was kept from shrinking. A renamed field there does not fail loudly, it deserializes to null: attachments stop resolving and a restore produces empty notes. Verified against the release mapping that the classes stay unrenamed. --- app/proguard-rules.pro | 10 +- .../mynotes/data/database/dao/TagsDao.java | 4 + .../data/preferences/SafePreferences.java | 7 +- .../data/sync/SyncMutationCoordinator.java | 142 +++++++++++++++- .../attach/NoteAttachmentRelocator.java | 157 ++++++++++++++++++ .../utils/backup/local/ZipBackupHelper.java | 34 +++- .../sync/SyncMutationCoordinatorTest.java | 58 +++++++ .../attach/AttachmentCleanerTest.java | 41 +++++ .../attach/AttachmentUrlTest.java | 44 +++++ .../attach/NoteAttachmentRelocatorTest.java | 142 ++++++++++++++++ 10 files changed, 628 insertions(+), 11 deletions(-) create mode 100644 app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java create mode 100644 app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index 6a5b6322..a8b5fb8b 100644 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -88,4 +88,12 @@ public static *** i(...); public static *** w(...); public static *** e(...); -} \ No newline at end of file +} +# Gson maps these by field name, and only the classes under data.model are kept above. +# Everything below is parsed from data the app itself wrote earlier — note attachments, local +# backups, Google Keep imports — so a renamed field silently deserializes to null rather than +# failing loudly: attachments stop resolving and a restore produces empty notes. +-keep class com.pasich.mynotes.extendedEditor.models.** { *; } +-keep class com.pasich.mynotes.utils.backup.models.** { *; } +-keepclassmembers class com.pasich.mynotes.extendedEditor.models.** { *; } +-keepclassmembers class com.pasich.mynotes.utils.backup.models.** { *; } diff --git a/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java b/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java index 31aac047..8ee0ac99 100644 --- a/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java +++ b/app/src/main/java/com/pasich/mynotes/data/database/dao/TagsDao.java @@ -20,6 +20,10 @@ public interface TagsDao { @Query("SELECT * FROM tags WHERE id = :id LIMIT 1") Tag getTagSync(long id); + /** Tags are referenced by name from a note, so the name is their real identity. */ + @Query("SELECT * FROM tags WHERE name = :name LIMIT 1") + Tag getTagByNameSync(String name); + @Query("DELETE FROM tags WHERE id = :id") void deleteById(long id); diff --git a/app/src/main/java/com/pasich/mynotes/data/preferences/SafePreferences.java b/app/src/main/java/com/pasich/mynotes/data/preferences/SafePreferences.java index c2b64dff..222b5d59 100644 --- a/app/src/main/java/com/pasich/mynotes/data/preferences/SafePreferences.java +++ b/app/src/main/java/com/pasich/mynotes/data/preferences/SafePreferences.java @@ -63,7 +63,12 @@ public boolean commitAll(java.util.Map values) { SharedPreferences.Editor editor = prefs.edit(); for (java.util.Map.Entry entry : values.entrySet()) { Object value = entry.getValue(); - if (value instanceof Integer) { + if (value == null) { + // Matches putString(key, null), which removes the key and lets the default + // apply. A backup whose JSON carries an explicit null for a string preference + // must restore to defaults, not abort the whole restore. + editor.remove(entry.getKey()); + } else if (value instanceof Integer) { editor.putInt(entry.getKey(), (Integer) value); } else if (value instanceof Boolean) { editor.putBoolean(entry.getKey(), (Boolean) value); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java index 7bc2cdba..842aeb22 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java @@ -44,6 +44,11 @@ interface StableIdGenerator { String nextStableId(); } + /** Repoints a restored note's attachments when its row id had to change. */ + interface AttachmentRelocation { + void relocate(@NonNull Note note, int previousId); + } + private final TransactionExecutor transactionExecutor; private final NoteDao noteDao; private final TaskDao taskDao; @@ -53,12 +58,16 @@ interface StableIdGenerator { private final SyncMetadataDao syncMetadataDao; private final TimeProvider timeProvider; private final StableIdGenerator stableIdGenerator; + private final AttachmentRelocation attachmentRelocation; private final Object legacyImportLock = new Object(); private long legacyImportTimestamp = -1L; private long legacyImportExpiresAt = -1L; @Inject - public SyncMutationCoordinator(@NonNull AppDatabase database) { + public SyncMutationCoordinator( + @dagger.hilt.android.qualifiers.ApplicationContext @NonNull + android.content.Context context, + @NonNull AppDatabase database) { this( new TransactionExecutor() { @Override @@ -79,7 +88,22 @@ public T call() { database.transactionsNote(), database.syncMetadataDao(), System::currentTimeMillis, - SyncMetadata::newStableId); + SyncMetadata::newStableId, + (note, previousId) -> { + com.pasich.mynotes.extendedEditor.attach.NoteAttachmentRelocator.Result moved = + com.pasich.mynotes.extendedEditor.attach.NoteAttachmentRelocator + .relocate( + com.pasich.mynotes.extendedEditor.attach + .AttachmentStorage.baseDirPath(context), + previousId, + note.getId(), + note.getAttachments(), + note.getValueJson()); + if (moved.changed) { + note.setAttachments(moved.attachmentsJson); + note.setValueJson(moved.valueJson); + } + }); } SyncMutationCoordinator( @@ -92,6 +116,31 @@ public T call() { @NonNull SyncMetadataDao syncMetadataDao, @NonNull TimeProvider timeProvider, @NonNull StableIdGenerator stableIdGenerator) { + this( + transactionExecutor, + noteDao, + taskDao, + tagsDao, + taskCategoryDao, + transactions, + syncMetadataDao, + timeProvider, + stableIdGenerator, + (note, previousId) -> {}); + } + + SyncMutationCoordinator( + @NonNull TransactionExecutor transactionExecutor, + @NonNull NoteDao noteDao, + @NonNull TaskDao taskDao, + @NonNull TagsDao tagsDao, + @NonNull TaskCategoryDao taskCategoryDao, + @NonNull Transactions transactions, + @NonNull SyncMetadataDao syncMetadataDao, + @NonNull TimeProvider timeProvider, + @NonNull StableIdGenerator stableIdGenerator, + @NonNull AttachmentRelocation attachmentRelocation) { + this.attachmentRelocation = attachmentRelocation; this.transactionExecutor = transactionExecutor; this.noteDao = noteDao; this.taskDao = taskDao; @@ -114,10 +163,12 @@ public long insertTag(@NonNull Tag tag) { }); } - public void insertTags(List tags) { - if (tags == null || tags.isEmpty()) return; + public void insertTags(List incoming) { + if (incoming == null || incoming.isEmpty()) return; transactionExecutor.run( () -> { + List tags = withoutTagsAlreadyPresent(incoming); + if (tags.isEmpty()) return null; long timestamp = resolveBatchTimestamp( SyncMetadata.RECORD_TYPE_TAG, extractTagIds(tags)); @@ -210,19 +261,38 @@ public long insertNote(@NonNull Note note) { return transactionExecutor.run(() -> insertNoteInternal(note, timeProvider.now())); } - public void insertNotes(List notes) { - if (notes == null || notes.isEmpty()) return; + public void insertNotes(List incoming) { + if (incoming == null || incoming.isEmpty()) return; transactionExecutor.run( () -> { + List notes = withoutNotesAlreadyPresent(incoming); + if (notes.isEmpty()) return null; long timestamp = resolveBatchTimestamp( SyncMetadata.RECORD_TYPE_NOTE, extractNoteIds(notes)); + int[] previousIds = new int[notes.size()]; + for (int i = 0; i < notes.size(); i++) { + previousIds[i] = notes.get(i).getId(); + } releaseTakenNoteIds(notes); long[] insertedIds = noteDao.addNotes(notes); for (int i = 0; i < notes.size(); i++) { Note note = notes.get(i); int localId = resolveIntId(note.getId(), insertedIds[i]); note.setId(localId); + if (previousIds[i] > 0 && previousIds[i] != localId) { + // Its attachments were extracted under the old id and would + // otherwise share a folder with whichever note owns that id now. + attachmentRelocation.relocate(note, previousIds[i]); + noteDao.updateNoteContent( + localId, + note.getTitle(), + note.getValue(), + note.getValueJson(), + note.getDate(), + note.getTag(), + note.getAttachments()); + } touchRecord(SyncMetadata.RECORD_TYPE_NOTE, localId, timestamp); } return null; @@ -506,6 +576,66 @@ private long insertNoteInternal(@NonNull Note note, long timestamp) { return insertedId; } + /** + * Drops incoming notes that this device already holds. + * + *

Restore inserts rather than replaces, so that a backup taken on another device cannot + * destroy an unrelated note that happens to share a row id. The cost is that restoring a backup + * onto the library it came from would duplicate every note — and the restore dialog promises + * the opposite. A row whose id is taken by an identical note is therefore skipped: re-restoring + * is a no-op again, while a genuinely different note under the same id is still kept alongside + * the existing one instead of overwriting it. + */ + @NonNull + private List withoutNotesAlreadyPresent(@NonNull List incoming) { + List result = new ArrayList<>(incoming.size()); + for (Note note : incoming) { + Note existing = note.getId() > 0 ? noteDao.getNoteSync(note.getId()) : null; + if (existing == null || !isSameNoteContent(existing, note)) { + result.add(note); + } + } + return result; + } + + /** True when two rows carry the same user-visible note. */ + private static boolean isSameNoteContent(@NonNull Note existing, @NonNull Note incoming) { + return equalText(existing.getTitle(), incoming.getTitle()) + && equalText(existing.getValue(), incoming.getValue()) + && equalText(existing.getValueJson(), incoming.getValueJson()) + && equalText(existing.getTag(), incoming.getTag()) + && equalText(existing.getAttachments(), incoming.getAttachments()) + && existing.getDate() == incoming.getDate() + && existing.isTrash() == incoming.isTrash(); + } + + /** + * Drops incoming tags this device already has under the same name. + * + *

A note stores its tag by name, and the table has no unique index on it, so inserting a + * second row with an existing name shows the user the same tag twice with no way to tell them + * apart. + */ + @NonNull + private List withoutTagsAlreadyPresent(@NonNull List incoming) { + List result = new ArrayList<>(incoming.size()); + Set seen = new LinkedHashSet<>(); + for (Tag tag : incoming) { + String name = tag.getNameTag(); + if (name != null && !name.isEmpty()) { + if (!seen.add(name) || tagsDao.getTagByNameSync(name) != null) { + continue; + } + } + result.add(tag); + } + return result; + } + + private static boolean equalText(String first, String second) { + return first == null ? second == null : first.equals(second); + } + /** * Lets a restore keep its original IDs only where they are still free. * diff --git a/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java new file mode 100644 index 00000000..463270b0 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocator.java @@ -0,0 +1,157 @@ +package com.pasich.mynotes.extendedEditor.attach; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +/** + * Moves a restored note's attachments into the folder its new row id owns. + * + *

A ZIP backup stores attachments under the note id they had when it was taken, and restore + * extracts them verbatim. When that id is already in use the note is inserted under a fresh id, + * which used to leave two notes sharing one {@code note_<id>} directory: saving the older + * note then saw the restored note's files as orphans and deleted them. + * + *

Copies rather than moves, so a failure part-way leaves the original files exactly where the + * pre-restore state expects them; the leftovers are ordinary orphans that cleanup reclaims later. + * Deliberately free of {@code android.*} so the rewriting rules are unit-testable. + */ +public final class NoteAttachmentRelocator { + + /** The rewritten note fields, or the originals when nothing needed to move. */ + public static final class Result { + @Nullable public final String attachmentsJson; + @Nullable public final String valueJson; + public final boolean changed; + + Result(@Nullable String attachmentsJson, @Nullable String valueJson, boolean changed) { + this.attachmentsJson = attachmentsJson; + this.valueJson = valueJson; + this.changed = changed; + } + } + + private NoteAttachmentRelocator() {} + + /** + * Repoints every reference from {@code previousNoteId} to {@code newNoteId}. + * + * @param attachmentsRoot the app-private {@code attachments} directory. + * @param attachmentsJson the note's attachments column. + * @param valueJson the note's Editor.js blocks, which carry their own copies of the URLs. + */ + @NonNull + public static Result relocate( + @NonNull File attachmentsRoot, + int previousNoteId, + int newNoteId, + @Nullable String attachmentsJson, + @Nullable String valueJson) { + if (previousNoteId <= 0 || newNoteId <= 0 || previousNoteId == newNoteId) { + return new Result(attachmentsJson, valueJson, false); + } + String movedAttachments = attachmentsJson; + boolean changed = false; + + if (attachmentsJson != null && !attachmentsJson.trim().isEmpty()) { + try { + JsonArray entries = JsonParser.parseString(attachmentsJson).getAsJsonArray(); + for (JsonElement element : entries) { + if (!element.isJsonObject()) continue; + JsonObject entry = element.getAsJsonObject(); + if (!entry.has("url") || !entry.get("url").isJsonPrimitive()) continue; + String rewritten = + moveReference( + attachmentsRoot, + previousNoteId, + newNoteId, + entry.get("url").getAsString()); + if (rewritten != null) { + entry.addProperty("url", rewritten); + changed = true; + } + } + if (changed) { + movedAttachments = entries.toString(); + } + } catch (RuntimeException unreadable) { + // Unreadable metadata is left exactly as it was; nothing here is worth guessing. + return new Result(attachmentsJson, valueJson, false); + } + } + + String movedValueJson = valueJson; + if (changed && valueJson != null && !valueJson.trim().isEmpty()) { + try { + JsonArray blocks = JsonParser.parseString(valueJson).getAsJsonArray(); + boolean rewroteBlock = false; + for (JsonElement element : blocks) { + if (!element.isJsonObject()) continue; + JsonObject data = element.getAsJsonObject().getAsJsonObject("data"); + if (data == null) continue; + JsonObject file = data.getAsJsonObject("file"); + if (file == null || !file.has("url") || !file.get("url").isJsonPrimitive()) { + continue; + } + String rewritten = + moveReference( + attachmentsRoot, + previousNoteId, + newNoteId, + file.get("url").getAsString()); + if (rewritten != null) { + file.addProperty("url", rewritten); + rewroteBlock = true; + } + } + if (rewroteBlock) { + movedValueJson = blocks.toString(); + } + } catch (RuntimeException unreadable) { + // Keep the blocks untouched rather than risk corrupting the note's content. + movedValueJson = valueJson; + } + } + + return new Result(movedAttachments, movedValueJson, changed); + } + + /** + * Copies one referenced file into the new note's folder and returns its new URL. + * + * @return the rewritten URL, or {@code null} when the reference does not belong to the old note + * or its file is not there to copy. + */ + @Nullable + private static String moveReference( + @NonNull File attachmentsRoot, int previousNoteId, int newNoteId, @NonNull String url) { + AttachmentUrl parsed = AttachmentUrl.parse(url); + if (parsed == null || !parsed.getNoteFolder().equals("note_" + previousNoteId)) { + return null; + } + File source = parsed.resolveWithin(attachmentsRoot); + if (source == null || !source.isFile()) { + return null; + } + File targetFolder = new File(attachmentsRoot, "note_" + newNoteId); + File target = new File(targetFolder, parsed.getFileName()); + try { + if (!targetFolder.isDirectory() && !targetFolder.mkdirs()) { + return null; + } + if (!target.exists()) { + Files.copy(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException | SecurityException failure) { + return null; + } + return AttachmentUrl.canonical(newNoteId, parsed.getFileName()); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java b/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java index 2489e5b6..3bc05517 100644 --- a/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java +++ b/app/src/main/java/com/pasich/mynotes/utils/backup/local/ZipBackupHelper.java @@ -6,11 +6,13 @@ import android.content.Context; import android.net.Uri; +import android.util.Log; import com.google.gson.Gson; import com.pasich.mynotes.utils.backup.models.JsonBackup; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -21,6 +23,8 @@ /** New ZIP-based backup format. Structure: My_Notes_Backup.json attachments/note_/file.ext */ public class ZipBackupHelper { + private static final String TAG = "ZipBackupHelper"; + /** Detect ZIP by magic header "PK" */ public static boolean isZip(byte[] data) { return data.length > 2 && data[0] == 0x50 && data[1] == 0x4B; @@ -102,10 +106,22 @@ public static JsonBackup readZipBackup(Context ctx, Uri uri) throws Exception { // ================== attachments/... ================== else if (entry.getName().startsWith(ATTACHMENTS_BASE_DIR)) { - File out = new File(ctx.getFilesDir(), entry.getName()); + // A backup file is untrusted input: it can be edited, or come from + // somewhere else entirely. "attachments/../../databases/notes" also starts + // with the prefix above, so without resolving the path first an archive + // could write anywhere the app can write. + File out = safeAttachmentTarget(ctx, entry.getName()); + if (out == null || entry.isDirectory()) { + Log.w(TAG, "Skipping a backup entry outside the attachment directory"); + zis.closeEntry(); + continue; + } File parent = out.getParentFile(); - assert parent != null; - if (!parent.exists()) parent.mkdirs(); + if (parent != null && !parent.exists() && !parent.mkdirs()) { + Log.w(TAG, "Could not create the attachment directory for a backup entry"); + zis.closeEntry(); + continue; + } try (FileOutputStream fos = new FileOutputStream(out)) { byte[] data = new byte[4096]; @@ -123,4 +139,16 @@ else if (entry.getName().startsWith(ATTACHMENTS_BASE_DIR)) { return backup != null ? backup : new JsonBackup().error(); } + + /** + * Resolves one archive entry inside the attachment directory, or {@code null} if it escapes. + * + * @param entryName the raw name from the archive, which is attacker-controlled. + */ + private static File safeAttachmentTarget(Context ctx, String entryName) throws IOException { + File root = new File(ctx.getFilesDir(), ATTACHMENTS_BASE_DIR).getCanonicalFile(); + File resolved = new File(ctx.getFilesDir(), entryName).getCanonicalFile(); + String prefix = root.getPath() + File.separator; + return resolved.getPath().startsWith(prefix) ? resolved : null; + } } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java index 48f4b263..d96b5f04 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java @@ -4,6 +4,7 @@ import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -63,6 +64,63 @@ public T run( new QueueStableIdGenerator("stable-a", "stable-b", "stable-c")); } + @Test + public void insertNotes_skipsANoteThisDeviceAlreadyHasUnchanged() { + // Restoring a backup onto the library it came from must stay a no-op: restore inserts + // rather than replaces, so without this every note would be duplicated. + Note existing = new Note().create("Title", "Body", 10L, "work"); + existing.setId(5); + Note fromBackup = new Note().create("Title", "Body", 10L, "work"); + fromBackup.setId(5); + when(noteDao.getNoteSync(5)).thenReturn(existing); + + coordinator.insertNotes(new java.util.ArrayList<>(java.util.List.of(fromBackup))); + + verify(noteDao, never()).addNotes(org.mockito.ArgumentMatchers.anyList()); + } + + @Test + public void insertNotes_keepsADifferentNoteThatHappensToShareARowId() { + // A backup from another device can reuse an id for entirely different content; that note + // has to survive alongside the local one rather than overwrite it. + Note existing = new Note().create("Local", "Local body", 10L, ""); + existing.setId(5); + Note fromBackup = new Note().create("Other", "Other body", 20L, ""); + fromBackup.setId(5); + when(noteDao.getNoteSync(5)).thenReturn(existing); + when(noteDao.addNotes(org.mockito.ArgumentMatchers.anyList())).thenReturn(new long[] {77L}); + + coordinator.insertNotes(new java.util.ArrayList<>(java.util.List.of(fromBackup))); + + assertThat(fromBackup.getId()).isEqualTo(77); + } + + @Test + public void insertTags_skipsATagNameThisDeviceAlreadyHas() { + Tag existing = new Tag().create("work"); + existing.id = 3; + when(tagsDao.getTagByNameSync("work")).thenReturn(existing); + Tag fromBackup = new Tag().create("work"); + fromBackup.id = 9; + + coordinator.insertTags(new java.util.ArrayList<>(java.util.List.of(fromBackup))); + + // A note references its tag by name, so a second row with the same name is the same tag + // shown twice with no way to tell them apart. + verify(tagsDao, never()).addTags(org.mockito.ArgumentMatchers.anyList()); + } + + @Test + public void insertTags_dropsRepeatsWithinOneRestoreBatch() { + Tag first = new Tag().create("work"); + Tag duplicate = new Tag().create("work"); + when(tagsDao.addTags(org.mockito.ArgumentMatchers.anyList())).thenReturn(new long[] {4L}); + + coordinator.insertTags(new java.util.ArrayList<>(java.util.List.of(first, duplicate))); + + assertThat(first.getId()).isEqualTo(4L); + } + @Test public void insertNote_createsMetadataRowWithStableIdAndTimestamp() { Note note = new Note().create("Title", "Body", 10L, ""); diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleanerTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleanerTest.java index 5e4d0b55..87245e30 100644 --- a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleanerTest.java +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentCleanerTest.java @@ -149,6 +149,47 @@ public void treatsAnEmptyReferenceListAsAFullClean() throws Exception { assertThat(orphan.exists()).isFalse(); } + @Test + public void reportsNoFolderWhenTheNoteHasNoAttachmentDirectory() { + AttachmentCleaner.Result result = AttachmentCleaner.cleanup(attachmentsRoot, 99, "[]"); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.NO_FOLDER); + } + + @Test + public void abortsOnANullEntryInTheAttachmentList() throws Exception { + File kept = write("1731000000000_882134.jpg", "keep"); + + AttachmentCleaner.Result result = AttachmentCleaner.cleanup(attachmentsRoot, 42, "[null]"); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.ABORTED_UNRESOLVED_REFERENCE); + assertThat(kept.exists()).isTrue(); + } + + @Test + public void treatsMissingMetadataAsNothingToClean() throws Exception { + File orphan = write("1731000000009_000001.tmp", "drop"); + + // A note that has never had an attachment stores null here. + AttachmentCleaner.Result result = AttachmentCleaner.cleanup(attachmentsRoot, 42, null); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.CLEANED); + assertThat(orphan.exists()).isFalse(); + } + + @Test + public void aReferenceToAnotherNotesFolderDoesNotProtectThisOne() throws Exception { + // The reference resolves, so cleanup proceeds; it simply protects nothing here. + File orphan = write("1731000000009_000001.tmp", "drop"); + + AttachmentCleaner.Result result = + AttachmentCleaner.cleanup( + attachmentsRoot, 42, json("editorjs://attachments/note_7/other.png")); + + assertThat(result).isEqualTo(AttachmentCleaner.Result.CLEANED); + assertThat(orphan.exists()).isFalse(); + } + private File write(String name, String content) throws Exception { File file = new File(noteFolder, name); Files.write(file.toPath(), content.getBytes(StandardCharsets.UTF_8)); diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrlTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrlTest.java index c6ebf550..da6ca18d 100644 --- a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrlTest.java +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/AttachmentUrlTest.java @@ -115,6 +115,50 @@ public void resolvesInsideTheAttachmentRoot() throws Exception { assertThat(parsed.resolveWithin(root)).isEqualTo(file.getCanonicalFile()); } + @Test + public void ignoresAQueryOrFragmentAfterThePath() { + AttachmentUrl withQuery = + AttachmentUrl.parse("editorjs://attachments/note_4/photo.png?v=2"); + AttachmentUrl withFragment = + AttachmentUrl.parse("editorjs://attachments/note_4/photo.png#top"); + + assertThat(withQuery).isNotNull(); + assertThat(withQuery.getFileName()).isEqualTo("photo.png"); + assertThat(withFragment).isNotNull(); + assertThat(withFragment.getFileName()).isEqualTo("photo.png"); + } + + @Test + public void acceptsAnUppercaseScheme() { + AttachmentUrl parsed = AttachmentUrl.parse("EDITORJS://attachments/note_4/photo.png"); + + assertThat(parsed).isNotNull(); + assertThat(parsed.canonical()).isEqualTo("editorjs://attachments/note_4/photo.png"); + } + + @Test + public void twoReferencesToTheSameFileAreEqual() { + AttachmentUrl fromCanonical = AttachmentUrl.parse("editorjs://attachments/note_4/a.png"); + AttachmentUrl fromLegacy = AttachmentUrl.parse("file://attachments/note_4/a.png"); + + assertThat(fromCanonical).isEqualTo(fromLegacy); + assertThat(fromCanonical.hashCode()).isEqualTo(fromLegacy.hashCode()); + assertThat(fromCanonical.toString()).isEqualTo("editorjs://attachments/note_4/a.png"); + assertThat(fromCanonical) + .isNotEqualTo(AttachmentUrl.parse("editorjs://attachments/note_4/b.png")); + } + + @Test + public void resolvingAgainstAMissingRootStillStaysInsideIt() throws Exception { + File root = new File(temporaryFolder.getRoot(), "not-created-yet"); + AttachmentUrl parsed = AttachmentUrl.parse("editorjs://attachments/note_9/photo.png"); + + assertThat(parsed).isNotNull(); + File resolved = parsed.resolveWithin(root); + assertThat(resolved).isNotNull(); + assertThat(resolved.getPath()).startsWith(root.getCanonicalPath() + File.separator); + } + @Test public void canonicalRefusesToBuildAnUnsafeReference() { try { diff --git a/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java new file mode 100644 index 00000000..7b51a176 --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/extendedEditor/attach/NoteAttachmentRelocatorTest.java @@ -0,0 +1,142 @@ +package com.pasich.mynotes.extendedEditor.attach; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Restoring a backup onto a device whose note ids already collide. + * + *

The archive stores attachments under the id the note had when the backup was taken. When that + * id is taken the note is inserted under a new one, and without relocation two notes end up sharing + * one folder — saving the older note then deletes the restored note's files as orphans. + */ +public class NoteAttachmentRelocatorTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private File root; + + @Before + public void setUp() throws Exception { + root = temporaryFolder.newFolder("attachments"); + } + + @Test + public void copiesReferencedFilesIntoTheNewNoteFolderAndRewritesTheUrls() throws Exception { + File original = seed(5, "1731000000000_882134.jpg", "photo bytes"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.relocate( + root, 5, 12, attachmentsJson(5, "1731000000000_882134.jpg"), null); + + assertThat(result.changed).isTrue(); + assertThat(result.attachmentsJson).contains("note_12"); + assertThat(result.attachmentsJson).doesNotContain("note_5"); + + File moved = new File(new File(root, "note_12"), "1731000000000_882134.jpg"); + assertThat(moved.isFile()).isTrue(); + assertThat(contentOf(moved)).isEqualTo("photo bytes"); + // Copied, not moved: the pre-restore state still points at the original. + assertThat(original.isFile()).isTrue(); + } + + @Test + public void rewritesTheEditorBlocksThatCarryTheSameUrls() throws Exception { + seed(5, "1731000000000_882134.jpg", "photo bytes"); + String blocks = + "[{\"type\":\"attaches\",\"data\":{\"file\":{\"url\":\"" + + AttachmentStorage.urlFor(5, "1731000000000_882134.jpg") + + "\",\"name\":\"photo.jpg\"}}}]"; + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.relocate( + root, 5, 12, attachmentsJson(5, "1731000000000_882134.jpg"), blocks); + + assertThat(result.changed).isTrue(); + // The column feeds the file list; the blocks are what the editor renders. + assertThat(result.valueJson).contains("note_12"); + assertThat(result.valueJson).doesNotContain("note_5"); + } + + @Test + public void leavesReferencesThatBelongToAnotherNoteAlone() throws Exception { + seed(7, "other.png", "not mine"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.relocate( + root, 5, 12, attachmentsJson(7, "other.png"), null); + + assertThat(result.changed).isFalse(); + assertThat(result.attachmentsJson).contains("note_7"); + assertThat(new File(new File(root, "note_12"), "other.png").exists()).isFalse(); + } + + @Test + public void doesNothingWhenTheIdDidNotChange() throws Exception { + seed(5, "photo.png", "bytes"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.relocate(root, 5, 5, attachmentsJson(5, "photo.png"), null); + + assertThat(result.changed).isFalse(); + } + + @Test + public void skipsAReferenceWhoseFileIsNotThere() { + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.relocate(root, 5, 12, attachmentsJson(5, "gone.png"), null); + + assertThat(result.changed).isFalse(); + assertThat(result.attachmentsJson).contains("note_5"); + } + + @Test + public void leavesUnreadableMetadataUntouched() { + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.relocate(root, 5, 12, "{not json", null); + + assertThat(result.changed).isFalse(); + assertThat(result.attachmentsJson).isEqualTo("{not json"); + } + + @Test + public void keepsAnExistingTargetFileRatherThanOverwritingIt() throws Exception { + seed(5, "photo.png", "restored bytes"); + File existing = seed(12, "photo.png", "the note already here"); + + NoteAttachmentRelocator.Result result = + NoteAttachmentRelocator.relocate( + root, 5, 12, attachmentsJson(5, "photo.png"), null); + + assertThat(result.changed).isTrue(); + // Overwriting would destroy the file the note that owns note_12 is using. + assertThat(contentOf(existing)).isEqualTo("the note already here"); + } + + private File seed(int noteId, String name, String content) throws Exception { + File folder = new File(root, "note_" + noteId); + assertThat(folder.mkdirs() || folder.isDirectory()).isTrue(); + File file = new File(folder, name); + Files.write(file.toPath(), content.getBytes(StandardCharsets.UTF_8)); + return file; + } + + private static String contentOf(File file) throws Exception { + return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + } + + private static String attachmentsJson(int noteId, String name) { + return "[{\"url\":\"" + + AttachmentStorage.urlFor(noteId, name) + + "\",\"name\":\"" + + name + + "\"}]"; + } +} From e969b8d6d4dc018ba13074195651338d70164f21 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 17:21:41 +0300 Subject: [PATCH 09/16] ci: report combined test coverage on the pull request and stabilise the emulator - The instrumentation job failed with "Timeout waiting for emulator to boot". reactivecircus/android-emulator-runner needs the KVM udev rule on GitHub-hosted Linux runners; without it the x86_64 emulator runs unaccelerated and boot times out before a single test runs. It had passed on earlier commits, so this was flaky rather than broken, and the rule makes it deterministic. - Coverage was uploaded as a downloadable artifact, which nobody opens during review. A coverage job now merges the unit and instrumentation reports and posts a single comment showing overall coverage and the coverage of the files the pull request changes, updating it on each push. - Instrumentation coverage is collected at all, which is what makes those numbers honest: the Room store, the DAOs and the preference adapters are only reachable on a device, so the unit-only report showed them as untested. It reported RoomSyncStore at 0% while twenty on-device tests exercised it. Measured with both reports: data/sync 58.1% to 72.4%, extendedEditor/attach 52.6% to 67.2%, data/database 0% to 38.8%. --- .github/workflows/ci.yml | 70 ++++++++++++++++++++++++++++++++++++++-- app/build.gradle | 4 +++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f788d4b..95f06eb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,11 @@ on: pull_request: types: [opened, synchronize, reopened] +# The coverage step comments on the pull request; everything else only reads. +permissions: + contents: read + pull-requests: write + jobs: editor: name: Build notes editor and audit dependencies @@ -98,8 +103,8 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: debug-unit-test-coverage - path: app/build/reports/coverage/ + name: coverage-unit + path: app/build/reports/coverage/test/debug/report.xml if-no-files-found: error - name: Run lint @@ -127,10 +132,69 @@ jobs: with: log-accepted-android-sdk-licenses: 'false' + # Without this the x86_64 emulator runs unaccelerated on GitHub-hosted Linux runners + # and boot times out before any test runs. Required by reactivecircus/android-emulator-runner. + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Run sync integration and migration tests uses: reactivecircus/android-emulator-runner@v2 with: api-level: 35 target: google_apis arch: x86_64 - script: ./gradlew :app:connectedDebugAndroidTest --no-daemon --stacktrace + # The coverage variant also runs the tests, and the Room store, the DAOs and the + # preference adapters are only reachable here — without this report they look untested. + script: ./gradlew :app:createDebugAndroidTestCoverageReport --no-daemon --stacktrace + + - name: Upload instrumentation coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-instrumentation + path: app/build/reports/coverage/androidTest/debug/connected/report.xml + if-no-files-found: error + + coverage: + name: Report test coverage on the pull request + runs-on: ubuntu-latest + needs: [build, instrumentation] + # Report whatever exists even if a test job failed, so a coverage drop is still visible. + if: always() + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Download unit-test coverage + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: coverage-unit + path: coverage/unit + + - name: Download instrumentation coverage + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: coverage-instrumentation + path: coverage/instrumentation + + # Posts, and on later pushes updates, a single comment showing overall coverage and the + # coverage of the files this PR actually changed. Both reports are passed together so the + # numbers reflect the unit and on-device suites combined. + - name: Comment coverage + uses: madrapps/jacoco-report@v1.7.1 + with: + paths: | + ${{ github.workspace }}/coverage/unit/report.xml + ${{ github.workspace }}/coverage/instrumentation/report.xml + token: ${{ secrets.GITHUB_TOKEN }} + title: Test coverage (unit + instrumentation) + update-comment: true + min-coverage-overall: 0 + min-coverage-changed-files: 0 diff --git a/app/build.gradle b/app/build.gradle index 74d86e45..b85478b2 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -115,6 +115,10 @@ android { minifyEnabled false shrinkResources false enableUnitTestCoverage true + // The Room store, the DAOs and the preference adapters are only reachable on a + // device, so without this they report 0% and the PR comment understates what is + // actually tested. + enableAndroidTestCoverage true proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' } From ce9236537aedc6ed6cb0575a71e4351f886bae3a Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 17:53:24 +0300 Subject: [PATCH 10/16] fix(sync): apply received theme and settings immediately Settings arriving with a sync were stored correctly but stayed invisible until the user navigated away and came back. Two separate reasons, both outside the sync code: - Light/dark is owned by AppCompatDelegate. Refreshing the preference caches (ThemePreferencesCache.refresh) only reloads the values; nothing called applyCurrentThemeMode, so the mode kept whatever the process started with. - Theme, dynamic colour and UI scale are read when an activity is created (BaseActivity), so the screen already on display never picked them up. The app does have a path for this, but it is wired to the settings screen's activity result, which a sync never goes through. - commitListPreferences now applies the theme mode after refreshing the caches, posted to the main thread because it runs on a background thread for both a sync apply and a backup restore. - RoomSyncStore distinguishes writing the same values from actually changing them, by comparing the digest either side of the write, and reports it once. - BackupActivity redraws itself after a successful sync that changed settings. Deliberately not redrawn while conflicts are pending: recreating the activity would dismiss the dialog the user is choosing a version in, and the values are stored either way, so they still take effect on the next screen. Identical values do not redraw either, which would otherwise flicker on every sync. --- .../pasich/mynotes/db/RoomSyncStoreTest.java | 32 +++++++++++++++++++ .../preferences/AppPreferencesHelper.java | 6 ++++ .../mynotes/data/sync/RoomSyncStore.java | 24 ++++++++++++++ .../ui/view/activity/BackupActivity.java | 32 +++++++++++++++++++ app/src/main/res/values-be/strings.xml | 1 + app/src/main/res/values-de/strings.xml | 1 + app/src/main/res/values-en-rGB/strings.xml | 1 + app/src/main/res/values-es/strings.xml | 1 + app/src/main/res/values-fr/strings.xml | 1 + app/src/main/res/values-it/strings.xml | 1 + app/src/main/res/values-kk/strings.xml | 1 + app/src/main/res/values-pl/strings.xml | 1 + app/src/main/res/values-ru/strings.xml | 1 + app/src/main/res/values-uk/strings.xml | 1 + app/src/main/res/values/strings.xml | 1 + 15 files changed, 105 insertions(+) diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java index 20887800..fc97f21b 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java @@ -626,6 +626,38 @@ public void recoveryFinishesAConflictWhosePreferenceWriteLandedBeforeTheCrash() assertThat(metadata.updatedAt).isGreaterThan(0L); } + @Test + public void applyingChangedPreferences_reportsThatTheScreenMustRedraw() throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + preferencesStore.readState(); + long conflictId = seedPreferencesConflict(9, 11); + + preferencesStore.resolveConflict(conflictId, SyncResolution.KEEP_DRIVE); + + // Theme and UI scale are read when an activity is created, so the visible screen has to + // be told; without this a theme from another device stayed invisible until the user + // navigated away and back. + assertThat(preferencesStore.consumeAppliedPreferencesChange()).isTrue(); + // The flag is consumed, so a later sync that changes nothing does not redraw. + assertThat(preferencesStore.consumeAppliedPreferencesChange()).isFalse(); + } + + @Test + public void applyingIdenticalPreferences_doesNotAskForARedraw() throws Exception { + PreferencesAdapter adapter = new PreferencesAdapter(); + adapter.current.set(preferencesWithTheme(11)); + RoomSyncStore preferencesStore = new RoomSyncStore(context, db, adapter.helper); + preferencesStore.readState(); + long conflictId = seedPreferencesConflict(9, 11); + + preferencesStore.resolveConflict(conflictId, SyncResolution.KEEP_DRIVE); + + // Same values in, same values out: recreating the screen would be a visible flicker for + // no reason. + assertThat(preferencesStore.consumeAppliedPreferencesChange()).isFalse(); + } + /** A preferences adapter whose durability can be turned off. */ private static final class PreferencesAdapter { private final PreferenceHelper helper = mock(PreferenceHelper.class); diff --git a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java index 01ad98e4..d62f5f80 100644 --- a/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java +++ b/app/src/main/java/com/pasich/mynotes/data/preferences/AppPreferencesHelper.java @@ -137,6 +137,12 @@ public boolean commitListPreferences(PreferencesBackup preferences) { } appCache.refresh(); themeCache.refresh(); + // Refreshing the caches only reloads the values. Light/dark is owned by + // AppCompatDelegate, which has to be told, or a theme arriving from another device sat + // in storage until the next activity was created. Posted to the main thread because this + // runs on a background thread for both a sync apply and a backup restore. + new android.os.Handler(android.os.Looper.getMainLooper()) + .post(themeCache::applyCurrentThemeMode); return true; } diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index 16ca407d..c8241551 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -65,6 +65,16 @@ public final class RoomSyncStore implements SyncStore { */ private final Map localAttachments = new ConcurrentHashMap<>(); + /** + * Set when an apply actually changed the visible settings, so the screen can redraw. + * + *

Theme, dynamic colour and UI scale are read when an activity is created, so a version + * arriving from another device was stored correctly but only became visible after the user + * navigated away and back. + */ + private final java.util.concurrent.atomic.AtomicBoolean appliedPreferencesChange = + new java.util.concurrent.atomic.AtomicBoolean(false); + public RoomSyncStore( @NonNull Context context, @NonNull AppDatabase database, @@ -499,6 +509,7 @@ private void finishJournal(@NonNull SyncPendingPreferencesEntity pending) { */ private void commitPendingPreferences( @NonNull PreferencesBackup backup, @NonNull String expectedDigest) throws IOException { + String before = livePreferencesDigest(); boolean committed; try { committed = preferenceHelper.commitListPreferences(backup); @@ -508,6 +519,9 @@ private void commitPendingPreferences( if (!committed) { throw new IOException("Could not commit synchronized preferences"); } + if (!expectedDigest.equals(before)) { + appliedPreferencesChange.set(true); + } // The digest doubles as the snapshot-build baseline, so recording it here keeps the next // build from treating a freshly received version as a local edit. preferences.edit().putString(PREFERENCES_HASH, expectedDigest).commit(); @@ -693,6 +707,16 @@ else if (SyncMetadata.RECORD_TYPE_CATEGORY.equals(metadata.recordType)) else if ("tag".equals(metadata.recordType)) database.tagsDao().deleteById(metadata.localId); } + /** + * Whether the last apply changed the settings, clearing the flag as it reports. + * + *

The caller is the visible screen, which redraws itself so a received theme takes effect at + * once rather than at the next activity creation. + */ + public boolean consumeAppliedPreferencesChange() { + return appliedPreferencesChange.getAndSet(false); + } + public List getConflicts() { return database.syncConflictDao().getAll(); } diff --git a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java index 30652cde..ed8cbda6 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java +++ b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java @@ -463,14 +463,46 @@ private void finishSync(SyncState state) { null, SnackBarInfo.Success, Snackbar.LENGTH_LONG); + boolean settingsArrived = + roomSyncStore != null && roomSyncStore.consumeAppliedPreferencesChange(); if (conflicts > 0) { showNextConflictDialog(); + } else if (settingsArrived) { + // Theme, dynamic colour and UI scale are read when an activity is created, so a + // settings version received from another device was stored but stayed invisible + // until the user navigated away and back. + // + // Only with no conflicts: redrawing while the user is choosing between versions + // would dismiss that dialog. The values are stored either way, so they still + // take effect on the next screen. + applyReceivedPreferences(); } } else { finishSyncError(new IllegalStateException(state.getErrorMessage())); } } + /** + * Redraws the screen so settings that arrived with a sync take effect immediately. + * + *

Delayed so the sync result stays readable for a moment before the screen rebuilds. + */ + private void applyReceivedPreferences() { + onInfoSnack( + getString(R.string.sync_preferences_applied), + null, + SnackBarInfo.Success, + Snackbar.LENGTH_LONG); + binding.getRoot() + .postDelayed( + () -> { + if (!isFinishing() && !isDestroyed()) { + recreate(); + } + }, + 1500L); + } + private void finishSyncError(Exception error) { if (isFinishing() || isDestroyed()) { return; diff --git a/app/src/main/res/values-be/strings.xml b/app/src/main/res/values-be/strings.xml index 16acab3e..4b6b0d20 100644 --- a/app/src/main/res/values-be/strings.xml +++ b/app/src/main/res/values-be/strings.xml @@ -423,6 +423,7 @@ Вырашыць канфлікты сінхранізацыі (%1$d засталося) Пакінуць лакальную версію Пакінуць версію з Google Drive + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 682b7652..ce2a8058 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -423,6 +423,7 @@ Synchronisierungskonflikte lösen (%1$d verbleibend) Lokale Version behalten Google-Drive-Version behalten + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values-en-rGB/strings.xml b/app/src/main/res/values-en-rGB/strings.xml index 05f4fda1..279bfad8 100644 --- a/app/src/main/res/values-en-rGB/strings.xml +++ b/app/src/main/res/values-en-rGB/strings.xml @@ -467,6 +467,7 @@ Resolve sync conflicts (%1$d remaining) Keep local version Keep Google Drive version + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 9560908d..03ea467b 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -424,6 +424,7 @@ Resolver conflictos de sincronización (%1$d restantes) Mantener versión local Mantener versión de Google Drive + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index 5e20f3b2..64981446 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -419,6 +419,7 @@ Résoudre les conflits de synchronisation (%1$d restants) Conserver la version locale Conserver la version Google Drive + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index efef81f5..0ab45b3b 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -421,6 +421,7 @@ Risolvi i conflitti di sincronizzazione (%1$d rimanenti) Mantieni la versione locale Mantieni la versione Google Drive + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values-kk/strings.xml b/app/src/main/res/values-kk/strings.xml index ddd3bcfc..58929e7f 100644 --- a/app/src/main/res/values-kk/strings.xml +++ b/app/src/main/res/values-kk/strings.xml @@ -420,6 +420,7 @@ Синхрондау қайшылықтарын шешу (%1$d қалды) Жергілікті нұсқаны сақтау Google Drive нұсқасын сақтау + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index 89ae82c2..8b768203 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -424,6 +424,7 @@ Rozwiąż konflikty synchronizacji (%1$d pozostało) Zachowaj wersję lokalną Zachowaj wersję z Google Drive + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 05eeecd3..0145147e 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -428,6 +428,7 @@ Разрешить конфликты синхронизации (%1$d осталось) Оставить локальную версию Оставить версию из Google Drive + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index e959553a..bf832f51 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -465,6 +465,7 @@ Розв’язати конфлікти синхронізації (%1$d залишилось) Залишити локальну версію Залишити версію з Google Drive + Застосовано налаштування з Google Drive Keep version 1 Keep version 2 Version %1$d — %2$s diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 925b622c..8a2572b6 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -345,6 +345,7 @@ Resolve sync conflicts (%1$d remaining) Keep local Keep Drive + Settings received from Google Drive applied Keep version 1 Keep version 2 Version %1$d — %2$s From a36cc63b77344418991b2f3ad613fcff1f454da3 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 18:06:37 +0300 Subject: [PATCH 11/16] feat(sync): compare the two versions in the conflict dialog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Variant A of the redesign. The dialog rendered both versions into a single message string, which left the one decision it exists for unsupportable: - the two versions were separated only by a newline, so comparing them meant reading the text - winnerUpdatedAt and loserUpdatedAt were already on the row but never shown, and "which one is mine" is usually answered by the time - each side was cut to 120 characters from the start, so a difference past that point never reached the screen at all - Later, Keep version 1 and Keep version 2 sat as three buttons of equal weight, though two of them discard someone's version for good Each version is now its own selectable card carrying its true origin, its timestamp, a "newer" marker, and a preview with the differing span emphasised. The confirm button acts on the selection, so there is one destructive action instead of two, and Later stops competing with them. The preview window is centred on the difference rather than cut from the head, which is what kept a change near the end of a note off screen. The decision logic moved out of the Activity into SyncConflictPresentation: which text to show, where the two versions diverge, which side is newer and what a version even is (text, deletion, settings, untitled). It has no android.* dependency, so all of it is unit-tested — including that a Drive-vs-Drive conflict never claims either side came from this device. The winner starts selected, so tapping through without reading changes nothing. --- .../ui/sync/SyncConflictPresentation.java | 260 +++++++++++++++++ .../ui/view/activity/BackupActivity.java | 216 ++++++++------ .../main/res/layout/dialog_sync_conflict.xml | 170 +++++++++++ app/src/main/res/values-be/strings.xml | 9 +- app/src/main/res/values-de/strings.xml | 9 +- app/src/main/res/values-en-rGB/strings.xml | 9 +- app/src/main/res/values-es/strings.xml | 9 +- app/src/main/res/values-fr/strings.xml | 9 +- app/src/main/res/values-it/strings.xml | 9 +- app/src/main/res/values-kk/strings.xml | 9 +- app/src/main/res/values-pl/strings.xml | 9 +- app/src/main/res/values-ru/strings.xml | 9 +- app/src/main/res/values-uk/strings.xml | 9 +- app/src/main/res/values/strings.xml | 9 +- .../ui/sync/SyncConflictPresentationTest.java | 274 ++++++++++++++++++ 15 files changed, 867 insertions(+), 152 deletions(-) create mode 100644 app/src/main/java/com/pasich/mynotes/ui/sync/SyncConflictPresentation.java create mode 100644 app/src/main/res/layout/dialog_sync_conflict.xml create mode 100644 app/src/test/java/com/pasich/mynotes/ui/sync/SyncConflictPresentationTest.java diff --git a/app/src/main/java/com/pasich/mynotes/ui/sync/SyncConflictPresentation.java b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncConflictPresentation.java new file mode 100644 index 00000000..189d7141 --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/ui/sync/SyncConflictPresentation.java @@ -0,0 +1,260 @@ +package com.pasich.mynotes.ui.sync; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.pasich.mynotes.data.database.entities.SyncConflictEntity; +import com.pasich.mynotes.data.sync.SyncMetadata; + +/** + * Turns a stored conflict into the two versions a person actually compares. + * + *

The dialog used to render both versions into one string, with no timestamps and a fragment cut + * at a fixed 120 characters — so a difference near the end of a note never reached the screen and + * "which one is mine" was unanswerable. Everything here comes from fields the conflict row already + * carries; nothing new is read from the database. + * + *

Free of {@code android.*} on purpose: which text is shown, where the difference is, and which + * side is newer are the parts worth testing, and they are testable only if they live outside the + * Activity. + */ +public final class SyncConflictPresentation { + + /** How a version should be described, so the caller supplies the localized wording. */ + public enum Kind { + /** Ordinary text taken from the payload. */ + TEXT, + /** The version is a deletion. */ + DELETED, + /** A settings payload, which has no single readable title. */ + SETTINGS, + /** Readable, but every candidate field was empty. */ + UNTITLED + } + + /** One side of the conflict, ready to render. */ + public static final class Version { + /** True only when this version genuinely came from this device. */ + public final boolean local; + + public final long updatedAt; + public final boolean newer; + @NonNull public final Kind kind; + + /** Preview text, already windowed around the difference. Empty unless {@link Kind#TEXT}. */ + @NonNull public final String preview; + + /** Range within {@link #preview} that differs from the other version. */ + public final int highlightStart; + + public final int highlightEnd; + + Version( + boolean local, + long updatedAt, + boolean newer, + @NonNull Kind kind, + @NonNull String preview, + int highlightStart, + int highlightEnd) { + this.local = local; + this.updatedAt = updatedAt; + this.newer = newer; + this.kind = kind; + this.preview = preview; + this.highlightStart = highlightStart; + this.highlightEnd = highlightEnd; + } + + public boolean hasHighlight() { + return highlightEnd > highlightStart; + } + } + + /** Longest preview shown in the dialog before it is windowed. */ + static final int PREVIEW_LIMIT = 140; + + @NonNull public final Version winner; + @NonNull public final Version alternative; + @NonNull public final String recordType; + + private SyncConflictPresentation( + @NonNull Version winner, @NonNull Version alternative, @NonNull String recordType) { + this.winner = winner; + this.alternative = alternative; + this.recordType = recordType; + } + + @NonNull + public static SyncConflictPresentation of(@NonNull SyncConflictEntity conflict) { + Kind winnerKind = + kindOf(conflict.recordType, conflict.winnerTombstone, conflict.winnerJson); + Kind loserKind = kindOf(conflict.recordType, conflict.loserTombstone, conflict.loserJson); + + String winnerText = + winnerKind == Kind.TEXT ? readable(conflict.recordType, conflict.winnerJson) : ""; + String loserText = + loserKind == Kind.TEXT ? readable(conflict.recordType, conflict.loserJson) : ""; + + int[] range = differenceRange(winnerText, loserText); + Window winnerWindow = window(winnerText, range[0], range[1]); + Window loserWindow = window(loserText, range[0], range[2]); + + boolean winnerNewer = conflict.winnerUpdatedAt >= conflict.loserUpdatedAt; + return new SyncConflictPresentation( + new Version( + "LOCAL".equals(conflict.winnerSource), + conflict.winnerUpdatedAt, + winnerNewer, + winnerKind, + winnerWindow.text, + winnerWindow.start, + winnerWindow.end), + new Version( + "LOCAL".equals(conflict.loserSource), + conflict.loserUpdatedAt, + !winnerNewer, + loserKind, + loserWindow.text, + loserWindow.start, + loserWindow.end), + conflict.recordType); + } + + @NonNull + private static Kind kindOf( + @NonNull String recordType, boolean tombstone, @Nullable String recordJson) { + if (tombstone) { + return Kind.DELETED; + } + if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(recordType)) { + return Kind.SETTINGS; + } + String text = readable(recordType, recordJson); + return text.isEmpty() ? Kind.UNTITLED : Kind.TEXT; + } + + /** + * Pulls the most descriptive text out of a stored version. + * + *

Notes and tags are serialized through Gson's short field aliases, so probing "title" and + * "name" never matched them. + */ + @NonNull + static String readable(@NonNull String recordType, @Nullable String recordJson) { + if (recordJson == null || recordJson.isEmpty()) { + return ""; + } + try { + JsonObject root = JsonParser.parseString(recordJson).getAsJsonObject(); + if (root.has("deletedAt") && !root.get("deletedAt").isJsonNull()) { + return ""; + } + JsonObject payload = root.getAsJsonObject("payload"); + if (payload == null) { + return ""; + } + StringBuilder result = new StringBuilder(); + for (String key : labelKeys(recordType)) { + if (!payload.has(key) || payload.get(key).isJsonNull()) continue; + if (!payload.get(key).isJsonPrimitive()) continue; + String value = payload.get(key).getAsString().trim(); + if (value.isEmpty()) continue; + if (result.length() > 0) result.append(" — "); + result.append(value); + } + return result.toString(); + } catch (RuntimeException unreadable) { + return ""; + } + } + + /** Payload keys carrying human-readable text, most specific first. */ + @NonNull + static String[] labelKeys(@NonNull String recordType) { + if (SyncMetadata.RECORD_TYPE_NOTE.equals(recordType)) { + return new String[] {"b", "c"}; // Note.title, Note.value + } + if (SyncMetadata.RECORD_TYPE_TAG.equals(recordType)) { + return new String[] {"b"}; // Tag.nameTag + } + if (SyncMetadata.RECORD_TYPE_TASK.equals(recordType)) { + return new String[] {"title", "description"}; + } + if (SyncMetadata.RECORD_TYPE_CATEGORY.equals(recordType)) { + return new String[] {"name"}; + } + return new String[0]; + } + + /** + * Locates where two versions stop agreeing. + * + * @return {@code {start, endInFirst, endInSecond}} — the shared prefix length and, for each + * side, where its differing part ends. Equal strings give a zero-length range. + */ + @NonNull + static int[] differenceRange(@NonNull String first, @NonNull String second) { + int prefix = 0; + int shortest = Math.min(first.length(), second.length()); + while (prefix < shortest && first.charAt(prefix) == second.charAt(prefix)) { + prefix++; + } + if (prefix == first.length() && prefix == second.length()) { + return new int[] {0, 0, 0}; + } + int suffix = 0; + while (suffix < shortest - prefix + && first.charAt(first.length() - 1 - suffix) + == second.charAt(second.length() - 1 - suffix)) { + suffix++; + } + return new int[] {prefix, first.length() - suffix, second.length() - suffix}; + } + + /** Preview text plus the highlight range inside it. */ + static final class Window { + @NonNull final String text; + final int start; + final int end; + + Window(@NonNull String text, int start, int end) { + this.text = text; + this.start = start; + this.end = end; + } + } + + /** + * Trims a version to preview length, keeping the difference on screen. + * + *

A fixed head-of-string cut is what hid the difference whenever it fell past the limit, so + * the window is centred on the differing range instead and marked with ellipses. + */ + @NonNull + static Window window(@NonNull String text, int diffStart, int diffEnd) { + if (text.length() <= PREVIEW_LIMIT) { + return new Window(text, clamp(diffStart, text.length()), clamp(diffEnd, text.length())); + } + int start = clamp(diffStart, text.length()); + int end = clamp(diffEnd, text.length()); + int centre = (start + end) / 2; + int from = Math.max(0, centre - PREVIEW_LIMIT / 2); + int to = Math.min(text.length(), from + PREVIEW_LIMIT); + from = Math.max(0, to - PREVIEW_LIMIT); + + String head = from > 0 ? "…" : ""; + String tail = to < text.length() ? "…" : ""; + String body = text.substring(from, to); + int shift = head.length() - from; + return new Window( + head + body + tail, + clamp(start + shift, head.length() + body.length()), + clamp(end + shift, head.length() + body.length())); + } + + private static int clamp(int value, int max) { + return Math.max(0, Math.min(value, max)); + } +} diff --git a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java index ed8cbda6..39ed2c8c 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java +++ b/app/src/main/java/com/pasich/mynotes/ui/view/activity/BackupActivity.java @@ -10,22 +10,30 @@ import android.app.Dialog; import android.content.ActivityNotFoundException; import android.content.Intent; +import android.graphics.Color; import android.net.Uri; import android.os.Bundle; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.ForegroundColorSpan; +import android.text.style.StyleSpan; import android.util.Log; import android.view.Menu; import android.view.MenuItem; +import android.view.View; +import android.widget.RadioButton; +import android.widget.TextView; import androidx.activity.OnBackPressedCallback; import androidx.activity.result.ActivityResultLauncher; import androidx.activity.result.contract.ActivityResultContracts; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import com.google.android.material.card.MaterialCardView; +import com.google.android.material.color.MaterialColors; import com.google.android.material.dialog.MaterialAlertDialogBuilder; import com.google.android.material.snackbar.Snackbar; import com.google.android.material.tabs.TabLayoutMediator; import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import com.pasich.mynotes.R; import com.pasich.mynotes.base.activity.BaseActivity; import com.pasich.mynotes.base.view.BackupOptionsCallback; @@ -35,13 +43,13 @@ import com.pasich.mynotes.data.preferences.PreferenceHelper; import com.pasich.mynotes.data.sync.RoomSyncStore; import com.pasich.mynotes.data.sync.SyncBundleCodec; -import com.pasich.mynotes.data.sync.SyncMetadata; import com.pasich.mynotes.data.sync.SyncResolution; import com.pasich.mynotes.data.sync.SyncSnapshot; import com.pasich.mynotes.data.sync.SyncState; import com.pasich.mynotes.databinding.ActivityBackupBinding; import com.pasich.mynotes.ui.contract.BackupContract; import com.pasich.mynotes.ui.presenter.BackupPresenter; +import com.pasich.mynotes.ui.sync.SyncConflictPresentation; import com.pasich.mynotes.ui.sync.SyncCoordinator; import com.pasich.mynotes.ui.sync.SyncCoordinatorFactory; import com.pasich.mynotes.ui.view.dialogs.BackupOptionsDialog; @@ -565,20 +573,133 @@ private void showConflictDialog(List unresolved) { return; } SyncConflictEntity conflict = unresolved.get(0); + SyncConflictPresentation presentation = SyncConflictPresentation.of(conflict); + + View body = getLayoutInflater().inflate(R.layout.dialog_sync_conflict, null, false); + ((TextView) body.findViewById(R.id.conflict_summary)) + .setText(R.string.sync_conflict_explain); + + MaterialCardView firstCard = body.findViewById(R.id.version_one_card); + MaterialCardView secondCard = body.findViewById(R.id.version_two_card); + RadioButton firstRadio = body.findViewById(R.id.version_one_radio); + RadioButton secondRadio = body.findViewById(R.id.version_two_radio); + + bindConflictVersion( + body, + presentation.winner, + R.id.version_one_origin, + R.id.version_one_newer, + R.id.version_one_time, + R.id.version_one_preview); + bindConflictVersion( + body, + presentation.alternative, + R.id.version_two_origin, + R.id.version_two_newer, + R.id.version_two_time, + R.id.version_two_preview); + + // The deterministic winner is what a sync already applied, so it starts selected: a user + // who taps through without reading changes nothing. + boolean[] keepWinner = {true}; + Runnable paint = + () -> { + firstCard.setChecked(keepWinner[0]); + secondCard.setChecked(!keepWinner[0]); + firstRadio.setChecked(keepWinner[0]); + secondRadio.setChecked(!keepWinner[0]); + }; + firstCard.setOnClickListener( + view -> { + keepWinner[0] = true; + paint.run(); + }); + secondCard.setOnClickListener( + view -> { + keepWinner[0] = false; + paint.run(); + }); + paint.run(); + new MaterialAlertDialogBuilder(this) .setTitle(getString(R.string.sync_conflict_title, unresolved.size())) - .setMessage(buildConflictMessage(conflict)) + .setView(body) .setNegativeButton(R.string.sync_conflict_later, null) - .setNeutralButton( - R.string.sync_conflict_keep_winner, - (dialog, which) -> resolveConflict(conflict.id, SyncResolution.KEEP_WINNER)) .setPositiveButton( - R.string.sync_conflict_keep_alternative, + R.string.sync_conflict_keep_selected, (dialog, which) -> - resolveConflict(conflict.id, SyncResolution.KEEP_ALTERNATIVE)) + resolveConflict( + conflict.id, + keepWinner[0] + ? SyncResolution.KEEP_WINNER + : SyncResolution.KEEP_ALTERNATIVE)) .show(); } + /** Fills one version card, highlighting the part that differs from the other version. */ + private void bindConflictVersion( + @NonNull View body, + @NonNull SyncConflictPresentation.Version version, + int originId, + int newerId, + int timeId, + int previewId) { + ((TextView) body.findViewById(originId)) + .setText( + version.local + ? R.string.sync_conflict_local_label + : R.string.sync_conflict_drive_label); + + TextView newer = body.findViewById(newerId); + newer.setText(R.string.sync_conflict_newer); + newer.setVisibility(version.newer ? View.VISIBLE : View.GONE); + + ((TextView) body.findViewById(timeId)) + .setText( + DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT) + .format(new Date(version.updatedAt))); + + TextView preview = body.findViewById(previewId); + preview.setText(conflictPreview(version)); + } + + /** + * Renders a version's preview, marking the differing span. + * + *

The difference is emphasised rather than the whole text restyled, because the point of the + * card is to answer "what changed" at a glance. + */ + @NonNull + private CharSequence conflictPreview(@NonNull SyncConflictPresentation.Version version) { + switch (version.kind) { + case DELETED: + return getString(R.string.sync_conflict_deleted); + case SETTINGS: + return getString(R.string.settings); + case UNTITLED: + return getString(R.string.sync_conflict_untitled); + default: + break; + } + if (!version.hasHighlight()) { + return version.preview; + } + SpannableString text = new SpannableString(version.preview); + int accent = + MaterialColors.getColor(this, androidx.appcompat.R.attr.colorPrimary, Color.GRAY); + text.setSpan( + new ForegroundColorSpan(accent), + version.highlightStart, + version.highlightEnd, + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new StyleSpan(android.graphics.Typeface.BOLD), + version.highlightStart, + version.highlightEnd, + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + return text; + } + private void resolveConflict(long conflictId, SyncResolution resolution) { syncCoordinator.resolveConflict( conflictId, @@ -644,83 +765,6 @@ private CharSequence formatLastSync(@NonNull SyncState state) { return getString(R.string.sync_last_sync_value, value); } - @NonNull - private String buildConflictMessage(@NonNull SyncConflictEntity conflict) { - return getString( - R.string.sync_conflict_version, - versionLabel(1, conflict.winnerSource), - describeConflictPayload(conflict.recordType, conflict.winnerJson)) - + "\n" - + getString( - R.string.sync_conflict_version, - versionLabel(2, conflict.loserSource), - describeConflictPayload(conflict.recordType, conflict.loserJson)); - } - - /** - * Names one side of a conflict by its position and its true origin. - * - *

A conflict between two Drive bundle heads has no local side, so the two versions are - * numbered and each is labelled with where it actually came from. Calling an arbitrary remote - * version "this device" told the user something untrue about data they were about to discard. - */ - @NonNull - private String versionLabel(int position, @NonNull String source) { - int origin = - "LOCAL".equals(source) - ? R.string.sync_conflict_local_label - : R.string.sync_conflict_drive_label; - return getString(R.string.sync_conflict_version_label, position, getString(origin)); - } - - @NonNull - private String describeConflictPayload(@NonNull String recordType, @NonNull String recordJson) { - if (SyncMetadata.RECORD_TYPE_PREFERENCES.equals(recordType)) { - return getString(R.string.settings); - } - try { - JsonObject root = JsonParser.parseString(recordJson).getAsJsonObject(); - JsonElement deletedAt = root.get("deletedAt"); - if (deletedAt != null && !deletedAt.isJsonNull()) { - return getString(R.string.sync_conflict_deleted); - } - JsonObject payload = root.getAsJsonObject("payload"); - if (payload == null) return getString(R.string.sync_conflict_deleted); - for (String key : conflictLabelKeys(recordType)) { - if (!payload.has(key) || payload.get(key).isJsonNull()) continue; - String value = payload.get(key).getAsString().trim(); - if (value.isEmpty()) continue; - return value.length() > 120 ? value.substring(0, 120) + "…" : value; - } - } catch (Exception ignored) { - } - return getString(R.string.sync_conflict_untitled); - } - - /** - * Payload keys that carry a human-readable label, most specific first. - * - *

Note and Tag are serialized through Gson's short field aliases, so probing "title" and - * "name" never matched them: every note and tag conflict showed the same placeholder for both - * the local and the Drive version, leaving no way to tell them apart before choosing one. - */ - @NonNull - private static String[] conflictLabelKeys(@NonNull String recordType) { - if (SyncMetadata.RECORD_TYPE_NOTE.equals(recordType)) { - return new String[] {"b", "c"}; // Note.title, Note.value - } - if (SyncMetadata.RECORD_TYPE_TAG.equals(recordType)) { - return new String[] {"b"}; // Tag.nameTag - } - if (SyncMetadata.RECORD_TYPE_TASK.equals(recordType)) { - return new String[] {"title", "description"}; - } - if (SyncMetadata.RECORD_TYPE_CATEGORY.equals(recordType)) { - return new String[] {"name"}; - } - return new String[0]; - } - private int unresolvedConflictCount(@NonNull List conflicts) { return unresolvedConflicts(conflicts).size(); } diff --git a/app/src/main/res/layout/dialog_sync_conflict.xml b/app/src/main/res/layout/dialog_sync_conflict.xml new file mode 100644 index 00000000..08324604 --- /dev/null +++ b/app/src/main/res/layout/dialog_sync_conflict.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/res/values-be/strings.xml b/app/src/main/res/values-be/strings.xml index 4b6b0d20..68c56eee 100644 --- a/app/src/main/res/values-be/strings.xml +++ b/app/src/main/res/values-be/strings.xml @@ -421,15 +421,12 @@ Праглядзець канфлікты Праглядзець канфлікты (%1$d) Вырашыць канфлікты сінхранізацыі (%1$d засталося) - Пакінуць лакальную версію - Пакінуць версію з Google Drive Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Пазней + Keep one version — the other will be discarded. + Keep selected + newer Канфлікт сінхранізацыі вырашаны - %1$s • %2$s Лакальная версія Версія з Google Drive Выдалена diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index ce2a8058..ec18783e 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -421,15 +421,12 @@ Konflikte prüfen Konflikte prüfen (%1$d) Synchronisierungskonflikte lösen (%1$d verbleibend) - Lokale Version behalten - Google-Drive-Version behalten Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Später + Keep one version — the other will be discarded. + Keep selected + newer Synchronisierungskonflikt gelöst - %1$s • %2$s Lokale Version Google-Drive-Version Gelöscht diff --git a/app/src/main/res/values-en-rGB/strings.xml b/app/src/main/res/values-en-rGB/strings.xml index 279bfad8..822ef736 100644 --- a/app/src/main/res/values-en-rGB/strings.xml +++ b/app/src/main/res/values-en-rGB/strings.xml @@ -465,15 +465,12 @@ Review conflicts Review conflicts (%1$d) Resolve sync conflicts (%1$d remaining) - Keep local version - Keep Google Drive version Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Later + Keep one version — the other will be discarded. + Keep selected + newer Sync conflict resolved - %1$s • %2$s Local version Google Drive version Deleted diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 03ea467b..2555e28b 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -422,15 +422,12 @@ Revisar conflictos Revisar conflictos (%1$d) Resolver conflictos de sincronización (%1$d restantes) - Mantener versión local - Mantener versión de Google Drive Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Más tarde + Keep one version — the other will be discarded. + Keep selected + newer Conflicto de sincronización resuelto - %1$s • %2$s Versión local Versión de Google Drive Eliminado diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index 64981446..a25d246f 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -417,15 +417,12 @@ Examiner les conflits Examiner les conflits (%1$d) Résoudre les conflits de synchronisation (%1$d restants) - Conserver la version locale - Conserver la version Google Drive Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Plus tard + Keep one version — the other will be discarded. + Keep selected + newer Conflit de synchronisation résolu - %1$s • %2$s Version locale Version Google Drive Supprimé diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 0ab45b3b..9122efc6 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -419,15 +419,12 @@ Rivedi conflitti Rivedi conflitti (%1$d) Risolvi i conflitti di sincronizzazione (%1$d rimanenti) - Mantieni la versione locale - Mantieni la versione Google Drive Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Più tardi + Keep one version — the other will be discarded. + Keep selected + newer Conflitto di sincronizzazione risolto - %1$s • %2$s Versione locale Versione Google Drive Eliminato diff --git a/app/src/main/res/values-kk/strings.xml b/app/src/main/res/values-kk/strings.xml index 58929e7f..ea6646ca 100644 --- a/app/src/main/res/values-kk/strings.xml +++ b/app/src/main/res/values-kk/strings.xml @@ -418,15 +418,12 @@ Қайшылықтарды қарау Қайшылықтарды қарау (%1$d) Синхрондау қайшылықтарын шешу (%1$d қалды) - Жергілікті нұсқаны сақтау - Google Drive нұсқасын сақтау Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Кейінірек + Keep one version — the other will be discarded. + Keep selected + newer Синхрондау қайшылығы шешілді - %1$s • %2$s Жергілікті нұсқа Google Drive нұсқасы Жойылды diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index 8b768203..4f55dfd3 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -422,15 +422,12 @@ Przejrzyj konflikty Przejrzyj konflikty (%1$d) Rozwiąż konflikty synchronizacji (%1$d pozostało) - Zachowaj wersję lokalną - Zachowaj wersję z Google Drive Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Później + Keep one version — the other will be discarded. + Keep selected + newer Konflikt synchronizacji rozwiązany - %1$s • %2$s Wersja lokalna Wersja z Google Drive Usunięto diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 0145147e..2e3dec06 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -426,15 +426,12 @@ Просмотреть конфликты Просмотреть конфликты (%1$d) Разрешить конфликты синхронизации (%1$d осталось) - Оставить локальную версию - Оставить версию из Google Drive Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Позже + Keep one version — the other will be discarded. + Keep selected + newer Конфликт синхронизации разрешён - %1$s • %2$s Локальная версия Версия из Google Drive Удалено diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index bf832f51..2c48418a 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -463,15 +463,12 @@ Переглянути конфлікти Переглянути конфлікти (%1$d) Розв’язати конфлікти синхронізації (%1$d залишилось) - Залишити локальну версію - Залишити версію з Google Drive Застосовано налаштування з Google Drive - Keep version 1 - Keep version 2 - Version %1$d — %2$s Пізніше + Залиште одну версію — друга буде відкинута. + Залишити обрану + новіша Конфлікт синхронізації розв’язано - %1$s • %2$s Локальна версія Версія з Google Drive Видалено diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 8a2572b6..1a722f9d 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -343,15 +343,12 @@ Review conflicts Review conflicts (%1$d) Resolve sync conflicts (%1$d remaining) - Keep local - Keep Drive Settings received from Google Drive applied - Keep version 1 - Keep version 2 - Version %1$d — %2$s Later + Keep one version — the other will be discarded. + Keep selected + newer Conflict resolved locally - %1$s: %2$s Local Drive Deleted version diff --git a/app/src/test/java/com/pasich/mynotes/ui/sync/SyncConflictPresentationTest.java b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncConflictPresentationTest.java new file mode 100644 index 00000000..2c5ac0ed --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/ui/sync/SyncConflictPresentationTest.java @@ -0,0 +1,274 @@ +package com.pasich.mynotes.ui.sync; + +import static com.google.common.truth.Truth.assertThat; + +import com.pasich.mynotes.data.database.entities.SyncConflictEntity; +import com.pasich.mynotes.data.sync.SyncMetadata; +import org.junit.Test; + +/** + * What the conflict dialog shows for a stored conflict. + * + *

The old dialog rendered both versions into one string with no timestamps and cut the text at a + * fixed 120 characters from the start, so a difference near the end of a note never reached the + * screen. These are the rules that replace it. + */ +public class SyncConflictPresentationTest { + + private static final String NOTE = "550e8400-e29b-41d4-a716-446655440000"; + + @Test + public void namesEachSideByItsOwnOrigin() { + SyncConflictPresentation presentation = + SyncConflictPresentation.of( + conflict( + "note", + "LOCAL", + "REMOTE", + note("Список", "Молоко"), + note("Список", "Хліб"))); + + assertThat(presentation.winner.local).isTrue(); + assertThat(presentation.alternative.local).isFalse(); + } + + @Test + public void aDriveVersusDriveConflictClaimsNeitherSideIsLocal() { + SyncConflictPresentation presentation = + SyncConflictPresentation.of( + conflict("note", "REMOTE", "REMOTE", note("A", "one"), note("A", "two"))); + + // Naming an arbitrary remote version "this device" is what the version-addressed + // resolution model exists to stop; the dialog must not reintroduce it. + assertThat(presentation.winner.local).isFalse(); + assertThat(presentation.alternative.local).isFalse(); + } + + @Test + public void marksTheNewerSideWhicheverItIs() { + SyncConflictEntity newerWinner = + conflict("note", "LOCAL", "REMOTE", note("A", "x"), note("A", "y")); + newerWinner.winnerUpdatedAt = 200L; + newerWinner.loserUpdatedAt = 100L; + + SyncConflictPresentation presentation = SyncConflictPresentation.of(newerWinner); + + assertThat(presentation.winner.newer).isTrue(); + assertThat(presentation.alternative.newer).isFalse(); + assertThat(presentation.winner.updatedAt).isEqualTo(200L); + assertThat(presentation.alternative.updatedAt).isEqualTo(100L); + } + + @Test + public void marksTheAlternativeNewerWhenItIs() { + SyncConflictEntity newerAlternative = + conflict("note", "LOCAL", "REMOTE", note("A", "x"), note("A", "y")); + newerAlternative.winnerUpdatedAt = 100L; + newerAlternative.loserUpdatedAt = 300L; + + SyncConflictPresentation presentation = SyncConflictPresentation.of(newerAlternative); + + assertThat(presentation.winner.newer).isFalse(); + assertThat(presentation.alternative.newer).isTrue(); + } + + @Test + public void readsNoteTitleAndBodyThroughTheirSerializedAliases() { + SyncConflictPresentation presentation = + SyncConflictPresentation.of( + conflict( + "note", + "LOCAL", + "REMOTE", + note("Список покупок", "Молоко"), + note("Список покупок", "Хліб"))); + + // Note serializes title as "b" and value as "c"; probing "title"/"value" matched nothing + // and every note conflict showed the same placeholder on both sides. + assertThat(presentation.winner.kind).isEqualTo(SyncConflictPresentation.Kind.TEXT); + assertThat(presentation.winner.preview).contains("Список покупок"); + assertThat(presentation.winner.preview).contains("Молоко"); + assertThat(presentation.alternative.preview).contains("Хліб"); + } + + @Test + public void highlightsOnlyThePartThatDiffers() { + SyncConflictPresentation presentation = + SyncConflictPresentation.of( + conflict( + "note", + "LOCAL", + "REMOTE", + note("Покупки", "Молоко, хліб, кава"), + note("Покупки", "Молоко, хліб, сир"))); + + String winner = presentation.winner.preview; + assertThat(presentation.winner.hasHighlight()).isTrue(); + assertThat( + winner.substring( + presentation.winner.highlightStart, + presentation.winner.highlightEnd)) + .isEqualTo("кава"); + String alternative = presentation.alternative.preview; + assertThat( + alternative.substring( + presentation.alternative.highlightStart, + presentation.alternative.highlightEnd)) + .isEqualTo("сир"); + } + + @Test + public void keepsADifferenceVisibleEvenWhenItIsPastThePreviewLimit() { + String shared = repeat("одне й те саме ", 30); + SyncConflictPresentation presentation = + SyncConflictPresentation.of( + conflict( + "note", + "LOCAL", + "REMOTE", + note("Довга", shared + "КАВА"), + note("Довга", shared + "СИР"))); + + // Cutting the head of the string is exactly how the old dialog hid this. + assertThat(presentation.winner.preview.length()) + .isAtMost(SyncConflictPresentation.PREVIEW_LIMIT + 2); + assertThat(presentation.winner.preview).contains("КАВА"); + assertThat( + presentation.winner.preview.substring( + presentation.winner.highlightStart, + presentation.winner.highlightEnd)) + .isEqualTo("КАВА"); + } + + @Test + public void reportsADeletedVersionAsADeletion() { + SyncConflictEntity conflict = + conflict("note", "LOCAL", "REMOTE", note("A", "body"), tombstone()); + conflict.loserTombstone = true; + + SyncConflictPresentation presentation = SyncConflictPresentation.of(conflict); + + assertThat(presentation.alternative.kind).isEqualTo(SyncConflictPresentation.Kind.DELETED); + assertThat(presentation.winner.kind).isEqualTo(SyncConflictPresentation.Kind.TEXT); + } + + @Test + public void reportsAPreferencesConflictAsSettings() { + SyncConflictPresentation presentation = + SyncConflictPresentation.of( + conflict( + SyncMetadata.RECORD_TYPE_PREFERENCES, + "LOCAL", + "REMOTE", + "{\"payload\":{\"c\":1}}", + "{\"payload\":{\"c\":2}}")); + + assertThat(presentation.winner.kind).isEqualTo(SyncConflictPresentation.Kind.SETTINGS); + assertThat(presentation.alternative.kind).isEqualTo(SyncConflictPresentation.Kind.SETTINGS); + } + + @Test + public void reportsAReadableButEmptyVersionAsUntitled() { + SyncConflictPresentation presentation = + SyncConflictPresentation.of( + conflict("note", "LOCAL", "REMOTE", note("", ""), note("", ""))); + + assertThat(presentation.winner.kind).isEqualTo(SyncConflictPresentation.Kind.UNTITLED); + } + + @Test + public void survivesUnreadableStoredJson() { + SyncConflictPresentation presentation = + SyncConflictPresentation.of( + conflict("note", "LOCAL", "REMOTE", "{not json", note("A", "b"))); + + // A corrupt row must still render something rather than take the dialog down. + assertThat(presentation.winner.kind).isEqualTo(SyncConflictPresentation.Kind.UNTITLED); + assertThat(presentation.alternative.kind).isEqualTo(SyncConflictPresentation.Kind.TEXT); + } + + @Test + public void identicalTextProducesNoHighlight() { + SyncConflictPresentation presentation = + SyncConflictPresentation.of( + conflict("note", "LOCAL", "REMOTE", note("A", "same"), note("A", "same"))); + + assertThat(presentation.winner.hasHighlight()).isFalse(); + assertThat(presentation.alternative.hasHighlight()).isFalse(); + } + + @Test + public void differenceRangeFindsTheChangedMiddle() { + int[] range = SyncConflictPresentation.differenceRange("abcXYZdef", "abcQdef"); + + assertThat(range[0]).isEqualTo(3); + assertThat("abcXYZdef".substring(range[0], range[1])).isEqualTo("XYZ"); + assertThat("abcQdef".substring(range[0], range[2])).isEqualTo("Q"); + } + + @Test + public void differenceRangeHandlesAPureAppend() { + int[] range = SyncConflictPresentation.differenceRange("abc", "abcdef"); + + assertThat("abc".substring(range[0], range[1])).isEmpty(); + assertThat("abcdef".substring(range[0], range[2])).isEqualTo("def"); + } + + @Test + public void differenceRangeHandlesAnEmptySide() { + int[] range = SyncConflictPresentation.differenceRange("", "abc"); + + assertThat(range[0]).isEqualTo(0); + assertThat("abc".substring(range[0], range[2])).isEqualTo("abc"); + } + + private static String repeat(String value, int times) { + StringBuilder result = new StringBuilder(value.length() * times); + for (int i = 0; i < times; i++) result.append(value); + return result.toString(); + } + + private static String note(String title, String body) { + return "{\"type\":\"note\",\"id\":\"" + + NOTE + + "\",\"updatedAt\":\"2026-08-31T12:00:00Z\",\"deletedAt\":null," + + "\"payload\":{\"b\":\"" + + title + + "\",\"c\":\"" + + body + + "\"}}"; + } + + private static String tombstone() { + return "{\"type\":\"note\",\"id\":\"" + + NOTE + + "\",\"updatedAt\":\"2026-08-31T12:00:00Z\"," + + "\"deletedAt\":\"2026-08-31T12:00:05Z\"}"; + } + + private static SyncConflictEntity conflict( + String recordType, + String winnerSource, + String loserSource, + String winnerJson, + String loserJson) { + return new SyncConflictEntity( + recordType, + NOTE, + "pair", + winnerSource, + loserSource, + "winner-version", + "loser-version", + winnerJson, + loserJson, + 200L, + 100L, + false, + false, + "PENDING", + false, + 1L, + 0L); + } +} From dbe18bdc25365f43662727e4d50181e5e349ebbe Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 19:38:43 +0300 Subject: [PATCH 12/16] fix(sync): stop every attachment-free note conflicting with itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by running a real two-device sync against Google Drive rather than the fake server: a second sync with nothing changed in between reported a conflict for every note, both sides showing identical text and identical timestamps. Two asymmetries between a locally built record and the same record decoded back from a bundle, either of which is enough to make the canonical hashes differ: - addAttachmentMetadata returned early only when the attachments column was null or blank. The editor stores "[]" for a note that simply has none, so the loop ran zero times and still wrote attachmentsManifest, attachmentHashes and attachmentNames as empty. A decoded record carries no attachment fields at all. Empty ones are no longer written. - normalizeNoteAttachmentFields removed attachmentsManifest and attachmentHashes before rebuilding them, but not attachmentNames, which it only re-added when non-empty. A payload that already carried the key kept it on the wire, so the decoded record differed from the local one that produced it. It is now cleared like the other two. The effect was a conflict per note on every sync, forever, with a fresh bundle republished each time — for any note without an attachment, which is most of them. An existing test asserted attachmentHashes was an empty array for a note with no attachments, pinning the first half of this in place; it now asserts the fields are absent. Added round-trip tests that encode a locally built record and require the decoded one to hash identically, at the codec level and end to end from the Room store on a device. --- .../pasich/mynotes/db/RoomSyncStoreTest.java | 51 +++++++++++++++++-- .../mynotes/data/sync/RoomSyncStore.java | 8 +++ .../mynotes/data/sync/SyncBundleCodec.java | 5 ++ .../data/sync/SyncBundleCodecTest.java | 35 +++++++++++++ 4 files changed, 94 insertions(+), 5 deletions(-) diff --git a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java index fc97f21b..6f1d88db 100644 --- a/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java +++ b/app/src/androidTest/java/com/pasich/mynotes/db/RoomSyncStoreTest.java @@ -200,11 +200,13 @@ public void readSnapshot_acceptsANoteWithNoAttachments() throws Exception { SnapshotBuildResult result = store.buildSnapshot(); assertThat(result.isPublishable()).isTrue(); - assertThat( - onlyNote(result.requireSnapshot()) - .getPayload() - .getAsJsonArray("attachmentHashes")) - .isEmpty(); + // Absent, not an empty array: a decoded remote record carries no attachment fields at + // all, so emitting empty ones here made the two shapes hash differently and every + // attachment-free note conflicted with itself on every sync. + com.google.gson.JsonObject payload = onlyNote(result.requireSnapshot()).getPayload(); + assertThat(payload.has("attachmentHashes")).isFalse(); + assertThat(payload.has("attachmentsManifest")).isFalse(); + assertThat(payload.has("attachmentNames")).isFalse(); } @Test @@ -325,6 +327,45 @@ private File resolveFirstAttachment(String attachmentsJson) { return com.pasich.mynotes.extendedEditor.attach.AttachmentStorage.resolve(context, url); } + @Test + public void aLocallyBuiltNoteSurvivesABundleRoundTripUnchanged() throws Exception { + // The editor stores "[]" for a note that simply has no attachments. + int noteId = seedNote("Alpha note", "milk bread coffee", "[]"); + assertThat(noteId).isGreaterThan(0); + + SyncRecord local = onlyNote(store.readSnapshot()); + com.pasich.mynotes.data.sync.SyncBundleCodec codec = + new com.pasich.mynotes.data.sync.SyncBundleCodec(); + byte[] bundle = codec.encode(store.readSnapshot(), java.time.Instant.now()); + SyncRecord decoded = + codec.decode(new ByteArrayInputStream(bundle)) + .getSnapshot() + .find(SyncRecord.Type.NOTE, local.getId()); + + // Empty attachment arrays were written locally but never survive the wire, so the two + // shapes hashed differently and every attachment-free note conflicted with itself on + // every sync — reproduced on a device before this was fixed. + assertThat(decoded).isNotNull(); + assertThat(decoded.getCanonicalPayloadHash()).isEqualTo(local.getCanonicalPayloadHash()); + } + + @Test + public void aNoteWithAnAttachmentAlsoSurvivesTheRoundTripUnchanged() throws Exception { + seedNoteWithAttachment("photo.png", "photo bytes".getBytes(StandardCharsets.UTF_8)); + + SyncRecord local = onlyNote(store.readSnapshot()); + com.pasich.mynotes.data.sync.SyncBundleCodec codec = + new com.pasich.mynotes.data.sync.SyncBundleCodec(); + byte[] bundle = codec.encode(store.readSnapshot(), java.time.Instant.now()); + SyncRecord decoded = + codec.decode(new ByteArrayInputStream(bundle)) + .getSnapshot() + .find(SyncRecord.Type.NOTE, local.getId()); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getCanonicalPayloadHash()).isEqualTo(local.getCanonicalPayloadHash()); + } + @Test public void clearAfterDisconnect_dropsStatusConflictsAndCachedBlobs() throws Exception { store.writeState(SyncState.success("google-drive", java.time.Instant.now(), 0)); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java index c8241551..0cd191a7 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/RoomSyncStore.java @@ -1274,6 +1274,14 @@ private boolean addAttachmentMetadata( manifest.add(manifestEntry); } if (!complete) return false; + if (manifest.size() == 0) { + // A note whose attachments column is "[]" — which is what the editor stores for a + // note that simply has none — used to get three empty arrays here, while a decoded + // remote record carries no attachment fields at all. The two shapes hashed + // differently, so every attachment-free note reported a conflict against itself on + // every sync and republished a bundle each time. + return true; + } payload.add("attachmentsManifest", manifest); payload.add("attachmentHashes", hashes); payload.add("attachmentNames", names); diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java index 925a3167..98e089c7 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncBundleCodec.java @@ -222,6 +222,11 @@ private static void normalizeNoteAttachmentFields(JsonObject note) throws IOExce } note.remove("attachmentsManifest"); note.remove("attachmentHashes"); + // Cleared as well as rebuilt. Only the two above were removed, so a payload that already + // carried an attachmentNames key kept it on the wire even when the rebuilt map was + // empty, and a decoded record then hashed differently from the local one that produced + // it — a conflict against itself on every sync. + note.remove("attachmentNames"); if (attachmentIds.size() > 0) { note.add("attachmentIds", attachmentIds); } diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java index b4158fc6..e157bcbb 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncBundleCodecTest.java @@ -268,6 +268,41 @@ public void roundTrip_ofALocallyBuiltNoteWithAnAttachment_hashesIdentically() th .isEqualTo(localRecord.getCanonicalPayloadHash()); } + @Test + public void encode_dropsEmptyAttachmentFieldsInsteadOfLeakingThemToTheWire() throws Exception { + // The shape an older client wrote for a note with no attachments. + JsonObject payload = new JsonObject(); + payload.addProperty("title", "Shopping"); + payload.addProperty("value", "Body"); + payload.add("attachmentsManifest", new JsonArray()); + payload.add("attachmentHashes", new JsonArray()); + payload.add("attachmentNames", new JsonObject()); + SyncRecord local = + SyncRecord.live( + SyncRecord.Type.NOTE, + NOTE_ID, + Instant.parse("2026-08-31T12:00:01Z"), + payload); + + SyncBundleCodec codec = new SyncBundleCodec(); + byte[] bundle = + codec.encode( + new SyncSnapshot(java.util.Collections.singletonList(local)), + Instant.parse("2026-08-31T12:00:00Z")); + SyncRecord decoded = + codec.decode(new ByteArrayInputStream(bundle)) + .getSnapshot() + .find(SyncRecord.Type.NOTE, NOTE_ID); + + // None of the three may survive: a decoded record carries no attachment fields for a + // note without attachments, so leaving one behind makes the two shapes hash differently. + assertThat(decoded.getPayload().has("attachmentNames")).isFalse(); + assertThat(decoded.getPayload().has("attachmentsManifest")).isFalse(); + assertThat(decoded.getPayload().has("attachmentHashes")).isFalse(); + assertThat(unzipToStrings(bundle).get(SyncBundleCodec.ENTRY_RECORDS)) + .doesNotContain("attachmentNames"); + } + private static SyncRecord task(String title) { JsonObject payload = new JsonObject(); payload.addProperty("title", title); From 201528151aa4d29fdf1d589cdaebe0638ad3ecda Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 20:07:08 +0300 Subject: [PATCH 13/16] fix(backup): stop a restore destroying one of two incoming notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reproduced on a device: a backup holding notes with ids 1 and 2, restored onto a library where id 1 is taken, ended with one of the two gone and no error. addNotes and addTags are REPLACE inserts. A row whose id is already taken has its id cleared so it is inserted as new, but that happened in the same batch as rows keeping an explicit id, so the reassigned row was handed the next autoincrement value — which was exactly the id a later row in the same batch then claimed. The REPLACE overwrote it, silently. Rows are now inserted in two groups, the ones keeping their id first, which leaves the autoincrement counter past every explicit id in the batch. Restoring onto an empty library still preserves every id exactly, and each row's attachment relocation and metadata are settled per group. Verified on the device that the same restore now keeps all three notes: the local one untouched, the free id preserved, the reassigned one placed after it. --- .../data/sync/SyncMutationCoordinator.java | 132 ++++++++++-------- .../sync/SyncMutationCoordinatorTest.java | 47 +++++++ 2 files changed, 119 insertions(+), 60 deletions(-) diff --git a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java index 842aeb22..d3bc990b 100644 --- a/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java +++ b/app/src/main/java/com/pasich/mynotes/data/sync/SyncMutationCoordinator.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; import javax.inject.Inject; @@ -172,18 +173,36 @@ public void insertTags(List incoming) { long timestamp = resolveBatchTimestamp( SyncMetadata.RECORD_TYPE_TAG, extractTagIds(tags)); - releaseTakenTagIds(tags); - long[] insertedIds = tagsDao.addTags(tags); - for (int i = 0; i < tags.size(); i++) { - Tag tag = tags.get(i); - long localId = resolveLongId(tag.getId(), insertedIds[i]); - tag.id = localId; - touchRecord(SyncMetadata.RECORD_TYPE_TAG, localId, timestamp); + + // Same REPLACE-insert collision as notes; see insertNotes. + List keepingId = new ArrayList<>(); + List reassigned = new ArrayList<>(); + for (Tag tag : tags) { + if (tag.getId() > 0 && tagsDao.getTagSync(tag.getId()) != null) { + tag.id = 0; + reassigned.add(tag); + } else { + keepingId.add(tag); + } } + assignInsertedTagIds(keepingId, timestamp); + assignInsertedTagIds(reassigned, timestamp); return null; }); } + /** Inserts one group of tags and settles each tag's final id and metadata. */ + private void assignInsertedTagIds(@NonNull List tags, long timestamp) { + if (tags.isEmpty()) return; + long[] insertedIds = tagsDao.addTags(tags); + for (int i = 0; i < tags.size(); i++) { + Tag tag = tags.get(i); + long localId = resolveLongId(tag.getId(), insertedIds[i]); + tag.id = localId; + touchRecord(SyncMetadata.RECORD_TYPE_TAG, localId, timestamp); + } + } + public void updateTag(@NonNull Tag tag) { transactionExecutor.run( () -> { @@ -270,35 +289,58 @@ public void insertNotes(List incoming) { long timestamp = resolveBatchTimestamp( SyncMetadata.RECORD_TYPE_NOTE, extractNoteIds(notes)); - int[] previousIds = new int[notes.size()]; - for (int i = 0; i < notes.size(); i++) { - previousIds[i] = notes.get(i).getId(); - } - releaseTakenNoteIds(notes); - long[] insertedIds = noteDao.addNotes(notes); - for (int i = 0; i < notes.size(); i++) { - Note note = notes.get(i); - int localId = resolveIntId(note.getId(), insertedIds[i]); - note.setId(localId); - if (previousIds[i] > 0 && previousIds[i] != localId) { - // Its attachments were extracted under the old id and would - // otherwise share a folder with whichever note owns that id now. - attachmentRelocation.relocate(note, previousIds[i]); - noteDao.updateNoteContent( - localId, - note.getTitle(), - note.getValue(), - note.getValueJson(), - note.getDate(), - note.getTag(), - note.getAttachments()); + + // Split by whether the row id is still free. Inserting the ones that keep + // their id first leaves the autoincrement counter past all of them, which is + // what stops a reassigned note being handed an id a later note in the same + // batch is about to claim: addNotes is a REPLACE insert, so that collision + // silently destroyed one of the two restored notes. + List keepingId = new ArrayList<>(); + List reassigned = new ArrayList<>(); + Map previousIds = new java.util.IdentityHashMap<>(); + for (Note note : notes) { + previousIds.put(note, note.getId()); + if (note.getId() > 0 && noteDao.getNoteSync(note.getId()) != null) { + note.setId(0); + reassigned.add(note); + } else { + keepingId.add(note); } - touchRecord(SyncMetadata.RECORD_TYPE_NOTE, localId, timestamp); } + + assignInsertedNoteIds(keepingId, timestamp, previousIds); + assignInsertedNoteIds(reassigned, timestamp, previousIds); return null; }); } + /** Inserts one group and settles each note's final id, metadata and attachment folder. */ + private void assignInsertedNoteIds( + @NonNull List notes, long timestamp, @NonNull Map previousIds) { + if (notes.isEmpty()) return; + long[] insertedIds = noteDao.addNotes(notes); + for (int i = 0; i < notes.size(); i++) { + Note note = notes.get(i); + int previous = previousIds.get(note); + int localId = resolveIntId(note.getId(), insertedIds[i]); + note.setId(localId); + if (previous > 0 && previous != localId) { + // Its attachments were extracted under the old id and would otherwise share a + // folder with whichever note owns that id now. + attachmentRelocation.relocate(note, previous); + noteDao.updateNoteContent( + localId, + note.getTitle(), + note.getValue(), + note.getValueJson(), + note.getDate(), + note.getTag(), + note.getAttachments()); + } + touchRecord(SyncMetadata.RECORD_TYPE_NOTE, localId, timestamp); + } + } + public void updateNoteContent(@NonNull Note note) { transactionExecutor.run( () -> { @@ -636,36 +678,6 @@ private static boolean equalText(String first, String second) { return first == null ? second == null : first.equals(second); } - /** - * Lets a restore keep its original IDs only where they are still free. - * - *

Backups carry the IDs the notes had when the backup was taken, and {@code addNotes} is a - * REPLACE insert. Restoring onto a device that already holds notes therefore destroyed every - * note whose ID happened to collide — silently, with no way back. Sync made that worse: the - * restored content inherited the destroyed note's stable ID through {@code ensureMetadataRow}, - * {@code touch()} cleared its tombstone, and the replacement propagated to every other device, - * overwriting the cloud copy too. - * - *

A colliding note is now inserted as a new row instead. Restoring onto an empty library — - * the ordinary case, and the one after a reinstall — still preserves every ID exactly. - */ - private void releaseTakenNoteIds(@NonNull List notes) { - for (Note note : notes) { - if (note.getId() > 0 && noteDao.getNoteSync(note.getId()) != null) { - note.setId(0); - } - } - } - - /** Same protection for a restored tag list; see {@link #releaseTakenNoteIds}. */ - private void releaseTakenTagIds(@NonNull List tags) { - for (Tag tag : tags) { - if (tag.getId() > 0 && tagsDao.getTagSync(tag.getId()) != null) { - tag.id = 0; - } - } - } - private void touchRecords(@NonNull String recordType, List localIds, long timestamp) { if (localIds == null || localIds.isEmpty()) return; for (Integer localId : localIds) { diff --git a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java index d96b5f04..876fb0ec 100644 --- a/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java +++ b/app/src/test/java/com/pasich/mynotes/data/sync/SyncMutationCoordinatorTest.java @@ -64,6 +64,53 @@ public T run( new QueueStableIdGenerator("stable-a", "stable-b", "stable-c")); } + @Test + public void insertNotes_insertsIdKeepingNotesBeforeReassignedOnes() { + // A restore where one incoming id is taken and another is free. addNotes is a REPLACE + // insert, so if the reassigned note is inserted first it takes the next autoincrement id + // — which is exactly the id the second note is about to claim — and one of the two is + // silently destroyed. Reproduced on a device before this ordering was introduced. + Note taken = new Note().create("Alpha", "body", 10L, ""); + taken.setId(1); + Note free = new Note().create("Beta", "body", 20L, ""); + free.setId(2); + when(noteDao.getNoteSync(1)).thenReturn(new Note().create("Occupant", "", 5L, "")); + when(noteDao.getNoteSync(2)).thenReturn(null); + when(noteDao.addNotes(org.mockito.ArgumentMatchers.anyList())) + .thenReturn(new long[] {2L}) + .thenReturn(new long[] {3L}); + + coordinator.insertNotes(new java.util.ArrayList<>(java.util.List.of(taken, free))); + + org.mockito.ArgumentCaptor batches = + org.mockito.ArgumentCaptor.forClass(java.util.List.class); + verify(noteDao, org.mockito.Mockito.times(2)).addNotes(batches.capture()); + java.util.List captured = batches.getAllValues(); + assertThat(((Note) captured.get(0).get(0)).getTitle()).isEqualTo("Beta"); + assertThat(((Note) captured.get(1).get(0)).getTitle()).isEqualTo("Alpha"); + // Both survive, with the reassigned one placed beyond the id the other kept. + assertThat(free.getId()).isEqualTo(2); + assertThat(taken.getId()).isEqualTo(3); + } + + @Test + public void insertNotes_keepsEveryIdWhenNoneAreTaken() { + Note first = new Note().create("One", "body", 10L, ""); + first.setId(4); + Note second = new Note().create("Two", "body", 20L, ""); + second.setId(5); + when(noteDao.addNotes(org.mockito.ArgumentMatchers.anyList())) + .thenReturn(new long[] {4L, 5L}); + + coordinator.insertNotes(new java.util.ArrayList<>(java.util.List.of(first, second))); + + // The ordinary restore onto an empty library must still preserve ids exactly. + verify(noteDao, org.mockito.Mockito.times(1)) + .addNotes(org.mockito.ArgumentMatchers.anyList()); + assertThat(first.getId()).isEqualTo(4); + assertThat(second.getId()).isEqualTo(5); + } + @Test public void insertNotes_skipsANoteThisDeviceAlreadyHasUnchanged() { // Restoring a backup onto the library it came from must stay a no-op: restore inserts From d1d1f48ebc62cf0f71d0f6224c380f1b8cb59010 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 20:55:05 +0300 Subject: [PATCH 14/16] fix: remove doubled status bar inset on tasks and help MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setupEdgeToEdgeInsets pads the root by systemBars.top and returns the insets unconsumed, so CoordinatorLayout went on dispatching them to its children. Both layouts also set fitsSystemWindows on their AppBarLayout, which applied the same top inset a second time — 118px of status bar became 236px, and the dead strip above the toolbar read as an empty second app bar. Every other screen carries fitsSystemWindows on the root alone; these two now match. Verified on device: the toolbar starts at the status bar edge (118) instead of below a blank band (236). --- app/src/main/res/layout/activity_help.xml | 1 - app/src/main/res/layout/activity_tasks.xml | 1 - 2 files changed, 2 deletions(-) diff --git a/app/src/main/res/layout/activity_help.xml b/app/src/main/res/layout/activity_help.xml index 95b9e40c..308f9fc1 100644 --- a/app/src/main/res/layout/activity_help.xml +++ b/app/src/main/res/layout/activity_help.xml @@ -14,7 +14,6 @@ android:layout_width="match_parent" android:layout_height="wrap_content" android:background="?attr/colorSurface" - android:fitsSystemWindows="true" app:elevation="0dp"> From c75b3e46b7b78d0954fb7d7d311abe69fd2df746 Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 21:25:05 +0300 Subject: [PATCH 15/16] feat: hide sync when the device has no Google Play services Sign-in runs through Credential Manager and the Drive scope is granted by the authorization client, so both sit on Play services. Nothing checked whether they were there: on a device without them every control on the account tab led to a failure the user could not act on. The check reads the package table rather than asking GoogleApiAvailability. That needs no dependency and, unlike loading a Play services class, cannot fail for the reason it is checking. The manifest declares the package under because API 30+ package visibility would otherwise report it absent on every device. isConfigured() already gates create(), so an unavailable device takes the path a build without google-services.json takes. That path hid both groups and left the tab blank, which reads as a broken screen; it now shows what is wrong, translated into all eleven locales. GoogleDriveSyncWorker bails on the same check and moves its Firebase calls inside the try. A scheduled worker that throws is rerun and takes the process down in the background, where nobody can see why. Verified on device: with Play services present the account tab and a real sync are unchanged; with the lookup pointed at an absent package the notice renders and the backup and import tabs keep working. 248 unit tests, 60 instrumentation tests, 0 failures; lint and R8 clean. --- app/src/main/AndroidManifest.xml | 4 + .../data/sync/GoogleDriveSyncWorker.java | 23 ++++-- .../ui/sync/SyncCoordinatorFactory.java | 14 +++- .../fragment/mydata/AccountSyncFragment.java | 4 + .../utils/auth/PlayServicesAvailability.java | 78 +++++++++++++++++++ .../main/res/layout/fragment_account_sync.xml | 37 +++++++++ app/src/main/res/values-be/strings.xml | 2 + app/src/main/res/values-de/strings.xml | 2 + app/src/main/res/values-en-rGB/strings.xml | 2 + app/src/main/res/values-es/strings.xml | 2 + app/src/main/res/values-fr/strings.xml | 2 + app/src/main/res/values-it/strings.xml | 2 + app/src/main/res/values-kk/strings.xml | 2 + app/src/main/res/values-pl/strings.xml | 2 + app/src/main/res/values-ru/strings.xml | 2 + app/src/main/res/values-uk/strings.xml | 2 + app/src/main/res/values/strings.xml | 2 + .../auth/PlayServicesAvailabilityTest.java | 58 ++++++++++++++ 18 files changed, 230 insertions(+), 10 deletions(-) create mode 100644 app/src/main/java/com/pasich/mynotes/utils/auth/PlayServicesAvailability.java create mode 100644 app/src/test/java/com/pasich/mynotes/utils/auth/PlayServicesAvailabilityTest.java diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index f68fd882..8c05d027 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -25,6 +25,10 @@ + + + Sign-in and the Drive scope both run through Play services, so on a device without them + * every control on the account tab would lead to a failure the user cannot act on. Returning + * false here makes {@link #create} yield null, which is the path that already shows the tab as + * unavailable. + */ public static boolean isConfigured(@NonNull Activity activity) { - return !activity.getString(R.string.default_web_client_id).trim().isEmpty(); + return !activity.getString(R.string.default_web_client_id).trim().isEmpty() + && PlayServicesAvailability.isAvailable(activity); } /** The authorization object has to be kept by the caller so it can forward activity results. */ diff --git a/app/src/main/java/com/pasich/mynotes/ui/view/fragment/mydata/AccountSyncFragment.java b/app/src/main/java/com/pasich/mynotes/ui/view/fragment/mydata/AccountSyncFragment.java index 2dfb8258..be7fd759 100644 --- a/app/src/main/java/com/pasich/mynotes/ui/view/fragment/mydata/AccountSyncFragment.java +++ b/app/src/main/java/com/pasich/mynotes/ui/view/fragment/mydata/AccountSyncFragment.java @@ -124,6 +124,9 @@ public void render( @NonNull CharSequence lastSyncText) { if (binding == null) return; boolean signedIn = profile.isSignedIn(); + // The three groups are mutually exclusive; rendering real state always clears the + // unavailable notice so a recreated view cannot show both. + binding.syncUnavailableGroup.setVisibility(View.GONE); binding.signedInGroup.setVisibility(signedIn ? View.VISIBLE : View.GONE); binding.signedOutGroup.setVisibility(signedIn ? View.GONE : View.VISIBLE); if (!signedIn) { @@ -152,5 +155,6 @@ public void showUnavailable() { if (binding == null) return; binding.signedInGroup.setVisibility(View.GONE); binding.signedOutGroup.setVisibility(View.GONE); + binding.syncUnavailableGroup.setVisibility(View.VISIBLE); } } diff --git a/app/src/main/java/com/pasich/mynotes/utils/auth/PlayServicesAvailability.java b/app/src/main/java/com/pasich/mynotes/utils/auth/PlayServicesAvailability.java new file mode 100644 index 00000000..aa00d98e --- /dev/null +++ b/app/src/main/java/com/pasich/mynotes/utils/auth/PlayServicesAvailability.java @@ -0,0 +1,78 @@ +package com.pasich.mynotes.utils.auth; + +import android.content.Context; +import android.content.pm.ApplicationInfo; +import android.content.pm.PackageManager; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +/** + * Whether Google Play services can serve this device at all. + * + *

Every part of sync sits on Play services: Credential Manager signs in through it and the Drive + * scope is authorized by it. Without it the sign-in button leads nowhere, so the account tab has to + * know before it offers anything. + * + *

The usual answer is {@code GoogleApiAvailability}, which means linking play-services-base and + * loading a Play services class to ask whether Play services exist — the one question that must + * stay answerable when they do not. Reading the package table instead needs no dependency and + * cannot fail for the reason it is checking. + */ +public final class PlayServicesAvailability { + + /** The package behind every Google Play services API. */ + public static final String PLAY_SERVICES_PACKAGE = "com.google.android.gms"; + + /** + * The single fact this class needs, behind a seam so every outcome is testable. + * + *

An Activity is not constructible in a plain JVM test and this project carries no + * Robolectric, so the branches would otherwise only ever run on a device that has Play services + * — which is exactly the case that does not need checking. + */ + public interface Lookup { + /** + * @return {@code TRUE} when the package is installed and enabled, {@code FALSE} when it is + * installed but disabled, {@code null} when it is not installed. + */ + @Nullable + Boolean isPackageEnabled(@NonNull String packageName); + } + + private PlayServicesAvailability() { + // no instance + } + + /** True only when Play services are installed and the user has not disabled them. */ + public static boolean isAvailable(@Nullable Context context) { + return context != null && isAvailable(packageLookup(context)); + } + + static boolean isAvailable(@NonNull Lookup lookup) { + try { + return Boolean.TRUE.equals(lookup.isPackageEnabled(PLAY_SERVICES_PACKAGE)); + } catch (RuntimeException unanswerable) { + // A dead package manager or a ROM that refuses the query must not take the app down; + // an unanswerable question is answered "no", which only hides sync. + return false; + } + } + + @NonNull + static Lookup packageLookup(@NonNull Context context) { + return packageName -> { + PackageManager packages = context.getPackageManager(); + if (packages == null) { + return null; + } + try { + ApplicationInfo info = packages.getApplicationInfo(packageName, 0); + return info.enabled; + } catch (PackageManager.NameNotFoundException absent) { + // Also how the package table answers when the manifest does not declare the + // package in , which is why it does. + return null; + } + }; + } +} diff --git a/app/src/main/res/layout/fragment_account_sync.xml b/app/src/main/res/layout/fragment_account_sync.xml index 9b7369e1..02a52211 100644 --- a/app/src/main/res/layout/fragment_account_sync.xml +++ b/app/src/main/res/layout/fragment_account_sync.xml @@ -11,6 +11,43 @@ android:paddingTop="20dp" android:paddingBottom="20dp"> + + + + + + + + + + Уліковы запіс Вы не ўвайшлі Сінхранізацыя і копія на Google Дыску даступныя пасля ўваходу. + Сінхранізацыя недаступная + Сінхранізацыя і копія на Google Дыску патрабуюць сэрвісаў Google Play, якіх няма на гэтай прыладзе. Сінхранізацыя Сінхранізацыя з Google Drive diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index ec18783e..931eaf09 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -462,6 +462,8 @@ Konto Nicht angemeldet Synchronisierung und Google-Drive-Backup sind nach der Anmeldung verfügbar. + Synchronisierung nicht verfügbar + Synchronisierung und Google-Drive-Backup benötigen die Google-Play-Dienste, die auf diesem Gerät fehlen. Synchronisierung Google-Drive-Synchronisierung diff --git a/app/src/main/res/values-en-rGB/strings.xml b/app/src/main/res/values-en-rGB/strings.xml index 822ef736..69567ba8 100644 --- a/app/src/main/res/values-en-rGB/strings.xml +++ b/app/src/main/res/values-en-rGB/strings.xml @@ -509,6 +509,8 @@ Account Not signed in Sync and Google Drive backup become available after you sign in. + Sync unavailable + Sync and Google Drive backup need Google Play services, which this device does not have. Synchronisation Google Drive Sync diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 2555e28b..20afc507 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -463,6 +463,8 @@ Cuenta No has iniciado sesión La sincronización y la copia en Google Drive están disponibles tras iniciar sesión. + Sincronización no disponible + La sincronización y la copia en Google Drive necesitan los servicios de Google Play, que no están en este dispositivo. Sincronización Sincronización con Google Drive diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index a25d246f..59d23ae6 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -458,6 +458,8 @@ Compte Non connecté La synchronisation et la sauvegarde Google Drive sont disponibles après connexion. + Synchronisation indisponible + La synchronisation et la sauvegarde Google Drive nécessitent les services Google Play, absents de cet appareil. Synchronisation Synchronisation Google Drive diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 9122efc6..0ce04eb4 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -460,6 +460,8 @@ Account Non hai eseguito l’accesso Sincronizzazione e backup su Google Drive sono disponibili dopo l’accesso. + Sincronizzazione non disponibile + Sincronizzazione e backup su Google Drive richiedono i servizi Google Play, assenti su questo dispositivo. Sincronizzazione Sincronizzazione con Google Drive diff --git a/app/src/main/res/values-kk/strings.xml b/app/src/main/res/values-kk/strings.xml index ea6646ca..509e1c6e 100644 --- a/app/src/main/res/values-kk/strings.xml +++ b/app/src/main/res/values-kk/strings.xml @@ -459,6 +459,8 @@ Есептік жазба Сіз кірмегенсіз Синхрондау және Google Дискідегі көшірме кіргеннен кейін қолжетімді. + Синхрондау қолжетімсіз + Синхрондау және Google Дискідегі көшірме бұл құрылғыда жоқ Google Play қызметтерін қажет етеді. Синхрондау Google Drive-пен синхрондау diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index 4f55dfd3..3d704560 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -463,6 +463,8 @@ Konto Nie zalogowano Synchronizacja i kopia na Dysku Google są dostępne po zalogowaniu. + Synchronizacja niedostępna + Synchronizacja i kopia na Dysku Google wymagają usług Google Play, których nie ma na tym urządzeniu. Synchronizacja Synchronizacja z Google Drive diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 2e3dec06..b6ed3eaa 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -467,6 +467,8 @@ Аккаунт Вы не вошли Синхронизация и копия на Google Диске доступны после входа. + Синхронизация недоступна + Синхронизация и копия на Google Диске требуют сервисов Google Play, которых нет на этом устройстве. Синхронизация Синхронизация с Google Drive diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index 2c48418a..723aee71 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -483,4 +483,6 @@ Акаунт Ви не увійшли Синхронізація та копія на Google Диску доступні після входу. + Синхронізація недоступна + Синхронізація та копія на Google Диску потребують сервісів Google Play, яких немає на цьому пристрої. diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 1a722f9d..7188e8a8 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -511,4 +511,6 @@ Account Not signed in Sync and Google Drive backup become available after you sign in. + Sync unavailable + Sync and Google Drive backup need Google Play services, which this device does not have. diff --git a/app/src/test/java/com/pasich/mynotes/utils/auth/PlayServicesAvailabilityTest.java b/app/src/test/java/com/pasich/mynotes/utils/auth/PlayServicesAvailabilityTest.java new file mode 100644 index 00000000..a9b1085e --- /dev/null +++ b/app/src/test/java/com/pasich/mynotes/utils/auth/PlayServicesAvailabilityTest.java @@ -0,0 +1,58 @@ +package com.pasich.mynotes.utils.auth; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; + +/** + * Whether sync is offered at all on a given device. + * + *

Every branch here decides whether the account tab shows controls or a notice, and only the + * happy one ever runs on a development device — which is why they are pinned rather than trusted. + */ +public class PlayServicesAvailabilityTest { + + @Test + public void offersSyncWhenPlayServicesAreInstalledAndEnabled() { + assertThat(PlayServicesAvailability.isAvailable(packageName -> Boolean.TRUE)).isTrue(); + } + + @Test + public void withholdsSyncWhenPlayServicesAreNotInstalled() { + assertThat(PlayServicesAvailability.isAvailable(packageName -> null)).isFalse(); + } + + @Test + public void withholdsSyncWhenPlayServicesAreInstalledButDisabled() { + // A user can disable the package in system settings; the APIs then fail the same way as + // on a device that never had it. + assertThat(PlayServicesAvailability.isAvailable(packageName -> Boolean.FALSE)).isFalse(); + } + + @Test + public void withholdsSyncRatherThanCrashingWhenThePackageTableRefusesToAnswer() { + // A dead package manager throws from a binder call. Sync hiding itself is recoverable; + // taking the backup screen down with it is not. + assertThat( + PlayServicesAvailability.isAvailable( + packageName -> { + throw new IllegalStateException("package manager is dead"); + })) + .isFalse(); + } + + @Test + public void asksAboutThePlayServicesPackage() { + String[] asked = new String[1]; + PlayServicesAvailability.isAvailable( + packageName -> { + asked[0] = packageName; + return Boolean.TRUE; + }); + + // The manifest declares this exact package under ; a mismatch would make the + // lookup report "absent" on every API 30+ device. + assertThat(asked[0]).isEqualTo("com.google.android.gms"); + assertThat(asked[0]).isEqualTo(PlayServicesAvailability.PLAY_SERVICES_PACKAGE); + } +} From ece1cb9ea624c3ae70a706e45389a5e34998c95a Mon Sep 17 00:00:00 2001 From: pasichDev Date: Fri, 4 Sep 2026 21:50:16 +0300 Subject: [PATCH 16/16] release: ship the sync work as 2.6.50 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2.6.47, 2.6.48 and 2.6.49 were tagged but never published, so no user is on them and no changelog entry needs to describe the path through them. The three entries collapse into one written for the only update anyone will actually make, 2.6.46 to 2.6.50 — and the 2.6.49 entry claimed a staged rollout had completed for users who never received it. versionCode moves to 50 because 49 is already taken by the v2.6.49 tag, which points at a commit this branch builds on. --- CHANGELOG.md | 64 ++++++++++++++++++------------------------------ app/build.gradle | 2 +- 2 files changed, 25 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 166f2b5e..7d052508 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,54 +1,38 @@ # CHANGELOG -## [2.6.49] - 01.09.2026 - -**Improvements** - -- **Google Drive sync for everyone:** The staged rollout is complete. All signed-in users who - explicitly confirm their first sync can now sync their notes, tasks, tags, preferences, and - attachments across devices. - -## [2.6.48] - 01.09.2026 - -**Improvements** - -- **Safer Google Drive sync:** Sync snapshots are now published as immutable files and merged - deterministically, so a concurrent device update cannot overwrite another device's data. -- **Efficient background sync:** Periodic sync now runs only on unmetered networks when the battery - is not low, and unchanged data no longer creates an extra Drive snapshot. -- **Quality visibility:** Added JaCoCo unit-test coverage reports to CI for every pull request. - -**Fixes** - -- Sync now requires explicit first-sync confirmation in the coordinator itself, preventing any - caller from bypassing the data-upload review. -- Fixed updates to an existing Drive sync bundle on Android/JDK configurations that reject HTTP - PATCH requests. - -## [2.6.47] - 01.09.2026 +## [2.6.50] - 04.09.2026 **New** -- **Google Drive sync:** Optionally keep notes, tasks, tags, preferences, and attachments in sync - across devices while continuing to work offline. Your data is merged safely before a sync is - published, and the first sync clearly explains what may be uploaded. -- **Your data:** Added an Account tab with Google sign-in, sync status, a manual sync action, and - an optional background-sync switch. Backup, export, and import remain available in their own - tabs. +- **Google Drive sync:** Keep notes, tasks, tags, settings, and attachments in sync across your + devices, while the app keeps working fully offline. Sync is optional and off until you sign in. + The first sync explains exactly what will be uploaded and waits for your confirmation. +- **Your data:** The backup screen gained an Account tab — sign in, see sync status and when the + last sync ran, sync on demand, and turn on background sync. Backup, export, and import stay in + their own tabs and work without an account. +- **Choosing between two versions:** When the same note was edited on two devices, the app now + shows both versions side by side with their times, marks the newer one, and highlights exactly + where they differ, so you pick a version instead of guessing which side is yours. **Improvements** -- Attachments are deduplicated and verified during sync, reducing unnecessary uploads while - protecting file integrity. -- The app now remains fully usable when Google services are unavailable or when you choose not to - sign in. -- Updated translations across all supported languages for the new sync and account experience. +- Attachments are uploaded once and verified by content, so the same image shared between notes + never travels twice and a damaged upload is detected rather than trusted. +- Background sync runs only on unmetered networks and not on a low battery, and it skips + publishing entirely when nothing has changed. +- The Account tab now says plainly when sync cannot be offered on a device that has no Google + Play services, instead of showing controls that lead nowhere. +- Translations updated across all supported languages for sync and the account screen. **Fixes** -- Fixed several sync stability issues, including leaving the screen during an active sync and - preserving the time of the last successful sync. -- Fixed Google sign-in compatibility on Android 8.0 and 8.1. +- Restoring a backup no longer loses a note when the backup mixes restored and renumbered + entries, and a note's attachments now follow it into the restored note. +- Leaving the screen during a sync, or a sync interrupted partway, no longer loses the time of + the last successful sync or the edits that were being uploaded. +- Fixed a blank strip drawn above the toolbar on the Tasks and Help screens, which looked like a + second, empty app bar. +- Fixed Google sign-in on Android 8.0 and 8.1. ## [2.6.46] - 18.05.2026 diff --git a/app/build.gradle b/app/build.gradle index b85478b2..d9d502d8 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -21,7 +21,7 @@ apply from: "$projectDir/gradle/libs-task.gradle" apply from: "$projectDir/gradle/changelog-task.gradle" -def appVersionCode = 49 +def appVersionCode = 50 def appVersionName = "2.6.${appVersionCode}" def gitCommitHashProvider = providers.exec {