Fix incorrect handling of fresh clone

src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs

@@ -1,7 +1,8 @@
-// Copyright (c) 2016-2018 SIL International
+// Copyright (c) 2016-2018 SIL International
 // This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
 using LfMerge.Core.Actions.Infrastructure;
 using LfMerge.Core.DataConverters;
+using LfMerge.Core.FieldWorks;
 using LfMerge.Core.LanguageForge.Model;
 using LfMergeBridge.LfMergeModel;
 using MongoDB.Bson;
@@ -12,6 +13,7 @@
 using System.Linq;
 using SIL.LCModel;
 using SIL.LCModel.Core.KernelInterfaces;
+using SIL.LCModel.Infrastructure;
 
 namespace LfMerge.Core.Tests.Lcm
 {
@@ -639,6 +641,110 @@ public void Action_RunTwiceWithTheSameEntryDeletedEachTime_ShouldCountJustOneDel
 				Is.EqualTo("Language Forge S/R"));
 		}
 
+		// The tests below are what a fresh clone looks like: LfMerge's working copy already holds work
+		// FLEx users did after LF's last sync, which LF's copy in Mongo knows nothing about.
+
+		private static void EditInFieldWorks(FwProject project, ILexEntry entry, string newLexeme)
+		{
+			UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", project.Cache.ActionHandlerAccessor, () =>
+			{
+				entry.LexemeFormOA.Form.SetVernacularDefaultWritingSystem(newLexeme);
+				entry.DateModified = DateTime.Now;
+			});
+		}
+
+		[Test]
+		public void Action_EntryEditedOnlyInFieldWorksSinceLastSync_KeepsFieldWorksVersion()
+		{
+			// Setup
+			var lfProj = _lfProj;
+			SutLcmToMongo.Run(lfProj);
+			FwProject fwProject = lfProj.FieldWorksProject;
+			ILexEntry editedInFlex = LcmTestHelper.GetEntry(fwProject, Guid.Parse(TestEntryGuidStr));
+			EditInFieldWorks(fwProject, editedInFlex, "edited in FLEx after LF's last sync");
+
+			string vernacularWS = fwProject.Cache.LanguageProject.DefaultVernacularWritingSystem.Id;
+			LfLexEntry editedInLf = _conn.GetLfLexEntryByGuid(lfProj, Guid.Parse(KenEntryGuidStr));
+			editedInLf.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, "edited in LF after the last sync");
+			editedInLf.AuthorInfo.ModifiedDate = DateTime.UtcNow;
+			_conn.UpdateMockLfLexEntry(editedInLf);
+
+			// Exercise
+			SutMongoToLcm.Run(lfProj);
+
+			// Verify
+			Assert.That(editedInFlex.LexemeFormOA.Form.VernacularDefaultWritingSystem.Text,
+				Is.EqualTo("edited in FLEx after LF's last sync"));
+			ILexEntry ken = LcmTestHelper.GetEntry(fwProject, Guid.Parse(KenEntryGuidStr));
+			Assert.That(ken.LexemeFormOA.Form.VernacularDefaultWritingSystem.Text,
+				Is.EqualTo("edited in LF after the last sync"));
+			Assert.That(_counts.Added,    Is.EqualTo(0));
+			Assert.That(_counts.Modified, Is.EqualTo(1));
+			Assert.That(_counts.Deleted,  Is.EqualTo(0));
+		}
+
+		[Test]
+		public void Action_EntryDeletedInFieldWorksSinceLastSync_IsNotRecreated()
+		{
+			// Setup
+			var lfProj = _lfProj;
+			SutLcmToMongo.Run(lfProj);
+			FwProject fwProject = lfProj.FieldWorksProject;
+			Guid kenGuid = Guid.Parse(KenEntryGuidStr);
+			LcmTestHelper.DeleteEntry(fwProject, kenGuid);
+
+			// Exercise
+			SutMongoToLcm.Run(lfProj);
+
+			// Verify
+			ILexEntry ken;
+			Assert.That(fwProject.ServiceLocator.GetInstance<ILexEntryRepository>().TryGetObject(kenGuid, out ken), Is.False);
+			Assert.That(_counts.Added,    Is.EqualTo(0));
+			Assert.That(_counts.Modified, Is.EqualTo(0));
+			Assert.That(_counts.Deleted,  Is.EqualTo(0));
+		}
+
+		[Test]
+		public void Action_LfDeletionOlderThanLastSync_DoesNotDeleteTheFieldWorksEntry()
+		{
+			// Setup: an entry LF deleted before the last sync, which FieldWorks has since restored
+			var lfProj = _lfProj;
+			SutLcmToMongo.Run(lfProj);
+			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
+			entry.IsDeleted = true;
+			entry.DateModified = _conn.GetLastSyncedDate(lfProj).Value.AddDays(-1);
+			_conn.UpdateRecord(lfProj, entry);
+
+			// Exercise
+			SutMongoToLcm.Run(lfProj);
+
+			// Verify
+			ILexEntry stillThere;
+			Assert.That(lfProj.FieldWorksProject.ServiceLocator.GetInstance<ILexEntryRepository>().TryGetObject(entryGuid, out stillThere), Is.True);
+			Assert.That(_counts.Deleted, Is.EqualTo(0));
+		}
+
+		[Test]
+		public void Action_NeverSyncedProjectWhoseEntriesLfMergeWrote_KeepsFieldWorksVersion()
+		{
+			// Setup: an initial clone whose transfer to Mongo never got as far as recording the sync
+			var lfProj = _lfProj;
+			SutLcmToMongo.Run(lfProj);
+			_conn.SetLastSyncedDate(lfProj, MagicValues.UnixEpoch);
+			FwProject fwProject = lfProj.FieldWorksProject;
+			ILexEntry editedInFlex = LcmTestHelper.GetEntry(fwProject, Guid.Parse(TestEntryGuidStr));
+			EditInFieldWorks(fwProject, editedInFlex, "edited in FLEx after the initial clone");
+
+			// Exercise
+			SutMongoToLcm.Run(lfProj);
+
+			// Verify
+			Assert.That(editedInFlex.LexemeFormOA.Form.VernacularDefaultWritingSystem.Text,
+				Is.EqualTo("edited in FLEx after the initial clone"));
+			Assert.That(_counts.Modified, Is.EqualTo(0));
+		}
+
 		// TODO: Move custom field tests to their own test class, and move these helper functions with them
 		public void AddCustomFieldToSense(LfSense sense, string fieldName, BsonValue fieldValue)
 		{

src/LfMerge.Core.Tests/TestDoubles.cs

@@ -188,10 +188,22 @@ public void UpdateMockLfLexEntry(BsonDocument mockData)
 			UpdateMockLfLexEntry(data);
 		}
 
+		// The user SampleData's entries were last modified by
+		private static readonly ObjectId LfUserRef = ObjectId.Parse("561b666c0f87096a35c3cf2d");
+
+		/// <summary>
+		/// Store an entry the way an LF user's edit does: LF's MapperModel::write stamps DateModified,
+		/// and LexEntryCommands::updateEntry records who made the edit. LfMerge's own writes go
+		/// through UpdateRecord, which does neither.
+		/// </summary>
 		public void UpdateMockLfLexEntry(LfLexEntry mockData)
 		{
 			Guid guid = mockData.Guid ?? Guid.Empty;
-			_storedLfLexEntries[guid] = DeepCopy(mockData);
+			var stored = DeepCopy(mockData);
+			stored.DateModified = DateTime.UtcNow;
+			if (stored.AuthorInfo != null && stored.AuthorInfo.ModifiedByUserRef == null)
+				stored.AuthorInfo.ModifiedByUserRef = LfUserRef;
+			_storedLfLexEntries[guid] = stored;
 		}
 
 		public void UpdateMockOptionList(BsonDocument mockData)
@@ -290,6 +302,9 @@ public bool RemoveRecord(ILfProject project, Guid guid)
 
 		public bool SetLastSyncedDate(ILfProject project, DateTime? newSyncedDate)
 		{
+			// Mongo keeps dates to the millisecond, as DeepCopy does for the entries' dates
+			if (newSyncedDate != null)
+				newSyncedDate = newSyncedDate.Value.AddTicks(-(newSyncedDate.Value.Ticks % TimeSpan.TicksPerMillisecond));
 			_storedLastSyncDate[project.ProjectCode] = newSyncedDate;
 			// Also update on the fake project record, since EnsureCloneAction looks at the project record to check its initial-clone logic
 			if (_projectRecordFactory != null)
@@ -366,6 +381,7 @@ public override MongoProjectRecord Create(ILfProject project)
 			MongoProjectRecord record;
 			if (_projectRecords.TryGetValue(project.ProjectCode, out record))
 			{
+				RefreshLastSyncedDate(record, project);
 				return record;
 			}
 			else
@@ -396,9 +412,19 @@ public override MongoProjectRecord Create(ILfProject project)
 					Config = sampleConfig
 				};
 				_projectRecords.Add(project.ProjectCode, record);
+				RefreshLastSyncedDate(record, project);
 				return record;
 			}
 		}
+
+		/// The real factory reads the record from Mongo every time, so it always sees the last sync's
+		/// date. Each instance of this double keeps its own records, and the connection double only
+		/// updates the most recently created instance's, so take the date from the connection instead.
+		private void RefreshLastSyncedDate(MongoProjectRecord record, ILfProject project)
+		{
+			var testDouble = Connection as MongoConnectionDouble;
+			if (testDouble != null) record.LastSyncedDate = testDouble.GetLastSyncedDate(project);
+		}
 	}
 
 	class LanguageDepotProjectDouble: ILanguageDepotProject

src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs

@@ -1,4 +1,4 @@
-// Copyright (c) 2016-2018 SIL International
+// Copyright (c) 2016-2018 SIL International
 // This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
 using System;
 using System.Collections.Generic;
@@ -40,6 +40,11 @@ public class ConvertMongoToLcmLexicon
 		private int _wsEn;
 		private ConvertMongoToLcmCustomField _convertCustomField;
 
+		// Entries where LF's copy differs from FieldWorks but no LF user has touched it since the
+		// last sync, so FieldWorks holds the newer version (see EditedInLfSinceLastSync)
+		private int _changedOnlyInFieldWorks;
+		private int _absentOnlyFromFieldWorks;
+
 		// Shorter names to use in this class since MagicStrings.LfOptionListCodeForGrammaticalInfo
 		// (etc.) are real mouthfuls
 		private const string GrammarListCode = MagicStrings.LfOptionListCodeForGrammaticalInfo;
@@ -143,6 +148,8 @@ public ConversionError<LfLexEntry> RunConversion()
 			_convertCustomField = new ConvertMongoToLcmCustomField(Cache, ServiceLocator, Logger, _wsEn);
 
 			IEnumerable<LfLexEntry> lexicon = GetLexicon(LfProject);
+			_changedOnlyInFieldWorks = 0;
+			_absentOnlyFromFieldWorks = 0;
 			UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", Cache.ActionHandlerAccessor, () =>
 				{
 					#if false  // Once we allow LanguageForge to create optionlist items with "canonical" values (parts of speech, semantic domains, etc.), uncomment this block
@@ -164,6 +171,16 @@ public ConversionError<LfLexEntry> RunConversion()
 						}
 					}
 				});
+			if (_changedOnlyInFieldWorks > 0 || _absentOnlyFromFieldWorks > 0)
+			{
+				// Normally zero: LfMerge's working copy is what LF last synced with, so the two copies
+				// can only differ where LF users have edited since. Anything else means the working copy
+				// has moved on without LF, as a fresh clone of a project FLEx has kept working on has.
+				Logger.Warning("MongoToLcm: {0} entries differ from FieldWorks and {1} are missing from it, though no LF user has " +
+					"edited them since the last sync ({2}); kept FieldWorks's version of all of them",
+					_changedOnlyInFieldWorks, _absentOnlyFromFieldWorks,
+					LastSyncedDate == null ? "never synced" : LastSyncedDate.Value.ToString("u"));
+			}
 			// Comment conversion gets run AFTER lexicon conversion, so that any comments on new entries are handled correctly in FW
 			var commCvtr = new ConvertMongoToLcmComments(Connection, LfProject, exceptions, Logger, Progress);
 			var commErrors = commCvtr.RunConversion(entryObjectIdToGuidMappings);
@@ -494,9 +511,66 @@ private IMoForm CreateOwnedLexemeForm(ILexEntry owner, string morphologyType)
 			return result;
 		}
 
+		/// <summary>
+		/// The last sync's date, or null if the project has never been synced.
+		/// </summary>
+		private DateTime? LastSyncedDate
+		{
+			get
+			{
+				DateTime? lastSynced = ProjectRecord.LastSyncedDate;
+				return lastSynced == null || lastSynced.Value <= MagicValues.UnixEpoch ? null : lastSynced;
+			}
+		}
+
+		/// <summary>
+		/// Whether an LF user has edited, created or deleted this entry since the last sync, or ever,
+		/// if the project has never been synced.
+		///
+		/// Mongo holds FieldWorks as it was at the last sync plus what LF users have done since, so
+		/// those are the only entries LF has anything to say about. Comparing LF's copy with
+		/// FieldWorks's instead is only right while LfMerge's working copy is still exactly what it
+		/// last synced: on a fresh clone of a project FLEx has kept working on, every entry FLEx
+		/// changed since then would look changed in LF, and LF's older copy would be written over it.
+		///
+		/// LF stamps an entry's DateModified on every edit and deletion (MapperModel::write), and
+		/// LfMerge's own writes to Mongo all come before it records LastSyncedDate. Both dates come
+		/// from the server's clock, so a FLEx user's clock being wrong cannot affect this. Mongo keeps
+		/// them to the millisecond, and a tie counts as edited: that falls back to comparing LF's copy
+		/// with FieldWorks's, which is what LfMerge has always done.
+		///
+		/// A project that has never been synced has no date to go by, though its entries can still be
+		/// LfMerge's: the initial clone's transfer to Mongo does not always get as far as recording a
+		/// sync. LfMerge clears the user references on every entry it writes, and LF sets one on every
+		/// edit and creation; a deletion sets none, but deleting what FieldWorks lacks is harmless.
+		/// </summary>
+		private bool EditedInLfSinceLastSync(LfLexEntry lfEntry)
+		{
+			DateTime? lastSynced = LastSyncedDate;
+			if (lastSynced == null)
+				return lfEntry.IsDeleted || (lfEntry.AuthorInfo != null &&
+					(lfEntry.AuthorInfo.ModifiedByUserRef != null || lfEntry.AuthorInfo.CreatedByUserRef != null));
+			return lfEntry.DateModified >= lastSynced.Value;
+		}
+
 		private void LfLexEntryToLcmLexEntry(LfLexEntry lfEntry)
 		{
 			Guid guid = lfEntry.Guid ?? Guid.Empty;
+			if (!EditedInLfSinceLastSync(lfEntry))
+			{
+				// Nothing for FieldWorks here. Where it differs, it is the newer one, and the transfer
+				// back to Mongo after Send/Receive will bring LF up to date with it.
+				ILexEntry existing;
+				if (lfEntry.IsDeleted)
+				{
+					// An LF deletion that an earlier sync has already sent
+				}
+				else if (!GetInstance<ILexEntryRepository>().TryGetObject(guid, out existing))
+					_absentOnlyFromFieldWorks++;
+				else if (lfEntry.AuthorInfo == null || lfEntry.AuthorInfo.ModifiedDate.ToLocalTime() != existing.DateModified)
+					_changedOnlyInFieldWorks++;
+				return;
+			}
 			bool createdEntry = false;
 			bool wantCreation = !lfEntry.IsDeleted;
 			ILexEntry LcmEntry = GetOrCreateEntryByGuid(guid, wantCreation, out createdEntry);
@@ -541,6 +615,14 @@ private void LfLexEntryToLcmLexEntry(LfLexEntry lfEntry)
 					return;
 				}
 			}
+			if (!createdEntry && LastSyncedDate != null && LcmEntry.DateModified.ToUniversalTime() > LastSyncedDate.Value)
+			{
+				// Only possible when the working copy has moved on without LF (or a FLEx user's clock
+				// is ahead). LF's edit still wins, as it always has; this records what it replaced.
+				Logger.Warning("MongoToLcm: entry {0} ({1}) was edited in FieldWorks on {2:u} as well as in LF since the last sync; " +
+					"LF's version replaces FieldWorks's", guid, ConvertUtilities.EntryNameForDebugging(lfEntry),
+					LcmEntry.DateModified.ToUniversalTime());
+			}
 
 			// Fields in order by lfEntry property, except for Senses and CustomFields, which are handled at the end
 			SetMultiStringFrom(LcmEntry.CitationForm, lfEntry.CitationForm);