Set up end to end Send/Receive testing with LexBox

src/LfMerge.Core.Tests/E2E/E2ETestBase.cs

@@ -0,0 +1,234 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+using Autofac;
+using LfMerge.Core.Actions;
+using LfMerge.Core.FieldWorks;
+using LfMerge.Core.MongoConnector;
+using NUnit.Framework;
+using NUnit.Framework.Interfaces;
+using SIL.LCModel;
+using SIL.TestUtilities;
+
+namespace LfMerge.Core.Tests.E2E
+{
+	[TestFixture]
+	[Category("LongRunning")]
+	[Category("IntegrationTests")]
+	public class E2ETestBase
+	{
+		public LfMerge.Core.Logging.ILogger Logger => MainClass.Logger;
+		public TemporaryFolder TempFolderForClass { get; set; }
+		public TemporaryFolder TempFolderForTest { get; set; }
+		public string Sena3ZipPath { get; set; }
+		private readonly HashSet<Guid> ProjectIdsToDelete = [];
+		public SRTestEnvironment TestEnv { get; set; }
+
+		public MongoConnectionDouble _mongoConnection;
+		public MongoProjectRecordFactory _recordFactory;
+
+		public E2ETestBase()
+		{
+		}
+
+		private static string TestName => TestContext.CurrentContext.Test.Name;
+		private static string TestNameForPath => string.Concat(TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars
+
+		[OneTimeSetUp]
+		public async Task FixtureSetup()
+		{
+			// Ensure top-level /tmp/LfMergeSRTests folder and subfolders exist
+			var tempPath = Path.Join(Path.GetTempPath(), "LfMergeSRTests");
+			Directory.CreateDirectory(tempPath);
+			var testDataPath = Path.Join(tempPath, "data");
+			Directory.CreateDirectory(testDataPath);
+			var lcmDataPath = Path.Join(tempPath, "lcm-common");
+			Directory.CreateDirectory(lcmDataPath);
+			Environment.SetEnvironmentVariable("FW_CommonAppData", lcmDataPath);
+
+			var derivedClassName = this.GetType().Name;
+			TempFolderForClass = new TemporaryFolder(Path.Join(tempPath, derivedClassName));
+
+			// Ensure sena-3.zip is available to all tests as a starting point
+			Sena3ZipPath = Path.Join(testDataPath, "sena-3.zip");
+			if (!File.Exists(Sena3ZipPath)) {
+				using var testEnv = new SRTestEnvironment(TempFolderForTest);
+				if (!await testEnv.IsLexBoxAvailable()) {
+					Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox.");
+				}
+				await testEnv.Login();
+				await testEnv.DownloadProjectBackup("sena-3", Sena3ZipPath);
+			}
+		}
+
+		[OneTimeTearDown]
+		public void FixtureTeardown()
+		{
+			Environment.SetEnvironmentVariable("FW_CommonAppData", null);
+			var result = TestContext.CurrentContext.Result;
+			var nonSuccess = result.FailCount + result.InconclusiveCount + result.WarningCount;
+			// Only delete class temp folder if we passed or skipped all tests
+			if (nonSuccess == 0) TempFolderForClass.Dispose();
+		}
+
+		[SetUp]
+		public async Task TestSetup()
+		{
+			TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath);
+			TestEnv = new SRTestEnvironment(TempFolderForTest);
+			if (!await TestEnv.IsLexBoxAvailable()) {
+				Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox.");
+			}
+			await TestEnv.Login();
+
+			MagicStrings.SetMinimalModelVersion(LcmCache.ModelVersion);
+			_mongoConnection = MainClass.Container.Resolve<IMongoConnection>() as MongoConnectionDouble;
+			if (_mongoConnection == null)
+				throw new AssertionException("E2E tests need a mock MongoConnection that stores data in order to work.");
+			_recordFactory = MainClass.Container.Resolve<MongoProjectRecordFactory>() as MongoProjectRecordFactoryDouble;
+			if (_recordFactory == null)
+				throw new AssertionException("E2E tests need a mock MongoProjectRecordFactory in order to work.");
+		}
+
+		[TearDown]
+		public async Task TestTeardown()
+		{
+			var outcome = TestContext.CurrentContext.Result.Outcome;
+			var success = outcome == ResultState.Success || outcome == ResultState.Ignored;
+			// Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation
+			TestEnv.DeleteTempFolderDuringCleanup = success;
+			// On failure, also leave LexBox project(s) in place for post-test investigation, even though this might tend to clutter things up a little
+			if (success) {
+				foreach (var projId in ProjectIdsToDelete) {
+					await TestEnv.DeleteLexBoxProject(projId);
+				}
+			}
+			ProjectIdsToDelete.Clear();
+			TestEnv.Dispose();
+		}
+
+		public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode);
+		public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata");
+
+		public string CloneRepoFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null)
+		{
+			var projUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(code);
+			newCode ??= code;
+			var dest = TestFolderForProject(newCode);
+			if (waitTime is null) {
+				MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest);
+			} else {
+				var start = DateTime.UtcNow;
+				var success = false;
+				while (!success) {
+					try {
+						MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest);
+					} catch {
+						if (DateTime.UtcNow > start + waitTime) {
+							throw; // Give up
+						}
+						System.Threading.Thread.Sleep(250);
+						continue;
+					}
+					// If we got this far, no exception so we succeeded
+					success = true;
+				}
+			}
+			return dest;
+		}
+
+		public FwProject CloneFwProjectFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null)
+		{
+			var dest = CloneRepoFromLexbox(code, newCode, waitTime);
+			var dirInfo = new DirectoryInfo(dest);
+			if (!dirInfo.Exists) throw new InvalidOperationException($"Failed to clone {code} from lexbox, cannot create FwProject");
+			var dirname = dirInfo.Name;
+			var fwdataPath = Path.Join(dest, $"{dirname}.fwdata");
+			MercurialTestHelper.ChangeBranch(dest, "tip");
+			LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath);
+			var settings = new LfMergeSettingsDouble(TempFolderForTest.Path);
+			return new FwProject(settings, dirname);
+		}
+
+		public async Task<string> CreateEmptyFlexProjectInLexbox()
+		{
+			var randomGuid = Guid.NewGuid();
+			var testCode = $"sr-{randomGuid}";
+			var testPath = TestFolderForProject(testCode);
+			MercurialTestHelper.InitializeHgRepo(testPath);
+			MercurialTestHelper.CreateFlexRepo(testPath);
+			// Now create project in LexBox
+			var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid);
+			var pushUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(testCode).AbsoluteUri;
+			MercurialTestHelper.HgPush(testPath, pushUrl);
+			if (result.Id.HasValue) ProjectIdsToDelete.Add(result.Id.Value);
+			return testCode;
+		}
+
+		public async Task<string> CreateNewProjectFromTemplate(string origZipPath)
+		{
+			var randomGuid = Guid.NewGuid();
+			var testCode = $"sr-{randomGuid}";
+			var testPath = TestFolderForProject(testCode);
+			// Now create project in LexBox
+			var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid);
+			await TestEnv.ResetAndUploadZip(testCode, origZipPath);
+			if (result.Id.HasValue) ProjectIdsToDelete.Add(result.Id.Value);
+			return testCode;
+		}
+
+		public Task<string> CreateNewProjectFromSena3() => CreateNewProjectFromTemplate(Sena3ZipPath);
+
+		public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null)
+		{
+			TestEnv.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg);
+		}
+
+		public async Task<LanguageForgeProject> CreateLfProjectFromSena3()
+		{
+			var projCode = await CreateNewProjectFromSena3();
+			var projPath = CloneRepoFromLexbox(projCode, waitTime:TimeSpan.FromSeconds(5));
+			MercurialTestHelper.ChangeBranch(projPath, "tip");
+			var fwdataPath = Path.Join(projPath, $"{projCode}.fwdata");
+			LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath);
+
+			// Do an initial clone from LexBox to the mock LF
+			var lfProject = LanguageForgeProject.Create(projCode, TestEnv.Settings);
+			lfProject.IsInitialClone = true;
+			var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, TestEnv.NullLogger, _mongoConnection, _recordFactory);
+			transferLcmToMongo.Run(lfProject);
+
+			return lfProject;
+		}
+
+		public void SendReceiveToLexbox(LanguageForgeProject lfProject)
+		{
+			TestEnv.Settings.LanguageDepotRepoUri = SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri;
+			var syncAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger);
+			syncAction.Run(lfProject);
+		}
+
+		public (string, DateTime, DateTime) UpdateFwGloss(FwProject project, Guid entryId, Func<string, string> textConverter)
+		{
+			var fwEntry = LcmTestHelper.GetEntry(project, entryId);
+			Assert.That(fwEntry, Is.Not.Null);
+			var origModifiedDate = fwEntry.DateModified;
+			var unchangedGloss = LcmTestHelper.UpdateAnalysisText(project, fwEntry.SensesOS[0].Gloss, textConverter);
+			return (unchangedGloss, origModifiedDate, fwEntry.DateModified);
+		}
+
+		public (string, DateTime, DateTime) UpdateLfGloss(LanguageForgeProject lfProject, Guid entryId, string wsId, Func<string, string> textConverter)
+		{
+			var lfEntry = _mongoConnection.GetLfLexEntryByGuid(entryId);
+			Assert.That(lfEntry, Is.Not.Null);
+			var unchangedGloss = lfEntry.Senses[0].Gloss[wsId].Value;
+			lfEntry.Senses[0].Gloss["pt"].Value = textConverter(unchangedGloss);
+			// Remember that in LfMerge it's AuthorInfo that corresponds to the Lcm modified date
+			DateTime origModifiedDate = lfEntry.AuthorInfo.ModifiedDate;
+			lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow;
+			_mongoConnection.UpdateRecord(lfProject, lfEntry);
+			return (unchangedGloss, origModifiedDate, lfEntry.AuthorInfo.ModifiedDate);
+		}
+	}
+}

src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Threading.Tasks;
+using NUnit.Framework;
+using System.Text.RegularExpressions;
+
+namespace LfMerge.Core.Tests.E2E
+{
+	[TestFixture]
+	[Category("LongRunning")]
+	[Category("IntegrationTests")]
+	public class LexboxSendReceiveTests : E2ETestBase
+	{
+		// This test will often trigger a race condition in LexBox that causes the *next* test to fail
+		// when `hg clone` returns 404. This is because of hgweb's directory cache, which only refreshes
+		// if it hasn't been refreshed more than N seconds ago (default 20, LexBox currently uses 5).
+		// Which means that even though CreateLfProjectFromSena3 has created the LexBox project, LexBox's
+		// copy of hgweb doesn't see it yet so it can't be cloned.
+		//
+		// The solution will be to adjust CloneRepoFromLexbox to take a parameter that is the number of
+		// seconds to wait for the project to become visible, and retry 404's until that much time has elapsed.
+		// Then only throw an exception if hgweb is still returning 404 after its dir cache should be refreshed.
+		[Test]
+		public async Task E2E_CheckFwProjectCreation()
+		{
+			var code = await CreateEmptyFlexProjectInLexbox();
+			Console.WriteLine($"Created new project {code}");
+		}
+
+		[Test]
+		public async Task E2E_LFDataChangedLDDataChanged_LFWins()
+		{
+			// Setup
+
+			var lfProject = await CreateLfProjectFromSena3();
+			var fwProjectCode = Regex.Replace(lfProject.ProjectCode, "^sr-", "fw-");
+			var fwProject = CloneFwProjectFromLexbox(lfProject.ProjectCode, fwProjectCode);
+
+			// Modify FW data first, then push to Lexbox
+			Guid entryId = LcmTestHelper.GetFirstEntry(fwProject).Guid;
+			var (unchangedGloss, origFwDateModified, fwDateModified) = UpdateFwGloss(fwProject, entryId, text => text + " - changed in FW");
+			CommitAndPush(fwProject, lfProject.ProjectCode, fwProjectCode, "Modified gloss in FW");
+
+			// Modify LF data second
+			var (_, origLfDateModified, _) = UpdateLfGloss(lfProject, entryId, "pt", text => text + " - changed in LF");
+
+			// Exercise
+
+			SendReceiveToLexbox(lfProject);
+
+			// Verify
+
+			// LF side should win conflict since its modified date was later
+			var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(entryId);
+			Assert.That(lfEntryAfterSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF"));
+			// LF's modified dates should have been updated by the sync action
+			Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(origLfDateModified));
+			// Remember that FieldWorks's DateModified is stored in local time for some incomprehensible reason...
+			Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(fwDateModified.ToUniversalTime()));
+		}
+	}
+}

src/LfMerge.Core.Tests/LcmTestHelper.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using LfMerge.Core.FieldWorks;
+using SIL.LCModel;
+using SIL.LCModel.Infrastructure;
+
+namespace LfMerge.Core.Tests
+{
+	public static class LcmTestHelper
+	{
+		public static IEnumerable<ILexEntry> GetEntries(FwProject project)
+		{
+			return project?.ServiceLocator?.LanguageProject?.LexDbOA?.Entries ?? [];
+		}
+
+		public static int CountEntries(FwProject project)
+		{
+			var repo = project?.ServiceLocator?.GetInstance<ILexEntryRepository>();
+			return repo.Count;
+		}
+
+		public static ILexEntry GetEntry(FwProject project, Guid guid)
+		{
+			var repo = project?.ServiceLocator?.GetInstance<ILexEntryRepository>();
+			return repo.GetObject(guid);
+		}
+
+		public static ILexEntry GetFirstEntry(FwProject project)
+		{
+			var repo = project?.ServiceLocator?.GetInstance<ILexEntryRepository>();
+			return repo.AllInstances().First();
+		}
+
+		public static string? GetVernacularText(IMultiUnicode field)
+		{
+			return field.BestVernacularAlternative?.Text;
+		}
+
+		public static string? GetAnalysisText(IMultiUnicode field)
+		{
+			return field.BestAnalysisAlternative?.Text;
+		}
+
+		public static void SetVernacularText(FwProject project, IMultiUnicode field, string newText)
+		{
+			var accessor = project.Cache.ActionHandlerAccessor;
+			UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", accessor, () => {
+				field.SetVernacularDefaultWritingSystem(newText);
+			});
+		}
+
+		public static void SetAnalysisText(FwProject project, IMultiUnicode field, string newText)
+		{
+			var accessor = project.Cache.ActionHandlerAccessor;
+			UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", accessor, () => {
+				field.SetAnalysisDefaultWritingSystem(newText);
+			});
+		}
+
+		public static string UpdateVernacularText(FwProject project, IMultiUnicode field, Func<string, string> textConverter)
+		{
+			var oldText = field.BestVernacularAlternative?.Text;
+			if (oldText != null)
+			{
+				var newText = textConverter(oldText);
+				SetVernacularText(project, field, newText);
+			}
+			return oldText;
+		}
+
+		public static string UpdateAnalysisText(FwProject project, IMultiUnicode field, Func<string, string> textConverter)
+		{
+			var oldText = field.BestAnalysisAlternative?.Text;
+			if (oldText != null)
+			{
+				var newText = textConverter(oldText);
+				SetAnalysisText(project, field, newText);
+			}
+			return oldText;
+		}
+	}
+}