From 7a4873e72c462b7fa5c691283679ac114efe1770 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 25 Jul 2024 10:23:03 +0700 Subject: [PATCH 01/73] Add MkFwData and SplitFwData commands to LfMerge Will be used in end-to-end testing scenarios --- LfMerge.sln | 16 +++++ src/MkFwData/.editorconfig | 27 +++++++++ src/MkFwData/.gitattributes | 1 + src/MkFwData/MkFwData.csproj | 25 ++++++++ src/MkFwData/Program.cs | 95 ++++++++++++++++++++++++++++++ src/SplitFwData/.editorconfig | 27 +++++++++ src/SplitFwData/.gitattributes | 1 + src/SplitFwData/Program.cs | 76 ++++++++++++++++++++++++ src/SplitFwData/SplitFwData.csproj | 25 ++++++++ 9 files changed, 293 insertions(+) create mode 100644 src/MkFwData/.editorconfig create mode 100644 src/MkFwData/.gitattributes create mode 100644 src/MkFwData/MkFwData.csproj create mode 100644 src/MkFwData/Program.cs create mode 100644 src/SplitFwData/.editorconfig create mode 100644 src/SplitFwData/.gitattributes create mode 100644 src/SplitFwData/Program.cs create mode 100644 src/SplitFwData/SplitFwData.csproj diff --git a/LfMerge.sln b/LfMerge.sln index 090e3358..fcb18ba4 100644 --- a/LfMerge.sln +++ b/LfMerge.sln @@ -26,6 +26,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LfMerge.Core.Tests", "src\L EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LfMergeAuxTool", "src\LfMergeAuxTool\LfMergeAuxTool.csproj", "{28882F30-358B-4E1C-A934-076D9EE6ACFC}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MkFwData", "src\MkFwData\MkFwData.csproj", "{5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SplitFwData", "src\SplitFwData\SplitFwData.csproj", "{23DF39D3-5C50-4832-A64E-022396430390}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -81,6 +85,18 @@ Global {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Release|Any CPU.ActiveCfg = Release|Any CPU {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Release|Any CPU.Build.0 = Release|Any CPU + {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU + {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU + {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Release|Any CPU.Build.0 = Release|Any CPU + {23DF39D3-5C50-4832-A64E-022396430390}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23DF39D3-5C50-4832-A64E-022396430390}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23DF39D3-5C50-4832-A64E-022396430390}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU + {23DF39D3-5C50-4832-A64E-022396430390}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU + {23DF39D3-5C50-4832-A64E-022396430390}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23DF39D3-5C50-4832-A64E-022396430390}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/MkFwData/.editorconfig b/src/MkFwData/.editorconfig new file mode 100644 index 00000000..d4047b80 --- /dev/null +++ b/src/MkFwData/.editorconfig @@ -0,0 +1,27 @@ +# Copyright (c) 2016-2021 SIL International +# This software is licensed under the MIT license (http://opensource.org/licenses/MIT) + +root = false + +# Defaults +[*] +indent_style = space +indent_size = tab +tab_width = 4 +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +max_line_length = 98 + +[*.cs] +indent_style = space +tab_width = 4 + +# Settings Visual Studio uses for the generated files +[*.{csproj,resx,settings,targets,vcxproj*,vdproj,xml,yml,props,md}] +indent_style = space +indent_size = 2 + +# Generated file +[*.sln] +end_of_line = crlf diff --git a/src/MkFwData/.gitattributes b/src/MkFwData/.gitattributes new file mode 100644 index 00000000..cf3363d0 --- /dev/null +++ b/src/MkFwData/.gitattributes @@ -0,0 +1 @@ +* text=auto whitespace=space-before-tab,tab-in-indent,blank-at-eol,tabwidth=4 diff --git a/src/MkFwData/MkFwData.csproj b/src/MkFwData/MkFwData.csproj new file mode 100644 index 00000000..afbe05a3 --- /dev/null +++ b/src/MkFwData/MkFwData.csproj @@ -0,0 +1,25 @@ + + + + Exe + net8.0 + net8.0 + enable + enable + true + $(MSBuildProjectDirectory) + + + + + + + + + + + + + + + diff --git a/src/MkFwData/Program.cs b/src/MkFwData/Program.cs new file mode 100644 index 00000000..b7203743 --- /dev/null +++ b/src/MkFwData/Program.cs @@ -0,0 +1,95 @@ +using Chorus.VcsDrivers.Mercurial; +using SIL.Progress; +using System.CommandLine; + +class Program +{ + static async Task Main(string[] args) + { + var rootCommand = new RootCommand("Make .fwdata file"); + + var verboseOption = new Option( + ["--verbose", "-v"], + "Display verbose output" + ); + rootCommand.AddGlobalOption(verboseOption); + + var quietOption = new Option( + ["--quiet", "-q"], + "Suppress all output (overrides --verbose if present)" + ); + rootCommand.AddGlobalOption(quietOption); + + var filename = new Argument( + "file", + "Name of .fwdata file to create, or directory to create it in" + ); + rootCommand.Add(filename); + + var hgRevOption = new Option( + ["--rev", "-r"], + "Revision to check out (default \"tip\")" + ); + hgRevOption.SetDefaultValue("tip"); + rootCommand.Add(hgRevOption); + + var cleanupOption = new Option( + ["--cleanup", "-c"], + "Clean repository after creating .fwdata file (deletes every other file except .fwdata)" + ); + rootCommand.Add(cleanupOption); + + rootCommand.SetHandler(Run, filename, verboseOption, quietOption, hgRevOption, cleanupOption); + + return await rootCommand.InvokeAsync(args); + } + + static FileInfo LocateFwDataFile(string input) + { + if (Directory.Exists(input)) { + var dirInfo = new DirectoryInfo(input); + var fname = dirInfo.Name + ".fwdata"; + return new FileInfo(Path.Join(input, fname)); + } else if (File.Exists(input)) { + return new FileInfo(input); + } else if (File.Exists(input + ".fwdata")) { + return new FileInfo(input + ".fwdata"); + } else { + if (input.EndsWith(".fwdata")) return new FileInfo(input); + return new FileInfo(input + ".fwdata"); + } + } + + static Task Run(string filename, bool verbose, bool quiet, string rev, bool cleanup) + { + IProgress progress = quiet ? new NullProgress() : new ConsoleProgress(); + progress.ShowVerbose = verbose; + var file = LocateFwDataFile(filename); + if (file.Exists) { + progress.WriteWarning("File {0} already exists and will be overwritten", file.FullName); + } + var dir = file.Directory; + if (dir == null || !dir.Exists) { + progress.WriteError("Could not find directory {0}. MkFwData needs a Mercurial repo to work with.", dir?.FullName ?? "(null)"); + return Task.FromResult(1); + } + string name = file.FullName; + progress.WriteMessage("Checking out {0}", rev); + var result = HgRunner.Run($"hg checkout {rev}", dir.FullName, 30, progress); + if (result.ExitCode != 0) + { + progress.WriteMessage("Could not find Mercurial repo in directory {0}. MkFwData needs a Mercurial repo to work with.", dir.FullName ?? "(null)"); + return Task.FromResult(result.ExitCode); + } + progress.WriteVerbose("Creating {0} ...", name); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(progress, writeVerbose: true, name); + progress.WriteMessage("Created {0}", name); + if (cleanup) + { + progress.WriteVerbose("Cleaning up..."); + HgRunner.Run($"hg checkout null", dir.FullName, 30, progress); + HgRunner.Run($"hg purge --no-confirm --exclude *.fwdata --exclude hgRunner.log", dir.FullName, 30, progress); + } + return Task.FromResult(0); + } +} diff --git a/src/SplitFwData/.editorconfig b/src/SplitFwData/.editorconfig new file mode 100644 index 00000000..d4047b80 --- /dev/null +++ b/src/SplitFwData/.editorconfig @@ -0,0 +1,27 @@ +# Copyright (c) 2016-2021 SIL International +# This software is licensed under the MIT license (http://opensource.org/licenses/MIT) + +root = false + +# Defaults +[*] +indent_style = space +indent_size = tab +tab_width = 4 +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +max_line_length = 98 + +[*.cs] +indent_style = space +tab_width = 4 + +# Settings Visual Studio uses for the generated files +[*.{csproj,resx,settings,targets,vcxproj*,vdproj,xml,yml,props,md}] +indent_style = space +indent_size = 2 + +# Generated file +[*.sln] +end_of_line = crlf diff --git a/src/SplitFwData/.gitattributes b/src/SplitFwData/.gitattributes new file mode 100644 index 00000000..cf3363d0 --- /dev/null +++ b/src/SplitFwData/.gitattributes @@ -0,0 +1 @@ +* text=auto whitespace=space-before-tab,tab-in-indent,blank-at-eol,tabwidth=4 diff --git a/src/SplitFwData/Program.cs b/src/SplitFwData/Program.cs new file mode 100644 index 00000000..110a6f09 --- /dev/null +++ b/src/SplitFwData/Program.cs @@ -0,0 +1,76 @@ +using Chorus.VcsDrivers.Mercurial; +using SIL.Progress; +using System.CommandLine; + +class Program +{ + static async Task Main(string[] args) + { + var rootCommand = new RootCommand("Make .fwdata file"); + + var verboseOption = new Option( + ["--verbose", "-v"], + "Display verbose output" + ); + rootCommand.AddGlobalOption(verboseOption); + + var quietOption = new Option( + ["--quiet", "-q"], + "Suppress all output (overrides --verbose if present)" + ); + rootCommand.AddGlobalOption(quietOption); + + var filename = new Argument( + "file", + "Name of .fwdata file to split" + ); + rootCommand.Add(filename); + + var cleanupOption = new Option( + ["--cleanup", "-c"], + "Delete .fwdata file after splitting" + ); + rootCommand.Add(cleanupOption); + + rootCommand.SetHandler(Run, filename, verboseOption, quietOption, cleanupOption); + + return await rootCommand.InvokeAsync(args); + } + + static FileInfo? LocateFwDataFile(string input) + { + if (Directory.Exists(input)) { + var dirInfo = new DirectoryInfo(input); + var fname = dirInfo.Name + ".fwdata"; + return new FileInfo(Path.Join(input, fname)); + } else if (File.Exists(input)) { + return new FileInfo(input); + } else if (File.Exists(input + ".fwdata")) { + return new FileInfo(input + ".fwdata"); + } else { + return null; + } + } + + static Task Run(string filename, bool verbose, bool quiet, bool cleanup) + { + IProgress progress = quiet ? new NullProgress() : new ConsoleProgress(); + progress.ShowVerbose = verbose; + var file = LocateFwDataFile(filename); + if (file == null || !file.Exists) { + progress.WriteError("Could not find {0}", filename); + return Task.FromResult(1); + } + string name = file.FullName; + progress.WriteVerbose("Splitting {0} ...", name); + LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(progress, writeVerbose: true, name); + progress.WriteMessage("Finished splitting {0}", name); + if (cleanup) + { + progress.WriteVerbose("Cleaning up..."); + var fwdataFile = new FileInfo(name); + if (fwdataFile.Exists) { fwdataFile.Delete(); progress.WriteVerbose("Deleted {0}", fwdataFile.FullName); } else { progress.WriteVerbose("File not found, so not deleting: {0}", fwdataFile.FullName); } + } + return Task.FromResult(0); + } +} diff --git a/src/SplitFwData/SplitFwData.csproj b/src/SplitFwData/SplitFwData.csproj new file mode 100644 index 00000000..afbe05a3 --- /dev/null +++ b/src/SplitFwData/SplitFwData.csproj @@ -0,0 +1,25 @@ + + + + Exe + net8.0 + net8.0 + enable + enable + true + $(MSBuildProjectDirectory) + + + + + + + + + + + + + + + From ab49c3bbc67a7537c0e230d953a4ea69efa28ee9 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 25 Jul 2024 10:31:24 +0700 Subject: [PATCH 02/73] Start working on test environment for E2E tests --- .../LfMerge.Core.Tests.csproj | 1 + src/LfMerge.Core.Tests/SRTestEnvironment.cs | 94 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 src/LfMerge.Core.Tests/SRTestEnvironment.cs diff --git a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj index e016ab04..d5afef7a 100644 --- a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj +++ b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj @@ -44,6 +44,7 @@ See full changelog at https://github.com/sillsdev/LfMerge/blob/develop/CHANGELOG + diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs new file mode 100644 index 00000000..0506acc2 --- /dev/null +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -0,0 +1,94 @@ +using System; +using System.ComponentModel; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading.Tasks; +using Bugsnag.Payload; +using LfMerge.Core.Logging; +using LfMerge.Core.Settings; +using NUnit.Framework; +using SIL.LCModel; +using SIL.TestUtilities; +using TusDotNetClient; + +namespace LfMerge.Core.Tests +{ + /// + /// Test environment for end-to-end testing, i.e. Send/Receive with a real LexBox instance + /// + public class SRTestEnvironment + { + public ILogger Logger => MainClass.Logger; + public Uri LexboxUrl { get; init; } + private TemporaryFolder TempFolder { get; init; } + private HttpClient Http { get; init; } = new HttpClient(); + private string Jwt { get; set; } + + public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProtocol = "http", int lexboxPort = 80, string lexboxUsername = "admin", string lexboxPassword = "pass") + { + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname, lexboxHostname); + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPrivateHostname, lexboxHostname); + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol, lexboxProtocol); + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_HgUsername, lexboxUsername); + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_TrustToken, lexboxPassword); + LexboxUrl = new Uri($"{lexboxProtocol}://{WebUtility.UrlEncode(lexboxUsername)}:{WebUtility.UrlEncode(lexboxPassword)}@{lexboxHostname}:{lexboxPort}"); + TempFolder = new TemporaryFolder(TestName + Path.GetRandomFileName()); + } + + public async Task Login() + { + var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername); + var lexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken); + var loginResult = await Http.PostAsJsonAsync(new Uri(LexboxUrl, "api/login"), new { EmailOrUsername=lexboxUsername, Password=lexboxPassword }); + Jwt = await loginResult.Content.ReadAsStringAsync(); + Http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", Jwt); + } + + public void InitRepo(string code, string dest) + { + var sourceUrl = new Uri(LexboxUrl, $"hg/{code}"); + MercurialTestHelper.CloneRepo(sourceUrl.AbsoluteUri, dest); + } + + public void InitRepo(string code) => InitRepo(code, Path.Join(TempFolder.Path, code)); + + public async Task ResetAndUploadZip(string code, string zipPath) + { + var resetUrl = new Uri(LexboxUrl, $"api/project/resetProject/{code}"); + await Http.PostAsync(resetUrl, null); + await UploadZip(code, zipPath); + } + + public async Task ResetToEmpty(string code) + { + var resetUrl = new Uri(LexboxUrl, $"api/project/resetProject/{code}"); + await Http.PostAsync(resetUrl, null); + var finishResetUrl = new Uri(LexboxUrl, $"api/project/finishResetProject/{code}"); + await Http.PostAsync(finishResetUrl, null); + } + + public async Task UploadZip(string code, string zipPath) + { + var sourceUrl = new Uri(LexboxUrl, $"api/project/upload-zip/{code}"); + var file = new FileInfo(zipPath); + var client = new TusClient(); + var fileUrl = await client.CreateAsync(sourceUrl.AbsolutePath, file.Length, []); + await client.UploadAsync(fileUrl, file); + } + + private string TestName + { + get + { + var testName = TestContext.CurrentContext.Test.Name; + var firstInvalidChar = testName.IndexOfAny(Path.GetInvalidPathChars()); + if (firstInvalidChar >= 0) + testName = testName.Substring(0, firstInvalidChar); + return testName; + } + } + + } +} From 96f5d3900b137e2c4b9918ba075728de1f1a61df Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 25 Jul 2024 11:43:44 +0700 Subject: [PATCH 03/73] Better auth handling for E2E tests --- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 0506acc2..b7d35b3f 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -22,6 +22,7 @@ public class SRTestEnvironment { public ILogger Logger => MainClass.Logger; public Uri LexboxUrl { get; init; } + public Uri LexboxUrlBasicAuth { get; init; } private TemporaryFolder TempFolder { get; init; } private HttpClient Http { get; init; } = new HttpClient(); private string Jwt { get; set; } @@ -33,14 +34,20 @@ public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProto Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol, lexboxProtocol); Environment.SetEnvironmentVariable(MagicStrings.EnvVar_HgUsername, lexboxUsername); Environment.SetEnvironmentVariable(MagicStrings.EnvVar_TrustToken, lexboxPassword); - LexboxUrl = new Uri($"{lexboxProtocol}://{WebUtility.UrlEncode(lexboxUsername)}:{WebUtility.UrlEncode(lexboxPassword)}@{lexboxHostname}:{lexboxPort}"); + LexboxUrl = new Uri($"{lexboxProtocol}://{lexboxHostname}:{lexboxPort}"); + LexboxUrlBasicAuth = new Uri($"{lexboxProtocol}://{WebUtility.UrlEncode(lexboxUsername)}:{WebUtility.UrlEncode(lexboxPassword)}@{lexboxHostname}:{lexboxPort}"); TempFolder = new TemporaryFolder(TestName + Path.GetRandomFileName()); } - public async Task Login() + public Task Login() { var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername); var lexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken); + return LoginAs(lexboxUsername, lexboxPassword); + } + + public async Task LoginAs(string lexboxUsername, string lexboxPassword) + { var loginResult = await Http.PostAsJsonAsync(new Uri(LexboxUrl, "api/login"), new { EmailOrUsername=lexboxUsername, Password=lexboxPassword }); Jwt = await loginResult.Content.ReadAsStringAsync(); Http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", Jwt); @@ -48,7 +55,7 @@ public async Task Login() public void InitRepo(string code, string dest) { - var sourceUrl = new Uri(LexboxUrl, $"hg/{code}"); + var sourceUrl = new Uri(LexboxUrlBasicAuth, $"hg/{code}"); MercurialTestHelper.CloneRepo(sourceUrl.AbsoluteUri, dest); } @@ -74,6 +81,7 @@ public async Task UploadZip(string code, string zipPath) var sourceUrl = new Uri(LexboxUrl, $"api/project/upload-zip/{code}"); var file = new FileInfo(zipPath); var client = new TusClient(); + client.AdditionalHeaders["Authorization"] = $"Bearer {Jwt}"; var fileUrl = await client.CreateAsync(sourceUrl.AbsolutePath, file.Length, []); await client.UploadAsync(fileUrl, file); } From 5dd0a6050e47ace3d20c3075cfa995c5224014a6 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 25 Jul 2024 15:44:45 +0700 Subject: [PATCH 04/73] Add methods for resetting projects in LexBox This will enable setting projects to specific commits before the test. --- src/LfMerge.Core.Tests/MercurialTestHelper.cs | 6 +++ src/LfMerge.Core.Tests/SRTestEnvironment.cs | 54 +++++++++++++++---- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/LfMerge.Core.Tests/MercurialTestHelper.cs b/src/LfMerge.Core.Tests/MercurialTestHelper.cs index 5ae2f89a..d25cfb32 100644 --- a/src/LfMerge.Core.Tests/MercurialTestHelper.cs +++ b/src/LfMerge.Core.Tests/MercurialTestHelper.cs @@ -81,6 +81,12 @@ public static void CloneRepo(string sourceRepo, string destinationRepo) RunHgCommand(destinationRepo, $"clone {sourceRepo} ."); } + public static void CloneRepoAtRevnum(string sourceRepo, string destinationRepo, int revnum) + { + Directory.CreateDirectory(destinationRepo); + RunHgCommand(destinationRepo, $"clone {sourceRepo} -U -r {revnum} ."); + } + public static void ChangeBranch(string repoPath, string newBranch) { RunHgCommand(repoPath, $"update {newBranch}"); diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index b7d35b3f..60535a82 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -1,11 +1,10 @@ using System; -using System.ComponentModel; using System.IO; +using System.IO.Compression; using System.Net; using System.Net.Http; using System.Net.Http.Json; using System.Threading.Tasks; -using Bugsnag.Payload; using LfMerge.Core.Logging; using LfMerge.Core.Settings; using NUnit.Framework; @@ -24,7 +23,9 @@ public class SRTestEnvironment public Uri LexboxUrl { get; init; } public Uri LexboxUrlBasicAuth { get; init; } private TemporaryFolder TempFolder { get; init; } - private HttpClient Http { get; init; } = new HttpClient(); + private HttpClient Http { get; init; } + private HttpClientHandler Handler { get; init; } = new HttpClientHandler(); + private CookieContainer Cookies { get; init; } = new CookieContainer(); private string Jwt { get; set; } public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProtocol = "http", int lexboxPort = 80, string lexboxUsername = "admin", string lexboxPassword = "pass") @@ -37,6 +38,8 @@ public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProto LexboxUrl = new Uri($"{lexboxProtocol}://{lexboxHostname}:{lexboxPort}"); LexboxUrlBasicAuth = new Uri($"{lexboxProtocol}://{WebUtility.UrlEncode(lexboxUsername)}:{WebUtility.UrlEncode(lexboxPassword)}@{lexboxHostname}:{lexboxPort}"); TempFolder = new TemporaryFolder(TestName + Path.GetRandomFileName()); + Handler.CookieContainer = Cookies; + Http = new HttpClient(Handler); } public Task Login() @@ -48,9 +51,11 @@ public Task Login() public async Task LoginAs(string lexboxUsername, string lexboxPassword) { - var loginResult = await Http.PostAsJsonAsync(new Uri(LexboxUrl, "api/login"), new { EmailOrUsername=lexboxUsername, Password=lexboxPassword }); - Jwt = await loginResult.Content.ReadAsStringAsync(); - Http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", Jwt); + var loginResult = await Http.PostAsync(new Uri(LexboxUrl, "api/login"), JsonContent.Create(new { EmailOrUsername=lexboxUsername, Password=lexboxPassword })); + var cookies = Cookies.GetCookies(LexboxUrl); + Jwt = cookies[".LexBoxAuth"].Value; + // Http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", Jwt); + // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. } public void InitRepo(string code, string dest) @@ -78,14 +83,45 @@ public async Task ResetToEmpty(string code) public async Task UploadZip(string code, string zipPath) { - var sourceUrl = new Uri(LexboxUrl, $"api/project/upload-zip/{code}"); + var sourceUrl = new Uri(LexboxUrl, $"/api/project/upload-zip/{code}"); var file = new FileInfo(zipPath); var client = new TusClient(); - client.AdditionalHeaders["Authorization"] = $"Bearer {Jwt}"; - var fileUrl = await client.CreateAsync(sourceUrl.AbsolutePath, file.Length, []); + // client.AdditionalHeaders["Authorization"] = $"Bearer {Jwt}"; // Once we set up for LexBox OAuth, we'll use Bearer auth instead + var cookies = Cookies.GetCookies(LexboxUrl); + var authCookie = cookies[".LexBoxAuth"].ToString(); + client.AdditionalHeaders["cookie"] = authCookie; + var fileUrl = await client.CreateAsync(sourceUrl.AbsoluteUri, file.Length, ("filetype", "application/zip")); await client.UploadAsync(fileUrl, file); } + public async Task DownloadProjectBackup(string code) + { + var backupUrl = new Uri(LexboxUrl, $"api/project/backupProject/{code}"); + var result = await Http.GetAsync(backupUrl); + var filename = result.Content.Headers.ContentDisposition?.FileName; + var savePath = Path.Join(TempFolder.Path, filename); + using (var outStream = File.Create(savePath)) + { + await result.Content.CopyToAsync(outStream); + } + } + + public async Task RollbackProjectToRev(string code, int revnum) + { + // Negative rev numbers will be interpreted as Mercurial does: -1 is the tip revision, -2 is one back from the tip, etc. + // I.e. rolling back to rev -2 will remove the most recent commit + var backupUrl = new Uri(LexboxUrl, $"api/project/backupProject/{code}"); + var result = await Http.GetAsync(backupUrl); + var zipStream = await result.Content.ReadAsStreamAsync(); + var projectDir = Path.Join(TempFolder.Path, code); + ZipFile.ExtractToDirectory(zipStream, projectDir); + var clonedDir = Path.Join(TempFolder.Path, $"{code}-{revnum}"); + MercurialTestHelper.CloneRepoAtRevnum(projectDir, clonedDir, revnum); + var zipPath = Path.Join(TempFolder.Path, $"{code}-{revnum}.zip"); + ZipFile.CreateFromDirectory(clonedDir, zipPath); + await ResetAndUploadZip(code, zipPath); + } + private string TestName { get From 05a8c93dba3797a67385718dc57f64cd30d3d9f3 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 25 Jul 2024 15:45:36 +0700 Subject: [PATCH 05/73] Add a basic unit test to exercise project reset This test can now exercise the "reset project to known revision" functionality in LexBox, proving that it works. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/LfMerge.Core.Tests/E2E/BasicTests.cs diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs new file mode 100644 index 00000000..b513fe21 --- /dev/null +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -0,0 +1,21 @@ +using System; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace LfMerge.Core.Tests.E2E +{ + [TestFixture] + [Category("LongRunning")] + [Category("IntegrationTests")] + public class BasicTests + { + [Test] + public async Task CheckProjectBackupDownloading() + { + var env = new SRTestEnvironment(); + await env.Login(); + await env.RollbackProjectToRev("sena-3", -1); // Should make no changes + // await env.RollbackProjectToRev("sena-3", -2); // Should remove one commit + } + } +} From 554bd2ea770b586e818a5a89c124b0c0a642dd6b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 30 Jul 2024 13:55:58 +0700 Subject: [PATCH 06/73] Add helper method to clone Lcm project from LexBox Give it a project code, get an FwProject back, it does the rest. (It's the caller's job to dispose of the FwProject when you're done, though). --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 11 ++++ src/LfMerge.Core.Tests/LcmTestHelper.cs | 51 +++++++++++++++++++ src/LfMerge.Core.Tests/MercurialTestHelper.cs | 9 +++- src/LfMerge.Core/MagicStrings.cs | 1 + 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 src/LfMerge.Core.Tests/LcmTestHelper.cs diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index b513fe21..6df9daf8 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading.Tasks; using NUnit.Framework; @@ -17,5 +18,15 @@ public async Task CheckProjectBackupDownloading() await env.RollbackProjectToRev("sena-3", -1); // Should make no changes // await env.RollbackProjectToRev("sena-3", -2); // Should remove one commit } + + [Test] + public async Task CheckProjectCloning() + { + await LcmTestHelper.LexboxLogin("admin", "pass"); + var sena3 = LcmTestHelper.CloneFromLexbox("sena-3"); + var entries = sena3.ServiceLocator.LanguageProject.LexDbOA.Entries; + Console.WriteLine($"Project has {entries.Count()} entries"); + sena3.Dispose(); + } } } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs new file mode 100644 index 00000000..609366b1 --- /dev/null +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -0,0 +1,51 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading.Tasks; +using LfMerge.Core.FieldWorks; +using SIL.PlatformUtilities; +using SIL.Progress; + +namespace LfMerge.Core.Tests +{ + public static class LcmTestHelper + { + public static string HgCommand => + Path.Combine(TestEnvironment.FindGitRepoRoot(), "Mercurial", + Platform.IsWindows ? "hg.exe" : "hg"); + + public static string LexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; + public static string LexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; + public static string LexboxPort = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriPort) ?? (LexboxProtocol == "http" ? "80" : "443"); + public static Uri LexboxUrl = new Uri($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); + + public static string BaseDir = Path.Combine(Path.GetTempPath(), nameof(LcmTestHelper)); + + public static HttpClientHandler Handler { get; set; } = new HttpClientHandler(); + public static CookieContainer Cookies => Handler.CookieContainer; + public static HttpClient Http { get; set; } = new HttpClient(Handler); + + public static async Task LexboxLogin(string username, string password) + { + await Http.PostAsJsonAsync(new Uri(LexboxUrl, "/api/login"), new { EmailOrUsername=username, Password=password }); + var cookies = Cookies.GetCookies(LexboxUrl); + return cookies[".LexBoxAuth"].Value; + } + + public static FwProject CloneFromLexbox(string code, string? dest = null) + { + var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); + var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; + dest ??= Path.Combine(BaseDir, "webwork", code); + MercurialTestHelper.CloneRepo(withAuth.Uri.AbsoluteUri, dest); + var fwdataPath = Path.Join(dest, $"{code}.fwdata"); + var progress = new NullProgress(); + MercurialTestHelper.ChangeBranch(dest, "tip"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(progress, false, fwdataPath); + var settings = new LfMergeSettingsDouble(BaseDir); + return new FwProject(settings, code); + } + } +} diff --git a/src/LfMerge.Core.Tests/MercurialTestHelper.cs b/src/LfMerge.Core.Tests/MercurialTestHelper.cs index d25cfb32..3f63ae50 100644 --- a/src/LfMerge.Core.Tests/MercurialTestHelper.cs +++ b/src/LfMerge.Core.Tests/MercurialTestHelper.cs @@ -81,10 +81,15 @@ public static void CloneRepo(string sourceRepo, string destinationRepo) RunHgCommand(destinationRepo, $"clone {sourceRepo} ."); } - public static void CloneRepoAtRevnum(string sourceRepo, string destinationRepo, int revnum) + public static void CloneRepoAtRev(string sourceRepo, string destinationRepo, string rev) { Directory.CreateDirectory(destinationRepo); - RunHgCommand(destinationRepo, $"clone {sourceRepo} -U -r {revnum} ."); + RunHgCommand(destinationRepo, $"clone {sourceRepo} -U -r {rev} ."); + } + + public static void CloneRepoAtRevnum(string sourceRepo, string destinationRepo, int revnum) + { + CloneRepoAtRev(sourceRepo, destinationRepo, revnum.ToString()); } public static void ChangeBranch(string repoPath, string newBranch) diff --git a/src/LfMerge.Core/MagicStrings.cs b/src/LfMerge.Core/MagicStrings.cs index 25f9cbc6..f0966513 100644 --- a/src/LfMerge.Core/MagicStrings.cs +++ b/src/LfMerge.Core/MagicStrings.cs @@ -30,6 +30,7 @@ static MagicStrings() public const string EnvVar_LanguageDepotPublicHostname = "LFMERGE_LANGUAGE_DEPOT_HG_PUBLIC_HOSTNAME"; public const string EnvVar_LanguageDepotPrivateHostname = "LFMERGE_LANGUAGE_DEPOT_HG_PRIVATE_HOSTNAME"; public const string EnvVar_LanguageDepotUriProtocol = "LFMERGE_LANGUAGE_DEPOT_HG_PROTOCOL"; + public const string EnvVar_LanguageDepotUriPort = "LFMERGE_LANGUAGE_DEPOT_HG_PORT"; public const string EnvVar_TrustToken = "LANGUAGE_DEPOT_TRUST_TOKEN"; public const string EnvVar_HgUsername = "LANGUAGE_DEPOT_HG_USERNAME"; From 4d5d4c956de7969c6740bc7c0d276eacc13586d6 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 30 Jul 2024 14:09:22 +0700 Subject: [PATCH 07/73] Add GetEntries helper method --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 2 +- src/LfMerge.Core.Tests/LcmTestHelper.cs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index 6df9daf8..c1d56d68 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -24,7 +24,7 @@ public async Task CheckProjectCloning() { await LcmTestHelper.LexboxLogin("admin", "pass"); var sena3 = LcmTestHelper.CloneFromLexbox("sena-3"); - var entries = sena3.ServiceLocator.LanguageProject.LexDbOA.Entries; + var entries = LcmTestHelper.GetEntries(sena3); Console.WriteLine($"Project has {entries.Count()} entries"); sena3.Dispose(); } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index 609366b1..9cd47cff 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -1,10 +1,12 @@ using System; +using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http; using System.Net.Http.Json; using System.Threading.Tasks; using LfMerge.Core.FieldWorks; +using SIL.LCModel; using SIL.PlatformUtilities; using SIL.Progress; @@ -47,5 +49,10 @@ public static FwProject CloneFromLexbox(string code, string? dest = null) var settings = new LfMergeSettingsDouble(BaseDir); return new FwProject(settings, code); } + + public static IEnumerable GetEntries(FwProject project) + { + return project?.ServiceLocator?.LanguageProject?.LexDbOA?.Entries ?? []; + } } } From 41c58ed3d34d66dd4162db572bde96176c201f8e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 30 Jul 2024 14:14:38 +0700 Subject: [PATCH 08/73] Deal with disposable FwProjects better --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index c1d56d68..c209d26d 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -23,10 +23,9 @@ public async Task CheckProjectBackupDownloading() public async Task CheckProjectCloning() { await LcmTestHelper.LexboxLogin("admin", "pass"); - var sena3 = LcmTestHelper.CloneFromLexbox("sena-3"); + using var sena3 = LcmTestHelper.CloneFromLexbox("sena-3"); var entries = LcmTestHelper.GetEntries(sena3); Console.WriteLine($"Project has {entries.Count()} entries"); - sena3.Dispose(); } } } From 83d6e21b780bacb6b93b6e6bc4ac8310379aad48 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 30 Jul 2024 14:59:30 +0700 Subject: [PATCH 09/73] Add helper method to get a single entry --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 3 +++ src/LfMerge.Core.Tests/LcmTestHelper.cs | 18 ++++++++++++++++++ src/LfMerge.Core.Tests/MercurialTestHelper.cs | 5 +++++ 3 files changed, 26 insertions(+) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index c209d26d..ce0b5344 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -26,6 +26,9 @@ public async Task CheckProjectCloning() using var sena3 = LcmTestHelper.CloneFromLexbox("sena-3"); var entries = LcmTestHelper.GetEntries(sena3); Console.WriteLine($"Project has {entries.Count()} entries"); + var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); + var citationForm = entry.CitationForm.BestVernacularAlternative.Text; + Assert.That(citationForm, Is.EqualTo("ambuka")); } } } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index 9cd47cff..14deb0e3 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -28,6 +28,7 @@ public static class LcmTestHelper public static HttpClientHandler Handler { get; set; } = new HttpClientHandler(); public static CookieContainer Cookies => Handler.CookieContainer; public static HttpClient Http { get; set; } = new HttpClient(Handler); + public static IProgress NullProgress = new NullProgress(); public static async Task LexboxLogin(string username, string password) { @@ -50,9 +51,26 @@ public static FwProject CloneFromLexbox(string code, string? dest = null) return new FwProject(settings, code); } + public static void CommitChanges(FwProject project, string code, string? commitMsg = null) + { + if (!project.IsDisposed) project.Dispose(); + commitMsg ??= "Auto-commit"; + var projectDir = Path.Combine(BaseDir, "webwork", code); + var fwdataPath = Path.Join(projectDir, $"{code}.fwdata"); + LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(NullProgress, false, fwdataPath); + MercurialTestHelper.HgCommit(projectDir, commitMsg); + MercurialTestHelper.HgPush(projectDir); + } + public static IEnumerable GetEntries(FwProject project) { return project?.ServiceLocator?.LanguageProject?.LexDbOA?.Entries ?? []; } + + public static ILexEntry GetEntry(FwProject project, Guid guid) + { + var repo = project?.ServiceLocator?.GetInstance(); + return repo.GetObject(guid); + } } } diff --git a/src/LfMerge.Core.Tests/MercurialTestHelper.cs b/src/LfMerge.Core.Tests/MercurialTestHelper.cs index 3f63ae50..16bebb29 100644 --- a/src/LfMerge.Core.Tests/MercurialTestHelper.cs +++ b/src/LfMerge.Core.Tests/MercurialTestHelper.cs @@ -63,6 +63,11 @@ public static void HgCreateBranch(string repoPath, int branchName) RunHgCommand(repoPath, $"branch -f \"{branchName}\""); } + public static void HgPush(string repoPath) + { + RunHgCommand(repoPath, $"push"); + } + public static void CreateFlexRepo(string lDProjectFolderPath, int modelVersion = 0) { if (modelVersion <= 0) From 2b4e84a9970b321d3a86edafc9f4fccfc19c6ed7 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 30 Jul 2024 15:29:29 +0700 Subject: [PATCH 10/73] Add helper for commit and push Also add test demonstrating that the helper works, and we can modify an entry's citation form, then push the modified entry to LexBox. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 8 ++++++++ src/LfMerge.Core.Tests/LcmTestHelper.cs | 4 +++- src/LfMerge.Core.Tests/MercurialTestHelper.cs | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index ce0b5344..0c5d04df 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -2,6 +2,8 @@ using System.Linq; using System.Threading.Tasks; using NUnit.Framework; +using SIL.LCModel.Core.Text; +using SIL.LCModel.Infrastructure; namespace LfMerge.Core.Tests.E2E { @@ -29,6 +31,12 @@ public async Task CheckProjectCloning() var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); var citationForm = entry.CitationForm.BestVernacularAlternative.Text; Assert.That(citationForm, Is.EqualTo("ambuka")); + var ws = entry.CitationForm.BestVernacularAlternative.get_WritingSystem(0); + // TODO: Move undo/redo stuff into a helper method in LcmTestHelper, as it quickly gets repetitive + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", sena3.Cache.ActionHandlerAccessor, () => { + entry.CitationForm.set_String(ws, "something"); + }); + LcmTestHelper.CommitChanges(sena3, "sena-3"); } } } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index 14deb0e3..dd1d4346 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -53,13 +53,15 @@ public static FwProject CloneFromLexbox(string code, string? dest = null) public static void CommitChanges(FwProject project, string code, string? commitMsg = null) { + var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); + var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; if (!project.IsDisposed) project.Dispose(); commitMsg ??= "Auto-commit"; var projectDir = Path.Combine(BaseDir, "webwork", code); var fwdataPath = Path.Join(projectDir, $"{code}.fwdata"); LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(NullProgress, false, fwdataPath); MercurialTestHelper.HgCommit(projectDir, commitMsg); - MercurialTestHelper.HgPush(projectDir); + MercurialTestHelper.HgPush(projectDir, withAuth.Uri.AbsoluteUri); } public static IEnumerable GetEntries(FwProject project) diff --git a/src/LfMerge.Core.Tests/MercurialTestHelper.cs b/src/LfMerge.Core.Tests/MercurialTestHelper.cs index 16bebb29..4ebfde07 100644 --- a/src/LfMerge.Core.Tests/MercurialTestHelper.cs +++ b/src/LfMerge.Core.Tests/MercurialTestHelper.cs @@ -63,9 +63,9 @@ public static void HgCreateBranch(string repoPath, int branchName) RunHgCommand(repoPath, $"branch -f \"{branchName}\""); } - public static void HgPush(string repoPath) + public static void HgPush(string repoPath, string remoteUri) { - RunHgCommand(repoPath, $"push"); + RunHgCommand(repoPath, $"push {remoteUri}"); } public static void CreateFlexRepo(string lDProjectFolderPath, int modelVersion = 0) From 3a54e8164f0cee10b7d7175a98cf75526b507bf2 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 31 Jul 2024 09:16:20 +0700 Subject: [PATCH 11/73] Improve LcmTestHelper method for setting text Now test code doesn't have to deal with undo/redo accessors. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 8 +------- src/LfMerge.Core.Tests/LcmTestHelper.cs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index 0c5d04df..e14f1fe6 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -2,8 +2,6 @@ using System.Linq; using System.Threading.Tasks; using NUnit.Framework; -using SIL.LCModel.Core.Text; -using SIL.LCModel.Infrastructure; namespace LfMerge.Core.Tests.E2E { @@ -31,11 +29,7 @@ public async Task CheckProjectCloning() var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); var citationForm = entry.CitationForm.BestVernacularAlternative.Text; Assert.That(citationForm, Is.EqualTo("ambuka")); - var ws = entry.CitationForm.BestVernacularAlternative.get_WritingSystem(0); - // TODO: Move undo/redo stuff into a helper method in LcmTestHelper, as it quickly gets repetitive - UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", sena3.Cache.ActionHandlerAccessor, () => { - entry.CitationForm.set_String(ws, "something"); - }); + LcmTestHelper.SetVernacularText(sena3, entry.CitationForm, "something"); LcmTestHelper.CommitChanges(sena3, "sena-3"); } } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index dd1d4346..e742b202 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using LfMerge.Core.FieldWorks; using SIL.LCModel; +using SIL.LCModel.Infrastructure; using SIL.PlatformUtilities; using SIL.Progress; @@ -74,5 +75,21 @@ public static ILexEntry GetEntry(FwProject project, Guid guid) var repo = project?.ServiceLocator?.GetInstance(); return repo.GetObject(guid); } + + public static void SetVernacularText(FwProject project, IMultiUnicode field, string newText) + { + var accessor = project.Cache.ActionHandlerAccessor; + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", accessor, () => { + field.SetVernacularDefaultWritingSystem(newText); + }); + } + + public static void SetAnalysisText(FwProject project, IMultiUnicode field, string newText) + { + var accessor = project.Cache.ActionHandlerAccessor; + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", accessor, () => { + field.SetAnalysisDefaultWritingSystem(newText); + }); + } } } From f28c6efd113d2d58872115779b8fb75eca3b851f Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 31 Jul 2024 09:29:06 +0700 Subject: [PATCH 12/73] Add test to ensure commit is working So far the test is failing, so it's not working yet. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 9 ++++++ src/LfMerge.Core.Tests/LcmTestHelper.cs | 36 +++++++++++++++++++----- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index e14f1fe6..34ae2507 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -31,6 +31,15 @@ public async Task CheckProjectCloning() Assert.That(citationForm, Is.EqualTo("ambuka")); LcmTestHelper.SetVernacularText(sena3, entry.CitationForm, "something"); LcmTestHelper.CommitChanges(sena3, "sena-3"); + + using var sena4 = LcmTestHelper.CloneFromLexbox("sena-3", "sena-4"); + entry = LcmTestHelper.GetEntry(sena4, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); + citationForm = entry.CitationForm.BestVernacularAlternative.Text; + Assert.That(citationForm, Is.EqualTo("something")); + LcmTestHelper.UpdateVernacularText(sena4, entry.CitationForm, (s) => $"{s}XYZ"); + citationForm = entry.CitationForm.BestVernacularAlternative.Text; + Assert.That(citationForm, Is.EqualTo("somethingXYZ")); + LcmTestHelper.CommitChanges(sena4, "sena-3", "sena-4"); } } } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index e742b202..b6786ac2 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -38,28 +38,30 @@ public static async Task LexboxLogin(string username, string password) return cookies[".LexBoxAuth"].Value; } - public static FwProject CloneFromLexbox(string code, string? dest = null) + public static FwProject CloneFromLexbox(string code, string? newCode = null) { var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; - dest ??= Path.Combine(BaseDir, "webwork", code); + newCode ??= code; + var dest = Path.Combine(BaseDir, "webwork", newCode); MercurialTestHelper.CloneRepo(withAuth.Uri.AbsoluteUri, dest); - var fwdataPath = Path.Join(dest, $"{code}.fwdata"); + var fwdataPath = Path.Join(dest, $"{newCode}.fwdata"); var progress = new NullProgress(); MercurialTestHelper.ChangeBranch(dest, "tip"); LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(progress, false, fwdataPath); var settings = new LfMergeSettingsDouble(BaseDir); - return new FwProject(settings, code); + return new FwProject(settings, newCode); } - public static void CommitChanges(FwProject project, string code, string? commitMsg = null) + public static void CommitChanges(FwProject project, string code, string? localCode = null, string? commitMsg = null) { + localCode ??= code; var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; if (!project.IsDisposed) project.Dispose(); commitMsg ??= "Auto-commit"; - var projectDir = Path.Combine(BaseDir, "webwork", code); - var fwdataPath = Path.Join(projectDir, $"{code}.fwdata"); + var projectDir = Path.Combine(BaseDir, "webwork", localCode); + var fwdataPath = Path.Join(projectDir, $"{localCode}.fwdata"); LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(NullProgress, false, fwdataPath); MercurialTestHelper.HgCommit(projectDir, commitMsg); MercurialTestHelper.HgPush(projectDir, withAuth.Uri.AbsoluteUri); @@ -91,5 +93,25 @@ public static void SetAnalysisText(FwProject project, IMultiUnicode field, strin field.SetAnalysisDefaultWritingSystem(newText); }); } + + public static void UpdateVernacularText(FwProject project, IMultiUnicode field, Func textConverter) + { + var oldText = field.BestVernacularAlternative?.Text; + if (oldText != null) + { + var newText = textConverter(oldText); + SetVernacularText(project, field, newText); + } + } + + public static void UpdateAnalysisText(FwProject project, IMultiUnicode field, Func textConverter) + { + var oldText = field.BestAnalysisAlternative?.Text; + if (oldText != null) + { + var newText = textConverter(oldText); + SetAnalysisText(project, field, newText); + } + } } } From d2665f58ee86ba970fd200ccb5221a07a5b7fb8c Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 7 Aug 2024 16:16:41 +0700 Subject: [PATCH 13/73] Start of new SRTestBase class which will autoreset Now, if you tag a test with `[Property("projectCode", "sena-3")]` it will automatically restore the sena-3 project to its original state at the end of the test. This will ensure idempotence even if tests fail. --- src/LfMerge.Core.Tests/SRTestBase.cs | 101 ++++++++++++++++++++ src/LfMerge.Core.Tests/SRTestEnvironment.cs | 22 ++++- 2 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 src/LfMerge.Core.Tests/SRTestBase.cs diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs new file mode 100644 index 00000000..4e576e7c --- /dev/null +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -0,0 +1,101 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading.Tasks; +using LfMerge.Core.Logging; +using LfMerge.Core.Settings; +using NUnit.Framework; +using NUnit.Framework.Interfaces; +using NUnit.Framework.Internal; +using SIL.LCModel; +using SIL.TestUtilities; +using TusDotNetClient; + +namespace LfMerge.Core.Tests +{ + /// + /// Test base class for end-to-end testing, i.e. Send/Receive with a real LexBox instance + /// + public class SRTestBase + { + public ILogger Logger => MainClass.Logger; + public Uri LexboxUrl { get; init; } + public Uri LexboxUrlBasicAuth { get; init; } + private TemporaryFolder TempFolder { get; init; } + private HttpClient Http { get; init; } + private HttpClientHandler Handler { get; init; } = new HttpClientHandler(); + private CookieContainer Cookies { get; init; } = new CookieContainer(); + private string Jwt { get; set; } + private string TipRevToRestore { get; set; } = ""; + + public SRTestBase(string lexboxHostname = "localhost", string lexboxProtocol = "http", int lexboxPort = 80, string lexboxUsername = "admin", string lexboxPassword = "pass") + { + // TODO: Just get an SRTestEnvironment instead of all this + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname, lexboxHostname); + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPrivateHostname, lexboxHostname); + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol, lexboxProtocol); + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_HgUsername, lexboxUsername); + Environment.SetEnvironmentVariable(MagicStrings.EnvVar_TrustToken, lexboxPassword); + LexboxUrl = new Uri($"{lexboxProtocol}://{lexboxHostname}:{lexboxPort}"); + LexboxUrlBasicAuth = new Uri($"{lexboxProtocol}://{WebUtility.UrlEncode(lexboxUsername)}:{WebUtility.UrlEncode(lexboxPassword)}@{lexboxHostname}:{lexboxPort}"); + TempFolder = new TemporaryFolder(TestName + Path.GetRandomFileName()); + Handler.CookieContainer = Cookies; + Http = new HttpClient(Handler); + } + + public Task Login() + { + var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername); + var lexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken); + return LoginAs(lexboxUsername, lexboxPassword); + } + + public async Task LoginAs(string lexboxUsername, string lexboxPassword) + { + var loginResult = await Http.PostAsync(new Uri(LexboxUrl, "api/login"), JsonContent.Create(new { EmailOrUsername=lexboxUsername, Password=lexboxPassword })); + var cookies = Cookies.GetCookies(LexboxUrl); + Jwt = cookies[".LexBoxAuth"].Value; + // Http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", Jwt); + // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. + } + + private string TestName + { + get + { + var testName = TestContext.CurrentContext.Test.Name; + var firstInvalidChar = testName.IndexOfAny(Path.GetInvalidPathChars()); + if (firstInvalidChar >= 0) + testName = testName.Substring(0, firstInvalidChar); + return testName; + } + } + + [SetUp] + public async Task BackupRemoteProject() + { + var env = new SRTestEnvironment(); // TODO: Instance property + var test = TestContext.CurrentContext.Test; + if (test.Properties.ContainsKey("projectCode")) { + var code = test.Properties.Get("projectCode") as string; + TipRevToRestore = await env.GetTipRev(code); + } else { + TipRevToRestore = ""; + } + } + + [TearDown] + public async Task RestoreRemoteProject() + { + var env = new SRTestEnvironment(); // TODO: Instance property + var test = TestContext.CurrentContext.Test; + if (!string.IsNullOrEmpty(TipRevToRestore) && test.Properties.ContainsKey("projectCode")) { + var code = test.Properties.Get("projectCode") as string; + await env.RollbackProjectToRev(code, TipRevToRestore); + } + } + } +} diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 60535a82..498bb562 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -106,7 +106,7 @@ public async Task DownloadProjectBackup(string code) } } - public async Task RollbackProjectToRev(string code, int revnum) + public async Task RollbackProjectToRev(string code, string rev) { // Negative rev numbers will be interpreted as Mercurial does: -1 is the tip revision, -2 is one back from the tip, etc. // I.e. rolling back to rev -2 will remove the most recent commit @@ -115,13 +115,27 @@ public async Task RollbackProjectToRev(string code, int revnum) var zipStream = await result.Content.ReadAsStreamAsync(); var projectDir = Path.Join(TempFolder.Path, code); ZipFile.ExtractToDirectory(zipStream, projectDir); - var clonedDir = Path.Join(TempFolder.Path, $"{code}-{revnum}"); - MercurialTestHelper.CloneRepoAtRevnum(projectDir, clonedDir, revnum); - var zipPath = Path.Join(TempFolder.Path, $"{code}-{revnum}.zip"); + var clonedDir = Path.Join(TempFolder.Path, $"{code}-{rev}"); + MercurialTestHelper.CloneRepoAtRev(projectDir, clonedDir, rev); + var zipPath = Path.Join(TempFolder.Path, $"{code}-{rev}.zip"); ZipFile.CreateFromDirectory(clonedDir, zipPath); await ResetAndUploadZip(code, zipPath); } + public Task RollbackProjectToRev(string code, int revnum) + { + return RollbackProjectToRev(code, revnum.ToString()); + } + + public record TipJson(string Node); + + public async Task GetTipRev(string code) + { + var tipUrl = new Uri(LexboxUrl, $"/hg/{code}/file/tip?style=json"); + var result = await Http.GetFromJsonAsync(tipUrl); + return result.Node; + } + private string TestName { get From f273aa6790a3c9e81e0248990c68f610d2958933 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 7 Aug 2024 16:31:30 +0700 Subject: [PATCH 14/73] Efficiency: don't rollback if already at right rev --- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 498bb562..4019c057 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -110,6 +110,9 @@ public async Task RollbackProjectToRev(string code, string rev) { // Negative rev numbers will be interpreted as Mercurial does: -1 is the tip revision, -2 is one back from the tip, etc. // I.e. rolling back to rev -2 will remove the most recent commit + if (rev == "-1") return; // Already at tip, nothing to do + var currentTip = await GetTipRev(code); + if (rev == currentTip) return; // Already at tip, nothing to do var backupUrl = new Uri(LexboxUrl, $"api/project/backupProject/{code}"); var result = await Http.GetAsync(backupUrl); var zipStream = await result.Content.ReadAsStreamAsync(); From 4b844e907769a611d0b6f74ed4c83d789a413650 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 7 Aug 2024 16:38:32 +0700 Subject: [PATCH 15/73] SRTestBase sets up a test environment for itself --- src/LfMerge.Core.Tests/SRTestBase.cs | 28 ++++++++------------- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 5 +++- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 4e576e7c..8e01d074 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -5,7 +5,6 @@ using System.Net.Http; using System.Net.Http.Json; using System.Threading.Tasks; -using LfMerge.Core.Logging; using LfMerge.Core.Settings; using NUnit.Framework; using NUnit.Framework.Interfaces; @@ -21,7 +20,7 @@ namespace LfMerge.Core.Tests /// public class SRTestBase { - public ILogger Logger => MainClass.Logger; + public LfMerge.Core.Logging.ILogger Logger => MainClass.Logger; public Uri LexboxUrl { get; init; } public Uri LexboxUrlBasicAuth { get; init; } private TemporaryFolder TempFolder { get; init; } @@ -30,20 +29,17 @@ public class SRTestBase private CookieContainer Cookies { get; init; } = new CookieContainer(); private string Jwt { get; set; } private string TipRevToRestore { get; set; } = ""; + private SRTestEnvironment TestEnv { get; init; } - public SRTestBase(string lexboxHostname = "localhost", string lexboxProtocol = "http", int lexboxPort = 80, string lexboxUsername = "admin", string lexboxPassword = "pass") + public SRTestBase() { // TODO: Just get an SRTestEnvironment instead of all this - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname, lexboxHostname); - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPrivateHostname, lexboxHostname); - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol, lexboxProtocol); - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_HgUsername, lexboxUsername); - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_TrustToken, lexboxPassword); - LexboxUrl = new Uri($"{lexboxProtocol}://{lexboxHostname}:{lexboxPort}"); - LexboxUrlBasicAuth = new Uri($"{lexboxProtocol}://{WebUtility.UrlEncode(lexboxUsername)}:{WebUtility.UrlEncode(lexboxPassword)}@{lexboxHostname}:{lexboxPort}"); - TempFolder = new TemporaryFolder(TestName + Path.GetRandomFileName()); - Handler.CookieContainer = Cookies; - Http = new HttpClient(Handler); + var lexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; + var lexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; + var lexboxPort = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriPort) ?? "80"; + var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername) ?? "admin"; + var lexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; + TestEnv = new SRTestEnvironment(lexboxHostname, lexboxProtocol, lexboxPort, lexboxUsername, lexboxPassword); } public Task Login() @@ -77,11 +73,10 @@ private string TestName [SetUp] public async Task BackupRemoteProject() { - var env = new SRTestEnvironment(); // TODO: Instance property var test = TestContext.CurrentContext.Test; if (test.Properties.ContainsKey("projectCode")) { var code = test.Properties.Get("projectCode") as string; - TipRevToRestore = await env.GetTipRev(code); + TipRevToRestore = await TestEnv.GetTipRev(code); } else { TipRevToRestore = ""; } @@ -90,11 +85,10 @@ public async Task BackupRemoteProject() [TearDown] public async Task RestoreRemoteProject() { - var env = new SRTestEnvironment(); // TODO: Instance property var test = TestContext.CurrentContext.Test; if (!string.IsNullOrEmpty(TipRevToRestore) && test.Properties.ContainsKey("projectCode")) { var code = test.Properties.Get("projectCode") as string; - await env.RollbackProjectToRev(code, TipRevToRestore); + await TestEnv.RollbackProjectToRev(code, TipRevToRestore); } } } diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 4019c057..24bb55f1 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -28,7 +28,7 @@ public class SRTestEnvironment private CookieContainer Cookies { get; init; } = new CookieContainer(); private string Jwt { get; set; } - public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProtocol = "http", int lexboxPort = 80, string lexboxUsername = "admin", string lexboxPassword = "pass") + public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProtocol = "http", string lexboxPort = "80", string lexboxUsername = "admin", string lexboxPassword = "pass") { Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname, lexboxHostname); Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPrivateHostname, lexboxHostname); @@ -42,6 +42,9 @@ public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProto Http = new HttpClient(Handler); } + public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProtocol = "http", int lexboxPort = 80, string lexboxUsername = "admin", string lexboxPassword = "pass") + : this(lexboxHostname, lexboxProtocol, lexboxPort.ToString(), lexboxUsername, lexboxPassword) { } + public Task Login() { var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername); From 6f93dec67755d2e80338df244356abe0214d8b3d Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 7 Aug 2024 16:54:34 +0700 Subject: [PATCH 16/73] Simpler way to turn test names into filenames --- src/LfMerge.Core.Tests/SRTestBase.cs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 8e01d074..3c64ba5f 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -58,17 +58,8 @@ public async Task LoginAs(string lexboxUsername, string lexboxPassword) // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. } - private string TestName - { - get - { - var testName = TestContext.CurrentContext.Test.Name; - var firstInvalidChar = testName.IndexOfAny(Path.GetInvalidPathChars()); - if (firstInvalidChar >= 0) - testName = testName.Substring(0, firstInvalidChar); - return testName; - } - } + private string TestName => TestContext.CurrentContext.Test.Name; + private string TestNameForPath => string.Join("", TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars [SetUp] public async Task BackupRemoteProject() From fd82dfcbdf66f8ebe88e9bb8993f913279aa66c4 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 8 Aug 2024 09:14:09 +0700 Subject: [PATCH 17/73] Remove overload of SRTestEnvironment constructor It's causing overload ambiguity when you create an SRTestEnvironment with no parameters. We'll just pass the port number in as a string. --- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 24bb55f1..14adb4d8 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -42,9 +42,6 @@ public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProto Http = new HttpClient(Handler); } - public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProtocol = "http", int lexboxPort = 80, string lexboxUsername = "admin", string lexboxPassword = "pass") - : this(lexboxHostname, lexboxProtocol, lexboxPort.ToString(), lexboxUsername, lexboxPassword) { } - public Task Login() { var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername); From 9d15c3bc843fb0628780a82055e1bf80ea59f8dd Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 8 Aug 2024 09:31:16 +0700 Subject: [PATCH 18/73] Much better setup and teardown in SRTestBase Now we have one folder per test, and it's deleted afterwards if the test passed. If the test failed, the folder is preserved, but it will be deleted the next time the test runs, so investigate before re-running tests if you need the files in the folder. --- src/LfMerge.Core.Tests/SRTestBase.cs | 69 +++++++++++++++------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 3c64ba5f..b04ed5b2 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -1,17 +1,9 @@ using System; using System.IO; -using System.IO.Compression; -using System.Net; -using System.Net.Http; -using System.Net.Http.Json; using System.Threading.Tasks; -using LfMerge.Core.Settings; using NUnit.Framework; using NUnit.Framework.Interfaces; -using NUnit.Framework.Internal; -using SIL.LCModel; using SIL.TestUtilities; -using TusDotNetClient; namespace LfMerge.Core.Tests { @@ -21,47 +13,55 @@ namespace LfMerge.Core.Tests public class SRTestBase { public LfMerge.Core.Logging.ILogger Logger => MainClass.Logger; - public Uri LexboxUrl { get; init; } - public Uri LexboxUrlBasicAuth { get; init; } - private TemporaryFolder TempFolder { get; init; } - private HttpClient Http { get; init; } - private HttpClientHandler Handler { get; init; } = new HttpClientHandler(); - private CookieContainer Cookies { get; init; } = new CookieContainer(); - private string Jwt { get; set; } + public TemporaryFolder TempFolderForClass { get; set; } + public TemporaryFolder TempFolderForTest { get; set; } private string TipRevToRestore { get; set; } = ""; - private SRTestEnvironment TestEnv { get; init; } + private SRTestEnvironment TestEnv { get; set; } public SRTestBase() { - // TODO: Just get an SRTestEnvironment instead of all this + } + + private string TestName => TestContext.CurrentContext.Test.Name; + private string TestNameForPath => string.Join("", TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars + + [OneTimeSetUp] + public async Task FixtureSetup() + { + // Log in to LexBox as admin var lexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; var lexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; var lexboxPort = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriPort) ?? "80"; var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername) ?? "admin"; var lexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; TestEnv = new SRTestEnvironment(lexboxHostname, lexboxProtocol, lexboxPort, lexboxUsername, lexboxPassword); + await TestEnv.Login(); + + // Ensure we don't delete top-level /tmp/LfMergeSRTests folder if it already exists + var tempPath = Path.Combine(Path.GetTempPath(), "LfMergeSRTests"); + var rootTempFolder = Directory.Exists(tempPath) ? TemporaryFolder.TrackExisting(tempPath) : new TemporaryFolder(tempPath); + + // But the folder for this specific test suite should be deleted if it already exists + var derivedClassName = this.GetType().Name; + TempFolderForClass = new TemporaryFolder(rootTempFolder, derivedClassName); } - public Task Login() + [OneTimeTearDown] + public void FixtureTeardown() { - var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername); - var lexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken); - return LoginAs(lexboxUsername, lexboxPassword); + var result = TestContext.CurrentContext.Result; + var nonSuccess = result.FailCount + result.InconclusiveCount + result.WarningCount; + // Only delete class temp folder if we passed or skipped all tests + if (nonSuccess == 0) TempFolderForClass.Dispose(); } - public async Task LoginAs(string lexboxUsername, string lexboxPassword) + [SetUp] + public async Task TestSetup() { - var loginResult = await Http.PostAsync(new Uri(LexboxUrl, "api/login"), JsonContent.Create(new { EmailOrUsername=lexboxUsername, Password=lexboxPassword })); - var cookies = Cookies.GetCookies(LexboxUrl); - Jwt = cookies[".LexBoxAuth"].Value; - // Http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", Jwt); - // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. + TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath); + await BackupRemoteProject(); } - private string TestName => TestContext.CurrentContext.Test.Name; - private string TestNameForPath => string.Join("", TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars - - [SetUp] public async Task BackupRemoteProject() { var test = TestContext.CurrentContext.Test; @@ -74,6 +74,13 @@ public async Task BackupRemoteProject() } [TearDown] + public async Task TestTeardown() + { + await RestoreRemoteProject(); + // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation + if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) TempFolderForTest.Dispose(); + } + public async Task RestoreRemoteProject() { var test = TestContext.CurrentContext.Test; From 8399906b19a3b413bcb89b3bc468228afbc694aa Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 8 Aug 2024 09:51:43 +0700 Subject: [PATCH 19/73] Clone, commit, and push tests are now repeatable Now that temp folders are set up properly, we can run tests that clone, commit, and push repeatedly: the LexBox project is automatically returned to its original state at the end of the test, and the temp folders are cleaned up prior to the next test being run. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 17 ++++++++--------- src/LfMerge.Core.Tests/LcmTestHelper.cs | 12 +++++------- src/LfMerge.Core.Tests/SRTestBase.cs | 13 ++++++++++++- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index 34ae2507..e22f50dc 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -8,38 +8,37 @@ namespace LfMerge.Core.Tests.E2E [TestFixture] [Category("LongRunning")] [Category("IntegrationTests")] - public class BasicTests + public class BasicTests : SRTestBase { [Test] public async Task CheckProjectBackupDownloading() { - var env = new SRTestEnvironment(); - await env.Login(); - await env.RollbackProjectToRev("sena-3", -1); // Should make no changes - // await env.RollbackProjectToRev("sena-3", -2); // Should remove one commit + await TestEnv.RollbackProjectToRev("sena-3", -1); // Should make no changes + // await TestEnv.RollbackProjectToRev("sena-3", -2); // Should remove one commit } [Test] + [Property("projectCode", "sena-3")] // This will cause it to auto-reset the project afterwards public async Task CheckProjectCloning() { await LcmTestHelper.LexboxLogin("admin", "pass"); - using var sena3 = LcmTestHelper.CloneFromLexbox("sena-3"); + using var sena3 = CloneFromLexbox("sena-3"); var entries = LcmTestHelper.GetEntries(sena3); Console.WriteLine($"Project has {entries.Count()} entries"); var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); var citationForm = entry.CitationForm.BestVernacularAlternative.Text; Assert.That(citationForm, Is.EqualTo("ambuka")); LcmTestHelper.SetVernacularText(sena3, entry.CitationForm, "something"); - LcmTestHelper.CommitChanges(sena3, "sena-3"); + CommitChanges(sena3, "sena-3"); - using var sena4 = LcmTestHelper.CloneFromLexbox("sena-3", "sena-4"); + using var sena4 = CloneFromLexbox("sena-3", "sena-4"); entry = LcmTestHelper.GetEntry(sena4, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); citationForm = entry.CitationForm.BestVernacularAlternative.Text; Assert.That(citationForm, Is.EqualTo("something")); LcmTestHelper.UpdateVernacularText(sena4, entry.CitationForm, (s) => $"{s}XYZ"); citationForm = entry.CitationForm.BestVernacularAlternative.Text; Assert.That(citationForm, Is.EqualTo("somethingXYZ")); - LcmTestHelper.CommitChanges(sena4, "sena-3", "sena-4"); + CommitChanges(sena4, "sena-3", "sena-4"); } } } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index b6786ac2..d9fb3dcc 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -24,8 +24,6 @@ public static class LcmTestHelper public static string LexboxPort = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriPort) ?? (LexboxProtocol == "http" ? "80" : "443"); public static Uri LexboxUrl = new Uri($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); - public static string BaseDir = Path.Combine(Path.GetTempPath(), nameof(LcmTestHelper)); - public static HttpClientHandler Handler { get; set; } = new HttpClientHandler(); public static CookieContainer Cookies => Handler.CookieContainer; public static HttpClient Http { get; set; } = new HttpClient(Handler); @@ -38,29 +36,29 @@ public static async Task LexboxLogin(string username, string password) return cookies[".LexBoxAuth"].Value; } - public static FwProject CloneFromLexbox(string code, string? newCode = null) + public static FwProject CloneFromLexbox(string code, string baseDir, string? newCode = null) { var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; newCode ??= code; - var dest = Path.Combine(BaseDir, "webwork", newCode); + var dest = Path.Combine(baseDir, "webwork", newCode); MercurialTestHelper.CloneRepo(withAuth.Uri.AbsoluteUri, dest); var fwdataPath = Path.Join(dest, $"{newCode}.fwdata"); var progress = new NullProgress(); MercurialTestHelper.ChangeBranch(dest, "tip"); LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(progress, false, fwdataPath); - var settings = new LfMergeSettingsDouble(BaseDir); + var settings = new LfMergeSettingsDouble(baseDir); return new FwProject(settings, newCode); } - public static void CommitChanges(FwProject project, string code, string? localCode = null, string? commitMsg = null) + public static void CommitChanges(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) { localCode ??= code; var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; if (!project.IsDisposed) project.Dispose(); commitMsg ??= "Auto-commit"; - var projectDir = Path.Combine(BaseDir, "webwork", localCode); + var projectDir = Path.Combine(baseDir, "webwork", localCode); var fwdataPath = Path.Join(projectDir, $"{localCode}.fwdata"); LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(NullProgress, false, fwdataPath); MercurialTestHelper.HgCommit(projectDir, commitMsg); diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index b04ed5b2..d3770db4 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Threading.Tasks; +using LfMerge.Core.FieldWorks; using NUnit.Framework; using NUnit.Framework.Interfaces; using SIL.TestUtilities; @@ -16,7 +17,7 @@ public class SRTestBase public TemporaryFolder TempFolderForClass { get; set; } public TemporaryFolder TempFolderForTest { get; set; } private string TipRevToRestore { get; set; } = ""; - private SRTestEnvironment TestEnv { get; set; } + public SRTestEnvironment TestEnv { get; set; } public SRTestBase() { @@ -89,5 +90,15 @@ public async Task RestoreRemoteProject() await TestEnv.RollbackProjectToRev(code, TipRevToRestore); } } + + public FwProject CloneFromLexbox(string code, string? newCode = null) + { + return LcmTestHelper.CloneFromLexbox(code, TempFolderForTest.Path, newCode); + } + + public void CommitChanges(FwProject project, string code, string? localCode = null, string? commitMsg = null) + { + LcmTestHelper.CommitChanges(project, code, TempFolderForTest.Path, localCode, commitMsg); + } } } From 37fb990ede61cfcd1860cecbf961915fe4acce09 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 8 Aug 2024 13:16:14 +0700 Subject: [PATCH 20/73] Fix the CommitChanges helper function Turns out the .fwdata file isn't updated until you call .Commit() on the action handler. --- src/LfMerge.Core.Tests/LcmTestHelper.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index d9fb3dcc..c804216a 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -56,6 +56,7 @@ public static void CommitChanges(FwProject project, string code, string baseDir, localCode ??= code; var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; + project.Cache.ActionHandlerAccessor.Commit(); if (!project.IsDisposed) project.Dispose(); commitMsg ??= "Auto-commit"; var projectDir = Path.Combine(baseDir, "webwork", localCode); From 700955593c103b44ab8d5d8428fb9fe13b42db7b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 8 Aug 2024 15:09:35 +0700 Subject: [PATCH 21/73] Fix Mercurial commits committing too much Also renamed CommitChanges to CommitAndPush. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 5 ++--- src/LfMerge.Core.Tests/LcmTestHelper.cs | 3 ++- src/LfMerge.Core.Tests/MercurialTestHelper.cs | 5 +++++ src/LfMerge.Core.Tests/SRTestBase.cs | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index e22f50dc..b93017de 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -24,12 +24,11 @@ public async Task CheckProjectCloning() await LcmTestHelper.LexboxLogin("admin", "pass"); using var sena3 = CloneFromLexbox("sena-3"); var entries = LcmTestHelper.GetEntries(sena3); - Console.WriteLine($"Project has {entries.Count()} entries"); var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); var citationForm = entry.CitationForm.BestVernacularAlternative.Text; Assert.That(citationForm, Is.EqualTo("ambuka")); LcmTestHelper.SetVernacularText(sena3, entry.CitationForm, "something"); - CommitChanges(sena3, "sena-3"); + CommitAndPush(sena3, "sena-3"); using var sena4 = CloneFromLexbox("sena-3", "sena-4"); entry = LcmTestHelper.GetEntry(sena4, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); @@ -38,7 +37,7 @@ public async Task CheckProjectCloning() LcmTestHelper.UpdateVernacularText(sena4, entry.CitationForm, (s) => $"{s}XYZ"); citationForm = entry.CitationForm.BestVernacularAlternative.Text; Assert.That(citationForm, Is.EqualTo("somethingXYZ")); - CommitChanges(sena4, "sena-3", "sena-4"); + CommitAndPush(sena4, "sena-3", "sena-4"); } } } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index c804216a..59046eba 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -51,7 +51,7 @@ public static FwProject CloneFromLexbox(string code, string baseDir, string? new return new FwProject(settings, newCode); } - public static void CommitChanges(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) + public static void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) { localCode ??= code; var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); @@ -62,6 +62,7 @@ public static void CommitChanges(FwProject project, string code, string baseDir, var projectDir = Path.Combine(baseDir, "webwork", localCode); var fwdataPath = Path.Join(projectDir, $"{localCode}.fwdata"); LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(NullProgress, false, fwdataPath); + MercurialTestHelper.HgClean(projectDir); // Ensure ConfigurationSettings, etc., don't get committed MercurialTestHelper.HgCommit(projectDir, commitMsg); MercurialTestHelper.HgPush(projectDir, withAuth.Uri.AbsoluteUri); } diff --git a/src/LfMerge.Core.Tests/MercurialTestHelper.cs b/src/LfMerge.Core.Tests/MercurialTestHelper.cs index 4ebfde07..61d03dcc 100644 --- a/src/LfMerge.Core.Tests/MercurialTestHelper.cs +++ b/src/LfMerge.Core.Tests/MercurialTestHelper.cs @@ -53,6 +53,11 @@ public static void HgAddFile(string repoPath, string file) RunHgCommand(repoPath, $"add {file}"); } + public static void HgClean(string repoPath) + { + RunHgCommand(repoPath, $"purge --no-confirm"); + } + public static void HgCommit(string repoPath, string message) { RunHgCommand(repoPath, $"commit -A -u dummyUser -m \"{message}\""); diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index d3770db4..d7a324ee 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -96,9 +96,9 @@ public FwProject CloneFromLexbox(string code, string? newCode = null) return LcmTestHelper.CloneFromLexbox(code, TempFolderForTest.Path, newCode); } - public void CommitChanges(FwProject project, string code, string? localCode = null, string? commitMsg = null) + public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null) { - LcmTestHelper.CommitChanges(project, code, TempFolderForTest.Path, localCode, commitMsg); + LcmTestHelper.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); } } } From 1bb8789595c8fa7e715fa0349ff3de239496de56 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 9 Aug 2024 09:55:13 +0700 Subject: [PATCH 22/73] Make CallLfMergeBridge public so tests can use it Also make it static so that tests can supply their own dependencies without needing to create an instance of ConvertMongoToLcmComments or ConvertLcmToMongoComments in order to use the method. --- .../ConvertLcmToMongoComments.cs | 21 +++++++++++++--- .../ConvertMongoToLcmComments.cs | 25 ++++++++++++++----- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoComments.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoComments.cs index b8e35015..eca44e4a 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoComments.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoComments.cs @@ -288,25 +288,38 @@ public ILexEntry OwningEntry(Guid guidOfUnknownLcmObject) } private GetChorusNotesResponse CallLfMergeBridge(List comments, out string bridgeOutput) + { + return CallLfMergeBridge(comments, out bridgeOutput, _project.FwDataPath, _progress, _logger); + } + + public static GetChorusNotesResponse CallLfMergeBridge(List comments, out string bridgeOutput, string fwDataPath, IProgress progress, ILogger logger) { // Call into LF Bridge to do the work. bridgeOutput = string.Empty; var bridgeInput = new LfMergeBridge.GetChorusNotesInput { LfComments = comments }; var options = new Dictionary { - {"-p", _project.FwDataPath}, + {"-p", fwDataPath}, }; LfMergeBridge.LfMergeBridge.ExtraInputData.Add(options, bridgeInput); - if (!LfMergeBridge.LfMergeBridge.Execute("Language_Forge_Get_Chorus_Notes", _progress, + if (!LfMergeBridge.LfMergeBridge.Execute("Language_Forge_Get_Chorus_Notes", progress, options, out bridgeOutput)) { - _logger.Error("Got an error from Language_Forge_Get_Chorus_Notes: {0}", bridgeOutput); + logger.Error("Got an error from Language_Forge_Get_Chorus_Notes: {0}", bridgeOutput); return null; } else { var success = LfMergeBridge.LfMergeBridge.ExtraOutputData.TryGetValue(options, out var outputObject); - return outputObject as GetChorusNotesResponse; + if (success) + { + return outputObject as GetChorusNotesResponse; + } + else + { + logger.Error("Language_Forge_Get_Chorus_Notes failed to return any data. Its output was: {0}", bridgeOutput); + return null; + } } } } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmComments.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmComments.cs index 8f70fb4e..47d6f89a 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmComments.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmComments.cs @@ -102,31 +102,44 @@ private LfLexEntry GetLexEntry(ObjectId idOfEntry) } private WriteToChorusNotesResponse CallLfMergeBridge(List> lfComments, out string bridgeOutput) + { + return CallLfMergeBridge(lfComments, out bridgeOutput, _project.FwDataPath, _progress, _logger); + } + + public static WriteToChorusNotesResponse CallLfMergeBridge(List> lfComments, out string bridgeOutput, string fwDataPath, IProgress progress, ILogger logger) { bridgeOutput = string.Empty; var options = new Dictionary { - {"-p", _project.FwDataPath}, + {"-p", fwDataPath}, }; try { var bridgeInput = new WriteToChorusNotesInput { LfComments = lfComments }; LfMergeBridge.LfMergeBridge.ExtraInputData.Add(options, bridgeInput); - if (!LfMergeBridge.LfMergeBridge.Execute("Language_Forge_Write_To_Chorus_Notes", _progress, + if (!LfMergeBridge.LfMergeBridge.Execute("Language_Forge_Write_To_Chorus_Notes", progress, options, out bridgeOutput)) { - _logger.Error("Got an error from Language_Forge_Write_To_Chorus_Notes: {0}", bridgeOutput); + logger.Error("Got an error from Language_Forge_Write_To_Chorus_Notes: {0}", bridgeOutput); return null; } else { var success = LfMergeBridge.LfMergeBridge.ExtraOutputData.TryGetValue(options, out var outputObject); - return outputObject as WriteToChorusNotesResponse; + if (success) + { + return outputObject as WriteToChorusNotesResponse; + } + else + { + logger.Error("Language_Forge_Write_To_Chorus_Notes failed to return any data. Its output was: {0}", bridgeOutput); + return null; + } } } catch (NullReferenceException) { - _logger.Debug("Got an exception. Before rethrowing it, here is what LfMergeBridge sent:"); - _logger.Debug("{0}", bridgeOutput); + logger.Debug("Got an exception. Before rethrowing it, here is what LfMergeBridge sent:"); + logger.Debug("{0}", bridgeOutput); throw; } } From c9e68541ba51b73da9780575d1e66a5fd754aaff Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 9 Aug 2024 11:31:27 +0700 Subject: [PATCH 23/73] Add basic (rudimentary, even) test for comments This causes a .ChorusNotes file to show up in the sena-3 project in LexBox, thereby proving that it can be done. Later we'll turn this into a real round-trip test of some kind. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 24 +++++++++++++++++++++++- src/LfMerge.Core.Tests/SRTestBase.cs | 3 +++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index b93017de..448dfde9 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -1,7 +1,10 @@ using System; -using System.Linq; +using System.Collections.Generic; using System.Threading.Tasks; +using LfMerge.Core.DataConverters; +using LfMergeBridge.LfMergeModel; using NUnit.Framework; +using SIL.Progress; namespace LfMerge.Core.Tests.E2E { @@ -39,5 +42,24 @@ public async Task CheckProjectCloning() Assert.That(citationForm, Is.EqualTo("somethingXYZ")); CommitAndPush(sena4, "sena-3", "sena-4"); } + + [Test] + [Property("projectCode", "sena-3")] + public async Task SendReceiveComments() + { + await LcmTestHelper.LexboxLogin("admin", "pass"); + using var sena3 = CloneFromLexbox("sena-3"); + var entries = LcmTestHelper.GetEntries(sena3); + var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); + var comment = new LfComment { + Guid = new Guid("6864b40d-6ad8-4c42-9590-114c0b8495c8"), + Content = "Comment for test", + // Let's see if that's enough + }; + var comments = new List> { new KeyValuePair("6864b40d-6ad8-4c42-9590-114c0b8495c8", comment) }; + var result = ConvertMongoToLcmComments.CallLfMergeBridge(comments, out var lfmergeBridgeOutput, FwDataPathForProject("sena-3"), new NullProgress(), TestEnv.Logger); + Console.WriteLine(lfmergeBridgeOutput); + CommitAndPush(sena3, "sena-3"); + } } } diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index d7a324ee..22869809 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -91,6 +91,9 @@ public async Task RestoreRemoteProject() } } + public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode); + public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata"); + public FwProject CloneFromLexbox(string code, string? newCode = null) { return LcmTestHelper.CloneFromLexbox(code, TempFolderForTest.Path, newCode); From fa873e27a4cac22c278eccc6741edb168e5c8ba5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 10:02:07 +0700 Subject: [PATCH 24/73] Refactor, moving HTTP stuff out of LcmTestHelper LcmTestHelper was the wrong place to put CommitAndPush since that needs to deal with URLs, and URLs are already being dealt with in SRTestEnv. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 2 - src/LfMerge.Core.Tests/LcmTestHelper.cs | 52 --------------------- src/LfMerge.Core.Tests/SRTestBase.cs | 16 +++++-- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 18 +++++++ 4 files changed, 31 insertions(+), 57 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index 448dfde9..fa976b3a 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -24,7 +24,6 @@ public async Task CheckProjectBackupDownloading() [Property("projectCode", "sena-3")] // This will cause it to auto-reset the project afterwards public async Task CheckProjectCloning() { - await LcmTestHelper.LexboxLogin("admin", "pass"); using var sena3 = CloneFromLexbox("sena-3"); var entries = LcmTestHelper.GetEntries(sena3); var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); @@ -47,7 +46,6 @@ public async Task CheckProjectCloning() [Property("projectCode", "sena-3")] public async Task SendReceiveComments() { - await LcmTestHelper.LexboxLogin("admin", "pass"); using var sena3 = CloneFromLexbox("sena-3"); var entries = LcmTestHelper.GetEntries(sena3); var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index 59046eba..a2564d28 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -15,58 +15,6 @@ namespace LfMerge.Core.Tests { public static class LcmTestHelper { - public static string HgCommand => - Path.Combine(TestEnvironment.FindGitRepoRoot(), "Mercurial", - Platform.IsWindows ? "hg.exe" : "hg"); - - public static string LexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; - public static string LexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; - public static string LexboxPort = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriPort) ?? (LexboxProtocol == "http" ? "80" : "443"); - public static Uri LexboxUrl = new Uri($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); - - public static HttpClientHandler Handler { get; set; } = new HttpClientHandler(); - public static CookieContainer Cookies => Handler.CookieContainer; - public static HttpClient Http { get; set; } = new HttpClient(Handler); - public static IProgress NullProgress = new NullProgress(); - - public static async Task LexboxLogin(string username, string password) - { - await Http.PostAsJsonAsync(new Uri(LexboxUrl, "/api/login"), new { EmailOrUsername=username, Password=password }); - var cookies = Cookies.GetCookies(LexboxUrl); - return cookies[".LexBoxAuth"].Value; - } - - public static FwProject CloneFromLexbox(string code, string baseDir, string? newCode = null) - { - var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); - var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; - newCode ??= code; - var dest = Path.Combine(baseDir, "webwork", newCode); - MercurialTestHelper.CloneRepo(withAuth.Uri.AbsoluteUri, dest); - var fwdataPath = Path.Join(dest, $"{newCode}.fwdata"); - var progress = new NullProgress(); - MercurialTestHelper.ChangeBranch(dest, "tip"); - LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(progress, false, fwdataPath); - var settings = new LfMergeSettingsDouble(baseDir); - return new FwProject(settings, newCode); - } - - public static void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) - { - localCode ??= code; - var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); - var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; - project.Cache.ActionHandlerAccessor.Commit(); - if (!project.IsDisposed) project.Dispose(); - commitMsg ??= "Auto-commit"; - var projectDir = Path.Combine(baseDir, "webwork", localCode); - var fwdataPath = Path.Join(projectDir, $"{localCode}.fwdata"); - LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(NullProgress, false, fwdataPath); - MercurialTestHelper.HgClean(projectDir); // Ensure ConfigurationSettings, etc., don't get committed - MercurialTestHelper.HgCommit(projectDir, commitMsg); - MercurialTestHelper.HgPush(projectDir, withAuth.Uri.AbsoluteUri); - } - public static IEnumerable GetEntries(FwProject project) { return project?.ServiceLocator?.LanguageProject?.LexDbOA?.Entries ?? []; diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 22869809..100aed5a 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -29,7 +29,7 @@ public SRTestBase() [OneTimeSetUp] public async Task FixtureSetup() { - // Log in to LexBox as admin + // Log in to LexBox as admin so we get a login cookie var lexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; var lexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; var lexboxPort = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriPort) ?? "80"; @@ -96,12 +96,22 @@ public async Task RestoreRemoteProject() public FwProject CloneFromLexbox(string code, string? newCode = null) { - return LcmTestHelper.CloneFromLexbox(code, TempFolderForTest.Path, newCode); + var projUrl = new Uri(TestEnv.LexboxUrl, $"/hg/{code}"); + var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; // TODO: extract this bit to its own method returning a project URL with auth + newCode ??= code; + var dest = Path.Combine(TempFolderForTest.Path, "webwork", newCode); + MercurialTestHelper.CloneRepo(withAuth.Uri.AbsoluteUri, dest); + var fwdataPath = Path.Join(dest, $"{newCode}.fwdata"); + MercurialTestHelper.ChangeBranch(dest, "tip"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); + var settings = new LfMergeSettingsDouble(TempFolderForTest.Path); + return new FwProject(settings, newCode); + } public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null) { - LcmTestHelper.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); + TestEnv.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); } } } diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 14adb4d8..dcd478a3 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -5,6 +5,7 @@ using System.Net.Http; using System.Net.Http.Json; using System.Threading.Tasks; +using LfMerge.Core.FieldWorks; using LfMerge.Core.Logging; using LfMerge.Core.Settings; using NUnit.Framework; @@ -26,6 +27,7 @@ public class SRTestEnvironment private HttpClient Http { get; init; } private HttpClientHandler Handler { get; init; } = new HttpClientHandler(); private CookieContainer Cookies { get; init; } = new CookieContainer(); + public static SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); private string Jwt { get; set; } public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProtocol = "http", string lexboxPort = "80", string lexboxUsername = "admin", string lexboxPassword = "pass") @@ -139,6 +141,22 @@ public async Task GetTipRev(string code) return result.Node; } + public void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) + { + localCode ??= code; + var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); + var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; + project.Cache.ActionHandlerAccessor.Commit(); + if (!project.IsDisposed) project.Dispose(); + commitMsg ??= "Auto-commit"; + var projectDir = Path.Combine(baseDir, "webwork", localCode); + var fwdataPath = Path.Join(projectDir, $"{localCode}.fwdata"); + LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(NullProgress, false, fwdataPath); + MercurialTestHelper.HgClean(projectDir); // Ensure ConfigurationSettings, etc., don't get committed + MercurialTestHelper.HgCommit(projectDir, commitMsg); + MercurialTestHelper.HgPush(projectDir, withAuth.Uri.AbsoluteUri); + } + private string TestName { get From 5055ee2c7bea273caef8f1cb0345e7d13d88bce2 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 10:25:31 +0700 Subject: [PATCH 25/73] Add more-efficient entry count method --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 8 ++++++-- src/LfMerge.Core.Tests/LcmTestHelper.cs | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index fa976b3a..c8c0f303 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -25,8 +25,10 @@ public async Task CheckProjectBackupDownloading() public async Task CheckProjectCloning() { using var sena3 = CloneFromLexbox("sena-3"); - var entries = LcmTestHelper.GetEntries(sena3); + var entryCount = LcmTestHelper.CountEntries(sena3); + Assert.That(entryCount, Is.EqualTo(1462)); var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); + Assert.That(entry, Is.Not.Null); var citationForm = entry.CitationForm.BestVernacularAlternative.Text; Assert.That(citationForm, Is.EqualTo("ambuka")); LcmTestHelper.SetVernacularText(sena3, entry.CitationForm, "something"); @@ -47,8 +49,10 @@ public async Task CheckProjectCloning() public async Task SendReceiveComments() { using var sena3 = CloneFromLexbox("sena-3"); - var entries = LcmTestHelper.GetEntries(sena3); + var entryCount = LcmTestHelper.CountEntries(sena3); + Assert.That(entryCount, Is.EqualTo(1462)); var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); + Assert.That(entry, Is.Not.Null); var comment = new LfComment { Guid = new Guid("6864b40d-6ad8-4c42-9590-114c0b8495c8"), Content = "Comment for test", diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index a2564d28..82f7e0f0 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -20,6 +20,12 @@ public static IEnumerable GetEntries(FwProject project) return project?.ServiceLocator?.LanguageProject?.LexDbOA?.Entries ?? []; } + public static int CountEntries(FwProject project) + { + var repo = project?.ServiceLocator?.GetInstance(); + return repo.Count; + } + public static ILexEntry GetEntry(FwProject project, Guid guid) { var repo = project?.ServiceLocator?.GetInstance(); From 78bc1f6a455b3d7b65c39783f24c5ac18edfcf91 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 10:37:52 +0700 Subject: [PATCH 26/73] Refactor SRTestEnv to keep track of URL parts This will enable other code to just ask the test environment what the lexbox username and password are, and not have to look it up from the environment variables all the time. --- src/LfMerge.Core.Tests/SRTestBase.cs | 9 ++---- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 35 ++++++++++++++------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 100aed5a..734fc850 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -30,12 +30,7 @@ public SRTestBase() public async Task FixtureSetup() { // Log in to LexBox as admin so we get a login cookie - var lexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; - var lexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; - var lexboxPort = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriPort) ?? "80"; - var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername) ?? "admin"; - var lexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; - TestEnv = new SRTestEnvironment(lexboxHostname, lexboxProtocol, lexboxPort, lexboxUsername, lexboxPassword); + TestEnv = new SRTestEnvironment(); await TestEnv.Login(); // Ensure we don't delete top-level /tmp/LfMergeSRTests folder if it already exists @@ -97,7 +92,7 @@ public async Task RestoreRemoteProject() public FwProject CloneFromLexbox(string code, string? newCode = null) { var projUrl = new Uri(TestEnv.LexboxUrl, $"/hg/{code}"); - var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; // TODO: extract this bit to its own method returning a project URL with auth + var withAuth = new UriBuilder(projUrl) { UserName = TestEnv.LexboxUsername, Password = TestEnv.LexboxPassword }; // TODO: extract this bit to its own method returning a project URL with auth newCode ??= code; var dest = Path.Combine(TempFolderForTest.Path, "webwork", newCode); MercurialTestHelper.CloneRepo(withAuth.Uri.AbsoluteUri, dest); diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index dcd478a3..1c2e05e7 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -23,6 +23,16 @@ public class SRTestEnvironment public ILogger Logger => MainClass.Logger; public Uri LexboxUrl { get; init; } public Uri LexboxUrlBasicAuth { get; init; } + private string? _lexboxHostname; + public string LexboxHostname => _lexboxHostname ?? Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; + private string? _lexboxProtocol; + public string LexboxProtocol => _lexboxProtocol ?? Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; + private string? _lexboxPort; + public string LexboxPort => _lexboxPort ?? (LexboxProtocol == "http" ? "80" : "443"); + private string? _lexboxUsername; + public string LexboxUsername => _lexboxUsername ?? Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername) ?? "admin"; + private string? _lexboxPassword; + public string LexboxPassword => _lexboxPassword ?? Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; private TemporaryFolder TempFolder { get; init; } private HttpClient Http { get; init; } private HttpClientHandler Handler { get; init; } = new HttpClientHandler(); @@ -30,15 +40,20 @@ public class SRTestEnvironment public static SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); private string Jwt { get; set; } - public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProtocol = "http", string lexboxPort = "80", string lexboxUsername = "admin", string lexboxPassword = "pass") + public SRTestEnvironment(string? lexboxHostname = null, string? lexboxProtocol = null, string? lexboxPort = null, string? lexboxUsername = null, string? lexboxPassword = null) { - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname, lexboxHostname); - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPrivateHostname, lexboxHostname); - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol, lexboxProtocol); - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_HgUsername, lexboxUsername); - Environment.SetEnvironmentVariable(MagicStrings.EnvVar_TrustToken, lexboxPassword); - LexboxUrl = new Uri($"{lexboxProtocol}://{lexboxHostname}:{lexboxPort}"); - LexboxUrlBasicAuth = new Uri($"{lexboxProtocol}://{WebUtility.UrlEncode(lexboxUsername)}:{WebUtility.UrlEncode(lexboxPassword)}@{lexboxHostname}:{lexboxPort}"); + _lexboxHostname = lexboxHostname; + _lexboxProtocol = lexboxProtocol; + _lexboxPort = lexboxPort; + _lexboxUsername = lexboxUsername; + _lexboxPassword = lexboxPassword; + if (lexboxHostname is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname, lexboxHostname); + if (lexboxHostname is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPrivateHostname, lexboxHostname); + if (lexboxProtocol is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol, lexboxProtocol); + if (lexboxUsername is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_HgUsername, lexboxUsername); + if (lexboxPassword is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_TrustToken, lexboxPassword); + LexboxUrl = new Uri($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); + LexboxUrlBasicAuth = new Uri($"{LexboxProtocol}://{WebUtility.UrlEncode(LexboxUsername)}:{WebUtility.UrlEncode(LexboxPassword)}@{LexboxHostname}:{LexboxPort}"); TempFolder = new TemporaryFolder(TestName + Path.GetRandomFileName()); Handler.CookieContainer = Cookies; Http = new HttpClient(Handler); @@ -46,9 +61,7 @@ public SRTestEnvironment(string lexboxHostname = "localhost", string lexboxProto public Task Login() { - var lexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername); - var lexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken); - return LoginAs(lexboxUsername, lexboxPassword); + return LoginAs(LexboxUsername, LexboxPassword); } public async Task LoginAs(string lexboxUsername, string lexboxPassword) From 6c857ab530828fada922569077a930fdd35c66bf Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 10:41:51 +0700 Subject: [PATCH 27/73] Add SRTestEnv method for project URLs with auth --- src/LfMerge.Core.Tests/SRTestBase.cs | 5 ++--- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 734fc850..4f49de8a 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -91,11 +91,10 @@ public async Task RestoreRemoteProject() public FwProject CloneFromLexbox(string code, string? newCode = null) { - var projUrl = new Uri(TestEnv.LexboxUrl, $"/hg/{code}"); - var withAuth = new UriBuilder(projUrl) { UserName = TestEnv.LexboxUsername, Password = TestEnv.LexboxPassword }; // TODO: extract this bit to its own method returning a project URL with auth + var projUrl = TestEnv.LexboxUrlForProjectWithAuth(code); newCode ??= code; var dest = Path.Combine(TempFolderForTest.Path, "webwork", newCode); - MercurialTestHelper.CloneRepo(withAuth.Uri.AbsoluteUri, dest); + MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); var fwdataPath = Path.Join(dest, $"{newCode}.fwdata"); MercurialTestHelper.ChangeBranch(dest, "tip"); LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 1c2e05e7..3da01b62 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -73,9 +73,12 @@ public async Task LoginAs(string lexboxUsername, string lexboxPassword) // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. } + public Uri LexboxUrlForProject(string code) => new Uri(LexboxUrl, $"hg/{code}"); + public Uri LexboxUrlForProjectWithAuth(string code) => new Uri(LexboxUrlBasicAuth, $"hg/{code}"); + public void InitRepo(string code, string dest) { - var sourceUrl = new Uri(LexboxUrlBasicAuth, $"hg/{code}"); + var sourceUrl = LexboxUrlForProjectWithAuth(code); MercurialTestHelper.CloneRepo(sourceUrl.AbsoluteUri, dest); } From 213a7df50107b276012b6bc74207c415777154c6 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 10:58:06 +0700 Subject: [PATCH 28/73] Fix reset-project method failing on existing files If you run multiple tests that want to reset the same project, the second one was erroring out because the zip file already existed. We now use random names for the files and dirs in the reset-project method so that this cannot occur. --- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 3da01b62..46036ba2 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -134,11 +134,11 @@ public async Task RollbackProjectToRev(string code, string rev) var backupUrl = new Uri(LexboxUrl, $"api/project/backupProject/{code}"); var result = await Http.GetAsync(backupUrl); var zipStream = await result.Content.ReadAsStreamAsync(); - var projectDir = Path.Join(TempFolder.Path, code); + var projectDir = TempFolder.GetPathForNewTempFile(false); ZipFile.ExtractToDirectory(zipStream, projectDir); - var clonedDir = Path.Join(TempFolder.Path, $"{code}-{rev}"); + var clonedDir = TempFolder.GetPathForNewTempFile(false); MercurialTestHelper.CloneRepoAtRev(projectDir, clonedDir, rev); - var zipPath = Path.Join(TempFolder.Path, $"{code}-{rev}.zip"); + var zipPath = TempFolder.GetPathForNewTempFile(false); ZipFile.CreateFromDirectory(clonedDir, zipPath); await ResetAndUploadZip(code, zipPath); } From 7dfef7556e3fca9d96a1e8415dddb71390d07630 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 13:57:42 +0700 Subject: [PATCH 29/73] Add ability to make GraphQL queries to LexBox Plus a basic test demonstrating creating a new project, which will eventually move to the base class or the SRTestEnvironment class. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 15 +++++++ src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs | 44 +++++++++++++++++++ .../LfMerge.Core.Tests.csproj | 2 + src/LfMerge.Core.Tests/SRTestEnvironment.cs | 37 ++++++++++++++++ 4 files changed, 98 insertions(+) create mode 100644 src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index c8c0f303..40ccf74b 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -63,5 +63,20 @@ public async Task SendReceiveComments() Console.WriteLine(lfmergeBridgeOutput); CommitAndPush(sena3, "sena-3"); } + + [Test] + // Will eventually move this code into SRTestEnv as a helper method + // This is a testbed where I'm working out the necessary steps + public async Task UploadNewProject() + { + var randomGuid = Guid.NewGuid(); + var testCode = $"sr-{randomGuid}"; + var testPath = TestFolderForProject(testCode); + MercurialTestHelper.InitializeHgRepo(testPath); + MercurialTestHelper.CreateFlexRepo(testPath); + // Now create project in LexBox + var result = await TestEnv.CreateLexBoxProject(testCode); + Console.WriteLine(result); + } } } diff --git a/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs b/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs new file mode 100644 index 00000000..70ee2088 --- /dev/null +++ b/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs @@ -0,0 +1,44 @@ +using System; + +namespace LfMerge.Core.Tests.LexboxGraphQLTypes +{ + public enum ProjectType + { + Unknown = 0, + FLEx = 1, + // WeSay = 2, + // OneStoryEditor = 3, + // OurWord = 4, + // AdaptIt = 5, + } + public enum RetentionPolicy + { + Unknown = 0, + Verified = 1, + Test = 2, + Dev = 3, + Training = 4, + } + + public record CreateProjectInput( + Guid? Id, + string Name, + string Description, + string Code, + ProjectType Type, + RetentionPolicy RetentionPolicy, + bool IsConfidential, + Guid? ProjectManagerId, + Guid? OrgId + ); + + public enum CreateProjectResult + { + Created, + Requested + } + + public record CreateProjectResponse(Guid? Id, CreateProjectResult Result); + public record CreateProject(CreateProjectResponse CreateProjectResponse); + public record CreateProjectGqlResponse(CreateProject CreateProject); +} diff --git a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj index d5afef7a..f84dc02a 100644 --- a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj +++ b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj @@ -32,6 +32,8 @@ See full changelog at https://github.com/sillsdev/LfMerge/blob/develop/CHANGELOG + + diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 46036ba2..c6dd0fd7 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -5,6 +5,10 @@ using System.Net.Http; using System.Net.Http.Json; using System.Threading.Tasks; +using GraphQL; +using GraphQL.Client.Abstractions; +using GraphQL.Client.Http; +using GraphQL.Client.Serializer.SystemTextJson; using LfMerge.Core.FieldWorks; using LfMerge.Core.Logging; using LfMerge.Core.Settings; @@ -39,6 +43,7 @@ public class SRTestEnvironment private CookieContainer Cookies { get; init; } = new CookieContainer(); public static SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); private string Jwt { get; set; } + private GraphQLHttpClient GqlClient { get; init; } public SRTestEnvironment(string? lexboxHostname = null, string? lexboxProtocol = null, string? lexboxPort = null, string? lexboxUsername = null, string? lexboxPassword = null) { @@ -57,6 +62,8 @@ public SRTestEnvironment(string? lexboxHostname = null, string? lexboxProtocol = TempFolder = new TemporaryFolder(TestName + Path.GetRandomFileName()); Handler.CookieContainer = Cookies; Http = new HttpClient(Handler); + var lexboxGqlEndpoint = new Uri(LexboxUrl, "/api/graphql"); + GqlClient = new GraphQLHttpClient(lexboxGqlEndpoint, new SystemTextJsonSerializer(), Http); } public Task Login() @@ -76,6 +83,36 @@ public async Task LoginAs(string lexboxUsername, string lexboxPassword) public Uri LexboxUrlForProject(string code) => new Uri(LexboxUrl, $"hg/{code}"); public Uri LexboxUrlForProjectWithAuth(string code) => new Uri(LexboxUrlBasicAuth, $"hg/{code}"); + public async Task CreateLexBoxProject(string code, string? name = null, string? description = null, Guid? managerId = null, Guid? orgId = null) + { + name ??= code; + description ??= $"Auto-created project for test {TestName}"; + var mutation = """ + mutation createProject($input: CreateProjectInput!) { + createProject(input: $input) { + createProjectResponse { + id + result + } + errors { + ... on DbError { + code + } + } + } + } + """; + var projId = Guid.NewGuid(); + var input = new LexboxGraphQLTypes.CreateProjectInput(projId, name, description, code, LexboxGraphQLTypes.ProjectType.FLEx, LexboxGraphQLTypes.RetentionPolicy.Dev, false, managerId, orgId); + var request = new GraphQLRequest { + Query = mutation, + Variables = new { input }, + }; + var response = await GqlClient.SendMutationAsync(request); + Assert.That(response.Errors, Is.Null.Or.Empty); + return response.Data.CreateProject.CreateProjectResponse; + } + public void InitRepo(string code, string dest) { var sourceUrl = LexboxUrlForProjectWithAuth(code); From 3c90d12f52f1c3977855ccb7873b0f7a21deab7d Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 14:27:43 +0700 Subject: [PATCH 30/73] Add other project types even if we won't use them Even if we won't use other project types than FLEx, might as well keep the enum the same as what LexBox uses, so that if we get a project type response that we didn't expect, System.Text.Json will know how to deserialize it correctly. --- src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs b/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs index 70ee2088..17d8ed91 100644 --- a/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs +++ b/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs @@ -6,10 +6,10 @@ public enum ProjectType { Unknown = 0, FLEx = 1, - // WeSay = 2, - // OneStoryEditor = 3, - // OurWord = 4, - // AdaptIt = 5, + WeSay = 2, + OneStoryEditor = 3, + OurWord = 4, + AdaptIt = 5, } public enum RetentionPolicy { From 43c02e70465fe2844c48596c4e74edd7d972e8dd Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 15:04:20 +0700 Subject: [PATCH 31/73] Ensure sena-3.zip is available to all tests This gives us a known starting point for all end-to-end tests, without needing to check the .zip file into the LfMerge repo. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 6 ++++-- src/LfMerge.Core.Tests/SRTestBase.cs | 12 +++++++++++- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 8 ++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index 40ccf74b..73333df9 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -75,8 +75,10 @@ public async Task UploadNewProject() MercurialTestHelper.InitializeHgRepo(testPath); MercurialTestHelper.CreateFlexRepo(testPath); // Now create project in LexBox - var result = await TestEnv.CreateLexBoxProject(testCode); - Console.WriteLine(result); + var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); + Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); + Assert.That(result.Id, Is.EqualTo(randomGuid)); + await TestEnv.ResetAndUploadZip(testCode, Sena3ZipPath); } } } diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 4f49de8a..46ee33ed 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -16,6 +16,8 @@ public class SRTestBase public LfMerge.Core.Logging.ILogger Logger => MainClass.Logger; public TemporaryFolder TempFolderForClass { get; set; } public TemporaryFolder TempFolderForTest { get; set; } + public TemporaryFolder TestDataFolder { get; set; } + public string Sena3ZipPath { get; set; } private string TipRevToRestore { get; set; } = ""; public SRTestEnvironment TestEnv { get; set; } @@ -33,13 +35,21 @@ public async Task FixtureSetup() TestEnv = new SRTestEnvironment(); await TestEnv.Login(); - // Ensure we don't delete top-level /tmp/LfMergeSRTests folder if it already exists + // Ensure we don't delete top-level /tmp/LfMergeSRTests folder and data subfolder if they already exist var tempPath = Path.Combine(Path.GetTempPath(), "LfMergeSRTests"); var rootTempFolder = Directory.Exists(tempPath) ? TemporaryFolder.TrackExisting(tempPath) : new TemporaryFolder(tempPath); + var testDataPath = Path.Combine(tempPath, "data"); + TestDataFolder = Directory.Exists(testDataPath) ? TemporaryFolder.TrackExisting(testDataPath) : new TemporaryFolder(testDataPath); // But the folder for this specific test suite should be deleted if it already exists var derivedClassName = this.GetType().Name; TempFolderForClass = new TemporaryFolder(rootTempFolder, derivedClassName); + + // Ensure sena-3.zip is available to all tests as a starting point + Sena3ZipPath = Path.Combine(TestDataFolder.Path, "sena-3.zip"); + if (!File.Exists(Sena3ZipPath)) { + await TestEnv.DownloadProjectBackup("sena-3", Sena3ZipPath); + } } [OneTimeTearDown] diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index c6dd0fd7..72ee81a0 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -83,8 +83,9 @@ public async Task LoginAs(string lexboxUsername, string lexboxPassword) public Uri LexboxUrlForProject(string code) => new Uri(LexboxUrl, $"hg/{code}"); public Uri LexboxUrlForProjectWithAuth(string code) => new Uri(LexboxUrlBasicAuth, $"hg/{code}"); - public async Task CreateLexBoxProject(string code, string? name = null, string? description = null, Guid? managerId = null, Guid? orgId = null) + public async Task CreateLexBoxProject(string code, Guid? projId = null, string? name = null, string? description = null, Guid? managerId = null, Guid? orgId = null) { + projId ??= Guid.NewGuid(); name ??= code; description ??= $"Auto-created project for test {TestName}"; var mutation = """ @@ -102,7 +103,6 @@ ... on DbError { } } """; - var projId = Guid.NewGuid(); var input = new LexboxGraphQLTypes.CreateProjectInput(projId, name, description, code, LexboxGraphQLTypes.ProjectType.FLEx, LexboxGraphQLTypes.RetentionPolicy.Dev, false, managerId, orgId); var request = new GraphQLRequest { Query = mutation, @@ -149,12 +149,12 @@ public async Task UploadZip(string code, string zipPath) await client.UploadAsync(fileUrl, file); } - public async Task DownloadProjectBackup(string code) + public async Task DownloadProjectBackup(string code, string? destZipPath = null) { var backupUrl = new Uri(LexboxUrl, $"api/project/backupProject/{code}"); var result = await Http.GetAsync(backupUrl); var filename = result.Content.Headers.ContentDisposition?.FileName; - var savePath = Path.Join(TempFolder.Path, filename); + var savePath = destZipPath ?? Path.Join(TempFolder.Path, filename); using (var outStream = File.Create(savePath)) { await result.Content.CopyToAsync(outStream); From b0c2fe9cd8f4e642929c00608bb7fce64cb13d64 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 15:11:36 +0700 Subject: [PATCH 32/73] Move CreateProject into SRTestBase Now any test method can just create a new project with a single line. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 12 ++---------- src/LfMerge.Core.Tests/SRTestBase.cs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index 73333df9..9c3fdb0e 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -69,16 +69,8 @@ public async Task SendReceiveComments() // This is a testbed where I'm working out the necessary steps public async Task UploadNewProject() { - var randomGuid = Guid.NewGuid(); - var testCode = $"sr-{randomGuid}"; - var testPath = TestFolderForProject(testCode); - MercurialTestHelper.InitializeHgRepo(testPath); - MercurialTestHelper.CreateFlexRepo(testPath); - // Now create project in LexBox - var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); - Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); - Assert.That(result.Id, Is.EqualTo(randomGuid)); - await TestEnv.ResetAndUploadZip(testCode, Sena3ZipPath); + var testCode = await CreateNewProjectFromSena3(); + Console.WriteLine($"Created {testCode}"); } } } diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 46ee33ed..08a1a5ce 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -113,6 +113,24 @@ public FwProject CloneFromLexbox(string code, string? newCode = null) } + public async Task CreateNewProjectFromTemplate(string origZipPath) + { + var randomGuid = Guid.NewGuid(); + var testCode = $"sr-{randomGuid}"; + var testPath = TestFolderForProject(testCode); + MercurialTestHelper.InitializeHgRepo(testPath); + MercurialTestHelper.CreateFlexRepo(testPath); + // Now create project in LexBox + var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); + Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); + Assert.That(result.Id, Is.EqualTo(randomGuid)); + await TestEnv.ResetAndUploadZip(testCode, origZipPath); + // TODO: Add code in TearDown to delete this project if test successful, so we don't clutter local LexBox with leftovers from successful tests + return testCode; + } + + public Task CreateNewProjectFromSena3() => CreateNewProjectFromTemplate(Sena3ZipPath); + public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null) { TestEnv.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); From 99e738496c417f59dda53611e9ae81e1d7c56000 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 15:13:45 +0700 Subject: [PATCH 33/73] Move TestTearDown to be next to TestSetUp --- src/LfMerge.Core.Tests/SRTestBase.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 08a1a5ce..44d3013d 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -68,6 +68,14 @@ public async Task TestSetup() await BackupRemoteProject(); } + [TearDown] + public async Task TestTeardown() + { + await RestoreRemoteProject(); + // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation + if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) TempFolderForTest.Dispose(); + } + public async Task BackupRemoteProject() { var test = TestContext.CurrentContext.Test; @@ -79,14 +87,6 @@ public async Task BackupRemoteProject() } } - [TearDown] - public async Task TestTeardown() - { - await RestoreRemoteProject(); - // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation - if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) TempFolderForTest.Dispose(); - } - public async Task RestoreRemoteProject() { var test = TestContext.CurrentContext.Test; From 9bd0a0973034f1d1db523191060ef8de72e65608 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 15:29:58 +0700 Subject: [PATCH 34/73] Auto-delete created projects on test success Failed tests will still leave the project in LexBox so that it can be examined later. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 2 -- src/LfMerge.Core.Tests/SRTestBase.cs | 12 +++++++-- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 29 ++++++++++++++++++++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index 9c3fdb0e..3eac0968 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -65,8 +65,6 @@ public async Task SendReceiveComments() } [Test] - // Will eventually move this code into SRTestEnv as a helper method - // This is a testbed where I'm working out the necessary steps public async Task UploadNewProject() { var testCode = await CreateNewProjectFromSena3(); diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 44d3013d..0210a58f 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -19,6 +19,7 @@ public class SRTestBase public TemporaryFolder TestDataFolder { get; set; } public string Sena3ZipPath { get; set; } private string TipRevToRestore { get; set; } = ""; + private Guid? ProjectIdToDelete { get; set; } public SRTestEnvironment TestEnv { get; set; } public SRTestBase() @@ -73,7 +74,14 @@ public async Task TestTeardown() { await RestoreRemoteProject(); // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation - if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) TempFolderForTest.Dispose(); + if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) { + TempFolderForTest.Dispose(); + if (ProjectIdToDelete is not null) { + var projId = ProjectIdToDelete.Value; + ProjectIdToDelete = null; + await TestEnv.DeleteLexBoxProject(projId); + } + } } public async Task BackupRemoteProject() @@ -125,7 +133,7 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); Assert.That(result.Id, Is.EqualTo(randomGuid)); await TestEnv.ResetAndUploadZip(testCode, origZipPath); - // TODO: Add code in TearDown to delete this project if test successful, so we don't clutter local LexBox with leftovers from successful tests + ProjectIdToDelete = result.Id; return testCode; } diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 72ee81a0..d2e4bf27 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.IO.Compression; +using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Json; @@ -109,10 +110,36 @@ ... on DbError { Variables = new { input }, }; var response = await GqlClient.SendMutationAsync(request); - Assert.That(response.Errors, Is.Null.Or.Empty); + Assert.That(response.Errors, Is.Null.Or.Empty, () => string.Join("\n", response.Errors.Select(error => error.Message))); return response.Data.CreateProject.CreateProjectResponse; } + public async Task DeleteLexBoxProject(Guid projectId) + { + var mutation = """ + mutation SoftDeleteProject($input: SoftDeleteProjectInput!) { + softDeleteProject(input: $input) { + project { + id, + deletedDate + } + errors { + ... on Error { + message + } + } + } + } + """; + var input = new { projectId }; + var request = new GraphQLRequest { + Query = mutation, + Variables = new { input }, + }; + var response = await GqlClient.SendMutationAsync(request); + Assert.That(response.Errors, Is.Null.Or.Empty, () => string.Join("\n", response.Errors.Select(error => error.Message))); + } + public void InitRepo(string code, string dest) { var sourceUrl = LexboxUrlForProjectWithAuth(code); From bf52e496ddfae69fb846dbec870255176a7b2f83 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 15:51:59 +0700 Subject: [PATCH 35/73] Slight GraphQL improvement --- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index d2e4bf27..c3c97f36 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -120,8 +120,7 @@ public async Task DeleteLexBoxProject(Guid projectId) mutation SoftDeleteProject($input: SoftDeleteProjectInput!) { softDeleteProject(input: $input) { project { - id, - deletedDate + id } errors { ... on Error { From 238049a25025aceb74e78138927477fe4a2d5e08 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 13 Aug 2024 17:02:10 +0700 Subject: [PATCH 36/73] Attempt to debug teardown race condition Somehow, if the UploadNewProject test is run after the other BasicTests, Mercurial isn't finding the newly-created project when the UploadZip step runs. However, if UploadNewProject is run in isolation, then Mercurial finds it. This is apparently a race condition involving Mercurial's new project registration, which we can reproduce consistently here. To reproduce, run: dotnet test -l "console;verbosity=normal" --filter=Foo_ We should see: About to reset sr-f7f322b8-b6d8-47e9-bb2c-7dd30426a5a1 About to upload /tmp/LfMergeSRTests/data/sena-3.zip to sr-f7f322b8-b6d8-47e9-bb2c-7dd30426a5a1 Done with reset and upload for sr-f7f322b8-b6d8-47e9-bb2c-7dd30426a5a1 Created sr-f7f322b8-b6d8-47e9-bb2c-7dd30426a5a1 Teardown for Foo_UploadNewProject About to delete project ID f7f322b8-b6d8-47e9-bb2c-7dd30426a5a1 Successfully deleted project ID f7f322b8-b6d8-47e9-bb2c-7dd30426a5a1 Teardown for Foo_UploadNewProject completed But instead we see: About to reset sr-f7f322b8-b6d8-47e9-bb2c-7dd30426a5a1 About to upload /tmp/LfMergeSRTests/data/sena-3.zip to sr-f7f322b8-b6d8-47e9-bb2c-7dd30426a5a1 Teardown for Foo_UploadNewProject Teardown for Foo_UploadNewProject completed And the "Done with reset and upload" step is never reached, becaue the LexBox server returns a 404 on the project when we upload the zip file. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 4 ++-- src/LfMerge.Core.Tests/SRTestBase.cs | 5 +++++ src/LfMerge.Core.Tests/SRTestEnvironment.cs | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index 3eac0968..4f0d96aa 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -46,7 +46,7 @@ public async Task CheckProjectCloning() [Test] [Property("projectCode", "sena-3")] - public async Task SendReceiveComments() + public async Task Foo_SendReceiveComments() { using var sena3 = CloneFromLexbox("sena-3"); var entryCount = LcmTestHelper.CountEntries(sena3); @@ -65,7 +65,7 @@ public async Task SendReceiveComments() } [Test] - public async Task UploadNewProject() + public async Task Foo_UploadNewProject() { var testCode = await CreateNewProjectFromSena3(); Console.WriteLine($"Created {testCode}"); diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 0210a58f..f16c6c26 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -73,6 +73,7 @@ public async Task TestSetup() public async Task TestTeardown() { await RestoreRemoteProject(); + Console.WriteLine($"Teardown for {TestName}"); // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) { TempFolderForTest.Dispose(); @@ -82,6 +83,7 @@ public async Task TestTeardown() await TestEnv.DeleteLexBoxProject(projId); } } + Console.WriteLine($"Teardown for {TestName} completed"); } public async Task BackupRemoteProject() @@ -132,6 +134,9 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); Assert.That(result.Id, Is.EqualTo(randomGuid)); + // Uncomment the two lines below to fix the race condition + // Console.WriteLine("Sleeping..."); + // System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5)); // Wait for Mercurial to "catch up" with new project creation await TestEnv.ResetAndUploadZip(testCode, origZipPath); ProjectIdToDelete = result.Id; return testCode; diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index c3c97f36..2f76a101 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -111,11 +111,13 @@ ... on DbError { }; var response = await GqlClient.SendMutationAsync(request); Assert.That(response.Errors, Is.Null.Or.Empty, () => string.Join("\n", response.Errors.Select(error => error.Message))); + Console.WriteLine($"Created project {response.Data.CreateProject.CreateProjectResponse.Id}"); return response.Data.CreateProject.CreateProjectResponse; } public async Task DeleteLexBoxProject(Guid projectId) { + Console.WriteLine($"About to delete project ID {projectId}"); var mutation = """ mutation SoftDeleteProject($input: SoftDeleteProjectInput!) { softDeleteProject(input: $input) { @@ -137,6 +139,7 @@ ... on Error { }; var response = await GqlClient.SendMutationAsync(request); Assert.That(response.Errors, Is.Null.Or.Empty, () => string.Join("\n", response.Errors.Select(error => error.Message))); + Console.WriteLine($"Successfully deleted project ID {projectId}"); } public void InitRepo(string code, string dest) @@ -149,9 +152,12 @@ public void InitRepo(string code, string dest) public async Task ResetAndUploadZip(string code, string zipPath) { + Console.WriteLine($"About to reset {code}"); var resetUrl = new Uri(LexboxUrl, $"api/project/resetProject/{code}"); await Http.PostAsync(resetUrl, null); + Console.WriteLine($"About to upload {zipPath} to {code}"); await UploadZip(code, zipPath); + Console.WriteLine($"Done with reset and upload for {code}"); } public async Task ResetToEmpty(string code) From ef5f48006228aaac945fd29b7bb22a1aedc0f733 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 14 Aug 2024 10:06:31 +0700 Subject: [PATCH 37/73] Remove debug logging since race condition is fixed Race condition fixed in LexBox, so we don't need the Console.WriteLine calls any longer. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 4 ++-- src/LfMerge.Core.Tests/SRTestBase.cs | 5 ----- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 6 ------ 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs index 4f0d96aa..3eac0968 100644 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ b/src/LfMerge.Core.Tests/E2E/BasicTests.cs @@ -46,7 +46,7 @@ public async Task CheckProjectCloning() [Test] [Property("projectCode", "sena-3")] - public async Task Foo_SendReceiveComments() + public async Task SendReceiveComments() { using var sena3 = CloneFromLexbox("sena-3"); var entryCount = LcmTestHelper.CountEntries(sena3); @@ -65,7 +65,7 @@ public async Task Foo_SendReceiveComments() } [Test] - public async Task Foo_UploadNewProject() + public async Task UploadNewProject() { var testCode = await CreateNewProjectFromSena3(); Console.WriteLine($"Created {testCode}"); diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index f16c6c26..0210a58f 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -73,7 +73,6 @@ public async Task TestSetup() public async Task TestTeardown() { await RestoreRemoteProject(); - Console.WriteLine($"Teardown for {TestName}"); // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) { TempFolderForTest.Dispose(); @@ -83,7 +82,6 @@ public async Task TestTeardown() await TestEnv.DeleteLexBoxProject(projId); } } - Console.WriteLine($"Teardown for {TestName} completed"); } public async Task BackupRemoteProject() @@ -134,9 +132,6 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); Assert.That(result.Id, Is.EqualTo(randomGuid)); - // Uncomment the two lines below to fix the race condition - // Console.WriteLine("Sleeping..."); - // System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5)); // Wait for Mercurial to "catch up" with new project creation await TestEnv.ResetAndUploadZip(testCode, origZipPath); ProjectIdToDelete = result.Id; return testCode; diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 2f76a101..c3c97f36 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -111,13 +111,11 @@ ... on DbError { }; var response = await GqlClient.SendMutationAsync(request); Assert.That(response.Errors, Is.Null.Or.Empty, () => string.Join("\n", response.Errors.Select(error => error.Message))); - Console.WriteLine($"Created project {response.Data.CreateProject.CreateProjectResponse.Id}"); return response.Data.CreateProject.CreateProjectResponse; } public async Task DeleteLexBoxProject(Guid projectId) { - Console.WriteLine($"About to delete project ID {projectId}"); var mutation = """ mutation SoftDeleteProject($input: SoftDeleteProjectInput!) { softDeleteProject(input: $input) { @@ -139,7 +137,6 @@ ... on Error { }; var response = await GqlClient.SendMutationAsync(request); Assert.That(response.Errors, Is.Null.Or.Empty, () => string.Join("\n", response.Errors.Select(error => error.Message))); - Console.WriteLine($"Successfully deleted project ID {projectId}"); } public void InitRepo(string code, string dest) @@ -152,12 +149,9 @@ public void InitRepo(string code, string dest) public async Task ResetAndUploadZip(string code, string zipPath) { - Console.WriteLine($"About to reset {code}"); var resetUrl = new Uri(LexboxUrl, $"api/project/resetProject/{code}"); await Http.PostAsync(resetUrl, null); - Console.WriteLine($"About to upload {zipPath} to {code}"); await UploadZip(code, zipPath); - Console.WriteLine($"Done with reset and upload for {code}"); } public async Task ResetToEmpty(string code) From a18650e1f17c3f515c3c642c1fc0a924f8d4f32d Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 14 Aug 2024 10:52:33 +0700 Subject: [PATCH 38/73] Derive SRTestEnv from TestEnv class Tests continue to pass, so there are no hidden conflicts. --- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index c3c97f36..99b1d0dd 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -23,9 +23,8 @@ namespace LfMerge.Core.Tests /// /// Test environment for end-to-end testing, i.e. Send/Receive with a real LexBox instance /// - public class SRTestEnvironment + public class SRTestEnvironment : TestEnvironment { - public ILogger Logger => MainClass.Logger; public Uri LexboxUrl { get; init; } public Uri LexboxUrlBasicAuth { get; init; } private string? _lexboxHostname; From 221d6c3904aeccd9f0652c677fe522aed8d22920 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 14 Aug 2024 17:48:53 +0700 Subject: [PATCH 39/73] Set env var so FW will use temp folder for storage FieldWorks and liblcm use the FW_CommonAppData environment variable to locate the folder where they should store things like writing systems downloaded from SLDR. We want it set to a temp folder for E2E tests. --- src/LfMerge.Core.Tests/SRTestBase.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 0210a58f..9af2b4d7 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -17,6 +17,7 @@ public class SRTestBase public TemporaryFolder TempFolderForClass { get; set; } public TemporaryFolder TempFolderForTest { get; set; } public TemporaryFolder TestDataFolder { get; set; } + public TemporaryFolder LcmDataFolder { get; set; } public string Sena3ZipPath { get; set; } private string TipRevToRestore { get; set; } = ""; private Guid? ProjectIdToDelete { get; set; } @@ -41,6 +42,9 @@ public async Task FixtureSetup() var rootTempFolder = Directory.Exists(tempPath) ? TemporaryFolder.TrackExisting(tempPath) : new TemporaryFolder(tempPath); var testDataPath = Path.Combine(tempPath, "data"); TestDataFolder = Directory.Exists(testDataPath) ? TemporaryFolder.TrackExisting(testDataPath) : new TemporaryFolder(testDataPath); + var lcmDataPath = Path.Combine(tempPath, "lcm-common"); + LcmDataFolder = Directory.Exists(lcmDataPath) ? TemporaryFolder.TrackExisting(lcmDataPath) : new TemporaryFolder(lcmDataPath); + Environment.SetEnvironmentVariable("FW_CommonAppData", LcmDataFolder.Path); // But the folder for this specific test suite should be deleted if it already exists var derivedClassName = this.GetType().Name; @@ -56,6 +60,7 @@ public async Task FixtureSetup() [OneTimeTearDown] public void FixtureTeardown() { + Environment.SetEnvironmentVariable("FW_CommonAppData", null); var result = TestContext.CurrentContext.Result; var nonSuccess = result.FailCount + result.InconclusiveCount + result.WarningCount; // Only delete class temp folder if we passed or skipped all tests From 06ca00c972590a212e11496521bdabace0594dba Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 14 Aug 2024 17:58:49 +0700 Subject: [PATCH 40/73] Add beginnings of first "real" end-to-end test Currently pulls project down from LexBox and verifies data. Next step will be to change data in a FieldWorks project, push that to LexBox, and verify that LfMerge can resolve merge conflicts. --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 80 ++++++++++++ src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 127 +++++++++++++++++++ src/LfMerge.Core.Tests/SRTestBase.cs | 40 ++++-- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 11 +- src/LfMerge.Core.Tests/TestEnvironment.cs | 2 +- src/LfMerge.Core/LanguageForgeProject.cs | 4 +- 6 files changed, 247 insertions(+), 17 deletions(-) create mode 100644 src/LfMerge.Core.Tests/E2E/E2ETestBase.cs create mode 100644 src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs new file mode 100644 index 00000000..e7fcdbd9 --- /dev/null +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Autofac; +using LfMerge.Core.Actions; +using LfMerge.Core.DataConverters; +using LfMerge.Core.FieldWorks; +using LfMerge.Core.LanguageForge.Model; +using LfMerge.Core.MongoConnector; +using LfMerge.Core.Settings; +using LfMergeBridge.LfMergeModel; +using NUnit.Framework; +using SIL.LCModel; +using SIL.Progress; +using SIL.TestUtilities; + +namespace LfMerge.Core.Tests.E2E +{ + [TestFixture] + [Category("LongRunning")] + [Category("IntegrationTests")] + public class E2ETestBase : SRTestBase + { + public LfMergeSettings _lDSettings; + public TemporaryFolder _languageDepotFolder; + public LanguageForgeProject _lfProject; + public SynchronizeAction _synchronizeAction; + public MongoConnectionDouble _mongoConnection; + public MongoProjectRecordFactory _recordFactory; + public string _workDir; + public const string TestLangProj = "testlangproj"; + public const string TestLangProjModified = "testlangproj-modified"; + + public void LcmSendReceive(FwProject project, string code, string? localCode = null, string? commitMsg = null) + { + // LfMergeBridge.LfMergeBridge.Execute?? + } + + + [SetUp] + public void Setup() + { + MagicStrings.SetMinimalModelVersion(LcmCache.ModelVersion); + + // SyncActionTests used to create mock LD server -- not needed since we use real (local) LexBox here + + // _languageDepotFolder = new TemporaryFolder(TestContext.CurrentContext.Test.Name + Path.GetRandomFileName()); + // _lDSettings = new LfMergeSettingsDouble(_languageDepotFolder.Path); + // Directory.CreateDirectory(_lDSettings.WebWorkDirectory); + // LanguageDepotMock.ProjectFolderPath = + // Path.Combine(_lDSettings.WebWorkDirectory, TestLangProj); + // Directory.CreateDirectory(LanguageDepotMock.ProjectFolderPath); + // LanguageDepotMock.Server = new MercurialServer(LanguageDepotMock.ProjectFolderPath); + + // SyncActionTests used to create local LF project from the embedded test data -- here we will let each test do that + + // _lfProject = LanguageForgeProject.Create(TestLangProj); + + // TODO: get far enough to need to test this + _synchronizeAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger); + + // SyncActionTests used to set the current working directory so that Mercurial could be found; no longer needed now + + // _workDir = Directory.GetCurrentDirectory(); + // Directory.SetCurrentDirectory(ExecutionEnvironment.DirectoryOfExecutingAssembly); + + _mongoConnection = MainClass.Container.Resolve() as MongoConnectionDouble; + if (_mongoConnection == null) + throw new AssertionException("E2E tests need a mock MongoConnection that stores data in order to work."); + _recordFactory = MainClass.Container.Resolve() as MongoProjectRecordFactoryDouble; + if (_recordFactory == null) + throw new AssertionException("E2E tests need a mock MongoProjectRecordFactory in order to work."); + + } + + + } +} diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs new file mode 100644 index 00000000..ba376eed --- /dev/null +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using LfMerge.Core.DataConverters; +using LfMergeBridge.LfMergeModel; +using NUnit.Framework; +using SIL.Progress; + +using System.IO; +using System.Linq; +using LfMerge.Core.Actions; +using LfMerge.Core.LanguageForge.Model; +using SIL.LCModel; + +namespace LfMerge.Core.Tests.E2E +{ + [TestFixture] + [Category("LongRunning")] + [Category("IntegrationTests")] + public class TryOutE2ETests : E2ETestBase + { + + // TODO: Duplicate the test below using LexBox instead of a mock LanguageDepot server + + [Test] + public async Task E2E_LFDataChangedLDDataChanged_LFWins() + { + // Setup + + // Take testlangproj-modified and "upload" it to mock LD as proj code "testlangproj" + // TODO: Replace the above with uploading the modified FwProject to LexBox, or perhaps cloning the original, having FW modify it, then uploading the modified version + + var projCode = await CreateNewProjectFromSena3(); + Guid entryId = Guid.Parse("0006f482-a078-4cef-9c5a-8bd35b53cf72"); + + var projPath = CloneRepoFromLexbox(projCode); + Console.WriteLine("cloned"); + MercurialTestHelper.ChangeBranch(projPath, "tip"); + Console.WriteLine("on tip"); + var fwdataPath = Path.Combine(projPath, $"{projCode}.fwdata"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); + Console.WriteLine("reassembled"); + // TODO: Write a helper to simplify this process + + var lfProject = LanguageForgeProject.Create(projCode, TestEnv.Settings); + lfProject.IsInitialClone = true; + Console.WriteLine($"Will look for .fwdata file at {lfProject.FwDataPath}"); + + // Do an initial clone from mock LD to LF (in the form of the mock Mongo) + + var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, TestEnv.Logger, _mongoConnection, _recordFactory); + transferLcmToMongo.Run(lfProject); + + // Do some initial checks to make sure we got the right data + + var lcmObject = lfProject.FieldWorksProject?.ServiceLocator?.ObjectRepository?.GetObject(entryId); + Assert.That(lcmObject, Is.Not.Null); + var lcmEntry = lcmObject as ILexEntry; + Assert.That(lcmEntry, Is.Not.Null); + Assert.That(lcmEntry.CitationForm.BestVernacularAlternative.Text, Is.EqualTo("cibubu")); + + IEnumerable originalMongoData = _mongoConnection.GetLfLexEntries(); + LfLexEntry lfEntry = originalMongoData.First(e => e.Guid == entryId); + + Assert.That(lfEntry.CitationForm.BestString(["seh"]), Is.EqualTo("cibubu")); + + // TODO: Finish implementing the rest of the original test (below) from SynchronizeActionTests, then create helper methods for reusable parts + + // Capture original modified dates from the particular entry we're interested in + + // IEnumerable originalMongoData = _mongoConnection.GetLfLexEntries(); + // LfLexEntry lfEntry = originalMongoData.First(e => e.Guid == _testEntryGuid); + // DateTime originalLfDateModified = lfEntry.DateModified; + // DateTime originalLfAuthorInfoModifiedDate = lfEntry.AuthorInfo.ModifiedDate; + + // Modify the entry in LF, but not yet in LD + + // string unchangedGloss = lfEntry.Senses[0].Gloss["en"].Value; + // string fwChangedGloss = unchangedGloss + " - changed in FW"; + // string lfChangedGloss = unchangedGloss + " - changed in LF"; + // lfEntry.Senses[0].Gloss["en"].Value = lfChangedGloss; + // lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; + // _mongoConnection.UpdateRecord(_lfProject, lfEntry); + + // Verify that the LD project has the FW value for the gloss + + // _lDProject = new LanguageDepotMock(testProjectCode, _lDSettings); + // var lDcache = _lDProject.FieldWorksProject.Cache; + // var lDLcmEntry = lDcache.ServiceLocator.GetObject(_testEntryGuid) as ILexEntry; + // Assert.That(lDLcmEntry.SensesOS[0].Gloss.AnalysisDefaultWritingSystem.Text, Is.EqualTo(fwChangedGloss)); + + // Capture the original modified date from the entry in the FW/LCM project + + // DateTime originalLdDateModified = lDLcmEntry.DateModified; + + // Exercise + + // Capture date/time immediately before running, to make sure that SyncAction updates it + + // var sutSynchronize = new SynchronizeAction(_env.Settings, _env.Logger); + // var timeBeforeRun = DateTime.UtcNow; + + // Actually do the LfMerge Send/Receive with mock LD + + // sutSynchronize.Run(_lfProject); + + // Verify + + // Verify that the result of the conflict was LF winning + + // Assert.That(GetGlossFromMongoDb(_testEntryGuid), Is.EqualTo(lfChangedGloss)); + + // Verify that the modified dates got updated when LF won the merge conflict, even though LF's view of the data didn't change + + // LfLexEntry updatedLfEntry = _mongoConnection.GetLfLexEntries().First(e => e.Guid == _testEntryGuid); + // DateTime updatedLfDateModified = updatedLfEntry.DateModified; + // DateTime updatedLfAuthorInfoModifiedDate = updatedLfEntry.AuthorInfo.ModifiedDate; + // // LF had the same data previously; however it's a merge conflict so DateModified + // // got updated + // Assert.That(updatedLfDateModified, Is.GreaterThan(originalLfDateModified)); + // // But the LCM modified date (AuthorInfo.ModifiedDate in LF) should be updated. + // Assert.That(updatedLfAuthorInfoModifiedDate, Is.GreaterThan(originalLfAuthorInfoModifiedDate)); + // Assert.That(updatedLfDateModified, Is.GreaterThan(originalLdDateModified)); + // Assert.That(_mongoConnection.GetLastSyncedDate(_lfProject), Is.GreaterThanOrEqualTo(timeBeforeRun)); + } + } +} diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 9af2b4d7..b0b93560 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -33,10 +33,6 @@ public SRTestBase() [OneTimeSetUp] public async Task FixtureSetup() { - // Log in to LexBox as admin so we get a login cookie - TestEnv = new SRTestEnvironment(); - await TestEnv.Login(); - // Ensure we don't delete top-level /tmp/LfMergeSRTests folder and data subfolder if they already exist var tempPath = Path.Combine(Path.GetTempPath(), "LfMergeSRTests"); var rootTempFolder = Directory.Exists(tempPath) ? TemporaryFolder.TrackExisting(tempPath) : new TemporaryFolder(tempPath); @@ -50,6 +46,10 @@ public async Task FixtureSetup() var derivedClassName = this.GetType().Name; TempFolderForClass = new TemporaryFolder(rootTempFolder, derivedClassName); + // Log in to LexBox as admin so we get a login cookie + TestEnv = new SRTestEnvironment(TempFolderForClass); + await TestEnv.Login(); + // Ensure sena-3.zip is available to all tests as a starting point Sena3ZipPath = Path.Combine(TestDataFolder.Path, "sena-3.zip"); if (!File.Exists(Sena3ZipPath)) { @@ -109,24 +109,32 @@ public async Task RestoreRemoteProject() } } - public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode); + // TODO See if TempFolderForTest will work instead of TempFolderForClass... (need to refactor SRTestEnv to expect to be instantiated once per test in order to pull that off) + public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForClass.Path, "webwork", projectCode); public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata"); - public FwProject CloneFromLexbox(string code, string? newCode = null) + public string CloneRepoFromLexbox(string code, string? newCode = null) { var projUrl = TestEnv.LexboxUrlForProjectWithAuth(code); newCode ??= code; - var dest = Path.Combine(TempFolderForTest.Path, "webwork", newCode); + var dest = TestFolderForProject(newCode); MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); + return dest; + } + + public FwProject CloneFromLexbox(string code, string? newCode = null) + { + var dest = CloneRepoFromLexbox(code, newCode); + var dirInfo = new DirectoryInfo(dest); + if (!dirInfo.Exists) throw new InvalidOperationException($"Failed to clone {code} from lexbox, cannot create FwProject"); var fwdataPath = Path.Join(dest, $"{newCode}.fwdata"); MercurialTestHelper.ChangeBranch(dest, "tip"); LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); var settings = new LfMergeSettingsDouble(TempFolderForTest.Path); return new FwProject(settings, newCode); - } - public async Task CreateNewProjectFromTemplate(string origZipPath) + public async Task CreateNewFlexProject() { var randomGuid = Guid.NewGuid(); var testCode = $"sr-{randomGuid}"; @@ -137,6 +145,20 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); Assert.That(result.Id, Is.EqualTo(randomGuid)); + // TODO: Push that first commit to lexbox so the project is non-empty + ProjectIdToDelete = result.Id; + return testCode; + } + + public async Task CreateNewProjectFromTemplate(string origZipPath) + { + var randomGuid = Guid.NewGuid(); + var testCode = $"sr-{randomGuid}"; + var testPath = TestFolderForProject(testCode); + // Now create project in LexBox + var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); + Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); + Assert.That(result.Id, Is.EqualTo(randomGuid)); await TestEnv.ResetAndUploadZip(testCode, origZipPath); ProjectIdToDelete = result.Id; return testCode; diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 99b1d0dd..783d476b 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -45,7 +45,8 @@ public class SRTestEnvironment : TestEnvironment private string Jwt { get; set; } private GraphQLHttpClient GqlClient { get; init; } - public SRTestEnvironment(string? lexboxHostname = null, string? lexboxProtocol = null, string? lexboxPort = null, string? lexboxUsername = null, string? lexboxPassword = null) + public SRTestEnvironment(TemporaryFolder? tempFolder = null, string? lexboxHostname = null, string? lexboxProtocol = null, string? lexboxPort = null, string? lexboxUsername = null, string? lexboxPassword = null) + : base(true, true, true, tempFolder ?? new TemporaryFolder(TestName + Path.GetRandomFileName())) { _lexboxHostname = lexboxHostname; _lexboxProtocol = lexboxProtocol; @@ -59,11 +60,11 @@ public SRTestEnvironment(string? lexboxHostname = null, string? lexboxProtocol = if (lexboxPassword is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_TrustToken, lexboxPassword); LexboxUrl = new Uri($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); LexboxUrlBasicAuth = new Uri($"{LexboxProtocol}://{WebUtility.UrlEncode(LexboxUsername)}:{WebUtility.UrlEncode(LexboxPassword)}@{LexboxHostname}:{LexboxPort}"); - TempFolder = new TemporaryFolder(TestName + Path.GetRandomFileName()); + TempFolder = _languageForgeServerFolder; Handler.CookieContainer = Cookies; - Http = new HttpClient(Handler); + Http = new HttpClient(Handler); // TODO: Move to static constructor var lexboxGqlEndpoint = new Uri(LexboxUrl, "/api/graphql"); - GqlClient = new GraphQLHttpClient(lexboxGqlEndpoint, new SystemTextJsonSerializer(), Http); + GqlClient = new GraphQLHttpClient(lexboxGqlEndpoint, new SystemTextJsonSerializer(), Http); // TODO: Move to static constructor } public Task Login() @@ -235,7 +236,7 @@ public void CommitAndPush(FwProject project, string code, string baseDir, string MercurialTestHelper.HgPush(projectDir, withAuth.Uri.AbsoluteUri); } - private string TestName + private static string TestName { get { diff --git a/src/LfMerge.Core.Tests/TestEnvironment.cs b/src/LfMerge.Core.Tests/TestEnvironment.cs index cae38056..b641454c 100644 --- a/src/LfMerge.Core.Tests/TestEnvironment.cs +++ b/src/LfMerge.Core.Tests/TestEnvironment.cs @@ -22,7 +22,7 @@ namespace LfMerge.Core.Tests { public class TestEnvironment : IDisposable { - private readonly TemporaryFolder _languageForgeServerFolder; + protected readonly TemporaryFolder _languageForgeServerFolder; private readonly bool _resetLfProjectsDuringCleanup; private readonly bool _releaseSingletons; public LfMergeSettings Settings; diff --git a/src/LfMerge.Core/LanguageForgeProject.cs b/src/LfMerge.Core/LanguageForgeProject.cs index 8f2dec12..6c3991e9 100644 --- a/src/LfMerge.Core/LanguageForgeProject.cs +++ b/src/LfMerge.Core/LanguageForgeProject.cs @@ -20,13 +20,13 @@ public class LanguageForgeProject: ILfProject private readonly string _projectCode; private ILanguageDepotProject _languageDepotProject; - public static LanguageForgeProject Create(string projectCode) + public static LanguageForgeProject Create(string projectCode, LfMergeSettings settings = null) { LanguageForgeProject project; if (CachedProjects.TryGetValue(projectCode, out project)) return project; - project = new LanguageForgeProject(projectCode); + project = new LanguageForgeProject(projectCode, settings); CachedProjects.Add(projectCode, project); return project; } From 6af71695ba5b93c86414712380b8981bb911a56c Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 15 Aug 2024 13:08:16 +0700 Subject: [PATCH 41/73] Use static properties for SRTestEnv.HttpClient This simplifies the constructor a lot and ends up making tests that clone FieldWorks projects a lot easier to write. --- src/LfMerge.Core.Tests/SRTestBase.cs | 17 +++--- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 64 ++++++++------------- 2 files changed, 31 insertions(+), 50 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index b0b93560..d2a85788 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -46,14 +46,11 @@ public async Task FixtureSetup() var derivedClassName = this.GetType().Name; TempFolderForClass = new TemporaryFolder(rootTempFolder, derivedClassName); - // Log in to LexBox as admin so we get a login cookie - TestEnv = new SRTestEnvironment(TempFolderForClass); - await TestEnv.Login(); - // Ensure sena-3.zip is available to all tests as a starting point Sena3ZipPath = Path.Combine(TestDataFolder.Path, "sena-3.zip"); if (!File.Exists(Sena3ZipPath)) { - await TestEnv.DownloadProjectBackup("sena-3", Sena3ZipPath); + await SRTestEnvironment.Login(); + await SRTestEnvironment.DownloadProjectBackup("sena-3", Sena3ZipPath); } } @@ -71,6 +68,8 @@ public void FixtureTeardown() public async Task TestSetup() { TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath); + TestEnv = new SRTestEnvironment(TempFolderForTest); + await SRTestEnvironment.Login(); await BackupRemoteProject(); } @@ -109,8 +108,7 @@ public async Task RestoreRemoteProject() } } - // TODO See if TempFolderForTest will work instead of TempFolderForClass... (need to refactor SRTestEnv to expect to be instantiated once per test in order to pull that off) - public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForClass.Path, "webwork", projectCode); + public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode); public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata"); public string CloneRepoFromLexbox(string code, string? newCode = null) @@ -127,11 +125,12 @@ public FwProject CloneFromLexbox(string code, string? newCode = null) var dest = CloneRepoFromLexbox(code, newCode); var dirInfo = new DirectoryInfo(dest); if (!dirInfo.Exists) throw new InvalidOperationException($"Failed to clone {code} from lexbox, cannot create FwProject"); - var fwdataPath = Path.Join(dest, $"{newCode}.fwdata"); + var dirname = dirInfo.Name; + var fwdataPath = Path.Join(dest, $"{dirname}.fwdata"); MercurialTestHelper.ChangeBranch(dest, "tip"); LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); var settings = new LfMergeSettingsDouble(TempFolderForTest.Path); - return new FwProject(settings, newCode); + return new FwProject(settings, dirname); } public async Task CreateNewFlexProject() diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 783d476b..36c0a09f 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -25,58 +25,41 @@ namespace LfMerge.Core.Tests /// public class SRTestEnvironment : TestEnvironment { - public Uri LexboxUrl { get; init; } - public Uri LexboxUrlBasicAuth { get; init; } - private string? _lexboxHostname; - public string LexboxHostname => _lexboxHostname ?? Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; - private string? _lexboxProtocol; - public string LexboxProtocol => _lexboxProtocol ?? Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; - private string? _lexboxPort; - public string LexboxPort => _lexboxPort ?? (LexboxProtocol == "http" ? "80" : "443"); - private string? _lexboxUsername; - public string LexboxUsername => _lexboxUsername ?? Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername) ?? "admin"; - private string? _lexboxPassword; - public string LexboxPassword => _lexboxPassword ?? Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; - private TemporaryFolder TempFolder { get; init; } - private HttpClient Http { get; init; } - private HttpClientHandler Handler { get; init; } = new HttpClientHandler(); - private CookieContainer Cookies { get; init; } = new CookieContainer(); + public static string LexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; + public static string LexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; + public static string LexboxPort = LexboxProtocol == "http" ? "80" : "443"; + public static string LexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername) ?? "admin"; + public static string LexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; + public static Uri LexboxUrl = new Uri($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); + public static Uri LexboxUrlBasicAuth = new Uri($"{LexboxProtocol}://{WebUtility.UrlEncode(LexboxUsername)}:{WebUtility.UrlEncode(LexboxPassword)}@{LexboxHostname}:{LexboxPort}"); + public static CookieContainer Cookies { get; } = new CookieContainer(); + private static HttpClientHandler Handler = new HttpClientHandler { CookieContainer = Cookies }; + private static Lazy LazyHttp = new(() => new HttpClient(Handler)); + public static HttpClient Http => LazyHttp.Value; public static SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); + public static bool AlreadyLoggedIn = false; + private TemporaryFolder TempFolder { get; init; } private string Jwt { get; set; } - private GraphQLHttpClient GqlClient { get; init; } + private static Lazy LazyGqlClient = new(() => new GraphQLHttpClient(new Uri(LexboxUrl, "/api/graphql"), new SystemTextJsonSerializer(), Http)); + public static GraphQLHttpClient GqlClient => LazyGqlClient.Value; - public SRTestEnvironment(TemporaryFolder? tempFolder = null, string? lexboxHostname = null, string? lexboxProtocol = null, string? lexboxPort = null, string? lexboxUsername = null, string? lexboxPassword = null) + public SRTestEnvironment(TemporaryFolder? tempFolder = null) : base(true, true, true, tempFolder ?? new TemporaryFolder(TestName + Path.GetRandomFileName())) { - _lexboxHostname = lexboxHostname; - _lexboxProtocol = lexboxProtocol; - _lexboxPort = lexboxPort; - _lexboxUsername = lexboxUsername; - _lexboxPassword = lexboxPassword; - if (lexboxHostname is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname, lexboxHostname); - if (lexboxHostname is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPrivateHostname, lexboxHostname); - if (lexboxProtocol is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol, lexboxProtocol); - if (lexboxUsername is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_HgUsername, lexboxUsername); - if (lexboxPassword is not null) Environment.SetEnvironmentVariable(MagicStrings.EnvVar_TrustToken, lexboxPassword); - LexboxUrl = new Uri($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); - LexboxUrlBasicAuth = new Uri($"{LexboxProtocol}://{WebUtility.UrlEncode(LexboxUsername)}:{WebUtility.UrlEncode(LexboxPassword)}@{LexboxHostname}:{LexboxPort}"); - TempFolder = _languageForgeServerFolder; - Handler.CookieContainer = Cookies; - Http = new HttpClient(Handler); // TODO: Move to static constructor - var lexboxGqlEndpoint = new Uri(LexboxUrl, "/api/graphql"); - GqlClient = new GraphQLHttpClient(lexboxGqlEndpoint, new SystemTextJsonSerializer(), Http); // TODO: Move to static constructor + TempFolder = _languageForgeServerFolder; // Better name for what E2E tests use it for } - public Task Login() + public static Task Login() { return LoginAs(LexboxUsername, LexboxPassword); } - public async Task LoginAs(string lexboxUsername, string lexboxPassword) + public static async Task LoginAs(string lexboxUsername, string lexboxPassword) { + if (AlreadyLoggedIn) return; var loginResult = await Http.PostAsync(new Uri(LexboxUrl, "api/login"), JsonContent.Create(new { EmailOrUsername=lexboxUsername, Password=lexboxPassword })); var cookies = Cookies.GetCookies(LexboxUrl); - Jwt = cookies[".LexBoxAuth"].Value; + AlreadyLoggedIn = true; // Http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", Jwt); // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. } @@ -175,13 +158,12 @@ public async Task UploadZip(string code, string zipPath) await client.UploadAsync(fileUrl, file); } - public async Task DownloadProjectBackup(string code, string? destZipPath = null) + public static async Task DownloadProjectBackup(string code, string destZipPath) { var backupUrl = new Uri(LexboxUrl, $"api/project/backupProject/{code}"); var result = await Http.GetAsync(backupUrl); var filename = result.Content.Headers.ContentDisposition?.FileName; - var savePath = destZipPath ?? Path.Join(TempFolder.Path, filename); - using (var outStream = File.Create(savePath)) + using (var outStream = File.Create(destZipPath)) { await result.Content.CopyToAsync(outStream); } From adcd8f27e32e255212492f581601a6b9dde2ca01 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 15 Aug 2024 13:21:25 +0700 Subject: [PATCH 42/73] Make most SRTestEnv methods static Most methods in SRTestEnvironment now access only static properties such as the HttpClient, so most of them can now be made static as well. --- src/LfMerge.Core.Tests/SRTestBase.cs | 16 ++++---- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 45 ++++++++++----------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index d2a85788..fa3c74ab 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -27,8 +27,8 @@ public SRTestBase() { } - private string TestName => TestContext.CurrentContext.Test.Name; - private string TestNameForPath => string.Join("", TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars + private static string TestName => TestContext.CurrentContext.Test.Name; + private static string TestNameForPath => string.Join("", TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars [OneTimeSetUp] public async Task FixtureSetup() @@ -83,7 +83,7 @@ public async Task TestTeardown() if (ProjectIdToDelete is not null) { var projId = ProjectIdToDelete.Value; ProjectIdToDelete = null; - await TestEnv.DeleteLexBoxProject(projId); + await SRTestEnvironment.DeleteLexBoxProject(projId); } } } @@ -93,7 +93,7 @@ public async Task BackupRemoteProject() var test = TestContext.CurrentContext.Test; if (test.Properties.ContainsKey("projectCode")) { var code = test.Properties.Get("projectCode") as string; - TipRevToRestore = await TestEnv.GetTipRev(code); + TipRevToRestore = await SRTestEnvironment.GetTipRev(code); } else { TipRevToRestore = ""; } @@ -113,7 +113,7 @@ public async Task RestoreRemoteProject() public string CloneRepoFromLexbox(string code, string? newCode = null) { - var projUrl = TestEnv.LexboxUrlForProjectWithAuth(code); + var projUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(code); newCode ??= code; var dest = TestFolderForProject(newCode); MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); @@ -141,7 +141,7 @@ public async Task CreateNewFlexProject() MercurialTestHelper.InitializeHgRepo(testPath); MercurialTestHelper.CreateFlexRepo(testPath); // Now create project in LexBox - var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); + var result = await SRTestEnvironment.CreateLexBoxProject(testCode, randomGuid); Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); Assert.That(result.Id, Is.EqualTo(randomGuid)); // TODO: Push that first commit to lexbox so the project is non-empty @@ -155,7 +155,7 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) var testCode = $"sr-{randomGuid}"; var testPath = TestFolderForProject(testCode); // Now create project in LexBox - var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); + var result = await SRTestEnvironment.CreateLexBoxProject(testCode, randomGuid); Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); Assert.That(result.Id, Is.EqualTo(randomGuid)); await TestEnv.ResetAndUploadZip(testCode, origZipPath); @@ -167,7 +167,7 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null) { - TestEnv.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); + SRTestEnvironment.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); } } } diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 36c0a09f..6be8223e 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -25,22 +25,21 @@ namespace LfMerge.Core.Tests /// public class SRTestEnvironment : TestEnvironment { - public static string LexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; - public static string LexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; - public static string LexboxPort = LexboxProtocol == "http" ? "80" : "443"; - public static string LexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername) ?? "admin"; - public static string LexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; - public static Uri LexboxUrl = new Uri($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); - public static Uri LexboxUrlBasicAuth = new Uri($"{LexboxProtocol}://{WebUtility.UrlEncode(LexboxUsername)}:{WebUtility.UrlEncode(LexboxPassword)}@{LexboxHostname}:{LexboxPort}"); - public static CookieContainer Cookies { get; } = new CookieContainer(); - private static HttpClientHandler Handler = new HttpClientHandler { CookieContainer = Cookies }; - private static Lazy LazyHttp = new(() => new HttpClient(Handler)); + public static readonly string LexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; + public static readonly string LexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; + public static readonly string LexboxPort = LexboxProtocol == "http" ? "80" : "443"; + public static readonly string LexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername) ?? "admin"; + public static readonly string LexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; + public static readonly Uri LexboxUrl = new($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); + public static readonly Uri LexboxUrlBasicAuth = new($"{LexboxProtocol}://{WebUtility.UrlEncode(LexboxUsername)}:{WebUtility.UrlEncode(LexboxPassword)}@{LexboxHostname}:{LexboxPort}"); + public static readonly CookieContainer Cookies = new(); + private static readonly HttpClientHandler Handler = new() { CookieContainer = Cookies }; + private static readonly Lazy LazyHttp = new(() => new HttpClient(Handler)); public static HttpClient Http => LazyHttp.Value; - public static SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); - public static bool AlreadyLoggedIn = false; + public static readonly SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); + private static bool AlreadyLoggedIn = false; private TemporaryFolder TempFolder { get; init; } - private string Jwt { get; set; } - private static Lazy LazyGqlClient = new(() => new GraphQLHttpClient(new Uri(LexboxUrl, "/api/graphql"), new SystemTextJsonSerializer(), Http)); + private static readonly Lazy LazyGqlClient = new(() => new GraphQLHttpClient(new Uri(LexboxUrl, "/api/graphql"), new SystemTextJsonSerializer(), Http)); public static GraphQLHttpClient GqlClient => LazyGqlClient.Value; public SRTestEnvironment(TemporaryFolder? tempFolder = null) @@ -64,10 +63,10 @@ public static async Task LoginAs(string lexboxUsername, string lexboxPassword) // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. } - public Uri LexboxUrlForProject(string code) => new Uri(LexboxUrl, $"hg/{code}"); - public Uri LexboxUrlForProjectWithAuth(string code) => new Uri(LexboxUrlBasicAuth, $"hg/{code}"); + public static Uri LexboxUrlForProject(string code) => new Uri(LexboxUrl, $"hg/{code}"); + public static Uri LexboxUrlForProjectWithAuth(string code) => new Uri(LexboxUrlBasicAuth, $"hg/{code}"); - public async Task CreateLexBoxProject(string code, Guid? projId = null, string? name = null, string? description = null, Guid? managerId = null, Guid? orgId = null) + public static async Task CreateLexBoxProject(string code, Guid? projId = null, string? name = null, string? description = null, Guid? managerId = null, Guid? orgId = null) { projId ??= Guid.NewGuid(); name ??= code; @@ -97,7 +96,7 @@ ... on DbError { return response.Data.CreateProject.CreateProjectResponse; } - public async Task DeleteLexBoxProject(Guid projectId) + public static async Task DeleteLexBoxProject(Guid projectId) { var mutation = """ mutation SoftDeleteProject($input: SoftDeleteProjectInput!) { @@ -122,7 +121,7 @@ ... on Error { Assert.That(response.Errors, Is.Null.Or.Empty, () => string.Join("\n", response.Errors.Select(error => error.Message))); } - public void InitRepo(string code, string dest) + public static void InitRepo(string code, string dest) { var sourceUrl = LexboxUrlForProjectWithAuth(code); MercurialTestHelper.CloneRepo(sourceUrl.AbsoluteUri, dest); @@ -137,7 +136,7 @@ public async Task ResetAndUploadZip(string code, string zipPath) await UploadZip(code, zipPath); } - public async Task ResetToEmpty(string code) + public static async Task ResetToEmpty(string code) { var resetUrl = new Uri(LexboxUrl, $"api/project/resetProject/{code}"); await Http.PostAsync(resetUrl, null); @@ -145,7 +144,7 @@ public async Task ResetToEmpty(string code) await Http.PostAsync(finishResetUrl, null); } - public async Task UploadZip(string code, string zipPath) + public static async Task UploadZip(string code, string zipPath) { var sourceUrl = new Uri(LexboxUrl, $"/api/project/upload-zip/{code}"); var file = new FileInfo(zipPath); @@ -195,14 +194,14 @@ public Task RollbackProjectToRev(string code, int revnum) public record TipJson(string Node); - public async Task GetTipRev(string code) + public static async Task GetTipRev(string code) { var tipUrl = new Uri(LexboxUrl, $"/hg/{code}/file/tip?style=json"); var result = await Http.GetFromJsonAsync(tipUrl); return result.Node; } - public void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) + public static void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) { localCode ??= code; var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); From 985683de7d8a5ba7d367210efd0442a2bda02164 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 15 Aug 2024 13:27:58 +0700 Subject: [PATCH 43/73] Less spammy logs from TransferLcmToMongo in tests --- src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 2 +- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index ba376eed..7625c962 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -48,7 +48,7 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() // Do an initial clone from mock LD to LF (in the form of the mock Mongo) - var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, TestEnv.Logger, _mongoConnection, _recordFactory); + var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, SRTestEnvironment.NullLogger, _mongoConnection, _recordFactory); transferLcmToMongo.Run(lfProject); // Do some initial checks to make sure we got the right data diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 6be8223e..8c2fcaf8 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -37,6 +37,7 @@ public class SRTestEnvironment : TestEnvironment private static readonly Lazy LazyHttp = new(() => new HttpClient(Handler)); public static HttpClient Http => LazyHttp.Value; public static readonly SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); + public static readonly ILogger NullLogger = new NullLogger(); private static bool AlreadyLoggedIn = false; private TemporaryFolder TempFolder { get; init; } private static readonly Lazy LazyGqlClient = new(() => new GraphQLHttpClient(new Uri(LexboxUrl, "/api/graphql"), new SystemTextJsonSerializer(), Http)); From 70e34e823e899e89d175e8dba5d5a84f1e8fd9dc Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 15 Aug 2024 13:35:18 +0700 Subject: [PATCH 44/73] Create helper method for E2E LF project creation --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 16 ++++++++++++- src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 25 ++------------------ 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index e7fcdbd9..45b2aacd 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -72,9 +72,23 @@ public void Setup() _recordFactory = MainClass.Container.Resolve() as MongoProjectRecordFactoryDouble; if (_recordFactory == null) throw new AssertionException("E2E tests need a mock MongoProjectRecordFactory in order to work."); - } + public async Task CreateLfProjectFromSena3() + { + var projCode = await CreateNewProjectFromSena3(); + var projPath = CloneRepoFromLexbox(projCode); + MercurialTestHelper.ChangeBranch(projPath, "tip"); + var fwdataPath = Path.Combine(projPath, $"{projCode}.fwdata"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); + + // Do an initial clone from LexBox to the mock LF + var lfProject = LanguageForgeProject.Create(projCode, TestEnv.Settings); + lfProject.IsInitialClone = true; + var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, SRTestEnvironment.NullLogger, _mongoConnection, _recordFactory); + transferLcmToMongo.Run(lfProject); + return lfProject; + } } } diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index 7625c962..8a455ee9 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -27,32 +27,11 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() { // Setup - // Take testlangproj-modified and "upload" it to mock LD as proj code "testlangproj" - // TODO: Replace the above with uploading the modified FwProject to LexBox, or perhaps cloning the original, having FW modify it, then uploading the modified version - - var projCode = await CreateNewProjectFromSena3(); - Guid entryId = Guid.Parse("0006f482-a078-4cef-9c5a-8bd35b53cf72"); - - var projPath = CloneRepoFromLexbox(projCode); - Console.WriteLine("cloned"); - MercurialTestHelper.ChangeBranch(projPath, "tip"); - Console.WriteLine("on tip"); - var fwdataPath = Path.Combine(projPath, $"{projCode}.fwdata"); - LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); - Console.WriteLine("reassembled"); - // TODO: Write a helper to simplify this process - - var lfProject = LanguageForgeProject.Create(projCode, TestEnv.Settings); - lfProject.IsInitialClone = true; - Console.WriteLine($"Will look for .fwdata file at {lfProject.FwDataPath}"); - - // Do an initial clone from mock LD to LF (in the form of the mock Mongo) - - var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, SRTestEnvironment.NullLogger, _mongoConnection, _recordFactory); - transferLcmToMongo.Run(lfProject); + var lfProject = await CreateLfProjectFromSena3(); // Do some initial checks to make sure we got the right data + Guid entryId = Guid.Parse("0006f482-a078-4cef-9c5a-8bd35b53cf72"); var lcmObject = lfProject.FieldWorksProject?.ServiceLocator?.ObjectRepository?.GetObject(entryId); Assert.That(lcmObject, Is.Not.Null); var lcmEntry = lcmObject as ILexEntry; From 3fa366016ba3dabe2d70929258beaef0e8ae4d11 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 15 Aug 2024 13:48:56 +0700 Subject: [PATCH 45/73] Very slight performance improvement --- src/LfMerge.Core.Tests/SRTestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index fa3c74ab..52295b18 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -28,7 +28,7 @@ public SRTestBase() } private static string TestName => TestContext.CurrentContext.Test.Name; - private static string TestNameForPath => string.Join("", TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars + private static string TestNameForPath => string.Concat(TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars [OneTimeSetUp] public async Task FixtureSetup() From 01364cef9e6d2eac6b552251b1a97fa23dedf0a3 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 08:57:48 +0700 Subject: [PATCH 46/73] Clone LF and FW projects with different codes This will allow us to check out the LF and FW projects separately, to better simulate a true Send/Receive scenario. --- src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index 8a455ee9..a147e984 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -11,6 +11,7 @@ using LfMerge.Core.Actions; using LfMerge.Core.LanguageForge.Model; using SIL.LCModel; +using System.Text.RegularExpressions; namespace LfMerge.Core.Tests.E2E { @@ -28,6 +29,8 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() // Setup var lfProject = await CreateLfProjectFromSena3(); + var fwProjectCode = Regex.Replace(lfProject.ProjectCode, "^sr-", "fw-"); + var fwProject = CloneFromLexbox(lfProject.ProjectCode, fwProjectCode); // Do some initial checks to make sure we got the right data @@ -43,6 +46,14 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() Assert.That(lfEntry.CitationForm.BestString(["seh"]), Is.EqualTo("cibubu")); + // Check that FW project cloned correctly too + + var fwObject = fwProject?.ServiceLocator?.ObjectRepository?.GetObject(entryId); + Assert.That(fwObject, Is.Not.Null); + var fwEntry = fwObject as ILexEntry; + Assert.That(fwEntry, Is.Not.Null); + Assert.That(fwEntry.CitationForm.BestVernacularAlternative.Text, Is.EqualTo("cibubu")); + // TODO: Finish implementing the rest of the original test (below) from SynchronizeActionTests, then create helper methods for reusable parts // Capture original modified dates from the particular entry we're interested in From 5689335712af1ff06379a3ad210251de5f895bc9 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 10:42:24 +0700 Subject: [PATCH 47/73] Working end-to-end Send/Receive test Our first test from SynchronizeActionTests fully converted to do end-to-end testing with a real LexBox instance, and working. --- src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 103 +++++++------------ src/LfMerge.Core.Tests/LcmTestHelper.cs | 10 ++ src/LfMerge.Core.Tests/SRTestEnvironment.cs | 1 + src/LfMerge.Core.Tests/TestDoubles.cs | 2 + 4 files changed, 50 insertions(+), 66 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index a147e984..e4de5548 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -20,9 +20,6 @@ namespace LfMerge.Core.Tests.E2E [Category("IntegrationTests")] public class TryOutE2ETests : E2ETestBase { - - // TODO: Duplicate the test below using LexBox instead of a mock LanguageDepot server - [Test] public async Task E2E_LFDataChangedLDDataChanged_LFWins() { @@ -32,86 +29,60 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() var fwProjectCode = Regex.Replace(lfProject.ProjectCode, "^sr-", "fw-"); var fwProject = CloneFromLexbox(lfProject.ProjectCode, fwProjectCode); - // Do some initial checks to make sure we got the right data + // Modify FW data first, then push to Lexbox Guid entryId = Guid.Parse("0006f482-a078-4cef-9c5a-8bd35b53cf72"); - var lcmObject = lfProject.FieldWorksProject?.ServiceLocator?.ObjectRepository?.GetObject(entryId); - Assert.That(lcmObject, Is.Not.Null); - var lcmEntry = lcmObject as ILexEntry; - Assert.That(lcmEntry, Is.Not.Null); - Assert.That(lcmEntry.CitationForm.BestVernacularAlternative.Text, Is.EqualTo("cibubu")); - - IEnumerable originalMongoData = _mongoConnection.GetLfLexEntries(); - LfLexEntry lfEntry = originalMongoData.First(e => e.Guid == entryId); - - Assert.That(lfEntry.CitationForm.BestString(["seh"]), Is.EqualTo("cibubu")); - // Check that FW project cloned correctly too - - var fwObject = fwProject?.ServiceLocator?.ObjectRepository?.GetObject(entryId); - Assert.That(fwObject, Is.Not.Null); - var fwEntry = fwObject as ILexEntry; + var fwEntryRepo = fwProject.Cache.ServiceLocator.GetInstance(); + var fwEntry = fwEntryRepo.GetObject(entryId); Assert.That(fwEntry, Is.Not.Null); - Assert.That(fwEntry.CitationForm.BestVernacularAlternative.Text, Is.EqualTo("cibubu")); - - // TODO: Finish implementing the rest of the original test (below) from SynchronizeActionTests, then create helper methods for reusable parts - - // Capture original modified dates from the particular entry we're interested in + string unchangedGloss = LcmTestHelper.GetAnalysisText(fwEntry.SensesOS[0].Gloss); + LcmTestHelper.UpdateAnalysisText(fwProject, fwEntry.SensesOS[0].Gloss, text => text + " - changed in FW"); + DateTime fwDateModified = fwEntry.DateModified; + CommitAndPush(fwProject, lfProject.ProjectCode, fwProjectCode, "Modified gloss in FW"); - // IEnumerable originalMongoData = _mongoConnection.GetLfLexEntries(); - // LfLexEntry lfEntry = originalMongoData.First(e => e.Guid == _testEntryGuid); - // DateTime originalLfDateModified = lfEntry.DateModified; - // DateTime originalLfAuthorInfoModifiedDate = lfEntry.AuthorInfo.ModifiedDate; + // Modify LF data second, then launch Send/Receive - // Modify the entry in LF, but not yet in LD + var lfEntry = _mongoConnection.GetLfLexEntryByGuid(entryId); + // Verify LF entry not yet affected by FW change because Send/Receive has not happened yet + Assert.That(lfEntry.Senses[0].Gloss["pt"].Value, Is.EqualTo(unchangedGloss)); + // Capture original modified dates so we can check later that they've been updated + DateTime originalLfDateModified = lfEntry.DateModified; + DateTime originalLfAuthorInfoModifiedDate = lfEntry.AuthorInfo.ModifiedDate; - // string unchangedGloss = lfEntry.Senses[0].Gloss["en"].Value; - // string fwChangedGloss = unchangedGloss + " - changed in FW"; - // string lfChangedGloss = unchangedGloss + " - changed in LF"; - // lfEntry.Senses[0].Gloss["en"].Value = lfChangedGloss; - // lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; - // _mongoConnection.UpdateRecord(_lfProject, lfEntry); + lfEntry.Senses[0].Gloss["pt"].Value = unchangedGloss + " - changed in LF"; + lfEntry.AuthorInfo.ModifiedDate = lfEntry.DateModified = DateTime.UtcNow; + _mongoConnection.UpdateRecord(lfProject, lfEntry); - // Verify that the LD project has the FW value for the gloss + // Ensure LF project will S/R to local LexBox - // _lDProject = new LanguageDepotMock(testProjectCode, _lDSettings); - // var lDcache = _lDProject.FieldWorksProject.Cache; - // var lDLcmEntry = lDcache.ServiceLocator.GetObject(_testEntryGuid) as ILexEntry; - // Assert.That(lDLcmEntry.SensesOS[0].Gloss.AnalysisDefaultWritingSystem.Text, Is.EqualTo(fwChangedGloss)); - - // Capture the original modified date from the entry in the FW/LCM project - - // DateTime originalLdDateModified = lDLcmEntry.DateModified; + var saveEnv = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri); + Environment.SetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri, SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri); + Console.WriteLine(SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri); // Exercise - // Capture date/time immediately before running, to make sure that SyncAction updates it - - // var sutSynchronize = new SynchronizeAction(_env.Settings, _env.Logger); - // var timeBeforeRun = DateTime.UtcNow; + // Do LfMerge Send/Receive of LF project] + var lfEntryBeforeSR = _mongoConnection.GetLfLexEntryByGuid(entryId); + Assert.That(lfEntryBeforeSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF")); - // Actually do the LfMerge Send/Receive with mock LD - - // sutSynchronize.Run(_lfProject); + try { + var syncAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger); + var timeBeforeRun = DateTime.UtcNow; + Assert.That(timeBeforeRun, Is.GreaterThan(lfEntry.AuthorInfo.ModifiedDate)); // TODO: Since this is trivially true, we might be able to get rid of timeBeforeRun. + syncAction.Run(lfProject); + } finally { + Environment.SetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri, saveEnv); + } // Verify - // Verify that the result of the conflict was LF winning - - // Assert.That(GetGlossFromMongoDb(_testEntryGuid), Is.EqualTo(lfChangedGloss)); - - // Verify that the modified dates got updated when LF won the merge conflict, even though LF's view of the data didn't change + // Verify that the result of the conflict was LF winning, and LF's modified dates got updated by the sync action - // LfLexEntry updatedLfEntry = _mongoConnection.GetLfLexEntries().First(e => e.Guid == _testEntryGuid); - // DateTime updatedLfDateModified = updatedLfEntry.DateModified; - // DateTime updatedLfAuthorInfoModifiedDate = updatedLfEntry.AuthorInfo.ModifiedDate; - // // LF had the same data previously; however it's a merge conflict so DateModified - // // got updated - // Assert.That(updatedLfDateModified, Is.GreaterThan(originalLfDateModified)); - // // But the LCM modified date (AuthorInfo.ModifiedDate in LF) should be updated. - // Assert.That(updatedLfAuthorInfoModifiedDate, Is.GreaterThan(originalLfAuthorInfoModifiedDate)); - // Assert.That(updatedLfDateModified, Is.GreaterThan(originalLdDateModified)); - // Assert.That(_mongoConnection.GetLastSyncedDate(_lfProject), Is.GreaterThanOrEqualTo(timeBeforeRun)); + var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(entryId); + Assert.That(lfEntryAfterSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF")); + Assert.That(lfEntryAfterSR.DateModified, Is.GreaterThan(originalLfDateModified)); + Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(originalLfAuthorInfoModifiedDate)); } } } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index 82f7e0f0..e21b0590 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -32,6 +32,16 @@ public static ILexEntry GetEntry(FwProject project, Guid guid) return repo.GetObject(guid); } + public static string? GetVernacularText(IMultiUnicode field) + { + return field.BestVernacularAlternative?.Text; + } + + public static string? GetAnalysisText(IMultiUnicode field) + { + return field.BestAnalysisAlternative?.Text; + } + public static void SetVernacularText(FwProject project, IMultiUnicode field, string newText) { var accessor = project.Cache.ActionHandlerAccessor; diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 8c2fcaf8..12a3e024 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -47,6 +47,7 @@ public SRTestEnvironment(TemporaryFolder? tempFolder = null) : base(true, true, true, tempFolder ?? new TemporaryFolder(TestName + Path.GetRandomFileName())) { TempFolder = _languageForgeServerFolder; // Better name for what E2E tests use it for + Settings.CommitWhenDone = true; // For SR tests specifically, we *do* want changes to .fwdata files to be persisted } public static Task Login() diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs index 01a9a079..827209dd 100644 --- a/src/LfMerge.Core.Tests/TestDoubles.cs +++ b/src/LfMerge.Core.Tests/TestDoubles.cs @@ -417,6 +417,8 @@ class ChorusHelperDouble: ChorusHelper { public override string GetSyncUri(ILfProject project) { + var envOverride = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri); + if (envOverride != null) return envOverride; var server = LanguageDepotMock.Server; return server != null && server.IsStarted ? server.Url : LanguageDepotMock.ProjectFolderPath; } From b0158a4e5a6a44badb10390f8e52231d355e5a18 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 10:50:11 +0700 Subject: [PATCH 48/73] Improve LcmTestHelper API Slightly simplifies E2E test code --- src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 10 ++++------ src/LfMerge.Core.Tests/LcmTestHelper.cs | 6 ++++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index e4de5548..160a5e2b 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -33,11 +33,9 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() Guid entryId = Guid.Parse("0006f482-a078-4cef-9c5a-8bd35b53cf72"); - var fwEntryRepo = fwProject.Cache.ServiceLocator.GetInstance(); - var fwEntry = fwEntryRepo.GetObject(entryId); + var fwEntry = LcmTestHelper.GetEntry(fwProject, entryId); Assert.That(fwEntry, Is.Not.Null); - string unchangedGloss = LcmTestHelper.GetAnalysisText(fwEntry.SensesOS[0].Gloss); - LcmTestHelper.UpdateAnalysisText(fwProject, fwEntry.SensesOS[0].Gloss, text => text + " - changed in FW"); + string unchangedGloss = LcmTestHelper.UpdateAnalysisText(fwProject, fwEntry.SensesOS[0].Gloss, text => text + " - changed in FW"); DateTime fwDateModified = fwEntry.DateModified; CommitAndPush(fwProject, lfProject.ProjectCode, fwProjectCode, "Modified gloss in FW"); @@ -68,8 +66,6 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() try { var syncAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger); - var timeBeforeRun = DateTime.UtcNow; - Assert.That(timeBeforeRun, Is.GreaterThan(lfEntry.AuthorInfo.ModifiedDate)); // TODO: Since this is trivially true, we might be able to get rid of timeBeforeRun. syncAction.Run(lfProject); } finally { Environment.SetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri, saveEnv); @@ -82,7 +78,9 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(entryId); Assert.That(lfEntryAfterSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF")); Assert.That(lfEntryAfterSR.DateModified, Is.GreaterThan(originalLfDateModified)); + Assert.That(lfEntryAfterSR.DateModified, Is.GreaterThan(fwDateModified)); Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(originalLfAuthorInfoModifiedDate)); + Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(fwDateModified)); } } } diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index e21b0590..d05e3a48 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -58,7 +58,7 @@ public static void SetAnalysisText(FwProject project, IMultiUnicode field, strin }); } - public static void UpdateVernacularText(FwProject project, IMultiUnicode field, Func textConverter) + public static string UpdateVernacularText(FwProject project, IMultiUnicode field, Func textConverter) { var oldText = field.BestVernacularAlternative?.Text; if (oldText != null) @@ -66,9 +66,10 @@ public static void UpdateVernacularText(FwProject project, IMultiUnicode field, var newText = textConverter(oldText); SetVernacularText(project, field, newText); } + return oldText; } - public static void UpdateAnalysisText(FwProject project, IMultiUnicode field, Func textConverter) + public static string UpdateAnalysisText(FwProject project, IMultiUnicode field, Func textConverter) { var oldText = field.BestAnalysisAlternative?.Text; if (oldText != null) @@ -76,6 +77,7 @@ public static void UpdateAnalysisText(FwProject project, IMultiUnicode field, Fu var newText = textConverter(oldText); SetAnalysisText(project, field, newText); } + return oldText; } } } From ab3c21e9d973f0c278842843d3a7d4572f209310 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 11:25:04 +0700 Subject: [PATCH 49/73] Refactor common code, greatly simplifying E2E test The code for updating glosses in FW and LF can be extracted into the test base class, allowing the individual tests to become a lot more readable as it's clear at a glance what's being updated. --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 22 +++++++++++++++++ src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 25 +++++--------------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 45b2aacd..3a7125f5 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -90,5 +90,27 @@ public async Task CreateLfProjectFromSena3() return lfProject; } + + public (string, DateTime, DateTime) UpdateFwGloss(FwProject project, Guid entryId, Func textConverter) + { + var fwEntry = LcmTestHelper.GetEntry(project, entryId); + Assert.That(fwEntry, Is.Not.Null); + var origModifiedDate = fwEntry.DateModified; + var unchangedGloss = LcmTestHelper.UpdateAnalysisText(project, fwEntry.SensesOS[0].Gloss, textConverter); + return (unchangedGloss, origModifiedDate, fwEntry.DateModified); + } + + public (string, DateTime, DateTime) UpdateLfGloss(LanguageForgeProject lfProject, Guid entryId, string wsId, Func textConverter) + { + var lfEntry = _mongoConnection.GetLfLexEntryByGuid(entryId); + Assert.That(lfEntry, Is.Not.Null); + var unchangedGloss = lfEntry.Senses[0].Gloss[wsId].Value; + lfEntry.Senses[0].Gloss["pt"].Value = textConverter(unchangedGloss); + // Remember that in LfMerge it's AuthorInfo that corresponds to the Lcm modified date + DateTime origModifiedDate = lfEntry.AuthorInfo.ModifiedDate; + lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; + _mongoConnection.UpdateRecord(lfProject, lfEntry); + return (unchangedGloss, origModifiedDate, lfEntry.AuthorInfo.ModifiedDate); + } } } diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index 160a5e2b..eeb9cde5 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -33,24 +33,12 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() Guid entryId = Guid.Parse("0006f482-a078-4cef-9c5a-8bd35b53cf72"); - var fwEntry = LcmTestHelper.GetEntry(fwProject, entryId); - Assert.That(fwEntry, Is.Not.Null); - string unchangedGloss = LcmTestHelper.UpdateAnalysisText(fwProject, fwEntry.SensesOS[0].Gloss, text => text + " - changed in FW"); - DateTime fwDateModified = fwEntry.DateModified; + var (unchangedGloss, origFwDateModified, fwDateModified) = UpdateFwGloss(fwProject, entryId, text => text + " - changed in FW"); CommitAndPush(fwProject, lfProject.ProjectCode, fwProjectCode, "Modified gloss in FW"); - // Modify LF data second, then launch Send/Receive + // Modify LF data second - var lfEntry = _mongoConnection.GetLfLexEntryByGuid(entryId); - // Verify LF entry not yet affected by FW change because Send/Receive has not happened yet - Assert.That(lfEntry.Senses[0].Gloss["pt"].Value, Is.EqualTo(unchangedGloss)); - // Capture original modified dates so we can check later that they've been updated - DateTime originalLfDateModified = lfEntry.DateModified; - DateTime originalLfAuthorInfoModifiedDate = lfEntry.AuthorInfo.ModifiedDate; - - lfEntry.Senses[0].Gloss["pt"].Value = unchangedGloss + " - changed in LF"; - lfEntry.AuthorInfo.ModifiedDate = lfEntry.DateModified = DateTime.UtcNow; - _mongoConnection.UpdateRecord(lfProject, lfEntry); + var (_, origLfDateModified, _) = UpdateLfGloss(lfProject, entryId, "pt", text => text + " - changed in LF"); // Ensure LF project will S/R to local LexBox @@ -77,10 +65,9 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(entryId); Assert.That(lfEntryAfterSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF")); - Assert.That(lfEntryAfterSR.DateModified, Is.GreaterThan(originalLfDateModified)); - Assert.That(lfEntryAfterSR.DateModified, Is.GreaterThan(fwDateModified)); - Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(originalLfAuthorInfoModifiedDate)); - Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(fwDateModified)); + Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(origLfDateModified)); + // Remember that FieldWorks's DateModified is stored in local time for some incomprehensible reason... + Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(fwDateModified.ToUniversalTime())); } } } From f737703b7d19433594e691c01f8880d750ad1b69 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 11:33:02 +0700 Subject: [PATCH 50/73] Final refactoring of first E2E test The process of doing a Send/Receive to LexBox is also an obvious bit of code to extract into a common method call. With this, the whole E2E test ends up readable without needing to scroll. --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 14 ++++++++++++ src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 24 +++----------------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 3a7125f5..85bd911d 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -91,6 +91,20 @@ public async Task CreateLfProjectFromSena3() return lfProject; } + public void SendReceiveToLexbox(LanguageForgeProject lfProject) + { + // ChorusHelperDouble.GetSyncUri assumes presence of a LanguageDepotMock, but here we want a real LexBox instance so we override it via environment variable + var saveEnv = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri); + try { + var lexboxRepoUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri; + Environment.SetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri, lexboxRepoUrl); + var syncAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger); + syncAction.Run(lfProject); + } finally { + Environment.SetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri, saveEnv); + } + } + public (string, DateTime, DateTime) UpdateFwGloss(FwProject project, Guid entryId, Func textConverter) { var fwEntry = LcmTestHelper.GetEntry(project, entryId); diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index eeb9cde5..689f0bde 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -30,41 +30,23 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() var fwProject = CloneFromLexbox(lfProject.ProjectCode, fwProjectCode); // Modify FW data first, then push to Lexbox - Guid entryId = Guid.Parse("0006f482-a078-4cef-9c5a-8bd35b53cf72"); - var (unchangedGloss, origFwDateModified, fwDateModified) = UpdateFwGloss(fwProject, entryId, text => text + " - changed in FW"); CommitAndPush(fwProject, lfProject.ProjectCode, fwProjectCode, "Modified gloss in FW"); // Modify LF data second - var (_, origLfDateModified, _) = UpdateLfGloss(lfProject, entryId, "pt", text => text + " - changed in LF"); - // Ensure LF project will S/R to local LexBox - - var saveEnv = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri); - Environment.SetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri, SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri); - Console.WriteLine(SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri); - // Exercise - // Do LfMerge Send/Receive of LF project] - var lfEntryBeforeSR = _mongoConnection.GetLfLexEntryByGuid(entryId); - Assert.That(lfEntryBeforeSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF")); - - try { - var syncAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger); - syncAction.Run(lfProject); - } finally { - Environment.SetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri, saveEnv); - } + SendReceiveToLexbox(lfProject); // Verify - // Verify that the result of the conflict was LF winning, and LF's modified dates got updated by the sync action - + // LF side should win conflict since its modified date was later var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(entryId); Assert.That(lfEntryAfterSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF")); + // LF's modified dates should have been updated by the sync action Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(origLfDateModified)); // Remember that FieldWorks's DateModified is stored in local time for some incomprehensible reason... Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(fwDateModified.ToUniversalTime())); From aaf4f36886e621889b5b36224d012d4a54ee7e2e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 12:01:01 +0700 Subject: [PATCH 51/73] Clean up now-unused code in E2ETestBase --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 36 ----------------------- 1 file changed, 36 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 85bd911d..b4530614 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -23,49 +23,13 @@ namespace LfMerge.Core.Tests.E2E [Category("IntegrationTests")] public class E2ETestBase : SRTestBase { - public LfMergeSettings _lDSettings; - public TemporaryFolder _languageDepotFolder; - public LanguageForgeProject _lfProject; - public SynchronizeAction _synchronizeAction; public MongoConnectionDouble _mongoConnection; public MongoProjectRecordFactory _recordFactory; - public string _workDir; - public const string TestLangProj = "testlangproj"; - public const string TestLangProjModified = "testlangproj-modified"; - - public void LcmSendReceive(FwProject project, string code, string? localCode = null, string? commitMsg = null) - { - // LfMergeBridge.LfMergeBridge.Execute?? - } - [SetUp] public void Setup() { MagicStrings.SetMinimalModelVersion(LcmCache.ModelVersion); - - // SyncActionTests used to create mock LD server -- not needed since we use real (local) LexBox here - - // _languageDepotFolder = new TemporaryFolder(TestContext.CurrentContext.Test.Name + Path.GetRandomFileName()); - // _lDSettings = new LfMergeSettingsDouble(_languageDepotFolder.Path); - // Directory.CreateDirectory(_lDSettings.WebWorkDirectory); - // LanguageDepotMock.ProjectFolderPath = - // Path.Combine(_lDSettings.WebWorkDirectory, TestLangProj); - // Directory.CreateDirectory(LanguageDepotMock.ProjectFolderPath); - // LanguageDepotMock.Server = new MercurialServer(LanguageDepotMock.ProjectFolderPath); - - // SyncActionTests used to create local LF project from the embedded test data -- here we will let each test do that - - // _lfProject = LanguageForgeProject.Create(TestLangProj); - - // TODO: get far enough to need to test this - _synchronizeAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger); - - // SyncActionTests used to set the current working directory so that Mercurial could be found; no longer needed now - - // _workDir = Directory.GetCurrentDirectory(); - // Directory.SetCurrentDirectory(ExecutionEnvironment.DirectoryOfExecutingAssembly); - _mongoConnection = MainClass.Container.Resolve() as MongoConnectionDouble; if (_mongoConnection == null) throw new AssertionException("E2E tests need a mock MongoConnection that stores data in order to work."); From 16d03cd32786ceeb1f2e7b6d83aac50b5df5e666 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 12:07:59 +0700 Subject: [PATCH 52/73] Minor code alignment tweak --- src/LfMerge.Core.Tests/TestEnvironment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LfMerge.Core.Tests/TestEnvironment.cs b/src/LfMerge.Core.Tests/TestEnvironment.cs index b641454c..fc54909f 100644 --- a/src/LfMerge.Core.Tests/TestEnvironment.cs +++ b/src/LfMerge.Core.Tests/TestEnvironment.cs @@ -22,7 +22,7 @@ namespace LfMerge.Core.Tests { public class TestEnvironment : IDisposable { - protected readonly TemporaryFolder _languageForgeServerFolder; + protected readonly TemporaryFolder _languageForgeServerFolder; private readonly bool _resetLfProjectsDuringCleanup; private readonly bool _releaseSingletons; public LfMergeSettings Settings; From 38fce529f2bf5e3ae8d16c5987c448ac6543de4a Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 12:20:15 +0700 Subject: [PATCH 53/73] Remove MkFwData code; we don't need it after all The E2E tests aren't going to run the MkFwData executable, they're just going to call the LfMergeBridge API directly when needed. Saves time. --- LfMerge.sln | 16 ----- src/MkFwData/.editorconfig | 27 --------- src/MkFwData/.gitattributes | 1 - src/MkFwData/MkFwData.csproj | 25 -------- src/MkFwData/Program.cs | 95 ------------------------------ src/SplitFwData/.editorconfig | 27 --------- src/SplitFwData/.gitattributes | 1 - src/SplitFwData/Program.cs | 76 ------------------------ src/SplitFwData/SplitFwData.csproj | 25 -------- 9 files changed, 293 deletions(-) delete mode 100644 src/MkFwData/.editorconfig delete mode 100644 src/MkFwData/.gitattributes delete mode 100644 src/MkFwData/MkFwData.csproj delete mode 100644 src/MkFwData/Program.cs delete mode 100644 src/SplitFwData/.editorconfig delete mode 100644 src/SplitFwData/.gitattributes delete mode 100644 src/SplitFwData/Program.cs delete mode 100644 src/SplitFwData/SplitFwData.csproj diff --git a/LfMerge.sln b/LfMerge.sln index fcb18ba4..090e3358 100644 --- a/LfMerge.sln +++ b/LfMerge.sln @@ -26,10 +26,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LfMerge.Core.Tests", "src\L EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LfMergeAuxTool", "src\LfMergeAuxTool\LfMergeAuxTool.csproj", "{28882F30-358B-4E1C-A934-076D9EE6ACFC}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MkFwData", "src\MkFwData\MkFwData.csproj", "{5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SplitFwData", "src\SplitFwData\SplitFwData.csproj", "{23DF39D3-5C50-4832-A64E-022396430390}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -85,18 +81,6 @@ Global {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Release|Any CPU.ActiveCfg = Release|Any CPU {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Release|Any CPU.Build.0 = Release|Any CPU - {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU - {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU - {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5CDB086B-79DE-4EF4-BB48-4AEAEEB0827B}.Release|Any CPU.Build.0 = Release|Any CPU - {23DF39D3-5C50-4832-A64E-022396430390}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {23DF39D3-5C50-4832-A64E-022396430390}.Debug|Any CPU.Build.0 = Debug|Any CPU - {23DF39D3-5C50-4832-A64E-022396430390}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU - {23DF39D3-5C50-4832-A64E-022396430390}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU - {23DF39D3-5C50-4832-A64E-022396430390}.Release|Any CPU.ActiveCfg = Release|Any CPU - {23DF39D3-5C50-4832-A64E-022396430390}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/MkFwData/.editorconfig b/src/MkFwData/.editorconfig deleted file mode 100644 index d4047b80..00000000 --- a/src/MkFwData/.editorconfig +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2016-2021 SIL International -# This software is licensed under the MIT license (http://opensource.org/licenses/MIT) - -root = false - -# Defaults -[*] -indent_style = space -indent_size = tab -tab_width = 4 -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true -max_line_length = 98 - -[*.cs] -indent_style = space -tab_width = 4 - -# Settings Visual Studio uses for the generated files -[*.{csproj,resx,settings,targets,vcxproj*,vdproj,xml,yml,props,md}] -indent_style = space -indent_size = 2 - -# Generated file -[*.sln] -end_of_line = crlf diff --git a/src/MkFwData/.gitattributes b/src/MkFwData/.gitattributes deleted file mode 100644 index cf3363d0..00000000 --- a/src/MkFwData/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -* text=auto whitespace=space-before-tab,tab-in-indent,blank-at-eol,tabwidth=4 diff --git a/src/MkFwData/MkFwData.csproj b/src/MkFwData/MkFwData.csproj deleted file mode 100644 index afbe05a3..00000000 --- a/src/MkFwData/MkFwData.csproj +++ /dev/null @@ -1,25 +0,0 @@ - - - - Exe - net8.0 - net8.0 - enable - enable - true - $(MSBuildProjectDirectory) - - - - - - - - - - - - - - - diff --git a/src/MkFwData/Program.cs b/src/MkFwData/Program.cs deleted file mode 100644 index b7203743..00000000 --- a/src/MkFwData/Program.cs +++ /dev/null @@ -1,95 +0,0 @@ -using Chorus.VcsDrivers.Mercurial; -using SIL.Progress; -using System.CommandLine; - -class Program -{ - static async Task Main(string[] args) - { - var rootCommand = new RootCommand("Make .fwdata file"); - - var verboseOption = new Option( - ["--verbose", "-v"], - "Display verbose output" - ); - rootCommand.AddGlobalOption(verboseOption); - - var quietOption = new Option( - ["--quiet", "-q"], - "Suppress all output (overrides --verbose if present)" - ); - rootCommand.AddGlobalOption(quietOption); - - var filename = new Argument( - "file", - "Name of .fwdata file to create, or directory to create it in" - ); - rootCommand.Add(filename); - - var hgRevOption = new Option( - ["--rev", "-r"], - "Revision to check out (default \"tip\")" - ); - hgRevOption.SetDefaultValue("tip"); - rootCommand.Add(hgRevOption); - - var cleanupOption = new Option( - ["--cleanup", "-c"], - "Clean repository after creating .fwdata file (deletes every other file except .fwdata)" - ); - rootCommand.Add(cleanupOption); - - rootCommand.SetHandler(Run, filename, verboseOption, quietOption, hgRevOption, cleanupOption); - - return await rootCommand.InvokeAsync(args); - } - - static FileInfo LocateFwDataFile(string input) - { - if (Directory.Exists(input)) { - var dirInfo = new DirectoryInfo(input); - var fname = dirInfo.Name + ".fwdata"; - return new FileInfo(Path.Join(input, fname)); - } else if (File.Exists(input)) { - return new FileInfo(input); - } else if (File.Exists(input + ".fwdata")) { - return new FileInfo(input + ".fwdata"); - } else { - if (input.EndsWith(".fwdata")) return new FileInfo(input); - return new FileInfo(input + ".fwdata"); - } - } - - static Task Run(string filename, bool verbose, bool quiet, string rev, bool cleanup) - { - IProgress progress = quiet ? new NullProgress() : new ConsoleProgress(); - progress.ShowVerbose = verbose; - var file = LocateFwDataFile(filename); - if (file.Exists) { - progress.WriteWarning("File {0} already exists and will be overwritten", file.FullName); - } - var dir = file.Directory; - if (dir == null || !dir.Exists) { - progress.WriteError("Could not find directory {0}. MkFwData needs a Mercurial repo to work with.", dir?.FullName ?? "(null)"); - return Task.FromResult(1); - } - string name = file.FullName; - progress.WriteMessage("Checking out {0}", rev); - var result = HgRunner.Run($"hg checkout {rev}", dir.FullName, 30, progress); - if (result.ExitCode != 0) - { - progress.WriteMessage("Could not find Mercurial repo in directory {0}. MkFwData needs a Mercurial repo to work with.", dir.FullName ?? "(null)"); - return Task.FromResult(result.ExitCode); - } - progress.WriteVerbose("Creating {0} ...", name); - LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(progress, writeVerbose: true, name); - progress.WriteMessage("Created {0}", name); - if (cleanup) - { - progress.WriteVerbose("Cleaning up..."); - HgRunner.Run($"hg checkout null", dir.FullName, 30, progress); - HgRunner.Run($"hg purge --no-confirm --exclude *.fwdata --exclude hgRunner.log", dir.FullName, 30, progress); - } - return Task.FromResult(0); - } -} diff --git a/src/SplitFwData/.editorconfig b/src/SplitFwData/.editorconfig deleted file mode 100644 index d4047b80..00000000 --- a/src/SplitFwData/.editorconfig +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2016-2021 SIL International -# This software is licensed under the MIT license (http://opensource.org/licenses/MIT) - -root = false - -# Defaults -[*] -indent_style = space -indent_size = tab -tab_width = 4 -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true -max_line_length = 98 - -[*.cs] -indent_style = space -tab_width = 4 - -# Settings Visual Studio uses for the generated files -[*.{csproj,resx,settings,targets,vcxproj*,vdproj,xml,yml,props,md}] -indent_style = space -indent_size = 2 - -# Generated file -[*.sln] -end_of_line = crlf diff --git a/src/SplitFwData/.gitattributes b/src/SplitFwData/.gitattributes deleted file mode 100644 index cf3363d0..00000000 --- a/src/SplitFwData/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -* text=auto whitespace=space-before-tab,tab-in-indent,blank-at-eol,tabwidth=4 diff --git a/src/SplitFwData/Program.cs b/src/SplitFwData/Program.cs deleted file mode 100644 index 110a6f09..00000000 --- a/src/SplitFwData/Program.cs +++ /dev/null @@ -1,76 +0,0 @@ -using Chorus.VcsDrivers.Mercurial; -using SIL.Progress; -using System.CommandLine; - -class Program -{ - static async Task Main(string[] args) - { - var rootCommand = new RootCommand("Make .fwdata file"); - - var verboseOption = new Option( - ["--verbose", "-v"], - "Display verbose output" - ); - rootCommand.AddGlobalOption(verboseOption); - - var quietOption = new Option( - ["--quiet", "-q"], - "Suppress all output (overrides --verbose if present)" - ); - rootCommand.AddGlobalOption(quietOption); - - var filename = new Argument( - "file", - "Name of .fwdata file to split" - ); - rootCommand.Add(filename); - - var cleanupOption = new Option( - ["--cleanup", "-c"], - "Delete .fwdata file after splitting" - ); - rootCommand.Add(cleanupOption); - - rootCommand.SetHandler(Run, filename, verboseOption, quietOption, cleanupOption); - - return await rootCommand.InvokeAsync(args); - } - - static FileInfo? LocateFwDataFile(string input) - { - if (Directory.Exists(input)) { - var dirInfo = new DirectoryInfo(input); - var fname = dirInfo.Name + ".fwdata"; - return new FileInfo(Path.Join(input, fname)); - } else if (File.Exists(input)) { - return new FileInfo(input); - } else if (File.Exists(input + ".fwdata")) { - return new FileInfo(input + ".fwdata"); - } else { - return null; - } - } - - static Task Run(string filename, bool verbose, bool quiet, bool cleanup) - { - IProgress progress = quiet ? new NullProgress() : new ConsoleProgress(); - progress.ShowVerbose = verbose; - var file = LocateFwDataFile(filename); - if (file == null || !file.Exists) { - progress.WriteError("Could not find {0}", filename); - return Task.FromResult(1); - } - string name = file.FullName; - progress.WriteVerbose("Splitting {0} ...", name); - LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(progress, writeVerbose: true, name); - progress.WriteMessage("Finished splitting {0}", name); - if (cleanup) - { - progress.WriteVerbose("Cleaning up..."); - var fwdataFile = new FileInfo(name); - if (fwdataFile.Exists) { fwdataFile.Delete(); progress.WriteVerbose("Deleted {0}", fwdataFile.FullName); } else { progress.WriteVerbose("File not found, so not deleting: {0}", fwdataFile.FullName); } - } - return Task.FromResult(0); - } -} diff --git a/src/SplitFwData/SplitFwData.csproj b/src/SplitFwData/SplitFwData.csproj deleted file mode 100644 index afbe05a3..00000000 --- a/src/SplitFwData/SplitFwData.csproj +++ /dev/null @@ -1,25 +0,0 @@ - - - - Exe - net8.0 - net8.0 - enable - enable - true - $(MSBuildProjectDirectory) - - - - - - - - - - - - - - - From e48052e7d46e8655a188c1775527b920e38e5663 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 12:30:09 +0700 Subject: [PATCH 54/73] Delete more now-unused S/R test code Now that we're not modifying existing LexBox projects but rather cloning them and creating new projects for each test, we don't need the entire "roll project back to previous revision" code. --- src/LfMerge.Core.Tests/E2E/BasicTests.cs | 74 --------------------- src/LfMerge.Core.Tests/SRTestBase.cs | 24 +------ src/LfMerge.Core.Tests/SRTestEnvironment.cs | 33 --------- 3 files changed, 1 insertion(+), 130 deletions(-) delete mode 100644 src/LfMerge.Core.Tests/E2E/BasicTests.cs diff --git a/src/LfMerge.Core.Tests/E2E/BasicTests.cs b/src/LfMerge.Core.Tests/E2E/BasicTests.cs deleted file mode 100644 index 3eac0968..00000000 --- a/src/LfMerge.Core.Tests/E2E/BasicTests.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using LfMerge.Core.DataConverters; -using LfMergeBridge.LfMergeModel; -using NUnit.Framework; -using SIL.Progress; - -namespace LfMerge.Core.Tests.E2E -{ - [TestFixture] - [Category("LongRunning")] - [Category("IntegrationTests")] - public class BasicTests : SRTestBase - { - [Test] - public async Task CheckProjectBackupDownloading() - { - await TestEnv.RollbackProjectToRev("sena-3", -1); // Should make no changes - // await TestEnv.RollbackProjectToRev("sena-3", -2); // Should remove one commit - } - - [Test] - [Property("projectCode", "sena-3")] // This will cause it to auto-reset the project afterwards - public async Task CheckProjectCloning() - { - using var sena3 = CloneFromLexbox("sena-3"); - var entryCount = LcmTestHelper.CountEntries(sena3); - Assert.That(entryCount, Is.EqualTo(1462)); - var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); - Assert.That(entry, Is.Not.Null); - var citationForm = entry.CitationForm.BestVernacularAlternative.Text; - Assert.That(citationForm, Is.EqualTo("ambuka")); - LcmTestHelper.SetVernacularText(sena3, entry.CitationForm, "something"); - CommitAndPush(sena3, "sena-3"); - - using var sena4 = CloneFromLexbox("sena-3", "sena-4"); - entry = LcmTestHelper.GetEntry(sena4, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); - citationForm = entry.CitationForm.BestVernacularAlternative.Text; - Assert.That(citationForm, Is.EqualTo("something")); - LcmTestHelper.UpdateVernacularText(sena4, entry.CitationForm, (s) => $"{s}XYZ"); - citationForm = entry.CitationForm.BestVernacularAlternative.Text; - Assert.That(citationForm, Is.EqualTo("somethingXYZ")); - CommitAndPush(sena4, "sena-3", "sena-4"); - } - - [Test] - [Property("projectCode", "sena-3")] - public async Task SendReceiveComments() - { - using var sena3 = CloneFromLexbox("sena-3"); - var entryCount = LcmTestHelper.CountEntries(sena3); - Assert.That(entryCount, Is.EqualTo(1462)); - var entry = LcmTestHelper.GetEntry(sena3, new Guid("5db6e79d-de66-4ec6-84c1-af3cd170f90d")); - Assert.That(entry, Is.Not.Null); - var comment = new LfComment { - Guid = new Guid("6864b40d-6ad8-4c42-9590-114c0b8495c8"), - Content = "Comment for test", - // Let's see if that's enough - }; - var comments = new List> { new KeyValuePair("6864b40d-6ad8-4c42-9590-114c0b8495c8", comment) }; - var result = ConvertMongoToLcmComments.CallLfMergeBridge(comments, out var lfmergeBridgeOutput, FwDataPathForProject("sena-3"), new NullProgress(), TestEnv.Logger); - Console.WriteLine(lfmergeBridgeOutput); - CommitAndPush(sena3, "sena-3"); - } - - [Test] - public async Task UploadNewProject() - { - var testCode = await CreateNewProjectFromSena3(); - Console.WriteLine($"Created {testCode}"); - } - } -} diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 52295b18..866f37b6 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -19,7 +19,6 @@ public class SRTestBase public TemporaryFolder TestDataFolder { get; set; } public TemporaryFolder LcmDataFolder { get; set; } public string Sena3ZipPath { get; set; } - private string TipRevToRestore { get; set; } = ""; private Guid? ProjectIdToDelete { get; set; } public SRTestEnvironment TestEnv { get; set; } @@ -70,44 +69,23 @@ public async Task TestSetup() TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath); TestEnv = new SRTestEnvironment(TempFolderForTest); await SRTestEnvironment.Login(); - await BackupRemoteProject(); } [TearDown] public async Task TestTeardown() { - await RestoreRemoteProject(); // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) { TempFolderForTest.Dispose(); if (ProjectIdToDelete is not null) { var projId = ProjectIdToDelete.Value; ProjectIdToDelete = null; + // Also leave LexBox project in place for post-test investigation, even though this might tend to clutter things up a little await SRTestEnvironment.DeleteLexBoxProject(projId); } } } - public async Task BackupRemoteProject() - { - var test = TestContext.CurrentContext.Test; - if (test.Properties.ContainsKey("projectCode")) { - var code = test.Properties.Get("projectCode") as string; - TipRevToRestore = await SRTestEnvironment.GetTipRev(code); - } else { - TipRevToRestore = ""; - } - } - - public async Task RestoreRemoteProject() - { - var test = TestContext.CurrentContext.Test; - if (!string.IsNullOrEmpty(TipRevToRestore) && test.Properties.ContainsKey("projectCode")) { - var code = test.Properties.Get("projectCode") as string; - await TestEnv.RollbackProjectToRev(code, TipRevToRestore); - } - } - public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode); public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata"); diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 12a3e024..aab40235 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -170,39 +170,6 @@ public static async Task DownloadProjectBackup(string code, string destZipPath) } } - public async Task RollbackProjectToRev(string code, string rev) - { - // Negative rev numbers will be interpreted as Mercurial does: -1 is the tip revision, -2 is one back from the tip, etc. - // I.e. rolling back to rev -2 will remove the most recent commit - if (rev == "-1") return; // Already at tip, nothing to do - var currentTip = await GetTipRev(code); - if (rev == currentTip) return; // Already at tip, nothing to do - var backupUrl = new Uri(LexboxUrl, $"api/project/backupProject/{code}"); - var result = await Http.GetAsync(backupUrl); - var zipStream = await result.Content.ReadAsStreamAsync(); - var projectDir = TempFolder.GetPathForNewTempFile(false); - ZipFile.ExtractToDirectory(zipStream, projectDir); - var clonedDir = TempFolder.GetPathForNewTempFile(false); - MercurialTestHelper.CloneRepoAtRev(projectDir, clonedDir, rev); - var zipPath = TempFolder.GetPathForNewTempFile(false); - ZipFile.CreateFromDirectory(clonedDir, zipPath); - await ResetAndUploadZip(code, zipPath); - } - - public Task RollbackProjectToRev(string code, int revnum) - { - return RollbackProjectToRev(code, revnum.ToString()); - } - - public record TipJson(string Node); - - public static async Task GetTipRev(string code) - { - var tipUrl = new Uri(LexboxUrl, $"/hg/{code}/file/tip?style=json"); - var result = await Http.GetFromJsonAsync(tipUrl); - return result.Node; - } - public static void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) { localCode ??= code; From 6d99535753262e62d01d61eff14b82f12348da7e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 12:48:07 +0700 Subject: [PATCH 55/73] Fix creating empty FLEx projects in LexBox Not used in the current E2E test, but will be used in an upcoming test --- src/LfMerge.Core.Tests/SRTestBase.cs | 5 +++-- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 866f37b6..883f884a 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -111,7 +111,7 @@ public FwProject CloneFromLexbox(string code, string? newCode = null) return new FwProject(settings, dirname); } - public async Task CreateNewFlexProject() + public async Task CreateEmptyFlexProjectInLexbox() { var randomGuid = Guid.NewGuid(); var testCode = $"sr-{randomGuid}"; @@ -122,7 +122,8 @@ public async Task CreateNewFlexProject() var result = await SRTestEnvironment.CreateLexBoxProject(testCode, randomGuid); Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); Assert.That(result.Id, Is.EqualTo(randomGuid)); - // TODO: Push that first commit to lexbox so the project is non-empty + var pushUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(testCode).AbsoluteUri; + MercurialTestHelper.HgPush(testPath, pushUrl); ProjectIdToDelete = result.Id; return testCode; } diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index aab40235..7bc96f72 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -171,12 +171,17 @@ public static async Task DownloadProjectBackup(string code, string destZipPath) } public static void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) + { + project.Cache.ActionHandlerAccessor.Commit(); + if (!project.IsDisposed) project.Dispose(); + CommitAndPush(code, baseDir, localCode, commitMsg); + } + + public static void CommitAndPush(string code, string baseDir, string? localCode = null, string? commitMsg = null) { localCode ??= code; var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; - project.Cache.ActionHandlerAccessor.Commit(); - if (!project.IsDisposed) project.Dispose(); commitMsg ??= "Auto-commit"; var projectDir = Path.Combine(baseDir, "webwork", localCode); var fwdataPath = Path.Join(projectDir, $"{localCode}.fwdata"); From 8db472140f11276375e1b6fdf1fdbcd94c8c4f2f Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 12:50:42 +0700 Subject: [PATCH 56/73] Remove unused using statements --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 8 -------- src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 10 ---------- src/LfMerge.Core.Tests/LcmTestHelper.cs | 7 ------- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 4 ---- 4 files changed, 29 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index b4530614..fe5f65e5 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -1,20 +1,12 @@ using System; -using System.Collections.Generic; using System.IO; -using System.Linq; using System.Threading.Tasks; using Autofac; using LfMerge.Core.Actions; -using LfMerge.Core.DataConverters; using LfMerge.Core.FieldWorks; -using LfMerge.Core.LanguageForge.Model; using LfMerge.Core.MongoConnector; -using LfMerge.Core.Settings; -using LfMergeBridge.LfMergeModel; using NUnit.Framework; using SIL.LCModel; -using SIL.Progress; -using SIL.TestUtilities; namespace LfMerge.Core.Tests.E2E { diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index 689f0bde..e343ef15 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -1,16 +1,6 @@ using System; -using System.Collections.Generic; using System.Threading.Tasks; -using LfMerge.Core.DataConverters; -using LfMergeBridge.LfMergeModel; using NUnit.Framework; -using SIL.Progress; - -using System.IO; -using System.Linq; -using LfMerge.Core.Actions; -using LfMerge.Core.LanguageForge.Model; -using SIL.LCModel; using System.Text.RegularExpressions; namespace LfMerge.Core.Tests.E2E diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index d05e3a48..4ef80dd0 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -1,15 +1,8 @@ using System; using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Net.Http.Json; -using System.Threading.Tasks; using LfMerge.Core.FieldWorks; using SIL.LCModel; using SIL.LCModel.Infrastructure; -using SIL.PlatformUtilities; -using SIL.Progress; namespace LfMerge.Core.Tests { diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 7bc96f72..dd5d4f84 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -1,20 +1,16 @@ using System; using System.IO; -using System.IO.Compression; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Json; using System.Threading.Tasks; using GraphQL; -using GraphQL.Client.Abstractions; using GraphQL.Client.Http; using GraphQL.Client.Serializer.SystemTextJson; using LfMerge.Core.FieldWorks; using LfMerge.Core.Logging; -using LfMerge.Core.Settings; using NUnit.Framework; -using SIL.LCModel; using SIL.TestUtilities; using TusDotNetClient; From 779358e507189c04e9c9d5c04cd890257a32dd60 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 13:37:57 +0700 Subject: [PATCH 57/73] Demonstrate another hgweb race condition Run `dotnet test --filter E2E_` to see `hg clone` return 404 on a newly-created project. Won't happen every single time, but will happen often enough that it's likely to cause problems. Solution is going to be to make CloneRepoFromLexbox more resilient against temporary 404s. --- src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index e343ef15..bd0dccc4 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -10,6 +10,22 @@ namespace LfMerge.Core.Tests.E2E [Category("IntegrationTests")] public class TryOutE2ETests : E2ETestBase { + // This test will often trigger a race condition in LexBox that causes the *next* test to fail + // when `hg clone` returns 404. This is because of hgweb's directory cache, which only refreshes + // if it hasn't been refreshed more than N seconds ago (default 20, LexBox currently uses 5). + // Which means that even though CreateLfProjectFromSena3 has created the LexBox project, LexBox's + // copy of hgweb doesn't see it yet so it can't be cloned. + // + // The solution will be to adjust CloneRepoFromLexbox to take a parameter that is the number of + // seconds to wait for the project to become visible, and retry 404's until that much time has elapsed. + // Then only throw an exception if hgweb is still returning 404 after its dir cache should be refreshed. + [Test] + public async Task E2E_CheckFwProjectCreation() + { + var code = await CreateEmptyFlexProjectInLexbox(); + Console.WriteLine($"Created new project {code}"); + } + [Test] public async Task E2E_LFDataChangedLDDataChanged_LFWins() { From 80d45bba71aea1b33bd9023e854aed37318df97f Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 19 Aug 2024 13:59:30 +0700 Subject: [PATCH 58/73] Make CloneRepoFromLexbox resilient to temp 404s --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 2 +- src/LfMerge.Core.Tests/MercurialTestHelper.cs | 5 ++-- src/LfMerge.Core.Tests/SRTestBase.cs | 26 ++++++++++++++++--- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index fe5f65e5..0fafbb86 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -33,7 +33,7 @@ public void Setup() public async Task CreateLfProjectFromSena3() { var projCode = await CreateNewProjectFromSena3(); - var projPath = CloneRepoFromLexbox(projCode); + var projPath = CloneRepoFromLexbox(projCode, waitTime:TimeSpan.FromSeconds(5)); MercurialTestHelper.ChangeBranch(projPath, "tip"); var fwdataPath = Path.Combine(projPath, $"{projCode}.fwdata"); LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); diff --git a/src/LfMerge.Core.Tests/MercurialTestHelper.cs b/src/LfMerge.Core.Tests/MercurialTestHelper.cs index 61d03dcc..148bdf31 100644 --- a/src/LfMerge.Core.Tests/MercurialTestHelper.cs +++ b/src/LfMerge.Core.Tests/MercurialTestHelper.cs @@ -19,9 +19,10 @@ public static class MercurialTestHelper private static string RunHgCommand(string repoPath, string args) { var result = CommandLineRunner.Run(HgCommand, args, repoPath, 120, new NullProgress()); - Assert.That(result.ExitCode, Is.EqualTo(0), + if (result.ExitCode == 0) return result.StandardOutput; + throw new System.Exception( $"hg {args} failed.\nStdOut: {result.StandardOutput}\nStdErr: {result.StandardError}"); - return result.StandardOutput; + } public static void InitializeHgRepo(string projectFolderPath) diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 883f884a..68ad9076 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -89,18 +89,36 @@ public async Task TestTeardown() public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode); public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata"); - public string CloneRepoFromLexbox(string code, string? newCode = null) + public string CloneRepoFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) { var projUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(code); newCode ??= code; var dest = TestFolderForProject(newCode); - MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); + if (waitTime is null) { + MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); + } else { + var start = DateTime.UtcNow; + var success = false; + while (!success) { + try { + MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); + } catch { + if (DateTime.UtcNow > start + waitTime) { + throw; // Give up + } + System.Threading.Thread.Sleep(250); + continue; + } + // If we got this far, no exception so we succeeded + success = true; + } + } return dest; } - public FwProject CloneFromLexbox(string code, string? newCode = null) + public FwProject CloneFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) { - var dest = CloneRepoFromLexbox(code, newCode); + var dest = CloneRepoFromLexbox(code, newCode, waitTime); var dirInfo = new DirectoryInfo(dest); if (!dirInfo.Exists) throw new InvalidOperationException($"Failed to clone {code} from lexbox, cannot create FwProject"); var dirname = dirInfo.Name; From 3749cbedae4e48324ca3becb8720253b76508d5b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 21 Aug 2024 12:02:17 +0700 Subject: [PATCH 59/73] Make most SRTestEnv methods non-static again As requested in code review --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 4 +- src/LfMerge.Core.Tests/SRTestBase.cs | 17 +++---- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 49 +++++++++++++-------- 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 0fafbb86..d585ab61 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -36,12 +36,12 @@ public async Task CreateLfProjectFromSena3() var projPath = CloneRepoFromLexbox(projCode, waitTime:TimeSpan.FromSeconds(5)); MercurialTestHelper.ChangeBranch(projPath, "tip"); var fwdataPath = Path.Combine(projPath, $"{projCode}.fwdata"); - LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath); // Do an initial clone from LexBox to the mock LF var lfProject = LanguageForgeProject.Create(projCode, TestEnv.Settings); lfProject.IsInitialClone = true; - var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, SRTestEnvironment.NullLogger, _mongoConnection, _recordFactory); + var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, TestEnv.NullLogger, _mongoConnection, _recordFactory); transferLcmToMongo.Run(lfProject); return lfProject; diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs index 68ad9076..02ddc02c 100644 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ b/src/LfMerge.Core.Tests/SRTestBase.cs @@ -48,8 +48,9 @@ public async Task FixtureSetup() // Ensure sena-3.zip is available to all tests as a starting point Sena3ZipPath = Path.Combine(TestDataFolder.Path, "sena-3.zip"); if (!File.Exists(Sena3ZipPath)) { - await SRTestEnvironment.Login(); - await SRTestEnvironment.DownloadProjectBackup("sena-3", Sena3ZipPath); + var testEnv = new SRTestEnvironment(TempFolderForTest); + await testEnv.Login(); + await testEnv.DownloadProjectBackup("sena-3", Sena3ZipPath); } } @@ -68,7 +69,7 @@ public async Task TestSetup() { TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath); TestEnv = new SRTestEnvironment(TempFolderForTest); - await SRTestEnvironment.Login(); + await TestEnv.Login(); } [TearDown] @@ -81,7 +82,7 @@ public async Task TestTeardown() var projId = ProjectIdToDelete.Value; ProjectIdToDelete = null; // Also leave LexBox project in place for post-test investigation, even though this might tend to clutter things up a little - await SRTestEnvironment.DeleteLexBoxProject(projId); + await TestEnv.DeleteLexBoxProject(projId); } } } @@ -124,7 +125,7 @@ public FwProject CloneFromLexbox(string code, string? newCode = null, TimeSpan? var dirname = dirInfo.Name; var fwdataPath = Path.Join(dest, $"{dirname}.fwdata"); MercurialTestHelper.ChangeBranch(dest, "tip"); - LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(SRTestEnvironment.NullProgress, false, fwdataPath); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath); var settings = new LfMergeSettingsDouble(TempFolderForTest.Path); return new FwProject(settings, dirname); } @@ -137,7 +138,7 @@ public async Task CreateEmptyFlexProjectInLexbox() MercurialTestHelper.InitializeHgRepo(testPath); MercurialTestHelper.CreateFlexRepo(testPath); // Now create project in LexBox - var result = await SRTestEnvironment.CreateLexBoxProject(testCode, randomGuid); + var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); Assert.That(result.Id, Is.EqualTo(randomGuid)); var pushUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(testCode).AbsoluteUri; @@ -152,7 +153,7 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) var testCode = $"sr-{randomGuid}"; var testPath = TestFolderForProject(testCode); // Now create project in LexBox - var result = await SRTestEnvironment.CreateLexBoxProject(testCode, randomGuid); + var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); Assert.That(result.Id, Is.EqualTo(randomGuid)); await TestEnv.ResetAndUploadZip(testCode, origZipPath); @@ -164,7 +165,7 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null) { - SRTestEnvironment.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); + TestEnv.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); } } } diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index dd5d4f84..d6440312 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -28,34 +28,45 @@ public class SRTestEnvironment : TestEnvironment public static readonly string LexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; public static readonly Uri LexboxUrl = new($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); public static readonly Uri LexboxUrlBasicAuth = new($"{LexboxProtocol}://{WebUtility.UrlEncode(LexboxUsername)}:{WebUtility.UrlEncode(LexboxPassword)}@{LexboxHostname}:{LexboxPort}"); - public static readonly CookieContainer Cookies = new(); - private static readonly HttpClientHandler Handler = new() { CookieContainer = Cookies }; - private static readonly Lazy LazyHttp = new(() => new HttpClient(Handler)); - public static HttpClient Http => LazyHttp.Value; - public static readonly SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); - public static readonly ILogger NullLogger = new NullLogger(); - private static bool AlreadyLoggedIn = false; + public static Cookie AdminLoginCookie { get; set; } + public readonly CookieContainer Cookies = new(); + private HttpClientHandler Handler { get; init; } + private Lazy LazyHttp { get; init; } + public HttpClient Http => LazyHttp.Value; + public readonly SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); + public readonly ILogger NullLogger = new NullLogger(); + private bool AlreadyLoggedIn = false; private TemporaryFolder TempFolder { get; init; } - private static readonly Lazy LazyGqlClient = new(() => new GraphQLHttpClient(new Uri(LexboxUrl, "/api/graphql"), new SystemTextJsonSerializer(), Http)); - public static GraphQLHttpClient GqlClient => LazyGqlClient.Value; + private Lazy LazyGqlClient { get; init; } + public GraphQLHttpClient GqlClient => LazyGqlClient.Value; public SRTestEnvironment(TemporaryFolder? tempFolder = null) : base(true, true, true, tempFolder ?? new TemporaryFolder(TestName + Path.GetRandomFileName())) { + Handler = new() { CookieContainer = Cookies }; + LazyHttp = new(() => new HttpClient(Handler)); + LazyGqlClient = new(() => new GraphQLHttpClient(new Uri(LexboxUrl, "/api/graphql"), new SystemTextJsonSerializer(), Http)); TempFolder = _languageForgeServerFolder; // Better name for what E2E tests use it for Settings.CommitWhenDone = true; // For SR tests specifically, we *do* want changes to .fwdata files to be persisted } - public static Task Login() + public async Task Login() { - return LoginAs(LexboxUsername, LexboxPassword); + if (AlreadyLoggedIn) return; + if (AdminLoginCookie is null) { + await LoginAs(LexboxUsername, LexboxPassword); + } else { + Cookies.Add(AdminLoginCookie); + AlreadyLoggedIn = true; + } } - public static async Task LoginAs(string lexboxUsername, string lexboxPassword) + public async Task LoginAs(string lexboxUsername, string lexboxPassword) { if (AlreadyLoggedIn) return; var loginResult = await Http.PostAsync(new Uri(LexboxUrl, "api/login"), JsonContent.Create(new { EmailOrUsername=lexboxUsername, Password=lexboxPassword })); var cookies = Cookies.GetCookies(LexboxUrl); + AdminLoginCookie = cookies[".LexBoxAuth"]; AlreadyLoggedIn = true; // Http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", Jwt); // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. @@ -64,7 +75,7 @@ public static async Task LoginAs(string lexboxUsername, string lexboxPassword) public static Uri LexboxUrlForProject(string code) => new Uri(LexboxUrl, $"hg/{code}"); public static Uri LexboxUrlForProjectWithAuth(string code) => new Uri(LexboxUrlBasicAuth, $"hg/{code}"); - public static async Task CreateLexBoxProject(string code, Guid? projId = null, string? name = null, string? description = null, Guid? managerId = null, Guid? orgId = null) + public async Task CreateLexBoxProject(string code, Guid? projId = null, string? name = null, string? description = null, Guid? managerId = null, Guid? orgId = null) { projId ??= Guid.NewGuid(); name ??= code; @@ -94,7 +105,7 @@ ... on DbError { return response.Data.CreateProject.CreateProjectResponse; } - public static async Task DeleteLexBoxProject(Guid projectId) + public async Task DeleteLexBoxProject(Guid projectId) { var mutation = """ mutation SoftDeleteProject($input: SoftDeleteProjectInput!) { @@ -134,7 +145,7 @@ public async Task ResetAndUploadZip(string code, string zipPath) await UploadZip(code, zipPath); } - public static async Task ResetToEmpty(string code) + public async Task ResetToEmpty(string code) { var resetUrl = new Uri(LexboxUrl, $"api/project/resetProject/{code}"); await Http.PostAsync(resetUrl, null); @@ -142,7 +153,7 @@ public static async Task ResetToEmpty(string code) await Http.PostAsync(finishResetUrl, null); } - public static async Task UploadZip(string code, string zipPath) + public async Task UploadZip(string code, string zipPath) { var sourceUrl = new Uri(LexboxUrl, $"/api/project/upload-zip/{code}"); var file = new FileInfo(zipPath); @@ -155,7 +166,7 @@ public static async Task UploadZip(string code, string zipPath) await client.UploadAsync(fileUrl, file); } - public static async Task DownloadProjectBackup(string code, string destZipPath) + public async Task DownloadProjectBackup(string code, string destZipPath) { var backupUrl = new Uri(LexboxUrl, $"api/project/backupProject/{code}"); var result = await Http.GetAsync(backupUrl); @@ -166,14 +177,14 @@ public static async Task DownloadProjectBackup(string code, string destZipPath) } } - public static void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) + public void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) { project.Cache.ActionHandlerAccessor.Commit(); if (!project.IsDisposed) project.Dispose(); CommitAndPush(code, baseDir, localCode, commitMsg); } - public static void CommitAndPush(string code, string baseDir, string? localCode = null, string? commitMsg = null) + public void CommitAndPush(string code, string baseDir, string? localCode = null, string? commitMsg = null) { localCode ??= code; var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); From bea171310fb91a2af61d2d7b1d7e588a4f4c78ff Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 21 Aug 2024 12:17:12 +0700 Subject: [PATCH 60/73] Allow overriding repo URL in settings, not env vars This ensures that the override will only affect the test it's set in, rather than potentially affecting other tests if we forget to clear the environment variable in a `finally` block. --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 13 +++---------- src/LfMerge.Core.Tests/TestDoubles.cs | 7 +++++-- src/LfMerge.Core/Settings/LfMergeSettings.cs | 6 +++++- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index d585ab61..191ec613 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -49,16 +49,9 @@ public async Task CreateLfProjectFromSena3() public void SendReceiveToLexbox(LanguageForgeProject lfProject) { - // ChorusHelperDouble.GetSyncUri assumes presence of a LanguageDepotMock, but here we want a real LexBox instance so we override it via environment variable - var saveEnv = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri); - try { - var lexboxRepoUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri; - Environment.SetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri, lexboxRepoUrl); - var syncAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger); - syncAction.Run(lfProject); - } finally { - Environment.SetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri, saveEnv); - } + TestEnv.Settings.LanguageDepotRepoUri = SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri; + var syncAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger); + syncAction.Run(lfProject); } public (string, DateTime, DateTime) UpdateFwGloss(FwProject project, Guid entryId, Func textConverter) diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs index 827209dd..86eec1fb 100644 --- a/src/LfMerge.Core.Tests/TestDoubles.cs +++ b/src/LfMerge.Core.Tests/TestDoubles.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Linq.Expressions; +using Autofac; using Bugsnag.Payload; using LfMergeBridge.LfMergeModel; using IniParser.Model; @@ -417,8 +418,10 @@ class ChorusHelperDouble: ChorusHelper { public override string GetSyncUri(ILfProject project) { - var envOverride = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri); - if (envOverride != null) return envOverride; + var settings = MainClass.Container.Resolve(); + // Allow tests to override LanguageDepotRepoUri if necessary (e.g., the E2E tests which need ChorusHelperDouble but use a "real" LexBox instance) + if (!string.IsNullOrEmpty(settings.LanguageDepotRepoUri)) + return settings.LanguageDepotRepoUri; var server = LanguageDepotMock.Server; return server != null && server.IsStarted ? server.Url : LanguageDepotMock.ProjectFolderPath; } diff --git a/src/LfMerge.Core/Settings/LfMergeSettings.cs b/src/LfMerge.Core/Settings/LfMergeSettings.cs index ac399145..9f253ce3 100644 --- a/src/LfMerge.Core/Settings/LfMergeSettings.cs +++ b/src/LfMerge.Core/Settings/LfMergeSettings.cs @@ -64,7 +64,11 @@ public bool VerboseProgress { } } - public string LanguageDepotRepoUri => Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri) ?? DefaultLfMergeSettings.LanguageDepotRepoUri; + private string _languageDepotRepoUri; + public string LanguageDepotRepoUri { + get => _languageDepotRepoUri ?? Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri) ?? DefaultLfMergeSettings.LanguageDepotRepoUri; + set => _languageDepotRepoUri = value; + } // Settings calculated at runtime from sources other than environment variables From 946468dc0beaec14aeea0b73196184ed8103466b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 21 Aug 2024 12:19:01 +0700 Subject: [PATCH 61/73] Don't hardcode GUID in LexBox tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than assume that the test will be running against the sena-3 project, let's make the test grab an arbitrary entry (the first one that the LexEntryRepository knows about) and use it. Since the entry whose GUID we hardcoded was actually the first LexEntry sorted by GUID, this currently has no effect as we're actually testing the same entry we were before — but it will allow us to change the test later. --- src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 2 +- src/LfMerge.Core.Tests/LcmTestHelper.cs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index bd0dccc4..1cf06f3f 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -36,7 +36,7 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() var fwProject = CloneFromLexbox(lfProject.ProjectCode, fwProjectCode); // Modify FW data first, then push to Lexbox - Guid entryId = Guid.Parse("0006f482-a078-4cef-9c5a-8bd35b53cf72"); + Guid entryId = LcmTestHelper.GetFirstEntry(fwProject).Guid; var (unchangedGloss, origFwDateModified, fwDateModified) = UpdateFwGloss(fwProject, entryId, text => text + " - changed in FW"); CommitAndPush(fwProject, lfProject.ProjectCode, fwProjectCode, "Modified gloss in FW"); diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs index 4ef80dd0..d8dc1215 100644 --- a/src/LfMerge.Core.Tests/LcmTestHelper.cs +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using LfMerge.Core.FieldWorks; using SIL.LCModel; using SIL.LCModel.Infrastructure; @@ -25,6 +26,12 @@ public static ILexEntry GetEntry(FwProject project, Guid guid) return repo.GetObject(guid); } + public static ILexEntry GetFirstEntry(FwProject project) + { + var repo = project?.ServiceLocator?.GetInstance(); + return repo.AllInstances().First(); + } + public static string? GetVernacularText(IMultiUnicode field) { return field.BestVernacularAlternative?.Text; From 5c7059bbca894076bc4bffb711a4ab7856a3248a Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 21 Aug 2024 13:12:50 +0700 Subject: [PATCH 62/73] Consolidate E2E base classes into one base class Since we're not going to be creating any other test fixtures derived from SRTestBase, there's no point in having two base classes; it has become over-engineering now that we've decided on just one E2E test. --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 157 +++++++++++++++++++- src/LfMerge.Core.Tests/SRTestBase.cs | 171 ---------------------- 2 files changed, 155 insertions(+), 173 deletions(-) delete mode 100644 src/LfMerge.Core.Tests/SRTestBase.cs diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 191ec613..6e1d04c5 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -6,21 +6,78 @@ using LfMerge.Core.FieldWorks; using LfMerge.Core.MongoConnector; using NUnit.Framework; +using NUnit.Framework.Interfaces; using SIL.LCModel; +using SIL.TestUtilities; namespace LfMerge.Core.Tests.E2E { [TestFixture] [Category("LongRunning")] [Category("IntegrationTests")] - public class E2ETestBase : SRTestBase + public class E2ETestBase { + public LfMerge.Core.Logging.ILogger Logger => MainClass.Logger; + public TemporaryFolder TempFolderForClass { get; set; } + public TemporaryFolder TempFolderForTest { get; set; } + public TemporaryFolder TestDataFolder { get; set; } + public TemporaryFolder LcmDataFolder { get; set; } + public string Sena3ZipPath { get; set; } + private Guid? ProjectIdToDelete { get; set; } + public SRTestEnvironment TestEnv { get; set; } + public MongoConnectionDouble _mongoConnection; public MongoProjectRecordFactory _recordFactory; + public E2ETestBase() + { + } + + private static string TestName => TestContext.CurrentContext.Test.Name; + private static string TestNameForPath => string.Concat(TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars + + [OneTimeSetUp] + public async Task FixtureSetup() + { + // Ensure we don't delete top-level /tmp/LfMergeSRTests folder and data subfolder if they already exist + var tempPath = Path.Combine(Path.GetTempPath(), "LfMergeSRTests"); + var rootTempFolder = Directory.Exists(tempPath) ? TemporaryFolder.TrackExisting(tempPath) : new TemporaryFolder(tempPath); + var testDataPath = Path.Combine(tempPath, "data"); + TestDataFolder = Directory.Exists(testDataPath) ? TemporaryFolder.TrackExisting(testDataPath) : new TemporaryFolder(testDataPath); + var lcmDataPath = Path.Combine(tempPath, "lcm-common"); + LcmDataFolder = Directory.Exists(lcmDataPath) ? TemporaryFolder.TrackExisting(lcmDataPath) : new TemporaryFolder(lcmDataPath); + Environment.SetEnvironmentVariable("FW_CommonAppData", LcmDataFolder.Path); + + // But the folder for this specific test suite should be deleted if it already exists + var derivedClassName = this.GetType().Name; + TempFolderForClass = new TemporaryFolder(rootTempFolder, derivedClassName); + + // Ensure sena-3.zip is available to all tests as a starting point + Sena3ZipPath = Path.Combine(TestDataFolder.Path, "sena-3.zip"); + if (!File.Exists(Sena3ZipPath)) { + var testEnv = new SRTestEnvironment(TempFolderForTest); + await testEnv.Login(); + await testEnv.DownloadProjectBackup("sena-3", Sena3ZipPath); + } + } + + [OneTimeTearDown] + public void FixtureTeardown() + { + Environment.SetEnvironmentVariable("FW_CommonAppData", null); + var result = TestContext.CurrentContext.Result; + var nonSuccess = result.FailCount + result.InconclusiveCount + result.WarningCount; + // Only delete class temp folder if we passed or skipped all tests + if (nonSuccess == 0) TempFolderForClass.Dispose(); + } + [SetUp] - public void Setup() + public async Task TestSetup() { + TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath); + TestEnv = new SRTestEnvironment(TempFolderForTest); + await TestEnv.Login(); + MagicStrings.SetMinimalModelVersion(LcmCache.ModelVersion); _mongoConnection = MainClass.Container.Resolve() as MongoConnectionDouble; if (_mongoConnection == null) @@ -30,6 +87,102 @@ public void Setup() throw new AssertionException("E2E tests need a mock MongoProjectRecordFactory in order to work."); } + [TearDown] + public async Task TestTeardown() + { + // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation + if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) { + TempFolderForTest.Dispose(); + if (ProjectIdToDelete is not null) { + var projId = ProjectIdToDelete.Value; + ProjectIdToDelete = null; + // Also leave LexBox project in place for post-test investigation, even though this might tend to clutter things up a little + await TestEnv.DeleteLexBoxProject(projId); + } + } + } + + public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode); + public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata"); + + public string CloneRepoFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) + { + var projUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(code); + newCode ??= code; + var dest = TestFolderForProject(newCode); + if (waitTime is null) { + MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); + } else { + var start = DateTime.UtcNow; + var success = false; + while (!success) { + try { + MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); + } catch { + if (DateTime.UtcNow > start + waitTime) { + throw; // Give up + } + System.Threading.Thread.Sleep(250); + continue; + } + // If we got this far, no exception so we succeeded + success = true; + } + } + return dest; + } + + public FwProject CloneFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) + { + var dest = CloneRepoFromLexbox(code, newCode, waitTime); + var dirInfo = new DirectoryInfo(dest); + if (!dirInfo.Exists) throw new InvalidOperationException($"Failed to clone {code} from lexbox, cannot create FwProject"); + var dirname = dirInfo.Name; + var fwdataPath = Path.Join(dest, $"{dirname}.fwdata"); + MercurialTestHelper.ChangeBranch(dest, "tip"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath); + var settings = new LfMergeSettingsDouble(TempFolderForTest.Path); + return new FwProject(settings, dirname); + } + + public async Task CreateEmptyFlexProjectInLexbox() + { + var randomGuid = Guid.NewGuid(); + var testCode = $"sr-{randomGuid}"; + var testPath = TestFolderForProject(testCode); + MercurialTestHelper.InitializeHgRepo(testPath); + MercurialTestHelper.CreateFlexRepo(testPath); + // Now create project in LexBox + var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); + Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); + Assert.That(result.Id, Is.EqualTo(randomGuid)); + var pushUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(testCode).AbsoluteUri; + MercurialTestHelper.HgPush(testPath, pushUrl); + ProjectIdToDelete = result.Id; + return testCode; + } + + public async Task CreateNewProjectFromTemplate(string origZipPath) + { + var randomGuid = Guid.NewGuid(); + var testCode = $"sr-{randomGuid}"; + var testPath = TestFolderForProject(testCode); + // Now create project in LexBox + var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); + Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); + Assert.That(result.Id, Is.EqualTo(randomGuid)); + await TestEnv.ResetAndUploadZip(testCode, origZipPath); + ProjectIdToDelete = result.Id; + return testCode; + } + + public Task CreateNewProjectFromSena3() => CreateNewProjectFromTemplate(Sena3ZipPath); + + public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null) + { + TestEnv.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); + } + public async Task CreateLfProjectFromSena3() { var projCode = await CreateNewProjectFromSena3(); diff --git a/src/LfMerge.Core.Tests/SRTestBase.cs b/src/LfMerge.Core.Tests/SRTestBase.cs deleted file mode 100644 index 02ddc02c..00000000 --- a/src/LfMerge.Core.Tests/SRTestBase.cs +++ /dev/null @@ -1,171 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using LfMerge.Core.FieldWorks; -using NUnit.Framework; -using NUnit.Framework.Interfaces; -using SIL.TestUtilities; - -namespace LfMerge.Core.Tests -{ - /// - /// Test base class for end-to-end testing, i.e. Send/Receive with a real LexBox instance - /// - public class SRTestBase - { - public LfMerge.Core.Logging.ILogger Logger => MainClass.Logger; - public TemporaryFolder TempFolderForClass { get; set; } - public TemporaryFolder TempFolderForTest { get; set; } - public TemporaryFolder TestDataFolder { get; set; } - public TemporaryFolder LcmDataFolder { get; set; } - public string Sena3ZipPath { get; set; } - private Guid? ProjectIdToDelete { get; set; } - public SRTestEnvironment TestEnv { get; set; } - - public SRTestBase() - { - } - - private static string TestName => TestContext.CurrentContext.Test.Name; - private static string TestNameForPath => string.Concat(TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars - - [OneTimeSetUp] - public async Task FixtureSetup() - { - // Ensure we don't delete top-level /tmp/LfMergeSRTests folder and data subfolder if they already exist - var tempPath = Path.Combine(Path.GetTempPath(), "LfMergeSRTests"); - var rootTempFolder = Directory.Exists(tempPath) ? TemporaryFolder.TrackExisting(tempPath) : new TemporaryFolder(tempPath); - var testDataPath = Path.Combine(tempPath, "data"); - TestDataFolder = Directory.Exists(testDataPath) ? TemporaryFolder.TrackExisting(testDataPath) : new TemporaryFolder(testDataPath); - var lcmDataPath = Path.Combine(tempPath, "lcm-common"); - LcmDataFolder = Directory.Exists(lcmDataPath) ? TemporaryFolder.TrackExisting(lcmDataPath) : new TemporaryFolder(lcmDataPath); - Environment.SetEnvironmentVariable("FW_CommonAppData", LcmDataFolder.Path); - - // But the folder for this specific test suite should be deleted if it already exists - var derivedClassName = this.GetType().Name; - TempFolderForClass = new TemporaryFolder(rootTempFolder, derivedClassName); - - // Ensure sena-3.zip is available to all tests as a starting point - Sena3ZipPath = Path.Combine(TestDataFolder.Path, "sena-3.zip"); - if (!File.Exists(Sena3ZipPath)) { - var testEnv = new SRTestEnvironment(TempFolderForTest); - await testEnv.Login(); - await testEnv.DownloadProjectBackup("sena-3", Sena3ZipPath); - } - } - - [OneTimeTearDown] - public void FixtureTeardown() - { - Environment.SetEnvironmentVariable("FW_CommonAppData", null); - var result = TestContext.CurrentContext.Result; - var nonSuccess = result.FailCount + result.InconclusiveCount + result.WarningCount; - // Only delete class temp folder if we passed or skipped all tests - if (nonSuccess == 0) TempFolderForClass.Dispose(); - } - - [SetUp] - public async Task TestSetup() - { - TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath); - TestEnv = new SRTestEnvironment(TempFolderForTest); - await TestEnv.Login(); - } - - [TearDown] - public async Task TestTeardown() - { - // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation - if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) { - TempFolderForTest.Dispose(); - if (ProjectIdToDelete is not null) { - var projId = ProjectIdToDelete.Value; - ProjectIdToDelete = null; - // Also leave LexBox project in place for post-test investigation, even though this might tend to clutter things up a little - await TestEnv.DeleteLexBoxProject(projId); - } - } - } - - public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode); - public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata"); - - public string CloneRepoFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) - { - var projUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(code); - newCode ??= code; - var dest = TestFolderForProject(newCode); - if (waitTime is null) { - MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); - } else { - var start = DateTime.UtcNow; - var success = false; - while (!success) { - try { - MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); - } catch { - if (DateTime.UtcNow > start + waitTime) { - throw; // Give up - } - System.Threading.Thread.Sleep(250); - continue; - } - // If we got this far, no exception so we succeeded - success = true; - } - } - return dest; - } - - public FwProject CloneFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) - { - var dest = CloneRepoFromLexbox(code, newCode, waitTime); - var dirInfo = new DirectoryInfo(dest); - if (!dirInfo.Exists) throw new InvalidOperationException($"Failed to clone {code} from lexbox, cannot create FwProject"); - var dirname = dirInfo.Name; - var fwdataPath = Path.Join(dest, $"{dirname}.fwdata"); - MercurialTestHelper.ChangeBranch(dest, "tip"); - LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath); - var settings = new LfMergeSettingsDouble(TempFolderForTest.Path); - return new FwProject(settings, dirname); - } - - public async Task CreateEmptyFlexProjectInLexbox() - { - var randomGuid = Guid.NewGuid(); - var testCode = $"sr-{randomGuid}"; - var testPath = TestFolderForProject(testCode); - MercurialTestHelper.InitializeHgRepo(testPath); - MercurialTestHelper.CreateFlexRepo(testPath); - // Now create project in LexBox - var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); - Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); - Assert.That(result.Id, Is.EqualTo(randomGuid)); - var pushUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(testCode).AbsoluteUri; - MercurialTestHelper.HgPush(testPath, pushUrl); - ProjectIdToDelete = result.Id; - return testCode; - } - - public async Task CreateNewProjectFromTemplate(string origZipPath) - { - var randomGuid = Guid.NewGuid(); - var testCode = $"sr-{randomGuid}"; - var testPath = TestFolderForProject(testCode); - // Now create project in LexBox - var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); - Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); - Assert.That(result.Id, Is.EqualTo(randomGuid)); - await TestEnv.ResetAndUploadZip(testCode, origZipPath); - ProjectIdToDelete = result.Id; - return testCode; - } - - public Task CreateNewProjectFromSena3() => CreateNewProjectFromTemplate(Sena3ZipPath); - - public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null) - { - TestEnv.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); - } - } -} From 129aaa2c5949eac37df6974e9565bfa6f2251f9f Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 21 Aug 2024 13:13:57 +0700 Subject: [PATCH 63/73] Ignore E2E tests if LexBox isn't available Instead of the tests failing if LexBox isn't available, this will just mark them as ignored. --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 6 ++++++ src/LfMerge.Core.Tests/SRTestEnvironment.cs | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 6e1d04c5..8d328369 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -56,6 +56,9 @@ public async Task FixtureSetup() Sena3ZipPath = Path.Combine(TestDataFolder.Path, "sena-3.zip"); if (!File.Exists(Sena3ZipPath)) { var testEnv = new SRTestEnvironment(TempFolderForTest); + if (!await testEnv.IsLexBoxAvailable()) { + Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox."); + } await testEnv.Login(); await testEnv.DownloadProjectBackup("sena-3", Sena3ZipPath); } @@ -76,6 +79,9 @@ public async Task TestSetup() { TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath); TestEnv = new SRTestEnvironment(TempFolderForTest); + if (!await TestEnv.IsLexBoxAvailable()) { + Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox."); + } await TestEnv.Login(); MagicStrings.SetMinimalModelVersion(LcmCache.ModelVersion); diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index d6440312..2b200f32 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -72,6 +72,12 @@ public async Task LoginAs(string lexboxUsername, string lexboxPassword) // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. } + public async Task IsLexBoxAvailable() + { + var httpResponse = await Http.GetAsync(new Uri(LexboxUrl, "api/healthz")); + return httpResponse.IsSuccessStatusCode && httpResponse.Headers.TryGetValues("lexbox-version", out var _ignore); + } + public static Uri LexboxUrlForProject(string code) => new Uri(LexboxUrl, $"hg/{code}"); public static Uri LexboxUrlForProjectWithAuth(string code) => new Uri(LexboxUrlBasicAuth, $"hg/{code}"); From a5b54498d905c3b13690e75d05b1f8e70d57a931 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 21 Aug 2024 13:28:57 +0700 Subject: [PATCH 64/73] Don't throw exceptions if LexBox is unavailable --- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 2b200f32..afd9203e 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -74,8 +74,10 @@ public async Task LoginAs(string lexboxUsername, string lexboxPassword) public async Task IsLexBoxAvailable() { - var httpResponse = await Http.GetAsync(new Uri(LexboxUrl, "api/healthz")); - return httpResponse.IsSuccessStatusCode && httpResponse.Headers.TryGetValues("lexbox-version", out var _ignore); + try { + var httpResponse = await Http.GetAsync(new Uri(LexboxUrl, "api/healthz")); + return httpResponse.IsSuccessStatusCode && httpResponse.Headers.TryGetValues("lexbox-version", out var _ignore); + } catch { return false; } } public static Uri LexboxUrlForProject(string code) => new Uri(LexboxUrl, $"hg/{code}"); From 0af5fb24a6728c89596d8b3eb7e662d477be9c3d Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 22 Aug 2024 09:44:50 +0700 Subject: [PATCH 65/73] Refactor common assertions about project creation --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 4 ---- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 9 ++++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 8d328369..5b31775c 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -160,8 +160,6 @@ public async Task CreateEmptyFlexProjectInLexbox() MercurialTestHelper.CreateFlexRepo(testPath); // Now create project in LexBox var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); - Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); - Assert.That(result.Id, Is.EqualTo(randomGuid)); var pushUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(testCode).AbsoluteUri; MercurialTestHelper.HgPush(testPath, pushUrl); ProjectIdToDelete = result.Id; @@ -175,8 +173,6 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) var testPath = TestFolderForProject(testCode); // Now create project in LexBox var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); - Assert.That(result.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); - Assert.That(result.Id, Is.EqualTo(randomGuid)); await TestEnv.ResetAndUploadZip(testCode, origZipPath); ProjectIdToDelete = result.Id; return testCode; diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index afd9203e..aa3451e5 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -108,9 +108,12 @@ ... on DbError { Query = mutation, Variables = new { input }, }; - var response = await GqlClient.SendMutationAsync(request); - Assert.That(response.Errors, Is.Null.Or.Empty, () => string.Join("\n", response.Errors.Select(error => error.Message))); - return response.Data.CreateProject.CreateProjectResponse; + var gqlResponse = await GqlClient.SendMutationAsync(request); + Assert.That(gqlResponse.Errors, Is.Null.Or.Empty, () => string.Join("\n", gqlResponse.Errors.Select(error => error.Message))); + var response = gqlResponse.Data.CreateProject.CreateProjectResponse; + Assert.That(response.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); + Assert.That(response.Id, Is.EqualTo(projId)); + return response; } public async Task DeleteLexBoxProject(Guid projectId) From b3e258ddc510ea4cac145f986ee7a0d8d0ab2b70 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 22 Aug 2024 10:25:27 +0700 Subject: [PATCH 66/73] Replace unmaintained TUS library with newer one Newer library allows passing in an HttpClient, so we don't have to copy cookies around. --- .../LfMerge.Core.Tests.csproj | 2 +- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 35 +++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj index f84dc02a..daa93089 100644 --- a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj +++ b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj @@ -46,7 +46,7 @@ See full changelog at https://github.com/sillsdev/LfMerge/blob/develop/CHANGELOG - + diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index aa3451e5..0011b5c9 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -5,6 +5,8 @@ using System.Net.Http; using System.Net.Http.Json; using System.Threading.Tasks; +using BirdMessenger; +using BirdMessenger.Collections; using GraphQL; using GraphQL.Client.Http; using GraphQL.Client.Serializer.SystemTextJson; @@ -12,7 +14,6 @@ using LfMerge.Core.Logging; using NUnit.Framework; using SIL.TestUtilities; -using TusDotNetClient; namespace LfMerge.Core.Tests { @@ -164,17 +165,31 @@ public async Task ResetToEmpty(string code) await Http.PostAsync(finishResetUrl, null); } - public async Task UploadZip(string code, string zipPath) + public async Task TusUpload(Uri tusEndpoint, string path, string mimeType) + { + var file = new FileInfo(path); + if (!file.Exists) return; + var metadata = new MetadataCollection { { "filetype", mimeType } }; + var createOpts = new TusCreateRequestOption { + Endpoint = tusEndpoint, + UploadLength = file.Length, + Metadata = metadata, + }; + var createResponse = await Http.TusCreateAsync(createOpts); + + // That doesn't actually upload the file; TusPatchAsync does the actual upload + var fileStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); + var patchOpts = new TusPatchRequestOption { + FileLocation = createResponse.FileLocation, + Stream = fileStream, + }; + await Http.TusPatchAsync(patchOpts); + } + + public Task UploadZip(string code, string zipPath) { var sourceUrl = new Uri(LexboxUrl, $"/api/project/upload-zip/{code}"); - var file = new FileInfo(zipPath); - var client = new TusClient(); - // client.AdditionalHeaders["Authorization"] = $"Bearer {Jwt}"; // Once we set up for LexBox OAuth, we'll use Bearer auth instead - var cookies = Cookies.GetCookies(LexboxUrl); - var authCookie = cookies[".LexBoxAuth"].ToString(); - client.AdditionalHeaders["cookie"] = authCookie; - var fileUrl = await client.CreateAsync(sourceUrl.AbsoluteUri, file.Length, ("filetype", "application/zip")); - await client.UploadAsync(fileUrl, file); + return TusUpload(sourceUrl, zipPath, "application/zip"); } public async Task DownloadProjectBackup(string code, string destZipPath) From 5fe1d6dd982efa9611627f817efcc2cb7581e9a5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 22 Aug 2024 10:35:08 +0700 Subject: [PATCH 67/73] Dispose SRTestEnv during test TearDown method We keep the temp directory around if the test failed, but we dispose everything else. --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 19 ++++++++++--------- src/LfMerge.Core.Tests/TestEnvironment.cs | 4 +++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 5b31775c..30e1bd75 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -55,7 +55,7 @@ public async Task FixtureSetup() // Ensure sena-3.zip is available to all tests as a starting point Sena3ZipPath = Path.Combine(TestDataFolder.Path, "sena-3.zip"); if (!File.Exists(Sena3ZipPath)) { - var testEnv = new SRTestEnvironment(TempFolderForTest); + using var testEnv = new SRTestEnvironment(TempFolderForTest); if (!await testEnv.IsLexBoxAvailable()) { Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox."); } @@ -96,16 +96,17 @@ public async Task TestSetup() [TearDown] public async Task TestTeardown() { + var outcome = TestContext.CurrentContext.Result.Outcome; + var success = outcome == ResultState.Success || outcome == ResultState.Ignored; // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation - if (TestContext.CurrentContext.Result.Outcome == ResultState.Success) { - TempFolderForTest.Dispose(); - if (ProjectIdToDelete is not null) { - var projId = ProjectIdToDelete.Value; - ProjectIdToDelete = null; - // Also leave LexBox project in place for post-test investigation, even though this might tend to clutter things up a little - await TestEnv.DeleteLexBoxProject(projId); - } + TestEnv.DeleteTempFolderDuringCleanup = success; + // Also leave LexBox project in place for post-test investigation, even though this might tend to clutter things up a little + if (success && ProjectIdToDelete is not null) { + var projId = ProjectIdToDelete.Value; + ProjectIdToDelete = null; + await TestEnv.DeleteLexBoxProject(projId); } + TestEnv.Dispose(); } public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode); diff --git a/src/LfMerge.Core.Tests/TestEnvironment.cs b/src/LfMerge.Core.Tests/TestEnvironment.cs index fc54909f..4fbff2ca 100644 --- a/src/LfMerge.Core.Tests/TestEnvironment.cs +++ b/src/LfMerge.Core.Tests/TestEnvironment.cs @@ -25,6 +25,7 @@ public class TestEnvironment : IDisposable protected readonly TemporaryFolder _languageForgeServerFolder; private readonly bool _resetLfProjectsDuringCleanup; private readonly bool _releaseSingletons; + public bool DeleteTempFolderDuringCleanup { get; set; } = true; public LfMergeSettings Settings; private readonly MongoConnectionDouble _mongoConnection; public ILogger Logger => MainClass.Logger; @@ -107,7 +108,8 @@ public void Dispose() MainClass.Container = null; if (_resetLfProjectsDuringCleanup) LanguageForgeProjectAccessor.Reset(); - _languageForgeServerFolder?.Dispose(); + if (DeleteTempFolderDuringCleanup) + _languageForgeServerFolder?.Dispose(); Settings = null; if (_releaseSingletons) SingletonsContainer.Release(); From 52017e9f8d6044ccbcdd05412b77d59250a0c1e4 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 22 Aug 2024 10:44:00 +0700 Subject: [PATCH 68/73] Don't use TempFolder if we don't need auto-cleanup --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 25 +++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 30e1bd75..5f626836 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -39,21 +39,20 @@ public E2ETestBase() [OneTimeSetUp] public async Task FixtureSetup() { - // Ensure we don't delete top-level /tmp/LfMergeSRTests folder and data subfolder if they already exist - var tempPath = Path.Combine(Path.GetTempPath(), "LfMergeSRTests"); - var rootTempFolder = Directory.Exists(tempPath) ? TemporaryFolder.TrackExisting(tempPath) : new TemporaryFolder(tempPath); - var testDataPath = Path.Combine(tempPath, "data"); - TestDataFolder = Directory.Exists(testDataPath) ? TemporaryFolder.TrackExisting(testDataPath) : new TemporaryFolder(testDataPath); - var lcmDataPath = Path.Combine(tempPath, "lcm-common"); - LcmDataFolder = Directory.Exists(lcmDataPath) ? TemporaryFolder.TrackExisting(lcmDataPath) : new TemporaryFolder(lcmDataPath); - Environment.SetEnvironmentVariable("FW_CommonAppData", LcmDataFolder.Path); - - // But the folder for this specific test suite should be deleted if it already exists + // Ensure top-level /tmp/LfMergeSRTests folder and subfolders exist + var tempPath = Path.Join(Path.GetTempPath(), "LfMergeSRTests"); + Directory.CreateDirectory(tempPath); + var testDataPath = Path.Join(tempPath, "data"); + Directory.CreateDirectory(testDataPath); + var lcmDataPath = Path.Join(tempPath, "lcm-common"); + Directory.CreateDirectory(lcmDataPath); + Environment.SetEnvironmentVariable("FW_CommonAppData", lcmDataPath); + var derivedClassName = this.GetType().Name; - TempFolderForClass = new TemporaryFolder(rootTempFolder, derivedClassName); + TempFolderForClass = new TemporaryFolder(Path.Join(tempPath, derivedClassName)); // Ensure sena-3.zip is available to all tests as a starting point - Sena3ZipPath = Path.Combine(TestDataFolder.Path, "sena-3.zip"); + Sena3ZipPath = Path.Join(testDataPath, "sena-3.zip"); if (!File.Exists(Sena3ZipPath)) { using var testEnv = new SRTestEnvironment(TempFolderForTest); if (!await testEnv.IsLexBoxAvailable()) { @@ -191,7 +190,7 @@ public async Task CreateLfProjectFromSena3() var projCode = await CreateNewProjectFromSena3(); var projPath = CloneRepoFromLexbox(projCode, waitTime:TimeSpan.FromSeconds(5)); MercurialTestHelper.ChangeBranch(projPath, "tip"); - var fwdataPath = Path.Combine(projPath, $"{projCode}.fwdata"); + var fwdataPath = Path.Join(projPath, $"{projCode}.fwdata"); LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath); // Do an initial clone from LexBox to the mock LF From 1eea9039fc3f21c8f85bff555916ebd3cd358ca0 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 22 Aug 2024 11:29:34 +0700 Subject: [PATCH 69/73] Better name for cloning FW project from Lexbox --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 2 +- src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 5f626836..a9669f3a 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -138,7 +138,7 @@ public string CloneRepoFromLexbox(string code, string? newCode = null, TimeSpan? return dest; } - public FwProject CloneFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) + public FwProject CloneFwProjectFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) { var dest = CloneRepoFromLexbox(code, newCode, waitTime); var dirInfo = new DirectoryInfo(dest); diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs index 1cf06f3f..3e6225e1 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs @@ -33,7 +33,7 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins() var lfProject = await CreateLfProjectFromSena3(); var fwProjectCode = Regex.Replace(lfProject.ProjectCode, "^sr-", "fw-"); - var fwProject = CloneFromLexbox(lfProject.ProjectCode, fwProjectCode); + var fwProject = CloneFwProjectFromLexbox(lfProject.ProjectCode, fwProjectCode); // Modify FW data first, then push to Lexbox Guid entryId = LcmTestHelper.GetFirstEntry(fwProject).Guid; From 0d1e52f00f75cc524c2e30a9fcda6305b111ecac Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 22 Aug 2024 11:33:44 +0700 Subject: [PATCH 70/73] Handle needing to delete multiple LexBox projects --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index a9669f3a..4f6c3fb1 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using Autofac; @@ -23,7 +24,7 @@ public class E2ETestBase public TemporaryFolder TestDataFolder { get; set; } public TemporaryFolder LcmDataFolder { get; set; } public string Sena3ZipPath { get; set; } - private Guid? ProjectIdToDelete { get; set; } + private readonly HashSet ProjectIdsToDelete = []; public SRTestEnvironment TestEnv { get; set; } public MongoConnectionDouble _mongoConnection; @@ -99,12 +100,13 @@ public async Task TestTeardown() var success = outcome == ResultState.Success || outcome == ResultState.Ignored; // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation TestEnv.DeleteTempFolderDuringCleanup = success; - // Also leave LexBox project in place for post-test investigation, even though this might tend to clutter things up a little - if (success && ProjectIdToDelete is not null) { - var projId = ProjectIdToDelete.Value; - ProjectIdToDelete = null; - await TestEnv.DeleteLexBoxProject(projId); + // On failure, also leave LexBox project(s) in place for post-test investigation, even though this might tend to clutter things up a little + if (success) { + foreach (var projId in ProjectIdsToDelete) { + await TestEnv.DeleteLexBoxProject(projId); + } } + ProjectIdsToDelete.Clear(); TestEnv.Dispose(); } @@ -162,7 +164,7 @@ public async Task CreateEmptyFlexProjectInLexbox() var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); var pushUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(testCode).AbsoluteUri; MercurialTestHelper.HgPush(testPath, pushUrl); - ProjectIdToDelete = result.Id; + if (result.Id.HasValue) ProjectIdsToDelete.Add(result.Id.Value); return testCode; } @@ -174,7 +176,7 @@ public async Task CreateNewProjectFromTemplate(string origZipPath) // Now create project in LexBox var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); await TestEnv.ResetAndUploadZip(testCode, origZipPath); - ProjectIdToDelete = result.Id; + if (result.Id.HasValue) ProjectIdsToDelete.Add(result.Id.Value); return testCode; } From 68440129ada0a8346331fe13a29ea9d6bd070599 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 26 Aug 2024 13:25:29 +0700 Subject: [PATCH 71/73] Remove now-unused properties --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs index 4f6c3fb1..3d11a912 100644 --- a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -21,8 +21,6 @@ public class E2ETestBase public LfMerge.Core.Logging.ILogger Logger => MainClass.Logger; public TemporaryFolder TempFolderForClass { get; set; } public TemporaryFolder TempFolderForTest { get; set; } - public TemporaryFolder TestDataFolder { get; set; } - public TemporaryFolder LcmDataFolder { get; set; } public string Sena3ZipPath { get; set; } private readonly HashSet ProjectIdsToDelete = []; public SRTestEnvironment TestEnv { get; set; } From 0f10f6b6ec94efde5a885e1fb4a8e4a1b131eeda Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 26 Aug 2024 13:40:21 +0700 Subject: [PATCH 72/73] Dispose of FileStream after TUS upload The library doesn't do this for us. This would be unsafe if we weren't awaiting the result of TusPatchAsync, but by the time TusPatchAsync returns, the FileStream is no longer needed. --- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs index 0011b5c9..350321ed 100644 --- a/src/LfMerge.Core.Tests/SRTestEnvironment.cs +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -178,7 +178,7 @@ public async Task TusUpload(Uri tusEndpoint, string path, string mimeType) var createResponse = await Http.TusCreateAsync(createOpts); // That doesn't actually upload the file; TusPatchAsync does the actual upload - var fileStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); + using var fileStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); var patchOpts = new TusPatchRequestOption { FileLocation = createResponse.FileLocation, Stream = fileStream, From 9f09fc0240c0a5230136eb17a64630ff35a09e62 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 26 Aug 2024 13:42:37 +0700 Subject: [PATCH 73/73] Rename TryOutE2ETests to LexboxSendReceiveTests --- .../E2E/{TryOutE2ETests.cs => LexboxSendReceiveTests.cs} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/LfMerge.Core.Tests/E2E/{TryOutE2ETests.cs => LexboxSendReceiveTests.cs} (98%) diff --git a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs b/src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs similarity index 98% rename from src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs rename to src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs index 3e6225e1..358f83a6 100644 --- a/src/LfMerge.Core.Tests/E2E/TryOutE2ETests.cs +++ b/src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs @@ -8,7 +8,7 @@ namespace LfMerge.Core.Tests.E2E [TestFixture] [Category("LongRunning")] [Category("IntegrationTests")] - public class TryOutE2ETests : E2ETestBase + public class LexboxSendReceiveTests : E2ETestBase { // This test will often trigger a race condition in LexBox that causes the *next* test to fail // when `hg clone` returns 404. This is because of hgweb's directory cache, which only refreshes