Cleanup builds & engines

src/Serval/src/Serval.Shared/Models/IInitializableEntity.cs

@@ -0,0 +1,7 @@
+namespace Serval.Shared.Models;
+
+public interface IInitializableEntity : IEntity
+{
+    bool? IsInitialized { get; set; }
+    DateTime? DateCreated { get; set; }
+}

src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs

@@ -25,6 +25,9 @@ public static IServalBuilder AddTranslation(
         builder.Services.AddScoped<IPretranslationService, PretranslationService>();
         builder.Services.AddScoped<IEngineService, EngineService>();
 
+        builder.Services.AddSingleton<EngineCleanupService>();
+        builder.Services.AddSingleton<BuildCleanupService>();
+
         var translationOptions = new TranslationOptions();
         builder.Configuration?.GetSection(TranslationOptions.Key).Bind(translationOptions);
         if (configure is not null)

src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs

@@ -1296,7 +1296,8 @@ private Engine Map(TranslationEngineConfigDto source)
             Type = source.Type.ToPascalCase(),
             Owner = Owner,
             Corpora = [],
-            IsModelPersisted = source.IsModelPersisted
+            IsModelPersisted = source.IsModelPersisted,
+            IsInitialized = false
         };
     }
 
@@ -1308,7 +1309,8 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source)
             Name = source.Name,
             Pretranslate = Map(engine, source.Pretranslate),
             TrainOn = Map(engine, source.TrainOn),
-            Options = Map(source.Options)
+            Options = Map(source.Options),
+            IsInitialized = false
         };
     }
 

src/Serval/src/Serval.Translation/Models/Build.cs

@@ -1,6 +1,6 @@
 namespace Serval.Translation.Models;
 
-public record Build : IEntity
+public record Build : IInitializableEntity
 {
     public string Id { get; set; } = "";
     public int Revision { get; set; } = 1;
@@ -15,4 +15,6 @@ public record Build : IEntity
     public JobState State { get; init; } = JobState.Pending;
     public DateTime? DateFinished { get; init; }
     public IReadOnlyDictionary<string, object>? Options { get; init; }
+    public bool? IsInitialized { get; set; }
+    public DateTime? DateCreated { get; set; }
 }

src/Serval/src/Serval.Translation/Models/Engine.cs

@@ -1,6 +1,6 @@
 namespace Serval.Translation.Models;
 
-public record Engine : IOwnedEntity
+public record Engine : IOwnedEntity, IInitializableEntity
 {
     public string Id { get; set; } = "";
     public int Revision { get; set; } = 1;
@@ -16,4 +16,6 @@ public record Engine : IOwnedEntity
     public int ModelRevision { get; init; }
     public double Confidence { get; init; }
     public int CorpusSize { get; init; }
+    public bool? IsInitialized { get; set; }
+    public DateTime? DateCreated { get; set; }
 }

src/Serval/src/Serval.Translation/Services/BuildCleanupService.cs

@@ -0,0 +1,7 @@
+namespace Serval.Translation.Services;
+
+public class BuildCleanupService(
+    IServiceProvider services,
+    ILogger<BuildCleanupService> logger,
+    TimeSpan? timeout = null
+) : UninitializedCleanupService<Build>(services, logger, timeout) { }

src/Serval/src/Serval.Translation/Services/EngineCleanupService.cs

@@ -0,0 +1,29 @@
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Serval.Translation.Services;
+
+public class EngineCleanupService(
+    IServiceProvider services,
+    ILogger<EngineCleanupService> logger,
+    TimeSpan? timeout = null
+) : UninitializedCleanupService<Engine>(services, logger, timeout)
+{
+    public EngineService? EngineService { get; set; }
+
+    protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken)
+    {
+        EngineService = scope.ServiceProvider.GetRequiredService<EngineService>();
+        await base.DoWorkAsync(scope, cancellationToken);
+    }
+
+    protected override async Task DeleteEntityAsync(
+        IRepository<Engine> engines,
+        Engine engine,
+        CancellationToken cancellationToken
+    )
+    {
+        if (EngineService == null)
+            return;
+        await EngineService.DeleteAsync(engine.Id, cancellationToken);
+    }
+}

src/Serval/src/Serval.Translation/Services/EngineService.cs

@@ -120,9 +120,9 @@ await client.TrainSegmentPairAsync(
 
     public override async Task<Engine> CreateAsync(Engine engine, CancellationToken cancellationToken = default)
     {
-        bool updateIsModelPersisted = engine.IsModelPersisted is null;
         try
         {
+            engine.DateCreated = DateTime.UtcNow;
             await Entities.InsertAsync(engine, cancellationToken);
             TranslationEngineApi.TranslationEngineApiClient? client =
                 _grpcClientFactory.CreateClient<TranslationEngineApi.TranslationEngineApiClient>(engine.Type);
@@ -146,6 +146,15 @@ public override async Task<Engine> CreateAsync(Engine engine, CancellationToken
             {
                 IsModelPersisted = createResponse.IsModelPersisted
             };
+            await Entities.UpdateAsync(
+                engine,
+                u =>
+                {
+                    u.Set(e => e.IsInitialized, true);
+                    u.Set(e => e.IsModelPersisted, engine.IsModelPersisted);
+                },
+                cancellationToken: cancellationToken
+            );
         }
         catch (RpcException rpcex)
         {
@@ -164,14 +173,6 @@ public override async Task<Engine> CreateAsync(Engine engine, CancellationToken
             await Entities.DeleteAsync(engine, CancellationToken.None);
             throw;
         }
-        if (updateIsModelPersisted)
-        {
-            await Entities.UpdateAsync(
-                engine,
-                u => u.Set(e => e.IsModelPersisted, engine.IsModelPersisted),
-                cancellationToken: cancellationToken
-            );
-        }
         return engine;
     }
 
@@ -216,6 +217,7 @@ private Dictionary<string, List<int>> GetChapters(string fileLocation, string sc
 
     public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default)
     {
+        build.DateCreated = DateTime.UtcNow;
         Engine engine = await GetAsync(build.EngineRef, cancellationToken);
         await _builds.InsertAsync(build, cancellationToken);
 
@@ -292,6 +294,11 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellationTok
                 _logger.LogInformation("{request}", JsonSerializer.Serialize(request));
             }
             await client.StartBuildAsync(request, cancellationToken: cancellationToken);
+            await _builds.UpdateAsync(
+                b => b.Id == build.Id,
+                u => u.Set(e => e.IsInitialized, true),
+                cancellationToken: CancellationToken.None
+            );
         }
         catch
         {

src/Serval/src/Serval.Translation/Services/UnitializedEntityCleanupService.cs

@@ -0,0 +1,49 @@
+using Microsoft.Extensions.DependencyInjection;
+using SIL.ServiceToolkit.Services;
+
+namespace Serval.Translation.Services;
+
+public abstract class UninitializedCleanupService<T>(
+    IServiceProvider services,
+    ILogger<UninitializedCleanupService<T>> logger,
+    TimeSpan? timeout = null
+) : RecurrentTask($"{typeof(T)} Cleanup Service", services, RefreshPeriod, logger)
+    where T : IInitializableEntity
+{
+    private readonly ILogger<UninitializedCleanupService<T>> _logger = logger;
+    private readonly TimeSpan _timeout = timeout ?? TimeSpan.FromMinutes(2);
+    private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1);
+
+    protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Running build cleanup job");
+        var entities = scope.ServiceProvider.GetRequiredService<IRepository<T>>();
+        await CheckEntitiesAsync(entities, cancellationToken);
+    }
+
+    public async Task CheckEntitiesAsync(IRepository<T> entities, CancellationToken cancellationToken)
+    {
+        var now = DateTime.UtcNow;
+        IEnumerable<T> uninitializedEntities = (await entities.GetAllAsync(cancellationToken)).Where(b =>
+            !(b.IsInitialized ?? true) && (now - (b.DateCreated ?? DateTime.UtcNow)) > _timeout
+        );
+        foreach (T entity in uninitializedEntities)
+        {
+            _logger.LogInformation(
+                "Deleting {type} {id} because it was never successfully started",
+                typeof(T),
+                entity.Id
+            );
+            await DeleteEntityAsync(entities, entity, cancellationToken);
+        }
+    }
+
+    protected virtual async Task DeleteEntityAsync(
+        IRepository<T> entities,
+        T entity,
+        CancellationToken cancellationToken
+    )
+    {
+        await entities.DeleteAsync(e => e.Id == entity.Id, cancellationToken);
+    }
+}

src/Serval/test/Serval.Translation.Tests/Services/BuildCleanupServiceTests.cs

@@ -0,0 +1,56 @@
+namespace Serval.Translation.Services;
+
+[TestFixture]
+public class BuildCleanupServiceTests
+{
+    [Test]
+    public async Task CleanupAsync()
+    {
+        TestEnvironment env = new();
+        Assert.That(env.Builds.Count, Is.EqualTo(2));
+        await env.CheckBuildsAsync();
+        Assert.That(env.Builds.Count, Is.EqualTo(1));
+        Assert.That((await env.Builds.GetAllAsync())[0].Id, Is.EqualTo("build2"));
+    }
+
+    private class TestEnvironment
+    {
+        public MemoryRepository<Build> Builds { get; }
+
+        public TestEnvironment()
+        {
+            Builds = new MemoryRepository<Build>();
+            Builds.Add(
+                new Build
+                {
+                    Id = "build1",
+                    EngineRef = "engine1",
+                    IsInitialized = false,
+                    DateCreated = DateTime.UtcNow.Subtract(TimeSpan.FromHours(10))
+                }
+            );
+            Builds.Add(
+                new Build
+                {
+                    Id = "build2",
+                    EngineRef = "engine2",
+                    IsInitialized = true,
+                    DateCreated = DateTime.UtcNow.Subtract(TimeSpan.FromHours(10))
+                }
+            );
+
+            Service = new BuildCleanupService(
+                Substitute.For<IServiceProvider>(),
+                Substitute.For<ILogger<BuildCleanupService>>(),
+                TimeSpan.Zero
+            );
+        }
+
+        public BuildCleanupService Service { get; }
+
+        public async Task CheckBuildsAsync()
+        {
+            await Service.CheckEntitiesAsync(Builds, CancellationToken.None);
+        }
+    }
+}

src/Serval/test/Serval.Translation.Tests/Services/EngineCleanupServiceTests.cs

@@ -0,0 +1,99 @@
+using Google.Protobuf.WellKnownTypes;
+using MassTransit.Mediator;
+using Serval.Translation.V1;
+
+namespace Serval.Translation.Services;
+
+[TestFixture]
+public class EngineCleanupServiceTests
+{
+    [Test]
+    public async Task CleanupAsync()
+    {
+        TestEnvironment env = new();
+        Assert.That(env.Engines.Count, Is.EqualTo(2));
+        await env.CheckEnginesAsync();
+        Assert.That(env.Engines.Count, Is.EqualTo(1));
+        Assert.That((await env.Engines.GetAllAsync())[0].Id, Is.EqualTo("engine2"));
+    }
+
+    private class TestEnvironment
+    {
+        public MemoryRepository<Engine> Engines { get; }
+
+        public TestEnvironment()
+        {
+            Engines = new MemoryRepository<Engine>();
+            Engines.Add(
+                new Engine
+                {
+                    Id = "engine1",
+                    SourceLanguage = "en",
+                    TargetLanguage = "es",
+                    Type = "Nmt",
+                    Owner = "client1",
+                    IsInitialized = false,
+                    DateCreated = DateTime.UtcNow.Subtract(TimeSpan.FromHours(10))
+                }
+            );
+            Engines.Add(
+                new Engine
+                {
+                    Id = "engine2",
+                    SourceLanguage = "en",
+                    TargetLanguage = "es",
+                    Type = "Nmt",
+                    Owner = "client1",
+                    IsInitialized = true,
+                    DateCreated = DateTime.UtcNow.Subtract(TimeSpan.FromHours(10))
+                }
+            );
+
+            Service = new EngineCleanupService(
+                Substitute.For<IServiceProvider>(),
+                Substitute.For<ILogger<EngineCleanupService>>(),
+                TimeSpan.Zero
+            );
+
+            var translationServiceClient = Substitute.For<TranslationEngineApi.TranslationEngineApiClient>();
+            translationServiceClient.DeleteAsync(Arg.Any<DeleteRequest>()).Returns(CreateAsyncUnaryCall(new Empty()));
+
+            GrpcClientFactory grpcClientFactory = Substitute.For<GrpcClientFactory>();
+            grpcClientFactory
+                .CreateClient<TranslationEngineApi.TranslationEngineApiClient>("Nmt")
+                .Returns(translationServiceClient);
+
+            var engineService = new EngineService(
+                Engines,
+                new MemoryRepository<Build>(),
+                new MemoryRepository<Pretranslation>(),
+                Substitute.For<IScopedMediator>(),
+                grpcClientFactory,
+                Substitute.For<IOptionsMonitor<DataFileOptions>>(),
+                new MemoryDataAccessContext(),
+                new LoggerFactory(),
+                Substitute.For<IScriptureDataFileService>()
+            );
+
+            Service.EngineService = engineService;
+        }
+
+        public EngineCleanupService Service { get; }
+
+        public async Task CheckEnginesAsync()
+        {
+            await Service.CheckEntitiesAsync(Engines, CancellationToken.None);
+        }
+
+        private static AsyncUnaryCall<TResponse> CreateAsyncUnaryCall<TResponse>(TResponse response)
+        {
+            return new AsyncUnaryCall<TResponse>(
+                Task.FromResult(response),
+                Task.FromResult(new Metadata()),
+                () => Status.DefaultSuccess,
+                () => new Metadata(),
+                () => { }
+            );
+        }
+    }
+}