diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eb38ca443..f1bcb4954 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -128,6 +128,21 @@ jobs: semver: ${{ needs.set-release-version.outputs.version }} secrets: inherit + build-projections-last-changed-list: + name: Build projections last changed list + uses: Informatievlaanderen/build-pipeline/.github/workflows/build-image.yml@main + needs: [ set-release-version ] + if: ${{ (github.repository_owner == 'Informatievlaanderen') && (needs.set-release-version.outputs.version != 'none') }} + with: + registry: streetname-registry + build-target: Containerize_ProjectionsLastChangedList + image-file: sr-projections-last-changed-list-console-image.tar + image-name: projections-last-changed-list-console + test-project: StreetNameRegistry.Tests + build-project: StreetNameRegistry.Projections.LastChangedList.Console + semver: ${{ needs.set-release-version.outputs.version }} + secrets: inherit + build-projections-backoffice: name: Build Projections BackOffice uses: Informatievlaanderen/build-pipeline/.github/workflows/build-image.yml@main @@ -337,6 +352,7 @@ jobs: , build-projector , build-projections-backoffice , build-projections-syndication + , build-projections-last-changed-list , build-consumer , build-producer , build-producer-snapshot-oslo @@ -681,6 +697,7 @@ jobs: , 'projector' , 'projections-syndication' , 'projections-backoffice' + ; 'projections-last-changed-list-console' , 'consumer' , 'producer' , 'producer-snapshot-oslo' @@ -803,6 +820,7 @@ jobs: , 'projector' , 'projections-syndication' , 'projections-backoffice' + , 'projections-last-changed-list-console' , 'consumer' , 'producer' , 'producer-snapshot-oslo' @@ -865,6 +883,7 @@ jobs: , 'projector' , 'projections-syndication' , 'projections-backoffice' + , 'projections-last-changed-list-console' , 'consumer' , 'producer' , 'producer-snapshot-oslo' diff --git a/StreetNameRegistry.sln b/StreetNameRegistry.sln index b6245a244..8923fbdaa 100755 --- a/StreetNameRegistry.sln +++ b/StreetNameRegistry.sln @@ -81,6 +81,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StreetNameRegistry.Consumer EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StreetNameRegistry.Projections.Integration", "src\StreetNameRegistry.Projections.Integration\StreetNameRegistry.Projections.Integration.csproj", "{E69043F1-E6C9-484C-A961-C28C33013C91}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StreetNameRegistry.Projections.LastChangedList.Console", "src\StreetNameRegistry.Projections.LastChangedList.Console\StreetNameRegistry.Projections.LastChangedList.Console.csproj", "{2E20B1DF-A4E2-42BA-85F4-F335754C455B}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -191,6 +193,10 @@ Global {E69043F1-E6C9-484C-A961-C28C33013C91}.Debug|Any CPU.Build.0 = Debug|Any CPU {E69043F1-E6C9-484C-A961-C28C33013C91}.Release|Any CPU.ActiveCfg = Release|Any CPU {E69043F1-E6C9-484C-A961-C28C33013C91}.Release|Any CPU.Build.0 = Release|Any CPU + {2E20B1DF-A4E2-42BA-85F4-F335754C455B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2E20B1DF-A4E2-42BA-85F4-F335754C455B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2E20B1DF-A4E2-42BA-85F4-F335754C455B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2E20B1DF-A4E2-42BA-85F4-F335754C455B}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -222,6 +228,7 @@ Global {06F8F677-2696-4996-B4AE-B8E3B747A696} = {65EA04DE-5928-430F-92CA-24C11B6E5A00} {83C7D129-ACEA-4FC6-A55C-748D8D31135A} = {65EA04DE-5928-430F-92CA-24C11B6E5A00} {E69043F1-E6C9-484C-A961-C28C33013C91} = {65EA04DE-5928-430F-92CA-24C11B6E5A00} + {2E20B1DF-A4E2-42BA-85F4-F335754C455B} = {65EA04DE-5928-430F-92CA-24C11B6E5A00} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3B44FE80-30FE-47B7-A0DE-8610218FFF63} diff --git a/build.fsx b/build.fsx index 215eda5b9..95cb49802 100755 --- a/build.fsx +++ b/build.fsx @@ -113,6 +113,7 @@ Target.create "Containerize_ProducerSnapshotOslo" (fun _ -> containerize "Street Target.create "Containerize_MigratorStreetName" (fun _ -> containerize "StreetNameRegistry.Migrator.StreetName" "migrator-streetname") Target.create "Containerize_ProjectionsSyndication" (fun _ -> containerize "StreetNameRegistry.Projections.Syndication" "projections-syndication") Target.create "Containerize_ProjectionsBackOffice" (fun _ -> containerize "StreetNameRegistry.Projections.BackOffice" "projections-backoffice") +Target.create "Containerize_ProjectionsLastChangedList" (fun _ -> containerize "StreetNameRegistry.Projections.LastChangedList.Console" "projections-last-changed-list-console") Target.create "Containerize_SnapshotVerifier" (fun _ -> containerize "StreetNameRegistry.Snapshot.Verifier" "snapshot-verifier") Target.create "Containerize_ConsumerReadPostal" (fun _ -> containerize "StreetNameRegistry.Consumer.Read.Postal" "consumer-read-postal") // -------------------------------------------------------------------------------- diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/Dockerfile b/src/StreetNameRegistry.Projections.LastChangedList.Console/Dockerfile new file mode 100644 index 000000000..b8e9b7b84 --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/Dockerfile @@ -0,0 +1,23 @@ +FROM mcr.microsoft.com/dotnet/runtime-deps:6.0.3 + +# create group & user +RUN addgroup --gid 1000 --system app && adduser --uid 1000 -system app --gid 1000 + +# create work dir and set permissions as WORKDIR sets permissions as root +RUN mkdir /app && chown -R app:app /app +WORKDIR /app + +LABEL maintainer "Digitaal Vlaanderen " +LABEL registry="streetname-registry" + +COPY / /app +WORKDIR /app + +RUN apt-get update && \ + apt-get install curl jq -y && \ + chmod +x ./init.sh + +# switch to created user +USER app + +ENTRYPOINT ["./init.sh"] diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/LastChangedListStreetNameCacheValidator.cs b/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/LastChangedListStreetNameCacheValidator.cs new file mode 100644 index 000000000..5e831cc2c --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/LastChangedListStreetNameCacheValidator.cs @@ -0,0 +1,34 @@ +namespace StreetNameRegistry.Projections.LastChangedList.Console.Infrastructure +{ + using System.Threading; + using System.Threading.Tasks; + using Be.Vlaanderen.Basisregisters.ProjectionHandling.LastChangedList; + using Dapper; + using Microsoft.Data.SqlClient; + + public sealed class LastChangedListStreetNameCacheValidator : ICacheValidator + { + private readonly string _connectionString; + private readonly string _projectionName; + + public LastChangedListStreetNameCacheValidator(string connectionString, string projectionName) + { + _connectionString = connectionString; + _projectionName = projectionName; + } + + public async Task CanCache(long position, CancellationToken ct) + { + await using var connection = new SqlConnection(_connectionString); + + var sql = @"SELECT [Position] + FROM [streetname-registry].[StreetNameRegistryLegacy].[ProjectionStates] + WHERE [Name] = @Name + "; + + var projectionPosition = await connection.ExecuteScalarAsync(sql, new { Name = _projectionName }); + + return projectionPosition >= position; + } + } +} diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/Modules/LastChangedListConsoleModule.cs b/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/Modules/LastChangedListConsoleModule.cs new file mode 100644 index 000000000..c94659018 --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/Modules/LastChangedListConsoleModule.cs @@ -0,0 +1,98 @@ +namespace StreetNameRegistry.Projections.LastChangedList.Console.Infrastructure.Modules +{ + using System; + using StreetNameRegistry.Infrastructure; + using Autofac; + using Autofac.Extensions.DependencyInjection; + using Be.Vlaanderen.Basisregisters.Api.Exceptions; + using Be.Vlaanderen.Basisregisters.DataDog.Tracing.Microsoft; + using Be.Vlaanderen.Basisregisters.DependencyInjection; + using Be.Vlaanderen.Basisregisters.EventHandling; + using Be.Vlaanderen.Basisregisters.EventHandling.Autofac; + using Be.Vlaanderen.Basisregisters.ProjectionHandling.LastChangedList; + using Be.Vlaanderen.Basisregisters.ProjectionHandling.SqlStreamStore.Autofac; + using Be.Vlaanderen.Basisregisters.Projector; + using Be.Vlaanderen.Basisregisters.Projector.ConnectedProjections; + using Be.Vlaanderen.Basisregisters.Projector.Modules; + using Microsoft.Extensions.Configuration; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using LastChangedListContextMigrationFactory = LastChangedList.LastChangedListContextMigrationFactory; + + public class LastChangedListConsoleModule : Module + { + private readonly IConfiguration _configuration; + private readonly IServiceCollection _services; + private readonly ILoggerFactory _loggerFactory; + + public LastChangedListConsoleModule( + IConfiguration configuration, + IServiceCollection services, + ILoggerFactory loggerFactory) + { + _configuration = configuration; + _services = services; + _loggerFactory = loggerFactory; + } + + protected override void Load(ContainerBuilder builder) + { + _services.RegisterModule(new DataDogModule(_configuration)); + + RegisterProjectionSetup(builder); + + builder + .RegisterType() + .AsSelf(); + + builder.Populate(_services); + } + + private void RegisterProjectionSetup(ContainerBuilder builder) + { + builder + .RegisterModule( + new EventHandlingModule( + typeof(DomainAssemblyMarker).Assembly, + EventsJsonSerializerSettingsProvider.CreateSerializerSettings())) + .RegisterModule() + .RegisterEventstreamModule(_configuration) + .RegisterModule(new ProjectorModule(_configuration)); + + RegisterProjections(builder); + } + + private void RegisterProjections(ContainerBuilder builder) + { + var logger = _loggerFactory.CreateLogger(); + var connectionString = _configuration.GetConnectionString("LastChangedList"); + + builder.RegisterModule(new LastChangedListModule(connectionString, _configuration["DataDog:ServiceName"], _services, _loggerFactory)); + + logger.LogInformation( + "Added {Context} to services:" + + Environment.NewLine + + "\tSchema: {Schema}" + + Environment.NewLine + + "\tTableName: {TableName}", + nameof(LastChangedListContext), LastChangedListContext.Schema, LastChangedListContext.MigrationsHistoryTable); + + builder.Register(c => + new LastChangedListStreetNameCacheValidator( + _configuration.GetConnectionString("LegacyProjections"), + "StreetNameRegistry.Projections.Legacy.StreetNameDetailV2.StreetNameDetailProjectionsV2")) + .AsSelf(); + + builder + .RegisterProjectionMigrator( + _configuration, + _loggerFactory) + .RegisterProjectionMigrator( + _configuration, + _loggerFactory) + .RegisterProjections( + context => new LastChangedProjections(context.Resolve()), + ConnectedProjectionSettings.Default); + } + } +} diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/Program.cs b/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/Program.cs new file mode 100644 index 000000000..796fdd291 --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/Program.cs @@ -0,0 +1,159 @@ +namespace StreetNameRegistry.Projections.LastChangedList.Console.Infrastructure +{ + using System; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using Autofac; + using Autofac.Extensions.DependencyInjection; + using Be.Vlaanderen.Basisregisters.Aws.DistributedMutex; + using Be.Vlaanderen.Basisregisters.ProjectionHandling.LastChangedList; + using Console; + using Destructurama; + using Microsoft.Extensions.Configuration; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using Microsoft.Extensions.Logging; + using Modules; + using Serilog; + using Serilog.Debugging; + using Serilog.Extensions.Logging; + + public class Program + { + protected Program() + { } + + public static async Task Main(string[] args) + { + AppDomain.CurrentDomain.FirstChanceException += (_, eventArgs) => + Log.Debug( + eventArgs.Exception, + "FirstChanceException event raised in {AppDomain}.", + AppDomain.CurrentDomain.FriendlyName); + + AppDomain.CurrentDomain.UnhandledException += (_, eventArgs) => + Log.Fatal((Exception)eventArgs.ExceptionObject, "Encountered a fatal exception, exiting program."); + + Log.Information("Initializing StreetNameRegistry.Projections.LastChangedList.Console"); + + var host = new HostBuilder() + .ConfigureAppConfiguration((_, configurationBuilder) => + { + configurationBuilder + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: false) + .AddJsonFile($"appsettings.{Environment.MachineName.ToLowerInvariant()}.json", optional: true, reloadOnChange: false) + .AddEnvironmentVariables() + .AddCommandLine(args); + }) + .ConfigureLogging((hostContext, loggingBuilder) => + { + SelfLog.Enable(Console.WriteLine); + + Log.Logger = new LoggerConfiguration() //NOSONAR logging configuration is safe + .ReadFrom.Configuration(hostContext.Configuration) + .Enrich.FromLogContext() + .Enrich.WithMachineName() + .Enrich.WithThreadId() + .Enrich.WithEnvironmentUserName() + .Destructure.JsonNetTypes() + .CreateLogger(); + + loggingBuilder.ClearProviders(); + loggingBuilder.AddSerilog(Log.Logger); + }) + .ConfigureServices((hostContext, services) => + { + var healthChecksBuilder = services.AddHealthChecks(); + var connectionStrings = hostContext.Configuration + .GetSection("ConnectionStrings") + .GetChildren(); + + foreach (var connectionString in connectionStrings) + { + healthChecksBuilder.AddSqlServer( + connectionString.Value, + name: $"sqlserver-{connectionString.Key.ToLowerInvariant()}", + tags: new[] { "db", "sql", "sqlserver" }); + } + + healthChecksBuilder.AddDbContextCheck( + $"dbcontext-{nameof(LastChangedListContext).ToLowerInvariant()}", + tags: new[] { "db", "sql", "sqlserver" }); + + var origins = hostContext.Configuration + .GetSection("Cors") + .GetChildren() + .Select(c => c.Value) + .ToArray(); + + foreach (var origin in origins) + { + services.AddCors(options => + { + options.AddDefaultPolicy(builder => + { + builder.WithOrigins(origin); + }); + }); + } + }) + .UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer((hostContext, builder) => + { + var services = new ServiceCollection(); + var loggerFactory = new SerilogLoggerFactory(Log.Logger); + + builder.RegisterModule(new LastChangedListConsoleModule(hostContext.Configuration, services, loggerFactory)); + + builder + .RegisterType() + .As() + .SingleInstance(); + + builder.Populate(services); + }) + // .ConfigureWebHostDefaults(webHostBuilder => + // webHostBuilder + // .UseStartup() + // .UseKestrel()) + .UseConsoleLifetime() + .Build(); + + Log.Information("Starting StreetNameRegistry.Projections.LastChangedList.Console"); + + var logger = host.Services.GetRequiredService>(); + var configuration = host.Services.GetRequiredService(); + + try + { + await DistributedLock.RunAsync( + async () => { await host.RunAsync().ConfigureAwait(false); }, + DistributedLockOptions.LoadFromConfiguration(configuration), + logger) + .ConfigureAwait(false); + } + catch (AggregateException aggregateException) + { + foreach (var innerException in aggregateException.InnerExceptions) + { + logger.LogCritical(innerException, "Encountered a fatal exception, exiting program."); + } + } + catch (Exception e) + { + logger.LogCritical(e, "Encountered a fatal exception, exiting program."); + Log.CloseAndFlush(); + + // Allow some time for flushing before shutdown. + await Task.Delay(500, default); + throw; + } + finally + { + logger.LogInformation("Stopping..."); + } + } + } +} diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/Startup.cs b/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/Startup.cs new file mode 100644 index 000000000..3134f279d --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/Infrastructure/Startup.cs @@ -0,0 +1,65 @@ +// namespace StreetNameRegistry.Projections.LastChangedList.Console.Infrastructure +// { +// using System.Collections.Generic; +// using System.Linq; +// using System.Threading; +// using Be.Vlaanderen.Basisregisters.AspNetCore.Mvc.Formatters.Json; +// using Be.Vlaanderen.Basisregisters.Projector.ConnectedProjections; +// using Be.Vlaanderen.Basisregisters.Projector.Controllers; +// using Microsoft.AspNetCore.Builder; +// using Microsoft.AspNetCore.Http; +// using Microsoft.Extensions.Configuration; +// using Microsoft.Extensions.DependencyInjection; +// using Newtonsoft.Json; +// using SqlStreamStore; +// +// public class Startup +// { +// public void Configure(IApplicationBuilder app) +// { +// app.UseRouting(); +// app.UseCors(); +// +// app.UseHealthChecks("/health"); +// +// app.UseEndpoints(endpoints => +// { +// endpoints.MapGet("v1/projections", async context => +// { +// var configuration = app.ApplicationServices.GetRequiredService(); +// var manager = app.ApplicationServices.GetRequiredService(); +// var streamStore = app.ApplicationServices.GetRequiredService(); +// +// var baseUri = configuration.GetValue("BaseUrl").TrimEnd('/'); +// +// var registeredConnectedProjections = manager +// .GetRegisteredProjections() +// .ToList(); +// var projectionStates = await manager.GetProjectionStates(CancellationToken.None); +// var responses = registeredConnectedProjections.Aggregate( +// new List(), +// (list, projection) => +// { +// var projectionState = projectionStates.SingleOrDefault(x => x.Name == projection.Id); +// list.Add(new ProjectionResponse( +// projection, +// projectionState, +// baseUri)); +// return list; +// }); +// +// var streamPosition = await streamStore.ReadHeadPosition(); +// +// var projectionResponseList = new ProjectionResponseList(responses, baseUri) +// { +// StreamPosition = streamPosition +// }; +// +// var json = JsonConvert.SerializeObject(projectionResponseList, new JsonSerializerSettings().ConfigureDefaultForApi()); +// +// await context.Response.WriteAsync(json); +// }); +// }); +// } +// } +// } diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/ProjectionRunner.cs b/src/StreetNameRegistry.Projections.LastChangedList.Console/ProjectionRunner.cs new file mode 100644 index 000000000..baea0c7eb --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/ProjectionRunner.cs @@ -0,0 +1,40 @@ +namespace StreetNameRegistry.Projections.LastChangedList.Console +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Be.Vlaanderen.Basisregisters.Projector.ConnectedProjections; + using Microsoft.Extensions.Hosting; + using Microsoft.Extensions.Logging; + + public class ProjectionRunner : BackgroundService + { + private readonly IHostApplicationLifetime _hostApplicationLifetime; + private readonly ILogger _logger; + private readonly IConnectedProjectionsManager _projectionManager; + + public ProjectionRunner( + IHostApplicationLifetime hostApplicationLifetime, + IConnectedProjectionsManager projectionManager, + ILogger logger) + { + _hostApplicationLifetime = hostApplicationLifetime; + _projectionManager = projectionManager; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + await _projectionManager.Start(stoppingToken); + } + catch (Exception exception) + { + _logger.LogCritical(exception, $"Critical error occured in {nameof(ProjectionRunner)}."); + _hostApplicationLifetime.StopApplication(); + throw; + } + } + } +} diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/Properties/AssemblyInfo.cs b/src/StreetNameRegistry.Projections.LastChangedList.Console/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..36dac4c3d --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/Properties/AssemblyInfo.cs @@ -0,0 +1,7 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyDescription("StreetNameRegistry Projections LastChangedList Console")] + +[assembly: ComVisible(false)] +[assembly: Guid("6920d5b3-fc6d-4a11-bd77-419eccc02ebb")] diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/Properties/launchSettings.json b/src/StreetNameRegistry.Projections.LastChangedList.Console/Properties/launchSettings.json new file mode 100644 index 000000000..e9883370b --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/Properties/launchSettings.json @@ -0,0 +1,11 @@ +{ + "profiles": { + "StreetNameRegistry.Projections.LastChangedList.Console": { + "commandName": "Project", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "http://localhost:5000/" + } + } +} diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/StreetNameRegistry.Projections.LastChangedList.Console.csproj b/src/StreetNameRegistry.Projections.LastChangedList.Console/StreetNameRegistry.Projections.LastChangedList.Console.csproj new file mode 100644 index 000000000..dced43cb3 --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/StreetNameRegistry.Projections.LastChangedList.Console.csproj @@ -0,0 +1,26 @@ + + + + + true + false + + + + + + + + + + + + + + + + + + + + diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/appsettings.json b/src/StreetNameRegistry.Projections.LastChangedList.Console/appsettings.json new file mode 100644 index 000000000..82ccdd994 --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/appsettings.json @@ -0,0 +1,48 @@ +{ + "ConnectionStrings": { + "Events": "Server=(localdb)\\mssqllocaldb;Database=EFProviders.InMemory.StreetNameRegistry;Trusted_Connection=True;TrustServerCertificate=True;", + "LegacyProjections": "Server=(localdb)\\mssqllocaldb;Database=EFProviders.InMemory.StreetNameRegistry;Trusted_Connection=True;TrustServerCertificate=True;", + "LastChangedList": "Server=(localdb)\\mssqllocaldb;Database=EFProviders.InMemory.StreetNameRegistry;Trusted_Connection=True;TrustServerCertificate=True;", + "LastChangedListAdmin": "Server=(localdb)\\mssqllocaldb;Database=EFProviders.InMemory.StreetNameRegistry;Trusted_Connection=True;TrustServerCertificate=True;" + }, + + "BaseUrl": "https://api.staging-basisregisters.vlaanderen/", + + "DataDog": { + "Enabled": false, + "Debug": false, + "ServiceName": "streetname-registry-projections-last-changed-list-console-dev" + }, + + "DistributedLock": { + "Region": "eu-west-1", + "TableName": "__DistributedLocks__", + "LeasePeriodInMinutes": 5, + "ThrowOnFailedRenew": true, + "TerminateApplicationOnFailedRenew": true, + "Enabled": true + }, + + "Cors": [ + "http://localhost:3000", + "http://localhost:5000" + ], + + "Serilog": { + "MinimumLevel": { + "Default": "Information" + }, + "WriteTo": [ + { + "Name": "Console", + "Args": { + "formatter": "Serilog.Formatting.Compact.RenderedCompactJsonFormatter, Serilog.Formatting.Compact" + } + } + ], + "Properties": { + "Application": "StreetNameRegistry - Projections LastChangedList Console", + "ContainerId": "REPLACE_CONTAINERID" + } + } +} diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/init.sh b/src/StreetNameRegistry.Projections.LastChangedList.Console/init.sh new file mode 100644 index 000000000..7423ec9dd --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/init.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +CONTAINERID=$(curl -s http://169.254.170.2/v2/metadata | jq -r ".Containers[] | select(.Labels[\"com.amazonaws.ecs.container-name\"] | startswith(\"basisregisters-\") and endswith(\"-last-changed-list-console\")) | .DockerId") + +sed -i "s/REPLACE_CONTAINERID/$CONTAINERID/g" appsettings.json + +./StreetNameRegistry.Projections.LastChangedList.Console diff --git a/src/StreetNameRegistry.Projections.LastChangedList.Console/paket.references b/src/StreetNameRegistry.Projections.LastChangedList.Console/paket.references new file mode 100644 index 000000000..bd0391596 --- /dev/null +++ b/src/StreetNameRegistry.Projections.LastChangedList.Console/paket.references @@ -0,0 +1,18 @@ +Be.Vlaanderen.Basisregisters.Api +Be.Vlaanderen.Basisregisters.EventHandling.Autofac +Be.Vlaanderen.Basisregisters.DataDog.Tracing.Sql +Be.Vlaanderen.Basisregisters.GrAr.Contracts +Be.Vlaanderen.Basisregisters.ProjectionHandling.Runner.SqlServer +Be.Vlaanderen.Basisregisters.ProjectionHandling.SqlStreamStore.Autofac +Be.Vlaanderen.Basisregisters.Projector + +Microsoft.Extensions.DependencyInjection + +Dapper + +AspNetCore.HealthChecks.SqlServer + +SourceLink.Embed.AllSourceFiles +SourceLink.Copy.PdbFiles + + diff --git a/src/StreetNameRegistry.Projections.LastChangedList/LastChangedProjections.cs b/src/StreetNameRegistry.Projections.LastChangedList/LastChangedProjections.cs index c4168bbd1..efdd7ba27 100644 --- a/src/StreetNameRegistry.Projections.LastChangedList/LastChangedProjections.cs +++ b/src/StreetNameRegistry.Projections.LastChangedList/LastChangedProjections.cs @@ -11,10 +11,11 @@ namespace StreetNameRegistry.Projections.LastChangedList using StreetName.Events; using StreetName.Events.Crab; - [ConnectedProjectionName("Cache markering straatnamen")] + [ConnectedProjectionName(ProjectionName)] [ConnectedProjectionDescription("Projectie die markeert voor hoeveel straatnamen de gecachte data nog geüpdated moeten worden.")] public sealed class LastChangedProjections : LastChangedListConnectedProjection { + public const string ProjectionName = "Cache markering straatnamen"; private static readonly AcceptType[] SupportedAcceptTypes = { AcceptType.JsonLd }; public LastChangedProjections(ICacheValidator cacheValidator) diff --git a/src/StreetNameRegistry.Projector/Caches/CachesController.cs b/src/StreetNameRegistry.Projector/Caches/CachesController.cs index 494e87493..7ce3a5587 100644 --- a/src/StreetNameRegistry.Projector/Caches/CachesController.cs +++ b/src/StreetNameRegistry.Projector/Caches/CachesController.cs @@ -1,6 +1,7 @@ namespace StreetNameRegistry.Projector.Caches { using System; + using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using System.Threading; @@ -9,17 +10,26 @@ namespace StreetNameRegistry.Projector.Caches using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; + using SqlStreamStore; + using StreetNameRegistry.Projections.LastChangedList; [ApiVersion("1.0")] [ApiRoute("caches")] public class CachesController : ApiController { + private static Dictionary _projectionNameMapper = new Dictionary() + { + {"StreetNameRegistry.Projections.LastChangedList.LastChangedProjections", LastChangedProjections.ProjectionName} + }; + [HttpGet] public async Task Get( [FromServices] IConfiguration configuration, [FromServices] LastChangedListContext lastChangedListContext, + [FromServices] IReadonlyStreamStore streamStore, CancellationToken cancellationToken) { + var maxErrorTimeInSeconds = configuration.GetValue("Caches:LastChangedList:MaxErrorTimeInSeconds") ?? 60; var maxErrorTime = DateTimeOffset.UtcNow.AddSeconds(-1 * maxErrorTimeInSeconds); @@ -29,13 +39,28 @@ public async Task Get( .Where(r => r.ToBeIndexed && (r.LastError == null || r.LastError < maxErrorTime)) .CountAsync(cancellationToken); - return Ok(new[] + var positions = await lastChangedListContext.ProjectionStates.ToListAsync(cancellationToken); + var streamPosition = await streamStore.ReadHeadPosition(cancellationToken); + + var response = new List { - new { - name = "Cache detail straatnamen", - numberOfRecordsToProcess = numberOfRecords - } - }); + new + { + name = "Cache detail straatnamen", + numberOfRecordsToProcess = numberOfRecords + } + }; + + foreach (var position in positions) + { + response.Add(new + { + name = _projectionNameMapper.ContainsKey(position.Name) ? _projectionNameMapper[position.Name] : position.Name, + numberOfRecordsToProcess = streamPosition - position.Position + }); + } + + return Ok(response); } } } diff --git a/src/StreetNameRegistry.Projector/Infrastructure/Modules/ApiModule.cs b/src/StreetNameRegistry.Projector/Infrastructure/Modules/ApiModule.cs index e6d8c2f1e..748b83e0f 100644 --- a/src/StreetNameRegistry.Projector/Infrastructure/Modules/ApiModule.cs +++ b/src/StreetNameRegistry.Projector/Infrastructure/Modules/ApiModule.cs @@ -7,7 +7,6 @@ namespace StreetNameRegistry.Projector.Infrastructure.Modules using Be.Vlaanderen.Basisregisters.DependencyInjection; using Be.Vlaanderen.Basisregisters.EventHandling; using Be.Vlaanderen.Basisregisters.EventHandling.Autofac; - using Be.Vlaanderen.Basisregisters.ProjectionHandling.LastChangedList; using Be.Vlaanderen.Basisregisters.ProjectionHandling.SqlStreamStore.Autofac; using Be.Vlaanderen.Basisregisters.Projector; using Be.Vlaanderen.Basisregisters.Projector.ConnectedProjections; @@ -33,7 +32,6 @@ namespace StreetNameRegistry.Projector.Infrastructure.Modules using StreetNameRegistry.Projections.Wfs; using StreetNameRegistry.Projections.Wfs.StreetNameHelperV2; using StreetNameRegistry.Projections.Wms; - using LastChangedListContextMigrationFactory = StreetNameRegistry.Projections.LastChangedList.LastChangedListContextMigrationFactory; public sealed class ApiModule : Module { @@ -145,17 +143,6 @@ private void RegisterLastChangedProjections(ContainerBuilder builder) _configuration["DataDog:ServiceName"], _services, _loggerFactory)); - - builder - .RegisterProjectionMigrator( - _configuration, - _loggerFactory) - .RegisterProjectionMigrator( - _configuration, - _loggerFactory) - .RegisterProjections( - context => new LastChangedProjections(context.Resolve()), - ConnectedProjectionSettings.Default); } private void RegisterLegacyProjectionsV2(ContainerBuilder builder) diff --git a/src/StreetNameRegistry.Projector/Infrastructure/Startup.cs b/src/StreetNameRegistry.Projector/Infrastructure/Startup.cs index d8409b529..69e747d03 100755 --- a/src/StreetNameRegistry.Projector/Infrastructure/Startup.cs +++ b/src/StreetNameRegistry.Projector/Infrastructure/Startup.cs @@ -22,7 +22,6 @@ namespace StreetNameRegistry.Projector.Infrastructure using System.Reflection; using System.Threading; using Be.Vlaanderen.Basisregisters.Projector.ConnectedProjections; - using Microsoft.Extensions.Options; using StreetNameRegistry.Projections.Wfs; using StreetNameRegistry.Projections.Wms; using Microsoft.OpenApi.Models;