Merge pull request #44 from iPromKnight/zyclops

feat: completed generic ingestion with service discovery
iPromKnight · Nov 16, 2024 · 973e6fc · 973e6fc
2 parents a1544b5 + 0db28ba
commit 973e6fc
Show file tree

Hide file tree

Showing 69 changed files with 864 additions and 178 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,4 @@ riderModule.iml
 .DS_Store
 .idea/
 
-src/Zilean.DmmScraper/python/
+src/Zilean.Scraper/python/
diff --git a/.run/Zilean.DmmScraper.run.xml b/.run/Zilean.DmmScraper.run.xml
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -25,10 +25,12 @@
     <PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="9.0.0" />
     <PackageVersion Include="Scalar.AspNetCore" Version="1.2.37" />
     <PackageVersion Include="SimCube.Aspire" Version="1.0.5" />
+    <PackageVersion Include="Spectre.Console.Cli" Version="0.49.2-preview.0.50" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="Testcontainers" Version="4.0.0" />
     <PackageVersion Include="Testcontainers.PostgreSql" Version="4.0.0" />
-    <PackageVersion Include="Verify.Xunit" Version="28.2.1" />
+    <PackageVersion Include="KubernetesClient" Version="15.0.1" />
+    <PackageVersion Include="Verify.Xunit" Version="28.3.1" />
     <PackageVersion Include="xunit" Version="2.9.2" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2">
       <PrivateAssets>all</PrivateAssets>

diff --git a/Dockerfile b/Dockerfile
@@ -6,7 +6,7 @@ COPY . .
 RUN dotnet restore -a $TARGETARCH
 WORKDIR /build/src/Zilean.ApiService
 RUN dotnet publish -c Release --no-restore -a $TARGETARCH -o /app/out
-WORKDIR /build/src/Zilean.DmmScraper
+WORKDIR /build/src/Zilean.Scraper
 RUN dotnet publish -c Release --no-restore -a $TARGETARCH -o /app/out
 
 # Run Stage

diff --git a/Zilean.sln b/Zilean.sln
@@ -17,6 +17,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{49EC8AF3-9A9
 		README.md = README.md
 		eng\compose-dev.yaml = eng\compose-dev.yaml
 		eng\.dockerignore = eng\.dockerignore
+		eng\dev-settings.json = eng\dev-settings.json
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "k6", "k6", "{3F62995D-FA0F-434F-B90A-5E59C5960A4A}"
@@ -31,7 +32,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "github", "github", "{CF4C96
 		.github\workflows\cicd.yaml = .github\workflows\cicd.yaml
 	EndProjectSection
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Zilean.DmmScraper", "src\Zilean.DmmScraper\Zilean.DmmScraper.csproj", "{192B767E-7CFA-4F72-A9CA-35BE2CEE46FD}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Zilean.Scraper", "src\Zilean.Scraper\Zilean.Scraper.csproj", "{192B767E-7CFA-4F72-A9CA-35BE2CEE46FD}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Zilean.Shared", "src\Zilean.Shared\Zilean.Shared.csproj", "{85F3D012-6CA4-43D3-A08F-0A93FA739F46}"
 EndProject

diff --git a/Zilean.sln.DotSettings.user b/Zilean.sln.DotSettings.user
@@ -1,10 +1,19 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ABoundedChannelFullMode_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F655a81eeb02548579e1e5404da9d688c22938_003Fbf_003F4afb42f7_003FBoundedChannelFullMode_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ABulkConfig_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fd6b09cbbbc4e7bf228e82618e4e0859c91be3b93407de754227ae1a771ea2a6_003FBulkConfig_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ACallSiteFactory_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Ffc2027f7e776fc105cddb56b1a25eeb3895b3ae6f3aac854d786e63bd01f75e2_003FCallSiteFactory_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ACronExpression_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F2c78f2a38e28ff68373887f2c71ed6595fcac3a938221853fc708d5ef51f739_003FCronExpression_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADbContextBulkExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fd1f8f68864fcaf754b3aefb6cb6777e56a17cfd9217db17992967674aa5cd1d_003FDbContextBulkExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADbContextBulkTransaction_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F1dfe90a67dda13ce28d4a1975cde7e7445f47ecbb8b4d73dd1bcc0c01acd88_003FDbContextBulkTransaction_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc73b3c6c598640c592fd3c6fa226c286e90908_003F37_003F6c0b0159_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fbd1d5c50194fea68ff3559c160230b0ab50f5acf4ce3061bffd6d62958e2182_003FExceptionDispatchInfo_002Ecs_002Fz_003A2_002D1/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbd1d5c50194fea68ff3559c160230b0ab50f5acf4ce3061bffd6d62958e2182_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpProtocol_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F234a93d637d5e17f1de0ec244dc09d402281d71bab8d9d5bb507e14f1c26c9_003FHttpProtocol_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIScheduledEventConfiguration_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F1f1f4f12a09ac651d3b94226de6f7b5a6cda93c54e23d9707c1cf4a3c1a417_003FIScheduledEventConfiguration_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AJsonSerializer_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F5148e388a7db4994ad2ab3750386454116e910_003F72_003Fc63c4ed4_003FJsonSerializer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AOpenApiServiceCollectionExtensions_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fba408bbefe5c813576b4417369c21d24248b94d040ec72a9753bc546f9308190_003FOpenApiServiceCollectionExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AScheduledEvent_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fb691f6facce7874b361c15a72a4e2181eaa3456f6484b161d7313b6321136d3_003FScheduledEvent_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AServiceCollection_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F595e4f2361fd31cb66c7e9303f46cf8f2fe77d96eea9d0e42c17cae783aa74_003FServiceCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 
 
 

diff --git a/eng/dev-settings.json b/eng/dev-settings.json
@@ -0,0 +1,51 @@
+{
+  "Zilean": {
+    "Dmm": {
+      "EnableScraping": true,
+      "EnableEndpoint": true,
+      "ScrapeSchedule": "0 * * * *",
+      "MinimumReDownloadIntervalMinutes": 30,
+      "MaxFilteredResults": 200,
+      "MinimumScoreMatch": 0.85,
+      "ImportBatched": false
+    },
+    "Torznab": {
+      "EnableEndpoint": true
+    },
+    "Database": {
+      "ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=300;"
+    },
+    "Prowlarr": {
+      "EnableEndpoint": true
+    },
+    "Torrents": {
+      "EnableEndpoint": false
+    },
+    "Imdb": {
+      "EnableImportMatching": true,
+      "EnableEndpoint": true,
+      "MinimumScoreMatch": 0.85
+    },
+    "Ingestion": {
+      "ZurgInstances": [],
+      "ZileanInstances": [],
+      "EnableScraping": false,
+      "Kubernetes": {
+        "EnableServiceDiscovery": false,
+        "KubernetesSelectors": [
+          {
+            "UrlTemplate": "http://zurg.{0}:9999",
+            "LabelSelector": "app.elfhosted.com/name=zurg",
+            "EndpointType": 1
+          }
+        ],
+        "KubeConfigFile": "/$HOME/.kube/config"
+      },
+      "BatchSize": 500,
+      "MaxChannelSize": 5000,
+      "ScrapeSchedule": "0 * * * *",
+      "ZurgEndpointSuffix": "/debug/torrents",
+      "ZileanEndpointSuffix": "/torrents/all"
+    }
+  }
+}
diff --git a/src/Zilean.ApiService/Features/Bootstrapping/ServiceCollectionExtensions.cs b/src/Zilean.ApiService/Features/Bootstrapping/ServiceCollectionExtensions.cs
@@ -1,5 +1,3 @@
-using Zilean.Database;
-
 namespace Zilean.ApiService.Features.Bootstrapping;
 
 [ExcludeFromCodeCoverage]
@@ -11,15 +9,16 @@ public static IServiceCollection AddSwaggerSupport(this IServiceCollection servi
     public static IServiceCollection AddSchedulingSupport(this IServiceCollection services) =>
         services.AddScheduler();
 
-    public static IServiceCollection AddDataBootStrapping(this IServiceCollection services) =>
-        services.AddHostedService<BootstrapIndexesService>();
+    public static IServiceCollection AddStartupHostedService(this IServiceCollection services) =>
+        services.AddHostedService<StartupService>();
 
     public static IServiceCollection ConditionallyRegisterDmmJob(this IServiceCollection services, ZileanConfiguration configuration)
     {
         if (configuration.Dmm.EnableScraping)
         {
             services.AddTransient<DmmSyncJob>();
-            services.AddSingleton<DmmSyncOnDemandState>();
+            services.AddTransient<GenericSyncJob>();
+            services.AddSingleton<SyncOnDemandState>();
         }
 
         return services;
@@ -33,7 +32,14 @@ public static IServiceProvider SetupScheduling(this IServiceProvider provider, Z
                 {
                     scheduler.Schedule<DmmSyncJob>()
                         .Cron(configuration.Dmm.ScrapeSchedule)
-                        .PreventOverlapping(nameof(DmmSyncJob));
+                        .PreventOverlapping("SyncJobs");
+                }
+
+                if (configuration.Ingestion.EnableScraping)
+                {
+                    scheduler.Schedule<GenericSyncJob>()
+                        .Cron(configuration.Ingestion.ScrapeSchedule)
+                        .PreventOverlapping("SyncJobs");
                 }
             })
             .LogScheduledTaskProgress();

diff --git a/.../Bootstrapping/BootstrapIndexesService.cs → .../Features/Bootstrapping/StartupService.cs b/.../Bootstrapping/BootstrapIndexesService.cs → .../Features/Bootstrapping/StartupService.cs
@@ -1,8 +1,6 @@
-using Zilean.Database;
-
 namespace Zilean.ApiService.Features.Bootstrapping;
 
-public class BootstrapIndexesService(
+public class StartupService(
     ZileanConfiguration configuration,
     IShellExecutionService executionService,
     IServiceProvider serviceProvider,
@@ -14,7 +12,7 @@ public class BootstrapIndexesService(
 
     public async Task StartingAsync(CancellationToken cancellationToken)
     {
-        var logger = loggerFactory.CreateLogger<BootstrapIndexesService>();
+        var logger = loggerFactory.CreateLogger<StartupService>();
         logger.LogInformation("Applying Migrations...");
         await using var asyncScope = serviceProvider.CreateAsyncScope();
         var dbContext = asyncScope.ServiceProvider.GetRequiredService<ZileanDbContext>();
@@ -28,10 +26,16 @@ public async Task StartingAsync(CancellationToken cancellationToken)
 
     public async Task StartedAsync(CancellationToken cancellationToken)
     {
-        if (configuration.Dmm.EnableScraping && DmmSyncJob.ShouldRunOnStartup())
+        if (configuration.Dmm.EnableScraping)
         {
-            var dmmJob = new DmmSyncJob(executionService, loggerFactory.CreateLogger<DmmSyncJob>());
-            await dmmJob.Invoke();
+            await using var asyncScope = serviceProvider.CreateAsyncScope();
+            var dbContext = asyncScope.ServiceProvider.GetRequiredService<ZileanDbContext>();
+            var dmmJob = new DmmSyncJob(executionService, loggerFactory.CreateLogger<DmmSyncJob>(), dbContext);
+            var shouldRun = await dmmJob.ShouldRunOnStartup();
+            if (shouldRun)
+            {
+                await dmmJob.Invoke();
+            }
         }
     }
 }
diff --git a/src/Zilean.ApiService/Features/Bootstrapping/WebApplicationExtensions.cs b/src/Zilean.ApiService/Features/Bootstrapping/WebApplicationExtensions.cs
@@ -15,5 +15,6 @@ public static WebApplication MapZileanEndpoints(this WebApplication app, ZileanC
             .MapDmmEndpoints(configuration)
             .MapImdbEndpoints(configuration)
             .MapTorznabEndpoints(configuration)
+            .MapTorrentsEndpoints(configuration)
             .MapHealthCheckEndpoints();
 }
diff --git a/src/Zilean.ApiService/Features/Dmm/DmmSyncJob.cs b/src/Zilean.ApiService/Features/Dmm/DmmSyncJob.cs
diff --git a/src/Zilean.ApiService/Features/Dmm/DmmSyncOnDemandState.cs b/src/Zilean.ApiService/Features/Dmm/DmmSyncOnDemandState.cs
diff --git a/...n.ApiService/Features/Dmm/DmmEndpoints.cs → ...ervice/Features/Search/SearchEndpoints.cs b/...n.ApiService/Features/Dmm/DmmEndpoints.cs → ...ervice/Features/Search/SearchEndpoints.cs
@@ -1,6 +1,6 @@
-namespace Zilean.ApiService.Features.Dmm;
+namespace Zilean.ApiService.Features.Search;
 
-public static class DmmEndpoints
+public static class SearchEndpoints
 {
     private const string GroupName = "dmm";
     private const string Search = "/search";
@@ -34,15 +34,15 @@ private static RouteGroupBuilder Dmm(this RouteGroupBuilder group)
         return group;
     }
 
-    private static async Task PerformOnDemandScrape(HttpContext context, ILogger<GeneralInstance> logger, IShellExecutionService executionService, ILogger<DmmSyncJob> syncLogger, IMutex mutex, DmmSyncOnDemandState state)
+    private static async Task PerformOnDemandScrape(HttpContext context, ILogger<GeneralInstance> logger, IShellExecutionService executionService, ILogger<DmmSyncJob> syncLogger, IMutex mutex, SyncOnDemandState state, ZileanDbContext dbContext)
     {
         if (state.IsRunning)
         {
             logger.LogWarning("On-demand scrape already running.");
             return;
         }
 
-        logger.LogInformation("Trying to schedule on-demand scrape with a 5 minute timeout on lock acquisition.");
+        logger.LogInformation("Trying to schedule on-demand scrape with a 1 minute timeout on lock acquisition.");
 
         bool available = mutex.TryGetLock(nameof(DmmSyncJob), 1);
 
@@ -52,7 +52,7 @@ private static async Task PerformOnDemandScrape(HttpContext context, ILogger<Gen
             {
                 logger.LogInformation("On-demand scrape mutex lock acquired.");
                 state.IsRunning = true;
-                await new DmmSyncJob(executionService, syncLogger).Invoke();
+                await new DmmSyncJob(executionService, syncLogger, dbContext).Invoke();
             }
             finally
             {
@@ -91,7 +91,7 @@ private static async Task<Ok<TorrentInfo[]>> PerformSearch(HttpContext context,
         }
     }
 
-    private static async Task<Ok<TorrentInfo[]>> PerformFilteredSearch(HttpContext context, ITorrentInfoService torrentInfoService, ZileanConfiguration configuration, ILogger<DmmFilteredInstance> logger, [AsParameters] DmmFilteredRequest request)
+    private static async Task<Ok<TorrentInfo[]>> PerformFilteredSearch(HttpContext context, ITorrentInfoService torrentInfoService, ZileanConfiguration configuration, ILogger<DmmFilteredInstance> logger, [AsParameters] SearchFilteredRequest request)
     {
 
         try

diff --git a/...ervice/Features/Dmm/DmmFilteredRequest.cs → .../Features/Search/SearchFilteredRequest.cs b/...ervice/Features/Dmm/DmmFilteredRequest.cs → .../Features/Search/SearchFilteredRequest.cs
@@ -1,6 +1,6 @@
-namespace Zilean.ApiService.Features.Dmm;
+namespace Zilean.ApiService.Features.Search;
 
-public class DmmFilteredRequest
+public class SearchFilteredRequest
 {
     public string? Query { get; init; }
     public int? Season { get; init; }