Merge pull request #38 from iPromKnight/wipesometissue-corn

fix: Add filtering logic to exclude certain torrents during DMM scraping
iPromKnight · Nov 16, 2024 · d2d7c0e · d2d7c0e
2 parents f710926 + 50612a7
commit d2d7c0e
Showing 1 changed file with 10 additions and 2 deletions.
diff --git a/src/Zilean.DmmScraper/Features/Dmm/DmmScraping.cs b/src/Zilean.DmmScraper/Features/Dmm/DmmScraping.cs
@@ -107,7 +107,9 @@ await AnsiConsole.Progress()
 
                     logger.LogInformation("Distinct torrents: {Count}", distinctTorrents.Count);
 
-                    var finalizedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents);
+                    var parsedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents);
+
+                    var finalizedTorrents = parsedTorrents.Where(WipeSomeTissue).ToList();
 
                     await torrentInfoService.StoreTorrentInfo(finalizedTorrents);
                 }
@@ -150,7 +152,9 @@ await Parallel.ForEachAsync(files, parallelOptions, async (file, ct) =>
         {
             var distinctTorrents = torrents.DistinctBy(x => x.InfoHash).ToList();
 
-            var finalizedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents);
+            var parsedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents);
+
+            var finalizedTorrents = parsedTorrents.Where(WipeSomeTissue).ToList();
 
             logger.LogInformation("Parsed {Count} torrents", finalizedTorrents.Count);
 
@@ -185,4 +189,8 @@ private static async IAsyncEnumerable<ExtractedDmmEntry> ProcessFileAsync(string
             yield return torrent;
         }
     }
+
+    private static bool WipeSomeTissue(TorrentInfo torrent) =>
+        torrent.RawTitle.Contains(" XXX ", StringComparison.OrdinalIgnoreCase) &&
+        !torrent.ParsedTitle.Contains("XXX", StringComparison.OrdinalIgnoreCase);
 }