From fc11d46c75a051ac9a2a472463a00942810f27ee Mon Sep 17 00:00:00 2001
From: DmitriyLewen <dmitriy.lewen@smartforce.io>
Date: Thu, 26 Dec 2024 15:58:19 +0600
Subject: [PATCH] refactor(crawler): save only new versions

---
 pkg/crawler/crawler.go                            |  5 +++++
 pkg/crawler/crawler_test.go                       |  2 +-
 .../testdata/happy/abbot-with-db.json.golden      | 15 +++++++++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 pkg/crawler/testdata/happy/abbot-with-db.json.golden

diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go
index 33fb1d1..defe179 100644
--- a/pkg/crawler/crawler.go
+++ b/pkg/crawler/crawler.go
@@ -288,6 +288,11 @@ func (c *Crawler) crawlSHA1(ctx context.Context, baseURL string, meta *Metadata,
 			})
 		}
 
+		versions = lo.Filter(versions, func(v types.Version, _ int) bool {
+			_, ok := savedVersion[v.Version]
+			return !ok
+		})
+
 		foundVersions = append(foundVersions, versions...)
 	}
 
diff --git a/pkg/crawler/crawler_test.go b/pkg/crawler/crawler_test.go
index c53f5f6..55e51b4 100644
--- a/pkg/crawler/crawler_test.go
+++ b/pkg/crawler/crawler_test.go
@@ -67,7 +67,7 @@ func TestCrawl(t *testing.T) {
 				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1":        "testdata/happy/abbot-1.4.0.jar.sha1",
 				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.sha1":   "testdata/happy/abbot-1.4.0-lite.jar.sha1",
 			},
-			goldenPath: "testdata/happy/abbot.json.golden",
+			goldenPath: "testdata/happy/abbot-with-db.json.golden",
 			filePath:   "indexes/abbot/abbot.json",
 		},
 		{
diff --git a/pkg/crawler/testdata/happy/abbot-with-db.json.golden b/pkg/crawler/testdata/happy/abbot-with-db.json.golden
new file mode 100644
index 0000000..30e0a85
--- /dev/null
+++ b/pkg/crawler/testdata/happy/abbot-with-db.json.golden
@@ -0,0 +1,15 @@
+{
+  "GroupID": "abbot",
+  "ArtifactID": "abbot",
+  "Versions": [
+    {
+      "Version": "1.4.0-lite",
+      "SHA1": "BUerA3Bor6ICaSW9lL+5/Pzsl2E="
+    },
+    {
+      "Version": "1.4.0",
+      "SHA1": "ojY2RqndBZVWM7RQAQtZohr4pCM="
+    }
+  ],
+  "ArchiveType": "jar"
+}
\ No newline at end of file