refactor(crawler): save only new versions

aquasecurity · Dec 26, 2024 · fc11d46 · fc11d46
1 parent bbbe9d8
commit fc11d46
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 1 deletion.
diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go
@@ -288,6 +288,11 @@ func (c *Crawler) crawlSHA1(ctx context.Context, baseURL string, meta *Metadata,
 			})
 		}
 
+		versions = lo.Filter(versions, func(v types.Version, _ int) bool {
+			_, ok := savedVersion[v.Version]
+			return !ok
+		})
+
 		foundVersions = append(foundVersions, versions...)
 	}
 

diff --git a/pkg/crawler/crawler_test.go b/pkg/crawler/crawler_test.go
@@ -67,7 +67,7 @@ func TestCrawl(t *testing.T) {
 				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1":        "testdata/happy/abbot-1.4.0.jar.sha1",
 				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.sha1":   "testdata/happy/abbot-1.4.0-lite.jar.sha1",
 			},
-			goldenPath: "testdata/happy/abbot.json.golden",
+			goldenPath: "testdata/happy/abbot-with-db.json.golden",
 			filePath:   "indexes/abbot/abbot.json",
 		},
 		{

diff --git a/pkg/crawler/testdata/happy/abbot-with-db.json.golden b/pkg/crawler/testdata/happy/abbot-with-db.json.golden
@@ -0,0 +1,15 @@
+{
+  "GroupID": "abbot",
+  "ArtifactID": "abbot",
+  "Versions": [
+    {
+      "Version": "1.4.0-lite",
+      "SHA1": "BUerA3Bor6ICaSW9lL+5/Pzsl2E="
+    },
+    {
+      "Version": "1.4.0",
+      "SHA1": "ojY2RqndBZVWM7RQAQtZohr4pCM="
+    }
+  ],
+  "ArchiveType": "jar"
+}