feat: support multiple components (#85)

* fix: target_compatible_with excluding arm64 The 'ci: update ci and exclude windows' commit added 'target_compatible_with' arguments to some of the tests restricting them to linux but it also forced amd64 instead of allowing the tests to run in both linux/amd64 and linux/arm64. Before this fix, I would get SKIPPED tests when testing on linux/arm64 and now all the tests run. * chore: updating locks Just doing 'bazel run' on the locks (except apt_security which is lock-less) and runnign the tests for linux/amd64 and linux/arm64: ``` bazel run @bullseye//:lock bazel run @shared_dependencies//:lock bazel run @noble//:lock bazel test //... ``` produces these changes, which seem to be, mostly, * tabs vs 2 spaces in lock.json * keys with '_' instead of '~' * some tooling in MODULES.lock * feat: Support multiple components Add support for multiple sources.list components (e.g. "main contrib"). Before, all of the components were being parsed as one string which resulted in _fetch_package_index() failing. See https://wiki.debian.org/SourcesList#sources.list_format and https://wiki.debian.org/DebianRepository/Format#Components --------- Co-authored-by: Sahin Yort <[email protected]>
GoogleContainerTools · Sep 12, 2024 · 2ce7b47 · 2ce7b47
1 parent 8a2174a
commit 2ce7b47
Show file tree

Hide file tree

Showing 11 changed files with 55 additions and 11 deletions.
diff --git a/apt/private/resolve.bzl b/apt/private/resolve.bzl
@@ -39,16 +39,14 @@ def internal_resolve(rctx, yq_toolchain_prefix, manifest, include_transitive):
 
     sources = []
 
-    dist = None
     for src in manifest["sources"]:
-        (distr, _, comp) = src["channel"].partition(" ")
-        if not dist:
-            dist = distr
-        sources.append((
-            src["url"],
-            distr,
-            comp,
-        ))
+        distr, components = src["channel"].split(" ", 1)
+        for comp in components.split(" "):
+            sources.append((
+                src["url"],
+                distr,
+                comp,
+            ))
 
     pkgindex = package_index.new(rctx, sources = sources, archs = manifest["archs"])
     pkgresolution = package_resolution.new(index = pkgindex)

diff --git a/e2e/smoke/BUILD b/e2e/smoke/BUILD
@@ -63,6 +63,7 @@ PACKAGES = [
     "@bullseye//dpkg",
     "@bullseye//apt",
     "@bullseye//perl",
+    "@bullseye//nvidia-kernel-common",
 ]
 
 # Creates /var/lib/dpkg/status with installed package information.

diff --git a/e2e/smoke/bullseye.lock.json b/e2e/smoke/bullseye.lock.json
@@ -1190,6 +1190,15 @@
 			"url": "https://snapshot.debian.org/archive/debian/20240210T223313Z/pool/main/o/openssl/openssl_1.1.1w-0+deb11u1_amd64.deb",
 			"version": "1.1.1w-0+deb11u1"
 		},
+		{
+			"arch": "amd64",
+			"dependencies": [],
+			"key": "nvidia-kernel-common_20151021-p-13_amd64",
+			"name": "nvidia-kernel-common",
+			"sha256": "fa4b007bf64cf8cf0e9b3aaae5dd388fcec3a589ce2475f16d64347945691533",
+			"url": "https://snapshot.debian.org/archive/debian/20240210T223313Z/pool/contrib/n/nvidia-support/nvidia-kernel-common_20151021+13_amd64.deb",
+			"version": "20151021+13"
+		},
 		{
 			"arch": "arm64",
 			"dependencies": [],
@@ -2379,6 +2388,15 @@
 			"sha256": "d9159af073e95641e7eda440fa1d7623873b8c0034c9826a353f890bed107f3c",
 			"url": "https://snapshot.debian.org/archive/debian/20240210T223313Z/pool/main/o/openssl/openssl_1.1.1w-0+deb11u1_arm64.deb",
 			"version": "1.1.1w-0+deb11u1"
+		},
+		{
+			"arch": "arm64",
+			"dependencies": [],
+			"key": "nvidia-kernel-common_20151021-p-13_arm64",
+			"name": "nvidia-kernel-common",
+			"sha256": "5a1f10cffc5407a6b63dcdf3f72af842ad9e39a808bb9418a4805aa0be345c65",
+			"url": "https://snapshot.debian.org/archive/debian/20240210T223313Z/pool/contrib/n/nvidia-support/nvidia-kernel-common_20151021+13_arm64.deb",
+			"version": "20151021+13"
 		}
 	],
 	"version": 1

diff --git a/e2e/smoke/bullseye.yaml b/e2e/smoke/bullseye.yaml
@@ -10,7 +10,7 @@
 version: 1
 
 sources:
-  - channel: bullseye main
+  - channel: bullseye main contrib
     url: https://snapshot.debian.org/archive/debian/20240210T223313Z
   - channel: bullseye-security main
     url: https://snapshot.debian.org/archive/debian-security/20240210T223313Z
@@ -33,3 +33,5 @@ packages:
   - "perl"
   # test cacerts() compatibility
   - "ca-certificates"
+  # test multiple components: contrib
+  - "nvidia-kernel-common"
diff --git a/e2e/smoke/test_linux_amd64.yaml b/e2e/smoke/test_linux_amd64.yaml
@@ -18,6 +18,7 @@ commandTests:
       - ncurses-base/now 6\.2\+20201114-2\+deb11u2 all \[installed,local\]
       - perl/now 5\.32\.1-4\+deb11u3 amd64 \[installed,local\]
       - tzdata/now 2024a-0\+deb11u1 all \[installed,local\]
+      - nvidia-kernel-common/now 20151021\+13 amd64 \[installed,local\]
   - name: "whoami"
     command: "whoami"
     expectedOutput: [r00t]

diff --git a/e2e/smoke/test_linux_arm64.yaml b/e2e/smoke/test_linux_arm64.yaml
@@ -18,6 +18,7 @@ commandTests:
       - ncurses-base/now 6\.2\+20201114-2\+deb11u2 all \[installed,local\]
       - perl/now 5\.32\.1-4\+deb11u3 arm64 \[installed,local\]
       - tzdata/now 2024a-0\+deb11u1 all \[installed,local\]
+      - nvidia-kernel-common/now 20151021\+13 arm64 \[installed,local\]
   - name: "whoami"
     command: "whoami"
     expectedOutput: [r00t]

diff --git a/examples/debian_snapshot/BUILD.bazel b/examples/debian_snapshot/BUILD.bazel
@@ -64,6 +64,7 @@ PACKAGES = [
     "@bullseye//apt",
     "@bullseye//perl",
     "@bullseye//openssl",
+    "@bullseye//nvidia-kernel-common",
 ]
 
 # Creates /var/lib/dpkg/status with installed package information.

diff --git a/examples/debian_snapshot/bullseye.lock.json b/examples/debian_snapshot/bullseye.lock.json
@@ -1190,6 +1190,15 @@
 			"url": "https://snapshot-cloudflare.debian.org/archive/debian/20240210T223313Z/pool/main/o/openssl/openssl_1.1.1w-0+deb11u1_amd64.deb",
 			"version": "1.1.1w-0+deb11u1"
 		},
+		{
+			"arch": "amd64",
+			"dependencies": [],
+			"key": "nvidia-kernel-common_20151021-p-13_amd64",
+			"name": "nvidia-kernel-common",
+			"sha256": "fa4b007bf64cf8cf0e9b3aaae5dd388fcec3a589ce2475f16d64347945691533",
+			"url": "https://snapshot-cloudflare.debian.org/archive/debian/20240210T223313Z/pool/contrib/n/nvidia-support/nvidia-kernel-common_20151021+13_amd64.deb",
+			"version": "20151021+13"
+		},
 		{
 			"arch": "arm64",
 			"dependencies": [],
@@ -2379,6 +2388,15 @@
 			"sha256": "d9159af073e95641e7eda440fa1d7623873b8c0034c9826a353f890bed107f3c",
 			"url": "https://snapshot-cloudflare.debian.org/archive/debian/20240210T223313Z/pool/main/o/openssl/openssl_1.1.1w-0+deb11u1_arm64.deb",
 			"version": "1.1.1w-0+deb11u1"
+		},
+		{
+			"arch": "arm64",
+			"dependencies": [],
+			"key": "nvidia-kernel-common_20151021-p-13_arm64",
+			"name": "nvidia-kernel-common",
+			"sha256": "5a1f10cffc5407a6b63dcdf3f72af842ad9e39a808bb9418a4805aa0be345c65",
+			"url": "https://snapshot-cloudflare.debian.org/archive/debian/20240210T223313Z/pool/contrib/n/nvidia-support/nvidia-kernel-common_20151021+13_arm64.deb",
+			"version": "20151021+13"
 		}
 	],
 	"version": 1

diff --git a/examples/debian_snapshot/bullseye.yaml b/examples/debian_snapshot/bullseye.yaml
@@ -10,7 +10,7 @@
 version: 1
 
 sources:
-  - channel: bullseye main
+  - channel: bullseye main contrib
     url: https://snapshot-cloudflare.debian.org/archive/debian/20240210T223313Z
   - channel: bullseye-security main
     url: https://snapshot-cloudflare.debian.org/archive/debian-security/20240210T223313Z
@@ -33,3 +33,5 @@ packages:
   - "perl"
   # test cacerts() compatibility
   - "ca-certificates"
+  # test multiple components: contrib
+  - "nvidia-kernel-common"
diff --git a/examples/debian_snapshot/test_linux_amd64.yaml b/examples/debian_snapshot/test_linux_amd64.yaml
@@ -18,6 +18,7 @@ commandTests:
       - ncurses-base/now 6\.2\+20201114-2\+deb11u2 all \[installed,local\]
       - perl/now 5\.32\.1-4\+deb11u3 amd64 \[installed,local\]
       - tzdata/now 2024a-0\+deb11u1 all \[installed,local\]
+      - nvidia-kernel-common/now 20151021\+13 amd64 \[installed,local\]
   - name: "whoami"
     command: "whoami"
     expectedOutput: [r00t]

diff --git a/examples/debian_snapshot/test_linux_arm64.yaml b/examples/debian_snapshot/test_linux_arm64.yaml
@@ -18,6 +18,7 @@ commandTests:
       - ncurses-base/now 6\.2\+20201114-2\+deb11u2 all \[installed,local\]
       - perl/now 5\.32\.1-4\+deb11u3 arm64 \[installed,local\]
       - tzdata/now 2024a-0\+deb11u1 all \[installed,local\]
+      - nvidia-kernel-common/now 20151021\+13 arm64 \[installed,local\]
   - name: "whoami"
     command: "whoami"
     expectedOutput: [r00t]