From 12160299a6c65877005fda2a2b27459ddcd61b1c Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Thu, 9 Nov 2023 14:56:46 +0000 Subject: [PATCH] feat: Simplify packages and apps - import third party packages as flake inputs - one overall derivation instead of multiple derivations Signed-off-by: Brian McGee --- flake.lock | 19 +++++- flake.nix | 10 +-- nix/apps.nix | 40 ++++++++++++ nix/default.nix | 2 + nix/devshell.nix | 55 +++++++++------- nix/packages.nix | 104 ++++++++++++++++++++++++++++++ sbomnix.nix | 40 ------------ scripts/default.nix | 10 --- scripts/nixupdate/default.nix | 64 ------------------ scripts/nixupdate/nix_outdated.py | 2 +- scripts/repology/default.nix | 91 -------------------------- scripts/vulnxscan/default.nix | 74 --------------------- setup.py | 4 ++ 13 files changed, 205 insertions(+), 310 deletions(-) create mode 100644 nix/apps.nix create mode 100644 nix/packages.nix delete mode 100644 sbomnix.nix delete mode 100644 scripts/default.nix delete mode 100644 scripts/nixupdate/default.nix delete mode 100644 scripts/repology/default.nix delete mode 100644 scripts/vulnxscan/default.nix diff --git a/flake.lock b/flake.lock index 6efb1cb..87ff0db 100644 --- a/flake.lock +++ b/flake.lock @@ -147,7 +147,8 @@ "nix-fast-build": "nix-fast-build", "nix-visualize": "nix-visualize", "nixpkgs": "nixpkgs_2", - "treefmt-nix": "treefmt-nix" + "treefmt-nix": "treefmt-nix", + "vulnix": "vulnix" } }, "treefmt-nix": { @@ -169,6 +170,22 @@ "repo": "treefmt-nix", "type": "github" } + }, + "vulnix": { + "flake": false, + "locked": { + "lastModified": 1676379453, + "narHash": "sha256-KXvmnaMjv//zd4aSwu4qmbon1Iyzdod6CPms7LIxeVU=", + "owner": "henrirosten", + "repo": "vulnix", + "rev": "ad28b2924027a44a9b81493a0f9de1b0e8641005", + "type": "github" + }, + "original": { + "owner": "henrirosten", + "repo": "vulnix", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 4d207d0..6294ad9 100644 --- a/flake.nix +++ b/flake.nix @@ -26,8 +26,12 @@ }; }; nix-visualize = { - url = "github:craigmbooth/nix-visualize"; - flake = false; + url = "github:craigmbooth/nix-visualize"; + flake = false; + }; + vulnix = { + url = "github:henrirosten/vulnix"; + flake = false; }; }; @@ -45,8 +49,6 @@ imports = [ ./nix - ./scripts - ./sbomnix.nix ]; }; } diff --git a/nix/apps.nix b/nix/apps.nix new file mode 100644 index 0000000..939c702 --- /dev/null +++ b/nix/apps.nix @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 +{ + perSystem = {self', ...}: { + apps = let + inherit (self'.packages) sbomnix; + in { + # nix run .#repology_cli + repology_cli = { + type = "app"; + program = "${sbomnix}/bin/repology_cli"; + }; + + # nix run .#repology_cve + repology_cve = { + type = "app"; + program = "${sbomnix}/bin/repology_cve"; + }; + + # nix run .#nix_outdated + nix_outdated = { + type = "app"; + program = "${sbomnix}/bin/nix_outdated"; + }; + + # nix run .#nix_outdated + nixgraph = { + type = "app"; + program = "${sbomnix}/bin/nixgraph"; + }; + + # nix run .#vulnxscan + vulnxscan = { + type = "app"; + program = "${sbomnix}/bin/vulnxscan"; + }; + }; + }; +} diff --git a/nix/default.nix b/nix/default.nix index 7af51c0..6229b6d 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -3,8 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 { imports = [ + ./apps.nix ./checks.nix ./devshell.nix + ./packages.nix ./treefmt.nix ]; } diff --git a/nix/devshell.nix b/nix/devshell.nix index 282f6be..1806e0b 100644 --- a/nix/devshell.nix +++ b/nix/devshell.nix @@ -4,39 +4,44 @@ { perSystem = { pkgs, - self', + lib, inputs', ... }: { devShells.default = let - pythonPackages = pkgs.python3Packages; + pp = pkgs.python3Packages; in pkgs.mkShell rec { name = "sbomnix-dev-shell"; - buildInputs = [ - pkgs.coreutils - pkgs.curl - pkgs.gnugrep - pkgs.gnused - pkgs.graphviz - pkgs.grype - pkgs.gzip - pkgs.nix - pkgs.reuse - pythonPackages.beautifulsoup4 - pythonPackages.colorlog - pythonPackages.graphviz - pythonPackages.numpy - pythonPackages.packageurl-python - pythonPackages.packaging - pythonPackages.pandas - pythonPackages.requests - pythonPackages.requests-cache - pythonPackages.tabulate - pythonPackages.venvShellHook - pythonPackages.wheel - inputs'.nix-fast-build.packages.default + buildInputs = lib.flatten [ + (with pkgs; [ + coreutils + curl + gnugrep + gnused + graphviz + grype + gzip + nix + reuse + ]) + (with pp; [ + beautifulsoup4 + colorlog + graphviz + numpy + packageurl-python + packaging + pandas + requests + requests-cache + tabulate + venvShellHook + wheel + ]) + + [inputs'.nix-fast-build.packages.default] ]; venvDir = "venv"; postShellHook = '' diff --git a/nix/packages.nix b/nix/packages.nix new file mode 100644 index 0000000..00bc860 --- /dev/null +++ b/nix/packages.nix @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 +{inputs, ...}: { + perSystem = { + pkgs, + lib, + ... + }: let + pp = pkgs.python3Packages; + + # We use vulnix from 'https://github.com/henrirosten/vulnix' to get + # vulnix support for runtime-only scan ('-C' command-line option) + # which is currently not available in released version of vulnix. + vulnix = (import inputs.vulnix) { + inherit (inputs) nixpkgs; # required but not used as we provide pkgs + inherit pkgs lib; + }; + + nix-visualize = (import inputs.nix-visualize) {inherit pkgs;}; + + pyrate-limiter = pp.buildPythonPackage rec { + version = "2.10.0"; + pname = "pyrate-limiter"; + format = "pyproject"; + + src = pkgs.fetchFromGitHub { + owner = "vutran1710"; + repo = "PyrateLimiter"; + rev = "v${version}"; + hash = "sha256-CPusPeyTS+QyWiMHsU0ii9ZxPuizsqv0wQy3uicrDw0="; + }; + + propagatedBuildInputs = [ + pp.poetry-core + ]; + }; + + requests-ratelimiter = pp.buildPythonPackage rec { + version = "0.4.0"; + pname = "requests-ratelimiter"; + format = "pyproject"; + + src = pkgs.fetchFromGitHub { + owner = "JWCook"; + repo = pname; + rev = "v${version}"; + hash = "sha256-F9bfcwijyyKzlFKBJAC/5ETc4/hZpPhm2Flckku2z6M="; + }; + + propagatedBuildInputs = [pyrate-limiter pp.requests]; + }; + in { + packages = rec { + default = sbomnix; + + sbomnix = pp.buildPythonPackage rec { + pname = "sbomnix"; + version = pkgs.lib.removeSuffix "\n" (builtins.readFile ../VERSION); + format = "setuptools"; + + src = lib.cleanSource ../.; + + propagatedBuildInputs = lib.flatten [ + [ + pyrate-limiter + requests-ratelimiter + ] + [pkgs.reuse] + (with pp; [ + beautifulsoup4 + colorlog + graphviz + numpy + packageurl-python + packaging + pandas + requests + requests-cache + tabulate + ]) + ]; + + pythonImportsCheck = ["sbomnix"]; + + postInstall = '' + + wrapProgram $out/bin/sbomnix \ + --prefix PATH : ${lib.makeBinPath [pkgs.nix pkgs.graphviz]} + + wrapProgram $out/bin/nixgraph \ + --prefix PATH : ${lib.makeBinPath [pkgs.nix pkgs.graphviz]} + + wrapProgram $out/bin/nix_outdated \ + --prefix PATH : ${lib.makeBinPath [nix-visualize]} + + wrapProgram $out/bin/vulnxscan \ + --prefix PATH : ${lib.makeBinPath [pkgs.grype pkgs.nix vulnix]} + + ''; + }; + }; + }; +} diff --git a/sbomnix.nix b/sbomnix.nix deleted file mode 100644 index a6c5dd2..0000000 --- a/sbomnix.nix +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 -{ - perSystem = { - pkgs, - self', - ... - }: let - pythonPackages = pkgs.python3Packages; - in { - packages.sbomnix = pythonPackages.buildPythonPackage rec { - pname = "sbomnix"; - version = pkgs.lib.removeSuffix "\n" (builtins.readFile ./VERSION); - format = "setuptools"; - - src = ./.; - makeWrapperArgs = [ - "--prefix PATH : ${pkgs.lib.makeBinPath [pkgs.nix pkgs.graphviz]}" - ]; - - propagatedBuildInputs = [ - pkgs.reuse - pythonPackages.colorlog - pythonPackages.graphviz - pythonPackages.numpy - pythonPackages.packageurl-python - pythonPackages.packaging - pythonPackages.pandas - pythonPackages.requests - pythonPackages.tabulate - ]; - pythonImportsCheck = ["sbomnix"]; - }; - apps.nixgraph = { - type = "app"; - program = "${self'.packages.sbomnix}/bin/nixgraph"; - }; - }; -} diff --git a/scripts/default.nix b/scripts/default.nix deleted file mode 100644 index 5eb1519..0000000 --- a/scripts/default.nix +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 -{ - imports = [ - ./nixupdate - ./repology - ./vulnxscan - ]; -} diff --git a/scripts/nixupdate/default.nix b/scripts/nixupdate/default.nix deleted file mode 100644 index fd1f88a..0000000 --- a/scripts/nixupdate/default.nix +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 -{ inputs, ... }: { - perSystem = { - pkgs, - self', - ... - }: let - pythonPackages = pkgs.python3Packages; - in { - packages = { - - nix_visualize = (import "${inputs.nix-visualize}/default.nix") { inherit pkgs; }; - - nixupdate = let - inherit - (self'.packages) - repology_cli - nix_visualize - requests-ratelimiter - ; - in - pythonPackages.buildPythonPackage rec { - pname = "nixupdate"; - version = pkgs.lib.removeSuffix "\n" (builtins.readFile ../../VERSION); - format = "setuptools"; - - src = ../../.; - - makeWrapperArgs = [ - "--prefix PATH : ${pkgs.lib.makeBinPath [repology_cli nix_visualize]}" - ]; - - propagatedBuildInputs = [ - pkgs.reuse - requests-ratelimiter - pythonPackages.beautifulsoup4 - pythonPackages.colorlog - pythonPackages.graphviz - pythonPackages.numpy - pythonPackages.packageurl-python - pythonPackages.packaging - pythonPackages.pandas - pythonPackages.tabulate - pythonPackages.requests - pythonPackages.requests-cache - ]; - - postInstall = '' - install -vD scripts/nixupdate/nix_outdated.py $out/bin/nix_outdated.py - ''; - - pythonImportsCheck = ["sbomnix"]; - }; - }; - apps = { - nix_outdated = { - type = "app"; - program = "${self'.packages.nixupdate}/bin/nix_outdated.py"; - }; - }; - }; -} diff --git a/scripts/nixupdate/nix_outdated.py b/scripts/nixupdate/nix_outdated.py index 9915003..5755f71 100755 --- a/scripts/nixupdate/nix_outdated.py +++ b/scripts/nixupdate/nix_outdated.py @@ -90,7 +90,7 @@ def _run_repology_cli(sbompath): suffix = ".csv" with NamedTemporaryFile(delete=False, prefix=prefix, suffix=suffix) as f: cmd = ( - "repology_cli.py " + "repology_cli " f"--sbom_cdx={sbompath} --repository=nix_unstable --out={f.name}" ) exec_cmd(cmd.split()) diff --git a/scripts/repology/default.nix b/scripts/repology/default.nix deleted file mode 100644 index fa7901c..0000000 --- a/scripts/repology/default.nix +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 -{ - perSystem = { - pkgs, - self', - ... - }: let - pythonPackages = pkgs.python3Packages; - in { - packages = { - pyrate-limiter = pythonPackages.buildPythonPackage rec { - version = "2.10.0"; - pname = "pyrate-limiter"; - format = "pyproject"; - - src = pkgs.fetchFromGitHub { - owner = "vutran1710"; - repo = "PyrateLimiter"; - rev = "v${version}"; - hash = "sha256-CPusPeyTS+QyWiMHsU0ii9ZxPuizsqv0wQy3uicrDw0="; - }; - - propagatedBuildInputs = with pythonPackages; [ - poetry-core - ]; - }; - - requests-ratelimiter = let - inherit (self'.packages) pyrate-limiter; - in - pythonPackages.buildPythonPackage rec { - version = "0.4.0"; - pname = "requests-ratelimiter"; - format = "pyproject"; - - src = pkgs.fetchFromGitHub { - owner = "JWCook"; - repo = pname; - rev = "v${version}"; - hash = "sha256-F9bfcwijyyKzlFKBJAC/5ETc4/hZpPhm2Flckku2z6M="; - }; - - propagatedBuildInputs = with pythonPackages; [ - poetry-core - pyrate-limiter - requests - ]; - }; - - repology_cli = let - inherit (self'.packages) requests-ratelimiter; - in - pythonPackages.buildPythonPackage rec { - pname = "repology_cli"; - version = pkgs.lib.removeSuffix "\n" (builtins.readFile ../../VERSION); - format = "setuptools"; - - src = ../../.; - - propagatedBuildInputs = [ - pkgs.reuse - requests-ratelimiter - pythonPackages.beautifulsoup4 - pythonPackages.colorlog - pythonPackages.graphviz - pythonPackages.numpy - pythonPackages.packageurl-python - pythonPackages.packaging - pythonPackages.pandas - pythonPackages.tabulate - pythonPackages.requests - pythonPackages.requests-cache - ]; - - postInstall = '' - install -vD scripts/repology/repology_cli.py $out/bin/repology_cli.py - install -vD scripts/repology/repology_cve.py $out/bin/repology_cve.py - ''; - - pythonImportsCheck = ["sbomnix"]; - - meta = { - # TODO add more meta attributes - mainProgram = "repology_cli.py"; - }; - }; - }; - }; -} diff --git a/scripts/vulnxscan/default.nix b/scripts/vulnxscan/default.nix deleted file mode 100644 index 9d9ce86..0000000 --- a/scripts/vulnxscan/default.nix +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) -# -# SPDX-License-Identifier: Apache-2.0 -{ - perSystem = { - pkgs, - self', - ... - }: { - packages = let - pythonPackages = pkgs.python3Packages; - in { - vulnix = - pkgs - .vulnix - .overrideAttrs ( - _old: rec { - # We use vulnix from 'https://github.com/henrirosten/vulnix' to get - # vulnix support for runtime-only scan ('-C' command-line option) - # which is currently not available in released version of vulnix. - src = pkgs.fetchFromGitHub { - owner = "henrirosten"; - repo = "vulnix"; - rev = "ad28b2924027a44a9b81493a0f9de1b0e8641005"; - sha256 = "sha256-KXvmnaMjv//zd4aSwu4qmbon1Iyzdod6CPms7LIxeVU="; - }; - version = "1.10.2.dev0"; - name = "vulnix-${version}"; - } - ); - - vulnxscan = let - inherit (self'.packages) sbomnix repology_cli requests-ratelimiter vulnix; - in - pythonPackages.buildPythonPackage rec { - pname = "vulnxscan"; - version = pkgs.lib.removeSuffix "\n" (builtins.readFile ../../VERSION); - format = "setuptools"; - - src = ../../.; - - makeWrapperArgs = [ - "--prefix PATH : ${pkgs.lib.makeBinPath [sbomnix repology_cli pkgs.grype pkgs.nix vulnix]}" - ]; - - propagatedBuildInputs = [ - pkgs.reuse - requests-ratelimiter - pythonPackages.beautifulsoup4 - pythonPackages.colorlog - pythonPackages.graphviz - pythonPackages.numpy - pythonPackages.packageurl-python - pythonPackages.packaging - pythonPackages.pandas - pythonPackages.tabulate - pythonPackages.requests - pythonPackages.requests-cache - ]; - - postInstall = '' - install -vD scripts/vulnxscan/vulnxscan.py $out/bin/vulnxscan.py - ''; - - pythonImportsCheck = ["sbomnix"]; - - meta = { - # TODO add more meta attributes - mainProgram = "vulnxscan.py"; - }; - }; - }; - }; -} diff --git a/setup.py b/setup.py index 8e3829a..b42a8e4 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,10 @@ def project_path(*names): "console_scripts": [ "sbomnix = sbomnix.main:main", "nixgraph = nixgraph.main:main", + "nix_outdated = scripts.nixupdate.nix_outdated:main", + "vulnxscan = scripts.vulnxscan.vulnxscan:main", + "repology_cli = scripts.repology.repology_cli:main", + "repology_cve = scripts.repology.repology_cve:main" ] }, )