scripts: zephyr_module: Add URL, PURL and version to SPDX

Improve the SPDX with the current values: - URL: extracted from `git remote`. If more than one remote, URL is not set. - Version: extracted from `git rev-parse` (commit id). - PURL: generated from URL and Version. For zephyr, the tag is extracted, if present, and replace the commit id for the version field. Since official modules does not have tags, tags are not yet extracted for modules. Signed-off-by: Thomas Gagneret <[email protected]>
zephyrproject-rtos · Jan 23, 2024 · 71a0628 · 71a0628
1 parent 2141bb4
commit 71a0628
Show file tree

Hide file tree

Showing 9 changed files with 196 additions and 56 deletions.
diff --git a/.github/workflows/twister.yaml b/.github/workflows/twister.yaml
@@ -230,7 +230,7 @@ jobs:
           export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
           ./scripts/twister --subset ${{matrix.subset}}/${{ strategy.job-total }} ${TWISTER_COMMON} ${PUSH_OPTIONS}
           if [ "${{matrix.subset}}" = "1" ]; then
-            ./scripts/zephyr_module.py --twister-out module_tests.args
+            ./scripts/zephyr_module.py --zephyr-base ${ZEPHYR_BASE} --twister-out module_tests.args
             if [ -s module_tests.args ]; then
               ./scripts/twister +module_tests.args --outdir module_tests ${TWISTER_COMMON} ${PUSH_OPTIONS}
             fi
@@ -245,7 +245,7 @@ jobs:
           python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request
           ./scripts/twister --subset ${{matrix.subset}}/${{ strategy.job-total }} --load-tests testplan.json ${TWISTER_COMMON} ${PR_OPTIONS}
           if [ "${{matrix.subset}}" = "1" -a ${{needs.twister-build-prep.outputs.fullrun}} = 'True' ]; then
-            ./scripts/zephyr_module.py --twister-out module_tests.args
+            ./scripts/zephyr_module.py --zephyr-base ${ZEPHYR_BASE} --twister-out module_tests.args
             if [ -s module_tests.args ]; then
               ./scripts/twister +module_tests.args --outdir module_tests ${TWISTER_COMMON} ${PR_OPTIONS}
             fi
@@ -258,7 +258,7 @@ jobs:
           export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
           ./scripts/twister --subset ${{matrix.subset}}/${{ strategy.job-total }} ${TWISTER_COMMON} ${DAILY_OPTIONS}
           if [ "${{matrix.subset}}" = "1" ]; then
-            ./scripts/zephyr_module.py --twister-out module_tests.args
+            ./scripts/zephyr_module.py --zephyr-base ${ZEPHYR_BASE} --twister-out module_tests.args
             if [ -s module_tests.args ]; then
               ./scripts/twister +module_tests.args --outdir module_tests ${TWISTER_COMMON} ${DAILY_OPTIONS}
             fi

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1621,9 +1621,8 @@ if(CONFIG_BUILD_OUTPUT_BIN AND CONFIG_BUILD_OUTPUT_UF2)
   set(BYPRODUCT_KERNEL_UF2_NAME "${PROJECT_BINARY_DIR}/${KERNEL_UF2_NAME}" CACHE FILEPATH "Kernel uf2 file" FORCE)
 endif()
 
+set(KERNEL_META_PATH  ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "")
 if(CONFIG_BUILD_OUTPUT_META)
-  set(KERNEL_META_PATH  ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "")
-
   list(APPEND
     post_build_commands
     COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/zephyr_module.py
@@ -1637,6 +1636,9 @@ if(CONFIG_BUILD_OUTPUT_META)
     post_build_byproducts
     ${KERNEL_META_PATH}
   )
+else(CONFIG_BUILD_OUTPUT_META)
+  # Prevent spdx to use invalid data
+  file(REMOVE ${KERNEL_META_PATH})
 endif()
 
 # Cleanup intermediate files

diff --git a/cmake/modules/pre_dt.cmake b/cmake/modules/pre_dt.cmake
@@ -51,7 +51,7 @@ function(pre_dt_module_run)
   #
   # DTS directories can come from multiple places. Some places, like a
   # user's CMakeLists.txt can preserve symbolic links. Others, like
-  # scripts/zephyr_module.py --settings-out resolve them.
+  # scripts/zephyr_module.py --zephyr-base <ZEPHYR_BASE> --settings-out resolve them.
   unset(real_dts_root)
   foreach(dts_dir ${DTS_ROOT})
     file(REAL_PATH ${dts_dir} real_dts_dir)

diff --git a/scripts/ci/check_compliance.py b/scripts/ci/check_compliance.py
@@ -304,7 +304,7 @@ def get_modules(self, modules_file):
         zephyr_module_path = os.path.join(ZEPHYR_BASE, "scripts",
                                           "zephyr_module.py")
         cmd = [sys.executable, zephyr_module_path,
-               '--kconfig-out', modules_file]
+               '--zephyr-base', ZEPHYR_BASE, '--kconfig-out', modules_file]
         try:
             subprocess.run(cmd, check=True, stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)

diff --git a/scripts/kconfig/lint.py b/scripts/kconfig/lint.py
@@ -204,7 +204,7 @@ def init_kconfig():
 
     os.environ.update(
         srctree=TOP_DIR,
-        CMAKE_BINARY_DIR=modules_file_dir(),
+        CMAKE_BINARY_DIR=modules_file_dir(TOP_DIR),
         KCONFIG_DOC_MODE="1",
         ZEPHYR_BASE=TOP_DIR,
         SOC_DIR="soc",
@@ -215,13 +215,13 @@ def init_kconfig():
     kconf = kconfiglib.Kconfig(suppress_traceback=True)
 
 
-def modules_file_dir():
+def modules_file_dir(zephyr_base):
     # Creates Kconfig.modules in a temporary directory and returns the path to
     # the directory. Kconfig.modules brings in Kconfig files from modules.
 
     tmpdir = tempfile.mkdtemp()
     run((os.path.join("scripts", "zephyr_module.py"),
-         "--kconfig-out", os.path.join(tmpdir, "Kconfig.modules")))
+         "--zephyr-base", zephyr_base, "--kconfig-out", os.path.join(tmpdir, "Kconfig.modules")))
     return tmpdir
 
 

diff --git a/scripts/west_commands/zspdx/datatypes.py b/scripts/west_commands/zspdx/datatypes.py
@@ -68,6 +68,18 @@ def __init__(self):
         # SPDX ID, including "SPDXRef-"
         self.spdxID = ""
 
+        # package URL
+        self.url = ""
+
+        # package version
+        self.version = ""
+
+        # package revision
+        self.revision = ""
+
+        # package tags (for current commit)
+        self.tags = []
+
         # the Package's declared license
         self.declaredLicense = "NOASSERTION"
 

diff --git a/scripts/west_commands/zspdx/walker.py b/scripts/west_commands/zspdx/walker.py
@@ -4,6 +4,7 @@
 
 import os
 import yaml
+import re
 
 from west import log
 from west.util import west_topdir, WestNotFound
@@ -195,7 +196,7 @@ def setupBuildDocument(self):
         # add it to pending relationships queue
         self.pendingRelationships.append(rd)
 
-    def setupZephyrDocument(self, modules):
+    def setupZephyrDocument(self, zephyr, modules):
         # set up zephyr document
         cfgZephyr = DocumentConfig()
         cfgZephyr.name = "zephyr-sources"
@@ -215,13 +216,31 @@ def setupZephyrDocument(self, modules):
         cfgPackageZephyr.name = "zephyr-sources"
         cfgPackageZephyr.spdxID = "SPDXRef-zephyr-sources"
         cfgPackageZephyr.relativeBaseDir = relativeBaseDir
+        zephyr_url = zephyr.get("remote", "")
+        if zephyr_url:
+            cfgPackageZephyr.url = zephyr_url
+
+        zephyr_tags = zephyr.get("tags", "")
+        if zephyr_tags:
+            cfgPackageZephyr.tags = zephyr_tags
+            # Find tag vX.Y.Z
+            for tag in zephyr_tags:
+                tag = re.fullmatch(r'^v(?P<version>\d+\.\d+\.\d+)$', tag)
+                if tag:
+                    cfgPackageZephyr.version = tag.group('version')
+                    break
+
+        if zephyr.get("revision"):
+            cfgPackageZephyr.revision = zephyr.get("revision")
 
         pkgZephyr = Package(cfgPackageZephyr, self.docZephyr)
         self.docZephyr.pkgs[pkgZephyr.cfg.spdxID] = pkgZephyr
 
         for module in modules:
             module_name = module.get("name", None)
             module_path = module.get("path", None)
+            module_url = module.get("remote", None)
+            module_revision = module.get("revision", None)
 
             if not module_name:
                 log.err(f"cannot find module name in meta file; bailing")
@@ -236,6 +255,11 @@ def setupZephyrDocument(self, modules):
             cfgPackageZephyrModule.spdxID = "SPDXRef-" + module_name + "-sources"
             cfgPackageZephyrModule.relativeBaseDir = module_path
 
+            if module_revision:
+                cfgPackageZephyrModule.revision = module_revision
+
+            if module_url:
+                cfgPackageZephyrModule.url = module_url
             pkgZephyrModule = Package(cfgPackageZephyrModule, self.docZephyr)
             self.docZephyr.pkgs[pkgZephyrModule.cfg.spdxID] = pkgZephyrModule
 
@@ -250,6 +274,8 @@ def setupZephyrDocument(self, modules):
         # add it to pending relationships queue
         self.pendingRelationships.append(rd)
 
+        return True
+
     def setupSDKDocument(self):
         # set up SDK document
         cfgSDK = DocumentConfig()
@@ -287,7 +313,8 @@ def setupDocuments(self):
         try:
             with open(self.metaFile) as file:
                 content = yaml.load(file.read(), yaml.SafeLoader)
-                self.setupZephyrDocument(content["modules"])
+                if not self.setupZephyrDocument(content["zephyr"], content["modules"]):
+                    return False
         except (FileNotFoundError, yaml.YAMLError):
             log.err(f"cannot find a valid zephyr_meta.yml required for SPDX generation; bailing")
             return False

diff --git a/scripts/west_commands/zspdx/writer.py b/scripts/west_commands/zspdx/writer.py
@@ -8,6 +8,23 @@
 
 from zspdx.util import getHashes
 
+import re
+
+def build_purl(url, version=None):
+    purl = None
+    # This is designed to match repository with the following url pattern:
+    # '<protocol><base_url>/<namespace>/<package>
+    COMMON_GIT_URL_REGEX=r'((git@|http(s)?:\/\/)(?P<base_url>[\w\.@]+)(\/|:))(?P<namespace>[\w,\-,\_]+)\/(?P<package>[\w,\-,\_]+)(.git){0,1}((\/){0,1})$'
+
+    match = re.fullmatch(COMMON_GIT_URL_REGEX, url)
+    if match:
+        purl = f'pkg:{match.group("base_url")}/{match.group("namespace")}/{match.group("package")}'
+
+    if purl and (version or len(version) > 0):
+        purl += f'@{version}'
+
+    return purl
+
 # Output tag-value SPDX 2.2 content for the given Relationship object.
 # Arguments:
 #   1) f: file handle for SPDX document
@@ -51,13 +68,32 @@ def writePackageSPDX(f, pkg):
 
 PackageName: {pkg.cfg.name}
 SPDXID: {pkg.cfg.spdxID}
-PackageDownloadLocation: NOASSERTION
 PackageLicenseConcluded: {pkg.concludedLicense}
 """)
     f.write(f"""PackageLicenseDeclared: {pkg.cfg.declaredLicense}
 PackageCopyrightText: {pkg.cfg.copyrightText}
 """)
 
+    if len(pkg.cfg.url) > 0:
+        f.write(f"PackageDownloadLocation: {pkg.cfg.url}\n")
+    else:
+        f.write("PackageDownloadLocation: NOASSERTION\n")
+
+    tags = pkg.cfg.tags
+
+    if len(pkg.cfg.version) > 0:
+        f.write(f"PackageVersion: {pkg.cfg.version}\n")
+        cpe = f'cpe:2.3:o:zephyrproject:zephyr:{pkg.cfg.version}:-:*:*:*:*:*:*'
+        f.write(f"ExternalRef: SECURITY cpe23Type {cpe}\n")
+    elif len(pkg.cfg.revision) > 0:
+        f.write(f"PackageVersion: {pkg.cfg.revision}\n")
+        tags.append(pkg.cfg.revision)
+
+    for tag in tags:
+        purl = build_purl(pkg.cfg.url, tag)
+        if purl:
+            f.write(f"ExternalRef: PACKAGE_MANAGER purl {purl}\n")
+
     # flag whether files analyzed / any files present
     if len(pkg.files) > 0:
         if len(pkg.licenseInfoFromFiles) > 0: