scripts: zephyr_module: Add URL, version to SPDX

Improve the SPDX with the current values:
 - URL: extracted from `git remote`. If more than one remote, URL is not
 set.
 - Version: extracted from `git rev-parse` (commit id).
 - PURL and CPE for Zephyr: generated from URL and version.

For zephyr, the tag is extracted, if present, and replace the commit id for
the version field.
Since official modules does not have tags, tags are not yet extracted for
modules.

Fix: SPDX relations were missing when adding modules to zephyr.spdx.

Signed-off-by: Thomas Gagneret <[email protected]>

CMakeLists.txt

@@ -1621,9 +1621,8 @@ if(CONFIG_BUILD_OUTPUT_BIN AND CONFIG_BUILD_OUTPUT_UF2)
   set(BYPRODUCT_KERNEL_UF2_NAME "${PROJECT_BINARY_DIR}/${KERNEL_UF2_NAME}" CACHE FILEPATH "Kernel uf2 file" FORCE)
 endif()
 
+set(KERNEL_META_PATH  ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "")
 if(CONFIG_BUILD_OUTPUT_META)
-  set(KERNEL_META_PATH  ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "")
-
   list(APPEND
     post_build_commands
     COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/zephyr_module.py
@@ -1637,6 +1636,9 @@ if(CONFIG_BUILD_OUTPUT_META)
     post_build_byproducts
     ${KERNEL_META_PATH}
   )
+else(CONFIG_BUILD_OUTPUT_META)
+  # Prevent spdx to use invalid data
+  file(REMOVE ${KERNEL_META_PATH})
 endif()
 
 # Cleanup intermediate files

scripts/west_commands/zspdx/datatypes.py

@@ -68,6 +68,18 @@ def __init__(self):
         # SPDX ID, including "SPDXRef-"
         self.spdxID = ""
 
+        # package URL
+        self.url = ""
+
+        # package version
+        self.version = ""
+
+        # package revision
+        self.revision = ""
+
+        # package external references
+        self.externalReferences = []
+
         # the Package's declared license
         self.declaredLicense = "NOASSERTION"
 

scripts/west_commands/zspdx/walker.py

@@ -4,6 +4,7 @@
 
 import os
 import yaml
+import re
 
 from west import log
 from west.util import west_topdir, WestNotFound
@@ -35,6 +36,13 @@ def __init__(self):
 # build files, and corresponding source and SDK files, and gathers the
 # information needed to build the SPDX data classes.
 class Walker:
+    CPE23TYPE_REGEX = (
+        r'^cpe:2\.3:[aho\*\-](:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^'
+        r"`\{\|}~]))+(\?*|\*?))|[\*\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)|[\*\-]))(:(((\?*"
+        r'|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^`\{\|}~]))+(\?*|\*?))|[\*\-])){4}$'
+    )
+    PURL_REGEX = r"^pkg:.+(\/.+)?\/.+(@.+)?(\?.+)?(#.+)?$"
+
     # initialize with WalkerConfig
     def __init__(self, cfg):
         super(Walker, self).__init__()
@@ -69,6 +77,40 @@ def __init__(self, cfg):
         # SDK install path from parsed CMake cache
         self.sdkPath = ""
 
+    def _build_purl(self, url, version=None):
+        if not url:
+            return None
+
+        purl = None
+        # This is designed to match repository with the following url pattern:
+        # '<protocol><base_url>/<namespace>/<package>
+        COMMON_GIT_URL_REGEX=r'((git@|http(s)?:\/\/)(?P<base_url>[\w\.@]+)(\/|:))(?P<namespace>[\w,\-,\_]+)\/(?P<package>[\w,\-,\_]+)(.git){0,1}((\/){0,1})$'
+
+        match = re.fullmatch(COMMON_GIT_URL_REGEX, url)
+        if match:
+            purl = f'pkg:{match.group("base_url")}/{match.group("namespace")}/{match.group("package")}'
+
+        if purl and (version or len(version) > 0):
+            purl += f'@{version}'
+
+        return purl
+
+    def _normalize_module_name(self, module_name):
+        # Replace "_" by "-" since it's not allowed in spdx ID
+        return module_name.replace("_", "-")
+
+    def _add_describe_relationship(self, doc, cfgpackage):
+        # create DESCRIBES relationship data
+        rd = RelationshipData()
+        rd.ownerType = RelationshipDataElementType.DOCUMENT
+        rd.ownerDocument = doc
+        rd.otherType = RelationshipDataElementType.PACKAGEID
+        rd.otherPackageID = cfgpackage.spdxID
+        rd.rlnType = "DESCRIBES"
+
+        # add it to pending relationships queue
+        self.pendingRelationships.append(rd)
+
     # primary entry point
     def makeDocuments(self):
         # parse CMake cache file and get compiler path
@@ -162,16 +204,7 @@ def setupAppDocument(self):
         pkgApp = Package(cfgPackageApp, self.docApp)
         self.docApp.pkgs[pkgApp.cfg.spdxID] = pkgApp
 
-        # create DESCRIBES relationship data
-        rd = RelationshipData()
-        rd.ownerType = RelationshipDataElementType.DOCUMENT
-        rd.ownerDocument = self.docApp
-        rd.otherType = RelationshipDataElementType.PACKAGEID
-        rd.otherPackageID = cfgPackageApp.spdxID
-        rd.rlnType = "DESCRIBES"
-
-        # add it to pending relationships queue
-        self.pendingRelationships.append(rd)
+        self._add_describe_relationship(self.docApp, cfgPackageApp)
 
     def setupBuildDocument(self):
         # set up build document
@@ -195,7 +228,7 @@ def setupBuildDocument(self):
         # add it to pending relationships queue
         self.pendingRelationships.append(rd)
 
-    def setupZephyrDocument(self, modules):
+    def setupZephyrDocument(self, zephyr, modules):
         # set up zephyr document
         cfgZephyr = DocumentConfig()
         cfgZephyr.name = "zephyr-sources"
@@ -216,39 +249,66 @@ def setupZephyrDocument(self, modules):
         cfgPackageZephyr.spdxID = "SPDXRef-zephyr-sources"
         cfgPackageZephyr.relativeBaseDir = relativeBaseDir
 
+        zephyr_url = zephyr.get("remote", "")
+        if zephyr_url:
+            cfgPackageZephyr.url = zephyr_url
+
+        if zephyr.get("revision"):
+            cfgPackageZephyr.revision = zephyr.get("revision")
+
+        purl = None
+        zephyr_tags = zephyr.get("tags", "")
+        if zephyr_tags:
+            # Find tag vX.Y.Z
+            for tag in zephyr_tags:
+                tag = re.fullmatch(r'^v(?P<version>\d+\.\d+\.\d+)$', tag)
+                if tag:
+                    cfgPackageZephyr.version = tag.group('version')
+                    purl = self._build_purl(zephyr_url, cfgPackageZephyr.version)
+                    break
+
+        if purl:
+            cfgPackageZephyr.externalReferences.append(purl)
+
+        if len(cfgPackageZephyr.version) > 0:
+            cpe = f'cpe:2.3:o:zephyrproject:zephyr:{cfgPackageZephyr.version}:-:*:*:*:*:*:*'
+            cfgPackageZephyr.externalReferences.append(cpe)
+
         pkgZephyr = Package(cfgPackageZephyr, self.docZephyr)
         self.docZephyr.pkgs[pkgZephyr.cfg.spdxID] = pkgZephyr
 
+        self._add_describe_relationship(self.docZephyr, cfgPackageZephyr)
+
         for module in modules:
             module_name = module.get("name", None)
             module_path = module.get("path", None)
+            module_url = module.get("remote", None)
+            module_revision = module.get("revision", None)
 
             if not module_name:
                 log.err(f"cannot find module name in meta file; bailing")
                 return False
 
-            # Replace "_" by "-" since it's not allowed in spdx ID
-            module_name = module_name.replace("_", "-")
+            module_name = self._normalize_module_name(module_name)
 
             # set up zephyr sources package
             cfgPackageZephyrModule = PackageConfig()
             cfgPackageZephyrModule.name = module_name
             cfgPackageZephyrModule.spdxID = "SPDXRef-" + module_name + "-sources"
             cfgPackageZephyrModule.relativeBaseDir = module_path
 
+            if module_revision:
+                cfgPackageZephyrModule.revision = module_revision
+
+            if module_url:
+                cfgPackageZephyrModule.url = module_url
+
             pkgZephyrModule = Package(cfgPackageZephyrModule, self.docZephyr)
             self.docZephyr.pkgs[pkgZephyrModule.cfg.spdxID] = pkgZephyrModule
 
-        # create DESCRIBES relationship data
-        rd = RelationshipData()
-        rd.ownerType = RelationshipDataElementType.DOCUMENT
-        rd.ownerDocument = self.docZephyr
-        rd.otherType = RelationshipDataElementType.PACKAGEID
-        rd.otherPackageID = cfgPackageZephyr.spdxID
-        rd.rlnType = "DESCRIBES"
+            self._add_describe_relationship(self.docZephyr, cfgPackageZephyrModule)
 
-        # add it to pending relationships queue
-        self.pendingRelationships.append(rd)
+        return True
 
     def setupSDKDocument(self):
         # set up SDK document
@@ -287,7 +347,8 @@ def setupDocuments(self):
         try:
             with open(self.metaFile) as file:
                 content = yaml.load(file.read(), yaml.SafeLoader)
-                self.setupZephyrDocument(content["modules"])
+                if not self.setupZephyrDocument(content["zephyr"], content["modules"]):
+                    return False
         except (FileNotFoundError, yaml.YAMLError):
             log.err(f"cannot find a valid zephyr_meta.yml required for SPDX generation; bailing")
             return False

scripts/west_commands/zspdx/writer.py

@@ -8,6 +8,15 @@
 
 from zspdx.util import getHashes
 
+import re
+
+CPE23TYPE_REGEX = (
+    r'^cpe:2\.3:[aho\*\-](:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^'
+    r"`\{\|}~]))+(\?*|\*?))|[\*\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)|[\*\-]))(:(((\?*"
+    r'|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^`\{\|}~]))+(\?*|\*?))|[\*\-])){4}$'
+)
+PURL_REGEX = r"^pkg:.+(\/.+)?\/.+(@.+)?(\?.+)?(#.+)?$"
+
 # Output tag-value SPDX 2.2 content for the given Relationship object.
 # Arguments:
 #   1) f: file handle for SPDX document
@@ -51,13 +60,30 @@ def writePackageSPDX(f, pkg):
 
 PackageName: {pkg.cfg.name}
 SPDXID: {pkg.cfg.spdxID}
-PackageDownloadLocation: NOASSERTION
 PackageLicenseConcluded: {pkg.concludedLicense}
 """)
     f.write(f"""PackageLicenseDeclared: {pkg.cfg.declaredLicense}
 PackageCopyrightText: {pkg.cfg.copyrightText}
 """)
 
+    if len(pkg.cfg.url) > 0:
+        f.write(f"PackageDownloadLocation: {pkg.cfg.url}\n")
+    else:
+        f.write("PackageDownloadLocation: NOASSERTION\n")
+
+    if len(pkg.cfg.version) > 0:
+        f.write(f"PackageVersion: {pkg.cfg.version}\n")
+    elif len(pkg.cfg.revision) > 0:
+        f.write(f"PackageVersion: {pkg.cfg.revision}\n")
+
+    for ref in pkg.cfg.externalReferences:
+        if re.fullmatch(CPE23TYPE_REGEX, ref):
+            f.write(f"ExternalRef: SECURITY cpe23Type {ref}\n")
+        elif re.fullmatch(PURL_REGEX, ref):
+            f.write(f"ExternalRef: PACKAGE_MANAGER purl {ref}\n")
+        else:
+            log.wrn(f"Unknown external reference ({ref})")
+
     # flag whether files analyzed / any files present
     if len(pkg.files) > 0:
         if len(pkg.licenseInfoFromFiles) > 0: