scripts: zephyr_module: Add URL, PURL and version to SPDX

Improve the SPDX with the current values:
 - URL: extracted from `git remote`. If more than one remote, URL is not
 set.
 - Version: extracted from `git rev-parse` (commit id).
 - PURL: generated from URL and Version.

For zephyr, the tag is extracted, if present, and replace the commit id for
the version field.
Since official modules does not have tags, tags are not yet extracted for
modules.

Signed-off-by: Thomas Gagneret <[email protected]>

CMakeLists.txt

@@ -1621,9 +1621,8 @@ if(CONFIG_BUILD_OUTPUT_BIN AND CONFIG_BUILD_OUTPUT_UF2)
   set(BYPRODUCT_KERNEL_UF2_NAME "${PROJECT_BINARY_DIR}/${KERNEL_UF2_NAME}" CACHE FILEPATH "Kernel uf2 file" FORCE)
 endif()
 
+set(KERNEL_META_PATH  ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "")
 if(CONFIG_BUILD_OUTPUT_META)
-  set(KERNEL_META_PATH  ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "")
-
   list(APPEND
     post_build_commands
     COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/zephyr_module.py
@@ -1637,6 +1636,9 @@ if(CONFIG_BUILD_OUTPUT_META)
     post_build_byproducts
     ${KERNEL_META_PATH}
   )
+else(CONFIG_BUILD_OUTPUT_META)
+  # Prevent spdx to use invalid data
+  file(REMOVE ${KERNEL_META_PATH})
 endif()
 
 # Cleanup intermediate files

scripts/west_commands/zspdx/datatypes.py

@@ -68,6 +68,18 @@ def __init__(self):
         # SPDX ID, including "SPDXRef-"
         self.spdxID = ""
 
+        # package URL
+        self.url = ""
+
+        # package version
+        self.version = ""
+
+        # package revision
+        self.revision = ""
+
+        # package tags (for current commit)
+        self.tags = []
+
         # the Package's declared license
         self.declaredLicense = "NOASSERTION"
 

scripts/west_commands/zspdx/walker.py

@@ -4,6 +4,7 @@
 
 import os
 import yaml
+import re
 
 from west import log
 from west.util import west_topdir, WestNotFound
@@ -195,7 +196,7 @@ def setupBuildDocument(self):
         # add it to pending relationships queue
         self.pendingRelationships.append(rd)
 
-    def setupZephyrDocument(self, modules):
+    def setupZephyrDocument(self, zephyr, modules):
         # set up zephyr document
         cfgZephyr = DocumentConfig()
         cfgZephyr.name = "zephyr-sources"
@@ -215,13 +216,31 @@ def setupZephyrDocument(self, modules):
         cfgPackageZephyr.name = "zephyr-sources"
         cfgPackageZephyr.spdxID = "SPDXRef-zephyr-sources"
         cfgPackageZephyr.relativeBaseDir = relativeBaseDir
+        zephyr_url = zephyr.get("remote", "")
+        if zephyr_url:
+            cfgPackageZephyr.url = zephyr_url
+
+        zephyr_tags = zephyr.get("tags", "")
+        if zephyr_tags:
+            cfgPackageZephyr.tags = zephyr_tags
+            # Find tag vX.Y.Z
+            for tag in zephyr_tags:
+                tag = re.fullmatch(r'^v(?P<version>\d+\.\d+\.\d+)$', tag)
+                if tag:
+                    cfgPackageZephyr.version = tag.group('version')
+                    break
+
+        if zephyr.get("revision"):
+            cfgPackageZephyr.revision = zephyr.get("revision")
 
         pkgZephyr = Package(cfgPackageZephyr, self.docZephyr)
         self.docZephyr.pkgs[pkgZephyr.cfg.spdxID] = pkgZephyr
 
         for module in modules:
             module_name = module.get("name", None)
             module_path = module.get("path", None)
+            module_url = module.get("remote", None)
+            module_revision = module.get("revision", None)
 
             if not module_name:
                 log.err(f"cannot find module name in meta file; bailing")
@@ -236,6 +255,11 @@ def setupZephyrDocument(self, modules):
             cfgPackageZephyrModule.spdxID = "SPDXRef-" + module_name + "-sources"
             cfgPackageZephyrModule.relativeBaseDir = module_path
 
+            if module_revision:
+                cfgPackageZephyrModule.revision = module_revision
+
+            if module_url:
+                cfgPackageZephyrModule.url = module_url
             pkgZephyrModule = Package(cfgPackageZephyrModule, self.docZephyr)
             self.docZephyr.pkgs[pkgZephyrModule.cfg.spdxID] = pkgZephyrModule
 
@@ -250,6 +274,8 @@ def setupZephyrDocument(self, modules):
         # add it to pending relationships queue
         self.pendingRelationships.append(rd)
 
+        return True
+
     def setupSDKDocument(self):
         # set up SDK document
         cfgSDK = DocumentConfig()
@@ -287,7 +313,8 @@ def setupDocuments(self):
         try:
             with open(self.metaFile) as file:
                 content = yaml.load(file.read(), yaml.SafeLoader)
-                self.setupZephyrDocument(content["modules"])
+                if not self.setupZephyrDocument(content["zephyr"], content["modules"]):
+                    return False
         except (FileNotFoundError, yaml.YAMLError):
             log.err(f"cannot find a valid zephyr_meta.yml required for SPDX generation; bailing")
             return False

scripts/west_commands/zspdx/writer.py

@@ -8,6 +8,23 @@
 
 from zspdx.util import getHashes
 
+import re
+
+def build_purl(url, version=None):
+    purl = None
+    # This is designed to match repository with the following url pattern:
+    # '<protocol><base_url>/<namespace>/<package>
+    COMMON_GIT_URL_REGEX=r'((git@|http(s)?:\/\/)(?P<base_url>[\w\.@]+)(\/|:))(?P<namespace>[\w,\-,\_]+)\/(?P<package>[\w,\-,\_]+)(.git){0,1}((\/){0,1})$'
+
+    match = re.fullmatch(COMMON_GIT_URL_REGEX, url)
+    if match:
+        purl = f'pkg:{match.group("base_url")}/{match.group("namespace")}/{match.group("package")}'
+
+    if purl and (version or len(version) > 0):
+        purl += f'@{version}'
+
+    return purl
+
 # Output tag-value SPDX 2.2 content for the given Relationship object.
 # Arguments:
 #   1) f: file handle for SPDX document
@@ -51,13 +68,30 @@ def writePackageSPDX(f, pkg):
 
 PackageName: {pkg.cfg.name}
 SPDXID: {pkg.cfg.spdxID}
-PackageDownloadLocation: NOASSERTION
 PackageLicenseConcluded: {pkg.concludedLicense}
 """)
     f.write(f"""PackageLicenseDeclared: {pkg.cfg.declaredLicense}
 PackageCopyrightText: {pkg.cfg.copyrightText}
 """)
 
+    if len(pkg.cfg.url) > 0:
+        f.write(f"PackageDownloadLocation: {pkg.cfg.url}\n")
+    else:
+        f.write("PackageDownloadLocation: NOASSERTION\n")
+
+    tags = pkg.cfg.tags
+
+    if len(pkg.cfg.version) > 0:
+        f.write(f"PackageVersion: {pkg.cfg.version}\n")
+    elif len(pkg.cfg.revision) > 0:
+        f.write(f"PackageVersion: {pkg.cfg.revision}\n")
+        tags.append(pkg.cfg.revision)
+
+    for tag in tags:
+        purl = build_purl(pkg.cfg.url, tag)
+        if purl:
+            f.write(f"ExternalRef: PACKAGE_MANAGER purl {purl}\n")
+
     # flag whether files analyzed / any files present
     if len(pkg.files) > 0:
         if len(pkg.licenseInfoFromFiles) > 0:

scripts/zephyr_module.py

@@ -402,24 +402,7 @@ def process_twister(module, meta):
     return out
 
 
-def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
-                 propagate_state=False):
-    # Process zephyr_base, projects, and modules and create a dictionary
-    # with meta information for each input.
-    #
-    # The dictionary will contain meta info in the following lists:
-    # - zephyr:        path and revision
-    # - modules:       name, path, and revision
-    # - west-projects: path and revision
-    #
-    # returns the dictionary with said lists
-
-    meta = {'zephyr': None, 'modules': None, 'workspace': None}
-
-    workspace_dirty = False
-    workspace_extra = extra_modules is not None
-    workspace_off = False
-
+def _create_meta_project(project_path):
     def git_revision(path):
         rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'],
                               stdout=subprocess.PIPE,
@@ -447,9 +430,91 @@ def git_revision(path):
                 return revision, False
         return None, False
 
-    zephyr_revision, zephyr_dirty = git_revision(zephyr_base)
-    zephyr_project = {'path': zephyr_base,
-                      'revision': zephyr_revision}
+    def git_remote(path):
+        popen = subprocess.Popen(['git', 'remote'],
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE,
+                                 cwd=path)
+        stdout, stderr = popen.communicate()
+        stdout = stdout.decode('utf-8')
+
+        remotes_name = []
+        if not (popen.returncode or stderr):
+            remotes_name = stdout.rstrip().split('\n')
+
+        remote_url = None
+        if len(remotes_name) == 1:
+            remote = remotes_name[0]
+            popen = subprocess.Popen(['git', 'remote', 'get-url', remote],
+                                     stdout=subprocess.PIPE,
+                                     stderr=subprocess.PIPE,
+                                     cwd=path)
+            stdout, stderr = popen.communicate()
+            stdout = stdout.decode('utf-8')
+
+            if not (popen.returncode or stderr):
+                remote_url = stdout.rstrip()
+
+        return remote_url, len(remotes_name) > 1
+
+    def git_tags(path, revision):
+        if not revision or len(revision) == 0:
+            return None
+
+        popen = subprocess.Popen(['git', '-P', 'tag', '--points-at', revision],
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE,
+                                 cwd=path)
+        stdout, stderr = popen.communicate()
+        stdout = stdout.decode('utf-8')
+
+        tags = None
+        if not (popen.returncode or stderr):
+            tags = stdout.rstrip().splitlines()
+
+        return tags
+
+    workspace_dirty = False
+    path = PurePath(project_path).as_posix()
+
+    revision, dirty = git_revision(path)
+    workspace_dirty |= dirty
+    remote, dirty = git_remote(path)
+    workspace_dirty |= dirty
+    tags = git_tags(path, revision)
+
+    meta_project = {'path': path,
+                    'revision': revision}
+
+    if remote:
+        meta_project['remote'] = remote
+
+    if tags:
+        meta_project['tags'] = tags
+
+    return meta_project, workspace_dirty
+
+
+def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
+                 propagate_state=False):
+    # Process zephyr_base, projects, and modules and create a dictionary
+    # with meta information for each input.
+    #
+    # The dictionary will contain meta info in the following lists:
+    # - zephyr:        path and revision
+    # - modules:       name, path, and revision
+    # - west-projects: path and revision
+    #
+    # returns the dictionary with said lists
+
+    meta = {'zephyr': None, 'modules': None, 'workspace': None}
+
+    workspace_dirty = False
+    workspace_extra = extra_modules is not None
+    workspace_off = False
+
+    zephyr_project, zephyr_dirty = _create_meta_project(zephyr_base)
+
     meta['zephyr'] = zephyr_project
     meta['workspace'] = {}
     workspace_dirty |= zephyr_dirty
@@ -459,44 +524,39 @@ def git_revision(path):
         projects = west_projs['projects']
         meta_projects = []
 
-        # Special treatment of manifest project.
-        manifest_proj_path = PurePath(projects[0].posixpath).as_posix()
-        manifest_revision, manifest_dirty = git_revision(manifest_proj_path)
+        manifest_project, manifest_dirty = _create_meta_project(
+            projects[0].posixpath)
         workspace_dirty |= manifest_dirty
-        manifest_project = {'path': manifest_proj_path,
-                            'revision': manifest_revision}
         meta_projects.append(manifest_project)
 
         for project in projects[1:]:
-            project_path = PurePath(project.posixpath).as_posix()
-            revision, dirty = git_revision(project_path)
+            meta_project, dirty = _create_meta_project(project.posixpath)
             workspace_dirty |= dirty
-            if project.sha(MANIFEST_REV_BRANCH) != revision:
-                revision += '-off'
-                workspace_off = True
-            meta_project = {'path': project_path,
-                            'revision': revision}
             meta_projects.append(meta_project)
 
+            if project.sha(MANIFEST_REV_BRANCH) != meta_project['revision']:
+                meta_project['revision'] += '-off'
+                workspace_off = True
+            if meta_project.get('remote') and project.url != meta_project['remote']:
+                workspace_off = True
+
         meta.update({'west': {'manifest': west_projs['manifest_path'],
                               'projects': meta_projects}})
         meta['workspace'].update({'off': workspace_off})
 
     meta_projects = []
     for module in modules:
-        module_path = PurePath(module.project).as_posix()
-        revision, dirty = git_revision(module_path)
-        workspace_dirty |= dirty
-        meta_project = {'name': module.meta['name'],
-                        'path': module_path,
-                        'revision': revision}
-        meta_projects.append(meta_project)
+        meta_module, dirty = _create_meta_project(module.project)
+        meta_module['name'] = module.meta.get('name')
+        meta_projects.append(meta_module)
+
     meta['modules'] = meta_projects
 
     meta['workspace'].update({'dirty': workspace_dirty,
                               'extra': workspace_extra})
 
     if propagate_state:
+        zephyr_revision = zephyr_project['revision']
         if workspace_dirty and not zephyr_dirty:
             zephyr_revision += '-dirty'
         if workspace_extra:
@@ -506,6 +566,7 @@ def git_revision(path):
         zephyr_project.update({'revision': zephyr_revision})
 
         if west_projs is not None:
+            manifest_revision = manifest_project['revision']
             if workspace_dirty and not manifest_dirty:
                 manifest_revision += '-dirty'
             if workspace_extra:
@@ -517,7 +578,7 @@ def git_revision(path):
     return meta
 
 
-def west_projects(manifest = None):
+def west_projects(manifest=None):
     manifest_path = None
     projects = []
     # West is imported here, as it is optional
@@ -685,7 +746,8 @@ def main():
     for module in modules:
         kconfig += process_kconfig(module.project, module.meta)
         cmake += process_cmake(module.project, module.meta)
-        sysbuild_kconfig += process_sysbuildkconfig(module.project, module.meta)
+        sysbuild_kconfig += process_sysbuildkconfig(
+            module.project, module.meta)
         sysbuild_cmake += process_sysbuildcmake(module.project, module.meta)
         settings += process_settings(module.project, module.meta)
         twister += process_twister(module.project, module.meta)