scripts: zephyr_module: Add URL, PURL and version to SPDX

Improve the SPDX with the current values:
 - URL: extracted from `git remote`. If more than one remote, URL is not
 set.
 - Version: extracted from `git rev-parse` (commit id).
 - PURL: generated from URL and Version.

For zephyr, the tag is extracted, if present, and replace the commit id for
the version field.
Since official modules does not have tags, tags are not yet extracted for
modules.

Signed-off-by: Thomas Gagneret <[email protected]>

CMakeLists.txt

@@ -1637,6 +1637,9 @@ if(CONFIG_BUILD_OUTPUT_META)
     post_build_byproducts
     ${KERNEL_META_PATH}
   )
+else(CONFIG_BUILD_OUTPUT_META)
+  # Prevent spdx to use incorrect data
+  file(REMOVE ${KERNEL_META_PATH})
 endif()
 
 # Cleanup intermediate files

scripts/west_commands/zspdx/datatypes.py

@@ -68,6 +68,15 @@ def __init__(self):
         # SPDX ID, including "SPDXRef-"
         self.spdxID = ""
 
+        # package URL
+        self.url = ""
+
+        # package revision
+        self.revision = ""
+
+        # package tag (for current commit)
+        self.tags = []
+
         # the Package's declared license
         self.declaredLicense = "NOASSERTION"
 

scripts/west_commands/zspdx/walker.py

@@ -4,6 +4,7 @@
 
 import os
 import yaml
+import re
 
 from west import log
 from west.util import west_topdir, WestNotFound
@@ -195,7 +196,7 @@ def setupBuildDocument(self):
         # add it to pending relationships queue
         self.pendingRelationships.append(rd)
 
-    def setupZephyrDocument(self, modules):
+    def setupZephyrDocument(self, zephyr, modules):
         # set up zephyr document
         cfgZephyr = DocumentConfig()
         cfgZephyr.name = "zephyr-sources"
@@ -215,13 +216,30 @@ def setupZephyrDocument(self, modules):
         cfgPackageZephyr.name = "zephyr-sources"
         cfgPackageZephyr.spdxID = "SPDXRef-zephyr-sources"
         cfgPackageZephyr.relativeBaseDir = relativeBaseDir
+        zephyr_url = zephyr.get("remote", "")
+        if zephyr_url:
+            cfgPackageZephyr.url = zephyr_url
+
+        zephyr_tags = zephyr.get("tags", "")
+        if zephyr_tags:
+            # Find tag vX.Y.Z
+            for tag in zephyr_tags:
+                tag = re.fullmatch('^v\d+\.\d+\.\d+$', tag)
+                if tag:
+                    cfgPackageZephyr.revision = tag.string
+                    break
+
+        if len(cfgPackageZephyr.revision) == 0 and zephyr.get("revision"):
+            cfgPackageZephyr.revision = zephyr.get("revision")
 
         pkgZephyr = Package(cfgPackageZephyr, self.docZephyr)
         self.docZephyr.pkgs[pkgZephyr.cfg.spdxID] = pkgZephyr
 
         for module in modules:
             module_name = module.get("name", None)
             module_path = module.get("path", None)
+            module_url = module.get("remote", "")
+            module_revision = module.get("revision", None)
 
             if not module_name:
                 log.err(f"cannot find module name in meta file; bailing")
@@ -236,6 +254,11 @@ def setupZephyrDocument(self, modules):
             cfgPackageZephyrModule.spdxID = "SPDXRef-" + module_name + "-sources"
             cfgPackageZephyrModule.relativeBaseDir = module_path
 
+            if module_revision:
+                cfgPackageZephyrModule.revision = module_revision
+
+            if module_url:
+                cfgPackageZephyrModule.url = module_url
             pkgZephyrModule = Package(cfgPackageZephyrModule, self.docZephyr)
             self.docZephyr.pkgs[pkgZephyrModule.cfg.spdxID] = pkgZephyrModule
 
@@ -250,6 +273,8 @@ def setupZephyrDocument(self, modules):
         # add it to pending relationships queue
         self.pendingRelationships.append(rd)
 
+        return True
+
     def setupSDKDocument(self):
         # set up SDK document
         cfgSDK = DocumentConfig()
@@ -287,7 +312,8 @@ def setupDocuments(self):
         try:
             with open(self.metaFile) as file:
                 content = yaml.load(file.read(), yaml.SafeLoader)
-                self.setupZephyrDocument(content["modules"])
+                if not self.setupZephyrDocument(content["zephyr"], content["modules"]):
+                    return False
         except (FileNotFoundError, yaml.YAMLError):
             log.err(f"cannot find a valid zephyr_meta.yml required for SPDX generation; bailing")
             return False

scripts/west_commands/zspdx/writer.py

@@ -8,6 +8,23 @@
 
 from zspdx.util import getHashes
 
+import re
+
+def build_purl(url, version=None):
+    purl = None
+    # This is designed to match repository with the following url pattern:
+    # '<protocol><base_url>/<namespace>/<package>
+    COMMON_GIT_URL_REGEX='((git@|http(s)?:\/\/)(?P<base_url>[\w\.@]+)(\/|:))(?P<namespace>[\w,\-,\_]+)\/(?P<package>[\w,\-,\_]+)(.git){0,1}((\/){0,1})$'
+
+    match = re.fullmatch(COMMON_GIT_URL_REGEX, url)
+    if match:
+        purl = f'pkg:{match.group("base_url")}/{match.group("namespace")}/{match.group("package")}'
+
+    if purl and (version or len(version) > 0):
+        purl += f'@{version}'
+
+    return purl
+
 # Output tag-value SPDX 2.2 content for the given Relationship object.
 # Arguments:
 #   1) f: file handle for SPDX document
@@ -51,13 +68,24 @@ def writePackageSPDX(f, pkg):
 
 PackageName: {pkg.cfg.name}
 SPDXID: {pkg.cfg.spdxID}
-PackageDownloadLocation: NOASSERTION
 PackageLicenseConcluded: {pkg.concludedLicense}
 """)
     f.write(f"""PackageLicenseDeclared: {pkg.cfg.declaredLicense}
 PackageCopyrightText: {pkg.cfg.copyrightText}
 """)
 
+    if len(pkg.cfg.url) > 0:
+        f.write(f"PackageDownloadLocation: {pkg.cfg.url}\n")
+    else:
+        f.write(f"PackageDownloadLocation: NOASSERTION\n")
+
+    if len(pkg.cfg.revision) > 0:
+        purl = build_purl(pkg.cfg.url, pkg.cfg.revision)
+        if purl:
+            f.write(f"ExternalRef: PACKAGE_MANAGER purl {purl}\n")
+
+        f.write(f"PackageVersion: {pkg.cfg.revision}\n")
+
     # flag whether files analyzed / any files present
     if len(pkg.files) > 0:
         if len(pkg.licenseInfoFromFiles) > 0:

scripts/zephyr_module.py

@@ -402,6 +402,99 @@ def process_twister(module, meta):
     return out
 
 
+def _create_meta_project(project_path):
+    def git_revision(path):
+         rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE,
+                               cwd=path).wait()
+         if rc == 0:
+             # A git repo.
+             popen = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
+                                      stdout=subprocess.PIPE,
+                                      stderr=subprocess.PIPE,
+                                      cwd=path)
+             stdout, stderr = popen.communicate()
+             stdout = stdout.decode('utf-8')
+
+             if not (popen.returncode or stderr):
+                 revision = stdout.rstrip()
+
+                 rc = subprocess.Popen(['git', 'diff-index', '--quiet', 'HEAD',
+                                        '--'],
+                                       stdout=None,
+                                       stderr=None,
+                                       cwd=path).wait()
+                 if rc:
+                     return revision + '-dirty', True
+                 return revision, False
+         return None, False
+
+    def git_remote(path):
+        popen = subprocess.Popen(['git', 'remote'],
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE,
+                              cwd=path)
+        stdout, stderr = popen.communicate()
+        stdout = stdout.decode('utf-8')
+
+        remotes_name = []
+        if not (popen.returncode or stderr):
+            remotes_name = stdout.rstrip().split('\n')
+
+        remote_url = None
+        if len(remotes_name) == 1:
+            remote = remotes_name[0]
+            popen = subprocess.Popen(['git', 'remote', 'get-url', remote],
+                                     stdout=subprocess.PIPE,
+                                     stderr=subprocess.PIPE,
+                                     cwd=path)
+            stdout, stderr = popen.communicate()
+            stdout = stdout.decode('utf-8')
+
+            if not (popen.returncode or stderr):
+                remote_url = stdout.rstrip()
+
+        return remote_url, len(remotes_name) > 1
+
+    def git_tags(path, revision):
+        if not revision or len(revision) == 0:
+            return None
+
+        popen = subprocess.Popen(['git', '-P', 'tag', '--points-at', revision],
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE,
+                              cwd=path)
+        stdout, stderr = popen.communicate()
+        stdout = stdout.decode('utf-8')
+
+        tags = None
+        if not (popen.returncode or stderr):
+            tags = stdout.rstrip().splitlines()
+
+        return tags
+
+    workspace_dirty = False
+    path = PurePath(project_path).as_posix()
+
+    revision, dirty = git_revision(path)
+    workspace_dirty |= dirty
+    remote, dirty = git_remote(path)
+    workspace_dirty |= dirty
+    tags = git_tags(path, revision)
+
+    meta_project = {'path': path,
+                'revision': revision}
+
+    if remote:
+        meta_project['remote'] = remote
+
+    #if not workspace_dirty and tags:
+    if tags:
+        meta_project['tags'] = tags
+
+    return meta_project, workspace_dirty
+
 def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
                  propagate_state=False):
     # Process zephyr_base, projects, and modules and create a dictionary
@@ -420,77 +513,42 @@ def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
     workspace_extra = extra_modules is not None
     workspace_off = False
 
-    def git_revision(path):
-        rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'],
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE,
-                              cwd=path).wait()
-        if rc == 0:
-            # A git repo.
-            popen = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE,
-                                     cwd=path)
-            stdout, stderr = popen.communicate()
-            stdout = stdout.decode('utf-8')
+    zephyr_project, zephyr_dirty = _create_meta_project(zephyr_base)
 
-            if not (popen.returncode or stderr):
-                revision = stdout.rstrip()
-
-                rc = subprocess.Popen(['git', 'diff-index', '--quiet', 'HEAD',
-                                       '--'],
-                                      stdout=None,
-                                      stderr=None,
-                                      cwd=path).wait()
-                if rc:
-                    return revision + '-dirty', True
-                return revision, False
-        return None, False
-
-    zephyr_revision, zephyr_dirty = git_revision(zephyr_base)
-    zephyr_project = {'path': zephyr_base,
-                      'revision': zephyr_revision}
     meta['zephyr'] = zephyr_project
-    meta['workspace'] = {}
     workspace_dirty |= zephyr_dirty
 
+    meta['workspace'] = {}
+
     if west_projs is not None:
         from west.manifest import MANIFEST_REV_BRANCH
         projects = west_projs['projects']
         meta_projects = []
 
-        # Special treatment of manifest project.
-        manifest_proj_path = PurePath(projects[0].posixpath).as_posix()
-        manifest_revision, manifest_dirty = git_revision(manifest_proj_path)
+        manifest_project, manifest_dirty = _create_meta_project(projects[0].posixpath)
         workspace_dirty |= manifest_dirty
-        manifest_project = {'path': manifest_proj_path,
-                            'revision': manifest_revision}
         meta_projects.append(manifest_project)
 
         for project in projects[1:]:
-            project_path = PurePath(project.posixpath).as_posix()
-            revision, dirty = git_revision(project_path)
-            workspace_dirty |= dirty
-            if project.sha(MANIFEST_REV_BRANCH) != revision:
-                revision += '-off'
-                workspace_off = True
-            meta_project = {'path': project_path,
-                            'revision': revision}
+            meta_project, dirty = _create_meta_project(project.posixpath)
             meta_projects.append(meta_project)
 
+            if project.sha(MANIFEST_REV_BRANCH) != meta_project['revision']:
+                meta_project['revision'] += '-off'
+                workspace_off = True
+            if meta_project.get('remote') and project.url != meta_project['remote']:
+                workspace_off = True
+
         meta.update({'west': {'manifest': west_projs['manifest_path'],
                               'projects': meta_projects}})
         meta['workspace'].update({'off': workspace_off})
 
     meta_projects = []
     for module in modules:
-        module_path = PurePath(module.project).as_posix()
-        revision, dirty = git_revision(module_path)
-        workspace_dirty |= dirty
-        meta_project = {'name': module.meta['name'],
-                        'path': module_path,
-                        'revision': revision}
-        meta_projects.append(meta_project)
+        meta_module, dirty = _create_meta_project(module.project)
+        meta_module['name'] = module.meta.get('name')
+        meta_projects.append(meta_module)
+
     meta['modules'] = meta_projects
 
     meta['workspace'].update({'dirty': workspace_dirty,
@@ -667,7 +725,7 @@ def main():
                              list`""")
     parser.add_argument('-x', '--extra-modules', nargs='+',
                         help='List of extra modules to parse')
-    parser.add_argument('-z', '--zephyr-base',
+    parser.add_argument('-z', '--zephyr-base', required=True,
                         help='Path to zephyr repository')
     args = parser.parse_args()