Refactoring oss item (#213)

* Update graph converter to ossitem

---------

Signed-off-by: jiyeong.seok <[email protected]>

requirements.txt

@@ -4,7 +4,7 @@ lxml
 virtualenv
 pyyaml
 lastversion
-fosslight_util~=1.4.47
+fosslight_util>=2.0.0
 PyGithub
 requirements-parser
 defusedxml

src/fosslight_dependency/_analyze_dependency.py

@@ -28,7 +28,7 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
                        output_custom_dir='', app_name=const.default_app_name, github_token='', manifest_file_name=[],
                        direct=True):
     ret = True
-    package_sheet_list = []
+    package_dep_item_list = []
     cover_comment = ''
 
     if package_manager_name == const.PYPI:
@@ -60,7 +60,7 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
     else:
         logger.error(f"Not supported package manager name: {package_manager_name}")
         ret = False
-        return ret, package_sheet_list
+        return ret, package_dep_item_list
 
     if manifest_file_name:
         package_manager.set_manifest_file(manifest_file_name)
@@ -76,7 +76,8 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
             logger.info(f"Parse oss information with file: {f_name}")
 
             if os.path.isfile(f_name):
-                package_sheet_list.extend(package_manager.parse_oss_information(f_name))
+                package_manager.parse_oss_information(f_name)
+                package_dep_item_list.extend(package_manager.dep_items)
             else:
                 logger.error(f"Failed to open input file: {f_name}")
                 ret = False
@@ -90,4 +91,4 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
 
     del package_manager
 
-    return ret, package_sheet_list, cover_comment
+    return ret, package_dep_item_list, cover_comment

src/fosslight_dependency/_graph_convertor.py

@@ -8,24 +8,23 @@
 
 
 class GraphConvertor:
-    def __init__(self, package_list: Optional[list] = None):
+    def __init__(self, dep_items: Optional[list] = None):
         self._verticies = {}
         self._edges = []
-        if package_list:
-            self.init_list(package_list)
+        if dep_items:
+            self.init_list(dep_items)
 
-    def init_list(self, package_list: list):
+    def init_list(self, dep_items: list):
         """
-        Initialize package_list to self._verticies and self._edges
+        Initialize dep_items to self._verticies and self._edges
 
         Args:
-            package_list (list): List containing package information
+            dep_items : List containing package information
         """
         depend_on_package_dict = {}
-        for idx, package_info in enumerate(package_list):
-            package_name = package_info[0]
-            depend_on_packages_str = package_info[-1]
-            depend_on_packages = list(map((lambda x: x.strip()), depend_on_packages_str.split(",")))
+        for idx, file_item in enumerate(dep_items):
+            package_name = file_item.purl
+            depend_on_packages = file_item.depends_on
             self._verticies[package_name] = idx
             depend_on_package_dict[package_name] = depend_on_packages
         else:

src/fosslight_dependency/_package_manager.py

@@ -49,8 +49,8 @@ def __init__(self, package_manager_name, dn_url, input_dir, output_dir):
         self.manifest_file_name = []
         self.relation_tree = {}
         self.package_name = ''
-        self.purl_dict = {}
         self.cover_comment = ''
+        self.dep_items = []
 
         self.platform = platform.system()
         self.license_scanner_bin = check_license_scanner(self.platform)
@@ -67,6 +67,7 @@ def __del__(self):
         self.manifest_file_name = []
         self.relation_tree = {}
         self.package_name = ''
+        self.dep_items = []
 
     def run_plugin(self):
         ret = True
@@ -257,18 +258,6 @@ def parse_dependency_tree(self, f_name):
         except Exception as e:
             logger.warning(f'Fail to parse gradle dependency tree:{e}')
 
-    def change_dep_to_purl(self, sheet_list):
-        for oss_item in sheet_list:
-            try:
-                if len(oss_item) < 10:
-                    break
-                deps_list = oss_item[9]
-                deps_purl = list(filter(None, map(lambda x: self.purl_dict.get(x, ''), deps_list)))
-                oss_item[9] = ','.join(deps_purl)
-            except Exception as e:
-                logger.warning(f'Fail to change depend_on to purl:{e}')
-        return sheet_list
-
 
 def get_url_to_purl(url, pkg_manager, oss_name='', oss_version=''):
     purl_prefix = f'pkg:{pkg_manager}'

src/fosslight_dependency/dependency_item.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.oss_item import FileItem
+
+_logger = logging.getLogger(LOGGER_NAME)
+
+
+class DependencyItem(FileItem):
+    def __init__(self):
+        super().__init__("")
+
+        self._depends_on_raw = []  # name(version) format
+        self._depends_on = []  # purl format
+        self.purl = ""
+
+    def __del__(self):
+        pass
+
+    @property
+    def depends_on(self):
+        return self._depends_on
+
+    @depends_on.setter
+    def depends_on(self, value):
+        if not value:
+            self._depends_on = []
+        else:
+            if not isinstance(value, list):
+                value = value.split(",")
+            self._depends_on.extend(value)
+            self._depends_on = [item.strip() for item in self._depends_on]
+            self._depends_on = list(set(self._depends_on))
+
+    @property
+    def depends_on_raw(self):
+        return self._depends_on_raw
+
+    @depends_on_raw.setter
+    def depends_on_raw(self, value):
+        if not value:
+            self._depends_on_raw = []
+        else:
+            if not isinstance(value, list):
+                value = value.split(",")
+            self._depends_on_raw.extend(value)
+            self._depends_on_raw = [item.strip() for item in self._depends_on_raw]
+            self._depends_on_raw = list(set(self._depends_on_raw))
+
+    def get_print_array(self):
+        items = []
+        for oss in self.oss_items:
+            exclude = "Exclude" if self.exclude or oss.exclude else ""
+            lic = ",".join(oss.license)
+            depends_on = ",".join(self.depends_on) if len(self.depends_on) > 0 else ""
+
+            oss_item = [self.purl, oss.name, oss.version, lic, oss.download_location, oss.homepage,
+                        oss.copyright, exclude, oss.comment, depends_on]
+            items.append(oss_item)
+
+        return items
+
+    def get_print_json(self):
+        items = []
+        for oss in self.oss_items:
+            json_item = {}
+            json_item["name"] = oss.name
+            json_item["version"] = oss.version
+
+            if self.purl != "":
+                json_item["package url"] = self.purl
+            if len(oss.license) > 0:
+                json_item["license"] = oss.license
+            if oss.download_location != "":
+                json_item["download location"] = oss.download_location
+            if oss.homepage != "":
+                json_item["homepage"] = oss.homepage
+            if oss.copyright != "":
+                json_item["copyright text"] = oss.copyright
+            if self.exclude or oss.exclude:
+                json_item["exclude"] = True
+            if oss.comment != "":
+                json_item["comment"] = oss.comment
+            if len(self.depends_on) > 0:
+                json_item["depends on"] = self.depends_on
+
+            items.append(json_item)
+
+        return items
+
+
+def change_dependson_to_purl(purl_dict, dep_items):
+    for dep_item in dep_items:
+        try:
+            dep_item.depends_on = list(filter(None, map(lambda x: purl_dict.get(x, ''), dep_item.depends_on_raw)))
+
+        except Exception as e:
+            _logger.warning(f'Fail to change depend_on to purl:{e}')
+    return dep_items

src/fosslight_dependency/package_manager/Android.py

@@ -8,6 +8,8 @@
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager, get_url_to_purl
+from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
+from fosslight_util.oss_item import OssItem
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 
@@ -40,43 +42,41 @@ def check_input_path(self):
 
     def parse_oss_information(self, f_name):
         with open(f_name, 'r', encoding='utf8') as input_fp:
-            sheet_list = []
-
+            purl_dict = {}
             for i, line in enumerate(input_fp.readlines()):
-                comment = ''
+                dep_item = DependencyItem()
+                oss_item = OssItem()
                 split_str = line.strip().split("\t")
                 if i < 2:
                     continue
 
-                if len(split_str) == 9:
-                    idx, manifest_file, oss_name, oss_version, license_name, dn_loc, homepage, NA, NA = split_str
-                elif len(split_str) == 7:
-                    idx, manifest_file, oss_name, oss_version, license_name, dn_loc, homepage = split_str
+                if len(split_str) == 9 or len(split_str) == 7:
+                    _, _, oss_item.name, oss_item.version, oss_item.license, \
+                        oss_item.download_location, oss_item.homepage = split_str[:7]
                 else:
                     continue
-                purl = get_url_to_purl(dn_loc, 'maven')
-                self.purl_dict[f'{oss_name}({oss_version})'] = purl
+                dep_item.purl = get_url_to_purl(oss_item.download_location, 'maven')
+                purl_dict[f'{oss_item.name}({oss_item.version})'] = dep_item.purl
 
-                comment_list = []
-                deps_list = []
                 if self.direct_dep:
-                    dep_key = f"{oss_name}({oss_version})"
+                    dep_key = f"{oss_item.name}({oss_item.version})"
                     if self.total_dep_list:
                         if dep_key not in self.total_dep_list:
                             continue
                     if dep_key in self.direct_dep_list:
-                        comment_list.append('direct')
+                        oss_item.comment = 'direct'
                     else:
-                        comment_list.append('transitive')
+                        oss_item.comment = 'transitive'
                     try:
                         if dep_key in self.relation_tree:
-                            deps_list.extend(self.relation_tree[dep_key])
+                            dep_item.depends_on_raw = self.relation_tree[dep_key]
                     except Exception as e:
                         logger.error(f"Fail to find oss scope in dependency tree: {e}")
-                comment = ','.join(comment_list)
 
-                sheet_list.append([purl, oss_name, oss_version, license_name, dn_loc, homepage,
-                                  '', '', comment, deps_list])
-            sheet_list = self.change_dep_to_purl(sheet_list)
+                dep_item.oss_items.append(oss_item)
+                self.dep_items.append(dep_item)
+
+            if self.direct_dep:
+                self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
 
-        return sheet_list
+        return

src/fosslight_dependency/package_manager/Carthage.py

@@ -11,6 +11,8 @@
 from fosslight_dependency._package_manager import PackageManager
 from fosslight_dependency._package_manager import connect_github, get_github_license, check_and_run_license_scanner
 from fosslight_dependency._package_manager import get_url_to_purl
+from fosslight_dependency.dependency_item import DependencyItem
+from fosslight_util.oss_item import OssItem
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 
@@ -35,9 +37,8 @@ def __init__(self, input_dir, output_dir, github_token):
     def parse_oss_information(self, f_name):
         github = "github"
         checkout_dir_list = get_checkout_dirname()
-        comment = ''
+
         with open(f_name, 'r', encoding='utf8') as input_fp:
-            sheet_list = []
             g = ''
             if not checkout_dir_list:
                 g = connect_github(self.github_token)
@@ -47,21 +48,24 @@ def parse_oss_information(self, f_name):
                 # Ref. https://github.com/Carthage/Carthage/blob/master/Documentation/Artifacts.md
                 re_result = re.findall(r'(github|git)[\s]\"(\S*)\"[\s]\"(\S*)\"', line)
                 try:
+                    dep_item = DependencyItem()
+                    oss_item = OssItem()
                     repo = re_result[0][0]
                     oss_path = re_result[0][1]
                     if oss_path.endswith('.git'):
                         oss_path = oss_path[:-4]
                     oss_origin_name = oss_path.split('/')[-1]
-                    oss_name = self.package_manager_name + ":" + oss_origin_name
+                    oss_item.name = self.package_manager_name + ":" + oss_origin_name
 
                     if repo == github:
-                        homepage = self.dn_url + oss_path
+                        oss_item.homepage = self.dn_url + oss_path
                     else:
-                        homepage = oss_path
-                    dn_loc = homepage
-                    oss_version = re_result[0][2]
+                        oss_item.homepage = oss_path
+                    oss_item.download_location = oss_item.homepage
+                    oss_item.version = re_result[0][2]
 
-                    purl = get_url_to_purl(homepage, self.package_manager_name, oss_origin_name, oss_version)
+                    dep_item.purl = get_url_to_purl(oss_item.homepage, self.package_manager_name,
+                                                    oss_origin_name, oss_item.version)
 
                     license_name = ''
                     find_license = False
@@ -89,20 +93,17 @@ def parse_oss_information(self, f_name):
                             except Exception as e:
                                 logger.warning(f"Failed to get license with github api: {e}")
                                 license_name == ''
-
+                    oss_item.license = license_name
                     if self.direct_dep_list:
                         if oss_origin_name in self.direct_dep_list:
-                            comment = 'direct'
+                            oss_item.comment = 'direct'
                         else:
-                            comment = 'transitive'
-
-                    sheet_list.append([purl, oss_name, oss_version, license_name, dn_loc, homepage,
-                                      '', '', comment, ''])
-
+                            oss_item.comment = 'transitive'
+                    dep_item.oss_items.append(oss_item)
+                    self.dep_items.append(dep_item)
                 except Exception as e:
                     logger.warning(f"Failed to parse oss information: {e}")
-
-        return sheet_list
+        return
 
     def parse_direct_dependencies(self):
         self.direct_dep = True