diff --git a/requirements-dev.txt b/requirements-dev.txt
index b27ff0c..c36c3a3 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -3,4 +3,5 @@ pytest
pytest-cov
pytest-flake8
flake8==3.9.2
-tox-wheel
\ No newline at end of file
+tox-wheel
+fosslight-source
\ No newline at end of file
diff --git a/src/fosslight_util/compare_yaml.py b/src/fosslight_util/compare_yaml.py
index fcdcba4..67e0db3 100644
--- a/src/fosslight_util/compare_yaml.py
+++ b/src/fosslight_util/compare_yaml.py
@@ -4,9 +4,7 @@
# SPDX-License-Identifier: Apache-2.0
import logging
-import os
from fosslight_util.constant import LOGGER_NAME
-from fosslight_util.parsing_yaml import parsing_yml
logger = logging.getLogger(LOGGER_NAME)
VERSION = 'version'
@@ -14,12 +12,16 @@
NAME = 'name'
-def compare_yaml(before_file, after_file):
- before_oss_items, _, _ = parsing_yml(before_file, os.path.dirname(before_file))
- after_oss_items, _, _ = parsing_yml(after_file, os.path.dirname(after_file))
+def compare_yaml(before_fileitems, after_fileitems):
+ bf_raw = []
+ af_raw = []
+ for bf in before_fileitems:
+ bf_raw.extend(bf.get_print_json())
+ for af in after_fileitems:
+ af_raw.extend(af.get_print_json())
- before_items = get_merged_item(before_oss_items)
- after_items = get_merged_item(after_oss_items)
+ before_items = get_merged_item(bf_raw)
+ after_items = get_merged_item(af_raw)
new_before = []
for bi in before_items:
@@ -72,13 +74,18 @@ def compare_yaml(before_file, after_file):
def get_merged_item(oss_items):
item_list = []
for oi in oss_items:
- if oi.exclude:
+ if oi.get("exclude", None):
continue
- item_info = {NAME: oi.name, VERSION: oi.version, LICENSE: oi.license}
+ oi_name = oi.get("name", '')
+ oi_version = oi.get("version", '')
+ oi_license = oi.get("license", '')
+ if not (oi_name and oi_version and oi_license):
+ continue
+ item_info = {NAME: oi_name, VERSION: oi_version, LICENSE: oi_license}
- filtered = next(filter(lambda oss_dict: oss_dict[NAME] == oi.name and oss_dict[VERSION] == oi.version, item_list), None)
+ filtered = next(filter(lambda oss_dict: oss_dict[NAME] == oi_name and oss_dict[VERSION] == oi_version, item_list), None)
if filtered:
- filtered[LICENSE].extend(oi.license)
+ filtered[LICENSE].extend(oi_license)
filtered[LICENSE] = list(set(filtered[LICENSE]))
else:
item_list.append(item_info)
diff --git a/src/fosslight_util/constant.py b/src/fosslight_util/constant.py
index 10904bc..a9764a9 100755
--- a/src/fosslight_util/constant.py
+++ b/src/fosslight_util/constant.py
@@ -15,6 +15,17 @@
f'BIN_{FL_BINARY}': FL_BINARY,
f'DEP_{FL_DEPENDENCY}': FL_DEPENDENCY}
+FOSSLIGHT_SCANNER = 'fosslight_scanner'
+FOSSLIGHT_SOURCE = 'fosslight_source'
+FOSSLIGHT_DEPENDENCY = 'fosslight_dependency'
+FOSSLIGHT_BINARY = 'fosslight_binary'
+
+SHEET_NAME_FOR_SCANNER = {
+ FOSSLIGHT_SOURCE: 'SRC_FL_Source',
+ FOSSLIGHT_BINARY: 'BIN_FL_Binary',
+ FOSSLIGHT_DEPENDENCY: 'DEP_FL_Dependency'
+}
+
# Github : https://github.com/(owner)/(repo)
# npm : https://www.npmjs.com/package/(package)/v/(version)
# npm2 : https://www.npmjs.com/package/@(group)/(package)/v/(version)
diff --git a/src/fosslight_util/correct.py b/src/fosslight_util/correct.py
index d426375..c88265a 100644
--- a/src/fosslight_util/correct.py
+++ b/src/fosslight_util/correct.py
@@ -7,19 +7,16 @@
import os
import copy
import re
-from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE
from fosslight_util.parsing_yaml import parsing_yml
-import fosslight_util.constant as constant
-from fosslight_util.oss_item import OssItem
logger = logging.getLogger(LOGGER_NAME)
SBOM_INFO_YAML = r"sbom(-|_)info[\s\S]*.ya?ml"
-def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list):
+def correct_with_yaml(correct_filepath, path_to_scan, scan_item):
success = True
msg = ""
- correct_list = {}
correct_yaml = ""
if correct_filepath == "":
correct_filepath = path_to_scan
@@ -33,101 +30,60 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list):
if not correct_yaml:
msg = f"Cannot find sbom-info.yaml in {correct_filepath}."
success = False
- return success, msg, correct_list
+ return success, msg, scan_item
rel_path = os.path.relpath(path_to_scan, correct_filepath)
- yaml_oss_list, _, err_msg = parsing_yml(correct_yaml, os.path.dirname(correct_yaml), print_log=True)
-
+ yaml_file_list, _, err_msg = parsing_yml(correct_yaml, os.path.dirname(correct_yaml), print_log=True)
find_match = False
- matched_yaml = []
- for yitem in yaml_oss_list:
- matched_yaml.append([0]*len(yitem.source_name_or_path))
-
- for sheet_name, sheet_contents in scanner_oss_list.items():
- if sheet_name not in constant.supported_sheet_and_scanner.keys():
- continue
- correct_contents = copy.deepcopy(sheet_contents)
- scanner_name = constant.supported_sheet_and_scanner[sheet_name]
- matched_source_path_with_sbom = []
- for idx, oss_raw_item in enumerate(sheet_contents):
- if len(oss_raw_item) < 9:
- logger.warning(f"sheet list is too short ({len(oss_raw_item)}): {oss_raw_item}")
+ for scanner_name, _ in scan_item.file_items.items():
+ correct_fileitems = []
+ exclude_fileitems = []
+ for yaml_file_item in yaml_file_list:
+ yaml_path_exists = False
+ if yaml_file_item.source_name_or_path == '':
+ if scanner_name == FOSSLIGHT_SOURCE:
+ correct_item = copy.deepcopy(yaml_file_item)
+ correct_item.comment = 'Added by sbom-info.yaml'
+ correct_fileitems.append(correct_item)
continue
- oss_item = OssItem('')
- oss_item.set_sheet_item(oss_raw_item, scanner_name)
+ for idx, scan_file_item in enumerate(scan_item.file_items[scanner_name]):
+ oss_rel_path = os.path.normpath(os.path.join(rel_path, scan_file_item.source_name_or_path))
+ yi_path = yaml_file_item.source_name_or_path
+ if ((os.path.normpath(yi_path) == os.path.normpath(oss_rel_path)) or
+ ((os.path.normpath(oss_rel_path).startswith(os.path.normpath(yi_path.rstrip('*')))))):
+ correct_item = copy.deepcopy(scan_file_item)
+ correct_item.exclude = yaml_file_item.exclude
+ correct_item.oss_items = copy.deepcopy(yaml_file_item.oss_items)
+ correct_item.comment = ''
+ correct_item.comment = 'Loaded from sbom-info.yaml'
+ correct_fileitems.append(correct_item)
- matched_yi = []
- if not oss_item.source_name_or_path[0] in matched_source_path_with_sbom:
- oss_rel_path = os.path.normpath(os.path.join(rel_path, oss_item.source_name_or_path[0]))
- for y_idx, yi in enumerate(yaml_oss_list):
- if not yi.source_name_or_path:
- continue
- for ys_idx, yi_path in enumerate(yi.source_name_or_path):
- yi_item = copy.deepcopy(yi)
- if ((os.path.normpath(yi_path) == os.path.normpath(oss_rel_path)) or
- ((os.path.normpath(oss_rel_path).startswith(os.path.normpath(yi_path.rstrip('*')))))):
- find_match = True
- yi_item.source_name_or_path = []
- yi_item.source_name_or_path = oss_item.source_name_or_path[0]
- matched_source_path_with_sbom.append(oss_item.source_name_or_path[0])
- matched_yi.append(yi_item)
- matched_yaml[y_idx][ys_idx] = 1
- if len(matched_yi) > 0:
- for matched_yi_item in matched_yi:
- matched_oss_item = copy.deepcopy(matched_yi_item)
- if matched_oss_item.comment:
- matched_oss_item.comment += '/'
- matched_oss_item.comment += 'Loaded from sbom-info.yaml'
- if sheet_name == 'BIN_FL_Binary':
- matched_oss_item.bin_vulnerability = oss_item.bin_vulnerability
- matched_oss_item.bin_tlsh = oss_item.bin_tlsh
- matched_oss_item.bin_sha1 = oss_item.bin_sha1
- matched_oss_array = matched_oss_item.get_print_array(scanner_name)[0]
- correct_contents.append(matched_oss_array)
- oss_item.exclude = True
- if oss_item.comment:
- oss_item.comment += '/'
- oss_item.comment += 'Excluded by sbom-info.yaml'
- correct_contents[idx] = oss_item.get_print_array(scanner_name)[0]
- else:
- oss_item.exclude = True
- if oss_item.comment:
- oss_item.comment += '/'
- oss_item.comment += 'Excluded by sbom-info.yaml'
- correct_contents[idx] = oss_item.get_print_array(scanner_name)[0]
+ yaml_path_exists = True
+ exclude_fileitems.append(idx)
- if sheet_name == 'SRC_FL_Source':
- for n_idx, ni in enumerate(matched_yaml):
- y_item = copy.deepcopy(yaml_oss_list[n_idx])
- all_matched = False
- if sum(ni) != 0:
- not_matched_path = []
- for idx, id in enumerate(ni):
- if not id:
- not_matched_path.append(y_item.source_name_or_path[idx])
- y_item.source_name_or_path = []
- y_item.source_name_or_path = not_matched_path
- if len(not_matched_path) == 0:
- all_matched = True
- if y_item.comment:
- y_item.comment += '/'
- y_item.comment += 'Added by sbom-info.yaml'
- if not (y_item.source_name_or_path or all_matched):
- correct_contents.append(y_item.get_print_array()[0])
- continue
- for y_path in y_item.source_name_or_path:
- y_item_i = copy.deepcopy(y_item)
- if not os.path.exists(os.path.normpath(os.path.join(correct_filepath, y_path))):
- y_item_i.exclude = True
- y_item_i.source_name_or_path = []
- y_item_i.source_name_or_path = y_path
- correct_contents.append(y_item_i.get_print_array()[0])
- correct_list[sheet_name] = correct_contents
+ if not yaml_path_exists:
+ correct_item = copy.deepcopy(yaml_file_item)
+ if os.path.exists(os.path.normpath(yaml_file_item.source_name_or_path)):
+ correct_item.comment = 'Loaded from sbom-info.yaml'
+ correct_fileitems.append(correct_item)
+ else:
+ if scanner_name == FOSSLIGHT_SOURCE:
+ correct_item.exclude = True
+ correct_item.comment = 'Added by sbom-info.yaml'
+ correct_fileitems.append(correct_item)
+ if correct_fileitems:
+ scan_item.append_file_items(correct_fileitems, scanner_name)
+ find_match = True
+ if exclude_fileitems:
+ exclude_fileitems = list(set(exclude_fileitems))
+ for e_idx in exclude_fileitems:
+ scan_item.file_items[scanner_name][e_idx].exclude = True
+ scan_item.file_items[scanner_name][e_idx].comment = 'Excluded by sbom-info.yaml'
if not find_match:
success = False
err_msg = 'No match items in sbom-info.yaml'
- return success, err_msg, yaml_oss_list
+ return success, err_msg, scan_item
- return success, msg, correct_list
+ return success, msg, scan_item
diff --git a/src/fosslight_util/oss_item.py b/src/fosslight_util/oss_item.py
index 53e8a65..c4dce2c 100644
--- a/src/fosslight_util/oss_item.py
+++ b/src/fosslight_util/oss_item.py
@@ -5,46 +5,41 @@
import logging
import os
-from fosslight_util.constant import LOGGER_NAME, FL_DEPENDENCY, FL_BINARY
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SCANNER
+from fosslight_util.cover import CoverItem
+from typing import List, Dict
_logger = logging.getLogger(LOGGER_NAME)
class OssItem:
- def __init__(self, value):
- self._name = ""
- self._version = ""
+
+ def __init__(self, name="", version="", license="", dl_url=""):
+ self.name = name
+ self.version = version
self._license = []
- self._copyright = ""
+ self.license = license
+ self.download_location = dl_url
+ self.exclude = False
self.comment = ""
- self._exclude = False
self.homepage = ""
- self.relative_path = value
- self._source_name_or_path = []
- self.download_location = ""
- self._yocto_recipe = []
- self._yocto_package = []
- self.is_binary = False
- self._depends_on = []
- self.purl = ""
- self.bin_vulnerability = ""
- self.bin_tlsh = ""
- self.bin_sha1 = ""
+ self._copyright = ""
def __del__(self):
pass
@property
- def copyright(self):
- return self._copyright
+ def license(self):
+ return self._license
- @copyright.setter
- def copyright(self, value):
+ @license.setter
+ def license(self, value):
if value != "":
- if isinstance(value, list):
- value = "\n".join(value)
- value = value.strip()
- self._copyright = value
+ if not isinstance(value, list):
+ value = value.split(",")
+ self._license.extend(value)
+ self._license = [item.strip() for item in self._license]
+ self._license = list(set(self._license))
@property
def exclude(self):
@@ -58,13 +53,16 @@ def exclude(self, value):
self._exclude = False
@property
- def name(self):
- return self._name
+ def copyright(self):
+ return self._copyright
- @name.setter
- def name(self, value):
+ @copyright.setter
+ def copyright(self, value):
if value != "":
- self._name = value
+ if isinstance(value, list):
+ value = "\n".join(value)
+ value = value.strip()
+ self._copyright = value
@property
def version(self):
@@ -78,149 +76,144 @@ def version(self, value):
self._version = ""
@property
- def license(self):
- return self._license
-
- @license.setter
- def license(self, value):
- if not isinstance(value, list):
- value = value.split(",")
- self._license.extend(value)
- self._license = [item.strip() for item in self._license]
- self._license = list(set(self._license))
-
- @property
- def source_name_or_path(self):
- return self._source_name_or_path
+ def comment(self):
+ return self._comment
- @source_name_or_path.setter
- def source_name_or_path(self, value):
+ @comment.setter
+ def comment(self, value):
if not value:
- self._source_name_or_path = []
+ self._comment = ""
else:
- if not isinstance(value, list):
- value = value.split(",")
- self._source_name_or_path.extend(value)
- self._source_name_or_path = [item.strip() for item in self._source_name_or_path]
- self._source_name_or_path = list(set(self._source_name_or_path))
+ if self._comment:
+ self._comment = f"{self._comment} / {value}"
+ else:
+ self._comment = value
- @property
- def yocto_recipe(self):
- return self._yocto_recipe
- @yocto_recipe.setter
- def yocto_recipe(self, value):
- if not isinstance(value, list):
- value = value.split(",")
- self._yocto_recipe.extend(value)
- self._yocto_recipe = [item.strip() for item in self._yocto_recipe]
- self._yocto_recipe = list(set(self._yocto_recipe))
+class FileItem:
+ def __init__(self, value):
+ self.relative_path = value
+ self.source_name_or_path = ""
+ self._exclude = False
+ self._comment = ""
+ self.is_binary = False
+ self.oss_items: List[OssItem] = []
+
+ def __del__(self):
+ pass
@property
- def yocto_package(self):
- return self._yocto_package
+ def exclude(self):
+ return self._exclude
- @yocto_package.setter
- def yocto_package(self, value):
- if not isinstance(value, list):
- value = value.split(",")
- self._yocto_package.extend(value)
- self._yocto_package = [item.strip() for item in self._yocto_package]
- self._yocto_package = list(set(self._yocto_package))
+ @exclude.setter
+ def exclude(self, value):
+ if value:
+ self._exclude = True
+ else:
+ self._exclude = False
+ for oss in self.oss_items:
+ oss.exclude = value
@property
- def depends_on(self):
- return self._depends_on
+ def comment(self):
+ return self._comment
- @depends_on.setter
- def depends_on(self, value):
+ @comment.setter
+ def comment(self, value):
if not value:
- self._depends_on = []
- else:
- if not isinstance(value, list):
- value = value.split(",")
- self._depends_on.extend(value)
- self._depends_on = [item.strip() for item in self._depends_on]
- self._depends_on = list(set(self._depends_on))
-
- def set_sheet_item(self, item, scanner_name=''):
- if len(item) < 9:
- _logger.warning(f"sheet list is too short ({len(item)}): {item}")
- return
- if scanner_name == FL_DEPENDENCY:
- self.purl = item[0]
+ self._comment = ""
else:
- self.source_name_or_path = item[0]
- self.name = item[1]
- self.version = item[2]
- self.license = item[3]
- self.download_location = item[4]
- self.homepage = item[5]
- self.copyright = item[6]
- self.exclude = item[7]
- self.comment = item[8]
-
- if len(item) >= 10 and scanner_name == FL_DEPENDENCY:
- self.depends_on = item[9]
- if len(item) >= 10 and scanner_name == FL_BINARY:
- self.bin_vulnerability = item[9]
- if len(item) >= 12:
- self.bin_tlsh = item[10]
- self.bin_sha1 = item[11]
-
- def get_print_array(self, scanner_name=''):
+ if self._comment:
+ self._comment = f"{self._comment} / {value}"
+ else:
+ self._comment = value
+ for oss in self.oss_items:
+ oss.comment = value
+
+ def get_print_array(self):
items = []
- if scanner_name != FL_DEPENDENCY:
- if len(self.source_name_or_path) == 0:
- self.source_name_or_path.append("")
- if len(self.license) == 0:
- self.license.append("")
-
- exclude = "Exclude" if self.exclude else ""
- lic = ",".join(self.license)
- if scanner_name == FL_DEPENDENCY:
- items = [self.purl, self.name, self.version, lic,
- self.download_location, self.homepage, self.copyright, exclude, self.comment]
- if len(self.depends_on) > 0:
- items.append(",".join(self.depends_on))
- else:
- for source_name_or_path in self.source_name_or_path:
- if scanner_name == FL_BINARY:
- oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
- self.download_location, self.homepage, self.copyright, exclude, self.comment,
- self.bin_vulnerability, self.bin_tlsh, self.bin_sha1]
- else:
- oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
- self.download_location, self.homepage, self.copyright, exclude, self.comment]
- items.append(oss_item)
+
+ for oss in self.oss_items:
+ exclude = "Exclude" if self.exclude or oss.exclude else ""
+ lic = ",".join(oss.license)
+
+ oss_item = [os.path.join(self.relative_path, self.source_name_or_path), oss.name, oss.version, lic,
+ oss.download_location, oss.homepage, oss.copyright, exclude, oss.comment]
+ items.append(oss_item)
return items
def get_print_json(self):
- json_item = {}
- json_item["name"] = self.name
-
- json_item["version"] = self.version
- if len(self.source_name_or_path) > 0:
- json_item["source path"] = self.source_name_or_path
- if len(self.license) > 0:
- json_item["license"] = self.license
- if self.download_location != "":
- json_item["download location"] = self.download_location
- if self.homepage != "":
- json_item["homepage"] = self.homepage
- if self.copyright != "":
- json_item["copyright text"] = self.copyright
- if self.exclude:
- json_item["exclude"] = self.exclude
- if self.comment != "":
- json_item["comment"] = self.comment
- if len(self.depends_on) > 0:
- json_item["depends on"] = self.depends_on
- if self.purl != "":
- json_item["package url"] = self.purl
-
- return json_item
+ items = []
+
+ for oss in self.oss_items:
+ json_item = {}
+ json_item["name"] = oss.name
+ json_item["version"] = oss.version
+
+ if self.source_name_or_path != "":
+ json_item["source path"] = self.source_name_or_path
+ if len(oss.license) > 0:
+ json_item["license"] = oss.license
+ if oss.download_location != "":
+ json_item["download location"] = oss.download_location
+ if oss.homepage != "":
+ json_item["homepage"] = oss.homepage
+ if oss.copyright != "":
+ json_item["copyright text"] = oss.copyright
+ if self.exclude or oss.exclude:
+ json_item["exclude"] = True
+ if oss.comment != "":
+ json_item["comment"] = oss.comment
+ items.append(json_item)
+ return items
def invalid(cmd):
_logger.info('[{}] is invalid'.format(cmd))
+
+
+class ScannerItem:
+ def __init__(self, pkg_name, start_time=""):
+ self.cover = CoverItem(tool_name=pkg_name, start_time=start_time)
+ self.file_items: Dict[str, List[FileItem]] = {pkg_name: []} if pkg_name != FOSSLIGHT_SCANNER else {}
+ self.external_sheets: Dict[str, List[List[str]]] = {}
+
+ def set_cover_pathinfo(self, input_dir, path_to_exclude):
+ self.cover.input_path = input_dir
+ self.cover.exclude_path = ", ".join(path_to_exclude)
+
+ def set_cover_comment(self, value):
+ if value:
+ if self.cover.comment:
+ self.cover.comment = f"{self.cover.comment} / {value}"
+ else:
+ self.cover.comment = value
+
+ def get_cover_comment(self):
+ return [item.strip() for item in self.cover.comment.split(" / ")]
+
+ def append_file_items(self, file_item: List[FileItem], pkg_name=""):
+ if pkg_name == "":
+ if len(self.file_items.keys()) != 1:
+ _logger.error("Package name is not set. Cannot append file_item into ScannerItem.")
+ else:
+ pkg_name = list(self.file_items.keys())[0]
+ if pkg_name not in self.file_items:
+ self.file_items[pkg_name] = []
+ self.file_items[pkg_name].extend(file_item)
+
+ def get_print_array(self, scanner_name):
+ items = []
+ for file_item in self.file_items[scanner_name]:
+ items.extend(file_item.get_print_array())
+ return items
+
+ def get_print_json(self, scanner_name):
+ items = []
+ for file_item in self.file_items[scanner_name]:
+ items.extend(file_item.get_print_json())
+ return items
+
+ def __del__(self):
+ pass
diff --git a/src/fosslight_util/output_format.py b/src/fosslight_util/output_format.py
index d0bec60..6b30e1c 100644
--- a/src/fosslight_util/output_format.py
+++ b/src/fosslight_util/output_format.py
@@ -106,8 +106,8 @@ def check_output_formats(output='', formats=[], customized_format={}):
return success, msg, output_path, output_files, output_extensions
-def write_output_file(output_file_without_ext: str, file_extension: str, sheet_list: dict, extended_header: dict = {},
- hide_header: dict = {}, cover: str = "") -> Tuple[bool, str, str]:
+def write_output_file(output_file_without_ext: str, file_extension: str, scan_item, extended_header: dict = {},
+ hide_header: dict = {}) -> Tuple[bool, str, str]:
success = True
msg = ''
@@ -116,13 +116,13 @@ def write_output_file(output_file_without_ext: str, file_extension: str, sheet_l
result_file = output_file_without_ext + file_extension
if file_extension == '.xlsx':
- success, msg = write_result_to_excel(result_file, sheet_list, extended_header, hide_header, cover)
+ success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
elif file_extension == '.csv':
- success, msg, result_file = write_result_to_csv(result_file, sheet_list, False, extended_header)
+ success, msg, result_file = write_result_to_csv(result_file, scan_item, False, extended_header)
elif file_extension == '.json':
- success, msg = write_opossum(result_file, sheet_list)
+ success, msg = write_opossum(result_file, scan_item)
elif file_extension == '.yaml':
- success, msg, result_file = write_yaml(result_file, sheet_list, False)
+ success, msg, result_file = write_yaml(result_file, scan_item, False)
else:
success = False
msg = f'Not supported file extension({file_extension})'
diff --git a/src/fosslight_util/parsing_yaml.py b/src/fosslight_util/parsing_yaml.py
index c8fde32..3fda179 100644
--- a/src/fosslight_util/parsing_yaml.py
+++ b/src/fosslight_util/parsing_yaml.py
@@ -8,8 +8,8 @@
import os
import re
import sys
-from .constant import LOGGER_NAME
-from .oss_item import OssItem
+from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.oss_item import OssItem, FileItem
_logger = logging.getLogger(LOGGER_NAME)
SUPPORT_OSS_INFO_FILES = [r"oss-pkg-info[\s\S]*.ya?ml", r"sbom(-|_)info[\s\S]*.ya?ml"]
@@ -17,7 +17,7 @@
def parsing_yml(yaml_file, base_path, print_log=True):
- oss_list = []
+ fileitems = []
license_list = []
idx = 1
err_reason = ""
@@ -38,37 +38,65 @@ def parsing_yml(yaml_file, base_path, print_log=True):
err_reason = "empty"
if print_log:
_logger.warning(f"The yaml file is empty file: {yaml_file}")
- return oss_list, license_list, err_reason
+ return fileitems, license_list, err_reason
is_old_format = any(x in doc for x in OLD_YAML_ROOT_ELEMENT)
+ filepath_list = []
for root_element in doc:
oss_items = doc[root_element]
if oss_items:
if not isinstance(oss_items, list) or 'version' not in oss_items[0]:
raise AttributeError(f"- Ref. {EXAMPLE_OSS_PKG_INFO_LINK}")
for oss in oss_items:
- item = OssItem(relative_path)
- if not is_old_format:
- item.name = root_element
- for key, value in oss.items():
- if key:
- key = key.lower().strip()
- set_value_switch(item, key, value, yaml_file)
- oss_list.append(item)
- license_list.extend(item.license)
- idx += 1
+ source_paths = get_source_name_or_path_in_yaml(oss)
+ for source_path in source_paths:
+ if os.path.join(relative_path, source_path) not in filepath_list:
+ filepath_list.append(os.path.join(relative_path, source_path))
+ fileitem = FileItem(relative_path)
+ fileitem.source_name_or_path = source_path
+ fileitems.append(fileitem)
+ else:
+ fileitem = next((i for i in fileitems if i.source_name_or_path == source_path), None)
+ ossitem = OssItem()
+ if not is_old_format:
+ ossitem.name = root_element
+ for key, value in oss.items():
+ if key:
+ key = key.lower().strip()
+ set_value_switch(ossitem, key, value, yaml_file)
+ fileitem.oss_items.append(ossitem)
+ license_list.extend(ossitem.license)
+ idx += 1
except AttributeError as ex:
if print_log:
_logger.warning(f"Not supported yaml file format: {yaml_file} {ex}")
- oss_list = []
+ fileitems = []
err_reason = "not_supported"
except yaml.YAMLError:
if print_log:
_logger.warning(f"Error to parse yaml - skip to parse yaml file: {yaml_file}")
- oss_list = []
+ fileitems = []
err_reason = "yaml_error"
- return oss_list, set(license_list), err_reason
+
+ return fileitems, set(license_list), err_reason
+
+
+def get_source_name_or_path_in_yaml(oss):
+ source_name_or_path = []
+ find = False
+ for key in oss.keys():
+ if key in ['file name or path', 'source name or path', 'source path',
+ 'file', 'binary name', 'binary path']:
+ if isinstance(oss[key], list):
+ source_name_or_path = oss[key]
+ else:
+ source_name_or_path.append(oss[key])
+ find = True
+ break
+ if not find:
+ source_name_or_path.append('')
+ return source_name_or_path
def find_sbom_yaml_files(path_to_find):
@@ -101,9 +129,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
oss.download_location = value
elif key in ['license', 'license text']:
oss.license = value
- elif key in ['file name or path', 'source name or path', 'source path',
- 'file', 'binary name', 'binary path']:
- oss.source_name_or_path = value
elif key in ['copyright text', 'copyright']:
oss.copyright = value
elif key == 'exclude':
@@ -112,16 +137,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
oss.comment = value
elif key == 'homepage':
oss.homepage = value
- elif key == 'yocto_package':
- oss.yocto_package = value
- elif key == 'yocto_recipe':
- oss.yocto_recipe = value
- elif key == 'vulnerability link':
- oss.bin_vulnerability = value
- elif key == 'tlsh':
- oss.bin_tlsh = value
- elif key == 'sha1':
- oss.bin_sha1 = value
else:
if yaml_file != "":
_logger.debug(f"file:{yaml_file} - key:{key} cannot be parsed")
diff --git a/src/fosslight_util/read_excel.py b/src/fosslight_util/read_excel.py
index 2c20493..1ddb161 100644
--- a/src/fosslight_util/read_excel.py
+++ b/src/fosslight_util/read_excel.py
@@ -7,7 +7,7 @@
import pandas as pd
import json
from fosslight_util.constant import LOGGER_NAME
-from fosslight_util.oss_item import OssItem
+from fosslight_util.oss_item import OssItem, FileItem
from fosslight_util.parsing_yaml import set_value_switch
logger = logging.getLogger(LOGGER_NAME)
@@ -16,8 +16,8 @@
SHEET_PREFIX_TO_READ = ["bin", "bom", "src"]
-def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
- oss_report_items: List[OssItem] = []
+def read_oss_report(excel_file: str, sheet_names: str = "", basepath: str = "") -> List[FileItem]:
+ fileitems: List[FileItem] = []
xl_sheets: Dict[str, Any] = {}
all_sheet_to_read: List[str] = []
not_matched_sheet: List[str] = []
@@ -57,6 +57,7 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
elif (not sheet_name_prefix_match) and not_matched_sheet:
logger.warning(f"Not matched sheet name: {not_matched_sheet}")
+ filepath_list = []
for sheet_name, xl_sheet in xl_sheets.items():
_item_idx = {
"ID": IDX_CANNOT_FOUND,
@@ -87,11 +88,18 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
is_bin = True if sheet_name.lower().startswith(PREFIX_BIN) else False
for row_idx, row in xl_sheet.iterrows():
- item = OssItem("")
- item.is_binary = is_bin
valid_row = True
load_data_cnt = 0
-
+ source_path = row[1]
+ if source_path not in filepath_list:
+ filepath_list.append(source_path)
+ fileitem = FileItem(basepath)
+ fileitem.source_name_or_path = source_path
+ fileitems.append(fileitem)
+ else:
+ fileitem = next((i for i in fileitems if i.source_name_or_path == source_path), None)
+ fileitem.is_binary = is_bin
+ ossitem = OssItem()
for column_key, column_idx in column_keys.items():
if column_idx != IDX_CANNOT_FOUND:
cell_obj = xl_sheet.iloc[row_idx, column_idx]
@@ -101,13 +109,13 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
if column_key != "ID":
if column_key:
column_key = column_key.lower().strip()
- set_value_switch(item, column_key, cell_value)
+ set_value_switch(ossitem, column_key, cell_value)
load_data_cnt += 1
else:
valid_row = False if cell_value == "-" else True
if valid_row and load_data_cnt > 0:
- oss_report_items.append(item)
+ fileitem.oss_items.append(ossitem)
except Exception as error:
logger.error(f"Parsing a OSS Report: {error}")
- return oss_report_items
+ return fileitems
diff --git a/src/fosslight_util/write_excel.py b/src/fosslight_util/write_excel.py
index cd15b50..ba6ed71 100755
--- a/src/fosslight_util/write_excel.py
+++ b/src/fosslight_util/write_excel.py
@@ -7,162 +7,101 @@
import time
import logging
import os
-import platform
import pandas as pd
-import copy
from pathlib import Path
-import fosslight_util.constant as constant
+from fosslight_util.constant import LOGGER_NAME, SHEET_NAME_FOR_SCANNER, FOSSLIGHT_BINARY
from jsonmerge import merge
-from fosslight_util.cover import CoverItem
-from typing import Tuple
_HEADER = {'BIN (': ['ID', 'Binary Path', 'Source Code Path',
'NOTICE.html', 'OSS Name', 'OSS Version',
'License', 'Download Location', 'Homepage',
'Copyright Text', 'Exclude', 'Comment'],
- 'SRC': ['ID', 'Source Path', 'OSS Name',
- 'OSS Version', 'License', 'Download Location',
- 'Homepage', 'Copyright Text', 'Exclude',
- 'Comment'],
+ 'SRC': ['ID', 'Source Path', 'OSS Name', 'OSS Version',
+ 'License', 'Download Location', 'Homepage',
+ 'Copyright Text', 'Exclude', 'Comment'],
'BIN': ['ID', 'Binary Path', 'OSS Name', 'OSS Version',
'License', 'Download Location', 'Homepage',
- 'Copyright Text', 'Exclude', 'Comment']}
+ 'Copyright Text', 'Exclude', 'Comment',
+ 'Vulnerability Link', 'TLSH', 'SHA1'],
+ 'DEP': ['ID', 'Package URL', 'OSS Name', 'OSS Version',
+ 'License', 'Download Location', 'Homepage',
+ 'Copyright Text', 'Exclude', 'Comment',
+ 'Depends On']}
+
+BIN_HIDE_HEADER = {'TLSH', "SHA1"}
_OUTPUT_FILE_PREFIX = "FOSSLight-Report_"
-_EMPTY_ITEM_MSG = "* There is no item"\
- " to print in FOSSLight-Report.\n"
IDX_FILE = 0
IDX_EXCLUDE = 7
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
COVER_SHEET_NAME = 'Scanner Info'
-def write_excel_and_csv(filename_without_extension: str, sheet_list: dict, ignore_os: bool = False,
- extended_header: dict = {}, hide_header: dict = {}) -> Tuple[bool, str, str]:
- success = True
- error_msg = ""
- success_csv = True
- error_msg_csv = ""
- output_files = ""
- output_csv = ""
-
- is_not_null, sheet_list = remove_empty_sheet(sheet_list)
-
- if is_not_null:
- output_dir = os.path.dirname(filename_without_extension)
- Path(output_dir).mkdir(parents=True, exist_ok=True)
-
- success, error_msg = write_result_to_excel(f"{filename_without_extension}.xlsx",
- sheet_list,
- extended_header,
- hide_header)
-
- if ignore_os or platform.system() != "Windows":
- success_csv, error_msg_csv, output_csv = write_result_to_csv(f"{filename_without_extension}.csv",
- sheet_list, True, extended_header)
- if success:
- output_files = f"{filename_without_extension}.xlsx"
- else:
- error_msg = "[Error] Writing excel:" + error_msg
- if success_csv:
- if output_csv:
- output_files = f"{output_files}, {output_csv}" if output_files else output_csv
- else:
- error_msg += "\n[Error] Writing csv:" + error_msg_csv
- else:
- success = False
- error_msg = _EMPTY_ITEM_MSG
-
- return (success and success_csv), error_msg, output_files
-
-
-def remove_empty_sheet(sheet_items):
- skip_sheet_name = []
- cnt_sheet_to_print = 0
- final_sheet_to_print = {}
- success = False
- try:
- if sheet_items:
- for sheet_name, sheet_content in sheet_items.items():
- if len(sheet_content) > 0:
- final_sheet_to_print[sheet_name] = sheet_content
- cnt_sheet_to_print += 1
- else:
- skip_sheet_name.append(sheet_name)
- if cnt_sheet_to_print != 0:
- success = True
- if len(skip_sheet_name) > 0:
- logger.warn("* Empty sheet(not printed):" + str(skip_sheet_name))
- except Exception as ex:
- logger.warn("* Warning:"+str(ex))
-
- return success, final_sheet_to_print
-
-
-def get_header_row(sheet_name, sheet_content, extended_header={}):
+def get_header_row(sheet_name, extended_header={}):
selected_header = []
merged_headers = merge(_HEADER, extended_header)
- selected_header = merged_headers.get(sheet_name)
+ selected_header = merged_headers.get(sheet_name, [])
if not selected_header:
for header_key in merged_headers.keys():
if sheet_name.startswith(header_key):
selected_header = merged_headers[header_key]
break
- if len(sheet_content) > 0:
- if not selected_header:
- selected_header = sheet_content.pop(0)
- return selected_header, sheet_content
+ return selected_header
-def write_result_to_csv(output_file, sheet_list_origin, separate_sheet=False, extended_header={}):
+def write_result_to_csv(output_file, scan_item, separate_sheet=False, extended_header={}):
success = True
error_msg = ""
file_extension = ".csv"
output = ""
try:
- sheet_list = copy.deepcopy(sheet_list_origin)
- if sheet_list:
- output_files = []
- output_dir = os.path.dirname(output_file)
- Path(output_dir).mkdir(parents=True, exist_ok=True)
- if separate_sheet:
- filename = os.path.splitext(os.path.basename(output_file))[0]
- separate_output_file = os.path.join(output_dir, filename)
-
- merge_sheet = []
- for sheet_name, sheet_contents in sheet_list.items():
- row_num = 1
- header_row, sheet_content_without_header = get_header_row(sheet_name, sheet_contents[:], extended_header)
-
- if 'Copyright Text' in header_row:
- idx = header_row.index('Copyright Text')-1
- for item in sheet_content_without_header:
- item[idx] = item[idx].replace('\n', ', ')
- if not separate_sheet:
- merge_sheet.extend(sheet_content_without_header)
- if sheet_name == list(sheet_list.keys())[-1]:
- sheet_content_without_header = merge_sheet
- else:
- continue
+ output_files = []
+ output_dir = os.path.dirname(output_file)
+ Path(output_dir).mkdir(parents=True, exist_ok=True)
+ if separate_sheet:
+ filename = os.path.splitext(os.path.basename(output_file))[0]
+ separate_output_file = os.path.join(output_dir, filename)
+
+ merge_sheet = []
+ for scanner_name, _ in scan_item.file_items.items():
+ row_num = 1
+ sheet_name = ""
+ if scanner_name.lower() in SHEET_NAME_FOR_SCANNER:
+ sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]
+ elif extended_header:
+ sheet_name = list(extended_header.keys())[0]
+ sheet_content_without_header = scan_item.get_print_array(scanner_name)
+ header_row = get_header_row(sheet_name, extended_header)
+
+ if 'Copyright Text' in header_row:
+ idx = header_row.index('Copyright Text')-1
+ for item in sheet_content_without_header:
+ item[idx] = item[idx].replace('\n', ', ')
+ if not separate_sheet:
+ merge_sheet.extend(sheet_content_without_header)
+ if scanner_name == list(scan_item.file_items.keys())[-1]:
+ sheet_content_without_header = merge_sheet
else:
- output_file = separate_output_file + "_" + sheet_name + file_extension
- try:
- sheet_content_without_header = sorted(sheet_content_without_header,
- key=lambda x: (x[IDX_EXCLUDE], x[IDX_FILE] == "", x[IDX_FILE]))
- except Exception:
- pass
- with open(output_file, 'w', newline='') as file:
- writer = csv.writer(file, delimiter='\t')
- writer.writerow(header_row)
- for row_item in sheet_content_without_header:
- row_item.insert(0, row_num)
- writer.writerow(row_item)
- row_num += 1
- output_files.append(output_file)
- if output_files:
- output = ", ".join(output_files)
+ continue
+ else:
+ output_file = separate_output_file + "_" + sheet_name + file_extension
+ try:
+ sheet_content_without_header = sorted(sheet_content_without_header,
+ key=lambda x: (x[IDX_EXCLUDE], x[IDX_FILE] == "", x[IDX_FILE]))
+ except Exception:
+ pass
+ with open(output_file, 'w', newline='') as file:
+ writer = csv.writer(file, delimiter='\t')
+ writer.writerow(header_row)
+ for row_item in sheet_content_without_header:
+ row_item.insert(0, row_num)
+ writer.writerow(row_item)
+ row_num += 1
+ output_files.append(output_file)
+ if output_files:
+ output = ", ".join(output_files)
except Exception as ex:
error_msg = str(ex)
success = False
@@ -170,7 +109,7 @@ def write_result_to_csv(output_file, sheet_list_origin, separate_sheet=False, ex
return success, error_msg, output
-def write_result_to_excel(out_file_name, sheet_list, extended_header={}, hide_header={}, cover=""):
+def write_result_to_excel(out_file_name, scan_item, extended_header={}, hide_header={}):
success = True
error_msg = ""
@@ -179,21 +118,37 @@ def write_result_to_excel(out_file_name, sheet_list, extended_header={}, hide_he
Path(output_dir).mkdir(parents=True, exist_ok=True)
workbook = xlsxwriter.Workbook(out_file_name)
- if cover:
- write_cover_sheet(workbook, cover)
- if sheet_list:
- for sheet_name, sheet_contents in sheet_list.items():
- selected_header, sheet_content_without_header = get_header_row(sheet_name, sheet_contents[:], extended_header)
+ write_cover_sheet(workbook, scan_item.cover)
+ if scan_item.file_items and len(scan_item.file_items.keys()) > 0:
+ for scanner_name, _ in scan_item.file_items.items():
+ sheet_name = ""
+ if scanner_name.lower() in SHEET_NAME_FOR_SCANNER:
+ sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]
+ elif extended_header:
+ sheet_name = list(extended_header.keys())[0]
+ sheet_content_without_header = scan_item.get_print_array(scanner_name)
+ selected_header = get_header_row(sheet_name, extended_header)
try:
sheet_content_without_header = sorted(sheet_content_without_header,
key=lambda x: (x[IDX_EXCLUDE], x[IDX_FILE] == "", x[IDX_FILE]))
except Exception:
pass
+ if sheet_name:
+ worksheet = create_worksheet(workbook, sheet_name, selected_header)
+ write_result_to_sheet(worksheet, sheet_content_without_header)
+ if (scanner_name == FOSSLIGHT_BINARY) and (not hide_header):
+ hide_header = BIN_HIDE_HEADER
+ if hide_header:
+ hide_column(worksheet, selected_header, hide_header)
+
+ for sheet_name, content in scan_item.external_sheets.items():
+ if len(content) > 0:
+ selected_header = content.pop(0)
worksheet = create_worksheet(workbook, sheet_name, selected_header)
- write_result_to_sheet(worksheet, sheet_content_without_header)
-
+ write_result_to_sheet(worksheet, content)
if hide_header:
hide_column(worksheet, selected_header, hide_header)
+
workbook.close()
except Exception as ex:
error_msg = str(ex)
@@ -242,37 +197,12 @@ def create_worksheet(workbook, sheet_name, header_row):
current_time = str(time.time())
sheet_name = current_time
worksheet = workbook.add_worksheet(sheet_name)
- for col_num, value in enumerate(header_row):
- worksheet.write(0, col_num, value)
+ if header_row:
+ for col_num, value in enumerate(header_row):
+ worksheet.write(0, col_num, value)
return worksheet
-def merge_cover_comment(find_excel_dir, merge_files=''):
- FIND_EXTENSION = '.xlsx'
- merge_comment = []
- cover_comment = ''
- try:
- files = os.listdir(find_excel_dir)
-
- if len([name for name in files if name.endswith(FIND_EXTENSION)]) > 0:
- for file in files:
- if merge_files:
- if file not in merge_files:
- continue
- if file.endswith(FIND_EXTENSION):
- file = os.path.join(find_excel_dir, file)
- df_excel = pd.read_excel(file, sheet_name=COVER_SHEET_NAME, index_col=0, engine='openpyxl')
- if not df_excel.empty:
- tool_name = df_excel.loc[CoverItem.tool_name_key].values[0]
- comment = df_excel.loc[CoverItem.comment_key].values[0]
- merge_comment.append(str(f"[{tool_name}] {comment}"))
- cover_comment = '\n'.join(merge_comment)
- except Exception as ex:
- logger.warning(f'Fail to merge comment of Scanner info: {str(ex)}')
-
- return cover_comment
-
-
def merge_excels(find_excel_dir, final_out, merge_files='', cover=''):
success = True
msg = ""
diff --git a/src/fosslight_util/write_opossum.py b/src/fosslight_util/write_opossum.py
index 4c1a043..6657fe6 100644
--- a/src/fosslight_util/write_opossum.py
+++ b/src/fosslight_util/write_opossum.py
@@ -11,10 +11,9 @@
from datetime import datetime
from pathlib import Path
import traceback
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SOURCE
from typing import Dict, Optional, Tuple
-import fosslight_util.constant as constant
-
PACKAGE = {
'requirements.txt': 'pypi',
@@ -30,7 +29,7 @@
}
_attributionConfidence = 80
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
class AttributionItem():
@@ -51,7 +50,7 @@ def __init__(self,
self.excludeFromNotice = False
self.source_name = source_name
- if source_name == constant.FL_DEPENDENCY:
+ if source_name == FOSSLIGHT_DEPENDENCY:
self.preSelected = True
else:
self.preSelected = False
@@ -113,12 +112,12 @@ def get_externalAttribution_dict(self):
dict[licenseName] = self.licenseName
dict[preSelected] = self.preSelected
- if self.source_name == constant.FL_SOURCE or constant.FL_BINARY:
+ if self.source_name == FOSSLIGHT_SOURCE or FOSSLIGHT_BINARY:
dict[copyright] = self.copyright
dict[packageName] = self.packageName
dict[packageVersion] = self.packageVersion
dict[url] = self.url
- elif self.source_name == constant.FL_DEPENDENCY:
+ elif self.source_name == FOSSLIGHT_DEPENDENCY:
dict[copyright] = self.copyright
dict[packageName] = self.packageName
dict[packageVersion] = self.packageVersion
@@ -165,7 +164,7 @@ def make_frequentlicenses():
return frequentLicenses, success, error_msg
-def write_opossum(filename: str, sheet_list: dict) -> Tuple[bool, str]:
+def write_opossum(filename: str, scan_item) -> Tuple[bool, str]:
success = True
error_msg = ''
dict = {}
@@ -176,7 +175,7 @@ def write_opossum(filename: str, sheet_list: dict) -> Tuple[bool, str]:
_filesWithChildren_key = 'filesWithChildren'
_attributionBreakpoints_key = 'attributionBreakpoints'
- if sheet_list:
+ if scan_item:
output_dir = os.path.dirname(filename)
Path(output_dir).mkdir(parents=True, exist_ok=True)
@@ -189,14 +188,9 @@ def write_opossum(filename: str, sheet_list: dict) -> Tuple[bool, str]:
filesWithChildren_list = []
attributionBreakpoints_list = []
try:
- for sheet_name, sheet_contents in sheet_list.items():
- if sheet_name in constant.supported_sheet_and_scanner.keys():
- scanner = constant.supported_sheet_and_scanner.get(sheet_name)
- else:
- logger.warning("Not supported scanner(sheet_name):" + sheet_name)
- continue
-
- ret_resources_attribution = make_resources_and_attributions(sheet_contents, scanner, resources, fc_list)
+ for scanner_name, _ in scan_item.file_items.items():
+ sheet_contents = scan_item.get_print_array(scanner_name)
+ ret_resources_attribution = make_resources_and_attributions(sheet_contents, scanner_name, resources, fc_list)
success, rsc, ea, ra, fl, ab = ret_resources_attribution
if success:
dict[_resources_key].update(rsc)
@@ -255,14 +249,14 @@ def make_resources_and_attributions(sheet_items, scanner, resources, fc_list):
items = items[0:9]
path, oss_name, oss_version, license, url, homepage, copyright, exclude, comment = items
- if scanner == constant.FL_SOURCE:
+ if scanner == FOSSLIGHT_SOURCE:
if (os.path.join(os.sep, path) + os.sep) not in fc_list:
resources = make_resources(path, resources)
attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url)
- elif scanner == constant.FL_BINARY:
+ elif scanner == FOSSLIGHT_BINARY:
resources = make_resources(path, resources)
attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url)
- elif scanner == constant.FL_DEPENDENCY:
+ elif scanner == FOSSLIGHT_DEPENDENCY:
try:
packageType = PACKAGE[path]
except Exception:
diff --git a/src/fosslight_util/write_scancodejson.py b/src/fosslight_util/write_scancodejson.py
index 18f8175..7b87018 100644
--- a/src/fosslight_util/write_scancodejson.py
+++ b/src/fosslight_util/write_scancodejson.py
@@ -6,59 +6,59 @@
import logging
import os
import json
-import fosslight_util.constant as constant
-from fosslight_util.oss_item import OssItem
+from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.oss_item import ScannerItem
from typing import List
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
EMPTY_FILE_PATH = '-'
-def write_scancodejson(output_dir: str, output_filename: str, oss_list: List[OssItem]):
+def write_scancodejson(output_dir: str, output_filename: str, oss_list: List[ScannerItem]):
json_output = {}
json_output['headers'] = []
json_output['summary'] = {}
json_output['license_detections'] = []
json_output['files'] = []
- for oi in oss_list:
- if oi.exclude:
- continue
- if not oi.source_name_or_path:
- oi.source_name_or_path = EMPTY_FILE_PATH
- for item_path in oi.source_name_or_path:
- filtered = next(filter(lambda x: x['path'] == item_path, json_output['files']), None)
- if filtered:
- append_oss_item_in_filesitem(oi, filtered)
- else:
- json_output['files'] = add_item_in_files(oi, item_path, json_output['files'])
+ for file_items in oss_list.file_items.values():
+ for fi in file_items:
+ if fi.exclude:
+ continue
+ if fi.oss_items and (all(oss_item.exclude for oss_item in fi.oss_items)):
+ continue
+ if not fi.source_name_or_path:
+ fi.source_name_or_path = EMPTY_FILE_PATH
+ json_output['files'] = add_item_in_files(fi, json_output['files'])
+
with open(os.path.join(output_dir, output_filename), 'w') as f:
json.dump(json_output, f, sort_keys=False, indent=4)
-def append_oss_item_in_filesitem(item, files_item):
- if item.is_binary:
- files_item['is_binary'] = item.is_binary
- if item.name or item.version or item.license or item.copyright or item.download_location or item.comment:
+def append_oss_item_in_filesitem(oss_items, files_item):
+ for oi in oss_items:
+ if oi.exclude:
+ continue
oss_item = {}
- oss_item['name'] = item.name
- oss_item['version'] = item.version
- oss_item['license'] = item.license
- oss_item['copyright'] = item.copyright
- oss_item['download_location'] = item.download_location
- oss_item['comment'] = item.comment
+ oss_item['name'] = oi.name
+ oss_item['version'] = oi.version
+ oss_item['license'] = oi.license
+ oss_item['copyright'] = oi.copyright
+ oss_item['download_location'] = oi.download_location
+ oss_item['comment'] = oi.comment
files_item['oss'].append(oss_item)
+
return files_item
-def add_item_in_files(item, item_path, files_list):
+def add_item_in_files(file_item, files_list):
files_item = {}
- files_item['path'] = item_path
- files_item['name'] = os.path.basename(item_path)
- files_item['is_binary'] = item.is_binary
- files_item['base_name'], files_item['extension'] = os.path.splitext(os.path.basename(item_path))
+ files_item['path'] = file_item.source_name_or_path
+ files_item['name'] = os.path.basename(file_item.source_name_or_path)
+ files_item['is_binary'] = file_item.is_binary
+ files_item['base_name'], files_item['extension'] = os.path.splitext(os.path.basename(file_item.source_name_or_path))
files_item['oss'] = []
- files_item = append_oss_item_in_filesitem(item, files_item)
+ files_item = append_oss_item_in_filesitem(file_item.oss_items, files_item)
files_list.append(files_item)
return files_list
diff --git a/src/fosslight_util/write_spdx.py b/src/fosslight_util/write_spdx.py
index c00a735..e4e3008 100644
--- a/src/fosslight_util/write_spdx.py
+++ b/src/fosslight_util/write_spdx.py
@@ -21,10 +21,10 @@
from spdx.writers import xml
from spdx.writers import tagvalue
from fosslight_util.spdx_licenses import get_spdx_licenses_json, get_license_from_nick
-import fosslight_util.constant as constant
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_DEPENDENCY
import traceback
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
def get_license_list_version():
@@ -37,11 +37,11 @@ def get_license_list_version():
return version
-def write_spdx(output_file_without_ext, output_extension, sheet_list,
+def write_spdx(output_file_without_ext, output_extension, scan_item,
scanner_name, scanner_version, spdx_version=(2, 3)):
success = True
error_msg = ''
- if sheet_list:
+ if scan_item:
doc = Document(version=Version(*spdx_version),
data_license=License.from_identifier('CC0-1.0'),
namespace=f'http://spdx.org/spdxdocs/{scanner_name.lower()}-{uuid.uuid4()}',
@@ -60,44 +60,39 @@ def write_spdx(output_file_without_ext, output_extension, sheet_list,
try:
package_id = 0
root_package = False
- for sheet_name, sheet_contents in sheet_list.items():
- if sheet_name not in constant.supported_sheet_and_scanner.keys():
- continue
- scanner = constant.supported_sheet_and_scanner.get(sheet_name)
- for oss_item in sheet_contents:
- if len(oss_item) < 9:
- logger.warning(f"sheet list is too short ({len(oss_item)}): {oss_item}")
- continue
+ for scanner_name, _ in scan_item.file_items.items():
+ json_contents = scan_item.get_print_json(scanner_name)
+ for oss_item in json_contents:
package_id += 1
package = Package(spdx_id=f'SPDXRef-{package_id}')
- if oss_item[1] != '':
- package.name = oss_item[1] # required
+ if oss_item.get('name', '') != '':
+ package.name = oss_item.get('name', '') # required
else:
package.name = SPDXNone()
- if oss_item[2] != '':
- package.version = oss_item[2] # no required
+ if oss_item.get('version', '') != '':
+ package.version = oss_item.get('version', '') # no required
- if oss_item[4] != '':
- package.download_location = oss_item[4] # required
+ if oss_item.get('download location', '') != '':
+ package.download_location = oss_item.get('download location', '') # required
else:
package.download_location = SPDXNone()
- if scanner == constant.FL_DEPENDENCY:
+ if scanner_name == FOSSLIGHT_DEPENDENCY:
package.files_analyzed = False # If omitted, the default value of true is assumed.
else:
package.files_analyzed = True
- if oss_item[5] != '':
- package.homepage = oss_item[5] # no required
+ if oss_item.get('homepage', '') != '':
+ package.homepage = oss_item.get('homepage', '') # no required
- if oss_item[6] != '':
- package.cr_text = oss_item[3] # required
+ if oss_item.get('copyright text', '') != '':
+ package.cr_text = oss_item.get('copyright text', '') # required
else:
package.cr_text = SPDXNone()
- if oss_item[3] != '':
- lic_list = [check_input_license_format(lic.strip()) for lic in oss_item[3].split(',')]
+ if oss_item.get('license', []) != '':
+ lic_list = [check_input_license_format(lic.strip()) for lic in oss_item.get('license', [])]
first_lic = License.from_identifier(lic_list.pop(0))
while lic_list:
next_lic = License.from_identifier(lic_list.pop(0))
@@ -109,21 +104,21 @@ def write_spdx(output_file_without_ext, output_extension, sheet_list,
doc.add_package(package)
- if scanner == constant.FL_DEPENDENCY:
- spdx_id_packages.append([package.name, package.spdx_id])
- comment = oss_item[8]
- relation_tree[package.name] = {}
- relation_tree[package.name]['id'] = package.spdx_id
- relation_tree[package.name]['dep'] = []
+ if scanner_name == FOSSLIGHT_DEPENDENCY:
+ purl = oss_item.get('package url', '')
+ spdx_id_packages.append([purl, package.spdx_id])
+ comment = oss_item.get('comment', '')
+ relation_tree[purl] = {}
+ relation_tree[purl]['id'] = package.spdx_id
+ relation_tree[purl]['dep'] = []
if 'root package' in comment.split(','):
root_package = True
relationship = Relationship(f"{doc.spdx_id} DESCRIBES {package.spdx_id}")
doc.add_relationship(relationship)
- if len(oss_item) > 9:
- deps = oss_item[9]
- relation_tree[package.name]['dep'].extend([di.strip().split('(')[0] for di in deps.split(',')])
- if scanner == constant.FL_DEPENDENCY and len(relation_tree) > 0:
+ deps = oss_item.get('depends on', '')
+ relation_tree[purl]['dep'].extend([di.strip().split('(')[0] for di in deps])
+ if scanner_name == FOSSLIGHT_DEPENDENCY and len(relation_tree) > 0:
for pkg in relation_tree:
if len(relation_tree[pkg]['dep']) > 0:
pkg_spdx_id = relation_tree[pkg]['id']
diff --git a/src/fosslight_util/write_yaml.py b/src/fosslight_util/write_yaml.py
index 944afed..6a7e822 100644
--- a/src/fosslight_util/write_yaml.py
+++ b/src/fosslight_util/write_yaml.py
@@ -2,62 +2,49 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2022 LG Electronics Inc.
# SPDX-License-Identifier: Apache-2.0
-
-
import yaml
import logging
import os
-import copy
+import json
from pathlib import Path
-import fosslight_util.constant as constant
-from fosslight_util.oss_item import OssItem
-from fosslight_util.write_excel import _EMPTY_ITEM_MSG
+from fosslight_util.constant import LOGGER_NAME, SHEET_NAME_FOR_SCANNER
from typing import Tuple
-_logger = logging.getLogger(constant.LOGGER_NAME)
+_logger = logging.getLogger(LOGGER_NAME)
-def write_yaml(output_file, sheet_list_origin, separate_yaml=False) -> Tuple[bool, str, str]:
+def write_yaml(output_file, scan_item, separate_yaml=False) -> Tuple[bool, str, str]:
success = True
error_msg = ""
output = ""
try:
- sheet_list = copy.deepcopy(sheet_list_origin)
- if sheet_list:
- output_files = []
- output_dir = os.path.dirname(output_file)
-
- Path(output_dir).mkdir(parents=True, exist_ok=True)
- if separate_yaml:
- filename = os.path.splitext(os.path.basename(output_file))[0]
- separate_output_file = os.path.join(output_dir, filename)
-
- merge_sheet = []
- for sheet_name, sheet_contents in sheet_list.items():
- if sheet_name not in constant.supported_sheet_and_scanner.keys():
- continue
- scanner_name = constant.supported_sheet_and_scanner[sheet_name]
- sheet_contents_with_scanner = []
- for i in sheet_contents:
- i.insert(0, scanner_name)
- sheet_contents_with_scanner.append(i)
- if not separate_yaml:
- merge_sheet.extend(sheet_contents_with_scanner)
- else:
- output_file = f'{separate_output_file}_{sheet_name}.yaml'
- convert_sheet_to_yaml(sheet_contents_with_scanner, output_file)
- output_files.append(output_file)
+ output_files = []
+ output_dir = os.path.dirname(output_file)
+
+ Path(output_dir).mkdir(parents=True, exist_ok=True)
+ if separate_yaml:
+ filename = os.path.splitext(os.path.basename(output_file))[0]
+ separate_output_file = os.path.join(output_dir, filename)
+
+ merge_sheet = []
+ for scanner_name, _ in scan_item.file_items.items():
+ sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]
+ json_contents = scan_item.get_print_json(scanner_name)
if not separate_yaml:
- convert_sheet_to_yaml(merge_sheet, output_file)
+ merge_sheet.extend(json_contents)
+ else:
+ output_file = f'{separate_output_file}_{sheet_name}.yaml'
+ remove_duplicates_and_dump_yaml(json_contents, output_file)
output_files.append(output_file)
- if output_files:
- output = ", ".join(output_files)
- else:
- success = False
- error_msg = _EMPTY_ITEM_MSG
+ if not separate_yaml:
+ remove_duplicates_and_dump_yaml(merge_sheet, output_file)
+ output_files.append(output_file)
+
+ if output_files:
+ output = ", ".join(output_files)
except Exception as ex:
error_msg = str(ex)
success = False
@@ -68,37 +55,37 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False) -> Tuple[boo
return success, error_msg, output
-def convert_sheet_to_yaml(sheet_contents_with_scanner, output_file):
- sheet_contents_with_scanner = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents_with_scanner))]
+def remove_duplicates_and_dump_yaml(json_contents, output_file):
+ unique_json_strings = {json.dumps(e, sort_keys=True) for e in json_contents}
+ unique_json_contents = [json.loads(e) for e in unique_json_strings]
yaml_dict = {}
- for sheet_item in sheet_contents_with_scanner:
- item = OssItem('')
- item.set_sheet_item(sheet_item[1:], sheet_item[0])
- create_yaml_with_ossitem(item, yaml_dict)
+ for uitem in unique_json_contents:
+ create_yaml_with_ossitem(uitem, yaml_dict)
with open(output_file, 'w') as f:
yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
def create_yaml_with_ossitem(item, yaml_dict):
- item_json = item.get_print_json()
- item_name = item_json.pop("name")
+ item_name = item.pop("name")
if item_name not in yaml_dict.keys():
yaml_dict[item_name] = []
merged = False
for oss_info in yaml_dict[item_name]:
- if oss_info.get('version', '') == item.version and \
- oss_info.get('license', []) == item.license and \
- oss_info.get('copyright text', '') == item.copyright and \
- oss_info.get('homepage', '') == item.homepage and \
- oss_info.get('download location', '') == item.download_location and \
- oss_info.get('exclude', False) == item.exclude:
- oss_info.get('source path', []).extend(item.source_name_or_path)
+ if oss_info.get('version', '') == item.get('version', '') and \
+ oss_info.get('license', []) == item.get('license', []) and \
+ oss_info.get('copyright text', '') == item.get('copyright text', '') and \
+ oss_info.get('homepage', '') == item.get('homepage', '') and \
+ oss_info.get('download location', '') == item.get('download location', '') and \
+ oss_info.get('exclude', False) == item.get('exclude', False):
+ if isinstance(oss_info.get('source path', []), str):
+ oss_info['source path'] = [oss_info.get('source path', '')]
+ oss_info.get('source path', []).append(item.get('source path', ''))
oss_info.pop('comment', None)
merged = True
break
if not merged:
- yaml_dict[item_name].append(item_json)
+ yaml_dict[item_name].append(item)
diff --git a/tests/test_excel_and_csv.py b/tests/test_excel_and_csv.py
index f6f1295..2b14449 100755
--- a/tests/test_excel_and_csv.py
+++ b/tests/test_excel_and_csv.py
@@ -2,68 +2,37 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2021 LG Electronics Inc.
# SPDX-License-Identifier: Apache-2.0
-from fosslight_util.write_excel import write_excel_and_csv
from fosslight_util.output_format import write_output_file
from fosslight_util.set_log import init_log
from copy import deepcopy
+from fosslight_util.oss_item import ScannerItem, FileItem, OssItem
+from fosslight_util.constant import FOSSLIGHT_SOURCE
def main():
- logger, _result_log = init_log("test_result/excel_and_csv/log_write_excel_and_csv.txt")
+ logger, _result_log = init_log("test_result/excel_and_csv/log_write_excel_csv.txt")
- sheet_contents = {}
- src_sheet_items = [['run_scancode.py', 'fosslight_source',
- '3.0.6', 'Apache-2.0', 'https://github.com/LGE-OSS/fosslight_source',
- 'https://github.com/LGE-OSS/fosslight_source',
- 'Copyright (c) 2021 LG Electronics, Inc.',
- 'Exclude', 'Comment message'],
- ['', 'Enact',
- '', 'Apache-2.0', 'https://github.com/enactjs/enact',
- 'https://enactjs.com', 'Copyright (c) 2012-2021 LG Electronics',
- '', ''],
- ['dependency_unified.py', 'fosslight_dependency',
- '3.0.6', 'Apache-2.0', 'https://github.com/LGE-OSS/fosslight_dependency',
- 'https://github.com/LGE-OSS/fosslight_dependency',
- 'Copyright (c) 2020 LG Electronics, Inc.',
- '', '']]
+ scan_item = ScannerItem(FOSSLIGHT_SOURCE)
+ scan_item.set_cover_pathinfo('tests/test_excel_and_csv', '')
+ scan_item.set_cover_comment('This is a test comment')
- bin_sheet_items = [['dependency_unified.py', 'fosslight_dependency',
- '3.0.6', 'Apache-2.0', 'https://github.com/LGE-OSS/fosslight_dependency',
- 'https://github.com/LGE-OSS/fosslight_dependency', 'Copyright (c) 2020 LG Electronics, Inc.',
- '', 'Awesome Open Source'],
- ['askalono.exe', 'askalono',
- '0.4.3', 'Apache-2.0', 'https://github.com/jpeddicord/askalono',
- '', 'Copyright (c) 2018 Amazon.com, Inc. or its affiliates.',
- '', '']]
- sheet_items = [['ID', 'Binary Path', 'OSS Name', 'OSS Version',
- 'License', 'Download Location', 'Homepage',
- 'Copyright Text', 'Exclude', 'Comment'],
- ['dependency_unified.py', 'fosslight_dependency',
- '3.0.6', 'Apache-2.0', 'https://github.com/LGE-OSS/fosslight_dependency',
- 'https://github.com/LGE-OSS/fosslight_dependency', 'Copyright (c) 2020 LG Electronics, Inc.',
- 'Exclude', 'Awesome Open Source'],
- ['askalono.exe', 'askalono',
- '0.4.3', 'Apache-2.0', 'https://github.com/jpeddicord/askalono',
- '', 'Copyright (c) 2018 Amazon.com, Inc. or its affiliates.',
- '', '']]
+ file_item = FileItem('test_result/excel_and_csv')
+ oss_item = OssItem("test_name", "1.0.0", "Apache-2.0", "https://abc.com")
+ oss_item.comment = "test_name comment"
+ file_item.oss_items.append(oss_item)
+ oss_item2 = OssItem("test_name", "2.0.0", "MIT", "https://abc2.com")
+ file_item.oss_items.append(oss_item2)
+ file_item.comment = "all test comment"
- sheet_contents['SRC'] = src_sheet_items
- sheet_contents['BIN_TEST'] = bin_sheet_items
- sheet_contents['SRC_NULL'] = []
- sheet_contents['CUSTOM_HEADER_SHEET'] = sheet_items
-
- logger.warning("TESTING - Writing an excel and csv")
- success, msg, result_file = write_excel_and_csv(
- 'test_result/excel_and_csv/FOSSLight-Report', deepcopy(sheet_contents))
- logger.warning(f"|-- Result:{success}, file:{result_file}, error_msg:{msg}")
+ scan_item.append_file_items([file_item])
logger.warning("TESTING - Writing an excel")
- success, msg, result_file = write_output_file('test_result/excel_and_csv/excel/Test_Excel', '.xlsx', deepcopy(sheet_contents))
+ success, msg, result_file = write_output_file('test_result/excel_and_csv/excel/Test_Excel', '.xlsx', deepcopy(scan_item))
logger.warning(f"|-- Result:{success}, file:{result_file}, error_msg:{msg}")
logger.warning("TESTING - Writing an csv")
success, msg, result_file = write_output_file(
- 'test_result/excel_and_csv/csv/Test_Csv', '.csv', deepcopy(sheet_contents))
+ 'test_result/excel_and_csv/csv/Test_Csv', '.csv', deepcopy(scan_item))
logger.warning(f"|-- Result:{success}, file:{result_file}, error_msg:{msg}")
diff --git a/tests/test_opossum.py b/tests/test_opossum.py
index 2c93204..9f0e451 100644
--- a/tests/test_opossum.py
+++ b/tests/test_opossum.py
@@ -4,102 +4,30 @@
# SPDX-License-Identifier: Apache-2.0
from fosslight_util.write_opossum import write_opossum
from fosslight_util.set_log import init_log
+from fosslight_util.oss_item import ScannerItem, FileItem, OssItem
+from fosslight_util.constant import FOSSLIGHT_SOURCE
def main():
logger, _result_log = init_log("test_result/excel/log_write_opossum.txt")
logger.warning("TESTING - Writing an opossum")
- sheet_list = {'SRC_FL_Source': [
- ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''],
- ['lib/babel-polyfill.js', '', '', 'bsd-3-clause', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', '', ''],
- ['lib/babel-polyfill.js', '', '', 'facebook-patent-rights-2', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', '', ''],
- ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''],
- ['bower.json', '', '', 'mit', '', '', '', '', ''],
- ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''],
- ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''],
- ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''],
- ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.min.js', '', '', 'mit',
- '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \
- (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.umd.js', '', '', 'mit', '', '',
- 'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \
- and other contributors ,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\
- Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''],
- ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\
- Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''],
- ['src/event.js', '', '', 'mit', '', '', '', '', '']],
- 'BIN_FL_Binary': [
- ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''],
- ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']],
- 'SRC_FL_Dependency': [
- ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2',
- 'https://python-future.org', '', '', ''],
- ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0',
- 'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''],
- ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5',
- 'https://pandas.pydata.org', '', '', '']]}
+ scan_item = ScannerItem(FOSSLIGHT_SOURCE)
+ scan_item.set_cover_pathinfo('tests/test_excel_and_csv', '')
+ scan_item.set_cover_comment('This is a test comment')
- sheet_list2 = {'SRC_FL_Dependency': [
- ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2',
- 'https://python-future.org', '', '', ''],
- ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0',
- 'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''],
- ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5',
- 'https://pandas.pydata.org', '', '', '']],
- 'SRC_FL_Source': [
- ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''],
- ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''],
- ['bower.json', '', '', 'mit', '', '', '', '', ''],
- ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''],
- ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''],
- ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''],
- ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.min.js', '', '', 'mit',
- '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \
- (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.umd.js', '', '', 'mit', '', '',
- 'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \
- and other contributors ,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\
- Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''],
- ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\
- Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''],
- ['src/event.js', '', '', 'mit', '', '', '', '', '']],
- 'BIN_FL_Binary': [
- ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''],
- ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']]}
+ file_item = FileItem('test_result/excel_and_csv')
+ oss_item = OssItem("test_name", "1.0.0", "Apache-2.0", "https://abc.com")
+ oss_item.comment = "test_name comment"
+ file_item.oss_items.append(oss_item)
+ oss_item2 = OssItem("test_name", "2.0.0", "MIT", "https://abc2.com")
+ file_item.oss_items.append(oss_item2)
+ file_item.comment = "all test comment"
- success, msg = write_opossum(
- 'test_result/opossum/FL-TEST_opossum.json', sheet_list)
- logger.warning("Result:" + str(success) + ", error_msg:" + msg)
+ scan_item.append_file_items([file_item])
success, msg = write_opossum(
- 'test_result/opossum/FL-TEST2_opossum.json', sheet_list2)
+ 'test_result/opossum/FL-TEST_opossum.json', scan_item)
logger.warning("Result:" + str(success) + ", error_msg:" + msg)
diff --git a/tests/test_output_format.py b/tests/test_output_format.py
index e720724..2b84fb4 100644
--- a/tests/test_output_format.py
+++ b/tests/test_output_format.py
@@ -5,54 +5,30 @@
import sys
from fosslight_util.output_format import write_output_file
from fosslight_util.set_log import init_log
+from fosslight_util.oss_item import ScannerItem, FileItem, OssItem
+from fosslight_util.constant import FOSSLIGHT_SOURCE
def main():
logger, _result_log = init_log("test_result/output_format/log_write_output.txt")
- sheet_list = {'SRC_FL_Source': [
- ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''],
- ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''],
- ['bower.json', '', '', 'mit', '', '', '', '', ''],
- ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''],
- ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''],
- ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''],
- ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.min.js', '', '', 'mit',
- '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \
- (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.umd.js', '', '', 'mit', '', '',
- 'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \
- and other contributors ,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\
- Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''],
- ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\
- Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''],
- ['src/event.js', '', '', 'mit', '', '', '', '', '']],
- 'BIN_FL_Binary': [
- ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''],
- ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']],
- 'SRC_FL_Dependency': [
- ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2',
- 'https://python-future.org', '', '', ''],
- ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0',
- 'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''],
- ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5',
- 'https://pandas.pydata.org', '', '', '']]}
+ scan_item = ScannerItem(FOSSLIGHT_SOURCE)
+ scan_item.set_cover_pathinfo('tests/test_excel_and_csv', '')
+ scan_item.set_cover_comment('This is a test comment')
+
+ file_item = FileItem('test_result/excel_and_csv')
+ oss_item = OssItem("test_name", "1.0.0", "Apache-2.0", "https://abc.com")
+ oss_item.comment = "test_name comment"
+ file_item.oss_items.append(oss_item)
+ oss_item2 = OssItem("test_name", "2.0.0", "MIT", "https://abc2.com")
+ file_item.oss_items.append(oss_item2)
+ file_item.comment = "all test comment"
+
+ scan_item.append_file_items([file_item])
logger.warning("TESTING - Writing an excel output")
success, msg, result_file = write_output_file(
- 'test_result/output_format/FL-TEST_Excel', '.xlsx', sheet_list)
+ 'test_result/output_format/FL-TEST_Excel', '.xlsx', scan_item)
logger.warning(f"Result: {success} error_msg:: {msg}, result_file: {result_file}")
if not success:
@@ -60,7 +36,7 @@ def main():
logger.warning("TESTING - Writing an opossum output")
success, msg, result_file = write_output_file(
- 'test_result/output_format/FL-TEST_opossum', '.json', sheet_list)
+ 'test_result/output_format/FL-TEST_opossum', '.json', scan_item)
logger.warning(f"Result: {success} error_msg:: {msg}, result_file: {result_file}")
if not success:
diff --git a/tests/test_yaml.py b/tests/test_yaml.py
index b173055..f1e59fc 100644
--- a/tests/test_yaml.py
+++ b/tests/test_yaml.py
@@ -4,108 +4,32 @@
# SPDX-License-Identifier: Apache-2.0
from fosslight_util.write_yaml import write_yaml
from fosslight_util.set_log import init_log
+from fosslight_util.oss_item import ScannerItem, FileItem, OssItem
+from fosslight_util.constant import FOSSLIGHT_SOURCE
def main():
logger, _result_log = init_log("test_result/yaml/log_write_yaml.txt")
logger.warning("TESTING - Writing a yaml")
- sheet_list = {'SRC_FL_Source': [
- ['test/lib/not_license.js', '', '', '', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''],
- ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''],
- ['test/lib/babel-polyfill2.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', 'test_commend'],
- ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''],
- ['lib/babel-polyfill.js', '', '', 'bsd-3-clause', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', '', ''],
- ['lib/babel-polyfill.js', '', '', 'facebook-patent-rights-2', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', '', ''],
- ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''],
- ['bower.json', '', '', 'mit', '', '', '', '', ''],
- ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''],
- ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''],
- ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''],
- ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.min.js', '', '', 'mit',
- '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \
- (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.umd.js', '', '', 'mit', '', '',
- 'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \
- and other contributors ,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\
- Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''],
- ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\
- Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''],
- ['src/event.js', '', '', 'mit', '', '', '', '', '']],
- 'BIN_FL_Binary': [
- ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''],
- ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']],
- 'SRC_FL_Dependency': [
- ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2',
- 'https://python-future.org', '', '', ''],
- ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0',
- 'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''],
- ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5',
- 'https://pandas.pydata.org', '', '', '']]}
+ scan_item = ScannerItem(FOSSLIGHT_SOURCE)
+ scan_item.set_cover_pathinfo('tests/test_excel_and_csv', '')
+ scan_item.set_cover_comment('This is a test comment')
- sheet_list2 = {'SRC_FL_Dependency': [
- ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2',
- 'https://python-future.org', '', '', ''],
- ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0',
- 'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''],
- ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5',
- 'https://pandas.pydata.org', '', '', '']],
- 'SRC_FL_Source': [
- ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '',
- 'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''],
- ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''],
- ['bower.json', '', '', 'mit', '', '', '', '', ''],
- ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''],
- ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''],
- ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''],
- ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\
- The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
- (http://engelschall.com)', '', ''],
- ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.min.js', '', '', 'mit',
- '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \
- (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['dist/cytoscape.umd.js', '', '', 'mit', '', '',
- 'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \
- and other contributors ,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\
- Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''],
- ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\
- Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \
- Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
- ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''],
- ['src/event.js', '', '', 'mit', '', '', '', '', '']],
- 'BIN_FL_Binary': [
- ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''],
- ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']]}
+ file_item = FileItem('test_result/excel_and_csv')
+ oss_item = OssItem("test_name", "1.0.0", "Apache-2.0", "https://abc.com")
+ oss_item.comment = "test_name comment"
+ file_item.oss_items.append(oss_item)
+ oss_item2 = OssItem("test_name", "2.0.0", "MIT", "https://abc2.com")
+ file_item.oss_items.append(oss_item2)
+ file_item.comment = "all test comment"
+ oss_item3 = OssItem("test_name2", "1.0.0", "GPL-2.0,BSD-3-Clause", "https://abc3.com")
+ file_item.oss_items.append(oss_item3)
- success, msg, output = write_yaml(
- 'test_result/yaml/FL-TEST_yaml.yaml', sheet_list)
- logger.warning(f"Result: {str(success)}, error_msg: {msg}, Output_files: {output}")
+ scan_item.append_file_items([file_item])
success, msg, output = write_yaml(
- 'test_result/yaml/FL-TEST2_yaml.yaml', sheet_list2)
+ 'test_result/yaml/FL-TEST_yaml.yaml', scan_item)
logger.warning(f"Result: {str(success)}, error_msg: {msg}, Output_files: {output}")
diff --git a/tox.ini b/tox.ini
index 3176c9e..b525b55 100644
--- a/tox.ini
+++ b/tox.ini
@@ -68,9 +68,7 @@ commands =
# Test - writing excel
python tests/test_excel_and_csv.py
ls test_result/excel_and_csv/
- cat test_result/excel_and_csv/FOSSLight-Report_SRC.csv
- cat test_result/excel_and_csv/FOSSLight-Report_BIN_TEST.csv
- cat test_result/excel_and_csv/FOSSLight-Report_CUSTOM_HEADER_SHEET.csv
+ cat test_result/excel_and_csv/excel/Test_Excel.xlsx
cat test_result/excel_and_csv/csv/Test_Csv.csv
# Test - writing opossum
python tests/test_opossum.py
@@ -78,7 +76,6 @@ commands =
# Test - writing yaml
python tests/test_yaml.py
cat test_result/yaml/FL-TEST_yaml.yaml
- cat test_result/yaml/FL-TEST2_yaml.yaml
# Test - timer
python tests/test_timer.py
# Test - downloading source