Merge branch 'main' of https://github.com/cjho0316/fosslight_scanner

.bumpversion.cfg

@@ -2,7 +2,7 @@
 commit = True
 tag = False
 message = Bump version: {current_version} → {new_version}
-current_version = 1.7.30
+current_version = 2.0.1
 
 [bumpversion:file:setup.py]
 search = '{current_version}'

.github/workflows/docker-build-push.yml

@@ -0,0 +1,34 @@
+name: Docker Build and Push
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build and push
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        push: true
+        platforms: linux/amd64,linux/arm64
+        tags: |
+          fosslight/fosslight_scanner:latest
+          fosslight/fosslight_scanner:${{ github.event.release.tag_name }}

CHANGELOG.md

@@ -1,5 +1,39 @@
 # Changelog
 
+## v2.0.1 (09/09/2024)
+## Changes
+## 🔧 Maintenance
+
+- Revert "Remove prechecker" @dd-jy (#120)
+
+---
+
+## v2.0.0 (06/09/2024)
+## Changes
+## 🔧 Maintenance
+
+- Remove prechecker @dd-jy (#119)
+- Refactoring OSS item @dd-jy (#118)
+
+---
+
+## v1.7.31 (06/09/2024)
+## Changes
+## 🚀 Features
+
+- Setting.json with source_scanner selection @soonhong99 (#109)
+
+## 🐛 Hotfixes
+
+- Fix a bug related to path_to_exclude @soimkim (#116)
+
+## 🔧 Maintenance
+
+- Limit installation fosslight package @dd-jy (#117)
+- Add simple_mode parameter to CoverItem constructor @YongGoose (#108)
+
+---
+
 ## v1.7.30 (22/07/2024)
 ## Changes
 ## 🚀 Features
@@ -263,31 +297,3 @@
 
 - Change the report file name @dd-jy (#48)
 - Modify help msg if invalid input @bjk7119 (#47)
-
----
-
-## v1.7.3 (01/09/2022)
-## Changes
-## 🚀 Features
-
-- Support 'xlsx' report for Compare mode @dd-jy (#46)
-
-## 🔧 Maintenance
-
-- Change the required version of Python to 3.7 @soimkim (#45)
-
----
-
-## v1.7.2 (16/08/2022)
-## Changes
-## 🚀 Features
-
-- Support yaml format of FOSSLight Report @dd-jy (#42)
-
----
-
-## v1.7.1 (22/07/2022)
-## Changes
-## 🐛 Hotfixes
-
-- Change FL Reuse to FL Prechecker @bjk7119 (#43)

setup.py

@@ -15,7 +15,7 @@
 if __name__ == "__main__":
     setup(
         name='fosslight_scanner',
-        version='1.7.30',
+        version='2.0.1',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Scanner',

src/fosslight_scanner/_parse_setting.py

@@ -11,7 +11,7 @@ def parse_setting_json(data):
     dep_argument = data.get('dep_argument', '')
     output = data.get('output', '')
     format = data.get('format', '')
-    link = data.get('link', "")
+    link = data.get('link', '')
     db_url = data.get('db_url', '')
     timer = data.get('timer', False)
     raw = data.get('raw', False)
@@ -20,34 +20,49 @@ def parse_setting_json(data):
     correct_fpath = data.get('correct_fpath', '')
     ui = data.get('ui', False)
     exclude_path = data.get('exclude', [])
-
+    selected_source_scanner = data.get('selected_source_scanner', '')
+    source_write_json_file = data.get('source_write_json_file', False)
+    source_print_matched_text = data.get('source_print_matched_text', False)
+    source_time_out = data.get('source_time_out', 120)
+    binary_simple = data.get('binary_simple', False)
     str_lists = [mode, path, exclude_path]
-    strings = [dep_argument, output, format, db_url, correct_fpath, link]
-    booleans = [timer, raw, no_correction, ui]
+    strings = [
+        dep_argument, output, format, db_url,
+        correct_fpath, link, selected_source_scanner
+    ]
+    booleans = [timer, raw, no_correction, ui, source_write_json_file, source_print_matched_text, binary_simple]
+
     is_incorrect = False
 
     # check if json file is incorrect format
     for i, target in enumerate(str_lists):
-        if not (isinstance(target, list) and all(isinstance(item, str) for item in target)):
+        if not (isinstance(target, list) and
+                all(isinstance(item, str) for item in target)):
             is_incorrect = True
             str_lists[i] = []
 
     for i, target in enumerate(strings):
         if not isinstance(target, str):
             is_incorrect = True
-            str_lists[i] = ''
+            strings[i] = ''
 
     for i, target in enumerate(booleans):
         if not isinstance(target, bool):
             is_incorrect = True
-            str_lists[i] = False
+            booleans[i] = False
 
     if not isinstance(core, int):
         is_incorrect = True
         core = -1
 
+    if not isinstance(source_time_out, int):
+        is_incorrect = True
+        source_time_out = 120
+
     if is_incorrect:
         print('Ignoring some values with incorrect format in the setting file.')
 
     return mode, path, dep_argument, output, format, link, db_url, timer, \
-        raw, core, no_correction, correct_fpath, ui, exclude_path
+        raw, core, no_correction, correct_fpath, ui, exclude_path, \
+        selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
+        binary_simple

src/fosslight_scanner/_run_compare.py

@@ -14,7 +14,8 @@
 from bs4 import BeautifulSoup
 import fosslight_util.constant as constant
 from fosslight_util.compare_yaml import compare_yaml
-from fosslight_util.convert_excel_to_yaml import convert_excel_to_yaml
+from fosslight_util.read_excel import read_oss_report
+from fosslight_util.parsing_yaml import parsing_yml
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 ADD = "add"
@@ -255,10 +256,18 @@ def run_compare(before_f, after_f, output_path, output_file, file_ext, _start_ti
 
     result_file = get_comparison_result_filename(output_path, output_file, file_ext, _start_time)
 
-    if before_ext == XLSX_EXT:
-        convert_excel_to_yaml(before_f, before_yaml)
-        convert_excel_to_yaml(after_f, after_yaml)
-    compared_result = compare_yaml(before_yaml, after_yaml)
+    before_basepath = os.path.dirname(before_f)
+    after_basepath = os.path.dirname(after_f)
+    if XLSX_EXT == before_ext:
+        before_fileitems = read_oss_report(before_f, "", before_basepath)
+    elif YAML_EXT == before_ext:
+        before_fileitems, _, _ = parsing_yml(before_yaml, before_basepath)
+    if XLSX_EXT == after_ext:
+        after_fileitems = read_oss_report(after_f, after_basepath)
+    elif YAML_EXT == after_ext:
+        after_fileitems, _, _ = parsing_yml(after_yaml, after_basepath)
+
+    compared_result = compare_yaml(before_fileitems, after_fileitems)
     if compared_result != '':
         count_compared_result(compared_result)
         ret, result_file = write_compared_result(result_file, compared_result, file_ext, before_yaml, after_yaml)

src/fosslight_scanner/cli.py

@@ -4,68 +4,107 @@
 # SPDX-License-Identifier: Apache-2.0
 import sys
 import json
+import os
+import os.path
 from argparse import ArgumentParser
+
 from ._help import print_help_msg
 from .fosslight_scanner import run_main, PKG_NAME
 from ._parse_setting import parse_setting_json
 from fosslight_util.help import print_package_version
-import os.path
 
 
 def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
              raw, core, no_correction, correct_fpath, ui, setting, exclude_path):
+
+    selected_source_scanner = "all"
+    source_write_json_file = False
+    source_print_matched_text = False
+    source_time_out = 120
+    binary_simple = False
+
     if setting and os.path.isfile(setting):
         try:
             with open(setting, 'r', encoding='utf-8') as file:
                 data = json.load(file)
             s_mode, s_path, s_dep_argument, s_output, s_format, s_link, s_db_url, s_timer, s_raw, s_core, \
-                s_no_correction, s_correct_fpath, s_ui, s_exclude_path = parse_setting_json(data)
+                s_no_correction, s_correct_fpath, s_ui, s_exclude_path, \
+                s_selected_source_scanner, s_source_write_json_file, s_source_print_matched_text, \
+                s_source_time_out, s_binary_simple = parse_setting_json(data)
 
             # direct cli arguments have higher priority than setting file
-            mode = mode if mode else s_mode
-            path = path if path else s_path
-            dep_argument = dep_argument if dep_argument else s_dep_argument
-            output = output if output else s_output
-            format = format if format else s_format
-            link = link if link else s_link
-            db_url = db_url if db_url else s_db_url
-            timer = timer if timer else s_timer
-            raw = raw if raw else s_raw
-            core = core if core else s_core
-            no_correction = no_correction if no_correction else s_no_correction
-            correct_fpath = correct_fpath if correct_fpath else s_correct_fpath
-            ui = ui if ui else s_ui
-            exclude_path = exclude_path if exclude_path else s_exclude_path
+            mode = mode or s_mode
+            path = path or s_path
+            dep_argument = dep_argument or s_dep_argument
+            output = output or s_output
+            format = format or s_format
+            link = link or s_link
+            db_url = db_url or s_db_url
+            timer = timer or s_timer
+            raw = raw or s_raw
+            core = core if core != -1 else s_core
+            no_correction = no_correction or s_no_correction
+            correct_fpath = correct_fpath or s_correct_fpath
+            ui = ui or s_ui
+            exclude_path = exclude_path or s_exclude_path
+
+            # These options are only set from the setting file, not from CLI arguments
+            selected_source_scanner = s_selected_source_scanner or selected_source_scanner
+            source_write_json_file = s_source_write_json_file
+            source_print_matched_text = s_source_print_matched_text
+            source_time_out = s_source_time_out if s_source_time_out != 120 else source_time_out
+            binary_simple = s_binary_simple
 
         except Exception as e:
             print(f"Cannot open setting file: {e}")
     return mode, path, dep_argument, output, format, link, db_url, timer, \
-        raw, core, no_correction, correct_fpath, ui, exclude_path
+        raw, core, no_correction, correct_fpath, ui, exclude_path, \
+        selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
+        binary_simple
 
 
 def main():
-    parser = ArgumentParser(description='FOSSLight Scanner', prog='fosslight_scanner', add_help=False)
-    parser.add_argument('mode', nargs='*', help='source| dependency| binary| all| compare', default="")
-    parser.add_argument('--path', '-p', help='Path to analyze (In compare mode, two FOSSLight reports',
+    parser = ArgumentParser(description='FOSSLight Scanner',
+                            prog='fosslight_scanner', add_help=False)
+    parser.add_argument('mode', nargs='*',
+                        help='source| dependency| binary| all| compare',
+                        default="")
+    parser.add_argument('--path', '-p',
+                        help='Path to analyze (In compare mode, two FOSSLight reports',
                         dest='path', nargs='+', default="")
-    parser.add_argument('--wget', '-w', help='Link to be analyzed', type=str, dest='link', default="")
-    parser.add_argument('--format', '-f', help='Scanner output file format (excel,yaml), Compare mode (excel,html,yaml,json)',
+    parser.add_argument('--wget', '-w', help='Link to be analyzed',
+                        type=str, dest='link', default="")
+    parser.add_argument('--format', '-f',
+                        help='Scanner output file format (excel,yaml), Compare mode (excel,html,yaml,json)',
                         type=str, dest='format', default="")
-    parser.add_argument('--output', '-o', help='Output directory or file', type=str, dest='output', default="")
-    parser.add_argument('--dependency', '-d', help='Dependency arguments', type=str, dest='dep_argument', default="")
-    parser.add_argument('--url', '-u', help="DB Url", type=str, dest='db_url', default="")
-    parser.add_argument('--core', '-c', help='Number of processes to analyze source', type=int, dest='core', default=-1)
-    parser.add_argument('--raw', '-r', help='Keep raw data', action='store_true', dest='raw', default=False)
-    parser.add_argument('--timer', '-t', help='Hide the progress bar', action='store_true', dest='timer', default=False)
-    parser.add_argument('--version', '-v', help='Print version', action='store_true', dest='version', default=False)
-    parser.add_argument('--help', '-h', help='Print help message', action='store_true', dest='help')
-    parser.add_argument('--exclude', '-e', help='Path to exclude from analysis', dest='exclude_path', nargs='*', default=[])
-    parser.add_argument('--setting', '-s', help='Scanner json setting file', type=str, dest='setting', default="")
-    parser.add_argument('--no_correction', help='No correction with sbom-info.yaml',
+    parser.add_argument('--output', '-o', help='Output directory or file',
+                        type=str, dest='output', default="")
+    parser.add_argument('--dependency', '-d', help='Dependency arguments',
+                        type=str, dest='dep_argument', default="")
+    parser.add_argument('--url', '-u', help="DB Url",
+                        type=str, dest='db_url', default="")
+    parser.add_argument('--core', '-c',
+                        help='Number of processes to analyze source',
+                        type=int, dest='core', default=-1)
+    parser.add_argument('--raw', '-r', help='Keep raw data',
+                        action='store_true', dest='raw', default=False)
+    parser.add_argument('--timer', '-t', help='Hide the progress bar',
+                        action='store_true', dest='timer', default=False)
+    parser.add_argument('--version', '-v', help='Print version',
+                        action='store_true', dest='version', default=False)
+    parser.add_argument('--help', '-h', help='Print help message',
+                        action='store_true', dest='help')
+    parser.add_argument('--exclude', '-e', help='Path to exclude from analysis',
+                        dest='exclude_path', nargs='*', default=[])
+    parser.add_argument('--setting', '-s', help='Scanner json setting file',
+                        type=str, dest='setting', default="")
+    parser.add_argument('--no_correction',
+                        help='No correction with sbom-info.yaml',
                         action='store_true', required=False, default=False)
     parser.add_argument('--correct_fpath', help='Path to the sbom-info.yaml',
                         type=str, required=False, default='')
-    parser.add_argument('--ui', help='Generate UI mode result file', action='store_true', required=False, default=False)
+    parser.add_argument('--ui', help='Generate UI mode result file',
+                        action='store_true', required=False, default=False)
 
     try:
         args = parser.parse_args()
@@ -78,12 +117,17 @@ def main():
         print_package_version(PKG_NAME, "FOSSLight Scanner Version:")
     else:
         mode, path, dep_argument, output, format, link, db_url, timer, raw, core, no_correction, correct_fpath, \
-          ui, exclude_path = set_args(args.mode, args.path, args.dep_argument, args.output, args.format,
-                                      args.link, args.db_url, args.timer, args.raw, args.core, args.no_correction,
-                                      args.correct_fpath, args.ui, args.setting, args.exclude_path)
+            ui, exclude_path, selected_source_scanner, source_write_json_file, source_print_matched_text, \
+            source_time_out, binary_simple, = set_args(
+                args.mode, args.path, args.dep_argument, args.output,
+                args.format, args.link, args.db_url, args.timer, args.raw,
+                args.core, args.no_correction, args.correct_fpath, args.ui,
+                args.setting, args.exclude_path)
 
         run_main(mode, path, dep_argument, output, format, link, db_url, timer,
-                 raw, core, not no_correction, correct_fpath, ui, exclude_path)
+                 raw, core, not no_correction, correct_fpath, ui, exclude_path,
+                 selected_source_scanner, source_write_json_file, source_print_matched_text,
+                 source_time_out, binary_simple)
 
 
 if __name__ == "__main__":