Merge pull request #95 from SeongjunJo/exclude_path

Supports for excluding paths from analysis

Author: soimkim

README.md

@@ -80,6 +80,7 @@ Options:
                                  * Compare mode result file: supports excel, json, yaml, html
         -o <output>             Output directory or file
         -c <number>             Number of processes to analyze source
+        -e <path>               Path to exclude from analysis (ex, -e {dir} {file})
         -r                      Keep raw data
         -t                      Hide the progress bar
         -v                      Print FOSSLight Scanner version
@@ -96,12 +97,21 @@ Options:
 $ fosslight all -p /home/source_path -d "-a 'source /test/Projects/venv/bin/activate' -d 'deactivate'"
 ```
 
-### Ex 2. Download Link and analyze
+### Ex 2. Local Source Analysis with Path to Exclude
+```
+$ fosslight all -p /home/source_path -e temp_dir src/temp.py
+```
+
+### Ex 3. Download Link and analyze
 ```
 $ fosslight all -o test_result_wget -w "https://github.com/LGE-OSS/example.git"
 ```
+If you want to analyze private repository, set your github token like below.
+```
+$ fosslight all -w "https://[email protected]/Foo/private_repo
+```
 
-### Ex 3. Compare the BOM of two FOSSLight reports with yaml or excel format and check the oss status (change/add/delete)
+### Ex 4. Compare the BOM of two FOSSLight reports with yaml or excel format and check the oss status (change/add/delete)
 ```
 $ fosslight compare -p FOSSLight_before_proj.yaml FOSSLight_after_proj.yaml -f excel
 ```

requirements.txt

@@ -4,8 +4,8 @@ openpyxl
 progress
 pyyaml
 beautifulsoup4
-fosslight_util>=1.4.44
-fosslight_source>=1.7.7
-fosslight_dependency>=3.15.0
-fosslight_binary>=4.1.29
+fosslight_util>=1.4.45
+fosslight_source>=1.7.8
+fosslight_dependency>=3.15.1
+fosslight_binary>=4.1.30
 fosslight_prechecker>=3.0.27

src/fosslight_scanner/_help.py

@@ -27,6 +27,7 @@
             -w <link>\t\t    Link to be analyzed can be downloaded by wget or git clone
             -f <format>\t\t    FOSSLight Report file format (excel, yaml)
                                      * Compare mode result file: supports excel, json, yaml, html
+            -e <path>\t\t    Path to exclude from analysis (ex, -e {dir} {file})
             -o <output>\t\t    Output directory or file
             -c <number>\t\t    Number of processes to analyze source
             -r\t\t\t    Keep raw data

src/fosslight_scanner/cli.py

@@ -25,6 +25,7 @@ def main():
     parser.add_argument('--timer', '-t', help='Hide the progress bar', action='store_true', dest='timer', default=False)
     parser.add_argument('--version', '-v', help='Print version', action='store_true', dest='version', default=False)
     parser.add_argument('--help', '-h', help='Print help message', action='store_true', dest='help')
+    parser.add_argument('--exclude', '-e', help='Path to exclude from analysis', dest='exclude_path', nargs='*', default=[])
     parser.add_argument('--no_correction', help='No correction with sbom-info.yaml',
                         action='store_true', required=False, default=False)
     parser.add_argument('--correct_fpath', help='Path to the sbom-info.yaml',
@@ -45,7 +46,7 @@ def main():
             args.mode = ['all']
         run_main(args.mode, args.path, args.dep_argument, args.output, args.format,
                  args.link, args.db_url, args.timer, args.raw, args.core,
-                 not args.no_correction, args.correct_fpath, args.ui)
+                 not args.no_correction, args.correct_fpath, args.ui, args.exclude_path)
 
 
 if __name__ == "__main__":

src/fosslight_scanner/common.py

@@ -62,15 +62,15 @@ def run_analysis(path_to_run, params, func, str_run_start, output, exe_path):
     return return_value
 
 
-def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args):
+def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwargs):
     # return_idx == -1 : Raw return value itself
     logger.info(f"## Start to run {str_run_start}")
     success = True
     result = []
     try:
         if path_to_run != "":
             logger.info(f"|--- Path to analyze : {path_to_run}")
-            result = func(*args)
+            result = func(*args, **kwargs)
         else:
             logger.info("Analyzing path is missing...")
     except SystemExit:

src/fosslight_scanner/fosslight_scanner.py

@@ -48,7 +48,7 @@
 SCANNER_MODE = ["all", "compare", "reuse", "prechecker", "binary", "bin", "src", "source", "dependency", "dep"]
 
 
-def run_dependency(path_to_analyze, output_file_with_path, params=""):
+def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[]):
     result_list = []
 
     package_manager = ""
@@ -90,7 +90,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params=""):
                                             output_file_with_path,
                                             pip_activate_cmd, pip_deactivate_cmd,
                                             output_custom_dir, app_name,
-                                            github_token)
+                                            github_token, path_to_exclude=path_to_exclude)
         if success:
             result_list = result.get('SRC_FL_Dependency')
     except Exception as ex:
@@ -106,7 +106,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 remove_src_data=True, result_log={}, output_file="",
                 output_extension="", num_cores=-1, db_url="",
                 default_oss_name="", default_oss_version="", url="",
-                correct_mode=True, correct_fpath="", ui_mode=False):
+                correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]):
     final_excel_dir = output_path
     success = True
     temp_output_fiiles = []
@@ -136,8 +136,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 output_prechecker = os.path.join(_output_dir, output_files["PRECHECKER"])
                 success, result = call_analysis_api(src_path, "Prechecker Lint",
                                                     -1, prechecker_lint,
-                                                    abs_path, False,
-                                                    output_prechecker)
+                                                    abs_path, False, output_prechecker,
+                                                    exclude_path=path_to_exclude)
                 success_file, copied_file = copy_file(output_prechecker, output_path)
                 if success_file:
                     temp_output_fiiles.append(copied_file)
@@ -150,12 +150,15 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                                                             -1, source_analysis,
                                                             abs_path,
                                                             src_output,
-                                                            False, num_cores, False)
+                                                            False, num_cores, False,
+                                                            path_to_exclude=path_to_exclude)
                     else:  # Run fosslight_source by using docker image
                         src_output = os.path.join("output", output_files["SRC"])
                         output_rel_path = os.path.relpath(abs_path, os.getcwd())
                         command = shlex.quote(f"docker run -it -v {_output_dir}:/app/output "
                                               f"fosslight -p {output_rel_path} -o {src_output}")
+                        if path_to_exclude:
+                            command += f" -e {' '.join(path_to_exclude)}"
                         command_result = subprocess.run(command, stdout=subprocess.PIPE, text=True)
                         logger.info(f"Source Analysis Result:{command_result.stdout}")
 
@@ -168,7 +171,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                                                abs_path,
                                                os.path.join(_output_dir, output_files["BIN"]),
                                                "", db_url, False,
-                                               correct_mode, correct_fpath)
+                                               correct_mode, correct_fpath,
+                                               path_to_exclude=path_to_exclude)
                 if success:
                     output_binary_txt_raw = f"{output_files['BIN'].split('.')[0]}.txt"
                     success_file, copied_file = copy_file(os.path.join(_output_dir, output_binary_txt_raw),
@@ -177,7 +181,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                         temp_output_fiiles.append(copied_file)
 
             if run_dep:
-                run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]), dep_arguments)
+                run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]), dep_arguments, path_to_exclude)
 
         else:
             return
@@ -191,7 +195,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
         merge_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]]
         cover = CoverItem(tool_name=PKG_NAME,
                           start_time=_start_time,
-                          input_path=abs_path)
+                          input_path=abs_path,
+                          exclude_path=path_to_exclude)
         cover.comment = merge_cover_comment(_output_dir, merge_files)
 
         if output_extension == ".xlsx":
@@ -307,8 +312,9 @@ def init(output_path="", make_outdir=True):
     return os.path.isdir(_output_dir), output_root_dir, result_log
 
 
-def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format, url_to_analyze, db_url,
-             hide_progressbar=False, keep_raw_data=False, num_cores=-1, correct_mode=True, correct_fpath="", ui_mode=False):
+def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format, url_to_analyze,
+             db_url, hide_progressbar=False, keep_raw_data=False, num_cores=-1,
+             correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]):
     global _executed_path, _start_time
 
     output_file = ""
@@ -417,7 +423,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
                                 remove_downloaded_source, {}, output_file,
                                 output_extension, num_cores, db_url,
                                 default_oss_name, default_oss_version, url_to_analyze,
-                                correct_mode, correct_fpath, ui_mode)
+                                correct_mode, correct_fpath, ui_mode, path_to_exclude)
             else:
                 logger.error("No mode has been selected for analysis.")
         try:

tox.ini

@@ -25,9 +25,11 @@ norecursedirs = test_result_*
 [testenv:test_run]
 commands =
     rm -rf test_result_local_path
+    rm -rf test_result_exclude_path
     rm -rf test_result_wget
     fosslight -o test_result_local_path/test.xlsx -p tests -r
     fosslight binary source -o test_result_multi_mode/test.xlsx -p tests -r
+    fosslight -o test_result_exclude_path/test.xlsx -p tests -e test sample_license.txt
     fosslight dependency -o test_result_wget -w "https://github.com/LGE-OSS/example.git"
     ls test_result_wget
 
@@ -39,5 +41,6 @@ commands =
     fosslight -h
     fosslight all -o test_result_local_path/test.xlsx -p tests -r
     fosslight binary dependency -o test_result_multi_mode/test.xlsx -p tests -r
+    fosslight -o test_result_exclude_path/test.xlsx -p tests -e test sample_license.txt
     fosslight source -o test_result_wget -w "https://github.com/LGE-OSS/example.git"
     pytest -v --flake8