Update SCANOSS version and redirect log (#204)

* change minimum SCANOSS version to 1.18.0

Signed-off-by: Wonjae Park <[email protected]>

* Redirect SCANOSS output to log file

Signed-off-by: Wonjae Park <[email protected]>

* Add comment on info sheet for limit exceed

Signed-off-by: Wonjae Park <[email protected]>

* Fix flake8 fail

Signed-off-by: Wonjae Park <[email protected]>

---------

Signed-off-by: Wonjae Park <[email protected]>

Author: JustinWonjaePark

requirements.txt

@@ -1,5 +1,5 @@
 pyparsing
-scanoss<=1.14.0
+scanoss>=1.18.0
 XlsxWriter
 fosslight_util>=2.1.10
 PyYAML

src/fosslight_source/cli.py

@@ -148,7 +148,7 @@ def create_report_file(
     output_path: str = "", output_files: list = [],
     output_extensions: list = [], correct_mode: bool = True,
     correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [],
-    formats: list = [], excluded_file_list: list = []
+    formats: list = [], excluded_file_list: list = [], api_limit_exceed: bool = False
 ) -> 'ScannerItem':
     """
     Create report files for given scanned result.
@@ -212,6 +212,9 @@ def create_report_file(
     scan_item.set_cover_comment(f"Total number of files : {files_count}")
     scan_item.set_cover_comment(f"Removed files : {removed_files_count}")
 
+    if api_limit_exceed:
+        scan_item.set_cover_comment("(Some of) SCANOSS scan was skipped. (API limits being exceeded)")
+
     if not merged_result:
         if files_count < 1:
             scan_item.set_cover_comment("(No file detected.)")
@@ -350,14 +353,15 @@ def run_scanners(
                                                                                       print_matched_text, formats, called_by_cli,
                                                                                       time_out, correct_mode, correct_filepath)
         if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '':
-            scanoss_result = run_scanoss_py(path_to_scan, output_file_name, formats, True, write_json_file, num_cores,
-                                            path_to_exclude)
+            scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_file_name, formats, True, write_json_file,
+                                                              num_cores, path_to_exclude)
         if selected_scanner in SCANNER_TYPE:
             spdx_downloads = get_spdx_downloads(path_to_scan, path_to_exclude)
             merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads)
             scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
                                            print_matched_text, output_path, output_files, output_extensions, correct_mode,
-                                           correct_filepath, path_to_scan, path_to_exclude, formats, excluded_file_list)
+                                           correct_filepath, path_to_scan, path_to_exclude, formats, excluded_file_list,
+                                           api_limit_exceed)
         else:
             print_help_msg_source_scanner()
             result_log[RESULT_KEY] = "Unsupported scanner"

src/fosslight_source/run_scanoss.py

@@ -17,6 +17,8 @@
 import shutil
 from pathlib import Path
 from scanoss.scanner import Scanner, ScanType
+import io
+import contextlib
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 warnings.filterwarnings("ignore", category=FutureWarning)
@@ -75,7 +77,13 @@ def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: list =
             scan_options=ScanType.SCAN_SNIPPETS.value,
             nb_threads=num_threads if num_threads > 0 else 10
         )
-        scanner.scan_folder_with_options(scan_dir=path_to_scan)
+
+        output_buffer = io.StringIO()
+        with contextlib.redirect_stdout(output_buffer), contextlib.redirect_stderr(output_buffer):
+            scanner.scan_folder_with_options(scan_dir=path_to_scan)
+        captured_output = output_buffer.getvalue()
+        api_limit_exceed = "due to service limits being exceeded" in captured_output
+        logger.debug(f"{captured_output}")
 
         if os.path.isfile(output_json_file):
             total_files_to_excluded = []
@@ -117,4 +125,4 @@ def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: list =
     except Exception as error:
         logger.debug(f"Moving scanoss raw files failed.: {error}")
 
-    return scanoss_file_list
+    return scanoss_file_list, api_limit_exceed

tests/cli_test.py

@@ -26,7 +26,7 @@ def main():
     logger, result_item = init_log(os.path.join(output_dir, "fosslight_log_"+_start_time+".txt"))
 
     ret = run_scan(path_to_find_bin, fosslight_report_name, True, -1, True, True, [], False)
-    ret_scanoss = run_scanoss_py(path_to_find_bin, fosslight_report_name, [], False, True, -1)
+    ret_scanoss, api_limit_exceed = run_scanoss_py(path_to_find_bin, fosslight_report_name, [], False, True, -1)
 
     logger.warning("[Scan] Result: %s" % (ret[0]))
     logger.warning("[Scan] Result_msg: %s" % (ret[1]))