Add --ignore-errors to skip files with syntax errors and attempt to find requirements on a best-effort basis

Author: shumbo

README.rst

@@ -55,6 +55,7 @@ Usage
         --debug               Print debug information
         --ignore <dirs>...    Ignore extra directories, each separated by a comma
         --no-follow-links     Do not follow symbolic links in the project
+        --ignore-errors       Ignore errors while scanning files
         --encoding <charset>  Use encoding parameter for file open
         --savepath <file>     Save the list of requirements in the given file
         --print               Output the list of requirements in the standard output

pipreqs/pipreqs.py

@@ -20,6 +20,7 @@
                           $ export HTTPS_PROXY="https://10.10.1.10:1080"
     --debug               Print debug information
     --ignore <dirs>...    Ignore extra directories, each separated by a comma
+    --ignore-errors       Ignore errors while scanning files
     --no-follow-links     Do not follow symbolic links in the project
     --encoding <charset>  Use encoding parameter for file open
     --savepath <file>     Save the list of requirements in the given file
@@ -97,11 +98,10 @@ def _open(filename=None, mode="r"):
             file.close()
 
 
-def get_all_imports(path, encoding="utf-8", extra_ignore_dirs=None, follow_links=True):
+def get_all_imports(path, encoding="utf-8", extra_ignore_dirs=None, follow_links=True, ignore_errors=False):
     imports = set()
     raw_imports = set()
     candidates = []
-    ignore_errors = False
     ignore_dirs = [
         ".hg",
         ".svn",
@@ -133,9 +133,9 @@ def get_all_imports(path, encoding="utf-8", extra_ignore_dirs=None, follow_links
 
         for file_name in files:
             file_name = os.path.join(root, file_name)
-            contents = read_file_content(file_name, encoding)
 
             try:
+                contents = read_file_content(file_name, encoding)
                 tree = ast.parse(contents)
                 for node in ast.walk(tree):
                     if isinstance(node, ast.Import):
@@ -145,7 +145,7 @@ def get_all_imports(path, encoding="utf-8", extra_ignore_dirs=None, follow_links
                         raw_imports.add(node.module)
             except Exception as exc:
                 if ignore_errors:
-                    traceback.print_exc(exc)
+                    traceback.print_exc()
                     logging.warn("Failed on file: %s" % file_name)
                     continue
                 else:
@@ -504,6 +504,7 @@ def init(args):
     encoding = args.get("--encoding")
     extra_ignore_dirs = args.get("--ignore")
     follow_links = not args.get("--no-follow-links")
+    ignore_errors = args.get("--ignore-errors")
 
     scan_noteboooks = args.get("--scan-notebooks", False)
     handle_scan_noteboooks()
@@ -535,6 +536,7 @@ def init(args):
         encoding=encoding,
         extra_ignore_dirs=extra_ignore_dirs,
         follow_links=follow_links,
+        ignore_errors=ignore_errors,
     )
     candidates = get_pkg_names(candidates)
     logging.debug("Found imports: " + ", ".join(candidates))

tests/test_pipreqs.py

@@ -114,6 +114,13 @@ def test_invalid_python(self):
         """
         self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid)
 
+    def test_ignore_errors(self):
+        """
+        Test that invalid python files do not raise an exception when ignore_errors is True.
+        """
+        imports = pipreqs.get_all_imports(self.project_invalid, ignore_errors=True)
+        self.assertEqual(len(imports), 0)
+
     def test_get_imports_info(self):
         """
         Test to see that the right number of packages were found on PyPI