Merge pull request #15 from suutari-ai/coverage

Improve test coverage etc.

Author: JsseL

.travis.yml

@@ -4,10 +4,11 @@ cache: pip
 python:
   - "2.7"
   - "3.4"
+  - "3.5"
+  - "3.6"
 install:
   - pip install -r requirements-test.txt
 script:
   - py.test -vvv --cov database_sanitizer --cov-report=term-missing
 after_success:
   - curl -s https://codecov.io/bash | bash
-

database_sanitizer/__main__.py

@@ -10,11 +10,9 @@
 from .dump import run
 
 
-def main(args=None):
-    if args is None:
-        args = sys.argv[1:]
-
+def main(argv=sys.argv):
     parser = argparse.ArgumentParser(
+        prog=(argv[0] if len(argv) else "database-sanitizer"),
         description="Sanitizes contents of databases.",
     )
     parser.add_argument(
@@ -30,16 +28,16 @@ def main(args=None):
         type=str,
         dest="output",
         help=(
-            "Path to the file where the sanitized database will be written into. "
-            "If omitted, standard output will be used instead."
+            "Path to the file where the sanitized database will be written "
+            "into. If omitted, standard output will be used instead."
         ),
     )
     parser.add_argument(
         "url",
         help="Database URL to which to connect into and sanitize contents.",
     )
 
-    args = parser.parse_args(args=args)
+    args = parser.parse_args(args=argv[1:])
     output = sys.stdout
     config = None
 

database_sanitizer/config.py

@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 import importlib
+
 import six
 import yaml
 
@@ -22,7 +23,7 @@ class Configuration(object):
     """
     def __init__(self):
         self.sanitizers = {}
-        self.addon_packages = None
+        self.addon_packages = []
 
     @classmethod
     def from_file(cls, filename):
@@ -83,10 +84,8 @@ def load_addon_packages(self, config_data):
                 ),
             )
 
-        section_addons = section_config.get("addons")
+        section_addons = section_config.get("addons", [])
         if not isinstance(section_addons, list):
-            if section_addons is None:
-                return
             raise ConfigurationError(
                 "'config.addons' is %s instead of list" % (
                     type(section_addons),
@@ -102,7 +101,7 @@ def load_addon_packages(self, config_data):
                     ),
                 )
 
-        self.addon_packages = tuple(section_addons)
+        self.addon_packages = list(section_addons)
 
     def load_sanitizers(self, config_data):
         """
@@ -203,18 +202,17 @@ def find_sanitizer(self, name):
 
         # Phase 2: Look for the sanitizer under "addon" packages, if any of
         # such have been defined.
-        if self.addon_packages:
-            for addon_package_name in self.addon_packages:
-                module_name = "%s.%s" % (
-                    addon_package_name,
-                    module_name_suffix,
-                )
-                callback = self.find_sanitizer_from_module(
-                    module_name=module_name,
-                    function_name=function_name,
-                )
-                if callback:
-                    return callback
+        for addon_package_name in self.addon_packages:
+            module_name = "%s.%s" % (
+                addon_package_name,
+                module_name_suffix,
+            )
+            callback = self.find_sanitizer_from_module(
+                module_name=module_name,
+                function_name=function_name,
+            )
+            if callback:
+                return callback
 
         # Phase 3: Look from builtin sanitizers.
         module_name = "database_sanitizer.sanitizers.%s" % (module_name_suffix,)

database_sanitizer/dump/mysql.py

@@ -12,7 +12,6 @@
     get_mysqldump_args_and_env_from_url,
 )
 
-
 #: Regular expression which matches `INSERT INTO` statements produced by the
 #: `mysqldump` utility, even when extended inserts have been enabled.
 INSERT_INTO_PATTERN = re.compile(

database_sanitizer/dump/postgres.py

@@ -8,7 +8,6 @@
 
 from ..utils.postgres import decode_copy_value, encode_copy_value
 
-
 COPY_LINE_PATTERN = re.compile(
     r"^COPY \"(?P<schema>[^\"]*)\".\"(?P<table>[^\"]*)\" "
     r"\((?P<columns>.*)\) "

database_sanitizer/tests/_compat.py

@@ -2,13 +2,25 @@
 
 from __future__ import unicode_literals
 
-import pytest
-
 from collections import namedtuple
 
+import mock
+import pytest
+
+from .. import config
 from ..config import Configuration, ConfigurationError
 
-from ._compat import mock
+
+@mock.patch.object(config, 'open')
+@mock.patch('yaml.load')
+def test_from_file(mocked_yaml_load, mocked_open):
+    mocked_yaml_load.return_value = {}
+
+    Configuration.from_file('filename.yml')
+
+    assert mocked_open.call_args == (('filename.yml', 'rb'), {})
+    opened_file = mocked_open.return_value.__enter__.return_value
+    assert mocked_yaml_load.call_args == ((opened_file,), {})
 
 
 def test_load_config_data_must_be_dict():
@@ -22,13 +34,13 @@ def test_load_addon_packages():
     config = Configuration()
 
     config.load_addon_packages({})
-    assert config.addon_packages is None
+    assert config.addon_packages == []
 
     with pytest.raises(ConfigurationError):
         config.load_addon_packages({"config": "test"})
 
     config.load_addon_packages({"config": {}})
-    assert config.addon_packages is None
+    assert config.addon_packages == []
 
     with pytest.raises(ConfigurationError):
         config.load_addon_packages({"config": {"addons": "test"}})
@@ -43,7 +55,7 @@ def test_load_addon_packages():
             "test3",
         ],
     }})
-    assert config.addon_packages == ("test1", "test2", "test3")
+    assert config.addon_packages == ["test1", "test2", "test3"]
 
 
 def test_load_sanitizers():

database_sanitizer/tests/test_config.py

@@ -0,0 +1,53 @@
+import subprocess
+from io import BytesIO, StringIO
+
+import mock
+import pytest
+
+from database_sanitizer import dump
+
+EXPECTED_POPEN_KWARGS = {
+    'mysql://User:Pass@HostName/Db': {
+        'args': (
+            'mysqldump --complete-insert --extended-insert'
+            ' --net_buffer_length=10240 -h hostname -u User Db').split(),
+        'env': {'MYSQL_PWD': 'Pass'},
+        'stdout': subprocess.PIPE,
+    },
+    'postgres:///Db': {
+        'args': tuple((
+            'pg_dump --encoding=utf-8 --quote-all-identifiers'
+            ' --dbname postgres:///Db').split()),
+        'stdout': subprocess.PIPE,
+    },
+}
+
+for url in ['postgresql:///Db', 'postgis:///Db']:
+    EXPECTED_POPEN_KWARGS[url] = EXPECTED_POPEN_KWARGS['postgres:///Db'].copy()
+    EXPECTED_POPEN_KWARGS[url]['args'] = tuple(
+        ' '.join(EXPECTED_POPEN_KWARGS[url]['args'])
+        .replace('postgres', 'postgresql').split())
+
+
+@pytest.mark.parametrize('url', list(EXPECTED_POPEN_KWARGS))
+@mock.patch('subprocess.Popen')
+def test_run(mocked_popen, url):
+    mocked_popen.return_value.stdout = BytesIO(b'INPUT DUMP')
+    output = StringIO()
+    config = None
+    dump.run(url, output, config)
+
+    expected_popen_kwargs = EXPECTED_POPEN_KWARGS[url]
+    (popen_args, popen_kwargs) = mocked_popen.call_args
+    expected_popen_args = (
+        (expected_popen_kwargs.pop('args'),) if popen_args else ())
+    assert popen_args == expected_popen_args
+    assert popen_kwargs == expected_popen_kwargs
+
+
+@mock.patch('subprocess.Popen')
+def test_run_unknown_scheme(mocked_popen):
+    with pytest.raises(ValueError) as excinfo:
+        dump.run('unknown:///db', None, None)
+    assert str(excinfo.value) == "Unsupported database scheme: 'unknown'"
+    mocked_popen.assert_not_called()

database_sanitizer/tests/test_dump.py

@@ -3,8 +3,8 @@
 from __future__ import unicode_literals
 
 import io
-import pytest
 
+import pytest
 from six.moves.urllib import parse as urlparse
 
 from ..config import Configuration
@@ -15,7 +15,6 @@
     sanitize_from_stream,
 )
 
-
 MOCK_MYSQLDUMP_OUTPUT = b"""
 --- Fake MySQL database dump
 
@@ -89,7 +88,43 @@ def test_parse_column_names(text, expected_column_names):
         ("('test'),('test')", (("test",), ("test",))),
         ("(1,2),(3,4),", ((1, 2), (3, 4))),
         ("(TRUE),(FALSE),(NULL)", ((True,), (False,), (None,))),
+        ("(x')", ()),  # Invalid data
     ),
 )
 def test_parse_values(text, expected_values):
     assert tuple(parse_values(text)) == expected_values
+
+
+@pytest.mark.parametrize('config_type', [
+    'no-config', 'empty-config', 'single-column-config'])
+@pytest.mark.parametrize('data_label', ['ok', 'invalid'])
+def test_optimizations(config_type, data_label):
+    if config_type == 'no-config':
+        config = None
+        decoder_call_count = 0
+    else:
+        config = Configuration()
+        if config_type == 'empty-config':
+            decoder_call_count = 0
+        else:
+            assert config_type == 'single-column-config'
+            config.sanitizers["test.notes"] = (lambda x: x)
+            decoder_call_count = 3  # Number of rows in test table
+
+    data = {
+        'ok': MOCK_MYSQLDUMP_OUTPUT,
+        'invalid': INVALID_MOCK_MYSQLDUMP_OUTPUT,
+    }[data_label]
+
+    should_raise = (
+        config_type == 'single-column-config'
+        and data_label == 'invalid')
+
+    dump_stream = io.BytesIO(data)
+    if should_raise:
+        with pytest.raises(ValueError):
+            list(sanitize_from_stream(dump_stream, config))
+    else:
+        expected_output = data.decode('utf-8').splitlines()
+        result = list(sanitize_from_stream(dump_stream, config))
+        assert result == expected_output

database_sanitizer/tests/test_dump_mysql.py

@@ -3,19 +3,17 @@
 from __future__ import unicode_literals
 
 import io
-import pytest
-
 from collections import namedtuple
+
+import mock
+import pytest
 from six.moves.urllib import parse as urlparse
 
 from ..config import Configuration
-from ..dump.postgres import parse_column_names, parse_values, sanitize
 from ..dump import postgres as dump_postgres
+from ..dump.postgres import parse_column_names, parse_values, sanitize
 from ..utils.postgres import decode_copy_value
 
-from ._compat import mock
-
-
 MOCK_PG_DUMP_OUTPUT = b"""
 --- Fake PostgreSQL database dump