--delimiter and --quotechar, closes #223

Author: simonw

docs/cli.rst

@@ -474,6 +474,8 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite
     207368,920 Kirkham St,37.760210314285,-122.47073935813
     188702,1501 Evans Ave,37.7422086702947,-122.387293152263
 
+.. _cli_insert_csv_tsv:
+
 Inserting CSV or TSV data
 =========================
 
@@ -483,14 +485,33 @@ If your data is in CSV format, you can insert it using the ``--csv`` option::
 
 For tab-delimited data, use ``--tsv``::
 
-    $ sqlite-utils insert dogs.db dogs docs.tsv --tsv
+    $ sqlite-utils insert dogs.db dogs dogs.tsv --tsv
 
 Data is expected to be encoded as Unicode UTF-8. If your data is an another character encoding you can specify it using the ``--encoding`` option::
 
-    $ sqlite-utils insert dogs.db dogs docs.tsv --tsv --encoding=latin-1
+    $ sqlite-utils insert dogs.db dogs dogs.tsv --tsv --encoding=latin-1
 
 A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option.
 
+.. _cli_insert_csv_tsv_delimiter:
+
+Alternative delimiters and quote characters
+-------------------------------------------
+
+If your file uses a delimiter other than ``,`` or a quote character other than ``"`` you can specify them using the ``--delimiter`` and ``--quotechar`` options.
+
+Here's a CSV file that uses ``;`` for delimiters and the ``|`` symbol for quote characters::
+
+    name;description
+    Cleo;|Very fine; a friendly dog|
+    Pancakes;A local corgi
+
+You can import that using::
+
+    $ sqlite-utils insert dogs.db dogs dogs.csv --delimiter=";" --quotechar="|"
+
+Passing either ``--delimiter`` and ``--quotechar`` implies ``--csv``, so you can omit that option.
+
 .. _cli_insert_replace:
 
 Insert-replacing data

sqlite_utils/cli.py

@@ -1,4 +1,5 @@
 import base64
+from tests.test_cli import test_query_memory_does_not_create_file
 import click
 import codecs
 from click_default_group import DefaultGroup
@@ -601,6 +602,8 @@ def insert_upsert_options(fn):
             click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"),
             click.option("-c", "--csv", is_flag=True, help="Expect CSV"),
             click.option("--tsv", is_flag=True, help="Expect TSV"),
+            click.option("--delimiter", help="Delimiter to use for CSV files"),
+            click.option("--quotechar", help="Quote character to use for CSV/TSV"),
             click.option(
                 "--batch-size", type=int, default=100, help="Commit every X records"
             ),
@@ -640,6 +643,8 @@ def insert_upsert_implementation(
     nl,
     csv,
     tsv,
+    delimiter,
+    quotechar,
     batch_size,
     alter,
     upsert,
@@ -654,6 +659,8 @@ def insert_upsert_implementation(
 ):
     db = sqlite_utils.Database(path)
     _load_extensions(db, load_extension)
+    if delimiter or quotechar:
+        csv = True
     if (nl + csv + tsv) >= 2:
         raise click.ClickException("Use just one of --nl, --csv or --tsv")
     if encoding and not (csv or tsv):
@@ -665,7 +672,12 @@ def insert_upsert_implementation(
     if csv or tsv:
         dialect = "excel-tab" if tsv else "excel"
         with file_progress(json_file, silent=silent) as json_file:
-            reader = csv_std.reader(json_file, dialect=dialect)
+            csv_reader_args = {"dialect": dialect}
+            if delimiter:
+                csv_reader_args["delimiter"] = delimiter
+            if quotechar:
+                csv_reader_args["quotechar"] = quotechar
+            reader = csv_std.reader(json_file, **csv_reader_args)
             headers = next(reader)
             docs = (dict(zip(headers, row)) for row in reader)
     else:
@@ -720,6 +732,8 @@ def insert(
     nl,
     csv,
     tsv,
+    delimiter,
+    quotechar,
     batch_size,
     alter,
     encoding,
@@ -746,6 +760,8 @@ def insert(
             nl,
             csv,
             tsv,
+            delimiter,
+            quotechar,
             batch_size,
             alter=alter,
             upsert=False,
@@ -773,6 +789,8 @@ def upsert(
     csv,
     tsv,
     batch_size,
+    delimiter,
+    quotechar,
     alter,
     not_null,
     default,
@@ -794,6 +812,8 @@ def upsert(
             nl,
             csv,
             tsv,
+            delimiter,
+            quotechar,
             batch_size,
             alter=alter,
             upsert=True,

tests/test_cli.py

@@ -722,18 +722,28 @@ def test_insert_ignore(db_path, tmpdir):
 
 
 @pytest.mark.parametrize(
-    "content,option",
-    (("foo\tbar\tbaz\n1\t2\t3", "--tsv"), ("foo,bar,baz\n1,2,3", "--csv")),
+    "content,options",
+    [
+        ("foo\tbar\tbaz\n1\t2\tcat,dog", ["--tsv"]),
+        ('foo,bar,baz\n1,2,"cat,dog"', ["--csv"]),
+        ('foo;bar;baz\n1;2;"cat,dog"', ["--csv", "--delimiter", ";"]),
+        # --delimiter implies --csv:
+        ('foo;bar;baz\n1;2;"cat,dog"', ["--delimiter", ";"]),
+        ("foo,bar,baz\n1,2,|cat,dog|", ["--csv", "--quotechar", "|"]),
+        ("foo,bar,baz\n1,2,|cat,dog|", ["--quotechar", "|"]),
+    ],
 )
-def test_insert_csv_tsv(content, option, db_path, tmpdir):
+def test_insert_csv_tsv(content, options, db_path, tmpdir):
     db = Database(db_path)
     file_path = str(tmpdir / "insert.csv-tsv")
     open(file_path, "w").write(content)
     result = CliRunner().invoke(
-        cli.cli, ["insert", db_path, "data", file_path, option], catch_exceptions=False
+        cli.cli,
+        ["insert", db_path, "data", file_path] + options,
+        catch_exceptions=False,
     )
     assert 0 == result.exit_code
-    assert [{"foo": "1", "bar": "2", "baz": "3"}] == list(db["data"].rows)
+    assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows)
 
 
 @pytest.mark.parametrize(