Fix dbrestore on SQLite (#383)

Co-authored-by: Mark <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Mark Bakhit <[email protected]>
jazzband · Aug 23, 2024 · bc74b87 · bc74b87
1 parent 163bd9e
commit bc74b87
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 18 deletions.
diff --git a/dbbackup/db/sqlite.py b/dbbackup/db/sqlite.py
@@ -71,11 +71,23 @@ def restore_dump(self, dump):
         if not self.connection.is_usable():
             self.connection.connect()
         cursor = self.connection.cursor()
+        sql_command = b""
+        sql_is_complete = True
         for line in dump.readlines():
-            try:
-                cursor.execute(line.decode("UTF-8"))
-            except (OperationalError, IntegrityError) as err:
-                warnings.warn(f"Error in db restore: {err}")
+            sql_command = sql_command + line
+            line_str = line.decode("UTF-8")
+            if line_str.startswith("INSERT") and not line_str.endswith(");\n"):
+                sql_is_complete = False
+                continue
+            if not sql_is_complete and line_str.endswith(");\n"):
+                sql_is_complete = True
+
+            if sql_is_complete:
+                try:
+                    cursor.execute(sql_command.decode("UTF-8"))
+                except (OperationalError, IntegrityError) as err:
+                    warnings.warn(f"Error in db restore: {err}")
+                sql_command = b""
 
 
 class SqliteCPConnector(BaseDBConnector):

diff --git a/dbbackup/tests/settings.py b/dbbackup/tests/settings.py
@@ -29,6 +29,7 @@
     "dbbackup",
     "dbbackup.tests.testapp",
 )
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
 DATABASES = {
     "default": {

diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py
@@ -5,7 +5,7 @@
 from django.test import TestCase
 
 from dbbackup.db.sqlite import SqliteConnector, SqliteCPConnector
-from dbbackup.tests.testapp.models import CharModel
+from dbbackup.tests.testapp.models import CharModel, TextModel
 
 
 class SqliteConnectorTest(TestCase):
@@ -28,7 +28,17 @@ def test_create_dump_with_unicode(self):
         dump = connector.create_dump()
         self.assertTrue(dump.read())
 
+    def test_create_dump_with_newline(self):
+        TextModel.objects.create(
+            field=f'INSERT ({"foo" * 5000}\nbar\n WHERE \nbaz IS\n "great" );\n'
+        )
+
+        connector = SqliteConnector()
+        dump = connector.create_dump()
+        self.assertTrue(dump.read())
+
     def test_restore_dump(self):
+        TextModel.objects.create(field="T\nf\nw\nnl")
         connector = SqliteConnector()
         dump = connector.create_dump()
         connector.restore_dump(dump)

diff --git a/dbbackup/tests/testapp/migrations/0001_initial.py b/dbbackup/tests/testapp/migrations/0001_initial.py
@@ -3,6 +3,7 @@
 
 class Migration(migrations.Migration):
 
+    initial = True
     dependencies = []
 
     operations = [

diff --git a/dbbackup/tests/testapp/migrations/0002_textmodel.py b/dbbackup/tests/testapp/migrations/0002_textmodel.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.0.1 on 2022-04-27 22:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("testapp", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="TextModel",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("field", models.TextField()),
+            ],
+        ),
+    ]
diff --git a/dbbackup/tests/testapp/models.py b/dbbackup/tests/testapp/models.py
@@ -1,22 +1,14 @@
 from django.db import models
 
-___all__ = (
-    "CharModel",
-    "IntegerModel",
-    "TextModel",
-    "BooleanModel" "DateModel",
-    "DateTimeModel",
-    "ForeignKeyModel",
-    "ManyToManyModel",
-    "FileModel",
-    "TestModel",
-)
-
 
 class CharModel(models.Model):
     field = models.CharField(max_length=10)
 
 
+class TextModel(models.Model):
+    field = models.TextField()
+
+
 class ForeignKeyModel(models.Model):
     field = models.ForeignKey(CharModel, on_delete=models.CASCADE)
 

diff --git a/docs/changelog.rst b/docs/changelog.rst
@@ -4,7 +4,8 @@ Changelog
 Unreleased
 ----------
 
-* Add `--no-drop` option to `dbrestore` command to prevent dropping tables before restoring data.
+* Add --no-drop option to dbrestore command to prevent dropping tables before restoring data.
+* Fix bug where sqlite dbrestore would fail if field data contains the line break character.
 
 4.2.0 (2024-08-22)
 ------------------