✨ add MySQL 8.4 and MariaDB 11.4 support (#122)

techouse · web-flow · commit 042dec9b301f · 2024-07-23T16:13:28.000+01:00
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -306,6 +306,36 @@ jobs:
             experimental: false
             py: "3.12"
 
+          - toxenv: "python3.8"
+            db: "mariadb:11.4"
+            legacy_db: 0
+            experimental: false
+            py: "3.8"
+
+          - toxenv: "python3.9"
+            db: "mariadb:11.4"
+            legacy_db: 0
+            experimental: false
+            py: "3.9"
+
+          - toxenv: "python3.10"
+            db: "mariadb:11.4"
+            legacy_db: 0
+            experimental: false
+            py: "3.10"
+
+          - toxenv: "python3.11"
+            db: "mariadb:11.4"
+            legacy_db: 0
+            experimental: false
+            py: "3.11"
+
+          - toxenv: "python3.12"
+            db: "mariadb:11.4"
+            legacy_db: 0
+            experimental: false
+            py: "3.12"
+
           - toxenv: "python3.8"
             db: "mysql:5.5"
             legacy_db: 1
@@ -425,15 +455,46 @@ jobs:
             legacy_db: 0
             experimental: false
             py: "3.12"
+
+          - toxenv: "python3.8"
+            db: "mysql:8.4"
+            legacy_db: 0
+            experimental: true
+            py: "3.8"
+
+          - toxenv: "python3.9"
+            db: "mysql:8.4"
+            legacy_db: 0
+            experimental: true
+            py: "3.9"
+
+          - toxenv: "python3.10"
+            db: "mysql:8.4"
+            legacy_db: 0
+            experimental: true
+            py: "3.10"
+
+          - toxenv: "python3.11"
+            db: "mysql:8.4"
+            legacy_db: 0
+            experimental: true
+            py: "3.11"
+
+          - toxenv: "python3.12"
+            db: "mysql:8.4"
+            legacy_db: 0
+            experimental: true
+            py: "3.12"
     continue-on-error: ${{ matrix.experimental }}
     services:
       mysql:
-        image: "${{ matrix.db }}"
+        image: ${{ matrix.db }}
         ports:
           - 3306:3306
         env:
           MYSQL_ALLOW_EMPTY_PASSWORD: yes
-        options: "--name=mysqld"
+        options: >-
+          --name=mysqld
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.py }}
@@ -461,31 +522,52 @@ jobs:
           MYSQL_PORT: 3306
         run: |
           set -e
+          
           while :
           do
             sleep 1
             mysql -h127.0.0.1 -uroot -e 'select version()' && break
           done
+          
+          case "$DB" in
+            'mysql:8.0'|'mysql:8.4')
+              mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on"
+              docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}"
+              docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}"
+              docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}"
+              docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}"
+              docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}"
+              ;;
+          esac
+          
+          USER_CREATION_COMMANDS=''
+          WITH_PLUGIN=''
+
           if [ "$DB" == 'mysql:8.0' ]; then
             WITH_PLUGIN='with mysql_native_password'
-            mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on"
-            docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}"
-            docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}"
-            docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}"
-            docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}"
-            docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}"
-            mysql -uroot -h127.0.0.1 -e '
+            USER_CREATION_COMMANDS='
               CREATE USER
               user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256",
               nopass_sha256 IDENTIFIED WITH "sha256_password",
               user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2",
               nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password"
-              PASSWORD EXPIRE NEVER;'
-            mysql -uroot -h127.0.0.1 -e 'GRANT RELOAD ON *.* TO user_caching_sha2;'
-          else
-            WITH_PLUGIN=''
+              PASSWORD EXPIRE NEVER;
+              GRANT RELOAD ON *.* TO user_caching_sha2;'
+          elif [ "$DB" == 'mysql:8.4' ]; then
+            WITH_PLUGIN='with caching_sha2_password'
+            USER_CREATION_COMMANDS='
+              CREATE USER
+              user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2",
+              nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password"
+              PASSWORD EXPIRE NEVER;
+              GRANT RELOAD ON *.* TO user_caching_sha2;'
+          fi
+          
+          if [ ! -z "$USER_CREATION_COMMANDS" ]; then
+            mysql -uroot -h127.0.0.1 -e "$USER_CREATION_COMMANDS"
           fi
-          mysql -h127.0.0.1 -uroot -e "create database $MYSQL_DATABASE DEFAULT CHARACTER SET utf8mb4"
+          
+          mysql -h127.0.0.1 -uroot -e "create database $MYSQL_DATABASE DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
           mysql -h127.0.0.1 -uroot -e "create user $MYSQL_USER identified $WITH_PLUGIN by '${MYSQL_PASSWORD}'; grant all on ${MYSQL_DATABASE}.* to ${MYSQL_USER};"
           mysql -h127.0.0.1 -uroot -e "create user ${MYSQL_USER}@localhost identified $WITH_PLUGIN by '${MYSQL_PASSWORD}'; grant all on ${MYSQL_DATABASE}.* to ${MYSQL_USER}@localhost;"
       - name: Create db_credentials.json
diff --git a/README.md b/README.md
@@ -1,8 +1,8 @@
 [![PyPI](https://img.shields.io/pypi/v/sqlite3-to-mysql)](https://pypi.org/project/sqlite3-to-mysql/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/sqlite3-to-mysql)](https://pypistats.org/packages/sqlite3-to-mysql)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlite3-to-mysql)](https://pypi.org/project/sqlite3-to-mysql/)
-[![MySQL Support](https://img.shields.io/static/v1?label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0&color=2b5d80)](https://img.shields.io/static/v1?label=MySQL&message=5.6+|+5.7+|+8.0&color=2b5d80)
-[![MariaDB Support](https://img.shields.io/static/v1?label=MariaDB&message=5.5+|+10.0+|+10.1+|+10.2+|+10.3+|+10.4+|+10.5+|+10.6|+10.11&color=C0765A)](https://img.shields.io/static/v1?label=MariaDB&message=10.0+|+10.1+|+10.2+|+10.3+|+10.4+|+10.5&color=C0765A)
+[![MySQL Support](https://img.shields.io/static/v1?label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0+|+8.4&color=2b5d80)](https://img.shields.io/static/v1?label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0+|+8.4&color=2b5d80)
+[![MariaDB Support](https://img.shields.io/static/v1?label=MariaDB&message=5.5+|+10.0+|+10.1+|+10.2+|+10.3+|+10.4+|+10.5+|+10.6|+10.11+|+11.4&color=C0765A)](https://img.shields.io/static/v1?label=MariaDB&message=5.5|+10.0+|+10.1+|+10.2+|+10.3+|+10.4+|+10.5|+11.4&color=C0765A)
 [![GitHub license](https://img.shields.io/github/license/techouse/sqlite3-to-mysql)](https://github.com/techouse/sqlite3-to-mysql/blob/master/LICENSE)
 [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE-OF-CONDUCT.md)
 [![PyPI - Format](https://img.shields.io/pypi/format/sqlite3-to-mysql)]((https://pypi.org/project/sqlite3-to-mysql/))
diff --git a/pyproject.toml b/pyproject.toml
@@ -39,7 +39,7 @@ classifiers = [
 ]
 dependencies = [
     "Click>=8.1.3",
-    "mysql-connector-python>=8.2.0",
+    "mysql-connector-python>=9.0.0",
     "pytimeparse2",
     "python-dateutil>=2.9.0.post0",
     "types_python_dateutil",
diff --git a/requirements_dev.txt b/requirements_dev.txt
@@ -2,7 +2,7 @@ Click>=8.1.3
 docker>=6.1.3
 factory-boy
 Faker>=18.10.0
-mysql-connector-python>=8.2.0
+mysql-connector-python>=9.0.0
 PyMySQL>=1.0.3
 pytest>=7.3.1
 pytest-cov
diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py
@@ -107,7 +107,7 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]):
             kwargs.get("mysql_collation") or CharacterSet().get_default_collation(self._mysql_charset.lower())[0]
         )
         if not kwargs.get("mysql_collation") and self._mysql_collation == "utf8mb4_0900_ai_ci":
-            self._mysql_collation = "utf8mb4_general_ci"
+            self._mysql_collation = "utf8mb4_unicode_ci"
 
         self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys", False) or False
 
@@ -144,6 +144,8 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]):
                 port=self._mysql_port,
                 ssl_disabled=self._mysql_ssl_disabled,
                 use_pure=True,
+                charset=self._mysql_charset,
+                collation=self._mysql_collation,
             )
             if isinstance(_mysql_connection, mysql.connector.MySQLConnection):
                 self._mysql = _mysql_connection
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -293,6 +293,8 @@ def mysql_instance(mysql_credentials: MySQLCredentials, pytestconfig: Config) ->
                 password=mysql_credentials.password,
                 host=mysql_credentials.host,
                 port=mysql_credentials.port,
+                charset="utf8mb4",
+                collation="utf8mb4_unicode_ci",
             )
         except mysql.connector.Error as err:
             if err.errno == errorcode.CR_SERVER_LOST:
diff --git a/tests/func/sqlite3_to_mysql_test.py b/tests/func/sqlite3_to_mysql_test.py
@@ -348,6 +348,8 @@ def test_transfer_transfers_all_tables_in_sqlite_file(
                 host=mysql_credentials.host,
                 port=mysql_credentials.port,
                 database=mysql_credentials.database,
+                charset="utf8mb4",
+                collation="utf8mb4_unicode_ci",
             )
         )
         server_version: t.Tuple[int, ...] = mysql_connector_connection.get_server_version()
diff --git a/tests/func/test_cli.py b/tests/func/test_cli.py
@@ -77,6 +77,10 @@ def test_invalid_database_name(
                 "_".join(faker.words(nb=3)),
                 "-u",
                 faker.first_name().lower(),
+                "-h",
+                mysql_credentials.host,
+                "-P",
+                str(mysql_credentials.port),
             ],
         )
         assert result.exit_code > 0
@@ -99,6 +103,10 @@ def test_invalid_database_user(
                 mysql_credentials.database,
                 "-u",
                 faker.first_name().lower(),
+                "-h",
+                mysql_credentials.host,
+                "-P",
+                str(mysql_credentials.port),
             ],
         )
         assert result.exit_code > 0
@@ -123,6 +131,10 @@ def test_invalid_database_password(
                 mysql_credentials.user,
                 "--mysql-password",
                 faker.password(length=16),
+                "-h",
+                mysql_credentials.host,
+                "-P",
+                str(mysql_credentials.port),
             ],
         )
         assert result.exit_code > 0
@@ -145,6 +157,10 @@ def test_database_password_prompt(
                 "-u",
                 mysql_credentials.user,
                 "-p",
+                "-h",
+                mysql_credentials.host,
+                "-P",
+                str(mysql_credentials.port),
             ],
             input=mysql_credentials.password,
         )
@@ -168,6 +184,10 @@ def test_invalid_database_password_prompt(
                 "-u",
                 mysql_credentials.user,
                 "-p",
+                "-h",
+                mysql_credentials.host,
+                "-P",
+                str(mysql_credentials.port),
             ],
             input=faker.password(length=16),
         )
@@ -230,6 +250,10 @@ def test_mysql_skip_transfer_data(
                 "--mysql-password",
                 mysql_credentials.password,
                 "--mysql-skip-transfer-data",
+                "-h",
+                mysql_credentials.host,
+                "-P",
+                str(mysql_credentials.port),
             ],
         )
         assert result.exit_code == 0
@@ -254,6 +278,10 @@ def test_mysql_skip_create_tables(
                 "--mysql-password",
                 mysql_credentials.password,
                 "--mysql-skip-transfer-data",
+                "-h",
+                mysql_credentials.host,
+                "-P",
+                str(mysql_credentials.port),
             ],
         )
         assert result1.exit_code == 0
@@ -270,6 +298,10 @@ def test_mysql_skip_create_tables(
                 "--mysql-password",
                 mysql_credentials.password,
                 "--mysql-skip-create-tables",
+                "-h",
+                mysql_credentials.host,
+                "-P",
+                str(mysql_credentials.port),
             ],
         )
         assert result2.exit_code == 0
@@ -294,6 +326,10 @@ def test_mysql_skip_create_tables_and_transfer_data(
                 mysql_credentials.password,
                 "--mysql-skip-create-tables",
                 "--mysql-skip-transfer-data",
+                "-h",
+                mysql_credentials.host,
+                "-P",
+                str(mysql_credentials.port),
             ],
         )
         assert result.exit_code > 0

Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ classifiers = [`
`39`	`39`	`]`
`40`	`40`	`dependencies = [`
`41`	`41`	`"Click>=8.1.3",`
`42`		`- "mysql-connector-python>=8.2.0",`
	`42`	`+ "mysql-connector-python>=9.0.0",`
`43`	`43`	`"pytimeparse2",`
`44`	`44`	`"python-dateutil>=2.9.0.post0",`
`45`	`45`	`"types_python_dateutil",`
Original file line number	Diff line number	Diff line change
`@@ -293,6 +293,8 @@ def mysql_instance(mysql_credentials: MySQLCredentials, pytestconfig: Config) ->`
`293`	`293`	`password=mysql_credentials.password,`
`294`	`294`	`host=mysql_credentials.host,`
`295`	`295`	`port=mysql_credentials.port,`
	`296`	`+ charset="utf8mb4",`
	`297`	`+ collation="utf8mb4_unicode_ci",`
`296`	`298`	`)`
`297`	`299`	`except mysql.connector.Error as err:`
`298`	`300`	`if err.errno == errorcode.CR_SERVER_LOST:`
Original file line number	Diff line number	Diff line change
`@@ -348,6 +348,8 @@ def test_transfer_transfers_all_tables_in_sqlite_file(`
`348`	`348`	`host=mysql_credentials.host,`
`349`	`349`	`port=mysql_credentials.port,`
`350`	`350`	`database=mysql_credentials.database,`
	`351`	`+ charset="utf8mb4",`
	`352`	`+ collation="utf8mb4_unicode_ci",`
`351`	`353`	`)`
`352`	`354`	`)`
`353`	`355`	`server_version: t.Tuple[int, ...] = mysql_connector_connection.get_server_version()`