Merge pull request #3 from Altinity/develop

Merge from source
mailercloud-com · Jun 24, 2024 · 38ed249 · 38ed249
2 parents f3356c4 + 4101e44
commit 38ed249
Show file tree

Hide file tree

Showing 59 changed files with 1,948 additions and 368 deletions.
diff --git a/.github/workflows/testflows-sink-connector-lightweight.yml b/.github/workflows/testflows-sink-connector-lightweight.yml
@@ -93,7 +93,7 @@ jobs:
 
       - name: Upload artifacts to Altinity Test Reports S3 bucket
         if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
-        working-directory: sink-connector/tests/integration/logs
+        working-directory: sink-connector-lightweight/tests/integration/logs
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -109,3 +109,69 @@ jobs:
             sink-connector-lightweight/tests/integration/env/auto/configs/*.yml
           if-no-files-found: error
           retention-days: 60
+  testflows-lightweight-replicated:
+    runs-on: [self-hosted, on-demand, type-cpx51, image-x86-app-docker-ce]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v3
+        if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
+        with:
+          name: clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz
+
+      - name: Load Docker image
+        if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
+        run: |
+          docker load  < clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz
+          docker image ls
+
+      - name: Runner ssh command
+        working-directory: sink-connector/tests/integration
+        run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
+
+      - name: Install all dependencies
+        working-directory: sink-connector-lightweight/tests/integration
+        run: pip3 install -r requirements.txt
+
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d_%H%M%S')" >> $GITHUB_OUTPUT
+
+      - name: Add ~./local/bin to the PATH
+        if: always()
+        working-directory: sink-connector-lightweight/tests/integration
+        run: echo ~/.local/bin >> $GITHUB_PATH
+
+      - name: Run testflows tests
+        working-directory: sink-connector-lightweight/tests/integration
+        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto replicated table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
+
+      - name: Create tfs results report
+        if: always()
+        working-directory: sink-connector-lightweight/tests/integration/logs
+        run: cat raw.log | tfs report results | tfs document convert > report.html
+
+      - name: Create tfs coverage report
+        if: always()
+        working-directory: sink-connector-lightweight/tests/integration/logs
+        run: cat raw.log | tfs report coverage ../requirements/requirements.py | tfs document convert > coverage.html
+
+      - name: Upload artifacts to Altinity Test Reports S3 bucket
+        if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
+        working-directory: sink-connector-lightweight/tests/integration/logs
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: 'eu-west-2'
+        run: aws s3 cp . s3://altinity-test-reports/altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_sink_lw/ --recursive --exclude "*" --include "*.log" --include "*.html"
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: testflows-sink-connector-lightweight-replicated-artefacts
+          path: |
+            sink-connector-lightweight/tests/integration/logs/*.log
+            sink-connector-lightweight/tests/integration/env/auto_replicated/configs/*.yml
+          if-no-files-found: error
+          retention-days: 60
diff --git a/doc/Troubleshooting.md b/doc/Troubleshooting.md
@@ -51,3 +51,7 @@ There are two solutions to get past this error.
 
 ###  A slave with the same server_uuid/server_id as this slave has connected to the master;
 https://stackoverflow.com/questions/63523998/multiple-debezium-connector-for-one-mysql-db , As mentioned here the `database.server.id` configuration variable has to be unique if there are multiple connectors connecting to the same database.
+
+
+###  PostgreSQL - ERROR - Error starting connectorio.debezium.DebeziumException: Creation of replication slot failed; when setting up multiple connectors for the same database host, please make sure to use a distinct replication slot name for each.  
+Make sure to add `slot.name` to the configuration(config.yml) and change it to a unique name.
diff --git a/doc/quickstart.md b/doc/quickstart.md
@@ -31,7 +31,7 @@ sudo apt install clickhouse-client
 Use Docker Compose to start containers. 
 ```
 cd sink-connector-lightweight/docker
-export CLICKHOUSE_SINK_CONNECTOR_LT_IMAGE=altinityinfra/clickhouse-sink-connector:408-97b1d3d83ef93c1b76a2b1c4d9c544dc67fbbec3-lt
+export CLICKHOUSE_SINK_CONNECTOR_LT_IMAGE=altinity/clickhouse-sink-connector:2.1.0-lt
 docker compose -f docker-compose-mysql.yml up --renew-anon-volumes
 ```
 
@@ -139,4 +139,4 @@ schema.history.internal.jdbc.url: "jdbc:clickhouse://cloud_url:8443/altinity_sin
 ## References:
 1. [Sink Connector Configuration ](configuration.md)
 2. [MySQL Topologies supported](https://debezium.io/documentation/reference/2.5/connectors/mysql.html#setting-up-mysql)
-3. [MySQL Setup](https://debezium.io/documentation/reference/2.5/connectors/mysql.html#setting-up-mysql)
+3. [MySQL Setup](https://debezium.io/documentation/reference/2.5/connectors/mysql.html#setting-up-mysql)
diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/common.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/common.xml
@@ -0,0 +1,6 @@
+<clickhouse>
+    <timezone>Europe/Moscow</timezone>
+    <listen_host replace="replace">::</listen_host>
+    <path>/var/lib/clickhouse/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+</clickhouse>
diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ports.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ports.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <https_port>8443</https_port>
+    <tcp_port_secure>9440</tcp_port_secure>
+    <postgresql_port>9005</postgresql_port>
+    <mysql_port>9004</mysql_port>
+</clickhouse>
diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/remote.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/remote.xml
@@ -12,6 +12,10 @@
         <replicated_cluster>
             <shard>
                 <internal_replication>false</internal_replication>
+                <replica>
+                    <host>clickhouse</host>
+                    <port>9000</port>
+                </replica>
                 <replica>
                     <host>clickhouse1</host>
                     <port>9000</port>
@@ -27,6 +31,12 @@
             </shard>
         </replicated_cluster>
         <sharded_cluster>
+            <shard>
+                <replica>
+                    <host>clickhouse</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
             <shard>
                 <replica>
                     <host>clickhouse1</host>

diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ssl.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ssl.xml
@@ -0,0 +1,18 @@
+<clickhouse>
+    <openSSL>
+        <server>
+            <certificateFile>/etc/clickhouse-server/ssl/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/ssl/server.key</privateKeyFile>
+            <dhParamsFile>/etc/clickhouse-server/ssl/dhparam.pem</dhParamsFile>
+            <verificationMode>none</verificationMode>
+            <cacheSessions>true</cacheSessions>
+        </server>
+        <client>
+            <cacheSessions>true</cacheSessions>
+            <verificationMode>none</verificationMode>
+            <invalidCertificateHandler>
+                <name>AcceptCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+</clickhouse>
diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/storage.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/storage.xml
@@ -0,0 +1,99 @@
+<clickhouse>
+
+<storage_configuration>
+    <disks>
+        <s3_disk>
+          <type>s3</type>
+            <endpoint>http://minio:9001/root/data/</endpoint>
+            <access_key_id>minio</access_key_id>
+            <secret_access_key>minio123</secret_access_key>
+        </s3_disk>
+        <default>
+            <keep_free_space_bytes>1024</keep_free_space_bytes>
+        </default>
+        <new_disk>
+            <path>/mnt/new_disk/</path>
+            <keep_free_space_bytes>10485760</keep_free_space_bytes>
+        </new_disk>
+        <disk1>
+            <path>/var/lib/clickhouse/disk1/</path>
+        </disk1>
+        <disk2>
+            <path>/var/lib/clickhouse/disk2/</path>
+        </disk2>
+        <jbod1>
+          <path>/jbod1/</path>
+        </jbod1>
+        <jbod2>
+          <path>/jbod2/</path>
+            <keep_free_space_bytes>10485760</keep_free_space_bytes>
+        </jbod2>
+    </disks>
+    <policies>
+      <s3_policy>
+        <volumes>
+          <main>
+            <disk>s3_disk</disk>
+          </main>
+        </volumes>
+      </s3_policy>
+        <default>
+            <volumes>
+                <default>
+                    <disk>default</disk>
+                </default>
+            </volumes>
+        </default>
+        <my_policy>
+            <volumes>
+                <main>
+                    <disk>default</disk>
+                    <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
+                </main>
+                <external>
+                    <disk>new_disk</disk>
+                </external>
+            </volumes>
+        </my_policy>
+        <policy1>
+            <volumes>
+                <volume2>
+                    <disk>disk1</disk>
+                </volume2>
+            </volumes>
+        </policy1>
+        <policy2>
+            <volumes>
+                <volume2>
+                    <disk>disk2</disk>
+                </volume2>
+            </volumes>
+        </policy2>
+        <tiered_storage>
+            <volumes>
+                <hot>
+                    <disk>disk1</disk>
+                </hot>
+                <cold>
+                    <disk>s3_disk</disk>
+                </cold>
+            </volumes>
+            <move_factor>0.2</move_factor>
+        </tiered_storage>
+        <fast_med_and_slow>
+          <volumes>
+              <fast>
+                  <disk>jbod1</disk>
+                </fast>
+                <medium>
+                  <disk>jbod2</disk>
+                </medium>
+                <slow>
+                  <disk>s3_disk</disk>
+                </slow>
+            </volumes>
+        </fast_med_and_slow>
+    </policies>
+</storage_configuration>
+
+</clickhouse>
diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/dhparam.pem b/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/dhparam.pem
@@ -0,0 +1,8 @@
+-----BEGIN DH PARAMETERS-----
+MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR
+XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5
+Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb
+9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf
+slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb
+dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg==
+-----END DH PARAMETERS-----
diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.crt b/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
+BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow
+FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI
+4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T
+4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU
+7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj
+sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg
+pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC
+kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j
+4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr
+85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C
+L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD
++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L
+P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp
+0g==
+-----END CERTIFICATE-----
diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.key b/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn
+DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L
+OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus
+TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7
+JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F
+KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6
+ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP
+TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f
+7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN
+MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5
+212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu
+plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr
+/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu
+Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa
+CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g
+1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5
+bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq
+cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez
+on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K
+GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54
+nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU
+2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh
+vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa
+dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr
+vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1
+KCBtmIrQLqzMpnBpLNuSY+Q=
+-----END PRIVATE KEY-----
diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse1/config.d/macros.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse1/config.d/macros.xml
@@ -1,6 +1,7 @@
 <?xml version="1.0"?>
 <clickhouse>
     <macros>
+	    <cluster>replicated_cluster</cluster>
         <replica>clickhouse1</replica>
         <shard>01</shard>
 <!--        <shard2>01</shard2>-->

diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse2/config.d/macros.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse2/config.d/macros.xml
@@ -1,8 +1,9 @@
 <?xml version="1.0"?>
 <clickhouse>
     <macros>
+	    <cluster>replicated_cluster</cluster>
         <replica>clickhouse2</replica>
-        <shard>02</shard>
+        <shard>01</shard>
 <!--        <shard2>02</shard2>-->
     </macros>
 </clickhouse>
diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse3/config.d/macros.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse3/config.d/macros.xml
@@ -1,8 +1,8 @@
 <?xml version="1.0"?>
 <clickhouse>
     <macros>
+	    <cluster>replicated_cluster</cluster>
         <replica>clickhouse3</replica>
-        <shard>02</shard>
-<!--        <shard2>03</shard2>-->
+        <shard>01</shard>
     </macros>
 </clickhouse>
diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/clickhouse-service.yml b/sink-connector-lightweight/tests/integration/env/auto_replicated/clickhouse-service.yml
@@ -0,0 +1,31 @@
+version: '2.3'
+
+services:
+  clickhouse:
+#    image: clickhouse/clickhouse-integration-test:28741
+    image: altinityinfra/clickhouse-regression-multiarch:1.0
+    init: true
+    expose:
+      - "9000"
+      - "8123"
+      - "2181"
+    volumes:
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml"
+      - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml"
+      - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse"
+      - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge"
+    entrypoint: bash -c "tail -f /dev/null"
+    healthcheck:
+      test: echo 1
+      interval: 10s
+      timeout: 10s
+      retries: 3
+      start_period: 300s
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - label:disable
+