Replacing the redis and redis-stack-server images with redis-libs-tests image in test infrastructure (#3505)

* Replacing the redis image with redis-libs-tests image in test infrastructure

* Replacing redis-stack-server image usage with client-libs-test. Fixing lib version in setup.py

* Defining stack tag variable for the build and test github action

* Removing unused env var from build and test github actions

Author: petyaslavova

.github/actions/run-tests/action.yml

@@ -31,92 +31,91 @@ runs:
     - name: Setup Test environment
       env:
         REDIS_VERSION: ${{ inputs.redis-version }}
-        REDIS_IMAGE: "redis:${{ inputs.redis-version }}"
-        CLIENT_LIBS_TEST_IMAGE: "redislabs/client-libs-test:${{ inputs.redis-version }}"
+        CLIENT_LIBS_TEST_IMAGE_TAG: ${{ inputs.redis-version }}
       run: |
         set -e
-                
+
         echo "::group::Installing dependencies"
         pip install -U setuptools wheel
         pip install -r requirements.txt
         pip install -r dev_requirements.txt
         if [ "${{inputs.parser-backend}}" == "hiredis" ]; then
           pip install "hiredis${{inputs.hiredis-version}}"
-          echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g')" >> $GITHUB_ENV        
+          echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g')" >> $GITHUB_ENV
         else
           echo "PARSER_BACKEND=${{inputs.parser-backend}}" >> $GITHUB_ENV
         fi
         echo "::endgroup::"
-        
+
         echo "::group::Starting Redis servers"
         redis_major_version=$(echo "$REDIS_VERSION" | grep -oP '^\d+')
-        
+
         if (( redis_major_version < 8 )); then
           echo "Using redis-stack for module tests"
-        
-          # Mapping of redis version to stack version  
+
+          # Mapping of redis version to stack version
           declare -A redis_stack_version_mapping=(
-            ["7.4.2"]="7.4.0-v3"
-            ["7.2.7"]="7.2.0-v15"
-            ["6.2.17"]="6.2.6-v19"
+            ["7.4.2"]="rs-7.4.0-v2"
+            ["7.2.7"]="rs-7.2.0-v14"
+            ["6.2.17"]="rs-6.2.6-v18"
           )
-                  
+
           if [[ -v redis_stack_version_mapping[$REDIS_VERSION] ]]; then
-            export REDIS_STACK_IMAGE="redis/redis-stack-server:${redis_stack_version_mapping[$REDIS_VERSION]}"
+            export CLIENT_LIBS_TEST_STACK_IMAGE_TAG=${redis_stack_version_mapping[$REDIS_VERSION]}
             echo "REDIS_MOD_URL=redis://127.0.0.1:6479/0" >> $GITHUB_ENV
           else
             echo "Version not found in the mapping."
             exit 1
           fi
-                  
+
           if (( redis_major_version < 7 )); then
             export REDIS_STACK_EXTRA_ARGS="--tls-auth-clients optional --save ''"
             export REDIS_EXTRA_ARGS="--tls-auth-clients optional --save ''"
             echo "REDIS_MAJOR_VERSION=${redis_major_version}" >> $GITHUB_ENV
           fi
-        
+
           invoke devenv --endpoints=all-stack
         else
           echo "Using redis CE for module tests"
           echo "REDIS_MOD_URL=redis://127.0.0.1:6379" >> $GITHUB_ENV
           invoke devenv --endpoints all
-        fi        
-        
+        fi
+
         sleep 10 # time to settle
         echo "::endgroup::"
       shell: bash
 
     - name: Run tests
       run: |
         set -e
-        
+
         run_tests() {
           local protocol=$1
           local eventloop=""
-    
+
           if [ "${{inputs.event-loop}}" == "uvloop" ]; then
             eventloop="--uvloop"
           fi
-    
+
           echo "::group::RESP${protocol} standalone tests"
           echo "REDIS_MOD_URL=${REDIS_MOD_URL}"
-        
+
           if (( $REDIS_MAJOR_VERSION < 7 )) && [ "$protocol" == "3" ]; then
             echo "Skipping module tests: Modules doesn't support RESP3 for Redis versions < 7"
             invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} $eventloop --protocol="${protocol}" --extra-markers="not redismod and not cp_integration"
-          else        
+          else
             invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} $eventloop --protocol="${protocol}"
           fi
-        
+
           echo "::endgroup::"
-    
+
           if [ "$protocol" == "2" ] || [ "${{inputs.parser-backend}}" != 'hiredis' ]; then
             echo "::group::RESP${protocol} cluster tests"
             invoke cluster-tests $eventloop --protocol=${protocol}
             echo "::endgroup::"
           fi
         }
-  
+
         run_tests 2 "${{inputs.event-loop}}"
         run_tests 3 "${{inputs.event-loop}}"
       shell: bash

.github/workflows/integration.yaml

@@ -27,8 +27,7 @@ env:
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   # this speeds up coverage with Python 3.12: https://github.com/nedbat/coveragepy/issues/1665
   COVERAGE_CORE: sysmon
-  REDIS_IMAGE: redis:latest
-  REDIS_STACK_IMAGE: redis/redis-stack-server:latest
+  CURRENT_CLIENT_LIBS_TEST_STACK_IMAGE_TAG: 'rs-7.4.0-v2'
   CURRENT_REDIS_VERSION: '7.4.2'
 
 jobs:
@@ -180,9 +179,8 @@ jobs:
           python-version: 3.9
       - name: Run installed unit tests
         env:
-          REDIS_VERSION: ${{ env.CURRENT_REDIS_VERSION }}
-          REDIS_IMAGE: "redis:${{ env.CURRENT_REDIS_VERSION }}"
-          CLIENT_LIBS_TEST_IMAGE: "redislabs/client-libs-test:${{ env.CURRENT_REDIS_VERSION }}"
+          CLIENT_LIBS_TEST_IMAGE_TAG: ${{ env.CURRENT_REDIS_VERSION }}
+          CLIENT_LIBS_TEST_STACK_IMAGE_TAG: ${{ env.CURRENT_CLIENT_LIBS_TEST_STACK_IMAGE_TAG }}
         run: |
           bash .github/workflows/install_and_test.sh ${{ matrix.extension }}
 

.gitignore

@@ -23,3 +23,6 @@ docker/stunnel/keys
 /dockers/*/tls/*
 /dockers/standalone/
 /dockers/cluster/
+/dockers/replica/
+/dockers/sentinel/
+/dockers/redis-stack/

docker-compose.yml

@@ -1,9 +1,14 @@
 ---
+x-client-libs-stack-image: &client-libs-stack-image
+  image: "redislabs/client-libs-test:${CLIENT_LIBS_TEST_STACK_IMAGE_TAG:-rs-7.4.0-v2}"
+
+x-client-libs-image: &client-libs-image
+  image: "redislabs/client-libs-test:${CLIENT_LIBS_TEST_IMAGE_TAG:-7.4.2}"
 
 services:
 
   redis:
-    image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:7.4.1}
+    <<: *client-libs-image
     container_name: redis-standalone
     environment:
       - TLS_ENABLED=yes
@@ -24,20 +29,26 @@ services:
       - all
 
   replica:
-    image: ${REDIS_IMAGE:-redis:7.4.1}
+    <<: *client-libs-image
     container_name: redis-replica
     depends_on:
       - redis
-    command: redis-server --replicaof redis 6379 --protected-mode no --save ""
+    environment:
+      - TLS_ENABLED=no
+      - REDIS_CLUSTER=no
+      - PORT=6380
+    command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --replicaof redis 6379 --protected-mode no --save ""}
     ports:
-      - 6380:6379
+      - 6380:6380
+    volumes:
+      - "./dockers/replica:/redis/work"
     profiles:
       - replica
       - all-stack
       - all
 
   cluster:
-    image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:7.4.1}
+    <<: *client-libs-image
     container_name: redis-cluster
     environment:
       - REDIS_CLUSTER=yes
@@ -58,57 +69,38 @@ services:
       - all
 
   sentinel:
-    image: ${REDIS_IMAGE:-redis:7.4.1}
+    <<: *client-libs-image
     container_name: redis-sentinel
     depends_on:
       - redis
-    entrypoint: "redis-sentinel /redis.conf --port 26379"
+    environment:
+      - REDIS_CLUSTER=no
+      - NODES=3
+      - PORT=26379
+    command: ${REDIS_EXTRA_ARGS:---sentinel}
     ports:
       - 26379:26379
-    volumes:
-      - "./dockers/sentinel.conf:/redis.conf"
-    profiles:
-      - sentinel
-      - all-stack
-      - all
-
-  sentinel2:
-    image: ${REDIS_IMAGE:-redis:7.4.1}
-    container_name: redis-sentinel2
-    depends_on:
-      - redis
-    entrypoint: "redis-sentinel /redis.conf --port 26380"
-    ports:
       - 26380:26380
-    volumes:
-      - "./dockers/sentinel.conf:/redis.conf"
-    profiles:
-      - sentinel
-      - all-stack
-      - all
-
-  sentinel3:
-    image: ${REDIS_IMAGE:-redis:7.4.1}
-    container_name: redis-sentinel3
-    depends_on:
-      - redis
-    entrypoint: "redis-sentinel /redis.conf --port 26381"
-    ports:
       - 26381:26381
     volumes:
-      - "./dockers/sentinel.conf:/redis.conf"
+      - "./dockers/sentinel.conf:/redis/config-default/redis.conf"
+      - "./dockers/sentinel:/redis/work"
     profiles:
       - sentinel
       - all-stack
       - all
 
   redis-stack:
-    image: ${REDIS_STACK_IMAGE:-redis/redis-stack-server:latest}
+    <<: *client-libs-stack-image
     container_name: redis-stack
+    environment:
+      - REDIS_CLUSTER=no
+      - PORT=6379
+    command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --save ""}
     ports:
       - 6479:6379
-    environment:
-      - "REDIS_ARGS=${REDIS_STACK_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --save ''}"
+    volumes:
+      - "./dockers/redis-stack:/redis/work"
     profiles:
       - standalone
       - all-stack