test

Author: gene-redpanda

.tasks/ci.yml

@@ -1,12 +1,11 @@
 version: '3'
 
 vars:
-  ARTIFACT_DIR: "{{.PWD}}/artifacts"
   DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}'
 
 env:
-  ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections"
-  ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles"
+  ANSIBLE_COLLECTIONS_PATH: "{{.ANSIBLE_COLLECTIONS_PATH}}"
+  ANSIBLE_ROLES_PATH: "{{.ANSIBLE_ROLES_PATH}}"
 
 tasks:
   default:

.tasks/cloud.yml

@@ -8,7 +8,6 @@ vars:
   ALLOW_FORCE_DESTROY: '{{.ALLOW_FORCE_DESTROY | default "true"}}'
   VPC_ID: '{{.VPC_ID | default ""}}'
   DISTRO: '{{.DISTRO | default "ubuntu-focal"}}'
-  ARTIFACT_DIR: "{{.PWD}}/artifacts"
   PUBLIC_KEY: "{{.ARTIFACT_DIR}}/testkey.pub"
   ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini"
   MACHINE_ARCH: '{{.MACHINE_ARCH | default "x86_64"}}'

.tasks/connect.yml

@@ -1,7 +1,6 @@
 version: '3'
 
 vars:
-  ARTIFACT_DIR: "{{.PWD}}/artifacts"
   DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}'
   RPM_VERSION: '{{.RPM_VERSION | default "v1.0.0-7ae9d19"}}'
   SERVER_DIR: '{{.SERVER_DIR | default "/tmp"}}'
@@ -13,18 +12,18 @@ vars:
   PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey"
   ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini"
   CLIENT_KEY:
-    sh: echo "${CLIENT_KEY:-{{.PWD}}/ansible/tls/clients/client.key}"
+    sh: echo "${CLIENT_KEY:-{{.ROOT_DIR}}/ansible/tls/clients/client.key}"
   CLIENT_CERT:
-    sh: echo "${CLIENT_CERT:-{{.PWD}}/ansible/tls/ca/ca.crt}"
+    sh: echo "${CLIENT_CERT:-{{.ROOT_DIR}}/ansible/tls/ca/ca.crt}"
   EXTRA_INVENTORY: "{{.ARTIFACT_DIR}}/hosts2_{{.DEPLOYMENT_ID}}.ini"
   IS_USING_UNSTABLE: '{{.IS_USING_UNSTABLE | default "false"}}'
 
 env:
   OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES"
   ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log"
   ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}"
-  ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections"
-  ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles"
+  ANSIBLE_COLLECTIONS_PATH: "{{.ANSIBLE_COLLECTIONS_PATH}}"
+  ANSIBLE_ROLES_PATH: "{{.ANSIBLE_ROLES_PATH}}"
 
 tasks:
   default:
@@ -116,7 +115,7 @@ tasks:
       - |
         CONNECT_TARGET=$(awk '/^\[connect\]/{f=1; next} /^$/{f=0} f{print $1}' "{{.HOSTS_FILE}}" | head -n1)
         echo "Testing Connect API at https://$CONNECT_TARGET:8083..."
-        curl -vvvvv -k --cert {{.PWD}}/ansible/tls/clients/client.crt --key {{.CLIENT_KEY}} --cacert {{.CLIENT_CERT}} -X GET https://$CONNECT_TARGET:8083/connectors
+        curl -vvvvv -k --cert {{.ROOT_DIR}}/ansible/tls/clients/client.crt --key {{.CLIENT_KEY}} --cacert {{.CLIENT_CERT}} -X GET https://$CONNECT_TARGET:8083/connectors
 
   test-prometheus-exporter:
     desc: "Test Prometheus metrics endpoint"
@@ -125,7 +124,7 @@ tasks:
       - |
         PROMETHEUS_EXPORTER_TARGET=$(awk '/^\[connect\]/{f=1; next} /^$/{f=0} f{print $1}' "{{.HOSTS_FILE}}" | head -n1)
         echo "Testing Prometheus exporter at https://$PROMETHEUS_EXPORTER_TARGET:9404..."
-        curl -vvvvv -k --cert {{.PWD}}/ansible/tls/clients/client.crt --key {{.PWD}}/ansible/tls/clients/client.key --cacert {{.PWD}}/ansible/tls/ca/ca.crt -X GET https://$PROMETHEUS_EXPORTER_TARGET:9404/metrics
+        curl -vvvvv -k --cert {{.ROOT_DIR}}/ansible/tls/clients/client.crt --key {{.ROOT_DIR}}/ansible/tls/clients/client.key --cacert {{.ROOT_DIR}}/ansible/tls/ca/ca.crt -X GET https://$PROMETHEUS_EXPORTER_TARGET:9404/metrics
 
   mirror-connector:
     desc: "Create mirror source connector (plaintext)"

.tasks/console.yml

@@ -1,7 +1,6 @@
 version: '3'
 
 vars:
-  ARTIFACT_DIR: "{{.PWD}}/artifacts"
   DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}'
   PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey"
   ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini"
@@ -11,8 +10,8 @@ env:
   OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES"
   ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log"
   ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}"
-  ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections"
-  ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles"
+  ANSIBLE_COLLECTIONS_PATH: "{{.ANSIBLE_COLLECTIONS_PATH}}"
+  ANSIBLE_ROLES_PATH: "{{.ANSIBLE_ROLES_PATH}}"
 
 tasks:
   default:

.tasks/mirror.yml

@@ -1,7 +1,6 @@
 version: '3'
 
 vars:
-  ARTIFACT_DIR: "{{.PWD}}/artifacts"
   DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}'
   NUM_NODES: '{{.NUM_NODES | default "3"}}'
   ENABLE_MONITORING: '{{.ENABLE_MONITORING | default "true"}}'

.tasks/monitor.yml

@@ -1,7 +1,6 @@
 version: '3'
 
 vars:
-  ARTIFACT_DIR: "{{.PWD}}/artifacts"
   DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}'
   PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey"
   ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini"
@@ -12,8 +11,8 @@ env:
   OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES"
   ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log"
   ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}"
-  ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections"
-  ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles"
+  ANSIBLE_COLLECTIONS_PATH: "{{.ANSIBLE_COLLECTIONS_PATH}}"
+  ANSIBLE_ROLES_PATH: "{{.ANSIBLE_ROLES_PATH}}"
 
 tasks:
   default:

.tasks/redpanda.yml

@@ -1,7 +1,6 @@
 version: '3'
 
 vars:
-  ARTIFACT_DIR: "{{.PWD}}/artifacts"
   DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}'
   PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey"
   ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini"
@@ -16,8 +15,8 @@ env:
   OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES"
   ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log"
   ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}"
-  ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections"
-  ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles"
+  ANSIBLE_COLLECTIONS_PATH: "{{.ANSIBLE_COLLECTIONS_PATH}}"
+  ANSIBLE_ROLES_PATH: "{{.ANSIBLE_ROLES_PATH}}"
 
 tasks:
   default:

.tasks/test.yml

@@ -1,14 +1,13 @@
 version: '3'
 
 vars:
-  ARTIFACT_DIR: "{{.PWD}}/artifacts"
   DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}'
   RPK_PATH: "{{.ARTIFACT_DIR}}/bin/rpk"
   HOSTS_FILE: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini"
   TEST_TOPIC_NAME: '{{.TEST_TOPIC_NAME | default "testtopic"}}'
   PARTITION_COUNT: '{{.PARTITION_COUNT | default "3"}}'
   CA_CRT:
-    sh: echo "${CA_CRT:-{{.PWD}}/ansible/tls/ca/ca.crt}"
+    sh: echo "${CA_CRT:-{{.ROOT_DIR}}/ansible/tls/ca/ca.crt}"
   AWS_DEFAULT_REGION: '{{.AWS_DEFAULT_REGION | default "us-west-2"}}'
   BUCKET_NAME:
     sh: echo "$(echo {{.DEPLOYMENT_ID}} | sed 's/_/-/g')-bucket"

.tasks/tools.yml

@@ -1,16 +1,10 @@
 version: '3'
 
 vars:
-  ARTIFACT_DIR: "{{.PWD}}/artifacts"
-  ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections"
-  ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles"
-  MAC_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-darwin-amd64.zip"
-  LINUX_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-amd64.zip"
-  SSH_EMAIL: '{{.SSH_EMAIL | default "[email protected]"}}'
   CLIENT_NAME: '{{.CLIENT_NAME | default "client"}}'
-  CLIENT_DIR: "{{.PWD}}/ansible/tls/clients"
-  CA_DIR: "{{.PWD}}/ansible/tls/ca"
-  CERT_DIR: "{{.PWD}}/ansible/tls/certs"
+  CLIENT_DIR: "{{.ROOT_DIR}}/ansible/tls/clients"
+  CA_DIR: "{{.ROOT_DIR}}/ansible/tls/ca"
+  CERT_DIR: "{{.ROOT_DIR}}/ansible/tls/certs"
 
 tasks:
   default:
@@ -30,23 +24,32 @@ tasks:
   _ansible-collections:
     internal: true
     desc: "Install Ansible collections from requirements.yml"
+    sources:
+      - "{{.ROOT_DIR}}/requirements.yml"
+    generates:
+      - "{{.ANSIBLE_COLLECTIONS_PATH}}/ansible_collections"
     cmds:
       - mkdir -p {{.ANSIBLE_COLLECTIONS_PATH}}
-      - ansible-galaxy collection install -r {{.PWD}}/requirements.yml --force -p {{.ANSIBLE_COLLECTIONS_PATH}}
-    status:
-      - test -d {{.ANSIBLE_COLLECTIONS_PATH}}/ansible_collections
+      - ansible-galaxy collection install -r {{.ROOT_DIR}}/requirements.yml --force -p {{.ANSIBLE_COLLECTIONS_PATH}}
 
   _ansible-roles:
     internal: true
     desc: "Install Ansible roles from requirements.yml"
+    sources:
+      - "{{.ROOT_DIR}}/requirements.yml"
+    generates:
+      - "{{.ANSIBLE_ROLES_PATH}}"
     cmds:
       - mkdir -p {{.ANSIBLE_ROLES_PATH}}
-      - ansible-galaxy role install -r {{.PWD}}/requirements.yml --force -p {{.ANSIBLE_ROLES_PATH}}
-    status:
-      - test -d {{.ANSIBLE_ROLES_PATH}}
+      - ansible-galaxy role install -r {{.ROOT_DIR}}/requirements.yml --force -p {{.ANSIBLE_ROLES_PATH}}
 
   rpk:
     desc: "Install rpk CLI tool"
+    vars:
+      MAC_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-darwin-amd64.zip"
+      LINUX_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-amd64.zip"
+    generates:
+      - "{{.ARTIFACT_DIR}}/bin/rpk"
     cmds:
       - mkdir -p {{.ARTIFACT_DIR}}/tmp
       - mkdir -p {{.ARTIFACT_DIR}}/bin
@@ -61,21 +64,17 @@ tasks:
       - chmod 755 {{.ARTIFACT_DIR}}/bin/rpk
       - rm {{.ARTIFACT_DIR}}/tmp/rpk.zip
       - echo "rpk installed to {{.ARTIFACT_DIR}}/bin/rpk"
-    status:
-      - test -f {{.ARTIFACT_DIR}}/bin/rpk
 
   ssh-keys:
     desc: "Generate SSH keypair for deployment"
+    vars:
+      SSH_EMAIL: '{{.SSH_EMAIL | default "[email protected]"}}'
+    generates:
+      - "{{.ARTIFACT_DIR}}/testkey"
+      - "{{.ARTIFACT_DIR}}/testkey.pub"
     cmds:
-      - |
-        if [ ! -f artifacts/testkey ]; then
-          printf 'y\n' | ssh-keygen -t rsa -b 4096 -C "{{.SSH_EMAIL}}" -N "" -f artifacts/testkey && chmod 0700 artifacts/testkey
-          echo "SSH keypair generated"
-        else
-          echo "SSH keypair already exists"
-        fi
-    status:
-      - test -f artifacts/testkey
+      - printf 'y\n' | ssh-keygen -t rsa -b 4096 -C "{{.SSH_EMAIL}}" -N "" -f {{.ARTIFACT_DIR}}/testkey && chmod 0700 {{.ARTIFACT_DIR}}/testkey
+      - echo "SSH keypair generated"
 
   tls-certs:
     desc: "Generate client TLS certificates"
@@ -88,31 +87,37 @@ tasks:
 
   _cert-client-dir:
     internal: true
+    generates:
+      - "{{.CLIENT_DIR}}"
     cmds:
       - mkdir -p {{.CLIENT_DIR}}
-    status:
-      - test -d {{.CLIENT_DIR}}
 
   _cert-client-key:
     internal: true
+    generates:
+      - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key"
     cmds:
       - openssl genrsa -out {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key 2048
-    status:
-      - test -f {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key
 
   _cert-client-csr:
     internal: true
+    sources:
+      - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key"
+    generates:
+      - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr"
     cmds:
       - openssl req -new -key {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key -out {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr -subj "/CN={{.CLIENT_NAME}}"
-    status:
-      - test -f {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr
 
   _cert-client-crt:
     internal: true
+    sources:
+      - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr"
+      - "{{.CA_DIR}}/ca.crt"
+      - "{{.CA_DIR}}/ca.key"
+    generates:
+      - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.crt"
     cmds:
       - openssl x509 -req -in {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr -CA {{.CA_DIR}}/ca.crt -CAkey {{.CA_DIR}}/ca.key -CAcreateserial -out {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.crt -days 365 -sha256
-    status:
-      - test -f {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.crt
 
   clean-certs:
     desc: "Remove all TLS certificates"

Taskfile.yml

@@ -3,6 +3,11 @@ version: '3'
 
 dotenv: ['.env']
 
+vars:
+  ARTIFACT_DIR: "{{.ROOT_DIR}}/artifacts"
+  ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections"
+  ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles"
+
 includes:
   tools: .tasks/tools.yml
   cloud: .tasks/cloud.yml