Updating integration test mount-dags-gitsync to use two git-syncs

Author: Maleware

tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2

@@ -77,17 +77,22 @@ spec:
     dagsGitSync:
 {% if test_scenario['values']['access'] == 'ssh' %}
       - repo: ssh://git@github.com/stackable-airflow/dags.git
+        branch: test/git-sync-0
         credentials:
           sshPrivateKeySecretName: git-sync-ssh
+      - repo: ssh://git@github.com/stackable-airflow/dags.git
+        branch: test/git-sync-1
 {% endif %}
 {% if test_scenario['values']['access'] == 'https' %}
       - repo: https://github.com/stackable-airflow/dags
+        branch: test/git-sync-0
         credentials:
           basicAuthSecretName: git-credentials
+      - repo: https://github.com/stackable-airflow/dags
+        branch: test/git-sync-1
 {% endif %}
 {% if test_scenario['values']['executor'] == 'celery' %}
         # Just setting some values to increase the test coverage (defaults should work just fine)
-        branch: main
         wait: 5s
 {% endif %}
         gitSyncConf:

tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2

@@ -6,24 +6,24 @@ commands:
 
 {% if test_scenario['values']['executor'] == 'kubernetes' %}
   # check that the executor template configmap contains mounts and envs
-  # will expect 4 (2 from from the volume declaration + mounts to two containers, base and gitsync)
-  - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"'  | grep "test-cm-gitsync" | wc -l | grep 4
-  # will expect 2 (two containers, base and gitsync)
-  - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"'  | grep "AIRFLOW_TEST_VAR" | wc -l | grep 2
-  # will expect 1 (one container, gitsync)
+  # will expect 6 (2 from from the volume declaration + mounts to four containers, base and gitsync-0, gitsync-1 and multi-gitsync)
+  - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"'  | grep "test-cm-gitsync" | wc -l | grep 6
+  # will expect 4 (two containers, base and gitsync-0, gitsync-1 and multi-gitsync)
+  - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"'  | grep "AIRFLOW_TEST_VAR" | wc -l | grep 4
+  # will expect 2 (three container, gitsync-0, gitsync-1)
 {% if test_scenario['values']['access'] == 'https' %}
-  - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"'  | grep "GITSYNC_USERNAME" | wc -l | grep 1
-  - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"'  | grep "GITSYNC_PASSWORD" | wc -l | grep 1
+  - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"'  | grep "GITSYNC_USERNAME" | wc -l | grep 2
+  - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"'  | grep "GITSYNC_PASSWORD" | wc -l | grep 2
 {% endif %}
 {% else %}
   # check that the statefulset contains mounts and envs
-  # will expect 6 (2 from from the volume declaration + mounts to 3 containers, base and 2 gitsyncs, plus configmap restarter)
-  - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "test-cm-gitsync" | wc -l | grep 6
-  # will expect 3 (two containers, base and gitsync-1, and one initContainer gitsync-0)
-  - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "AIRFLOW_TEST_VAR" | wc -l | grep 3
-  # will expect 2 (one container, gitsync-1, and one initContainer gitsync-0)
+  # will expect 6 (2 from from the volume declaration + mounts to 5 containers, base and 2 gitsyncs plus configmap restarter)
+  - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "test-cm-gitsync" | wc -l | grep 7
+  # will expect 5 (three containers, base, gitsyinc-0 and gitsync-1, and two initContainer gitsync-0 and gitsync-1)
+  - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "AIRFLOW_TEST_VAR" | wc -l | grep 5
+  # will expect 4 (two container, gitsync-0, gitsync-1, and two initContainer gitsync-0 and gitsync-1)
 {% if test_scenario['values']['access'] == 'https' %}
-  - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_USERNAME" | wc -l | grep 2
-  - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_PASSWORD" | wc -l | grep 2
+  - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_USERNAME" | wc -l | grep 4
+  - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_PASSWORD" | wc -l | grep 4
 {% endif %}
 {% endif %}

tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py

@@ -21,7 +21,8 @@ def assert_metric(role, metric):
 ts = now.strftime("%Y-%m-%dT%H:%M:%S.%f") + now.strftime("%z")
 
 # Trigger a DAG run to create metrics
-dag_id = "sparkapp_dag"
+dag_id_0 = "sparkapp_dag_0"
+dag_id_1 = "sparkapp_dag_1"
 dag_data = {"logical_date": f"{ts}"}
 
 print(f"DAG-Data: {dag_data}")
@@ -51,13 +52,20 @@ def assert_metric(role, metric):
     "Content-Type": "application/json",
 }
 
+# Two DAGs from two different repos/branches will need to be triggered and activated
 # activate DAG
-response = requests.patch(
-    f"{rest_url}/dags/{dag_id}", headers=headers, json={"is_paused": False}
+response_0 = requests.patch(
+    f"{rest_url}/dags/{dag_id_0}", headers=headers, json={"is_paused": False}
+)
+response_1 = requests.patch(
+    f"{rest_url}/dags/{dag_id_1}", headers=headers, json={"is_paused": False}
 )
 # trigger DAG
-response = requests.post(
-    f"{rest_url}/dags/{dag_id}/dagRuns", headers=headers, json=dag_data
+response_0 = requests.post(
+    f"{rest_url}/dags/{dag_id_0}/dagRuns", headers=headers, json=dag_data
+)
+response_1 = requests.post(
+    f"{rest_url}/dags/{dag_id_1}/dagRuns", headers=headers, json=dag_data
 )
 
 # Wait for the metrics to be consumed by the statsd-exporter
@@ -69,12 +77,16 @@ def assert_metric(role, metric):
 while True:
     try:
         logging.info(f"Response code: {response.status_code}")
-        assert response.status_code == 200, "DAG run could not be triggered."
+        assert response_0.status_code == 200, f"DAG from git-sync 0 run could not be triggered because of status code: {response_0.status_code}"
+        assert response_1.status_code == 200, f"DAG from git-sync 1 run could not be triggered. {response_1.status_code}"
         # Worker is not deployed with the kubernetes executor so retrieve success metric from scheduler
         # (disable line-break flake checks)
         if (assert_metric("scheduler", "airflow_scheduler_heartbeat")) and (
             assert_metric(
-                "scheduler", "airflow_dagrun_duration_success_sparkapp_dag_count"
+                "scheduler", "airflow_dagrun_duration_success_sparkapp_dag_0_count"
+            )) and (
+            assert_metric(
+                "scheduler", "airflow_dagrun_duration_success_sparkapp_dag_1_count"
             )
         ):  # noqa: W503, W504
             break