wip: get logging tests to work post-merge

Author: adwk67

rust/operator-binary/src/airflow_controller.rs

@@ -981,6 +981,7 @@ fn build_server_rolegroup_statefulset(
             authentication_config,
             authorization_config,
             git_sync_resources,
+            &rolegroup_ref.role_group,
         )
         .context(BuildStatefulsetEnvVarsSnafu)?,
     );
@@ -1171,7 +1172,10 @@ fn build_server_rolegroup_statefulset(
             match_labels: Some(statefulset_match_labels.into()),
             ..LabelSelector::default()
         },
-        service_name: Some(rolegroup_ref.object_name()),
+        service_name: Some(format!(
+            "{name}-metrics",
+            name = rolegroup_ref.object_name()
+        )),
         template: pod_template,
         volume_claim_templates: pvcs,
         ..StatefulSetSpec::default()

rust/operator-binary/src/crd/mod.rs

@@ -568,7 +568,6 @@ impl AirflowRole {
                     command.extend(vec![
                         "prepare_signal_handlers".to_string(),
                         format!("containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &"),
-                        "airflow db migrate".to_string(),
                         "airflow api-server &".to_string(),
                     ]);
                 }

rust/operator-binary/src/env_vars.rs

@@ -75,6 +75,7 @@ pub enum Error {
 /// Return environment variables to be applied to the statefulsets for the scheduler, webserver (and worker,
 /// for clusters utilizing `celeryExecutor`: for clusters using `kubernetesExecutor` a different set will be
 /// used which is defined in [`build_airflow_template_envs`]).
+#[allow(clippy::too_many_arguments)]
 pub fn build_airflow_statefulset_envs(
     airflow: &v1alpha1::AirflowCluster,
     airflow_role: &AirflowRole,
@@ -83,11 +84,12 @@ pub fn build_airflow_statefulset_envs(
     auth_config: &AirflowClientAuthenticationDetailsResolved,
     authorization_config: &AirflowAuthorizationResolved,
     git_sync_resources: &git_sync::v1alpha1::GitSyncResources,
+    rolegroup: &String,
 ) -> Result<Vec<EnvVar>, Error> {
     let mut env: BTreeMap<String, EnvVar> = BTreeMap::new();
 
     env.extend(static_envs(git_sync_resources));
-    env.extend(execution_server_env_vars(airflow));
+    env.extend(execution_server_env_vars(airflow, rolegroup));
 
     // environment variables
     let env_vars = rolegroup_config.get(&PropertyNameKind::Env);
@@ -312,6 +314,8 @@ fn static_envs(
         ..Default::default()
     });
 
+    // Basic auth is only relevant to 2.x and can be removed once
+    // that version is no longer supported.
     env.insert(AIRFLOW_API_AUTH_BACKENDS.into(), EnvVar {
         name: AIRFLOW_API_AUTH_BACKENDS.into(),
         value: Some("airflow.api.auth.backend.basic_auth, airflow.api.auth.backend.session".into()),
@@ -374,7 +378,16 @@ pub fn build_airflow_template_envs(
     });
 
     env.extend(static_envs(git_sync_resources));
-    env.extend(execution_server_env_vars(airflow));
+
+    // It does not appear to be possible for kubernetesExecutors to work with
+    // multiple webserver rolegroups. For the celery case, each executor sets
+    // the execution server from the associated rolegroup, but for kubernetes
+    // workers this is not possible.
+    if let Some(webserver_role) = airflow.spec.webservers.as_ref() {
+        if let Some(rolegroup) = webserver_role.role_groups.iter().next() {
+            env.extend(execution_server_env_vars(airflow, rolegroup.0));
+        }
+    }
 
     // _STACKABLE_POST_HOOK will contain a command to create a shutdown hook that will be
     // evaluated in the wrapper for each stackable spark container: this is necessary for pods
@@ -453,31 +466,28 @@ fn authorization_env_vars(authorization_config: &AirflowAuthorizationResolved) -
     env
 }
 
-fn execution_server_env_vars(airflow: &v1alpha1::AirflowCluster) -> BTreeMap<String, EnvVar> {
+fn execution_server_env_vars(
+    airflow: &v1alpha1::AirflowCluster,
+    rolegroup: &String,
+) -> BTreeMap<String, EnvVar> {
     let mut env: BTreeMap<String, EnvVar> = BTreeMap::new();
 
-    if let Some(webserver_role) = airflow.spec.webservers.as_ref() {
-        if let Some(rolegroup) = webserver_role.role_groups.iter().next() {
-            if let Some(name) = airflow.metadata.name.as_ref() {
-                let webserver = format!(
-                    "{name}-webserver-{rolegroup}",
-                    name = name,
-                    rolegroup = rolegroup.0
-                );
-                tracing::info!("Webserver set [{webserver}]");
-
-                env.insert("AIRFLOW__CORE__EXECUTION_API_SERVER_URL".into(), EnvVar {
-                    name: "AIRFLOW__CORE__EXECUTION_API_SERVER_URL".into(),
-                    value: Some(format!("http://{webserver}:8080/execution/")),
-                    ..Default::default()
-                });
-                env.insert("AIRFLOW__CORE__BASE_URL".into(), EnvVar {
-                    name: "AIRFLOW__CORE__BASE_URL".into(),
-                    value: Some(format!("http://{webserver}:8080/")),
-                    ..Default::default()
-                });
-            }
-        }
+    if let Some(name) = airflow.metadata.name.as_ref() {
+        let webserver = format!("{name}-webserver-{rolegroup}-metrics",);
+        tracing::debug!("Webserver set [{webserver}]");
+
+        // These settings are new in 3.x and will have no affect with earlier versions.
+        env.insert("AIRFLOW__CORE__EXECUTION_API_SERVER_URL".into(), EnvVar {
+            name: "AIRFLOW__CORE__EXECUTION_API_SERVER_URL".into(),
+            value: Some(format!("http://{webserver}:8080/execution/")),
+            ..Default::default()
+        });
+        env.insert("AIRFLOW__CORE__BASE_URL".into(), EnvVar {
+            name: "AIRFLOW__CORE__BASE_URL".into(),
+            value: Some(format!("http://{webserver}:8080/")),
+            ..Default::default()
+        });
     }
+
     env
 }

…tes/kuttl/external-access/40-assert.yaml …/kuttl/external-access/40-assert.yaml.j2tests/templates/kuttl/external-access/40-assert.yaml renamed to tests/templates/kuttl/external-access/40-assert.yaml.j2 tests/templates/kuttl/external-access/40-assert.yaml renamed to tests/templates/kuttl/external-access/40-assert.yaml.j2

@@ -43,18 +43,6 @@ status:
 ---
 apiVersion: apps/v1
 kind: StatefulSet
-metadata:
-  name: airflow-worker-default
-spec:
-  template:
-    spec:
-      terminationGracePeriodSeconds: 300
-status:
-  readyReplicas: 2
-  replicas: 2
----
-apiVersion: apps/v1
-kind: StatefulSet
 metadata:
   name: airflow-scheduler-default
 spec:
@@ -76,15 +64,6 @@ status:
 ---
 apiVersion: policy/v1
 kind: PodDisruptionBudget
-metadata:
-  name: airflow-worker
-status:
-  expectedPods: 2
-  currentHealthy: 2
-  disruptionsAllowed: 1
----
-apiVersion: policy/v1
-kind: PodDisruptionBudget
 metadata:
   name: airflow-scheduler
 status:
@@ -112,3 +91,27 @@ metadata:
   name: airflow-webserver-external-unstable
 spec:
   type: NodePort # external-unstable
+
+{% if test_scenario['values']['executor'] == 'celery' %}
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: airflow-worker-default
+spec:
+  template:
+    spec:
+      terminationGracePeriodSeconds: 300
+status:
+  readyReplicas: 2
+  replicas: 2
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: airflow-worker
+status:
+  expectedPods: 2
+  currentHealthy: 2
+  disruptionsAllowed: 1
+{% endif %}

tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2

@@ -45,10 +45,21 @@ spec:
         replicas: 1
         config:
           listenerClass: test-cluster-internal-$NAMESPACE
+{% if test_scenario['values']['executor'] == 'celery' %}
   celeryExecutors:
     roleGroups:
       default:
         replicas: 2
+{% elif test_scenario['values']['executor'] == 'kubernetes' %}
+  kubernetesExecutors:
+    config:
+      resources:
+        cpu:
+          min: 100m
+          max: 500m
+        memory:
+          limit: 1Gi
+{% endif %}
   schedulers:
     roleGroups:
       default:

tests/templates/kuttl/ldap/70-install-airflow-python.yaml

@@ -18,5 +18,6 @@ spec:
       containers:
         - name: test-airflow-python
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+          imagePullPolicy: IfNotPresent
           stdin: true
           tty: true

tests/templates/kuttl/logging/50-install-airflow-python.yaml

@@ -18,5 +18,6 @@ spec:
       containers:
         - name: test-airflow-python
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+          imagePullPolicy: IfNotPresent
           stdin: true
           tty: true

tests/templates/kuttl/mount-dags-configmap/40-install-airflow-python.yaml

@@ -18,5 +18,6 @@ spec:
       containers:
         - name: test-airflow-python
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+          imagePullPolicy: IfNotPresent
           stdin: true
           tty: true

tests/templates/kuttl/mount-dags-gitsync/40-install-airflow-python.yaml

@@ -18,5 +18,6 @@ spec:
       containers:
         - name: test-airflow-python
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+          imagePullPolicy: IfNotPresent
           stdin: true
           tty: true

tests/templates/kuttl/oidc/50-install-test-container.yaml.j2

@@ -56,6 +56,7 @@ spec:
       containers:
         - name: python
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+          imagePullPolicy: IfNotPresent
           stdin: true
           tty: true
           resources: