Merge branch 'main' into fernst/update_airflow_210

.gitignore

@@ -1,7 +1,7 @@
 /git
 *.workspace
-dags/**/*.pyc
 /.idea
 /logs
 /db-data
+dags/*
 .DS_Store

docker/config/.env.aws

@@ -0,0 +1,2 @@
+# You can add your AWS environment variables here
+# We keep that file separate from .env.localrunner so that it can be updated by the startup script

docker/config/airflow.cfg

@@ -26,7 +26,7 @@ executor = SequentialExecutor
 # This defines the maximum number of task instances that can run concurrently in Airflow
 # regardless of scheduler count and worker count. Generally, this value is reflective of
 # the number of task instances with the running state in the metadata database.
-parallelism = 150
+parallelism = 32
 
 # The maximum number of task instances allowed to run concurrently in each DAG. To calculate
 # the number of tasks that is running concurrently for a DAG, add up the number of running
@@ -35,7 +35,7 @@ parallelism = 150
 #
 # An example scenario when this would be useful is when you want to stop a new dag with an early
 # start date from stealing all the executor slots in a cluster.
-max_active_tasks_per_dag = 150
+max_active_tasks_per_dag = 16
 
 # Are DAGs paused by default at creation
 dags_are_paused_at_creation = True
@@ -336,13 +336,13 @@ statsd_datadog_tags =
 [secrets]
 # Full class name of secrets backend to enable (will precede env vars and metastore in search path)
 # Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend
-backend =
+backend = airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend
 
 # The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class.
 # See documentation for the secrets backend you are using. JSON is expected.
 # Example for AWS Systems Manager ParameterStore:
 # ``{{"connections_prefix": "/airflow/connections", "profile_name": "default"}}``
-backend_kwargs = '{"connections_lookup_pattern":"^(?!aws_default$).*$"}'
+backend_kwargs = {"connections_prefix" : "airflow-prod/connection", "variables_prefix" : "airflow-prod/variable", "config_prefix": "airflow-prod/config", "connections_lookup_pattern":"^(?!aws_default$).*$"}
 
 [cli]
 # In what way should the cli access the API. The LocalClient will use the
@@ -457,7 +457,7 @@ reload_on_plugin_change = False
 secret_key = $SECRET_KEY
 
 # Number of workers to run the Gunicorn web server
-workers = 9
+workers = 4
 
 # The worker class gunicorn should use. Choices include
 # sync (default), eventlet, gevent
@@ -587,15 +587,15 @@ email_backend = airflow.utils.email.send_email_smtp
 # If you want airflow to send emails on retries, failure, and you want to use
 # the airflow.utils.email.send_email_smtp function, you have to configure an
 # smtp server here
-smtp_host = localhost
+smtp_host = smtp.sendgrid.net
 smtp_starttls = True
 smtp_ssl = False
 # Example: smtp_user = airflow
-# smtp_user =
+smtp_user = apikey
 # Example: smtp_password = airflow
-# smtp_password =
-smtp_port = 25
-smtp_mail_from = airflow@example.com
+smtp_password_secret = pwd/sendgrid_smtp_server
+smtp_port = 587
+smtp_mail_from = airflow@alan.com
 smtp_timeout = 30
 smtp_retry_limit = 5
 
@@ -836,7 +836,7 @@ schedule_after_task_execution = False
 
 # The scheduler can run multiple processes in parallel to parse dags.
 # This defines how many processes will run.
-parsing_processes = 7
+parsing_processes = 2
 
 # One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``.
 # The scheduler will list and sort the dag files to decide the parsing order.

docker/docker-compose-local.yml

@@ -21,6 +21,11 @@ services:
     environment:
       - LOAD_EX=n
       - EXECUTOR=Local
+      - AIRFLOW__WEBSERVER__WORKERS=1
+      - AIRFLOW__WEBSERVER__EXPOSE_CONFIG=true
+      - AIRFLOW__SENTRY__SENTRY_ON
+      - AIRFLOW__SENTRY__SENTRY_DSN
+      - SENTRY_ENVIRONMENT=dev
     logging:
       options:
         max-size: 10m
@@ -30,6 +35,7 @@ services:
       - "${PWD}/plugins:/usr/local/airflow/plugins"
       - "${PWD}/requirements:/usr/local/airflow/requirements"
       - "${PWD}/startup_script:/usr/local/airflow/startup"
+      - "${HOME}/.aws:/usr/local/airflow/.aws"
     ports:
       - "8080:8080"
     command: local-runner
@@ -40,3 +46,4 @@ services:
       retries: 3
     env_file:
       - ./config/.env.localrunner
+      - ./config/.env.aws

docker/script/bootstrap.sh

@@ -53,13 +53,13 @@ sudo mkdir mariadb_rpm
 sudo chown airflow /mariadb_rpm
 
 if [[ $(uname -p) == "aarch64" ]]; then
-  wget https://mirror.mariadb.org/yum/11.4/fedora38-aarch64/rpms/MariaDB-common-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
-  wget https://mirror.mariadb.org/yum/11.4/fedora38-aarch64/rpms/MariaDB-shared-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
-  wget https://mirror.mariadb.org/yum/11.4/fedora38-aarch64/rpms/MariaDB-devel-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
+  wget https://archive.mariadb.org/yum/11.4/fedora38-aarch64/rpms/MariaDB-common-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
+  wget https://archive.mariadb.org/yum/11.4/fedora38-aarch64/rpms/MariaDB-shared-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
+  wget https://archive.mariadb.org/yum/11.4/fedora38-aarch64/rpms/MariaDB-devel-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
 else
-  wget https://mirror.mariadb.org/yum/11.4/fedora38-amd64/rpms/MariaDB-common-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
-  wget https://mirror.mariadb.org/yum/11.4/fedora38-amd64/rpms/MariaDB-shared-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
-  wget https://mirror.mariadb.org/yum/11.4/fedora38-amd64/rpms/MariaDB-devel-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
+  wget https://archive.mariadb.org/yum/11.4/fedora38-amd64/rpms/MariaDB-common-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
+  wget https://archive.mariadb.org/yum/11.4/fedora38-amd64/rpms/MariaDB-shared-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
+  wget https://archive.mariadb.org/yum/11.4/fedora38-amd64/rpms/MariaDB-devel-11.4.2-1.fc38.$(uname -p).rpm -P /mariadb_rpm
 fi
 
 # install mariadb_devel and its dependencies

mwaa-local-env

@@ -68,7 +68,7 @@ test-requirements)
      echo "Container amazon/mwaa-local:$AIRFLOW_VERSION not built. Building locally."
      build_image
    fi
-   docker run -v $(pwd)/dags:/usr/local/airflow/dags -v $(pwd)/plugins:/usr/local/airflow/plugins -v $(pwd)/requirements:/usr/local/airflow/requirements -it amazon/mwaa-local:$AIRFLOW_VERSION test-requirements
+   docker run -v $(pwd)/dags:/usr/local/airflow/dags -v $(pwd)/plugins:/usr/local/airflow/plugins -v $(pwd)/requirements:/usr/local/airflow/requirements amazon/mwaa-local:$AIRFLOW_VERSION test-requirements
    ;;
 test-startup-script)
    BUILT_IMAGE=$(docker images -q amazon/mwaa-local:$AIRFLOW_VERSION)