chore(improvement): Implement makim and containers-sugar

.containers-sugar.yaml

@@ -0,0 +1,31 @@
+version: 1.0.0
+compose-app: docker-compose
+service-groups:
+  - name: default
+    project-name: egh-prod
+    compose-path:
+      - containers/compose-base.yaml
+      - containers/compose-prod.yaml
+    env-file: .env
+    services:
+      default: superset,airflow
+      list:
+        - name: redis
+        - name: flower
+        - name: minio
+        - name: superset
+        - name: airflow
+        - name: postgres
+
+  - name: dev
+    project-name: egh-dev
+    compose-path: 
+      - containers/compose-base.yaml
+      - containers/compose-dev.yaml
+    env-file: .env
+    services:
+      default: superset,airflow
+      list:
+        - name: superset
+        - name: airflow
+        - name: postgres

.github/workflows/main.yaml

@@ -14,7 +14,7 @@ env:
   _AIRFLOW_WWW_USER_LAST_NAME: Admin
   _AIRFLOW_WWW_USER_PASSWORD: airflow
   _AIRFLOW_WWW_USER_USERNAME: airflow
-  AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/docker/airflow
+  AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/containers/airflow
   AIRFLOW_HOME: /opt/airflow
   AIRFLOW_PORT: 8099
   AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth
@@ -72,15 +72,6 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: setup
-      run: make prepare-host
-
-    - name: Create environment variables file
-      run: |
-        export HOST_UID=$(id -u)
-        export HOST_GID=$(id -g)
-        envsubst < .env.tpl > .env
-
     - uses: conda-incubator/setup-miniconda@v2
       with:
         miniconda-version: "latest"
@@ -91,44 +82,57 @@ jobs:
         use-mamba: true
         miniforge-variant: Mambaforge
 
+    - name: Create environment variables file
+      run: |
+        export HOST_UID=$(id -u)
+        export HOST_GID=$(id -g)
+        envsubst < .env.tpl > .env
+
+    - name: setup
+      run: makim host.prepare
+
     - name: configure epigraphhub credentials
       run: |
         epigraphhub-config --name ci-epigraphhub \
           --db-uri "${POSTGRES_EPIGRAPH_USER}:${POSTGRES_EPIGRAPH_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_EPIGRAPH_DB}"
 
     - name: configure aws s3 credentials to minio
-      run: make dev-create-s3-credential
+      run: makim aws.dev-create-s3-credential
 
-    - name: build docker containers
-      run: make containers-build
+    - name: build containers-sugar
+      run: containers-sugar build
 
     - name: start services
       run: |
-        make containers-start-services SERVICES="superset airflow"
+        makim containers.start-services --services "superset,airflow"
 
     - name: wait for the services are properly working
       run: |
         docker ps
-        make containers-wait-all
+        makim containers.wait-all
 
     - name: Trigger all DAGs
       run: |
-        make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test owid 2022-10-25'
-        make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test foph 2022-10-25'
-        make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test colombia 2022-10-25'
-        make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test web_status_test 2022-10-25'
+        containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test owid 2022-10-25'
+        containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test foph 2022-10-25'
+        containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test colombia 2022-10-25'
+        containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test web_status_test 2022-10-25'
 
     - name: Test MinIO access
       run: aws --endpoint-url http://localhost:9000/ s3 ls
 
     - name: Generate logs
       if: ${{ failure() }}
       run: |
-        make containers-logs ARGS="--tail 1000" SERVICES="" > /tmp/containers-services.log
+        containers-sugar logs --extras "--tail 1000" --all > /tmp/containers-services.log
 
     - name: Archive log artifacts
       uses: actions/upload-artifact@v3
       if: ${{ failure() }}
       with:
         name: containers-services
         path: /tmp/containers-services.log
+
+    - name: Setup tmate session
+      if: "${{ failure() && (contains(github.event.pull_request.labels.*.name, 'ci:enable-debugging')) }}"
+      uses: mxschmitt/action-tmate@v3

.makim.yaml

@@ -0,0 +1,83 @@
+version: 1.0.0
+env-file: .env
+groups:
+
+  host:
+    targets:
+      prepare:
+        help: Preparing host
+        shell: bash
+        run: |
+          ./scripts/prepare-host.sh
+
+  aws:
+    targets:
+      dev-create-s3-credential:
+        help: Create AWS S3 Credentials
+        run: |
+          set -e
+          ./scripts/dev/create-s3-credential.sh
+
+
+  conda:
+    targets:
+      lock:
+        help: Configure a conda-lock
+        run: |
+          cd conda
+          rm -f conda-*.lock
+          conda-lock --conda which mamba \
+              -f prod.yaml  \
+              -p osx-64 \
+              -p linux-64 \
+              --kind explicit
+
+  containers:
+    targets:
+      wait:
+        help: Wait for a service
+        args:
+          timeout:
+            help: Define the timeout for the healthcheck
+            type: integer
+            default: 30
+          service:
+            help: Service name that would be awaited
+            type: string
+            required: True
+
+        run: |
+          timeout {{ args.timeout }} ./containers/scripts/healthcheck.sh {{ args.service }}
+
+      wait-all:
+        run: |
+          if $ENV == "dev":
+            makim containers.wait --service "postgres"
+
+          for service in ["postgres","airflow","superset","redis","flower","minio"]:
+            makim containers.wait --service @(service)
+
+      start-services:
+        help: bla
+        shell: bash
+        dependencies:
+          - targets: host.prepare
+        args:
+          services: 
+            help: Services names separated by comma
+            type: string
+            default: "airflow,superset"
+
+        run: |
+          set -e
+          if [ "${ENV}" = "dev" ]; then \
+            containers-sugar start --services "postgres"; \
+            makim containers.wait --service "postgres"; \
+          fi
+          containers-sugar start --services {{ args.services }}
+          makim containers.wait --service "airflow"
+
+      dev-prepare-db:
+        help: Prepare a database. Used for development
+        run: |
+          containers-sugar exec --extras="-T" --services superset   --cmd "bash /opt/EpiGraphHub/containers/postgresql/scripts/dev/prepare-db.sh"

README.md

@@ -166,40 +166,40 @@ $ conda activate epigraphhub
 
 ![fig2](https://user-images.githubusercontent.com/140123/165057193-c5a2b2a2-8f30-426d-9bac-8f559c01265d.png)
 
-The project provides three **compose** files, where one is the base
-definition (`containers/compose-base.yaml`) and the others are one for
-development (`containers/compose-dev.yaml`) and other for production
-(`containers/compose-prod.yaml`). Additionally, there is a **Makefile**
-file that provides `make` targets that helps to manage the docker services:
-
-- `containers-build`: builds docker images
-- `containers-start`: starts docker services
-- `containers-stop`: stops docker services
-- `containers-restart`: restarts docker services
-- `containers-logs`: shows docker
-- `containers-dev-prepare-db`: prepares the development version of the database
-- `containers-run-cron`: run the cron tasks
-- `containers-bash`: opens the docker service bash
-
-This `make` targets runs by default docker for development. For production,
+The project provides three **compose** files, where one is the base definition
+(`containers/compose-base.yaml`) and the others are one for development
+(`containers/compose-dev.yaml`) and other for production
+(`containers/compose-prod.yaml`). Additionally, there is a **.makim.yaml** file
+that provides `makim` targets that helps to define targets and dependencies and a
+**.containers.sugar.yaml** file that provides `containers-sugar` commands to
+manage the usage of containers:
+
+- `containers-sugar build`: builds docker images
+- `containers-sugar start`: starts docker services
+- `containers-sugar stop`: stops docker services
+- `containers-sugar restart`: restarts docker services
+- `containers-sugar logs`: shows docker
+- `makim containers.dev-prepare-db`: prepares the development version of the database
+
+This `makim` targets runs by default docker for development. For production,
 it is necessary to pass the argument `ENV=prod` , for example:
 
 ```bash
-make containers-build
+makim containers-sugar build
 ```
 
 A common workflow to prepare the system would be:
 
 1. Build the docker image:
 
 ```bash
-$ make containers-build
+$ containers-sugar build
 ```
 
 2. Start the services:
 
 ```bash
-$ make containers-start
+$ containers-sugar start
 ```
 
 Before moving forward, check if the services are working properly:
@@ -211,7 +211,7 @@ $ ./containers/healthcheck.sh epigraphhub
 3. Now, prepare the development database (skip it for production):
 
 ```bash
-$ make containers-prepare-db
+$ makim containers.dev-prepare-db
 ```
 
 ## AIRFLOW
@@ -231,7 +231,7 @@ For development, it is already created by the containers stack.
 In order to configure it locally, run:
 
 ```bash
-make dev-create-s3-credentials
+makim aws.dev-create-s3-credentials
 ```
 
 Note: it needs the AWS env variable on the environment (or from .env file).
@@ -291,4 +291,4 @@ Linked-in : https://www.linkedin.com/company/the-graph-network
 [linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=for-the-badge&logo=linkedin&colorB=555
 [linkedin-url]: https://www.linkedin.com/company/the-graph-network
 [discord-shield]: https://img.shields.io/discord/865572405229518878?color=%235460e6&label=DISCORD&style=for-the-badge
-[discord-url]: https://discord.gg/56thARPrnJ
+[discord-url]: https://discord.gg/56thARPrnJ

conda/base.yaml

@@ -12,3 +12,5 @@ dependencies:
   - pip
   - pip:
     - epigraphhub
+    - containers-sugar==1.4.3
+    - makim==1.6.4