Update new celery implementation #110
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| permissions: | |
| contents: write | |
| pages: write | |
| id-token: write | |
| env: | |
| AWS_REGION: eu-west-3 # ajustez si nécessaire | |
| DOCKER_BUILDKIT: 1 | |
| jobs: | |
| # ----------------------------------------------------------------------- | |
| # 1. TESTS | |
| # ----------------------------------------------------------------------- | |
| # setup-and-test: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - name: Checkout Repository | |
| # uses: actions/checkout@v4 | |
| # - name: Set up Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: "3.10" | |
| # - name: Set up Docker | |
| # uses: docker/setup-buildx-action@v3 | |
| # - name: Free up disk space | |
| # run: | | |
| # sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL | |
| # sudo apt-get clean | |
| # sudo docker system prune -af | |
| # sudo docker volume prune -f | |
| # rm -rf ~/.cache/pip | |
| # df -h | |
| # - name: Install Docker Compose | |
| # run: | | |
| # COMPOSE_VERSION=v2.24.5 | |
| # sudo curl -L "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| # sudo chmod +x /usr/local/bin/docker-compose | |
| # docker-compose --version | |
| # - name: Create .cv_deploy.env | |
| # shell: bash | |
| # run: | | |
| # { | |
| # echo "MODEL_PATH=${{ secrets.MODEL_PATH }}" | |
| # echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" | |
| # echo "REDIS_URL=${{ secrets.REDIS_URL }}" | |
| # echo "REDIS_HOST=${{ secrets.REDIS_HOST }}" | |
| # echo "REDIS_PORT=${{ secrets.REDIS_PORT }}" | |
| # echo "REDIS_PASSWORD=${{ secrets.REDIS_PASSWORD }}" | |
| # echo "POSTGRES_URL=${{ secrets.POSTGRES_URL }}" | |
| # echo "GEE_SERVICE_ACCOUNT_EMAIL=${{ secrets.GEE_SERVICE_ACCOUNT_EMAIL }}" | |
| # echo "GEE_SERVICE_ACCOUNT_KEY_FILE=/tmp/gee_key.json" | |
| # echo "SKIP_GEE_INIT=true" | |
| # echo "BLOB_ACCOUNT_URL=${{ secrets.BLOB_ACCOUNT_URL }}" | |
| # echo "BLOB_CONTAINER_NAME=${{ secrets.BLOB_CONTAINER_NAME }}" | |
| # echo "SERVER_URL=${{ secrets.SERVER_URL }}" | |
| # echo "BLOB_SAS_TOKEN=${{ secrets.BLOB_SAS_TOKEN }}" | |
| # } >> .cv_deploy.env | |
| # - name: Create coverage directory | |
| # run: mkdir -p coverage | |
| # - name: Run Tests | |
| # run: | | |
| # export DOCKER_BUILDKIT=1 | |
| # export PIP_NO_CACHE_DIR=1 | |
| # export DOCKER_DEFAULT_PLATFORM=linux/amd64 | |
| # docker-compose -f _ci_pipeline.yml up --build -d | |
| # docker-compose -f _ci_pipeline.yml run --rm -v ${{ github.workspace }}/coverage:/app/coverage testing | |
| # - name: Check for coverage file | |
| # run: | | |
| # test -f ./coverage/coverage.xml || { echo "Coverage file not found!"; exit 1; } | |
| # - name: Upload coverage reports to Codecov | |
| # uses: codecov/codecov-action@v4 | |
| # with: | |
| # files: ./coverage/coverage.xml | |
| # env: | |
| # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| # - name: Tear down testing environment | |
| # if: always() | |
| # run: docker-compose -f _ci_pipeline.yml down --volumes --remove-orphans | |
| # - name: Prune Docker resources (testing) | |
| # if: always() | |
| # run: docker system prune -af | |
| # # ----------------------------------------------------------------------- | |
| # 2. DEPLOY | |
| # ----------------------------------------------------------------------- | |
| deploy: | |
| # needs: [setup-and-test] | |
| if: github.ref == 'refs/heads/main' && github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| # ------------------------------------------------------------------- | |
| # Build source configuration JSON (includes env‑vars) | |
| # ------------------------------------------------------------------- | |
| - name: Build App Runner source config | |
| shell: bash | |
| run: | | |
| cat > source.json <<'EOF' | |
| { | |
| "AuthenticationConfiguration": { | |
| "ConnectionArn": "${{ secrets.AWS_CONNECTION_ARN }}" | |
| }, | |
| "AutoDeploymentsEnabled": false, | |
| "CodeRepository": { | |
| "RepositoryUrl": "https://github.com/${{ github.repository }}", | |
| "SourceCodeVersion": { "Type": "BRANCH", "Value": "main" }, | |
| "CodeConfiguration": { | |
| "ConfigurationSource": "API", | |
| "CodeConfigurationValues": { | |
| "Runtime": "PYTHON_3", | |
| "BuildCommand": "set -ex && mkdir -p cv_model local_uploads && python -m pip install --no-cache-dir --upgrade pip setuptools wheel && python -m pip install --no-cache-dir -r requirements.prod.txt && echo 'Build succeeded!'", | |
| "StartCommand": "python -m uvicorn app.main:app --host 0.0.0.0 --port 8080 --log-level debug", | |
| "Port": "8080", | |
| "RuntimeEnvironmentVariables": { | |
| "MODEL_PATH": "${{ secrets.MODEL_PATH }}", | |
| "OPENAI_API_KEY": "${{ secrets.OPENAI_API_KEY }}", | |
| "REDIS_URL": "${{ secrets.REDIS_URL }}", | |
| "REDIS_HOST": "${{ secrets.REDIS_HOST }}", | |
| "REDIS_PORT": "${{ secrets.REDIS_PORT }}", | |
| "REDIS_PASSWORD": "${{ secrets.REDIS_PASSWORD }}", | |
| "POSTGRES_URL": "${{ secrets.POSTGRES_URL }}", | |
| "GEE_SERVICE_ACCOUNT_EMAIL": "${{ secrets.GEE_SERVICE_ACCOUNT_EMAIL }}", | |
| "GEE_SERVICE_ACCOUNT_KEY_FILE": "/tmp/gee_key.json", | |
| "BLOB_ACCOUNT_URL": "${{ secrets.BLOB_ACCOUNT_URL }}", | |
| "BLOB_CONTAINER_NAME": "${{ secrets.BLOB_CONTAINER_NAME }}", | |
| "SERVER_URL": "${{ secrets.SERVER_URL }}", | |
| "BLOB_SAS_TOKEN": "${{ secrets.BLOB_SAS_TOKEN }}", | |
| "INSTALL_ADDITIONAL_PACKAGES": "true", | |
| "DEBUG": "true" | |
| }, | |
| "RuntimeEnvironmentSecrets": { | |
| "GEE_KEY_JSON": "${{ secrets.GEE_SECRET_ARN }}" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| EOF | |
| # ------------------------------------------------------------------- | |
| # Create or update the service | |
| # ------------------------------------------------------------------- | |
| - name: Deploy / update App Runner service | |
| shell: bash | |
| run: | | |
| SERVICE_ID=$(aws apprunner list-services \ | |
| --query "ServiceSummaryList[?ServiceName=='map-action-api'].ServiceId" \ | |
| --output text) | |
| if [ -z "$SERVICE_ID" ] || [ "$SERVICE_ID" == "None" ]; then | |
| echo "Creating new App Runner service…" | |
| aws apprunner create-service \ | |
| --service-name map-action-api \ | |
| --source-configuration file://source.json \ | |
| --instance-configuration Cpu=1\ vCPU,Memory=2\ GB,InstanceRoleArn=${{ secrets.APP_RUNNER_INSTANCE_ROLE_ARN }} | |
| else | |
| echo "Updating existing App Runner service…" | |
| # Get the full ARN for the service | |
| SERVICE_ARN=$(aws apprunner list-services \ | |
| --query "ServiceSummaryList[?ServiceName=='map-action-api'].ServiceArn" \ | |
| --output text) | |
| aws apprunner update-service \ | |
| --service-arn "$SERVICE_ARN" \ | |
| --source-configuration file://source.json \ | |
| --instance-configuration Cpu=1\ vCPU,Memory=2\ GB,InstanceRoleArn=${{ secrets.APP_RUNNER_INSTANCE_ROLE_ARN }} | |
| fi | |
| echo "Deployment initiated successfully." | |
| # ------------------------------------------------------------------- | |
| # Build Celery workers source configuration JSON | |
| # ------------------------------------------------------------------- | |
| - name: Build App Runner source config for Celery workers | |
| shell: bash | |
| run: | | |
| cat > celery_source.json <<'EOF' | |
| { | |
| "AuthenticationConfiguration": { | |
| "ConnectionArn": "${{ secrets.AWS_CONNECTION_ARN }}" | |
| }, | |
| "AutoDeploymentsEnabled": false, | |
| "CodeRepository": { | |
| "RepositoryUrl": "https://github.com/${{ github.repository }}", | |
| "SourceCodeVersion": { "Type": "BRANCH", "Value": "main" }, | |
| "CodeConfiguration": { | |
| "ConfigurationSource": "API", | |
| "CodeConfigurationValues": { | |
| "Runtime": "PYTHON_3", | |
| "BuildCommand": "set -ex && mkdir -p cv_model local_uploads && python -m pip install --no-cache-dir --upgrade pip setuptools wheel && python -m pip install --no-cache-dir -r requirements.prod.txt && echo 'Build succeeded!'", | |
| "StartCommand": "python -m uvicorn app.services.celery.health:app --host 0.0.0.0 --port 8080 & celery -A app.services.celery.celery_config.celery_app worker --loglevel=info", | |
| "Port": "8080", | |
| "RuntimeEnvironmentVariables": { | |
| "MODEL_PATH": "${{ secrets.MODEL_PATH }}", | |
| "OPENAI_API_KEY": "${{ secrets.OPENAI_API_KEY }}", | |
| "REDIS_URL": "${{ secrets.REDIS_URL }}", | |
| "REDIS_HOST": "${{ secrets.REDIS_HOST }}", | |
| "REDIS_PORT": "${{ secrets.REDIS_PORT }}", | |
| "REDIS_PASSWORD": "${{ secrets.REDIS_PASSWORD }}", | |
| "POSTGRES_URL": "${{ secrets.POSTGRES_URL }}", | |
| "GEE_SERVICE_ACCOUNT_EMAIL": "${{ secrets.GEE_SERVICE_ACCOUNT_EMAIL }}", | |
| "GEE_SERVICE_ACCOUNT_KEY_FILE": "/tmp/gee_key.json", | |
| "BLOB_ACCOUNT_URL": "${{ secrets.BLOB_ACCOUNT_URL }}", | |
| "BLOB_CONTAINER_NAME": "${{ secrets.BLOB_CONTAINER_NAME }}", | |
| "SERVER_URL": "${{ secrets.SERVER_URL }}", | |
| "BLOB_SAS_TOKEN": "${{ secrets.BLOB_SAS_TOKEN }}", | |
| "INSTALL_ADDITIONAL_PACKAGES": "true", | |
| "DEBUG": "true" | |
| }, | |
| "RuntimeEnvironmentSecrets": { | |
| "GEE_KEY_JSON": "${{ secrets.GEE_SECRET_ARN }}" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| EOF | |
| # ------------------------------------------------------------------- | |
| # Deploy Celery workers service | |
| # ------------------------------------------------------------------- | |
| - name: Deploy / update Celery workers App Runner service | |
| shell: bash | |
| run: | | |
| CELERY_SERVICE_ID=$(aws apprunner list-services \ | |
| --query "ServiceSummaryList[?ServiceName=='map-action-celery-workers'].ServiceId" \ | |
| --output text) | |
| if [ -z "$CELERY_SERVICE_ID" ] || [ "$CELERY_SERVICE_ID" == "None" ]; then | |
| echo "Creating new Celery workers App Runner service…" | |
| aws apprunner create-service \ | |
| --service-name map-action-celery-workers \ | |
| --source-configuration file://celery_source.json \ | |
| --instance-configuration Cpu=1\ vCPU,Memory=2\ GB,InstanceRoleArn=${{ secrets.APP_RUNNER_INSTANCE_ROLE_ARN }} | |
| else | |
| echo "Updating existing Celery workers App Runner service…" | |
| # Get the full ARN for the service | |
| CELERY_SERVICE_ARN=$(aws apprunner list-services \ | |
| --query "ServiceSummaryList[?ServiceName=='map-action-celery-workers'].ServiceArn" \ | |
| --output text) | |
| aws apprunner update-service \ | |
| --service-arn "$CELERY_SERVICE_ARN" \ | |
| --source-configuration file://celery_source.json \ | |
| --instance-configuration Cpu=1\ vCPU,Memory=2\ GB,InstanceRoleArn=${{ secrets.APP_RUNNER_INSTANCE_ROLE_ARN }} | |
| fi | |
| echo "Celery workers deployment initiated successfully." |