Fix: File deletion during cleanup job #171
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| permissions: | |
| contents: write | |
| pages: write | |
| id-token: write | |
| jobs: | |
| cleanup: | |
| runs-on: self-hosted | |
| steps: | |
| - name: Free up disk space | |
| run: | | |
| sudo docker system prune -af | |
| sudo docker volume prune -f | |
| sudo yum clean all | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| sudo df -h | |
| setup-and-test: | |
| needs: cleanup | |
| runs-on: self-hosted | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| continue-on-error: true | |
| - name: Fallback Check for Python Versions | |
| if: failure() | |
| run: | | |
| echo "Checking available Python versions..." | |
| curl -s https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json | jq '.[] | .version' | |
| exit 1 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| # Skip Docker Compose installation as it should be available on self-hosted runner | |
| - name: Check Docker and Docker Compose versions | |
| run: | | |
| docker --version | |
| docker-compose --version | |
| - name: Create .cv_deploy.env | |
| run: | | |
| touch .cv_deploy.env | |
| echo "MODEL_PATH=${{ secrets.MODEL_PATH }}" >> .cv_deploy.env | |
| echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .cv_deploy.env | |
| echo "SUPABASE_URL=${{ secrets.SUPABASE_URL }}" >> .cv_deploy.env | |
| echo "SUPABASE_KEY=${{ secrets.SUPABASE_ANON_KEY }}" >> .cv_deploy.env | |
| echo "USE_SUPABASE_STORAGE=${{ secrets.USE_SUPABASE_STORAGE }}" >> .cv_deploy.env | |
| echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .cv_deploy.env | |
| echo "POSTGRES_URL=${{ secrets.POSTGRES_URL }}" >> .cv_deploy.env | |
| echo "POSTGRES_USER=${{ secrets.POSTGRES_USER }}" >> .cv_deploy.env | |
| echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .cv_deploy.env | |
| echo "REDIS_URL=${{ secrets.REDIS_URL }}" >> .cv_deploy.env | |
| echo "COPERNICUS_CLIENT_ID=${{ secrets.COPERNICUS_CLIENT_ID }}" >> .cv_deploy.env | |
| echo "COPERNICUS_CLIENT_SECRET=${{ secrets.COPERNICUS_CLIENT_SECRET }}" >> .cv_deploy.env | |
| echo "REDIS_HOST=${{ secrets.REDIS_HOST }}" >> .cv_deploy.env | |
| echo "REDIS_PORT=${{ secrets.REDIS_PORT }}" >> .cv_deploy.env | |
| echo "REDIS_PASSWORD=${{ secrets.REDIS_PASSWORD }}" >> .cv_deploy.env | |
| echo "POSTGRES_URL=${{ secrets.POSTGRES_URL }}" >> .cv_deploy.env | |
| echo "GEE_SERVICE_ACCOUNT_EMAIL=${{ secrets.GEE_SERVICE_ACCOUNT_EMAIL }}" >> .cv_deploy.env | |
| echo "GEE_SERVICE_ACCOUNT_KEY_FILE=${{ secrets.GEE_SERVICE_ACCOUNT_KEY_FILE}}" >> .cv_deploy.env | |
| echo "SERVER_URL=${{ secrets.SERVER_URL }}" >> .cv_deploy.env | |
| echo "S3_BUCKET_NAME=${{ secrets.S3_BUCKET_NAME }}" >> .cv_deploy.env | |
| echo "AWS_REGION=${{ secrets.AWS_REGION }}" >> .cv_deploy.env | |
| shell: bash | |
| - name: Create coverage directory | |
| run: mkdir -p coverage | |
| - name: Run Tests | |
| run: | | |
| # Verify the .cv_deploy.env file exists | |
| ls -la .cv_deploy.env | |
| # Run the tests | |
| docker-compose -f _ci_pipeline.yml up --build -d | |
| docker-compose -f _ci_pipeline.yml run --rm -v ${{ github.workspace }}/coverage:/app/coverage testing | |
| - name: Check for coverage file | |
| run: | | |
| if [ ! -f ./coverage/coverage.xml ]; then | |
| echo "Coverage file not found!" | |
| exit 1 | |
| fi | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v4 | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| with: | |
| files: ./coverage/coverage.xml | |
| - name: Tear down testing environment | |
| if: always() # Ensure this runs even if previous steps fail | |
| run: | | |
| docker-compose -f _ci_pipeline.yml down --volumes --remove-orphans | |
| - name: Prune Docker resources (testing) | |
| if: always() # Ensure this runs even if previous steps fail | |
| run: | | |
| docker system prune -af | |
| deploy: | |
| needs: setup-and-test | |
| runs-on: self-hosted | |
| steps: | |
| - name: Clean up root-owned files | |
| run: | | |
| # Clean up any root-owned files that might prevent checkout | |
| if [ -d "/home/ec2-user/actions-runner/_work/Model_deploy/Model_deploy" ]; then | |
| sudo find /home/ec2-user/actions-runner/_work/Model_deploy/Model_deploy -type f -name "*.png" -exec sudo rm -f {} \; || true | |
| sudo find /home/ec2-user/actions-runner/_work/Model_deploy/Model_deploy/app/plots -type f -exec sudo rm -f {} \; || true | |
| sudo find /home/ec2-user/actions-runner/_work/Model_deploy/Model_deploy/local_uploads -type f -exec sudo rm -f {} \; || true | |
| sudo chmod -R 755 /home/ec2-user/actions-runner/_work/Model_deploy/Model_deploy || true | |
| fi | |
| continue-on-error: true | |
| - name: Checkout Repository | |
| uses: actions/checkout@v3 | |
| - name: Create .cv_deploy.env | |
| run: | | |
| echo "MODEL_PATH=${{ secrets.MODEL_PATH }}" >> .cv_deploy.env | |
| echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .cv_deploy.env | |
| echo "SUPABASE_URL=${{ secrets.SUPABASE_URL }}" >> .cv_deploy.env | |
| echo "SUPABASE_KEY=${{ secrets.SUPABASE_ANON_KEY }}" >> .cv_deploy.env | |
| echo "USE_SUPABASE_STORAGE=${{ secrets.USE_SUPABASE_STORAGE }}" >> .cv_deploy.env | |
| echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .cv_deploy.env | |
| echo "POSTGRES_URL=${{ secrets.POSTGRES_URL }}" >> .cv_deploy.env | |
| echo "POSTGRES_USER=${{ secrets.POSTGRES_USER }}" >> .cv_deploy.env | |
| echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .cv_deploy.env | |
| echo "REDIS_URL=${{ secrets.REDIS_URL }}" >> .cv_deploy.env | |
| echo "COPERNICUS_CLIENT_ID=${{ secrets.COPERNICUS_CLIENT_ID }}" >> .cv_deploy.env | |
| echo "COPERNICUS_CLIENT_SECRET=${{ secrets.COPERNICUS_CLIENT_SECRET }}" >> .cv_deploy.env | |
| echo "REDIS_HOST=${{ secrets.REDIS_HOST }}" >> .cv_deploy.env | |
| echo "REDIS_PORT=${{ secrets.REDIS_PORT }}" >> .cv_deploy.env | |
| echo "REDIS_PASSWORD=${{ secrets.REDIS_PASSWORD }}" >> .cv_deploy.env | |
| echo "POSTGRES_URL=${{ secrets.POSTGRES_URL }}" >> .cv_deploy.env | |
| echo "GEE_SERVICE_ACCOUNT_EMAIL=${{ secrets.GEE_SERVICE_ACCOUNT_EMAIL }}" >> .cv_deploy.env | |
| echo "GEE_SERVICE_ACCOUNT_KEY_FILE=${{ secrets.GEE_SERVICE_ACCOUNT_KEY_FILE}}" >> .cv_deploy.env | |
| echo "SERVER_URL=${{ secrets.SERVER_URL }}" >> .cv_deploy.env | |
| echo "S3_BUCKET_NAME=${{ secrets.S3_BUCKET_NAME }}" >> .cv_deploy.env | |
| echo "AWS_REGION=${{ secrets.AWS_REGION }}" >> .cv_deploy.env | |
| - name: Build and Deploy Docker Image | |
| run: | | |
| docker-compose -f _cd_pipeline.yaml build | |
| docker-compose -f _cd_pipeline.yaml down --remove-orphans | |
| docker-compose -f _cd_pipeline.yaml up --build -d | |
| - name: Post-deployment cleanup | |
| if: always() | |
| run: | | |
| docker system prune -af --volumes | |
| docker image prune -af | |
| docker volume prune -f |