separate daily test metrics

.github/workflows/daily-test.yml

@@ -18,7 +18,7 @@ env:
   ROLE_TO_ASSUME: ${{ secrets.ROLE_TO_ASSUME }}
   METRIC_NAME: ${{ secrets.METRIC_NAME }}
   METRIC_NAMESPACE: ${{ secrets.METRIC_NAMESPACE }}
-
+  REGION: us-east-1
 
 permissions:
   id-token: write
@@ -32,33 +32,46 @@ jobs:
       matrix:
         # Latest 2 Android versions. TODO: Upgrade to Appium 2 for OS 14 and above.
         os_list: ["12", "13"]
-        demo_download_url_list: ["$DEMO_APP_DOWNLOAD_LINK", "$DEMO_APP_DOWNLOAD_LINK_NO_VIDEO_CODECS"]
+        demo_flavor_list: ["default", "no-video-codecs"]
     outputs:
       job-status: ${{ job.status }}
+    continue-on-error: true
 
     steps:
       - name: Checkout test package
         uses: actions/checkout@v4
+        if: always()
         with:
           repository: awslabs/amazon-chime-sdk-apple
           token: ${{ secrets.GH_INTEG_READ_ONLY_PAT }}
           ref: automated-test-development
 
       - name: Get latest prod demo app
+        if: always()
         run: |
-          wget -O amazon-chime-sdk-app.apk ${{ matrix.demo_download_url_list }}
+          if [ "${{ matrix.demo_flavor_list }}" = "default" ]; then
+            wget -O amazon-chime-sdk-app.apk $DEMO_APP_DOWNLOAD_LINK;
+          elif [ "${{ matrix.demo_flavor_list }}" = "no-video-codecs" ]; then
+            wget -O amazon-chime-sdk-app.apk $DEMO_APP_DOWNLOAD_LINK_NO_VIDEO_CODECS;
+          else
+            echo "Error: Unsupported demo app flavor: ${{ matrix.demo_flavor_list }}";
+            exit 1;
+          fi
 
       - name: Setup Node.js - 15.x
         uses: actions/setup-node@v4
+        if: always()
         with:
           node-version: 15.x
 
       - name: Install Dependencies
+        if: always()
         run: |
           npm install @aws-sdk/client-cloudwatch
 
       - name: Run tests against specified Android versions
         id: tests
+        if: always()
         run: |
           id=$(curl -F '[email protected]' -F name=amazon-chime-sdk-app.apk -u "${{ secrets.SAUCE_USERNAME }}:${{ secrets.SAUCE_ACCESS_KEY }}" 'https://api.us-west-1.saucelabs.com/v1/storage/upload' |jq '.item.id')
           npm install
@@ -78,25 +91,29 @@ jobs:
 
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
+        if: always()
         with:
           role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
-          aws-region: us-east-1
+          aws-region: ${{ env.REGION }}
 
       - name: Setup Node.js - 18.x
         uses: actions/setup-node@v4
+        if: always()
         with:
           node-version: 18.x
 
       - name: Install Dependencies
+        if: always()
         run: |
           # Required to bydpass issue with canvas dependency: https://github.com/Automattic/node-canvas/issues/1862
           sudo apt-get install -y libcairo2-dev libjpeg-dev libpango1.0-dev libgif-dev build-essential g++
 
       - name: Send Metric to CloudWatch
+        if: always()
         run: |
           node -e "
           const { CloudWatchClient, PutMetricDataCommand } = require('@aws-sdk/client-cloudwatch');
-          const client = new CloudWatchClient({ region: 'us-east-1' });
+          const client = new CloudWatchClient({ region: process.env.REGION });
 
           const value = '${{ job.status }}' === 'failure' ? 0 : 1;
 
@@ -105,8 +122,9 @@ jobs:
             MetricData: [{
               MetricName: process.env.METRIC_NAME,
               Dimensions: [
-                { Name: 'Workflow', Value: 'DailyTest' },
-                { Name: 'Platform', Value: 'Android' }
+                { Name: 'Platform', Value: 'Android' },
+                { Name: 'OS', Value: ${{ matrix.os_list }} },
+                { Name: 'Flavor', Value: '${{ matrix.demo_flavor_list }}' }
               ],
               Value: value,
             }]