Attach render test artifacts AWS Device Farm to workflow (#2812)

.github/workflows/android-ci.yml

@@ -278,23 +278,6 @@ jobs:
             ./render-test/android/RenderTestsApp-${{ matrix.flavor }}.apk
             ./render-test/android/RenderTests-${{ matrix.flavor }}.apk
 
-      # these two can be (and should be) removed once merged into main
-      - name: Copy render tests and render test app
-        if: matrix.flavor == 'opengl'
-        run: |
-          cp ./render-test/android/RenderTestsApp-opengl.apk ./render-test/android/RenderTestsApp.apk
-          cp ./render-test/android/RenderTests-opengl.apk ./render-test/android/RenderTests.apk
-
-      - name: Store Render Test .apk files
-        if: matrix.flavor == 'opengl'
-        uses: actions/upload-artifact@v4
-        with:
-          name: android-render-tests
-          if-no-files-found: error
-          path: |
-            ./render-test/android/RenderTestsApp.apk
-            ./render-test/android/RenderTests.apk
-
   android-ci-result:
     runs-on: ubuntu-latest
     if: needs.pre_job.outputs.should_skip != 'true' && always()

.github/workflows/android-device-test.yml

@@ -29,7 +29,10 @@ jobs:
             appFile: RenderTestsApp-vulkan.apk,
             name: "Android Render Tests (Vulkan)",
             # Google Pixel 7 Pro
-            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564"
+            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564",
+            # android-render-test.yml
+            # see https://github.com/maplibre/ci-runners/tree/main/aws-device-farm/custom-test-envs
+            testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/c1fc7d3e-dfe3-4a31-9ee0-7b0f71b08872"
           },
           {
             artifactName: benchmarkAPKs,
@@ -165,7 +168,16 @@ jobs:
         if: failure() && env.run_device_test == 'true'
         run: |
           npm install
-          node scripts/aws-device-farm/log-test-spec-output.mjs ${{ steps.aws_device_farm_run.outputs.runArn }}
+          temp_dir="$(mktemp -d)"
+          node scripts/aws-device-farm/store-test-artifacts.mjs ${{ steps.aws_device_farm_run.outputs.runArn }} "$temp_dir"
+          zip -r test_artifacts.zip "$temp_dir"
+
+      - name: Upload Test Artifacts
+        if: failure() && env.run_device_test == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-artifacts
+          path: test_artifacts.zip
 
       - name: Generate another token (previous one could have expired)
         if: always()

render-test/android/app/src/main/AndroidManifest.xml

@@ -1,14 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-          android:versionCode="1"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:versionCode="1"
           android:versionName="1.0">
   <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
   <application
       android:allowBackup="false"
       android:fullBackupContent="false"
       android:icon="@mipmap/ic_launcher"
       android:label="@string/app_name"
-      android:hasCode="true">
+      android:hasCode="true"
+      android:debuggable="true"
+      tools:ignore="HardcodedDebugMode">
 
     <activity android:name="android.app.NativeActivity"
               android:label="@string/app_name"

scripts/aws-device-farm/store-test-artifacts.mjs

@@ -0,0 +1,69 @@
+import * as fs from "node:fs";
+import { Readable } from "node:stream";
+import { finished } from "node:stream/promises";
+import * as path from "node:path";
+import * as crypto from "node:crypto";
+
+import { ListArtifactsCommand, ListJobsCommand, ListSuitesCommand } from "@aws-sdk/client-device-farm";
+import { getDeviceFarmClient } from "./device-farm-client.mjs";
+
+/**
+ * @returns {never}
+ */
+function usage() {
+  console.error("Stores artifacts from AWS Device Farm run");
+  console.error(`Usage: node ${process.argv.at(1)} RUN_ARN OUTPUT_DIR`);
+  process.exit(1);
+}
+
+if (process.argv.length !== 4) usage();
+
+const arn = process.argv.at(2);
+const outputDirArg = process.argv.at(3);
+
+if (typeof arn !== 'string') usage();
+if (typeof outputDirArg !== 'string') usage();
+
+const outputDir = outputDirArg;
+
+if (!fs.existsSync(outputDir)) {
+  console.error("Output dir does not exist");
+  process.exit(1);
+}
+
+const deviceFarmClient = getDeviceFarmClient();
+
+/**
+ * Looks for the run with the provided ARN and returns the test spec output.
+ * 
+ * @param {string} arn 
+ * @returns string
+ */
+async function getTestSpecOutput(arn) {
+  const jobs = await deviceFarmClient.send(new ListJobsCommand({
+    arn
+  }));
+
+  await Promise.all((jobs.jobs || []).map(async (job) => {
+    const suites = await deviceFarmClient.send(new ListSuitesCommand({arn: job.arn}));
+    await Promise.all((suites.suites || []).map(async (suite) => {
+      const artifacts = await deviceFarmClient.send(new ListArtifactsCommand({
+        arn: suite.arn,
+        type: 'FILE'
+      }));
+      await Promise.all((artifacts.artifacts || []).map(async (artifact) => {
+        if (!artifact.name || !artifact.url || !artifact.type) return;
+        if (['TESTSPEC_OUTPUT', 'CUSTOMER_ARTIFACT', 'CUSTOMER_ARTIFACT_LOG', 'DEVICE_LOG'].includes(artifact.type)) {
+          const filename = `${artifact.name.replaceAll(' ', '_')}-${crypto.randomBytes(10).toString('hex')}.${artifact.extension}`;
+          const res = await fetch(artifact.url);
+          if (!res.ok || !res.body) return;
+          const destination = path.resolve(outputDir, filename);
+          const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
+          await finished(Readable.fromWeb(/** @type {any} **/ (res.body)).pipe(fileStream));
+        }
+      }));
+    }));
+  }));
+}
+
+await getTestSpecOutput(arn);