ci: Enhance robot-framework tests with version-specific baselines

* Moved existing baseline images to version-specific directories under `.ci/robot_framework/images/2_46/`. * Added new baseline images for nightly builds under `.ci/robot_framework/images/nightly/`. * Updated Robot Framework test scripts to use `TEST_WPEWEBKIT_VERSION` for selecting the correct baseline image path. * Modified `podman-robot/action.yml` to include `wpe_vers` as an input and export `TEST_WPEWEBKIT_VERSION`. * Updated GitHub workflows to pass `wpe_vers` to the `podman-robot` action, ensuring version-specific testing.
Igalia · Feb 20, 2025 · 1e95d76 · 1e95d76
1 parent d9e201e
commit 1e95d76
Show file tree

Hide file tree

Showing 24 changed files with 20 additions and 4 deletions.
diff --git a/.ci/robot_framework/images/glyphs.png → .ci/robot_framework/images/2_46/glyphs.png b/.ci/robot_framework/images/glyphs.png → .ci/robot_framework/images/2_46/glyphs.png
diff --git a/.ci/robot_framework/images/home-screen.png → ...bot_framework/images/2_46/home-screen.png b/.ci/robot_framework/images/home-screen.png → ...bot_framework/images/2_46/home-screen.png
diff --git a/.ci/robot_framework/images/init-screen.png → ...bot_framework/images/2_46/init-screen.png b/.ci/robot_framework/images/init-screen.png → ...bot_framework/images/2_46/init-screen.png
diff --git a/.ci/robot_framework/images/pinch-gesture.png → ...t_framework/images/2_46/pinch-gesture.png b/.ci/robot_framework/images/pinch-gesture.png → ...t_framework/images/2_46/pinch-gesture.png
diff --git a/.ci/robot_framework/images/search-screen.png → ...t_framework/images/2_46/search-screen.png b/.ci/robot_framework/images/search-screen.png → ...t_framework/images/2_46/search-screen.png
diff --git a/.ci/robot_framework/images/zoom-gesture.png → ...ot_framework/images/2_46/zoom-gesture.png b/.ci/robot_framework/images/zoom-gesture.png → ...ot_framework/images/2_46/zoom-gesture.png
diff --git a/.ci/robot_framework/images/nightly/glyphs.png b/.ci/robot_framework/images/nightly/glyphs.png
diff --git a/.ci/robot_framework/images/nightly/home-screen.png b/.ci/robot_framework/images/nightly/home-screen.png
diff --git a/.ci/robot_framework/images/nightly/init-screen.png b/.ci/robot_framework/images/nightly/init-screen.png
diff --git a/.ci/robot_framework/images/nightly/pinch-gesture.png b/.ci/robot_framework/images/nightly/pinch-gesture.png
diff --git a/.ci/robot_framework/images/nightly/search-screen.png b/.ci/robot_framework/images/nightly/search-screen.png
diff --git a/.ci/robot_framework/images/nightly/zoom-gesture.png b/.ci/robot_framework/images/nightly/zoom-gesture.png
diff --git a/.ci/robot_framework/tests/keywords_common.robot b/.ci/robot_framework/tests/keywords_common.robot
@@ -45,8 +45,9 @@ Remote Weston Capture Screenshot
 
 Remote Weston Check Screenshot
     [Arguments]    ${image}
+    ${TEST_WPEWEBKIT_VERSION}    Get Environment Variable    TEST_WPEWEBKIT_VERSION
     Remote Weston Capture Screenshot    ${image}
-    Compare Images   ${BASELINE_IMAGES_PATH}/${image}    ${image}    threshold=0.0050
+    Compare Images   ${BASELINE_IMAGES_PATH}/${TEST_WPEWEBKIT_VERSION}/${image}    ${image}    threshold=0.0050
 
 Webdriver Remote Start
     [Arguments]    @{other_params}

diff --git a/.ci/robot_framework/tests/keywords_touch_events.robot b/.ci/robot_framework/tests/keywords_touch_events.robot
@@ -21,6 +21,7 @@ Check Browser Touch Events Using Uinput
     ${TEST_BOARD_IP}    Get Environment Variable    TEST_BOARD_IP
     ${TEST_WEBSERVER_IP}    Get Environment Variable    TEST_WEBSERVER_IP
     ${TEST_WEBSERVER_PORT}    Get Environment Variable    TEST_WEBSERVER_PORT
+    ${TEST_WPEWEBKIT_VERSION}    Get Environment Variable    TEST_WPEWEBKIT_VERSION
     ${PAGE}    Set Variable    http://${TEST_WEBSERVER_IP}:${TEST_WEBSERVER_PORT}/robot_framework/html/vertical_scroll.html
     ${PAGE2}    Set Variable    http://${TEST_WEBSERVER_IP}:${TEST_WEBSERVER_PORT}/robot_framework/html/rbyers/paint.html
 
@@ -56,11 +57,11 @@ Check Browser Touch Events Using Uinput
     Go to    ${PAGE2}
     SSH Command    ${TEST_BOARD_IP}    /root/scripts/touch-two-fingers-gesture.py --duration 2 --steps 40 900 200 900 500 900 800 900 500
     Capture Page Screenshot    ${PINCH_GESTURE_IMAGE}
-    Compare Images   ${BASELINE_IMAGES_PATH}/${PINCH_GESTURE_IMAGE}    ${PINCH_GESTURE_IMAGE}    threshold=0.0005
+    Compare Images   ${BASELINE_IMAGES_PATH}/${TEST_WPEWEBKIT_VERSION}/${PINCH_GESTURE_IMAGE}    ${PINCH_GESTURE_IMAGE}    threshold=0.0005
 
     # Multitouch: Zoom
     Go to    ${PAGE2}
     SSH Command    ${TEST_BOARD_IP}    /root/scripts/touch-two-fingers-gesture.py --duration 2 --steps 40 900 500 900 200 900 500 900 800
     Capture Page Screenshot    ${ZOOM_GESTURE_IMAGE}
-    Compare Images   ${BASELINE_IMAGES_PATH}/${ZOOM_GESTURE_IMAGE}    ${ZOOM_GESTURE_IMAGE}    threshold=0.0005
+    Compare Images   ${BASELINE_IMAGES_PATH}/${TEST_WPEWEBKIT_VERSION}/${ZOOM_GESTURE_IMAGE}    ${ZOOM_GESTURE_IMAGE}    threshold=0.0005
 
diff --git a/.ci/robot_framework/tests/tests_013_glyphs.robot b/.ci/robot_framework/tests/tests_013_glyphs.robot
@@ -18,11 +18,12 @@ Resource   keywords_common.robot
 Verify no changes in the glyphs
     ${TEST_WEBSERVER_IP}    Get Environment Variable    TEST_WEBSERVER_IP
     ${TEST_WEBSERVER_PORT}    Get Environment Variable    TEST_WEBSERVER_PORT
+    ${TEST_WPEWEBKIT_VERSION}    Get Environment Variable    TEST_WPEWEBKIT_VERSION
     ${PAGE}    Set Variable    http://${TEST_WEBSERVER_IP}:${TEST_WEBSERVER_PORT}/robot_framework/html/glyphs.html
 
     Go to    ${PAGE}
     Wait Until Page Contains    Ready    timeout=10s
 
     Capture Page Screenshot    ${GLYPHS_PAGE_IMAGE}
-    Compare Images   ${BASELINE_IMAGES_PATH}/${GLYPHS_PAGE_IMAGE}    ${GLYPHS_PAGE_IMAGE}    threshold=0.0005
+    Compare Images   ${BASELINE_IMAGES_PATH}/${TEST_WPEWEBKIT_VERSION}/${GLYPHS_PAGE_IMAGE}    ${GLYPHS_PAGE_IMAGE}    threshold=0.0005
 
diff --git a/.github/actions/podman-robot/action.yml b/.github/actions/podman-robot/action.yml
@@ -13,6 +13,8 @@ inputs:
     required: true
   repo_release:
     default: 'scarthgap'
+  wpe_vers:
+    required: true
 runs:
   using: 'composite'
   steps:
@@ -68,6 +70,7 @@ runs:
         echo "#!/bin/sh" > setup-env-local.sh
         echo "export TEST_BOARD_IP=${IP}" >> setup-env-local.sh
         echo "export TEST_WEBSERVER_IP=${WEBSERVER_IP}" >> setup-env-local.sh
+        echo "export TEST_WPEWEBKIT_VERSION=${{ inputs.wpe_vers }}" >> setup-env-local.sh
         if ./run-tests.sh; then
           MESSAGE="Tests passed for \`${{ inputs.bitbake_buildname }}\` on \`${{ inputs.bitbake_target }}\`. Tests results: https://wk-contrib.igalia.com/yocto/meta-wpe-image/${TIMESTAMP}-${{ inputs.bitbake_buildname }}/robot/report.html"
           echo "* ✅️ ${MESSAGE}" >> ci-message.txt

diff --git a/.github/workflows/test-build-nightly.yml b/.github/workflows/test-build-nightly.yml
@@ -71,3 +71,4 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}
diff --git a/.github/workflows/test-build-raspberrypi3-mesa-nightly-manually.yml b/.github/workflows/test-build-raspberrypi3-mesa-nightly-manually.yml
@@ -66,3 +66,4 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}
diff --git a/.github/workflows/test-build-raspberrypi3-mesa-stable-manually.yml b/.github/workflows/test-build-raspberrypi3-mesa-stable-manually.yml
@@ -66,3 +66,4 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}
diff --git a/.github/workflows/test-build-raspberrypi5-nightly-manually.yml b/.github/workflows/test-build-raspberrypi5-nightly-manually.yml
@@ -66,4 +66,5 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}
 
diff --git a/.github/workflows/test-build-raspberrypi5-stable-manually.yml b/.github/workflows/test-build-raspberrypi5-stable-manually.yml
@@ -66,4 +66,5 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}
 
diff --git a/.github/workflows/test-build-wandboard-mesa-nightly-manually.yml b/.github/workflows/test-build-wandboard-mesa-nightly-manually.yml
@@ -66,3 +66,4 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}
diff --git a/.github/workflows/test-build-wandboard-mesa-stable-manually.yml b/.github/workflows/test-build-wandboard-mesa-stable-manually.yml
@@ -66,3 +66,4 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}
diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml
@@ -66,6 +66,7 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}
 
   build-raspberrypi-rest-stable:
     strategy:
@@ -124,6 +125,7 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}
 
   build-freescale-stable:
     strategy:
@@ -183,3 +185,4 @@ jobs:
           bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
           bitbake_machine: ${{ matrix.machine }}
           github_token: ${GITHUB_TOKEN}
+          wpe_vers: ${{ matrix.wpe_vers }}