Check the rendering of a few games against expected screenshots

.circleci/config.yml

@@ -25,6 +25,26 @@ parameters:
     type: string
     default: master
 
+commands:
+  install-gdevelop-runtime-dependencies:
+    steps:
+      - run:
+          name: Install the runtime dependencies of GDevelop
+          # Even in CLI mode, Electron needs these libraries and a display to
+          # start up on a headless machine.
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y \
+              libnss3 \
+              libasound2t64 \
+              libgbm1 \
+              libgtk-3-0 \
+              libxss1 \
+              libatk-bridge2.0-0 \
+              libdrm2 \
+              libxkbcommon0 \
+              xvfb
+
 jobs:
   install:
     docker:
@@ -165,22 +185,7 @@ jobs:
     steps:
       - checkout
 
-      - run:
-          name: Install the runtime dependencies of GDevelop
-          # Even in CLI mode, Electron needs these libraries and a display to
-          # start up on a headless machine.
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y \
-              libnss3 \
-              libasound2t64 \
-              libgbm1 \
-              libgtk-3-0 \
-              libxss1 \
-              libatk-bridge2.0-0 \
-              libdrm2 \
-              libxkbcommon0 \
-              xvfb
+      - install-gdevelop-runtime-dependencies
 
       - run:
           name: Install the dependencies of the scripts
@@ -244,6 +249,64 @@ jobs:
           name: Report whether the gameplay tests passed
           command: exit "$(cat /tmp/gameplay-tests-exit-code)"
 
+  # Check that the games are still drawn the same: their rendering tests
+  # (gameplay tests named "Rendering: ...") take screenshots compared with the
+  # images of `rendering-screenshots/` (see scripts/check-rendering-screenshots.js).
+  # The latest build of GDevelop is used: an image that differs while the game
+  # did not change is an early warning of a change of the rendering of GDevelop.
+  rendering-screenshots:
+    parameters:
+      scope:
+        description: >
+          "all" checks every game with rendering tests. "changed" only checks
+          the games whose project or expected images changed compared to `main`.
+        type: enum
+        enum: ['all', 'changed']
+        default: 'all'
+    resource_class: large
+    docker:
+      - image: cimg/node:24.13.0
+    working_directory: ~/GDevelop-examples
+
+    steps:
+      - checkout
+      - install-gdevelop-runtime-dependencies
+
+      - run:
+          name: Install the dependencies of the scripts
+          command: npm ci --omit=dev
+
+      - run:
+          name: Check the rendering of the games
+          no_output_timeout: 30m
+          # The failure is reported by the last step of the job, so that the
+          # screenshots and their differences are uploaded first.
+          command: |
+            set +e
+            SCOPE_ARGS=""
+            if [ "<< parameters.scope >>" = "changed" ]; then
+              SCOPE_ARGS="--only-changed --base-ref=origin/main"
+            fi
+            node scripts/check-rendering-screenshots.js $SCOPE_ARGS \
+              --gdevelop-branch=<< pipeline.parameters.gdevelop-branch >> \
+              --work-dir=/tmp/gdevelop-portable \
+              --artifacts-dir=/tmp/rendering-screenshots-artifacts \
+              --junit-path=/tmp/rendering-screenshots-results/results.xml
+            echo $? > /tmp/rendering-screenshots-exit-code
+
+      - store_test_results:
+          path: /tmp/rendering-screenshots-results
+      # One folder per game, with its screenshots (in
+      # `gameplay-test-screenshots/`) and, for those that differ, an image of
+      # the differences next to them (`.diff.png`, differences in magenta).
+      - store_artifacts:
+          path: /tmp/rendering-screenshots-artifacts
+          destination: rendering-screenshots
+
+      - run:
+          name: Report whether the games are drawn like their expected images
+          command: exit "$(cat /tmp/rendering-screenshots-exit-code)"
+
 workflows:
   tests:
     jobs:
@@ -271,9 +334,8 @@ workflows:
 
   # Every night (03:00 UTC): rebuild and redeploy `main`. The starters re-skinned
   # with each theme are generated by the build from the themes and assets the
-  # assets repository publishes, so this is how they catch up with it. Only this
-  # workflow runs on the schedule: the gameplay tests are not run again, as the
-  # games did not change.
+  # assets repository publishes, so this is how they catch up with it. The
+  # gameplay tests are not run again, as the games did not change.
   nightly-rebuild:
     triggers:
       - schedule:
@@ -306,6 +368,9 @@ workflows:
           name: gameplay-tests-all
           scope: 'all'
           parallelism: << pipeline.parameters.all-gameplay-tests-parallelism >>
+      - rendering-screenshots:
+          name: rendering-screenshots-all
+          scope: 'all'
 
   # On a branch / Pull Request: only check the games it modifies.
   changed-gameplay-tests:
@@ -319,3 +384,20 @@ workflows:
           name: gameplay-tests-changed
           scope: 'changed'
           parallelism: << pipeline.parameters.changed-gameplay-tests-parallelism >>
+      - rendering-screenshots:
+          name: rendering-screenshots-changed
+          scope: 'changed'
+
+  # Every night (04:17 UTC): check the rendering of the games with the latest
+  # build of GDevelop, which changes every day without any change here.
+  nightly-rendering-screenshots:
+    triggers:
+      - schedule:
+          cron: '17 4 * * *'
+          filters:
+            branches:
+              only: main
+    jobs:
+      - rendering-screenshots:
+          name: rendering-screenshots-nightly
+          scope: 'all'

.gitignore

@@ -11,3 +11,7 @@ gameplay-test-results.json
 gameplay-test-screenshots/
 /gameplay-tests-artifacts
 /gameplay-tests-results
+
+# Written by the rendering screenshots check.
+/rendering-screenshots-artifacts
+/rendering-screenshots-results

README.md

@@ -230,6 +230,34 @@ land on `main`, by triggering a CircleCI pipeline with the
 `run-all-gameplay-tests` parameter set to `true`. The number of parallel
 containers used is the `all-gameplay-tests-parallelism` pipeline parameter.
 
+### Rendering screenshots
+
+A few games have a _rendering test_: a gameplay test named `Rendering: ...`
+taking PNG screenshots (`harness.takeScreenshot(label, { width: 640, format: 'png' })`).
+The `rendering-screenshots` CI job runs them with the latest build of GDevelop
+and compares each screenshot with the expected image,
+`rendering-screenshots/<example>/<label>.png`. The games are drawn in software,
+which draws the same pixels from one run to the next: an image that differs
+when the game did not change is an early warning that the rendering of
+GDevelop changed. The job runs on `main`, every night, and on the branches
+changing these games, their images or the scripts.
+
+```bash
+# Compare the screenshots with the expected images.
+node scripts/check-rendering-screenshots.js
+
+# After a change of the rendering made on purpose (in GDevelop or in a game),
+# write the screenshots as the expected images, and check them in the diff.
+node scripts/check-rendering-screenshots.js --update
+```
+
+When a screenshot differs, an image of the differences (in magenta) is written
+next to it, in the artifacts of the job (under `rendering-screenshots/`).
+
+Texts are drawn with the font settings of the system, which differ from one
+machine to another: hide the layers showing texts in these screenshots
+(`hideLayers: ['UI']`).
+
 ## License
 
 All examples provided on this repository are MIT licensed, unless specified otherwise.