Added playwright ctrf reporter to track flaky tests

.github/workflows/ci.yml

@@ -350,6 +350,11 @@ jobs:
         STRIPE_PUBLISHABLE_KEY: ${{ secrets.STRIPE_PUBLISHABLE_KEY }}
         STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }}
 
+    - name: Run CTRF annotations
+      working-directory: ghost/core
+      run: npx github-actions-ctrf ./ctrf/ctrf-report.json
+      if: always()
+
     - uses: tryghost/actions/actions/slack-build@main
       if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/main'
       with:

.gitignore

@@ -126,6 +126,8 @@ Caddyfile
 
 # Playwright state with cookies it keeps across tests
 /ghost/core/playwright-state.json
+# Playwright ctrf report files
+/ghost/core/ctrf
 
 # Admin
 /ghost/admin/dist

ghost/core/package.json

@@ -252,6 +252,7 @@
     "mock-knex": "TryGhost/mock-knex#d8b93b1c20d4820323477f2c60db016ab3e73192",
     "nock": "13.3.3",
     "papaparse": "5.3.2",
+    "playwright-ctrf-json-reporter": "^0.0.17",
     "postcss": "8.4.39",
     "postcss-cli": "11.0.0",
     "rewire": "6.0.0",

ghost/core/playwright.config.js

@@ -8,7 +8,7 @@ const config = {
     // save trace on fail
     retries: process.env.CI ? 2 : 0,
     workers: process.env.CI ? '100%' : (process.env.PLAYWRIGHT_SLOWMO ? 1 : undefined),
-    reporter: process.env.CI ? [['list', {printSteps: true}], ['html']] : [['list', {printSteps: true}]],
+    reporter: process.env.CI ? [['list', {printSteps: true}], ['html'], ['playwright-ctrf-json-reporter', {}]] : [['list', {printSteps: true}], ['playwright-ctrf-json-reporter', {}]],
     use: {
         trace: 'retain-on-failure',
         // Use a single browser since we can't simultaneously test multiple browsers

yarn.lock

@@ -25182,6 +25182,11 @@ playwright-core@1.46.1:
   resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.46.1.tgz#28f3ab35312135dda75b0c92a3e5c0e7edb9cc8b"
   integrity sha512-h9LqIQaAv+CYvWzsZ+h3RsrqCStkBHlgo6/TJlFst3cOTlLghBQlJwPOZKQJTKNaD3QIB7aAVQ+gfWbN3NXB7A==
 
+playwright-ctrf-json-reporter@^0.0.17:
+  version "0.0.17"
+  resolved "https://registry.yarnpkg.com/playwright-ctrf-json-reporter/-/playwright-ctrf-json-reporter-0.0.17.tgz#e0c3c22c9b72a354cf8e5ad9c4d33747d9bce99e"
+  integrity sha512-pkKPQMjYR7EbFVZn7/TVbh0cx4vEOvFNqYARAy1lZC7gPwIgFb5JUXsJ//pQsnSb5twB4m9kjekcJMo3zxTXKg==
+
 playwright@1.46.1:
   version "1.46.1"
   resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.46.1.tgz#ea562bc48373648e10420a10c16842f0b227c218"