From 5929b330e43f9fe64234e29aaa147600d1333462 Mon Sep 17 00:00:00 2001 From: Otto Richter Date: Mon, 2 Dec 2024 02:44:44 +0100 Subject: [PATCH] tests(e2e): Prepare for visual regression testing - fix some configuration issues - mask or replace dynamic content from screenshots - explain the behaviour in the e2e README readme --- playwright.config.ts | 1 + tests/e2e/README.md | 49 +++++++++++++++++++++++++---------- tests/e2e/example.test.e2e.ts | 2 +- tests/e2e/release.test.e2e.ts | 8 +++--- tests/e2e/utils_e2e.ts | 31 +++++++++++++++++++--- 5 files changed, 68 insertions(+), 23 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index d8a0e3ecf0..3906232abc 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -103,4 +103,5 @@ export default { outputDir: 'tests/e2e/test-artifacts/', /* Folder for explicit snapshots for visual testing */ snapshotDir: 'tests/e2e/test-snapshots/', + snapshotPathTemplate: '{snapshotDir}/snapshots/{testFilePath}/{projectName}_{arg}{ext}', } satisfies PlaywrightTestConfig; diff --git a/tests/e2e/README.md b/tests/e2e/README.md index cc255e05d9..8d8858bfd5 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -155,20 +155,6 @@ For SQLite: make test-e2e-sqlite#example ``` -### Visual testing - -> **Warning** -> This is not currently used by most Forgejo contributors. -> Your help to improve the situation and allow for visual testing is appreciated. - -Although the main goal of e2e is assertion testing, we have added a framework for visual regression testing. If you are working on front-end features, please use the following: - - Check out `main`, `make clean frontend`, and run e2e tests with `VISUAL_TEST=1` to generate outputs. This will initially fail, as no screenshots exist. You can run the e2e tests again to assert that it passes. - - Check out your branch, `make clean frontend`, and run e2e tests with `VISUAL_TEST=1`. You should be able to assert that your front-end changes don't break any other tests unintentionally. - -`VISUAL_TEST=1` will create screenshots in tests/e2e/test-snapshots. The test will fail the first time this is enabled (until we get visual test image persistence figured out), because it will be testing against an empty screenshot folder. - -`ACCEPT_VISUAL=1` will overwrite the snapshot images with new images. - ## Tips and tricks @@ -216,6 +202,41 @@ you can alternatively use: await page.waitForURL('**/target.html'); ~~~ +### Visual testing + +Due to size and frequent updates, we do not host screenshots in the Forgejo repository. +However, it is good practice to ensure that your test is capable of generating relevant and stable screenshots. +Forgejo is regularly tested against visual regressions in a dedicated repository which contains the screenshots: +https://code.forgejo.org/forgejo/visual-browser-testing/ + +For tests that consume only the `page`, +screenshots are automatically created at the end of each test. + +If your test visits different relevant screens or pages during the test, +or creates a custom `page` from context +(e.g. for tests that require a signed-in user) +calling `await save_visual(page);` explicitly in relevant positions is encouraged. + +Please confirm locally that your screenshots are stable by performing several runs of your test. +When screenshots are available and reproducible, +check in your test without the screenshots. + +When your screenshots differ between runs, +for example because dynamic elements (e.g. timestamps, commit hashes etc) +change between runs, +mask these elements in the `save_visual` function in `utils_e2e.ts`. + +#### Working with screenshots + +The following environment variables control visual testing: + +`VISUAL_TEST=1` will create screenshots in tests/e2e/test-snapshots. + The test will fail the first time, + because the screenshots are not included with Forgejo. + Subsequent runs will comopare against your local copy of the screenshots. + +`ACCEPT_VISUAL=1` will overwrite the snapshot images with new images. + ### Only sign in if necessary Signing in takes time and is actually executed step-by-step. diff --git a/tests/e2e/example.test.e2e.ts b/tests/e2e/example.test.e2e.ts index 64818c4557..2405aee0c4 100644 --- a/tests/e2e/example.test.e2e.ts +++ b/tests/e2e/example.test.e2e.ts @@ -31,7 +31,7 @@ test('Register Form', async ({page}, workerInfo) => { await expect(page.locator('.secondary-nav span>img.ui.avatar')).toBeVisible(); await expect(page.locator('.ui.positive.message.flash-success')).toHaveText('Account was successfully created. Welcome!'); - save_visual(page); + await save_visual(page); }); // eslint-disable-next-line playwright/no-skipped-test diff --git a/tests/e2e/release.test.e2e.ts b/tests/e2e/release.test.e2e.ts index 373f23dfa7..fefa446c59 100644 --- a/tests/e2e/release.test.e2e.ts +++ b/tests/e2e/release.test.e2e.ts @@ -41,7 +41,7 @@ test('External Release Attachments', async ({browser, isMobile}, workerInfo) => await page.fill('input[name=attachment-new-name-2]', 'Test'); await page.fill('input[name=attachment-new-exturl-2]', 'https://forgejo.org/'); await page.click('.remove-rel-attach'); - save_visual(page); + await save_visual(page); await page.click('.button.small.primary'); // Validate release page and click edit @@ -53,7 +53,7 @@ test('External Release Attachments', async ({browser, isMobile}, workerInfo) => await expect(page.locator('.download[open] li:nth-of-type(2) a')).toHaveAttribute('href', '/user2/repo2/archive/2.0.tar.gz'); await expect(page.locator('.download[open] li:nth-of-type(3)')).toContainText('Test'); await expect(page.locator('.download[open] li:nth-of-type(3) a')).toHaveAttribute('href', 'https://forgejo.org/'); - save_visual(page); + await save_visual(page); await page.locator('.octicon-pencil').first().click(); // Validate edit page and edit the release @@ -68,7 +68,7 @@ test('External Release Attachments', async ({browser, isMobile}, workerInfo) => await expect(page.locator('.attachment_edit:visible')).toHaveCount(4); await page.locator('.attachment_edit:visible').nth(2).fill('Test3'); await page.locator('.attachment_edit:visible').nth(3).fill('https://gitea.com/'); - save_visual(page); + await save_visual(page); await page.click('.button.small.primary'); // Validate release page and click edit @@ -78,7 +78,7 @@ test('External Release Attachments', async ({browser, isMobile}, workerInfo) => await expect(page.locator('.download[open] li:nth-of-type(3) a')).toHaveAttribute('href', 'https://gitea.io/'); await expect(page.locator('.download[open] li:nth-of-type(4)')).toContainText('Test3'); await expect(page.locator('.download[open] li:nth-of-type(4) a')).toHaveAttribute('href', 'https://gitea.com/'); - save_visual(page); + await save_visual(page); await page.locator('.octicon-pencil').first().click(); // Delete release diff --git a/tests/e2e/utils_e2e.ts b/tests/e2e/utils_e2e.ts index a52495bcc6..09189e6826 100644 --- a/tests/e2e/utils_e2e.ts +++ b/tests/e2e/utils_e2e.ts @@ -4,6 +4,15 @@ export const test = baseTest.extend({ context: async ({browser}, use) => { return use(await test_context(browser)); }, + // see https://playwright.dev/docs/test-fixtures#adding-global-beforeeachaftereach-hooks + forEachTest: [async ({page}, use) => { + await use(); + // some tests create a new page which is not yet available here + // only operate on tests that make the URL available + if (page.url() !== 'about:blank') { + await save_visual(page); + } + }, {auto: true}], }); async function test_context(browser: Browser, options?: BrowserContextOptions) { @@ -66,14 +75,28 @@ export async function save_visual(page: Page) { // Optionally include visual testing if (process.env.VISUAL_TEST) { await page.waitForLoadState('domcontentloaded'); - // Mock page/version string - await page.locator('footer div.ui.left').evaluate((node) => node.innerHTML = 'MOCK'); + // Mock/replace dynamic content which can have different size (and thus cannot simply be masked below) + await page.locator('footer .left-links').evaluate((node) => node.innerHTML = 'MOCK'); + // replace timestamps in repos to mask them later down + await page.locator('.flex-item-body > relative-time').filter({hasText: /now|minute/}).evaluateAll((nodes) => { + for (const node of nodes) node.outerHTML = 'relative time in repo'; + }); + await page.locator('relative-time').evaluateAll((nodes) => { + for (const node of nodes) node.outerHTML = 'time element'; + }); + // used for instance for security keys + await page.locator('absolute-date').evaluateAll((nodes) => { + for (const node of nodes) node.outerHTML = 'time element'; + }); await expect(page).toHaveScreenshot({ fullPage: true, timeout: 20000, mask: [ - page.locator('.secondary-nav span>img.ui.avatar'), - page.locator('.ui.dropdown.jump.item span>img.ui.avatar'), + page.locator('.ui.avatar'), + page.locator('.sha'), + page.locator('#repo_migrating'), + // update order of recently created repos is not fully deterministic + page.locator('.flex-item-main').filter({hasText: 'relative time in repo'}), ], }); }