Enable parallel running of browser tests

refs: https://github.com/TryGhost/DevOps/issues/78

Instead of running a single instance of Ghost, we now run an instance of Ghost for each test worker.

This has the unfortunate effect that a test failing will close and restart a new instance of Ghost, but in general will be multiple times faster than sequential execution of tests.

ghost/core/core/cli/record-test.js

@@ -1,30 +0,0 @@
-const {chromium} = require('@playwright/test');
-const Command = require('./command');
-const playwrightConfig = require('../../playwright.config');
-const {globalSetup} = require('../../test/e2e-browser/utils');
-
-module.exports = class RecordTest extends Command {
-    setup() {
-        this.help('Use PlayWright to record a browser-based test');
-    }
-
-    permittedEnvironments() {
-        return ['testing-browser'];
-    }
-
-    async handle() {
-        await globalSetup({
-            projects: [playwrightConfig]
-        });
-
-        const browser = await chromium.launch({headless: false});
-
-        const context = await browser.newContext(playwrightConfig.use);
-
-        // Pause the page, and start recording manually.
-        const page = await context.newPage();
-        await page.goto('/ghost');
-
-        await page.pause();
-    }
-};

ghost/core/package.json

@@ -33,12 +33,10 @@
     "test:integration": "yarn test:base './test/integration' --timeout=10000",
     "test:e2e": "yarn test:base ./test/e2e-* --timeout=15000",
     "test:regression": "yarn test:base './test/regression' --timeout=60000",
-    "test:browser": "yarn test:browser:admin && yarn test:browser:portal",
+    "test:browser": "NODE_ENV=testing-browser playwright test",
     "test:browser:admin": "NODE_ENV=testing-browser  playwright test test/e2e-browser --project=admin",
     "test:browser:portal": "NODE_ENV=testing-browser playwright test test/e2e-browser --project=portal",
-    "test:browser:single": "NODE_ENV=testing-browser playwright test",
     "test:browser:setup": "npx playwright install",
-    "test:browser:record": "NODE_ENV=testing-browser yarn start record-test",
     "test:ci:e2e": "c8 -c ./.c8rc.e2e.json -o coverage-e2e yarn test:e2e -b --retries=2 --reporter=./test/utils/mocha-retry-reporter.js",
     "test:ci:regression": "yarn test:regression -b --retries=2 --reporter=./test/utils/mocha-retry-reporter.js",
     "test:ci:integration": "c8 -c ./.c8rc.e2e.json -o coverage-integration --lines 57 --functions 47 --branches 77 --statements 57 yarn test:integration -b --retries=2 --reporter=./test/utils/mocha-retry-reporter.js",

ghost/core/playwright.config.js

@@ -5,15 +5,13 @@ const config = {
     expect: {
         timeout: 10000
     },
-    workers: 1,
     reporter: [['list', {printSteps: true}]],
     use: {
         // Use a single browser since we can't simultaneously test multiple browsers
         browserName: 'chromium',
         headless: !process.env.PLAYWRIGHT_DEBUG,
-        baseURL: process.env.TEST_URL ?? 'http://127.0.0.1:2369',
-        // TODO: Where to put this
-        storageState: 'playwright-state.json'
+        // Port doesn't matter, overriden by baseURL fixture for each worker
+        baseURL: 'http://127.0.0.1:2368'
     },
     // separated tests to projects for better logging to console
     // portal tests are much more stable when running in the separate DB from admin tests
@@ -26,9 +24,7 @@ const config = {
             name: 'portal',
             testDir: 'test/e2e-browser/portal'
         }
-    ],
-    globalSetup: './test/e2e-browser/utils/global-setup',
-    globalTeardown: './test/e2e-browser/utils/global-teardown'
+    ]
 };
 
 module.exports = config;

ghost/core/test/e2e-browser/admin/announcement-bar-settings.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Announcement Bar Settings', () => {
     test('Bar hidden by default', async ({page}) => {

ghost/core/test/e2e-browser/admin/members.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {createMember, deleteAllMembers} = require('../utils/e2e-browser-utils');
 const fs = require('fs');
 

ghost/core/test/e2e-browser/admin/membership-settings.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {disconnectStripe, setupStripe, generateStripeIntegrationToken} = require('../utils');
 
 test.describe('Membership Settings', () => {

ghost/core/test/e2e-browser/admin/portal-settings.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Portal Settings', () => {
     test.describe('Links', () => {

ghost/core/test/e2e-browser/admin/posts.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Admin', () => {
     test.describe('Posts', () => {

ghost/core/test/e2e-browser/admin/private-site.spec.js

@@ -1,4 +1,5 @@
-const {test, expect} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Site Settings', () => {
     test.describe('Privacy setting', () => {

ghost/core/test/e2e-browser/admin/publishing.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {DateTime} = require('luxon');
 const {slugify} = require('@tryghost/string');
 const {createTier, createMember, createPostDraft, impersonateMember} = require('../utils');

ghost/core/test/e2e-browser/admin/setup.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Admin', () => {
     test.describe('Setup', () => {

ghost/core/test/e2e-browser/admin/site-settings.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {createPostDraft, createTier, disconnectStripe, generateStripeIntegrationToken, setupStripe} = require('../utils');
 
 const changeSubscriptionAccess = async (page, access) => {

ghost/core/test/e2e-browser/admin/tiers.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {createTier, createOffer, getUniqueName, getSlug, goToMembershipPage, openTierModal} = require('../utils');
 
 test.describe('Admin', () => {

ghost/core/test/e2e-browser/fixtures/ghost-test.js

@@ -0,0 +1,145 @@
+// express-test.js
+const config = require('../../../core/shared/config');
+const base = require('@playwright/test');
+const {promisify} = require('util');
+const {spawn, exec} = require('child_process');
+const {setupGhost, setupMailgun, enableLabs, setupStripe} = require('../utils/e2e-browser-utils');
+const {allowStripe} = require('../../utils/e2e-framework-mock-manager');
+const MailgunClient = require('@tryghost/mailgun-client');
+const sinon = require('sinon');
+const ObjectID = require('bson-objectid').default;
+
+const startWebhookServer = (port) => {
+    const command = `stripe listen --forward-to http://127.0.0.1:${port}/members/webhooks/stripe/`;
+    return spawn(command.split(' ')[0], command.split(' ').slice(1));
+};
+
+const getWebhookSecret = async () => {
+    const command = `stripe listen --print-secret ${process.env.CI ? `--api-key ${process.env.STRIPE_SECRET_KEY}` : ''}`.trim();
+    const webhookSecret = (await promisify(exec)(command)).stdout;
+    return webhookSecret.toString().trim();
+};
+
+const generateStripeIntegrationToken = async () => {
+    const publishableKey = process.env.STRIPE_PUBLISHABLE_KEY;
+    const secretKey = process.env.STRIPE_SECRET_KEY;
+    const accountId = process.env.STRIPE_ACCOUNT_ID ?? JSON.parse((await promisify(exec)('stripe get account')).stdout).id;
+
+    return Buffer.from(JSON.stringify({
+        a: secretKey,
+        p: publishableKey,
+        l: false,
+        i: accountId
+    })).toString('base64');
+};
+
+const stubMailgun = () => {
+    // We need to stub the Mailgun client before starting Ghost
+    sinon.stub(MailgunClient.prototype, 'getInstance').returns({
+        // @ts-ignore
+        messages: {
+            create: async function () {
+                return {
+                    id: `mailgun-mock-id-${ObjectID().toHexString()}`
+                };
+            }
+        }
+    });
+};
+
+// Global promises for webhook secret / Stripe integration token
+const webhookSecretPromise = getWebhookSecret();
+const stripeIntegrationTokenPromise = generateStripeIntegrationToken();
+
+module.exports = base.test.extend({
+    baseURL: async ({port, baseURL}, use) => {
+        // Replace the port in baseURL with the one we got from the port fixture
+        const url = new URL(baseURL);
+        url.port = port.toString();
+        await use(url.toString());
+    },
+
+    storageState: async ({ghost}, use) => {
+        await use(ghost.state);
+    },
+
+    // eslint-disable-next-line no-empty-pattern
+    port: [async ({}, use, workerInfo) => {
+        await use(2369 + workerInfo.parallelIndex);
+    }, {scope: 'worker'}],
+
+    ghost: [async ({browser, port}, use, workerInfo) => {
+        process.env.PORT = port;
+
+        const currentDate = new Date();
+        const formattedDate = `${currentDate.getFullYear()}-${String(currentDate.getMonth() + 1).padStart(2, '0')}-${String(currentDate.getDate()).padStart(2, '0')}-${String(currentDate.getHours()).padStart(2, '0')}-${String(currentDate.getMinutes()).padStart(2, '0')}-${String(currentDate.getSeconds()).padStart(2, '0')}`;
+        process.env.database__connection__filename = `/tmp/ghost-playwright.${workerInfo.workerIndex}.${formattedDate}.db`;
+        const sandbox = sinon.createSandbox();
+        const originalConfigGet = config.get.bind(config);
+        sandbox.stub(config, 'get').callsFake((key) => {
+            if (key === 'database:connection:filename') {
+                return process.env.database__connection__filename;
+            }
+            if (key === 'database') {
+                return {
+                    client: 'sqlite3',
+                    connection: {
+                        filename: process.env.database__connection__filename
+                    },
+                    useNullAsDefault: true,
+                    debug: false
+                };
+            }
+            if (key === 'server') {
+                return {
+                    port
+                };
+            }
+            if (key === 'url') {
+                return `http://127.0.0.1:${port}`;
+            }
+            return originalConfigGet(key);
+        });
+
+        const stripeServer = startWebhookServer(port);
+
+        process.env.WEBHOOK_SECRET = await webhookSecretPromise;
+
+        stubMailgun();
+
+        const {startGhost} = require('../../utils/e2e-framework');
+        const server = await startGhost({
+            frontend: true,
+            server: true,
+            backend: true
+        });
+
+        // StartGhost automatically disables network, so we need to re-enable it for Stripe
+        allowStripe();
+
+        const page = await browser.newPage({
+            baseURL: `http://127.0.0.1:${port}/`,
+            storageState: undefined
+        });
+
+        await setupGhost(page);
+        await setupStripe(page, await stripeIntegrationTokenPromise);
+        await setupMailgun(page);
+        await enableLabs(page);
+        const state = await page.context().storageState();
+
+        await page.close();
+
+        // Use the server in the tests.
+        await use({
+            server,
+            state
+        });
+
+        // Cleanup.
+        const {stopGhost} = require('../../utils/e2e-utils');
+        await stopGhost();
+        stripeServer.kill();
+        sandbox.restore();
+    }, {scope: 'worker', auto: true}]
+});

ghost/core/test/e2e-browser/portal/donations.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {createMember, impersonateMember, completeStripeSubscription} = require('../utils');
 
 test.describe('Portal', () => {

ghost/core/test/e2e-browser/portal/member-actions.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {createMember, impersonateMember} = require('../utils');
 
 /**

ghost/core/test/e2e-browser/portal/offers.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {deleteAllMembers, createTier, createOffer, completeStripeSubscription} = require('../utils');
 
 test.describe('Portal', () => {

ghost/core/test/e2e-browser/portal/tiers.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {deleteAllMembers, completeStripeSubscription} = require('../utils');
 
 test.describe('Portal', () => {

ghost/core/test/e2e-browser/portal/upgrade.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {completeStripeSubscription, createMember, impersonateMember} = require('../utils');
 
 test.describe('Portal', () => {

ghost/core/test/e2e-browser/utils/global-setup.js

@@ -1,114 +0,0 @@
-const config = require('../../../core/shared/config');
-const {promisify} = require('util');
-const {spawn, exec} = require('child_process');
-const {knex} = require('../../../core/server/data/db');
-const {setupGhost, setupStripe, setupMailgun, enableLabs} = require('./e2e-browser-utils');
-const {chromium} = require('@playwright/test');
-const {startGhost} = require('../../utils/e2e-framework');
-const {stopGhost} = require('../../utils/e2e-utils');
-const MailgunClient = require('@tryghost/mailgun-client');
-const sinon = require('sinon');
-const ObjectID = require('bson-objectid').default;
-const {allowStripe} = require('../../utils/e2e-framework-mock-manager');
-
-const startWebhookServer = () => {
-    const command = `stripe listen --forward-to ${config.getSiteUrl()}members/webhooks/stripe/ ${process.env.CI ? `--api-key ${process.env.STRIPE_SECRET_KEY}` : ''}`.trim();
-    spawn(command.split(' ')[0], command.split(' ').slice(1));
-};
-
-const getWebhookSecret = async () => {
-    const command = `stripe listen --print-secret ${process.env.CI ? `--api-key ${process.env.STRIPE_SECRET_KEY}` : ''}`.trim();
-    const webhookSecret = (await promisify(exec)(command)).stdout;
-    return webhookSecret.toString().trim();
-};
-
-const generateStripeIntegrationToken = async () => {
-    const inquirer = require('inquirer');
-    const stripeDatabaseKeys = {
-        publishableKey: 'stripe_connect_publishable_key',
-        secretKey: 'stripe_connect_secret_key',
-        liveMode: 'stripe_connect_livemode'
-    };
-    const publishableKey = process.env.STRIPE_PUBLISHABLE_KEY ?? (await knex('settings').select('value').where('key', stripeDatabaseKeys.publishableKey).first())?.value
-        ?? (await inquirer.prompt([{
-            message: 'Stripe publishable key (starts "pk_test_")',
-            type: 'password',
-            name: 'value'
-        }])).value;
-    const secretKey = process.env.STRIPE_SECRET_KEY ?? (await knex('settings').select('value').where('key', stripeDatabaseKeys.secretKey).first())?.value
-        ?? (await inquirer.prompt([{
-            message: 'Stripe secret key (starts "sk_test_")',
-            type: 'password',
-            name: 'value'
-        }])).value;
-
-    const accountId = process.env.STRIPE_ACCOUNT_ID ?? JSON.parse((await promisify(exec)('stripe get account')).stdout).id;
-
-    return Buffer.from(JSON.stringify({
-        a: secretKey,
-        p: publishableKey,
-        l: false,
-        i: accountId
-    })).toString('base64');
-};
-
-const stubMailgun = () => {
-    // We need to stub the Mailgun client before starting Ghost
-    sinon.stub(MailgunClient.prototype, 'getInstance').returns({
-        // @ts-ignore
-        messages: {
-            create: async function () {
-                return {
-                    id: `mailgun-mock-id-${ObjectID().toHexString()}`
-                };
-            }
-        }
-    });
-};
-
-/**
- * Setup the environment
- */
-const setup = async (playwrightConfig) => {
-    const usingRemoteServer = process.env.CI && process.env.TEST_URL;
-
-    let stripeConnectIntegrationToken;
-    if (!usingRemoteServer) {
-        startWebhookServer();
-        stripeConnectIntegrationToken = await generateStripeIntegrationToken();
-
-        process.env.WEBHOOK_SECRET = await getWebhookSecret();
-
-        // Stub out NodeMailer
-        stubMailgun();
-
-        await startGhost({
-            frontend: true,
-            server: true,
-            backend: true
-        });
-
-        // StartGhost automatically disables network, so we need to re-enable it for Stripe
-        allowStripe();
-    }
-
-    const {baseURL, storageState} = playwrightConfig.projects[0].use;
-    const browser = await chromium.launch();
-    const page = await browser.newPage({
-        baseURL
-    });
-    await setupGhost(page);
-    if (!usingRemoteServer) {
-        await setupStripe(page, stripeConnectIntegrationToken);
-        await setupMailgun(page);
-    }
-    await enableLabs(page);
-    await page.context().storageState({path: storageState});
-    await browser.close();
-
-    if (!usingRemoteServer) {
-        await stopGhost();
-    }
-};
-
-module.exports = setup;

ghost/core/test/e2e-browser/utils/global-teardown.js

@@ -1,8 +0,0 @@
-/**
- * Teardown the environment
- */
-const teardown = async () => {
-    // @NOTE: local environment should probably drop the db state here
-};
-
-module.exports = teardown;

ghost/core/test/e2e-browser/utils/index.js

@@ -1,5 +1,4 @@
 module.exports = {
     ...require('./e2e-browser-utils'),
-    ...require('./helpers'),
-    globalSetup: require('./global-setup')
+    ...require('./helpers')
 };