From c5bac09a42d0bf2a3a9b53fed3743291c2109e43 Mon Sep 17 00:00:00 2001 From: Bjorn Lu Date: Mon, 6 Mar 2023 22:55:44 +0800 Subject: [PATCH] Add page render benchmark (#6415) --- .github/workflows/benchmark.yml | 2 +- benchmark/bench/render.js | 122 +++++++++++++++++++++ benchmark/index.js | 2 + benchmark/make-project/_util.js | 10 ++ benchmark/make-project/render-default.js | 87 +++++++++++++++ benchmark/package.json | 2 + packages/integrations/timer/README.md | 3 + packages/integrations/timer/package.json | 43 ++++++++ packages/integrations/timer/src/index.ts | 34 ++++++ packages/integrations/timer/src/preview.ts | 36 ++++++ packages/integrations/timer/src/server.ts | 21 ++++ packages/integrations/timer/tsconfig.json | 10 ++ pnpm-lock.yaml | 19 ++++ 13 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 benchmark/bench/render.js create mode 100644 benchmark/make-project/render-default.js create mode 100644 packages/integrations/timer/README.md create mode 100644 packages/integrations/timer/package.json create mode 100644 packages/integrations/timer/src/index.ts create mode 100644 packages/integrations/timer/src/preview.ts create mode 100644 packages/integrations/timer/src/server.ts create mode 100644 packages/integrations/timer/tsconfig.json diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6f0b2a5707..89934f4928 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -95,7 +95,7 @@ jobs: continue-on-error: true with: issue-number: ${{ github.event.issue.number }} - message: | + body: | ${{ needs.benchmark.outputs.PR-BENCH }} ${{ needs.benchmark.outputs.MAIN-BENCH }} diff --git a/benchmark/bench/render.js b/benchmark/bench/render.js new file mode 100644 index 0000000000..59214b7881 --- /dev/null +++ b/benchmark/bench/render.js @@ -0,0 +1,122 @@ +import fs from 'fs/promises'; +import http from 'http'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { execaCommand } from 'execa'; +import { waitUntilBusy } from 'port-authority'; +import { markdownTable } from 'markdown-table'; +import { renderFiles } from '../make-project/render-default.js'; +import { astroBin } from './_util.js'; + +const port = 4322; + +export const defaultProject = 'render-default'; + +/** @typedef {{ avg: number, stdev: number, max: number }} Stat */ + +/** + * @param {URL} projectDir + * @param {URL} outputFile + */ +export async function run(projectDir, outputFile) { + const root = fileURLToPath(projectDir); + + console.log('Building...'); + await execaCommand(`${astroBin} build`, { + cwd: root, + stdio: 'inherit', + }); + + console.log('Previewing...'); + const previewProcess = execaCommand(`${astroBin} preview --port ${port}`, { + cwd: root, + stdio: 'inherit', + }); + + console.log('Waiting for server ready...'); + await waitUntilBusy(port, { timeout: 5000 }); + + console.log('Running benchmark...'); + const result = await benchmarkRenderTime(); + + console.log('Killing server...'); + if (!previewProcess.kill('SIGTERM')) { + console.warn('Failed to kill server process id:', previewProcess.pid); + } + + console.log('Writing results to', fileURLToPath(outputFile)); + await fs.writeFile(outputFile, JSON.stringify(result, null, 2)); + + console.log('Result preview:'); + console.log('='.repeat(10)); + console.log(`#### Render\n\n`); + console.log(printResult(result)); + console.log('='.repeat(10)); + + console.log('Done!'); +} + +async function benchmarkRenderTime() { + /** @type {Record} */ + const result = {}; + for (const fileName of Object.keys(renderFiles)) { + // Render each file 100 times and push to an array + for (let i = 0; i < 100; i++) { + const pathname = '/' + fileName.slice(0, -path.extname(fileName).length); + const renderTime = await fetchRenderTime(`http://localhost:${port}${pathname}`); + if (!result[pathname]) result[pathname] = []; + result[pathname].push(renderTime); + } + } + /** @type {Record} */ + const processedResult = {}; + for (const [pathname, times] of Object.entries(result)) { + // From the 100 results, calculate average, standard deviation, and max value + const avg = times.reduce((a, b) => a + b, 0) / times.length; + const stdev = Math.sqrt( + times.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / times.length + ); + const max = Math.max(...times); + processedResult[pathname] = { avg, stdev, max }; + } + return processedResult; +} + +/** + * @param {Record} result + */ +function printResult(result) { + return markdownTable( + [ + ['Page', 'Avg (ms)', 'Stdev (ms)', 'Max (ms)'], + ...Object.entries(result).map(([pathname, { avg, stdev, max }]) => [ + pathname, + avg.toFixed(2), + stdev.toFixed(2), + max.toFixed(2), + ]), + ], + { + align: ['l', 'r', 'r', 'r'], + } + ); +} + +/** + * Simple fetch utility to get the render time sent by `@astrojs/timer` in plain text + * @param {string} url + * @returns {Promise} + */ +function fetchRenderTime(url) { + return new Promise((resolve, reject) => { + const req = http.request(url, (res) => { + res.setEncoding('utf8'); + let data = ''; + res.on('data', (chunk) => (data += chunk)); + res.on('error', (e) => reject(e)); + res.on('end', () => resolve(+data)); + }); + req.on('error', (e) => reject(e)); + req.end(); + }); +} diff --git a/benchmark/index.js b/benchmark/index.js index 6ac76759c0..c05fafefa3 100755 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -12,6 +12,7 @@ astro-benchmark [options] Command [empty] Run all benchmarks memory Run build memory and speed test + render Run rendering speed test server-stress Run server stress test Options @@ -24,6 +25,7 @@ Options const commandName = args._[0]; const benchmarks = { memory: () => import('./bench/memory.js'), + 'render': () => import('./bench/render.js'), 'server-stress': () => import('./bench/server-stress.js'), }; diff --git a/benchmark/make-project/_util.js b/benchmark/make-project/_util.js index c0e17965b0..65c91dbf3f 100644 --- a/benchmark/make-project/_util.js +++ b/benchmark/make-project/_util.js @@ -1,2 +1,12 @@ export const loremIpsum = "Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."; + +export const loremIpsumHtml = loremIpsum + .replace(/Lorem/g, 'Lorem') + .replace(/Ipsum/g, 'Ipsum') + .replace(/dummy/g, 'dummy'); + +export const loremIpsumMd = loremIpsum + .replace(/Lorem/g, '**Lorem**') + .replace(/Ipsum/g, '_Ipsum_') + .replace(/dummy/g, '`dummy`'); diff --git a/benchmark/make-project/render-default.js b/benchmark/make-project/render-default.js new file mode 100644 index 0000000000..9dfe886098 --- /dev/null +++ b/benchmark/make-project/render-default.js @@ -0,0 +1,87 @@ +import fs from 'fs/promises'; +import { loremIpsumHtml, loremIpsumMd } from './_util.js'; + +// Map of files to be generated and tested for rendering. +// Ideally each content should be similar for comparison. +export const renderFiles = { + 'astro.astro': `\ +--- +const className = "text-red-500"; +const style = { color: "red" }; +const items = Array.from({ length: 1000 }, (_, i) => i); +--- + + + + My Site + + +

List

+
    + {items.map((item) => ( +
  • {item}
    • + ))} +
+ ${Array.from({ length: 1000 }) + .map(() => `

${loremIpsumHtml}

`) + .join('\n')} + +`, + 'md.md': `\ +# List + +${Array.from({ length: 1000 }, (_, i) => i) + .map((v) => `- ${v}`) + .join('\n')} + +${Array.from({ length: 1000 }) + .map(() => loremIpsumMd) + .join('\n\n')} +`, + 'mdx.mdx': `\ +export const className = "text-red-500"; +export const style = { color: "red" }; +export const items = Array.from({ length: 1000 }, (_, i) => i); + +# List + +
    + {items.map((item) => ( +
  • {item}
    • + ))} +
+ +${Array.from({ length: 1000 }) + .map(() => loremIpsumMd) + .join('\n\n')} +`, +}; + +/** + * @param {URL} projectDir + */ +export async function run(projectDir) { + await fs.rm(projectDir, { recursive: true, force: true }); + await fs.mkdir(new URL('./src/pages', projectDir), { recursive: true }); + + await Promise.all( + Object.entries(renderFiles).map(([name, content]) => { + return fs.writeFile(new URL(`./src/pages/${name}`, projectDir), content, 'utf-8'); + }) + ); + + await fs.writeFile( + new URL('./astro.config.js', projectDir), + `\ +import { defineConfig } from 'astro/config'; +import timer from '@astrojs/timer'; +import mdx from '@astrojs/mdx'; + +export default defineConfig({ + integrations: [mdx()], + output: 'server', + adapter: timer(), +});`, + 'utf-8' + ); +} diff --git a/benchmark/package.json b/benchmark/package.json index 34b486e978..85ba91e872 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -7,7 +7,9 @@ "astro-benchmark": "./index.js" }, "dependencies": { + "@astrojs/mdx": "workspace:*", "@astrojs/node": "workspace:*", + "@astrojs/timer": "workspace:*", "astro": "workspace:*", "autocannon": "^7.10.0", "execa": "^6.1.0", diff --git a/packages/integrations/timer/README.md b/packages/integrations/timer/README.md new file mode 100644 index 0000000000..81124745d8 --- /dev/null +++ b/packages/integrations/timer/README.md @@ -0,0 +1,3 @@ +# @astrojs/timer + +Like `@astrojs/node`, but returns the rendered time in milliseconds for the page instead of the page content itself. This is used for internal benchmarks only. diff --git a/packages/integrations/timer/package.json b/packages/integrations/timer/package.json new file mode 100644 index 0000000000..13203a2d55 --- /dev/null +++ b/packages/integrations/timer/package.json @@ -0,0 +1,43 @@ +{ + "name": "@astrojs/timer", + "description": "Preview server for benchmark", + "private": true, + "version": "0.0.0", + "type": "module", + "types": "./dist/index.d.ts", + "author": "withastro", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/withastro/astro.git", + "directory": "packages/integrations/timer" + }, + "keywords": [ + "withastro", + "astro-adapter" + ], + "bugs": "https://github.com/withastro/astro/issues", + "exports": { + ".": "./dist/index.js", + "./server.js": "./dist/server.js", + "./preview.js": "./dist/preview.js", + "./package.json": "./package.json" + }, + "scripts": { + "build": "astro-scripts build \"src/**/*.ts\" && tsc", + "build:ci": "astro-scripts build \"src/**/*.ts\"", + "dev": "astro-scripts dev \"src/**/*.ts\"" + }, + "dependencies": { + "@astrojs/webapi": "workspace:*", + "server-destroy": "^1.0.1" + }, + "peerDependencies": { + "astro": "workspace:^2.0.17" + }, + "devDependencies": { + "@types/server-destroy": "^1.0.1", + "astro": "workspace:*", + "astro-scripts": "workspace:*" + } +} diff --git a/packages/integrations/timer/src/index.ts b/packages/integrations/timer/src/index.ts new file mode 100644 index 0000000000..feeaa2122c --- /dev/null +++ b/packages/integrations/timer/src/index.ts @@ -0,0 +1,34 @@ +import type { AstroAdapter, AstroIntegration } from 'astro'; + +export function getAdapter(): AstroAdapter { + return { + name: '@astrojs/timer', + serverEntrypoint: '@astrojs/timer/server.js', + previewEntrypoint: '@astrojs/timer/preview.js', + exports: ['handler'], + }; +} + +export default function createIntegration(): AstroIntegration { + return { + name: '@astrojs/timer', + hooks: { + 'astro:config:setup': ({ updateConfig }) => { + updateConfig({ + vite: { + ssr: { + noExternal: ['@astrojs/timer'], + }, + }, + }); + }, + 'astro:config:done': ({ setAdapter, config }) => { + setAdapter(getAdapter()); + + if (config.output === 'static') { + console.warn(`[@astrojs/timer] \`output: "server"\` is required to use this adapter.`); + } + }, + }, + }; +} diff --git a/packages/integrations/timer/src/preview.ts b/packages/integrations/timer/src/preview.ts new file mode 100644 index 0000000000..1208830dd3 --- /dev/null +++ b/packages/integrations/timer/src/preview.ts @@ -0,0 +1,36 @@ +import type { CreatePreviewServer } from 'astro'; +import { createServer } from 'http'; +import enableDestroy from 'server-destroy'; + +const preview: CreatePreviewServer = async function ({ serverEntrypoint, host, port }) { + const ssrModule = await import(serverEntrypoint.toString()); + const ssrHandler = ssrModule.handler; + const server = createServer(ssrHandler); + server.listen(port, host); + enableDestroy(server); + + // eslint-disable-next-line no-console + console.log(`Preview server listening on http://${host}:${port}`); + + // Resolves once the server is closed + const closed = new Promise((resolve, reject) => { + server.addListener('close', resolve); + server.addListener('error', reject); + }); + + return { + host, + port, + closed() { + return closed; + }, + server, + stop: async () => { + await new Promise((resolve, reject) => { + server.destroy((err) => (err ? reject(err) : resolve(undefined))); + }); + }, + }; +}; + +export { preview as default }; diff --git a/packages/integrations/timer/src/server.ts b/packages/integrations/timer/src/server.ts new file mode 100644 index 0000000000..0f609fd50d --- /dev/null +++ b/packages/integrations/timer/src/server.ts @@ -0,0 +1,21 @@ +import { polyfill } from '@astrojs/webapi'; +import type { IncomingMessage, ServerResponse } from 'http'; +import type { SSRManifest } from 'astro'; +import { NodeApp } from 'astro/app/node'; + +polyfill(globalThis, { + exclude: 'window document', +}); + +export function createExports(manifest: SSRManifest) { + const app = new NodeApp(manifest); + return { + handler: async (req: IncomingMessage, res: ServerResponse) => { + const start = performance.now(); + await app.render(req); + const end = performance.now(); + res.write(end - start + ''); + res.end(); + }, + }; +} diff --git a/packages/integrations/timer/tsconfig.json b/packages/integrations/timer/tsconfig.json new file mode 100644 index 0000000000..44baf375c8 --- /dev/null +++ b/packages/integrations/timer/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../tsconfig.base.json", + "include": ["src"], + "compilerOptions": { + "allowJs": true, + "module": "ES2020", + "outDir": "./dist", + "target": "ES2020" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 25643eafdf..cebf0d4b39 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,7 +65,9 @@ importers: benchmark: specifiers: + '@astrojs/mdx': workspace:* '@astrojs/node': workspace:* + '@astrojs/timer': workspace:* astro: workspace:* autocannon: ^7.10.0 execa: ^6.1.0 @@ -74,7 +76,9 @@ importers: port-authority: ^2.0.1 pretty-bytes: ^6.0.0 dependencies: + '@astrojs/mdx': link:../packages/integrations/mdx '@astrojs/node': link:../packages/integrations/node + '@astrojs/timer': link:../packages/integrations/timer astro: link:../packages/astro autocannon: 7.10.0 execa: 6.1.0 @@ -3375,6 +3379,21 @@ importers: tailwindcss: 3.2.6_postcss@8.4.21 vite: 4.1.2 + packages/integrations/timer: + specifiers: + '@astrojs/webapi': workspace:* + '@types/server-destroy': ^1.0.1 + astro: workspace:* + astro-scripts: workspace:* + server-destroy: ^1.0.1 + dependencies: + '@astrojs/webapi': link:../../webapi + server-destroy: 1.0.1 + devDependencies: + '@types/server-destroy': 1.0.1 + astro: link:../../astro + astro-scripts: link:../../../scripts + packages/integrations/turbolinks: specifiers: astro: workspace:*