diff --git a/.github/workflows/continuous_benchmark.yml b/.github/workflows/continuous_benchmark.yml index a1d48bb8d9..1b382f8dc6 100644 --- a/.github/workflows/continuous_benchmark.yml +++ b/.github/workflows/continuous_benchmark.yml @@ -50,6 +50,7 @@ jobs: uses: CodSpeedHQ/action@513a19673a831f139e8717bf45ead67e47f00044 # v3.2.0 timeout-minutes: 30 with: - run: pnpm benchmark codspeed + working-directory: ./benchmark + run: pnpm bench token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/benchmark/bench/_util.js b/benchmark/bench/_util.js index b16a16e1ce..d9dfe5b190 100644 --- a/benchmark/bench/_util.js +++ b/benchmark/bench/_util.js @@ -19,3 +19,14 @@ export function calculateStat(numbers) { const max = Math.max(...numbers); return { avg, stdev, max }; } + +export async function makeProject(name) { + console.log('Making project:', name); + const projectDir = new URL(`../projects/${name}/`, import.meta.url); + + const makeProjectMod = await import(`../make-project/${name}.js`); + await makeProjectMod.run(projectDir); + + console.log('Finished making project:', name); + return projectDir; +} diff --git a/benchmark/bench/codspeed.bench.js b/benchmark/bench/codspeed.bench.js new file mode 100644 index 0000000000..4073ebed85 --- /dev/null +++ b/benchmark/bench/codspeed.bench.js @@ -0,0 +1,48 @@ +import { fileURLToPath } from 'node:url'; +import { exec } from 'tinyexec'; +import { beforeAll, bench, describe } from 'vitest'; +import { astroBin, makeProject } from './_util.js'; +let streamingApp; +let nonStreamingApp; +beforeAll(async () => { + const render = await makeProject('render-bench'); + const root = fileURLToPath(render); + await exec(astroBin, ['build'], { + nodeOptions: { + cwd: root, + stdio: 'inherit', + }, + }); + const entry = new URL('./dist/server/entry.mjs', `file://${root}`); + const { manifest, createApp } = await import(entry); + streamingApp = createApp(manifest, true); + nonStreamingApp = createApp(manifest, false); +}, 900000); + +describe('Bench rendering', () => { + bench('Rendering: streaming [true], .astro file', async () => { + const request = new Request(new URL('http://exmpale.com/astro')); + await streamingApp.render(request); + }); + bench('Rendering: streaming [true], .md file', async () => { + const request = new Request(new URL('http://exmpale.com/md')); + await streamingApp.render(request); + }); + bench('Rendering: streaming [true], .mdx file', async () => { + const request = new Request(new URL('http://exmpale.com/mdx')); + await streamingApp.render(request); + }); + + bench('Rendering: streaming [false], .astro file', async () => { + const request = new Request(new URL('http://exmpale.com/astro')); + await nonStreamingApp.render(request); + }); + bench('Rendering: streaming [false], .md file', async () => { + const request = new Request(new URL('http://exmpale.com/md')); + await nonStreamingApp.render(request); + }); + bench('Rendering: streaming [false], .mdx file', async () => { + const request = new Request(new URL('http://exmpale.com/mdx')); + await nonStreamingApp.render(request); + }); +}); diff --git a/benchmark/bench/codspeed.js b/benchmark/bench/codspeed.js deleted file mode 100644 index 2643b1ec8a..0000000000 --- a/benchmark/bench/codspeed.js +++ /dev/null @@ -1,64 +0,0 @@ -import path from 'node:path'; -import { withCodSpeed } from '@codspeed/tinybench-plugin'; -import { Bench } from 'tinybench'; -import { exec } from 'tinyexec'; -import { astroBin } from './_util.js'; - -export async function run({ memory: _memory, render, stress: _stress }) { - const options = { - iterations: 10, - }; - const bench = process.env.CODSPEED ? withCodSpeed(new Bench(options)) : new Bench(options); - await exec(astroBin, ['build'], { - nodeOptions: { - cwd: render.root, - stdio: 'inherit', - }, - }); - - const entry = new URL('./dist/server/entry.mjs', `file://${render.root}`); - const { manifest, createApp } = await import(entry); - const streamingApp = createApp(manifest, true); - const nonStreamingApp = createApp(manifest, false); - bench - .add('Rendering: streaming [true], .astro file', async () => { - console.info('Start task.'); - const request = new Request(new URL('http://exmpale.com/astro')); - await streamingApp.render(request); - console.info('Finish task.'); - }) - .add('Rendering: streaming [true], .md file', async () => { - console.info('Start task.'); - const request = new Request(new URL('http://exmpale.com/md')); - await streamingApp.render(request); - console.info('Finish task.'); - }) - .add('Rendering: streaming [true], .mdx file', async () => { - console.info('Start task.'); - const request = new Request(new URL('http://exmpale.com/mdx')); - await streamingApp.render(request); - console.info('Finish task.'); - }) - - .add('Rendering: streaming [false], .astro file', async () => { - console.info('Start task.'); - const request = new Request(new URL('http://exmpale.com/astro')); - await nonStreamingApp.render(request); - console.info('Finish task.'); - }) - .add('Rendering: streaming [false], .md file', async () => { - console.info('Start task.'); - const request = new Request(new URL('http://exmpale.com/md')); - await nonStreamingApp.render(request); - console.info('Finish task.'); - }) - .add('Rendering: streaming [false], .mdx file', async () => { - console.info('Start task.'); - const request = new Request(new URL('http://exmpale.com/mdx')); - await nonStreamingApp.render(request); - console.info('Finish task.'); - }); - - await bench.run(); - console.table(bench.table()); -} diff --git a/benchmark/index.js b/benchmark/index.js index 2f2846e1db..127fc4f5bf 100755 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -1,7 +1,8 @@ import mri from 'mri'; import fs from 'node:fs/promises'; import path from 'node:path'; -import {fileURLToPath, pathToFileURL} from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { makeProject } from './bench/_util.js'; const args = mri(process.argv.slice(2)); @@ -14,7 +15,6 @@ Command memory Run build memory and speed test render Run rendering speed test server-stress Run server stress test - codspeed Run codspeed test cli-startup Run CLI startup speed test Options @@ -30,7 +30,6 @@ const benchmarks = { render: () => import('./bench/render.js'), 'server-stress': () => import('./bench/server-stress.js'), 'cli-startup': () => import('./bench/cli-startup.js'), - codspeed: () => import('./bench/codspeed.js') }; if (commandName && !(commandName in benchmarks)) { @@ -39,26 +38,12 @@ if (commandName && !(commandName in benchmarks)) { } if (commandName) { - if (commandName === 'codspeed') { - const render = await makeProject('render-bench'); - const rootRender = fileURLToPath(render); - const bench = benchmarks[commandName]; - const benchMod = await bench(); - const payload = { - render: { - root: rootRender, - output: await getOutputFile('render') - }, - }; - await benchMod.run(payload); - } else { - // Run single benchmark - const bench = benchmarks[commandName]; - const benchMod = await bench(); - const projectDir = await makeProject(args.project || benchMod.defaultProject); - const outputFile = await getOutputFile(commandName); - await benchMod.run(projectDir, outputFile); - } + // Run single benchmark + const bench = benchmarks[commandName]; + const benchMod = await bench(); + const projectDir = await makeProject(args.project || benchMod.defaultProject); + const outputFile = await getOutputFile(commandName); + await benchMod.run(projectDir, outputFile); } else { // Run all benchmarks for (const name in benchmarks) { @@ -70,21 +55,10 @@ if (commandName) { } } -export async function makeProject(name) { - console.log('Making project:', name); - const projectDir = new URL(`./projects/${name}/`, import.meta.url); - - const makeProjectMod = await import(`./make-project/${name}.js`); - await makeProjectMod.run(projectDir); - - console.log('Finished making project:', name); - return projectDir; -} - /** * @param {string} benchmarkName */ -async function getOutputFile(benchmarkName) { +export async function getOutputFile(benchmarkName) { let file; if (args.output) { file = pathToFileURL(path.resolve(args.output)); diff --git a/benchmark/make-project/markdown-cc1.js b/benchmark/make-project/markdown-cc1.js index 6c83959601..1e3aaa5177 100644 --- a/benchmark/make-project/markdown-cc1.js +++ b/benchmark/make-project/markdown-cc1.js @@ -8,11 +8,13 @@ export async function run(projectDir) { await fs.rm(projectDir, { recursive: true, force: true }); await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true }); await fs.mkdir(new URL('./src/content/blog', projectDir), { recursive: true }); - await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./src/image.jpg', projectDir)); + await fs.copyFile( + new URL('./image.jpg', import.meta.url), + new URL('./src/image.jpg', projectDir), + ); const promises = []; - for (let i = 0; i < 10000; i++) { const content = `\ # Article ${i} @@ -24,11 +26,10 @@ ${loremIpsumMd} `; promises.push( - fs.writeFile(new URL(`./src/content/blog/article-${i}.md`, projectDir), content, 'utf-8') + fs.writeFile(new URL(`./src/content/blog/article-${i}.md`, projectDir), content, 'utf-8'), ); } - await fs.writeFile( new URL(`./src/pages/blog/[...slug].astro`, projectDir), `\ @@ -46,7 +47,7 @@ const { Content } = await entry.render();