diff --git a/resources/benchmark/args.ts b/resources/benchmark/args.ts index 0e3460c48b..ae26c313f3 100644 --- a/resources/benchmark/args.ts +++ b/resources/benchmark/args.ts @@ -6,15 +6,40 @@ import { localRepoPath } from '../utils.js'; import { LOCAL } from './config.js'; import { bold } from './output.js'; +export type Runtime = 'node' | 'deno' | 'bun'; + export interface BenchmarkArguments { benchmarks: Array; revisions: Array; + runtime: Runtime; } export function getArguments(argv: ReadonlyArray): BenchmarkArguments { - const revsIndex = argv.indexOf('--revs'); - const revisions = revsIndex === -1 ? [] : argv.slice(revsIndex + 1); - const benchmarks = revsIndex === -1 ? [...argv] : argv.slice(0, revsIndex); + const runtimeIndex = argv.indexOf('--runtime'); + const runtimeValue = + runtimeIndex === -1 + ? inferRuntimeFromExecPath(process.execPath) + : argv[runtimeIndex + 1]; + if ( + runtimeValue !== 'node' && + runtimeValue !== 'deno' && + runtimeValue !== 'bun' + ) { + throw new Error( + `Invalid --runtime value: "${runtimeValue}". Must be "node", "deno", or "bun".`, + ); + } + const runtime: Runtime = runtimeValue; + + const filteredArgv = + runtimeIndex === -1 + ? [...argv] + : [...argv.slice(0, runtimeIndex), ...argv.slice(runtimeIndex + 2)]; + + const revsIndex = filteredArgv.indexOf('--revs'); + const revisions = revsIndex === -1 ? [] : filteredArgv.slice(revsIndex + 1); + const benchmarks = + revsIndex === -1 ? [...filteredArgv] : filteredArgv.slice(0, revsIndex); switch (revisions.length) { case 0: @@ -33,7 +58,19 @@ export function getArguments(argv: ReadonlyArray): BenchmarkArguments { benchmarks.push(...findAllBenchmarks()); } - return { benchmarks, revisions }; + return { benchmarks, revisions, runtime }; +} + +function inferRuntimeFromExecPath(execPath: string): Runtime { + const executableName = path.basename(execPath).toLowerCase(); + + if (executableName.startsWith('deno')) { + return 'deno'; + } + if (executableName.startsWith('bun')) { + return 'bun'; + } + return 'node'; } function findAllBenchmarks(): Array { diff --git a/resources/benchmark/config.ts b/resources/benchmark/config.ts index 099e85af57..b347c18a9a 100644 --- a/resources/benchmark/config.ts +++ b/resources/benchmark/config.ts @@ -16,12 +16,31 @@ export const memorySamplesPerBenchmark = 10; export const pairedGreenThreshold = 0.95; export const pairedYellowThreshold = 0.8; -export const timingBenchmarkNodeFlags: ReadonlyArray = ['--expose-gc']; +export const timingCommandArgsByRuntime = { + node: ['--expose-gc'], + bun: ['--expose-gc'], + deno: [ + 'run', + '--allow-read', + '--allow-env', + '--allow-write', + '--v8-flags=--expose-gc', + ], +} as const; -export const memoryBenchmarkNodeFlags: ReadonlyArray = [ - '--predictable', - '--no-concurrent-sweeping', - '--no-minor-gc-task', - '--min-semi-space-size=1280', // 1.25GB - '--max-semi-space-size=1280', // 1.25GB -]; +export const memoryCommandArgsByRuntime = { + node: [ + '--no-concurrent-sweeping', + '--no-minor-gc-task', + '--min-semi-space-size=1280', // 1.25GB + '--max-semi-space-size=1280', // 1.25GB + ], + bun: [], + deno: [ + 'run', + '--allow-read', + '--allow-env', + '--allow-write', + '--v8-flags=--no-concurrent-sweeping,--no-minor-gc-task,--min-semi-space-size=1280,--max-semi-space-size=1280', + ], +} as const; diff --git a/resources/benchmark/output.ts b/resources/benchmark/output.ts index 400a2c1c70..42e1a487ca 100644 --- a/resources/benchmark/output.ts +++ b/resources/benchmark/output.ts @@ -5,6 +5,7 @@ type ColorFn = (value: number | string) => string; export function printBenchmarkResults( results: ReadonlyArray, + includesMemory: boolean, ): void { const nameMaxLen = maxBy(results, ({ name }) => name.length); const opsTop = maxBy(results, ({ ops }) => ops); @@ -29,9 +30,7 @@ export function printBenchmarkResults( grey('\xb1') + deviationStr() + cyan('%') + - grey(' x ') + - memPerOpStr() + - '/op' + + (includesMemory ? grey(' x ') + memPerOpStr() + '/op' : '') + grey(' (' + numSamples + ' runs sampled)'), ); diff --git a/resources/benchmark/run.ts b/resources/benchmark/run.ts index 913789a17e..2a966ee1c0 100644 --- a/resources/benchmark/run.ts +++ b/resources/benchmark/run.ts @@ -1,6 +1,8 @@ import assert from 'node:assert'; import path from 'node:path'; +import readline from 'node:readline'; +import type { Runtime } from './args.js'; import { getArguments } from './args.js'; import { maxTime, @@ -9,6 +11,7 @@ import { } from './config.js'; import { cyan, + grey, printBenchmarkResults, printPairedComparisons, red, @@ -28,11 +31,17 @@ import { export function runBenchmarks(): void { // Get the revisions and make things happen! - const { benchmarks, revisions } = getArguments(process.argv.slice(2)); + const { benchmarks, revisions, runtime } = getArguments( + process.argv.slice(2), + ); const benchmarkProjects = prepareBenchmarkProjects(revisions); + console.log(''); + console.log('\u2699\uFE0F Runtime: ' + runtime); + console.log(''); + for (const benchmark of benchmarks) { - runBenchmark(benchmark, benchmarkProjects); + runBenchmark(benchmark, benchmarkProjects, runtime); } } @@ -40,20 +49,28 @@ export function runBenchmarks(): void { function runBenchmark( benchmark: string, benchmarkProjects: ReadonlyArray, + runtime: Runtime, ): void { - const memorySamples: Array> = []; + const memorySamples: Array | undefined> = []; + const includesMemory = runtime !== 'bun'; + for (let i = 0; i < benchmarkProjects.length; ++i) { const modulePath = path.join(benchmarkProjects[i].projectPath, benchmark); if (i === 0) { - console.log('\u23F1 ' + getBenchmarkName(modulePath)); + console.log('\u23F1 ' + getBenchmarkName(modulePath, runtime)); + if (includesMemory) { + writeProgress(' completed ' + cyan(0) + ' memory tests...'); + } + } + + if (!includesMemory) { + continue; } try { - memorySamples[i] = collectMemorySamples(modulePath); - process.stdout.write( - ' completed ' + cyan(i + 1) + ' memory tests...\u000D', - ); + memorySamples[i] = collectMemorySamples(modulePath, runtime); + writeProgress(' completed ' + cyan(i + 1) + ' memory tests...'); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); @@ -63,11 +80,15 @@ function runBenchmark( return; } } - process.stdout.write('\n'); + if (includesMemory) { + endProgressLine(); + } else { + console.log(' ' + grey('skipping memory benchmarks for bun runtime')); + } let timingSamples: Array>; try { - timingSamples = collectTimingSamples(benchmark, benchmarkProjects); + timingSamples = collectTimingSamples(benchmark, benchmarkProjects, runtime); } catch { console.log(' ' + red('timing samples collection failed')); return; @@ -80,7 +101,7 @@ function runBenchmark( result = computeStats( benchmarkProjects[i].revision, timingSamples[i], - memorySamples[i], + includesMemory ? memorySamples[i] : undefined, ); } catch (error) { const errorMessage = @@ -96,7 +117,7 @@ function runBenchmark( console.log('\n'); - printBenchmarkResults(results); + printBenchmarkResults(results, includesMemory); printPairedComparisons( getPairedComparisons( benchmarkProjects.map(({ revision }) => revision), @@ -109,6 +130,7 @@ function runBenchmark( function collectTimingSamples( benchmark: string, benchmarkProjects: ReadonlyArray, + runtime: Runtime, ): Array> { const sampleGroups = benchmarkProjects.map((project) => ({ revision: project.revision, @@ -122,6 +144,7 @@ function collectTimingSamples( // pairwise revision comparison has stabilized. const start = Date.now(); let round = 0; + writeProgress(' completed ' + cyan(0) + ' timing rounds...'); while ( (Date.now() - start) / 1e3 < maxTime && (round < minTimingSamplesPerBenchmark || @@ -129,7 +152,7 @@ function collectTimingSamples( ) { for (const sampleGroup of shuffled(sampleGroups)) { try { - const sample = sampleTimingModule(sampleGroup.modulePath); + const sample = sampleTimingModule(sampleGroup.modulePath, runtime); assert(sample > 0); sampleGroup.samples.push(sample); @@ -142,13 +165,21 @@ function collectTimingSamples( } ++round; - process.stdout.write( - ' completed ' + cyan(round) + ' timing rounds...\u000D', - ); + writeProgress(' completed ' + cyan(round) + ' timing rounds...'); } return timingSamples; } +function writeProgress(message: string): void { + readline.clearLine(process.stdout, 0); + readline.cursorTo(process.stdout, 0); + process.stdout.write(message); +} + +function endProgressLine(): void { + process.stdout.write('\n'); +} + function shuffled(array: ReadonlyArray): Array { const shuffledArray = [...array]; // Fisher-Yates shuffle: walk backward and swap each slot with a random @@ -163,14 +194,17 @@ function shuffled(array: ReadonlyArray): Array { return shuffledArray; } -export function collectMemorySamples(modulePath: string): Array { +export function collectMemorySamples( + modulePath: string, + runtime: Runtime, +): Array { const samples: Array = []; for ( let sampleIndex = 0; sampleIndex < memorySamplesPerBenchmark; ++sampleIndex ) { - const sample = sampleMemoryModule(modulePath); + const sample = sampleMemoryModule(modulePath, runtime); assert(sample > 0); samples.push(sample); } diff --git a/resources/benchmark/statistics.ts b/resources/benchmark/statistics.ts index 52ebae2fd1..4e219759c8 100644 --- a/resources/benchmark/statistics.ts +++ b/resources/benchmark/statistics.ts @@ -29,7 +29,7 @@ interface LogRatioStats { export function computeStats( name: string, timingSamples: ReadonlyArray, - memorySamples: ReadonlyArray, + memorySamples: ReadonlyArray = [], ): BenchmarkResult { const { mean } = computeMeanStats(timingSamples); diff --git a/resources/benchmark/worker-utils.js b/resources/benchmark/worker-utils.js index c554c1fa77..48d2d8e1c4 100644 --- a/resources/benchmark/worker-utils.js +++ b/resources/benchmark/worker-utils.js @@ -20,7 +20,9 @@ export async function loadBenchmark(modulePath) { } export function writeResult(result) { - fs.writeFileSync(3, JSON.stringify(result)); + const resultFilePath = process.env.BENCHMARK_RESULT_FILE; + assert(resultFilePath != null); + fs.writeFileSync(resultFilePath, JSON.stringify(result)); } export function runWorker(main) { diff --git a/resources/benchmark/workers.ts b/resources/benchmark/workers.ts index bb95dfd565..ffdb070ae9 100644 --- a/resources/benchmark/workers.ts +++ b/resources/benchmark/workers.ts @@ -1,54 +1,105 @@ import assert from 'node:assert'; import childProcess from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { localRepoPath } from '../utils.js'; +import type { Runtime } from './args.js'; import { - memoryBenchmarkNodeFlags, - timingBenchmarkNodeFlags, + memoryCommandArgsByRuntime, + timingCommandArgsByRuntime, } from './config.js'; -export function getBenchmarkName(modulePath: string): string { +export function getBenchmarkName(modulePath: string, runtime: Runtime): string { return runWorkerFile( localRepoPath('resources/benchmark/worker-name.js'), modulePath, + timingCommandArgsByRuntime[runtime], + runtime, ) as string; } -export function sampleTimingModule(modulePath: string): number { +export function sampleTimingModule( + modulePath: string, + runtime: Runtime = inferRuntimeFromExecPath(process.execPath), +): number { return runWorkerFile( localRepoPath('resources/benchmark/worker-timing.js'), modulePath, - timingBenchmarkNodeFlags, + timingCommandArgsByRuntime[runtime], + runtime, ) as number; } -export function sampleMemoryModule(modulePath: string): number { +export function sampleMemoryModule( + modulePath: string, + runtime: Runtime = inferRuntimeFromExecPath(process.execPath), +): number { return runWorkerFile( localRepoPath('resources/benchmark/worker-memory.js'), modulePath, - memoryBenchmarkNodeFlags, + memoryCommandArgsByRuntime[runtime], + runtime, ) as number; } function runWorkerFile( workerPath: string, modulePath: string, - nodeFlags: ReadonlyArray = [], + commandArgs: ReadonlyArray, + runtime: Runtime, ): unknown { - const result = childProcess.spawnSync( - process.execPath, - [...nodeFlags, workerPath, modulePath], - { - stdio: ['inherit', 'inherit', 'inherit', 'pipe'], - env: { NODE_ENV: 'production' }, - }, + const resultPath = fs.mkdtempSync( + path.join(os.tmpdir(), 'graphql-benchmark-worker-'), ); - if (result.status !== 0) { - throw new Error(`Benchmark worker failed with "${result.status}" status.`); + const resultFilePath = path.join(resultPath, 'result.json'); + + const currentRuntime = inferRuntimeFromExecPath(process.execPath); + const execPath = + runtime === currentRuntime ? process.execPath : String(runtime); + const execArgs = [...commandArgs, workerPath, modulePath]; + + try { + const result = childProcess.spawnSync(execPath, execArgs, { + stdio: 'inherit', + env: { + ...process.env, + BENCHMARK_RESULT_FILE: resultFilePath, + NODE_ENV: 'production', + }, + }); + if (result.error != null) { + throw result.error; + } + if (result.signal != null) { + throw new Error( + `Benchmark worker terminated by signal "${result.signal}".`, + ); + } + if (result.status !== 0) { + throw new Error( + `Benchmark worker failed with "${result.status}" status.`, + ); + } + + const resultStr = fs.readFileSync(resultFilePath, 'utf8'); + assert(resultStr !== ''); + return JSON.parse(resultStr); + } finally { + fs.rmSync(resultPath, { recursive: true, force: true }); } +} + +function inferRuntimeFromExecPath(execPath: string): Runtime { + const executableName = path.basename(execPath).toLowerCase(); - const resultStr = result.output[3]?.toString(); - assert(resultStr != null); - return JSON.parse(resultStr); + if (executableName === 'deno' || executableName === 'deno.exe') { + return 'deno'; + } + if (executableName === 'bun' || executableName === 'bun.exe') { + return 'bun'; + } + return 'node'; }