internal: add runtime option to benchmark (#4697)

Author: yaacovCR

resources/benchmark/args.ts

@@ -6,15 +6,40 @@ import { localRepoPath } from '../utils.js';
 import { LOCAL } from './config.js';
 import { bold } from './output.js';
 
+export type Runtime = 'node' | 'deno' | 'bun';
+
 export interface BenchmarkArguments {
   benchmarks: Array<string>;
   revisions: Array<string>;
+  runtime: Runtime;
 }
 
 export function getArguments(argv: ReadonlyArray<string>): BenchmarkArguments {
-  const revsIndex = argv.indexOf('--revs');
-  const revisions = revsIndex === -1 ? [] : argv.slice(revsIndex + 1);
-  const benchmarks = revsIndex === -1 ? [...argv] : argv.slice(0, revsIndex);
+  const runtimeIndex = argv.indexOf('--runtime');
+  const runtimeValue =
+    runtimeIndex === -1
+      ? inferRuntimeFromExecPath(process.execPath)
+      : argv[runtimeIndex + 1];
+  if (
+    runtimeValue !== 'node' &&
+    runtimeValue !== 'deno' &&
+    runtimeValue !== 'bun'
+  ) {
+    throw new Error(
+      `Invalid --runtime value: "${runtimeValue}". Must be "node", "deno", or "bun".`,
+    );
+  }
+  const runtime: Runtime = runtimeValue;
+
+  const filteredArgv =
+    runtimeIndex === -1
+      ? [...argv]
+      : [...argv.slice(0, runtimeIndex), ...argv.slice(runtimeIndex + 2)];
+
+  const revsIndex = filteredArgv.indexOf('--revs');
+  const revisions = revsIndex === -1 ? [] : filteredArgv.slice(revsIndex + 1);
+  const benchmarks =
+    revsIndex === -1 ? [...filteredArgv] : filteredArgv.slice(0, revsIndex);
 
   switch (revisions.length) {
     case 0:
@@ -33,7 +58,19 @@ export function getArguments(argv: ReadonlyArray<string>): BenchmarkArguments {
     benchmarks.push(...findAllBenchmarks());
   }
 
-  return { benchmarks, revisions };
+  return { benchmarks, revisions, runtime };
+}
+
+function inferRuntimeFromExecPath(execPath: string): Runtime {
+  const executableName = path.basename(execPath).toLowerCase();
+
+  if (executableName.startsWith('deno')) {
+    return 'deno';
+  }
+  if (executableName.startsWith('bun')) {
+    return 'bun';
+  }
+  return 'node';
 }
 
 function findAllBenchmarks(): Array<string> {

resources/benchmark/config.ts

@@ -16,12 +16,31 @@ export const memorySamplesPerBenchmark = 10;
 export const pairedGreenThreshold = 0.95;
 export const pairedYellowThreshold = 0.8;
 
-export const timingBenchmarkNodeFlags: ReadonlyArray<string> = ['--expose-gc'];
+export const timingCommandArgsByRuntime = {
+  node: ['--expose-gc'],
+  bun: ['--expose-gc'],
+  deno: [
+    'run',
+    '--allow-read',
+    '--allow-env',
+    '--allow-write',
+    '--v8-flags=--expose-gc',
+  ],
+} as const;
 
-export const memoryBenchmarkNodeFlags: ReadonlyArray<string> = [
-  '--predictable',
-  '--no-concurrent-sweeping',
-  '--no-minor-gc-task',
-  '--min-semi-space-size=1280', // 1.25GB
-  '--max-semi-space-size=1280', // 1.25GB
-];
+export const memoryCommandArgsByRuntime = {
+  node: [
+    '--no-concurrent-sweeping',
+    '--no-minor-gc-task',
+    '--min-semi-space-size=1280', // 1.25GB
+    '--max-semi-space-size=1280', // 1.25GB
+  ],
+  bun: [],
+  deno: [
+    'run',
+    '--allow-read',
+    '--allow-env',
+    '--allow-write',
+    '--v8-flags=--no-concurrent-sweeping,--no-minor-gc-task,--min-semi-space-size=1280,--max-semi-space-size=1280',
+  ],
+} as const;

resources/benchmark/output.ts

@@ -5,6 +5,7 @@ type ColorFn = (value: number | string) => string;
 
 export function printBenchmarkResults(
   results: ReadonlyArray<BenchmarkResult>,
+  includesMemory: boolean,
 ): void {
   const nameMaxLen = maxBy(results, ({ name }) => name.length);
   const opsTop = maxBy(results, ({ ops }) => ops);
@@ -29,9 +30,7 @@ export function printBenchmarkResults(
         grey('\xb1') +
         deviationStr() +
         cyan('%') +
-        grey(' x ') +
-        memPerOpStr() +
-        '/op' +
+        (includesMemory ? grey(' x ') + memPerOpStr() + '/op' : '') +
         grey(' (' + numSamples + ' runs sampled)'),
     );
 

resources/benchmark/run.ts

@@ -1,6 +1,8 @@
 import assert from 'node:assert';
 import path from 'node:path';
+import readline from 'node:readline';
 
+import type { Runtime } from './args.js';
 import { getArguments } from './args.js';
 import {
   maxTime,
@@ -9,6 +11,7 @@ import {
 } from './config.js';
 import {
   cyan,
+  grey,
   printBenchmarkResults,
   printPairedComparisons,
   red,
@@ -28,32 +31,46 @@ import {
 
 export function runBenchmarks(): void {
   // Get the revisions and make things happen!
-  const { benchmarks, revisions } = getArguments(process.argv.slice(2));
+  const { benchmarks, revisions, runtime } = getArguments(
+    process.argv.slice(2),
+  );
   const benchmarkProjects = prepareBenchmarkProjects(revisions);
 
+  console.log('');
+  console.log('\u2699\uFE0F  Runtime: ' + runtime);
+  console.log('');
+
   for (const benchmark of benchmarks) {
-    runBenchmark(benchmark, benchmarkProjects);
+    runBenchmark(benchmark, benchmarkProjects, runtime);
   }
 }
 
 // Prepare all revisions and run benchmarks matching a pattern against them.
 function runBenchmark(
   benchmark: string,
   benchmarkProjects: ReadonlyArray<BenchmarkProject>,
+  runtime: Runtime,
 ): void {
-  const memorySamples: Array<Array<number>> = [];
+  const memorySamples: Array<Array<number> | undefined> = [];
+  const includesMemory = runtime !== 'bun';
+
   for (let i = 0; i < benchmarkProjects.length; ++i) {
     const modulePath = path.join(benchmarkProjects[i].projectPath, benchmark);
 
     if (i === 0) {
-      console.log('\u23F1   ' + getBenchmarkName(modulePath));
+      console.log('\u23F1   ' + getBenchmarkName(modulePath, runtime));
+      if (includesMemory) {
+        writeProgress('  completed ' + cyan(0) + ' memory tests...');
+      }
+    }
+
+    if (!includesMemory) {
+      continue;
     }
 
     try {
-      memorySamples[i] = collectMemorySamples(modulePath);
-      process.stdout.write(
-        '  completed ' + cyan(i + 1) + ' memory tests...\u000D',
-      );
+      memorySamples[i] = collectMemorySamples(modulePath, runtime);
+      writeProgress('  completed ' + cyan(i + 1) + ' memory tests...');
     } catch (error) {
       const errorMessage =
         error instanceof Error ? error.message : String(error);
@@ -63,11 +80,15 @@ function runBenchmark(
       return;
     }
   }
-  process.stdout.write('\n');
+  if (includesMemory) {
+    endProgressLine();
+  } else {
+    console.log('  ' + grey('skipping memory benchmarks for bun runtime'));
+  }
 
   let timingSamples: Array<Array<number>>;
   try {
-    timingSamples = collectTimingSamples(benchmark, benchmarkProjects);
+    timingSamples = collectTimingSamples(benchmark, benchmarkProjects, runtime);
   } catch {
     console.log('  ' + red('timing samples collection failed'));
     return;
@@ -80,7 +101,7 @@ function runBenchmark(
       result = computeStats(
         benchmarkProjects[i].revision,
         timingSamples[i],
-        memorySamples[i],
+        includesMemory ? memorySamples[i] : undefined,
       );
     } catch (error) {
       const errorMessage =
@@ -96,7 +117,7 @@ function runBenchmark(
 
   console.log('\n');
 
-  printBenchmarkResults(results);
+  printBenchmarkResults(results, includesMemory);
   printPairedComparisons(
     getPairedComparisons(
       benchmarkProjects.map(({ revision }) => revision),
@@ -109,6 +130,7 @@ function runBenchmark(
 function collectTimingSamples(
   benchmark: string,
   benchmarkProjects: ReadonlyArray<BenchmarkProject>,
+  runtime: Runtime,
 ): Array<Array<number>> {
   const sampleGroups = benchmarkProjects.map((project) => ({
     revision: project.revision,
@@ -122,14 +144,15 @@ function collectTimingSamples(
   // pairwise revision comparison has stabilized.
   const start = Date.now();
   let round = 0;
+  writeProgress('  completed ' + cyan(0) + ' timing rounds...');
   while (
     (Date.now() - start) / 1e3 < maxTime &&
     (round < minTimingSamplesPerBenchmark ||
       !havePairwiseComparisonsStabilized(timingSamples))
   ) {
     for (const sampleGroup of shuffled(sampleGroups)) {
       try {
-        const sample = sampleTimingModule(sampleGroup.modulePath);
+        const sample = sampleTimingModule(sampleGroup.modulePath, runtime);
 
         assert(sample > 0);
         sampleGroup.samples.push(sample);
@@ -142,13 +165,21 @@ function collectTimingSamples(
     }
 
     ++round;
-    process.stdout.write(
-      '  completed ' + cyan(round) + ' timing rounds...\u000D',
-    );
+    writeProgress('  completed ' + cyan(round) + ' timing rounds...');
   }
   return timingSamples;
 }
 
+function writeProgress(message: string): void {
+  readline.clearLine(process.stdout, 0);
+  readline.cursorTo(process.stdout, 0);
+  process.stdout.write(message);
+}
+
+function endProgressLine(): void {
+  process.stdout.write('\n');
+}
+
 function shuffled<T>(array: ReadonlyArray<T>): Array<T> {
   const shuffledArray = [...array];
   // Fisher-Yates shuffle: walk backward and swap each slot with a random
@@ -163,14 +194,17 @@ function shuffled<T>(array: ReadonlyArray<T>): Array<T> {
   return shuffledArray;
 }
 
-export function collectMemorySamples(modulePath: string): Array<number> {
+export function collectMemorySamples(
+  modulePath: string,
+  runtime: Runtime,
+): Array<number> {
   const samples: Array<number> = [];
   for (
     let sampleIndex = 0;
     sampleIndex < memorySamplesPerBenchmark;
     ++sampleIndex
   ) {
-    const sample = sampleMemoryModule(modulePath);
+    const sample = sampleMemoryModule(modulePath, runtime);
     assert(sample > 0);
     samples.push(sample);
   }

resources/benchmark/statistics.ts

@@ -29,7 +29,7 @@ interface LogRatioStats {
 export function computeStats(
   name: string,
   timingSamples: ReadonlyArray<number>,
-  memorySamples: ReadonlyArray<number>,
+  memorySamples: ReadonlyArray<number> = [],
 ): BenchmarkResult {
   const { mean } = computeMeanStats(timingSamples);
 

resources/benchmark/worker-utils.js

@@ -20,7 +20,9 @@ export async function loadBenchmark(modulePath) {
 }
 
 export function writeResult(result) {
-  fs.writeFileSync(3, JSON.stringify(result));
+  const resultFilePath = process.env.BENCHMARK_RESULT_FILE;
+  assert(resultFilePath != null);
+  fs.writeFileSync(resultFilePath, JSON.stringify(result));
 }
 
 export function runWorker(main) {