benchmark: report paired benchmark comparisons (#4689)

yaacovCR · web-flow · commit 1aa5a03b83b2 · 2026-04-24T04:56:22.000+03:00
diff --git a/resources/benchmark/config.ts b/resources/benchmark/config.ts
@@ -13,6 +13,9 @@ export const targetPairwiseComparisonIntervalHalfWidth = 2;
 
 export const memorySamplesPerBenchmark = 10;
 
+export const pairedGreenThreshold = 0.95;
+export const pairedYellowThreshold = 0.8;
+
 export const timingBenchmarkNodeFlags: ReadonlyArray<string> = ['--expose-gc'];
 
 export const memoryBenchmarkNodeFlags: ReadonlyArray<string> = [
diff --git a/resources/benchmark/output.ts b/resources/benchmark/output.ts
@@ -1,4 +1,7 @@
-import type { BenchmarkResult } from './types.js';
+import { pairedGreenThreshold, pairedYellowThreshold } from './config.js';
+import type { BenchmarkResult, PairedComparison } from './types.js';
+
+type ColorFn = (value: number | string) => string;
 
 export function printBenchmarkResults(
   results: ReadonlyArray<BenchmarkResult>,
@@ -54,6 +57,54 @@ export function printBenchmarkResults(
   }
 }
 
+export function printPairedComparisons(
+  pairedComparisons: ReadonlyArray<PairedComparison>,
+): void {
+  if (pairedComparisons.length === 0) {
+    return;
+  }
+
+  console.log('  ' + grey('paired round-by-round ops/sec changes:'));
+
+  const leftMaxLen = maxBy(
+    pairedComparisons,
+    ({ baselineRevision }) => baselineRevision.length,
+  );
+  const rightMaxLen = maxBy(
+    pairedComparisons,
+    ({ revision }) => revision.length,
+  );
+  const speedupMaxLen = maxBy(
+    pairedComparisons,
+    ({ speedupPercent }) => formatSignedPercent(speedupPercent).length,
+  );
+  const ciMaxLen = maxBy(
+    pairedComparisons,
+    (comparison) => formatConfidenceInterval(comparison).length,
+  );
+
+  for (const paired of pairedComparisons) {
+    const speedupColorFn = pairedSpeedupColorFn(paired);
+    console.log(
+      '  ' +
+        paired.baselineRevision.padEnd(leftMaxLen) +
+        ' -> ' +
+        paired.revision.padEnd(rightMaxLen) +
+        grey(' x ') +
+        speedupColorFn(
+          formatSignedPercent(paired.speedupPercent).padStart(speedupMaxLen),
+        ) +
+        ' ops/sec change ' +
+        grey(
+          '(95% CI ' +
+            formatConfidenceInterval(paired).padStart(ciMaxLen) +
+            ', ' +
+            paired.numPairs +
+            ' paired runs)',
+        ),
+    );
+  }
+}
 function beautifyBytes(bytes: number): string {
   const sizes = ['Bytes', 'KB', 'MB', 'GB'];
   const i = Math.floor(Math.log2(bytes) / 10);
@@ -64,10 +115,52 @@ function beautifyNumber(num: number): string {
   return Number(num.toFixed(num > 100 ? 0 : 2)).toLocaleString();
 }
 
+function formatSignedPercent(num: number): string {
+  const rounded = Number(num.toFixed(2));
+  const sign = rounded > 0 ? '+' : '';
+  return sign + rounded.toFixed(2) + '%';
+}
+
+function formatConfidenceInterval({
+  ciLowPercent,
+  ciHighPercent,
+}: PairedComparison): string {
+  return (
+    formatSignedPercent(ciLowPercent) +
+    ' to ' +
+    formatSignedPercent(ciHighPercent)
+  );
+}
+
 function maxBy<T>(array: ReadonlyArray<T>, fn: (obj: T) => number): number {
   return Math.max(...array.map(fn));
 }
 
+function pairedSpeedupColorFn({
+  speedupPercent,
+  ciLowPercent,
+  ciHighPercent,
+}: PairedComparison): ColorFn {
+  if (ciLowPercent <= 0 && ciHighPercent >= 0) {
+    return grey;
+  }
+
+  const relativeOps = 1 + speedupPercent / 100;
+  if (speedupPercent < 0) {
+    if (relativeOps > pairedGreenThreshold) {
+      return grey;
+    }
+
+    if (relativeOps > pairedYellowThreshold) {
+      return yellow;
+    }
+
+    return red;
+  }
+
+  return green;
+}
+
 export function bold(str: number | string): string {
   return '\u001b[1m' + str + '\u001b[0m';
 }
diff --git a/resources/benchmark/run.ts b/resources/benchmark/run.ts
@@ -7,10 +7,16 @@ import {
   memorySamplesPerBenchmark,
   minTimingSamplesPerBenchmark,
 } from './config.js';
-import { cyan, printBenchmarkResults, red } from './output.js';
+import {
+  cyan,
+  printBenchmarkResults,
+  printPairedComparisons,
+  red,
+} from './output.js';
 import { prepareBenchmarkProjects } from './projects.js';
 import {
   computeStats,
+  getPairedComparisons,
   havePairwiseComparisonsStabilized,
 } from './statistics.js';
 import type { BenchmarkProject, BenchmarkResult } from './types.js';
@@ -91,6 +97,12 @@ function runBenchmark(
   console.log('\n');
 
   printBenchmarkResults(results);
+  printPairedComparisons(
+    getPairedComparisons(
+      benchmarkProjects.map(({ revision }) => revision),
+      timingSamples,
+    ),
+  );
   console.log('');
 }
 
diff --git a/resources/benchmark/statistics.ts b/resources/benchmark/statistics.ts
@@ -4,7 +4,7 @@ import {
   NS_PER_SEC,
   targetPairwiseComparisonIntervalHalfWidth,
 } from './config.js';
-import type { BenchmarkResult } from './types.js';
+import type { BenchmarkResult, PairedComparison } from './types.js';
 
 // T-Distribution two-tailed critical values for 95% confidence.
 // See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm.
@@ -18,29 +18,67 @@ const tTable: { [v: number]: number } = {
 };
 const tTableInfinity = 1.96;
 
+interface LogRatioStats {
+  meanRatio: number;
+  lowRatio: number;
+  highRatio: number;
+  numSamples: number;
+}
+
 // Computes stats on benchmark results.
 export function computeStats(
   name: string,
   timingSamples: ReadonlyArray<number>,
   memorySamples: ReadonlyArray<number>,
 ): BenchmarkResult {
-  const { mean, marginOfError } = computeMeanStats(timingSamples);
-
-  let meanMemUsed = 0;
-  for (const memUsed of memorySamples) {
-    meanMemUsed += memUsed;
-  }
-  meanMemUsed /= memorySamples.length;
+  const { mean } = computeMeanStats(timingSamples);
 
   return {
     name,
-    memPerOp: Math.floor(meanMemUsed),
+    memPerOp: Math.floor(computeMean(memorySamples)),
     ops: NS_PER_SEC / mean,
-    deviation: (marginOfError / mean) * 100 || 0,
+    deviation: computeRelativeMarginOfError(timingSamples),
     numSamples: timingSamples.length,
   };
 }
 
+export function getPairedComparisons(
+  revisions: ReadonlyArray<string>,
+  timingSamplesByRevision: ReadonlyArray<ReadonlyArray<number>>,
+): Array<PairedComparison> {
+  const pairedComparisons: Array<PairedComparison> = [];
+
+  for (
+    let baselineIndex = 1;
+    baselineIndex < timingSamplesByRevision.length;
+    ++baselineIndex
+  ) {
+    const baselineSamples = timingSamplesByRevision[baselineIndex];
+
+    for (
+      let revisionIndex = 0;
+      revisionIndex < baselineIndex;
+      ++revisionIndex
+    ) {
+      const paired = computePairedComparison(
+        baselineSamples,
+        timingSamplesByRevision[revisionIndex],
+      );
+      if (paired == null) {
+        continue;
+      }
+
+      pairedComparisons.push({
+        baselineRevision: revisions[baselineIndex],
+        revision: revisions[revisionIndex],
+        ...paired,
+      });
+    }
+  }
+
+  return pairedComparisons;
+}
+
 export function havePairwiseComparisonsStabilized(
   timingSamplesByRevision: ReadonlyArray<ReadonlyArray<number>>,
 ): boolean {
@@ -56,15 +94,13 @@ export function havePairwiseComparisonsStabilized(
       revisionIndex < baselineIndex;
       ++revisionIndex
     ) {
-      const ciHalfWidthPercent = computeLogRatioRelativeMarginOfError(
-        getRoundLogRatios(
-          baselineSamples,
-          timingSamplesByRevision[revisionIndex],
-        ),
+      const paired = computePairedComparison(
+        baselineSamples,
+        timingSamplesByRevision[revisionIndex],
       );
       if (
-        ciHalfWidthPercent == null ||
-        ciHalfWidthPercent > targetPairwiseComparisonIntervalHalfWidth
+        paired == null ||
+        paired.ciHalfWidthPercent > targetPairwiseComparisonIntervalHalfWidth
       ) {
         return false;
       }
@@ -74,11 +110,52 @@ export function havePairwiseComparisonsStabilized(
   return true;
 }
 
-function computeLogRatioRelativeMarginOfError(
+function computeRelativeMarginOfError(samples: ReadonlyArray<number>): number {
+  const { mean, marginOfError } = computeMeanStats(samples);
+  return (marginOfError / mean) * 100 || 0;
+}
+
+function computeLogRatioStats(
   logRatios: ReadonlyArray<number>,
-): number | undefined {
-  const { marginOfError } = computeMeanStats(logRatios);
-  return Math.expm1(marginOfError) * 100;
+): LogRatioStats | undefined {
+  if (logRatios.length < 2) {
+    return;
+  }
+
+  const { mean, marginOfError } = computeMeanStats(logRatios);
+  return {
+    meanRatio: Math.exp(mean),
+    lowRatio: Math.exp(mean - marginOfError),
+    highRatio: Math.exp(mean + marginOfError),
+    numSamples: logRatios.length,
+  };
+}
+
+function computePairedComparison(
+  baselineSamples: ReadonlyArray<number>,
+  samples: ReadonlyArray<number>,
+): Omit<PairedComparison, 'baselineRevision' | 'revision'> | undefined {
+  const logRatioStats = computeLogRatioStats(
+    getRoundLogRatios(baselineSamples, samples),
+  );
+  if (logRatioStats == null) {
+    return;
+  }
+
+  const speedupPercent = (logRatioStats.meanRatio - 1) * 100;
+  const ciLowPercent = (logRatioStats.lowRatio - 1) * 100;
+  const ciHighPercent = (logRatioStats.highRatio - 1) * 100;
+
+  return {
+    speedupPercent,
+    ciLowPercent,
+    ciHighPercent,
+    ciHalfWidthPercent: Math.max(
+      Math.abs(speedupPercent - ciLowPercent),
+      Math.abs(ciHighPercent - speedupPercent),
+    ),
+    numPairs: logRatioStats.numSamples,
+  };
 }
 
 function getRoundLogRatios(
@@ -94,17 +171,21 @@ function getRoundLogRatios(
   return logRatios;
 }
 
+function computeMean(samples: ReadonlyArray<number>): number {
+  let mean = 0;
+  for (const sample of samples) {
+    mean += sample;
+  }
+  return mean / samples.length;
+}
+
 function computeMeanStats(samples: ReadonlyArray<number>): {
   mean: number;
   marginOfError: number;
 } {
   assert(samples.length > 1);
 
-  let mean = 0;
-  for (const sample of samples) {
-    mean += sample;
-  }
-  mean /= samples.length;
+  const mean = computeMean(samples);
 
   let variance = 0;
   for (const sample of samples) {
diff --git a/resources/benchmark/types.ts b/resources/benchmark/types.ts
@@ -10,3 +10,13 @@ export interface BenchmarkResult {
   deviation: number;
   numSamples: number;
 }
+
+export interface PairedComparison {
+  baselineRevision: string;
+  revision: string;
+  speedupPercent: number;
+  ciLowPercent: number;
+  ciHighPercent: number;
+  ciHalfWidthPercent: number;
+  numPairs: number;
+}
