diff --git a/README.md b/README.md index eebe95c7..863d168e 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,12 @@ Both the `Task` and `Bench` classes extend the `EventTarget` object. So you can bench.addEventListener('cycle', (evt) => { const task = evt.task!; }); + +// runs when timer saturation is detected for a task's measured samples +bench.addEventListener('warning', (evt) => { + const task = evt.task!; + const reason = evt.reason; // 'zero-dominated' | 'low-distinct' | 'zero-mad' +}); ``` #### [`TaskEvents`](https://tinylibs.github.io/tinybench/types/TaskEvents.html) @@ -286,6 +292,96 @@ const bench = new Bench({ }) ``` +## Timer Overhead Correction + +Each timer call (`performance.now()`, `process.hrtime.bigint()`, …) has a +non-zero call cost `C`. For a task whose true duration `X` is comparable +to `C`, the raw measured sample `X + C` is dominated by the timer rather +than the task. + +When `subtractTimerOverhead: true` is set, an estimate `Ĉ` is computed +once at construction time via [`calibrateTimerOverhead`](https://tinylibs.github.io/tinybench/functions/calibrateTimerOverhead.html), +and `Math.max(0, raw_sample - Ĉ)` is used as each non-overridden sample +before statistics are computed. + +```ts +const bench = new Bench({ subtractTimerOverhead: true }) +console.log(bench.timerOverhead) // calibrated Ĉ in ms (or undefined) +``` + +The calibration helper is also exported for direct use, with a +configurable estimator strategy (`'median'` default, or `'min'` / `'p05'`): + +```ts +import { calibrateTimerOverhead, hrtimeNowTimestampProvider } from 'tinybench' + +const overhead = calibrateTimerOverhead(hrtimeNowTimestampProvider, { + estimator: 'p05', + samples: 1024, + warmupSamples: 64, +}) +``` + +**Caveats.** + +- Incompatible with `concurrency: 'task'` — overhead is calibrated + sequentially and does not reflect concurrent execution cost. + Construction (and `run()`) throws if both are set. +- For sub-overhead measurements (`X ≈ Ĉ`) the `max(0, …)` clamp + truncates the lower tail and biases statistics; prefer + `overriddenDuration` (see below). +- On runtimes with a coarse timer (resolution >= 1 ms) the calibration + returns `0` and the option becomes a no-op. + +## Per-Sample Override (`overriddenDuration`) + +A task function may return an object containing `overriddenDuration` +(in ms). That value replaces the timer-measured sample directly, +bypassing both the timer and any overhead correction. Useful for +externally-timed work or sub-overhead measurements that the timer +cannot resolve. + +```ts +bench.add('externally-timed', () => { + const start = process.hrtime.bigint() + doWork() + const elapsedMs = Number(process.hrtime.bigint() - start) / 1e6 + return { overriddenDuration: elapsedMs } +}) +``` + +Overridden samples are excluded from `Task.detectedResolution` and +from timer-saturation detection. + +## Timer Diagnostics + +After `bench.run()` (or `runSync()`), each task exposes +`detectedResolution` — the smallest reproducibly observed positive +sample (in ms) among the timer-measured samples, or `undefined` when no +positive timer measurement was seen (e.g. every sample was overridden). + +```ts +const task = bench.getTask('foo') +console.log(task?.detectedResolution) // e.g. 0.000041 (≈ 41 ns) +``` + +When the timer's resolution dominates a task's measured distribution +(more than half zero samples, fewer than `max(3, min(10, ⌊n / 1000⌋))` +distinct values, or zero MAD with `n > 100`), tinybench dispatches a +`'warning'` event on both the task and the bench, carrying the matching +[`TimerSaturationReason`](https://tinylibs.github.io/tinybench/types/TimerSaturationReason.html): + +```ts +bench.addEventListener('warning', evt => { + console.warn(`timer-saturated: ${evt.task?.name} — ${evt.reason}`) +}) +``` + +The same heuristic and estimator are exposed as standalone helpers for +custom analysis: [`detectTimerSaturation`](https://tinylibs.github.io/tinybench/functions/detectTimerSaturation.html), +[`classifyTimerSaturation`](https://tinylibs.github.io/tinybench/functions/classifyTimerSaturation.html), +and [`estimateResolution`](https://tinylibs.github.io/tinybench/functions/estimateResolution.html). + ## Aborting Benchmarks Tinybench supports aborting benchmarks using `AbortSignal` at both the bench and task levels: diff --git a/src/bench.ts b/src/bench.ts index 608dfc09..cbc62f8f 100644 --- a/src/bench.ts +++ b/src/bench.ts @@ -24,6 +24,7 @@ import { BenchEvent } from './event' import { Task } from './task' import { assert, + calibrateTimerOverhead, defaultConvertTaskResultForConsoleTable, getTimestampProvider, runtime, @@ -95,6 +96,16 @@ export class Bench extends EventTarget implements BenchLike { */ readonly signal?: AbortSignal + /** + * Whether to subtract an estimated timestamp provider call overhead from + * each raw latency sample. + * + * Incompatible with `concurrency: 'task'`; the constraint is enforced + * at construction and at the start of {@link Bench.run}. + * @default false + */ + readonly subtractTimerOverhead: boolean + /** * A teardown function that runs after each task execution. */ @@ -120,6 +131,15 @@ export class Bench extends EventTarget implements BenchLike { */ readonly time: number + /** + * The estimated cost of one timestamp provider call in milliseconds. + * + * `undefined` when {@link subtractTimerOverhead} is `false`. + * Otherwise calibrated once at construction time via + * {@link calibrateTimerOverhead}. + */ + readonly timerOverhead: number | undefined + /** * A timestamp provider and its related functions. */ @@ -195,6 +215,14 @@ export class Bench extends EventTarget implements BenchLike { this.throws = restOptions.throws ?? false this.signal = restOptions.signal this.retainSamples = restOptions.retainSamples === true + this.subtractTimerOverhead = restOptions.subtractTimerOverhead === true + assert( + !(this.subtractTimerOverhead && this.concurrency === 'task'), + '`subtractTimerOverhead` cannot be used with `concurrency: "task"` — set `concurrency` to `null` or `"bench"`, or disable `subtractTimerOverhead`' + ) + this.timerOverhead = this.subtractTimerOverhead + ? calibrateTimerOverhead(this.timestampProvider) + : undefined if (this.signal) { this.signal.addEventListener( @@ -264,6 +292,10 @@ export class Bench extends EventTarget implements BenchLike { * @returns the tasks array */ async run (): Promise { + assert( + !(this.subtractTimerOverhead && this.concurrency === 'task'), + '`subtractTimerOverhead` cannot be used with `concurrency: "task"` — set `concurrency` to `null` or `"bench"`, or disable `subtractTimerOverhead`' + ) if (this.warmup) { await this.#warmupTasks() } diff --git a/src/event.ts b/src/event.ts index 470628f3..b679a9e3 100644 --- a/src/event.ts +++ b/src/event.ts @@ -4,6 +4,7 @@ import type { BenchEventsOptionalTask, BenchEventsWithError, BenchEventsWithTask, + TimerSaturationReason, } from './types' /** @@ -24,6 +25,20 @@ class BenchEvent< return this.#error as K extends BenchEventsWithError ? Error : undefined } + /** + * The reason a `'warning'` event was dispatched. + * @returns The {@link TimerSaturationReason} for `'warning'` events; + * `undefined` for every other event type and for `'warning'` events + * dispatched without a reason + */ + get reason (): K extends 'warning' + ? TimerSaturationReason | undefined + : undefined { + return this.#reason as K extends 'warning' + ? TimerSaturationReason | undefined + : undefined + } + /** * The task associated with the event. * @returns The task if the event type is one that includes a task; otherwise, undefined @@ -41,15 +56,25 @@ class BenchEvent< } #error?: Error + #reason?: TimerSaturationReason #task?: Task + constructor (type: 'warning', task: Task, reason?: TimerSaturationReason) constructor (type: BenchEventsWithError, task: Task, error: Error) constructor (type: BenchEventsWithTask, task: Task) constructor (type: BenchEventsOptionalTask, task?: Task) - constructor (type: BenchEvents, task?: Task, error?: Error) { + constructor ( + type: BenchEvents, + task?: Task, + errorOrReason?: Error | TimerSaturationReason + ) { super(type) this.#task = task - this.#error = error + if (typeof errorOrReason === 'string') { + this.#reason = errorOrReason + } else { + this.#error = errorOrReason + } } } diff --git a/src/index.ts b/src/index.ts index 5fadff4b..49418b54 100644 --- a/src/index.ts +++ b/src/index.ts @@ -32,9 +32,25 @@ export type { TaskResultStarted, TaskResultTimestampProviderInfo, TaskResultWithStatistics, + TimerSaturationReason, TimestampFn, TimestampFns, TimestampProvider, TimestampValue, } from './types' -export { formatNumber, hrtimeNow, mToNs, performanceNow as now, nToMs } from './utils' +export type { + CalibrateTimerOverheadOptions, + TimerOverheadEstimatorKind, +} from './utils' +export { + calibrateTimerOverhead, + classifyTimerSaturation, + detectTimerSaturation, + estimateResolution, + formatNumber, + hrtimeNow, + medianAbsoluteDeviation, + mToNs, + performanceNow as now, + nToMs, +} from './utils' diff --git a/src/task.ts b/src/task.ts index 6fefbd8a..8c076cbe 100644 --- a/src/task.ts +++ b/src/task.ts @@ -15,14 +15,18 @@ import type { TimestampProvider, TimestampValue, } from './types' +import type { TimerSaturationReason } from './types' import { BenchEvent } from './event' import { assert, + classifyTimerSaturation, computeStatistics, + estimateResolution, isFnAsyncResource, isPromiseLike, isValidSamples, + medianAbsoluteDeviation, sortSamples, toError, withConcurrency, @@ -70,6 +74,19 @@ export class Task extends EventTarget { options?: RemoveEventListenerOptionsArgument ) => void + /** + * The estimated effective timer resolution observed during the last run, + * computed as the smallest strictly positive latency sample that appears + * at least twice among the timer-measured samples (samples supplied via + * `overriddenDuration` are excluded). + * @returns The resolution in milliseconds, or `undefined` when no + * timer-measured strictly positive sample was observed (e.g. every + * sample was supplied via `overriddenDuration`) + */ + get detectedResolution (): number | undefined { + return this.#detectedResolution + } + /** * The name of the task. * @returns The task name as a string @@ -116,6 +133,11 @@ export class Task extends EventTarget { */ readonly #bench: BenchLike + /** + * The estimated effective timer resolution from the last run. + */ + #detectedResolution: number | undefined = undefined + /** * The task function */ @@ -217,6 +239,7 @@ export class Task extends EventTarget { */ reset (emit = true): void { this.#runs = 0 + this.#detectedResolution = undefined this.#result = this.#aborted ? abortedTaskResult : notStartedTaskResult if (emit) this.dispatchEvent(new BenchEvent('reset', this)) @@ -233,14 +256,14 @@ export class Task extends EventTarget { this.#result = { state: 'started' } this.dispatchEvent(new BenchEvent('start', this)) await this.#bench.setup(this, 'run') - const { error, samples: latencySamples } = await this.#benchmark( - 'run', - this.#bench.time, - this.#bench.iterations - ) + const { + error, + isOverridden, + samples: latencySamples, + } = await this.#benchmark('run', this.#bench.time, this.#bench.iterations) await this.#bench.teardown(this, 'run') - this.#processRunResult({ error, latencySamples }) + this.#processRunResult({ error, isOverridden, latencySamples }) return this } @@ -267,11 +290,11 @@ export class Task extends EventTarget { '`setup` function must be sync when using `runSync()`' ) - const { error, samples: latencySamples } = this.#benchmarkSync( - 'run', - this.#bench.time, - this.#bench.iterations - ) + const { + error, + isOverridden, + samples: latencySamples, + } = this.#benchmarkSync('run', this.#bench.time, this.#bench.iterations) const teardownResult = this.#bench.teardown(this, 'run') assert( @@ -279,7 +302,7 @@ export class Task extends EventTarget { '`teardown` function must be sync when using `runSync()`' ) - this.#processRunResult({ error, latencySamples }) + this.#processRunResult({ error, isOverridden, latencySamples }) return this } @@ -339,7 +362,8 @@ export class Task extends EventTarget { time: number, iterations: number ): Promise< - { error: Error; samples?: never } | { error?: never; samples?: Samples } + | { error: Error; isOverridden?: never; samples?: never } + | { error?: never; isOverridden?: boolean[]; samples?: Samples } > { try { if (this.#fnOpts.beforeAll) { @@ -348,6 +372,7 @@ export class Task extends EventTarget { let totalTime = 0 // ms const samples: number[] = [] + const isOverridden: boolean[] = [] const benchmarkTask = async () => { if (this.#aborted) { @@ -358,11 +383,12 @@ export class Task extends EventTarget { await this.#fnOpts.beforeEach.call(this, mode) } - const taskTime = this.#async + const { overridden, taskTime } = this.#async ? await this.#measure() : this.#measureSync() samples.push(taskTime) + isOverridden.push(overridden) totalTime += taskTime } finally { if (this.#fnOpts.afterEach != null) { @@ -395,7 +421,7 @@ export class Task extends EventTarget { await this.#fnOpts.afterAll.call(this, mode) } - return isValidSamples(samples) ? { samples } : {} + return isValidSamples(samples) ? { isOverridden, samples } : {} } catch (error) { return { error: toError(error) } } @@ -411,7 +437,9 @@ export class Task extends EventTarget { mode: 'run' | 'warmup', time: number, iterations: number - ): { error: Error; samples?: never } | { error?: never; samples?: Samples } { + ): + | { error: Error; isOverridden?: never; samples?: never } + | { error?: never; isOverridden?: boolean[]; samples?: Samples } { try { if (this.#fnOpts.beforeAll) { const beforeAllResult = this.#fnOpts.beforeAll.call(this, mode) @@ -423,6 +451,7 @@ export class Task extends EventTarget { let totalTime = 0 const samples: number[] = [] + const isOverridden: boolean[] = [] const benchmarkTask = () => { if (this.#aborted) { @@ -437,9 +466,10 @@ export class Task extends EventTarget { ) } - const taskTime = this.#measureSync() + const { overridden, taskTime } = this.#measureSync() samples.push(taskTime) + isOverridden.push(overridden) totalTime += taskTime } finally { if (this.#fnOpts.afterEach) { @@ -467,7 +497,7 @@ export class Task extends EventTarget { '`afterAll` function must be sync when using `runSync()`' ) } - return isValidSamples(samples) ? { samples } : {} + return isValidSamples(samples) ? { isOverridden, samples } : {} } catch (error) { return { error: toError(error) } } @@ -475,9 +505,10 @@ export class Task extends EventTarget { /** * Measures a single execution of the task function asynchronously. - * @returns The measured execution time + * @returns The measured execution time and whether it was supplied by the + * task function via `overriddenDuration` */ - async #measure (): Promise { + async #measure (): Promise<{ overridden: boolean; taskTime: number }> { const taskStart = this.#timestampFn() as unknown as number // eslint-disable-next-line no-useless-call const fnResult = await this.#fn.call(this) @@ -487,16 +518,17 @@ export class Task extends EventTarget { const overriddenDuration = getOverriddenDurationFromFnResult(fnResult) if (overriddenDuration !== undefined) { - return overriddenDuration + return { overridden: true, taskTime: overriddenDuration } } - return taskTime + return { overridden: false, taskTime } } /** * Measures a single execution of the task function synchronously. - * @returns The measured execution time + * @returns The measured execution time and whether it was supplied by the + * task function via `overriddenDuration` */ - #measureSync (): number { + #measureSync (): { overridden: boolean; taskTime: number } { const taskStart = this.#timestampFn() as unknown as number // eslint-disable-next-line no-useless-call const fnResult = this.#fn.call(this) @@ -510,9 +542,9 @@ export class Task extends EventTarget { ) const overriddenDuration = getOverriddenDurationFromFnResult(fnResult) if (overriddenDuration !== undefined) { - return overriddenDuration + return { overridden: true, taskTime: overriddenDuration } } - return taskTime + return { overridden: false, taskTime } } /** @@ -555,26 +587,95 @@ export class Task extends EventTarget { /** * Processes the result of a benchmark run and updates the task result. * Calculates statistics from the collected samples and dispatches appropriate events. - * @param options - An object containing the error and latency samples from the run + * + * Ordering: + * 1. Apply overhead correction in-place on the collection-order sample array + * (alignment with `isOverridden` preserved — `latencySamples[i]` still + * matches `isOverridden[i]` because no sort has been performed yet). + * Samples whose duration was supplied via `overriddenDuration` are skipped. + * 2. Build a measured-only view (excluding `overriddenDuration` samples) used + * for both `detectedResolution` and timer-saturation detection. Constant + * `overriddenDuration` values would otherwise be reported as the timer + * grain or trigger a spurious low-distinct-count warning. + * 3. Compute `detectedResolution` from the measured-only subset. + * 4. Sort the working array for the final statistics. + * 5. Compute the final statistics on the (possibly corrected) sorted samples. + * 6. Classify timer saturation on the measured-only subset. + * 7. Dispatch `'cycle'` and `'complete'` events; dispatch `'warning'` (carrying + * the {@link TimerSaturationReason}) if a saturation criterion fired. + * @param options - An object containing the run results * @param options.error - The error that occurred during the run, if any + * @param options.isOverridden - Parallel boolean array (collection order) indicating + * which samples were supplied by the task function via `overriddenDuration`, + * or `undefined` when overhead correction is disabled * @param options.latencySamples - The array of latency samples collected during the run */ #processRunResult ({ error, + isOverridden, latencySamples, }: { error?: Error + isOverridden?: boolean[] latencySamples?: number[] }): void { if (isValidSamples(latencySamples)) { this.#runs = latencySamples.length + const overhead = this.#bench.timerOverhead + const hasOverhead = overhead !== undefined && overhead > 0 + + // Phase 1 — Subtract overhead while isOverridden[i] is still aligned with + // latencySamples[i] (both in collection order, pre-sort). + if (hasOverhead) { + for (let i = 0; i < latencySamples.length; i++) { + if (isOverridden?.[i] !== true) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + latencySamples[i] = Math.max(0, latencySamples[i]! - overhead) + } + } + } + + // Phase 2 — Capture measured-only samples (alignment with isOverridden + // is still valid since the array has not been sorted yet). + const hasAnyOverridden = isOverridden?.some(v => v) ?? false + const measuredOnly: number[] = hasAnyOverridden + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + ? latencySamples.filter((_, i) => isOverridden![i] !== true) + : latencySamples + + // Phase 3 — Resolution diagnostic on the measured-only subset. + // Excluding `overriddenDuration` samples prevents a constant user + // value from being reported as the timer grain. `estimateResolution` + // is sort-invariant, so it can run before the working-array sort. + this.#detectedResolution = isValidSamples(measuredOnly) + ? estimateResolution(measuredOnly) + : undefined + + // Phase 4 — Single sort of the working array. sortSamples(latencySamples) + // Phase 5 — Final statistics on (possibly corrected) sorted samples. const latencyStatistics = computeStatistics( latencySamples, this.#retainSamples ) + + // Phase 6 — Saturation classification on the measured-only subset. + let saturationReason: TimerSaturationReason | undefined + if (measuredOnly === latencySamples) { + saturationReason = classifyTimerSaturation( + latencySamples, + latencyStatistics.mad + ) + } else if (isValidSamples(measuredOnly)) { + sortSamples(measuredOnly) + saturationReason = classifyTimerSaturation( + measuredOnly, + medianAbsoluteDeviation(measuredOnly) + ) + } + const latencyStatisticsMean = latencyStatistics.mean let totalTime = 0 @@ -606,6 +707,12 @@ export class Task extends EventTarget { totalTime, } /* eslint-enable perfectionist/sort-objects */ + + if (saturationReason !== undefined) { + const warningEv = new BenchEvent('warning', this, saturationReason) + this.dispatchEvent(warningEv) + this.#bench.dispatchEvent(warningEv) + } } else if (this.#aborted) { // If aborted with no samples, still set the aborted flag this.#result = abortedTaskResult diff --git a/src/types.ts b/src/types.ts index 3ca5310c..cc54f5eb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -23,6 +23,7 @@ export type BenchEvents = | 'reset' // when the reset method gets called | 'start' // when running the benchmarks gets started | 'warmup' // when the benchmarks start getting warmed up + | 'warning' // when timer saturation is detected for a task's latency samples /** * Bench events that may have an associated Task @@ -41,7 +42,7 @@ export type BenchEventsWithError = Extract */ export type BenchEventsWithTask = Extract< BenchEvents, - 'add' | 'cycle' | 'error' | 'remove' + 'add' | 'cycle' | 'error' | 'remove' | 'warning' > /** @@ -120,6 +121,13 @@ export interface BenchLike extends EventTarget { * The amount of time to run each task. */ time: number + /** + * The estimated cost of one timestamp provider call in milliseconds. + * + * Calibrated once at construction; `undefined` (or omitted) when timer + * overhead subtraction is disabled or unsupported by the implementation. + */ + readonly timerOverhead?: number /** * The timestamp provider used by the benchmark. */ @@ -183,6 +191,65 @@ export interface BenchOptions { */ signal?: AbortSignal + /** + * Whether to subtract an estimated timestamp provider call overhead from + * each raw latency sample. + * + * Each sample is measured as `t1 - t0` around a single call to the task + * function, so every raw sample is inflated by approximately one + * timestamp provider call cost `C`. When this option is `true`, an + * estimate `Ĉ` is computed once at construction time via + * {@link calibrateTimerOverhead}, and `max(0, raw_sample - Ĉ)` is used + * in place of each non-overridden sample before statistics are computed. + * + * **Statistics after correction.** All fields of {@link Statistics} are + * derived from the clamped corrected samples, not from the raw + * distribution. With `M` denoting the raw-sample mean: + * + * - **Clean-shift regime (`X >> Ĉ`).** The clamp `max(0, …)` rarely + * triggers, so the correction acts as a translation by `Ĉ`. Location + * statistics (`mean`, `min`, `max`, all percentiles) decrease by `Ĉ`; + * absolute-unit dispersion (`vr`, `sd`, `sem`, `moe`, `mad`, `aad`) + * is essentially unchanged. Because `rme = moe / mean`, it inflates + * by the deterministic factor `M / (M − Ĉ)` whenever `Ĉ > 0`. + * - **Sub-overhead regime (`X ≈ Ĉ`).** A non-trivial fraction of + * samples clamp to `0`, biasing the corrected mean upward, + * contracting `vr`/`sd`/`sem`/`moe`/`aad`, and compounding the + * `M / (M − Ĉ)` factor in `rme`. Once the cumulative mass of raw + * samples at or below `Ĉ` reaches a given quantile, that percentile + * collapses to `0`; in particular `p50` collapses once at least half + * of the raw samples satisfy `raw_sample ≤ Ĉ`, which then forces + * `mad` and `aad` toward `0`. Prefer `overriddenDuration` for + * sub-overhead measurements. + * + * **Three observable consequences of the clamp.** + * + * 1. `latency.min` may be exactly `0` even when no zero-duration sample + * was actually observed. + * 2. The throughput estimator substitutes `1000 / latency.mean` (or `0` + * when `mean === 0`) for every clamped sample. + * 3. {@link detectTimerSaturation} criterion `'zero-dominated'` cannot + * distinguish clamped samples from genuine zero-duration timer + * reads, so a `'warning'` event may be dispatched in the + * sub-overhead regime even when the timer itself is not saturated. + * + * **Caveat — `concurrency: "task"`.** The overhead is calibrated once + * at construction time with sequential timer calls. Setting both + * options causes the constructor (and `run()`) to throw, since the + * sequentially-calibrated estimate would not reflect the per-iteration + * timer call cost under concurrent execution. + * + * **Caveat — `overriddenDuration`.** Samples returned by the task + * function via `overriddenDuration` are intentional user values and + * are never modified by the correction. They are also excluded from + * {@link Task.detectedResolution} and from timer-saturation detection. + * + * On runtimes with a coarse timer (resolution >= 1 ms), the + * calibration returns `0` and this option becomes a no-op. + * @default false + */ + subtractTimerOverhead?: boolean + /** * Teardown function to run after each benchmark task (cycle). */ @@ -385,7 +452,6 @@ export type JSRuntime = | 'v8' | 'workerd' -/** /** * A function that returns the current timestamp. */ @@ -403,6 +469,7 @@ export interface ResolvedBenchOptions extends BenchOptions { iterations: NonNullable now: NonNullable setup: NonNullable + subtractTimerOverhead: NonNullable teardown: NonNullable throws: NonNullable time: NonNullable @@ -538,6 +605,7 @@ export type TaskEvents = Extract< | 'reset' // when the reset method gets called | 'start' // when running the task gets started | 'warmup' // when the task start getting warmed up + | 'warning' // when timer saturation is detected for the task's latency samples > /** @@ -672,6 +740,20 @@ export interface TaskResultWithStatistics { totalTime: number } +/** + * Reason a sample set is classified as timer-saturated. + * + * - `'zero-dominated'` — more than half of the samples are exactly zero. + * - `'low-distinct'` — distinct sample count is below + * `max(3, min(10, ⌊n / 1000⌋))`. + * - `'zero-mad'` — median absolute deviation is zero with more than 100 + * samples. + */ +export type TimerSaturationReason = + | 'low-distinct' + | 'zero-dominated' + | 'zero-mad' + /** * A timestamp function that returns either a number or bigint. */ diff --git a/src/utils.ts b/src/utils.ts index faa883e2..2f853c01 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -10,6 +10,7 @@ import type { Samples, SortedSamples, Statistics, + TimerSaturationReason, TimestampProvider, TimestampValue, } from './types' @@ -260,6 +261,104 @@ export const isValidSamples = ( return Array.isArray(value) && value.length !== 0 } +/** + * Classifies timer saturation in a latency sample set. + * + * Criteria are evaluated in the fixed order `'zero-dominated'` → + * `'low-distinct'` → `'zero-mad'`; the first match wins. Fewer than 10 + * samples are never classified — with so few measurements the criteria + * cannot reliably distinguish a deterministic fast function from one truly + * limited by the timer grain. + * + * The distinct-value count is computed in O(n) by exploiting the + * sorted-ascending invariant of `samples` and short-circuits as soon as + * the threshold is reached. + * @param samples - the latency samples, sorted ascending + * @param mad - the median absolute deviation (e.g. from + * {@link medianAbsoluteDeviation} or `computeStatistics`) + * @returns the saturation reason, or `undefined` when no criterion fires + */ +export const classifyTimerSaturation = ( + samples: SortedSamples, + mad: number +): TimerSaturationReason | undefined => { + const n = samples.length + if (n < 10) return undefined + + let zeroCount = 0 + for (const s of samples) { + if (s === 0) zeroCount++ + } + if (zeroCount * 2 > n) return 'zero-dominated' + + const distinctThreshold = Math.max(3, Math.min(10, Math.floor(n / 1000))) + let distinctCount = 1 + for (let i = 1; i < n; i++) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + if (samples[i]! !== samples[i - 1]!) { + distinctCount++ + if (distinctCount >= distinctThreshold) break + } + } + if (distinctCount < distinctThreshold) return 'low-distinct' + + if (n > 100 && mad === 0) return 'zero-mad' + + return undefined +} + +/** + * Detects timer saturation in a latency sample set. + * + * Boolean wrapper around {@link classifyTimerSaturation}; prefer the + * classifier when the specific reason is needed (e.g. to surface it on a + * `'warning'` event). + * @param samples - the latency samples, sorted ascending + * @param mad - the median absolute deviation + * @returns `true` when a saturation criterion fires, `false` otherwise + */ +export const detectTimerSaturation = ( + samples: SortedSamples, + mad: number +): boolean => classifyTimerSaturation(samples, mad) !== undefined + +/** + * Estimates the effective timer resolution from a latency sample set. + * + * The estimator returns the smallest strictly positive sample value that + * appears at least twice (the smallest reproducibly observed increment). + * Requiring two occurrences gives a 2/n breakdown point and avoids being + * pulled to an artificially low value by a single anomalous sample (cold + * cache, GC pause, hardware quirk). + * + * When no positive value appears more than once (e.g. a continuous + * sub-microsecond timer with all unique samples), falls back to the strict + * minimum of the positive values, which is the best available lower bound + * in that case. + * @param samples - the latency samples (sorted or unsorted) + * @returns the estimated resolution in milliseconds, or `undefined` when no + * strictly positive sample is observed + */ +export const estimateResolution = ( + samples: Samples +): number | undefined => { + const counts = new Map() + let fallbackMin = Number.POSITIVE_INFINITY + for (const s of samples) { + if (s > 0) { + counts.set(s, (counts.get(s) ?? 0) + 1) + if (s < fallbackMin) fallbackMin = s + } + } + if (fallbackMin === Number.POSITIVE_INFINITY) return undefined + + let robustMin = Number.POSITIVE_INFINITY + for (const [v, c] of counts) { + if (c >= 2 && v < robustMin) robustMin = v + } + return robustMin === Number.POSITIVE_INFINITY ? fallbackMin : robustMin +} + /** * Sorts samples in place. * @param samples - samples to sort @@ -332,6 +431,114 @@ const quantileSorted = ( */ export const sortFn = (a: number, b: number) => a - b +/** + * Options for {@link calibrateTimerOverhead}. + */ +export interface CalibrateTimerOverheadOptions { + /** + * Estimator used to reduce the distribution of strictly-positive + * back-to-back call deltas to a single overhead value. + * @default 'median' + */ + estimator?: TimerOverheadEstimatorKind + /** + * Number of back-to-back call pairs to measure during the collection phase. + * @default 1024 + */ + samples?: number + /** + * Number of discarded warm-up pairs executed before the collection phase, + * allowing the JIT to reach a steady compilation tier for both + * `provider.fn` and `provider.toMs`. + * @default 64 + */ + warmupSamples?: number +} + +/** + * Estimator strategy for {@link calibrateTimerOverhead}. + * + * - `'median'` — median of strictly-positive deltas (default). Robust to + * occasional OS-scheduling jitter and GC spikes at the cost of a slight + * upward bias on noisy hosts. + * - `'min'` — minimum of strictly-positive deltas. Captures the lowest + * observed call cost. + * - `'p05'` — 5th percentile of strictly-positive deltas. A compromise + * between robustness and tightness. + */ +export type TimerOverheadEstimatorKind = 'median' | 'min' | 'p05' + +/** + * Estimates the cost of a single `provider.fn()` call by repeatedly measuring + * back-to-back pairs and reducing the strictly-positive deltas to a single + * value via the chosen estimator. + * + * **Coarse-timer detection.** When the timer resolution `R` exceeds the call + * cost `C` (`C < R / 2`), the probability that any pair crosses a tick + * boundary is `C / R < 1 / 2`, so most pairs return a delta of zero. The + * positive deltas that do occur each equal exactly one tick `R`, not the + * call cost. To prevent catastrophic over-correction, the function returns + * `0` whenever fewer than half of the pairs produce a positive delta. + * + * **Bigint precision.** The subtraction is performed in the provider's + * native type before conversion to milliseconds (`toMs(b - a)`). For + * `hrtimeNow`, this preserves precision when absolute timestamps exceed + * `Number.MAX_SAFE_INTEGER` ns (≈ 104 days uptime). + * + * **JIT warmup.** A discarded warmup phase ensures `fn` and `toMs` are + * JIT-compiled to their steady-state tier before measurements begin. + * @param provider - the timestamp provider to calibrate + * @param options - calibration options + * @returns the estimated overhead in milliseconds, never negative; `0` when + * the timer resolution dominates or no positive delta is observed + */ +export const calibrateTimerOverhead = ( + provider: TimestampProvider, + options: CalibrateTimerOverheadOptions = {} +): number => { + const { estimator = 'median', samples = 1024, warmupSamples = 64 } = options + const { fn, toMs } = provider + + // `fn` returns TimestampValue (`bigint | number`); both operands always + // share a runtime type. Casting both to `bigint` lets the operator + // typecheck without a type predicate; at runtime the `-` operator is + // polymorphic for both numeric branches and `toMs` accepts either. + for (let i = 0; i < warmupSamples; i++) { + const a = fn() as bigint + const b = fn() as bigint + toMs(b - a) + } + + const deltas: number[] = [] + for (let i = 0; i < samples; i++) { + const a = fn() as bigint + const b = fn() as bigint + const delta = toMs(b - a) + if (delta > 0) deltas.push(delta) + } + + if (deltas.length * 2 < samples) return 0 + if (deltas.length === 0) return 0 + + deltas.sort(sortFn) + + if (estimator === 'min') { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return deltas[0]! + } + if (estimator === 'p05') { + const idx = Math.max(0, Math.ceil(deltas.length * 0.05) - 1) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return deltas[idx]! + } + const mid = deltas.length >> 1 + return (deltas.length & 1) === 1 + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + ? deltas[mid]! + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + : (deltas[mid - 1]! + deltas[mid]!) / 2 +} + /** * Computes the average absolute deviation from the mean. * @param samples - the sample @@ -406,6 +613,19 @@ export function absoluteDeviationMedian ( return 0 // should never reach here } +/** + * Computes the median absolute deviation (MAD) of a sorted sample set. + * + * Convenience wrapper that derives the median from the sorted input and + * forwards to `absoluteDeviationMedian`. Use when only `mad` is + * required and the cost of a full `computeStatistics` pass is + * unjustified (e.g. inside {@link classifyTimerSaturation}). + * @param samples - the sorted sample, length ≥ 1 + * @returns the median absolute deviation + */ +export const medianAbsoluteDeviation = (samples: SortedSamples): number => + absoluteDeviationMedian(samples, quantileSorted(samples, 0.5)) + /** * Computes the statistics of a sample. * The sample must be sorted. diff --git a/test/calibrate-timer-overhead.test.ts b/test/calibrate-timer-overhead.test.ts new file mode 100644 index 00000000..a38a9dbd --- /dev/null +++ b/test/calibrate-timer-overhead.test.ts @@ -0,0 +1,179 @@ +import { expect, test } from 'vitest' + +import type { TimestampProvider } from '../src/types' + +import { Bench } from '../src' +import { + calibrateTimerOverhead, + hrtimeNowTimestampProvider, + mToMs, + nToMs, + performanceNowTimestampProvider, +} from '../src/utils' + +/** + * Deterministic provider: pair `i` returns `(0, i + 1)`, so the i-th + * collected delta equals `(i + 1)` ns. With `samples = N` the sorted + * deltas (in ms) are `[1, 2, …, N] × 1e-6`. + * @returns a fresh provider with its own counter + */ +const makeAscendingPairProvider = (): TimestampProvider => { + let callCount = 0 + return { + fn: () => { + const idx = callCount++ + const pairIdx = idx >> 1 + return (idx & 1) === 1 ? pairIdx + 1 : 0 + }, + fromMs: ms => ms * 1_000_000, + name: 'asc-pairs', + toMs: nToMs, + } +} + +test('calibrateTimerOverhead returns a finite non-negative number with performanceNow', () => { + const overhead = calibrateTimerOverhead(performanceNowTimestampProvider) + expect(overhead).toBeTypeOf('number') + expect(overhead).toBeGreaterThanOrEqual(0) + expect(Number.isFinite(overhead)).toBe(true) +}) + +test('calibrateTimerOverhead returns a finite non-negative number with hrtimeNow', () => { + const overhead = calibrateTimerOverhead(hrtimeNowTimestampProvider) + expect(overhead).toBeTypeOf('number') + expect(overhead).toBeGreaterThanOrEqual(0) + expect(Number.isFinite(overhead)).toBe(true) +}) + +test('calibrateTimerOverhead returns 0 for a fixed-value provider', () => { + const fixedProvider: TimestampProvider = { + fn: () => 42, + fromMs: mToMs, + name: 'fixed', + toMs: mToMs, + } + expect(calibrateTimerOverhead(fixedProvider, { samples: 256 })).toBe(0) +}) + +test('calibrateTimerOverhead returns 0 for a coarse 1 ms timer provider', () => { + let counter = 0 + const coarseProvider: TimestampProvider = { + fn: () => Math.floor(counter++ / 64), + fromMs: mToMs, + name: 'coarse', + toMs: mToMs, + } + expect(calibrateTimerOverhead(coarseProvider, { samples: 1024 })).toBe(0) +}) + +test('calibrateTimerOverhead estimators are ordered min ≤ p05 ≤ median', () => { + const min = calibrateTimerOverhead(makeAscendingPairProvider(), { + estimator: 'min', + samples: 100, + warmupSamples: 0, + }) + const p05 = calibrateTimerOverhead(makeAscendingPairProvider(), { + estimator: 'p05', + samples: 100, + warmupSamples: 0, + }) + const median = calibrateTimerOverhead(makeAscendingPairProvider(), { + estimator: 'median', + samples: 100, + warmupSamples: 0, + }) + expect(min).toBe(1e-6) + expect(p05).toBe(5e-6) + expect(median).toBe(50.5e-6) +}) + +test("calibrateTimerOverhead 'p05' selects the index ⌈n·0.05⌉ − 1 delta", () => { + expect( + calibrateTimerOverhead(makeAscendingPairProvider(), { + estimator: 'p05', + samples: 20, + warmupSamples: 0, + }) + ).toBe(1e-6) + expect( + calibrateTimerOverhead(makeAscendingPairProvider(), { + estimator: 'p05', + samples: 21, + warmupSamples: 0, + }) + ).toBe(2e-6) + expect( + calibrateTimerOverhead(makeAscendingPairProvider(), { + estimator: 'p05', + samples: 200, + warmupSamples: 0, + }) + ).toBe(10e-6) +}) + +test('calibrateTimerOverhead with hrtimeNow returns a plausible overhead under 10 microseconds', () => { + const overhead = calibrateTimerOverhead(hrtimeNowTimestampProvider, { + estimator: 'median', + samples: 1024, + }) + if (overhead > 0) { + expect(overhead).toBeLessThan(0.01) + } else { + expect(overhead).toBe(0) + } +}) + +test('subtractTimerOverhead defaults to false and leaves timerOverhead undefined', () => { + const bench = new Bench() + expect(bench.subtractTimerOverhead).toBe(false) + expect(bench.timerOverhead).toBeUndefined() +}) + +test('subtractTimerOverhead: true populates a finite non-negative timerOverhead', () => { + const bench = new Bench({ subtractTimerOverhead: true }) + expect(bench.subtractTimerOverhead).toBe(true) + expect(bench.timerOverhead).toBeTypeOf('number') + expect(Number.isFinite(bench.timerOverhead)).toBe(true) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + expect(bench.timerOverhead!).toBeGreaterThanOrEqual(0) +}) + +test('subtractTimerOverhead: true does not produce negative latency samples', () => { + const bench = new Bench({ + iterations: 64, + subtractTimerOverhead: true, + time: 100, + warmup: false, + }) + bench.add('noop', () => { + // noop + }) + bench.runSync() + + const fooTask = bench.getTask('noop') + expect(fooTask).toBeDefined() + if (!fooTask) return + + expect(fooTask.result.state).toBe('completed') + if (fooTask.result.state !== 'completed') return + + expect(fooTask.result.latency.min).toBeGreaterThanOrEqual(0) + expect(fooTask.result.latency.mean).toBeGreaterThanOrEqual(0) +}) + +test('subtractTimerOverhead with concurrency: "task" throws at construction', () => { + expect( + () => new Bench({ concurrency: 'task', subtractTimerOverhead: true }) + ).toThrow(/cannot be used with `concurrency: "task"`/) +}) + +test('subtractTimerOverhead enforces concurrency invariant at run()', async () => { + const bench = new Bench({ subtractTimerOverhead: true }) + bench.add('noop', () => { + // noop + }) + ;(bench as { concurrency: 'bench' | 'task' | null }).concurrency = 'task' + await expect(bench.run()).rejects.toThrow( + /cannot be used with `concurrency: "task"`/ + ) +}) diff --git a/test/detected-resolution.test.ts b/test/detected-resolution.test.ts new file mode 100644 index 00000000..39c16847 --- /dev/null +++ b/test/detected-resolution.test.ts @@ -0,0 +1,72 @@ +import { expect, test } from 'vitest' + +import type { Samples } from '../src/types' + +import { Bench } from '../src' +import { estimateResolution } from '../src/utils' + +const asSamples = (arr: number[]): Samples => arr as unknown as Samples + +test('estimateResolution returns the smallest reproduced positive value', () => { + expect(estimateResolution(asSamples([0, 0, 0, 0.001, 1, 1, 1]))).toBe(1) + expect(estimateResolution(asSamples([0.5, 1, 1, 2, 2]))).toBe(1) + expect(estimateResolution(asSamples([0.1, 0.1, 0.5, 0.5]))).toBe(0.1) +}) + +test('estimateResolution falls back to strict min when no value repeats', () => { + expect(estimateResolution(asSamples([0, 0, 0.5, 1, 2]))).toBe(0.5) + expect(estimateResolution(asSamples([1, 2, 3]))).toBe(1) +}) + +test('estimateResolution returns the value itself when all positives are equal', () => { + expect(estimateResolution(asSamples([1, 1, 1, 1]))).toBe(1) +}) + +test('estimateResolution returns undefined when all samples are zero or negative', () => { + expect(estimateResolution(asSamples([0, 0, 0]))).toBeUndefined() + expect(estimateResolution(asSamples([0]))).toBeUndefined() + expect(estimateResolution(asSamples([0, -1, -0.5]))).toBeUndefined() +}) + +test('Task.detectedResolution is undefined before run', () => { + const bench = new Bench() + bench.add('foo', () => { + // noop + }) + const fooTask = bench.getTask('foo') + expect(fooTask).toBeDefined() + if (!fooTask) return + expect(fooTask.detectedResolution).toBeUndefined() +}) + +test('Task.detectedResolution is populated after a successful run', () => { + const bench = new Bench({ iterations: 64, time: 100, warmup: false }) + bench.add('foo', () => { + // noop + }) + bench.runSync() + const fooTask = bench.getTask('foo') + expect(fooTask).toBeDefined() + if (!fooTask) return + expect(fooTask.result.state).toBe('completed') + if (fooTask.result.state !== 'completed') return + + const resolution = fooTask.detectedResolution + expect(resolution).toBeDefined() + expect(resolution).toBeTypeOf('number') + expect(resolution).toBeGreaterThan(0) + expect(Number.isFinite(resolution)).toBe(true) +}) + +test('Task.detectedResolution is reset to undefined by reset()', () => { + const bench = new Bench({ iterations: 64, time: 100, warmup: false }) + bench.add('foo', () => { + // noop + }) + bench.runSync() + const fooTask = bench.getTask('foo') + expect(fooTask).toBeDefined() + if (!fooTask) return + fooTask.reset() + expect(fooTask.detectedResolution).toBeUndefined() +}) diff --git a/test/subtract-timer-overhead-alignment.test.ts b/test/subtract-timer-overhead-alignment.test.ts new file mode 100644 index 00000000..336f9cf4 --- /dev/null +++ b/test/subtract-timer-overhead-alignment.test.ts @@ -0,0 +1,92 @@ +import { expect, test } from 'vitest' + +import type { TimestampProvider } from '../src/types' + +import { Bench } from '../src' +import { nToMs } from '../src/utils' + +/** + * Deterministic provider where every back-to-back call pair yields a delta + * of exactly `stepNs` nanoseconds. Calibration converges on `stepNs / 1e6` + * ms; every measured raw `taskTime` equals that same overhead, so after + * `max(0, raw - Ĉ)` correction every measured sample becomes `0`. + * @param stepNs - the per-call increment in nanoseconds + * @returns a fresh provider with its own counter + */ +const makeStepProvider = (stepNs: number): TimestampProvider => { + let counter = 0 + return { + fn: () => { + counter += 1 + return counter * stepNs + }, + fromMs: ms => ms * 1_000_000, + name: 'det-step', + toMs: nToMs, + } +} + +test('subtractTimerOverhead aligns isOverridden with samples in mixed runs', async () => { + const iterations = 32 + const K = 100 + const stepNs = 1000 + const stepMs = stepNs / 1_000_000 + const bench = new Bench({ + iterations, + retainSamples: true, + subtractTimerOverhead: true, + time: 0, + timestampProvider: makeStepProvider(stepNs), + warmup: false, + }) + expect(bench.timerOverhead).toBe(stepMs) + + let i = 0 + bench.add('alternating', () => { + if ((i++ & 1) === 0) return { overriddenDuration: K } + return undefined + }) + await bench.run() + + const task = bench.getTask('alternating') + expect(task).toBeDefined() + if (!task) return + expect(task.result.state).toBe('completed') + if (task.result.state !== 'completed') return + + const samples = task.result.latency.samples + expect(samples).toBeDefined() + if (!samples) return + expect(samples.length).toBe(iterations) + expect(samples.filter(s => s === K).length).toBe(iterations / 2) + expect(samples.filter(s => s === 0).length).toBe(iterations / 2) + expect(samples.every(s => s >= 0)).toBe(true) + expect(task.result.latency.min).toBe(0) + expect(task.result.latency.max).toBe(K) +}) + +test('subtractTimerOverhead alignment holds with the real timer', async () => { + const iterations = 32 + const K = 0.0000001234567 + const bench = new Bench({ + iterations, + retainSamples: true, + subtractTimerOverhead: true, + time: 0, + warmup: false, + }) + let i = 0 + bench.add('alternating', () => { + if ((i++ & 1) === 0) return { overriddenDuration: K } + return undefined + }) + await bench.run() + + const task = bench.getTask('alternating') + if (task?.result.state !== 'completed') return + const samples = task.result.latency.samples + if (!samples) return + expect(samples.filter(s => s === K).length).toBe(iterations / 2) + expect(samples.filter(s => s !== K).every(s => s >= 0)).toBe(true) + expect(samples.filter(s => s !== K).length).toBe(iterations / 2) +}) diff --git a/test/subtract-timer-overhead-overridden.test.ts b/test/subtract-timer-overhead-overridden.test.ts new file mode 100644 index 00000000..dc9940a2 --- /dev/null +++ b/test/subtract-timer-overhead-overridden.test.ts @@ -0,0 +1,80 @@ +import { expect, test } from 'vitest' + +import { Bench } from '../src' + +test('subtractTimerOverhead does not modify samples returned via overriddenDuration', async () => { + const target = 1 + const bench = new Bench({ + iterations: 32, + subtractTimerOverhead: true, + time: 0, + warmup: false, + }) + bench.add('override-bench', () => { + return { overriddenDuration: target } + }) + await bench.run() + + const task = bench.getTask('override-bench') + expect(task).toBeDefined() + if (!task) return + expect(task.result.state).toBe('completed') + if (task.result.state !== 'completed') return + + expect(task.result.latency.mean).toBeCloseTo(target, 5) + expect(task.result.latency.min).toBeCloseTo(target, 5) + expect(task.result.latency.max).toBeCloseTo(target, 5) +}) + +test('warning event is not dispatched for fully-overridden constant duration (issue #10)', async () => { + const bench = new Bench({ + iterations: 32, + subtractTimerOverhead: true, + time: 0, + warmup: false, + }) + let warningCount = 0 + bench.addEventListener('warning', () => { + warningCount++ + }) + bench.add('override-bench', () => { + return { overriddenDuration: 0.05 } + }) + await bench.run() + + // All 32 samples are overridden → measuredOnly is empty → no saturation + // detection runs → no warning. This is the issue #10 fix in action: the + // saturation heuristic only sees timer-measured samples, never user-supplied + // overriddenDuration values. + expect(warningCount).toBe(0) +}) + +test('detectedResolution is undefined when every sample is overridden', async () => { + const benchA = new Bench({ + iterations: 64, + subtractTimerOverhead: false, + time: 100, + warmup: false, + }) + const benchB = new Bench({ + iterations: 64, + subtractTimerOverhead: true, + time: 100, + warmup: false, + }) + + benchA.add('regex', () => { + return { overriddenDuration: 0.001 } + }) + benchB.add('regex', () => { + return { overriddenDuration: 0.001 } + }) + + await benchA.run() + await benchB.run() + + const taskA = benchA.getTask('regex') + const taskB = benchB.getTask('regex') + expect(taskA?.detectedResolution).toBeUndefined() + expect(taskB?.detectedResolution).toBeUndefined() +}) diff --git a/test/utils-detect-timer-saturation.test.ts b/test/utils-detect-timer-saturation.test.ts new file mode 100644 index 00000000..244990c5 --- /dev/null +++ b/test/utils-detect-timer-saturation.test.ts @@ -0,0 +1,97 @@ +import { expect, test } from 'vitest' + +import type { SortedSamples } from '../src/types' + +import { + classifyTimerSaturation, + detectTimerSaturation, +} from '../src/utils' + +const asSorted = (arr: number[]): SortedSamples => + arr as unknown as SortedSamples + +test('detectTimerSaturation returns false for n below the minimum threshold', () => { + expect(detectTimerSaturation(asSorted([1]), 0)).toBe(false) + expect( + detectTimerSaturation(asSorted([1, 1, 1, 1, 1, 1, 1, 1, 1]), 0) + ).toBe(false) +}) + +test('detectTimerSaturation flags more than half zero samples (criterion A)', () => { + expect( + detectTimerSaturation(asSorted([0, 0, 0, 0, 0, 0, 1, 2, 3, 4]), 0) + ).toBe(true) +}) + +test('detectTimerSaturation does not flag exactly half zero samples', () => { + expect( + detectTimerSaturation(asSorted([0, 0, 0, 0, 0, 1, 2, 3, 4, 5]), 1) + ).toBe(false) +}) + +test('detectTimerSaturation flags degenerate distinct counts (criterion B)', () => { + expect( + detectTimerSaturation(asSorted(new Array(64).fill(1)), 0) + ).toBe(true) + const halfHalf = new Array(500) + .fill(1) + .concat(new Array(500).fill(2)) + expect(detectTimerSaturation(asSorted(halfHalf), 0.5)).toBe(true) +}) + +test('detectTimerSaturation flags zero MAD with more than 100 samples (criterion C)', () => { + const arr: number[] = [] + for (let i = 0; i < 120; i++) arr.push(5) + for (let i = 0; i < 80; i++) arr.push((i % 10) + 1) + arr.sort((a, b) => a - b) + expect(detectTimerSaturation(asSorted(arr), 0)).toBe(true) +}) + +test('detectTimerSaturation does not flag healthy spread samples', () => { + const arr: number[] = [] + let seed = 42 + for (let i = 0; i < 500; i++) { + seed = (seed * 1664525 + 1013904223) >>> 0 + arr.push(50 + ((seed >>> 0) / 0xffffffff - 0.5) * 10) + } + arr.sort((a, b) => a - b) + expect(detectTimerSaturation(asSorted(arr), 1.5)).toBe(false) +}) + +test('classifyTimerSaturation returns undefined for n below the minimum threshold', () => { + expect(classifyTimerSaturation(asSorted([1]), 0)).toBeUndefined() + expect( + classifyTimerSaturation(asSorted([1, 1, 1, 1, 1, 1, 1, 1, 1]), 0) + ).toBeUndefined() +}) + +test("classifyTimerSaturation returns 'zero-dominated' for criterion A", () => { + expect( + classifyTimerSaturation(asSorted([0, 0, 0, 0, 0, 0, 1, 2, 3, 4]), 0) + ).toBe('zero-dominated') +}) + +test("classifyTimerSaturation returns 'low-distinct' for criterion B", () => { + expect( + classifyTimerSaturation(asSorted(new Array(64).fill(1)), 0) + ).toBe('low-distinct') +}) + +test("classifyTimerSaturation returns 'zero-mad' for criterion C", () => { + const arr: number[] = [] + for (let i = 0; i < 120; i++) arr.push(5) + for (let i = 0; i < 80; i++) arr.push((i % 10) + 1) + arr.sort((a, b) => a - b) + expect(classifyTimerSaturation(asSorted(arr), 0)).toBe('zero-mad') +}) + +test('classifyTimerSaturation returns undefined for healthy spread samples', () => { + const arr: number[] = [] + let seed = 42 + for (let i = 0; i < 500; i++) { + seed = (seed * 1664525 + 1013904223) >>> 0 + arr.push(50 + ((seed >>> 0) / 0xffffffff - 0.5) * 10) + } + arr.sort((a, b) => a - b) + expect(classifyTimerSaturation(asSorted(arr), 1.5)).toBeUndefined() +}) diff --git a/test/warning-event-reason.test.ts b/test/warning-event-reason.test.ts new file mode 100644 index 00000000..e6167e41 --- /dev/null +++ b/test/warning-event-reason.test.ts @@ -0,0 +1,44 @@ +import { expect, test } from 'vitest' + +import type { TimerSaturationReason, TimestampProvider } from '../src/types' + +import { Bench } from '../src' +import { mToMs } from '../src/utils' + +const fixedZeroProvider: TimestampProvider = { + fn: () => 0, + fromMs: mToMs, + name: 'fixed-zero', + toMs: mToMs, +} + +test("warning event carries reason 'zero-dominated' for a constant-zero timer", async () => { + const bench = new Bench({ + iterations: 64, + time: 0, + timestampProvider: fixedZeroProvider, + warmup: false, + }) + let received: TimerSaturationReason | undefined + bench.addEventListener('warning', evt => { + received = evt.reason + }) + bench.add('zero-task', () => { + // noop + }) + await bench.run() + expect(received).toBe('zero-dominated') +}) + +test('non-warning events expose reason as undefined', async () => { + const bench = new Bench({ iterations: 8, time: 0, warmup: false }) + let cycleReason: unknown = 'untouched' + bench.addEventListener('cycle', evt => { + cycleReason = evt.reason + }) + bench.add('noop', () => { + // noop + }) + await bench.run() + expect(cycleReason).toBeUndefined() +})