test(sim): Phase 1d-tests D + G — board-kind coverage matrix + perf baseline
D (board-kinds-coverage): iterates every BoardKind in src/types/board.ts and asserts each has at least one gallery example across all six examples-*.ts modules. Surfaces real coverage gaps without inventing fixtures: 9 BoardKinds today have no demo circuit (esp32 variants that share QEMU backends with covered primaries + attiny85 + raspberry-pi-3 backend QEMU). All documented as ACCEPTED_UNCOVERED with rationale. Adding a new BoardKind without either an example or an entry in that set fails the test — enforces deliberate coverage decisions. G (solver-perf-baseline): opt-in via `CI_PERF=1` env var. For 6 canonical examples, measures `solveMs` 10× and asserts median under a per-example ceiling (generous tolerances for CI variance). Default-skipped because CI machine timings would flake; enabled on demand for regression checks after a solver change. Adding a new BoardKind or canonical example extends coverage automatically — no duplicated lists. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
bed2bd90ef
commit
68c19a6663
|
|
@ -0,0 +1,141 @@
|
|||
/**
|
||||
* Board-kind coverage matrix (Phase 1d-tests D).
|
||||
*
|
||||
* Every velxio `BoardKind` is exercised by at least one example
|
||||
* across all six `examples-*.ts` modules. Adding a new board kind
|
||||
* to `src/types/board.ts` without adding at least one gallery
|
||||
* example for it should fail this test — keeps the demo coverage in
|
||||
* sync with the supported hardware list.
|
||||
*
|
||||
* Deep behavioural tests for each board family (AVR / RP2040 /
|
||||
* ESP32 / ESP32-C3) live in their dedicated `*Simulator.test.ts`
|
||||
* files; this test only asserts the gallery side.
|
||||
*
|
||||
* Fidelity (memory `feedback_tests_import_real_code`): imports
|
||||
* `BOARD_KIND_LABELS` (the canonical list) + every examples-*.ts
|
||||
* source-of-truth. No duplicated board list.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { BOARD_KIND_LABELS, type BoardKind } from '../types/board';
|
||||
import { analogExamples } from '../data/examples-analog';
|
||||
import { digitalExamples } from '../data/examples-digital';
|
||||
import { hundredDaysExamples } from '../data/examples-100-days';
|
||||
import { epaperExamples } from '../data/examples-displays-epaper';
|
||||
import { picowWifiExamples } from '../data/examples-picow-wifi';
|
||||
import { circuitExamples } from '../data/examples-circuits';
|
||||
import type { ExampleProject } from '../data/examples';
|
||||
|
||||
const ALL_EXAMPLES: ExampleProject[] = [
|
||||
...analogExamples,
|
||||
...digitalExamples,
|
||||
...hundredDaysExamples,
|
||||
...epaperExamples,
|
||||
...picowWifiExamples,
|
||||
...circuitExamples,
|
||||
];
|
||||
|
||||
/**
|
||||
* Boards that intentionally have no gallery example today. They
|
||||
* exist in the type for future support but no canvas demo ships.
|
||||
* Adding to this list requires a comment explaining why — keeps the
|
||||
* coverage gap visible at code-review time.
|
||||
*/
|
||||
const ACCEPTED_UNCOVERED: ReadonlySet<BoardKind> = new Set([
|
||||
// Pi 3B runs on the backend (QEMU ARM64) — no in-browser canvas
|
||||
// example because it boots a full Linux image.
|
||||
'raspberry-pi-3',
|
||||
|
||||
// ESP32 Xtensa LX6 variants — all share the same QEMU backend as
|
||||
// the primary `esp32` boardKind (which IS covered). Adding a
|
||||
// dedicated example per variant adds zero engine coverage; the
|
||||
// canvas just renders a different SVG.
|
||||
'esp32-cam',
|
||||
'wemos-lolin32-lite',
|
||||
|
||||
// ESP32-S3 Xtensa LX7 family — share the `esp32-s3` QEMU backend.
|
||||
// No primary `esp32-s3` example today either; the family is
|
||||
// emulator-ready but lacks a demo circuit. Add one when product
|
||||
// wants S3 in the gallery.
|
||||
'esp32-s3',
|
||||
'xiao-esp32-s3',
|
||||
'arduino-nano-esp32',
|
||||
|
||||
// ESP32-C3 RISC-V family — share the `esp32-c3` backend. Same as
|
||||
// above; no primary C3 example exists.
|
||||
'esp32-c3',
|
||||
'xiao-esp32-c3',
|
||||
'aitewinrobot-esp32c3-supermini',
|
||||
|
||||
// ATtiny85 — fully supported via avr8js but no gallery example
|
||||
// showcases its limited (5 GPIO) form factor. Add one when
|
||||
// someone proposes a use case.
|
||||
'attiny85',
|
||||
]);
|
||||
|
||||
interface Coverage {
|
||||
byBoardType: Map<BoardKind, string[]>;
|
||||
byBoardsArray: Map<BoardKind, string[]>;
|
||||
}
|
||||
|
||||
function buildCoverage(): Coverage {
|
||||
const byBoardType = new Map<BoardKind, string[]>();
|
||||
const byBoardsArray = new Map<BoardKind, string[]>();
|
||||
for (const ex of ALL_EXAMPLES) {
|
||||
if (ex.boardType) {
|
||||
const list = byBoardType.get(ex.boardType) ?? [];
|
||||
list.push(ex.id);
|
||||
byBoardType.set(ex.boardType, list);
|
||||
}
|
||||
if (ex.boards && ex.boards.length > 0) {
|
||||
for (const b of ex.boards) {
|
||||
const list = byBoardsArray.get(b.boardKind as BoardKind) ?? [];
|
||||
list.push(ex.id);
|
||||
byBoardsArray.set(b.boardKind as BoardKind, list);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { byBoardType, byBoardsArray };
|
||||
}
|
||||
|
||||
describe('BoardKind gallery coverage matrix', () => {
|
||||
const { byBoardType, byBoardsArray } = buildCoverage();
|
||||
const allKinds = Object.keys(BOARD_KIND_LABELS) as BoardKind[];
|
||||
|
||||
it.each(allKinds.map((k) => [k] as const))(
|
||||
'BoardKind %s has at least one gallery example (or is accepted as uncovered)',
|
||||
(kind) => {
|
||||
const count =
|
||||
(byBoardType.get(kind)?.length ?? 0) + (byBoardsArray.get(kind)?.length ?? 0);
|
||||
if (ACCEPTED_UNCOVERED.has(kind)) {
|
||||
expect(count, `${kind} is in ACCEPTED_UNCOVERED but actually has ${count} examples — remove from the accepted list`).toBe(0);
|
||||
return;
|
||||
}
|
||||
expect(
|
||||
count,
|
||||
`${kind} has no gallery example. Either add an example to data/examples-*.ts OR add ${kind} to ACCEPTED_UNCOVERED with a comment.`,
|
||||
).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
|
||||
it('summary: every BoardKind appears in BOARD_KIND_LABELS', () => {
|
||||
// Self-check that the LABELS map is exhaustive vs the type union.
|
||||
// If you add a kind to the BoardKind union without an entry in
|
||||
// BOARD_KIND_LABELS, TypeScript already catches it. This is a
|
||||
// runtime double-check.
|
||||
for (const kind of allKinds) {
|
||||
expect(BOARD_KIND_LABELS[kind], `${kind} missing label`).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
it('reports BoardKind coverage stats (informational)', () => {
|
||||
const stats = allKinds.map((kind) => {
|
||||
const inBoardType = byBoardType.get(kind)?.length ?? 0;
|
||||
const inBoardsArray = byBoardsArray.get(kind)?.length ?? 0;
|
||||
return { kind, total: inBoardType + inBoardsArray, inBoardType, inBoardsArray };
|
||||
});
|
||||
stats.sort((a, b) => b.total - a.total);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[board-coverage]', stats.map((s) => `${s.kind}=${s.total}`).join(' '));
|
||||
expect(stats.length).toBe(allKinds.length);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
/**
|
||||
* Solver performance baseline (Phase 1d-tests G — opt-in).
|
||||
*
|
||||
* For the same canonical examples that `solver-determinism` locks,
|
||||
* measure `solveMs` 10 times each and assert the median solve fits
|
||||
* inside a pre-defined ceiling. Detects perf regressions when
|
||||
* someone refactors `NetlistBuilder`, swaps a model, or upgrades
|
||||
* ngspice.
|
||||
*
|
||||
* Gated behind `CI_PERF=1` because:
|
||||
* • Numerics are deterministic, but timings are CI-machine dependent.
|
||||
* A GitHub-hosted runner can be 2-3× slower than a developer
|
||||
* workstation; running this in every PR would flake.
|
||||
* • 10× runs × 6 examples × ~50-500 ms each ≈ 30 s extra wall time.
|
||||
*
|
||||
* Enable on demand:
|
||||
* CI_PERF=1 npx vitest run src/__tests__/solver-perf-baseline.test.ts
|
||||
*
|
||||
* Update ceilings in the table below after a deliberate solver
|
||||
* change. The values are deliberately generous (2-3× expected) to
|
||||
* tolerate CI variance while still catching 10× regressions.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { analogExamples } from '../data/examples-analog';
|
||||
import { digitalExamples } from '../data/examples-digital';
|
||||
import { exampleToBuildNetlistInput } from '../utils/exampleToBuildNetlistInput';
|
||||
import { solveInput } from './helpers/solveInput';
|
||||
import type { ExampleProject } from '../data/examples';
|
||||
|
||||
const PERF_ENABLED = process.env.CI_PERF === '1';
|
||||
|
||||
/**
|
||||
* Per-example median-solve-time ceilings, ms. Generous to tolerate
|
||||
* CI variance; tighten after empirical baselining on a real CI box.
|
||||
*/
|
||||
const CEILINGS_MS: Record<string, number> = {
|
||||
'an-voltage-divider': 100,
|
||||
'an-rc-low-pass': 100,
|
||||
'an-half-wave-rectifier': 2000, // .tran with sine generator → slower
|
||||
'an-bjt-switch': 200,
|
||||
'digital-and-two-switches': 100,
|
||||
'digital-not-inverter': 100,
|
||||
};
|
||||
|
||||
function findExample(id: string): ExampleProject | undefined {
|
||||
return [...analogExamples, ...digitalExamples].find((ex) => ex.id === id);
|
||||
}
|
||||
|
||||
function median(values: number[]): number {
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
if (sorted.length % 2) return sorted[mid]!;
|
||||
return ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2;
|
||||
}
|
||||
|
||||
describe.skipIf(!PERF_ENABLED)('Solver performance baseline (CI_PERF=1)', () => {
|
||||
for (const [id, ceiling] of Object.entries(CEILINGS_MS)) {
|
||||
const example = findExample(id);
|
||||
if (!example) {
|
||||
it.skip(`${id} (not found — ceiling list out of sync)`, () => {});
|
||||
continue;
|
||||
}
|
||||
it(`${id} — median solve ms ≤ ${ceiling}`, { timeout: 60_000 }, async () => {
|
||||
const input = exampleToBuildNetlistInput(example);
|
||||
const samples: number[] = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const r = await solveInput(input);
|
||||
samples.push(r.solveMs);
|
||||
}
|
||||
const med = median(samples);
|
||||
const all = samples.map((s) => s.toFixed(1)).join(', ');
|
||||
expect(
|
||||
med,
|
||||
`median=${med.toFixed(1)} ms exceeds ${ceiling} ms ceiling for ${id}. Samples: [${all}]`,
|
||||
).toBeLessThanOrEqual(ceiling);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// When the suite is disabled (default), expose a trivial it() so the
|
||||
// file still registers in test discovery and a CI dashboard can show
|
||||
// "skipped" instead of an empty file.
|
||||
describe.skipIf(PERF_ENABLED)('Solver performance baseline (disabled — set CI_PERF=1 to enable)', () => {
|
||||
it('placeholder', () => {
|
||||
expect(PERF_ENABLED).toBe(false);
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue