From 68c19a666328842b7d17923516a69b7c7858412b Mon Sep 17 00:00:00 2001 From: davidmonterocrespo24 Date: Fri, 15 May 2026 23:13:48 +0200 Subject: [PATCH] =?UTF-8?q?test(sim):=20Phase=201d-tests=20D=20+=20G=20?= =?UTF-8?q?=E2=80=94=20board-kind=20coverage=20matrix=20+=20perf=20baselin?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D (board-kinds-coverage): iterates every BoardKind in src/types/board.ts and asserts each has at least one gallery example across all six examples-*.ts modules. Surfaces real coverage gaps without inventing fixtures: 9 BoardKinds today have no demo circuit (esp32 variants that share QEMU backends with covered primaries + attiny85 + raspberry-pi-3 backend QEMU). All documented as ACCEPTED_UNCOVERED with rationale. Adding a new BoardKind without either an example or an entry in that set fails the test — enforces deliberate coverage decisions. G (solver-perf-baseline): opt-in via `CI_PERF=1` env var. For 6 canonical examples, measures `solveMs` 10× and asserts median under a per-example ceiling (generous tolerances for CI variance). Default-skipped because CI machine timings would flake; enabled on demand for regression checks after a solver change. Adding a new BoardKind or canonical example extends coverage automatically — no duplicated lists. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../__tests__/board-kinds-coverage.test.ts | 141 ++++++++++++++++++ .../__tests__/solver-perf-baseline.test.ts | 87 +++++++++++ 2 files changed, 228 insertions(+) create mode 100644 frontend/src/__tests__/board-kinds-coverage.test.ts create mode 100644 frontend/src/__tests__/solver-perf-baseline.test.ts diff --git a/frontend/src/__tests__/board-kinds-coverage.test.ts b/frontend/src/__tests__/board-kinds-coverage.test.ts new file mode 100644 index 00000000..3c177713 --- /dev/null +++ b/frontend/src/__tests__/board-kinds-coverage.test.ts @@ -0,0 +1,141 @@ +/** + * Board-kind coverage matrix (Phase 1d-tests D). + * + * Every velxio `BoardKind` is exercised by at least one example + * across all six `examples-*.ts` modules. Adding a new board kind + * to `src/types/board.ts` without adding at least one gallery + * example for it should fail this test — keeps the demo coverage in + * sync with the supported hardware list. + * + * Deep behavioural tests for each board family (AVR / RP2040 / + * ESP32 / ESP32-C3) live in their dedicated `*Simulator.test.ts` + * files; this test only asserts the gallery side. + * + * Fidelity (memory `feedback_tests_import_real_code`): imports + * `BOARD_KIND_LABELS` (the canonical list) + every examples-*.ts + * source-of-truth. No duplicated board list. + */ +import { describe, it, expect } from 'vitest'; +import { BOARD_KIND_LABELS, type BoardKind } from '../types/board'; +import { analogExamples } from '../data/examples-analog'; +import { digitalExamples } from '../data/examples-digital'; +import { hundredDaysExamples } from '../data/examples-100-days'; +import { epaperExamples } from '../data/examples-displays-epaper'; +import { picowWifiExamples } from '../data/examples-picow-wifi'; +import { circuitExamples } from '../data/examples-circuits'; +import type { ExampleProject } from '../data/examples'; + +const ALL_EXAMPLES: ExampleProject[] = [ + ...analogExamples, + ...digitalExamples, + ...hundredDaysExamples, + ...epaperExamples, + ...picowWifiExamples, + ...circuitExamples, +]; + +/** + * Boards that intentionally have no gallery example today. They + * exist in the type for future support but no canvas demo ships. + * Adding to this list requires a comment explaining why — keeps the + * coverage gap visible at code-review time. + */ +const ACCEPTED_UNCOVERED: ReadonlySet = new Set([ + // Pi 3B runs on the backend (QEMU ARM64) — no in-browser canvas + // example because it boots a full Linux image. + 'raspberry-pi-3', + + // ESP32 Xtensa LX6 variants — all share the same QEMU backend as + // the primary `esp32` boardKind (which IS covered). Adding a + // dedicated example per variant adds zero engine coverage; the + // canvas just renders a different SVG. + 'esp32-cam', + 'wemos-lolin32-lite', + + // ESP32-S3 Xtensa LX7 family — share the `esp32-s3` QEMU backend. + // No primary `esp32-s3` example today either; the family is + // emulator-ready but lacks a demo circuit. Add one when product + // wants S3 in the gallery. + 'esp32-s3', + 'xiao-esp32-s3', + 'arduino-nano-esp32', + + // ESP32-C3 RISC-V family — share the `esp32-c3` backend. Same as + // above; no primary C3 example exists. + 'esp32-c3', + 'xiao-esp32-c3', + 'aitewinrobot-esp32c3-supermini', + + // ATtiny85 — fully supported via avr8js but no gallery example + // showcases its limited (5 GPIO) form factor. Add one when + // someone proposes a use case. + 'attiny85', +]); + +interface Coverage { + byBoardType: Map; + byBoardsArray: Map; +} + +function buildCoverage(): Coverage { + const byBoardType = new Map(); + const byBoardsArray = new Map(); + for (const ex of ALL_EXAMPLES) { + if (ex.boardType) { + const list = byBoardType.get(ex.boardType) ?? []; + list.push(ex.id); + byBoardType.set(ex.boardType, list); + } + if (ex.boards && ex.boards.length > 0) { + for (const b of ex.boards) { + const list = byBoardsArray.get(b.boardKind as BoardKind) ?? []; + list.push(ex.id); + byBoardsArray.set(b.boardKind as BoardKind, list); + } + } + } + return { byBoardType, byBoardsArray }; +} + +describe('BoardKind gallery coverage matrix', () => { + const { byBoardType, byBoardsArray } = buildCoverage(); + const allKinds = Object.keys(BOARD_KIND_LABELS) as BoardKind[]; + + it.each(allKinds.map((k) => [k] as const))( + 'BoardKind %s has at least one gallery example (or is accepted as uncovered)', + (kind) => { + const count = + (byBoardType.get(kind)?.length ?? 0) + (byBoardsArray.get(kind)?.length ?? 0); + if (ACCEPTED_UNCOVERED.has(kind)) { + expect(count, `${kind} is in ACCEPTED_UNCOVERED but actually has ${count} examples — remove from the accepted list`).toBe(0); + return; + } + expect( + count, + `${kind} has no gallery example. Either add an example to data/examples-*.ts OR add ${kind} to ACCEPTED_UNCOVERED with a comment.`, + ).toBeGreaterThan(0); + }, + ); + + it('summary: every BoardKind appears in BOARD_KIND_LABELS', () => { + // Self-check that the LABELS map is exhaustive vs the type union. + // If you add a kind to the BoardKind union without an entry in + // BOARD_KIND_LABELS, TypeScript already catches it. This is a + // runtime double-check. + for (const kind of allKinds) { + expect(BOARD_KIND_LABELS[kind], `${kind} missing label`).toBeTruthy(); + } + }); + + it('reports BoardKind coverage stats (informational)', () => { + const stats = allKinds.map((kind) => { + const inBoardType = byBoardType.get(kind)?.length ?? 0; + const inBoardsArray = byBoardsArray.get(kind)?.length ?? 0; + return { kind, total: inBoardType + inBoardsArray, inBoardType, inBoardsArray }; + }); + stats.sort((a, b) => b.total - a.total); + // eslint-disable-next-line no-console + console.log('[board-coverage]', stats.map((s) => `${s.kind}=${s.total}`).join(' ')); + expect(stats.length).toBe(allKinds.length); + }); +}); diff --git a/frontend/src/__tests__/solver-perf-baseline.test.ts b/frontend/src/__tests__/solver-perf-baseline.test.ts new file mode 100644 index 00000000..b7dddc85 --- /dev/null +++ b/frontend/src/__tests__/solver-perf-baseline.test.ts @@ -0,0 +1,87 @@ +/** + * Solver performance baseline (Phase 1d-tests G — opt-in). + * + * For the same canonical examples that `solver-determinism` locks, + * measure `solveMs` 10 times each and assert the median solve fits + * inside a pre-defined ceiling. Detects perf regressions when + * someone refactors `NetlistBuilder`, swaps a model, or upgrades + * ngspice. + * + * Gated behind `CI_PERF=1` because: + * • Numerics are deterministic, but timings are CI-machine dependent. + * A GitHub-hosted runner can be 2-3× slower than a developer + * workstation; running this in every PR would flake. + * • 10× runs × 6 examples × ~50-500 ms each ≈ 30 s extra wall time. + * + * Enable on demand: + * CI_PERF=1 npx vitest run src/__tests__/solver-perf-baseline.test.ts + * + * Update ceilings in the table below after a deliberate solver + * change. The values are deliberately generous (2-3× expected) to + * tolerate CI variance while still catching 10× regressions. + */ +import { describe, it, expect } from 'vitest'; +import { analogExamples } from '../data/examples-analog'; +import { digitalExamples } from '../data/examples-digital'; +import { exampleToBuildNetlistInput } from '../utils/exampleToBuildNetlistInput'; +import { solveInput } from './helpers/solveInput'; +import type { ExampleProject } from '../data/examples'; + +const PERF_ENABLED = process.env.CI_PERF === '1'; + +/** + * Per-example median-solve-time ceilings, ms. Generous to tolerate + * CI variance; tighten after empirical baselining on a real CI box. + */ +const CEILINGS_MS: Record = { + 'an-voltage-divider': 100, + 'an-rc-low-pass': 100, + 'an-half-wave-rectifier': 2000, // .tran with sine generator → slower + 'an-bjt-switch': 200, + 'digital-and-two-switches': 100, + 'digital-not-inverter': 100, +}; + +function findExample(id: string): ExampleProject | undefined { + return [...analogExamples, ...digitalExamples].find((ex) => ex.id === id); +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2) return sorted[mid]!; + return ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2; +} + +describe.skipIf(!PERF_ENABLED)('Solver performance baseline (CI_PERF=1)', () => { + for (const [id, ceiling] of Object.entries(CEILINGS_MS)) { + const example = findExample(id); + if (!example) { + it.skip(`${id} (not found — ceiling list out of sync)`, () => {}); + continue; + } + it(`${id} — median solve ms ≤ ${ceiling}`, { timeout: 60_000 }, async () => { + const input = exampleToBuildNetlistInput(example); + const samples: number[] = []; + for (let i = 0; i < 10; i++) { + const r = await solveInput(input); + samples.push(r.solveMs); + } + const med = median(samples); + const all = samples.map((s) => s.toFixed(1)).join(', '); + expect( + med, + `median=${med.toFixed(1)} ms exceeds ${ceiling} ms ceiling for ${id}. Samples: [${all}]`, + ).toBeLessThanOrEqual(ceiling); + }); + } +}); + +// When the suite is disabled (default), expose a trivial it() so the +// file still registers in test discovery and a CI dashboard can show +// "skipped" instead of an empty file. +describe.skipIf(PERF_ENABLED)('Solver performance baseline (disabled — set CI_PERF=1 to enable)', () => { + it('placeholder', () => { + expect(PERF_ENABLED).toBe(false); + }); +});