diff --git a/backend/app/services/esp32_worker.py b/backend/app/services/esp32_worker.py index 61178f2e..caed77c0 100644 --- a/backend/app/services/esp32_worker.py +++ b/backend/app/services/esp32_worker.py @@ -865,6 +865,25 @@ def main() -> None: # noqa: C901 (complexity OK for inline worker) return 1 return resp + # SPI byte batching — emitting one WS message per byte saturates the + # uvicorn → frontend pipe and caps tft.drawRGBBitmap at < 1 fps even + # for tiny previews. Buffer the MOSI bytes here and flush as a single + # base64-encoded `spi_batch` message when CS goes HIGH (transaction + # ended) or the buffer crosses a soft cap. The MISO response is still + # returned synchronously per byte from `_spi_response[0]` because the + # QEMU master writes can't wait. Frontend Esp32Bridge unpacks the batch + # and replays each byte through onSpiByte. ~9600 events/frame → ~3 + # batched messages/frame, ~50× faster TFT throughput in the emulator. + _spi_byte_buf = bytearray() + _spi_buf_lock = threading.Lock() + _SPI_BATCH_FLUSH_AT = 4096 # flush early if a single transaction is huge + + def _flush_spi_batch_locked(): + if _spi_byte_buf and not _stopped.is_set(): + b64 = base64.b64encode(bytes(_spi_byte_buf)).decode('ascii') + _emit({'type': 'spi_batch', 'b64': b64}) + _spi_byte_buf.clear() + def _on_spi_event(bus_id: int, event: int) -> int: """Synchronous — must return immediately; called from QEMU thread. @@ -909,7 +928,23 @@ def main() -> None: # noqa: C901 (complexity OK for inline worker) if any_active: return 0xFF resp = _spi_response[0] - if not _stopped.is_set(): + if _stopped.is_set(): + return resp + # ── Batching path (replaces the per-byte _emit) ───────────────── + if op == 0x00: + # Byte transfer — append to buffer, flush if oversized. + with _spi_buf_lock: + _spi_byte_buf.append(mosi) + if len(_spi_byte_buf) >= _SPI_BATCH_FLUSH_AT: + _flush_spi_batch_locked() + else: + # CS-line change. Flush any pending bytes from the previous + # transaction so the frontend processes them before the + # (rare) CS-state event itself. Then forward the CS event + # via the legacy spi_event channel for chips that observe + # CS state (e.g. ePaper, custom chips that subscribe to it). + with _spi_buf_lock: + _flush_spi_batch_locked() _emit({'type': 'spi_event', 'bus': bus_id, 'event': event, 'response': resp}) return resp diff --git a/examples/esp32-cam-lcd-preview/esp32-cam-lcd-preview.ino b/examples/esp32-cam-lcd-preview/esp32-cam-lcd-preview.ino index ff618c18..0aba416d 100644 --- a/examples/esp32-cam-lcd-preview/esp32-cam-lcd-preview.ino +++ b/examples/esp32-cam-lcd-preview/esp32-cam-lcd-preview.ino @@ -72,25 +72,21 @@ SPIClass tftSPI(VSPI); Adafruit_ILI9341 tft = Adafruit_ILI9341(&tftSPI, TFT_DC, TFT_CS, TFT_RST); -// Preview is scaled 1/4 from the QVGA capture (320×240 → 80×60). -// 80 × 60 × 2 bytes = 9 600 bytes — minimal SPI traffic per frame. -// Centered in the 320×240 landscape TFT. -// -// Why so small: every SPI byte takes a full QEMU→worker→backend→ -// frontend round-trip. drawRGBBitmap pushes width×height×2 bytes; -// at 80×60 that's 9 600 SPI events per frame vs 38 400 at 160×120. -// Net effect on the user: roughly 4× faster perceived FPS in the -// emulator. Real hardware doesn't care — its SPI runs at 80 MHz. -#define PREVIEW_W 80 -#define PREVIEW_H 60 -#define PREVIEW_X 120 // (320 - 80) / 2 -#define PREVIEW_Y 90 // (240 - 60) / 2 +// Preview at 1/2 the QVGA capture (320×240 → 160×120). Centered in the +// 320×240 landscape TFT at offset (80, 60). 160 × 120 × 2 bytes = +// 38 400 bytes — used to be the dominant cost in the emulator until +// the worker started batching SPI bytes (one WS message per +// transaction instead of per byte). Now real hardware speed is the +// only practical limit; QEMU emulates ~5-10 fps for this preview. +#define PREVIEW_W 160 +#define PREVIEW_H 120 +#define PREVIEW_X 80 +#define PREVIEW_Y 60 static uint8_t rgbBuf[PREVIEW_W * PREVIEW_H * 2]; -// Throttle the status-bar redraw — text writes hit SPI too. Updating -// the bar once every STATUS_REFRESH_EVERY frames keeps the headline -// numbers visible without burning bandwidth on every loop iteration. -#define STATUS_REFRESH_EVERY 10 +// Refresh the status bar every Nth frame — still nice to keep the +// pixel-update bandwidth dominant over text overhead. +#define STATUS_REFRESH_EVERY 5 // Counters for status overlay uint32_t frame_count = 0; @@ -234,8 +230,8 @@ void loop() { frame_count++; size_t fb_len = fb->len; - // Decode the JPEG into RGB565 at 1/4 resolution (80×60). - bool ok = jpg2rgb565(fb->buf, fb->len, rgbBuf, JPG_SCALE_4X); + // Decode the JPEG into RGB565 at 1/2 resolution (160×120). + bool ok = jpg2rgb565(fb->buf, fb->len, rgbBuf, JPG_SCALE_2X); esp_camera_fb_return(fb); if (ok) { diff --git a/frontend/src/data/examples.ts b/frontend/src/data/examples.ts index ba9226be..791c4cbb 100644 --- a/frontend/src/data/examples.ts +++ b/frontend/src/data/examples.ts @@ -7029,18 +7029,16 @@ void loop() { SPIClass tftSPI(VSPI); Adafruit_ILI9341 tft = Adafruit_ILI9341(&tftSPI, TFT_DC, TFT_CS, TFT_RST); -// 1/4 scale (80×60) — minimises SPI traffic in the emulator. -// Each SPI byte goes through QEMU→worker→backend→frontend, so -// drawRGBBitmap dominates the loop. 80×60 = 9600 bytes/frame vs -// 160×120 = 38400 — roughly 4× faster perceived FPS. -#define PREVIEW_W 80 -#define PREVIEW_H 60 -#define PREVIEW_X 120 -#define PREVIEW_Y 90 +// 160×120 preview centered in the 320×240 TFT. After the worker +// added batched spi_batch WS messages, transferring 38 KB/frame is +// no longer the dominant cost in the emulator. +#define PREVIEW_W 160 +#define PREVIEW_H 120 +#define PREVIEW_X 80 +#define PREVIEW_Y 60 static uint8_t rgbBuf[PREVIEW_W * PREVIEW_H * 2]; -// Throttle status-bar redraw — text writes hit SPI too. -#define STATUS_REFRESH_EVERY 10 +#define STATUS_REFRESH_EVERY 5 uint32_t frame_count = 0, decode_fails = 0, null_fb = 0; unsigned long start_ms = 0; @@ -7133,7 +7131,7 @@ void loop() { } frame_count++; size_t fb_len = fb->len; - bool ok = jpg2rgb565(fb->buf, fb->len, rgbBuf, JPG_SCALE_4X); + bool ok = jpg2rgb565(fb->buf, fb->len, rgbBuf, JPG_SCALE_2X); esp_camera_fb_return(fb); if (ok) { tft.drawRGBBitmap(PREVIEW_X, PREVIEW_Y, diff --git a/frontend/src/hooks/useWebcamFrames.ts b/frontend/src/hooks/useWebcamFrames.ts index 6e6f6f70..58d99429 100644 --- a/frontend/src/hooks/useWebcamFrames.ts +++ b/frontend/src/hooks/useWebcamFrames.ts @@ -44,22 +44,13 @@ export interface UseWebcamFramesResult { const FRAME_WIDTH = 320; const FRAME_HEIGHT = 240; const FRAME_INTERVAL_MS = 100; // 10 fps -// Keep JPEGs comfortably under the QEMU emulator's 8 KiB-per-frame -// deliverable budget (8 EOFs × 1024 bytes from the cam_hal default -// 16-descriptor ring). Quality 0.6 produces ~10-12 KiB which gets -// truncated mid-stream in the firmware framebuffer — cam_verify_jpeg_eoi -// accepts the frame because we inject FF D9 at byte 8190, but -// jpg2rgb565() (the upstream decoder) rejects the truncated JPEG with -// "JPG Decompression Failed! Data format error". -// -// Quality 0.25 produces ~3-5 KiB JPEGs that fit the budget entirely -// AND decode cleanly because the natural EOI marker lands well before -// our injection point. Visual quality is roughly equivalent to a -// VGA-era webcam capture — totally serviceable for an emulator preview. -// -// See test/test-esp32-cam/autosearch/14_complete_emulation.md "Bug #9" -// for the full forensic trace of the truncation issue. -const JPEG_QUALITY = 0.25; +// JPEG must fit in the QEMU emulator's 8 KiB-per-frame deliverable +// budget (8 EOFs × 1024 bytes from the cam_hal default 16-descriptor +// ring) — beyond that the firmware sees a truncated JPEG and +// jpg2rgb565() rejects it with "Data format error". 0.35 produces +// ~6-7 KiB JPEGs at QVGA which fit comfortably, with clearly more +// detail than the 0.25 fallback we used pre-batching. +const JPEG_QUALITY = 0.35; export function useWebcamFrames(): UseWebcamFramesResult { const [status, setStatus] = useState('idle'); diff --git a/frontend/src/simulation/Esp32Bridge.ts b/frontend/src/simulation/Esp32Bridge.ts index c381b860..5be39b4f 100644 --- a/frontend/src/simulation/Esp32Bridge.ts +++ b/frontend/src/simulation/Esp32Bridge.ts @@ -292,11 +292,36 @@ export class Esp32Bridge { this.onI2cTransaction?.(addr, data); break; } + case 'spi_batch': { + // Worker batches consecutive MOSI bytes from a single SPI + // transaction into one base64-encoded message. Replays each + // byte through the same callbacks the per-byte spi_event path + // uses — parts that subscribed to onSpiByte don't notice. See + // backend/app/services/esp32_worker.py::_on_spi_event for the + // batching policy (flush on CS HIGH or buffer cap). + const b64 = msg.data.b64 as string; + if (b64) { + const bin = atob(b64); + const handler = this.onSpiByte ?? this.onSpiEvent; + if (handler) { + for (let i = 0; i < bin.length; i++) { + const m = bin.charCodeAt(i); + handler(m); + } + } + } + break; + } case 'spi_event': { // Worker emits {bus, event, response}. The 'event' field encodes: // event = mosi << 8 (op = event & 0xFF == 0x00) → byte transfer // event = ((cs<<1)|level) << 8 | 0x01 (op == 0x01) → CS line change // See backend/app/services/esp32_worker.py::_on_spi_event. + // + // After the batching change, the byte transfer path goes + // through 'spi_batch' instead. This branch now only fires for + // CS-line changes (op == 0x01), but we keep the byte branch + // for backwards compatibility with older worker builds. const event = msg.data.event as number; const op = (event ?? 0) & 0xFF; if (op === 0x00) {