fix(esp32): trim flash image before serializing, pad on QEMU attach

Issue #101 reproducer: an ESP32 sketch that pulls in Adafruit_SSD1306 +
Adafruit_GFX produced a "No response from server. Is the backend
running on port 8001?" error in the browser. The compile actually
succeeded backend-side, but the JSON response carrying the firmware
was ~5.5 MB of base64 — the ESP-IDF compiler builds a full 4 MB merged
flash image (mostly 0xFF padding), encodes it whole, and ships it. In
prod that response goes through nginx + Cloudflare, which buffer-fail
or RST the connection on payloads that big — axios then lands in the
"no response" branch with no HTTP status to surface.

Fix: trim the trailing 0xFF padding before serializing, re-pad to a
valid QEMU flash size (2/4/8/16 MB) just before mtd attach. Lossless:
bytes after `last_used` in the merge are 0xFF by construction, so
trim → pad reproduces the original image byte-for-byte.

Numbers from the reproducer (Adafruit_SSD1306 + Adafruit_GFX,
esp32:esp32:esp32 board):
  before: ~5.5 MB JSON response
  after:  539 KB JSON response (10× smaller)

backend/app/services/espidf_compiler.py
  _merge_flash_image now tracks `last_used` across the three placed
  sections (bootloader / partitions / app) and writes only
  flash[:last_used] to merged_flash.bin.

backend/app/services/esp32_flash_image.py (new)
  Shared `pad_to_flash_size(bytes) -> bytes` helper. Rounds up to the
  next valid QEMU flash size with a 4 MB minimum, matches the
  frontend's existing padToFlashSize logic in Esp32MicroPythonLoader.
  Raises ValueError on >16 MB inputs (would indicate a broken upstream
  merge, not anything user-recoverable).

backend/app/services/esp32_lib_bridge.py
backend/app/services/esp32_worker.py
  Both QEMU consumer paths (in-process and subprocess) call
  pad_to_flash_size right after base64.b64decode, before writing the
  tmp .bin that QEMU attaches with `-drive if=mtd,format=raw`.

Verified: smoke test confirms trim → pad → original is byte-exact.
Edge cases covered: small payloads pad up to the 4 MB minimum;
firmwares >16 MB are rejected loudly.

Closes #101

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/app/services/esp32_flash_image.py

@@ -0,0 +1,44 @@
+"""Helpers for ESP32 flash images shared between the in-process bridge and
+the subprocess worker.
+
+The ESP-IDF compiler in `espidf_compiler.py` builds a full 4 MB merged flash
+image and then **trims the trailing 0xFF padding** before serializing the
+binary into the JSON compile response (issue #101 — sending the full 4 MB
+gave a ~5.5 MB base64 string that nginx / Cloudflare would buffer-fail with
+"No response from server"). The frontend stores the trimmed bytes; this
+module re-pads them on the receiving side just before QEMU attaches the
+image as an MTD drive — QEMU rejects flash sizes that aren't a power-of-2
+megabyte (2 / 4 / 8 / 16 MB).
+
+This is a lossless round trip: bytes after `last_used` in the compiler's
+merge are 0xFF by construction, so trim → pad reproduces the original
+image byte-for-byte.
+"""
+
+from __future__ import annotations
+
+# Sizes QEMU's esp32-picsimlab MTD layer accepts. ESP32 / S3 / C3 builds
+# all default to 4 MB (CONFIG_ESPTOOLPY_FLASHSIZE_4MB). We only ever
+# round UP — never down.
+_VALID_FLASH_SIZES = [s * 1024 * 1024 for s in (2, 4, 8, 16)]
+_MIN_FLASH_BYTES = 4 * 1024 * 1024
+
+
+def pad_to_flash_size(fw_bytes: bytes) -> bytes:
+    """Pad `fw_bytes` with 0xFF up to the next valid QEMU flash size.
+
+    Returns the input unchanged when it's already at or above a valid size.
+    Raises `ValueError` if the firmware exceeds the largest size we accept
+    (16 MB) — at that point something is wrong with the upstream merge.
+    """
+    target = next(
+        (s for s in _VALID_FLASH_SIZES if s >= max(len(fw_bytes), _MIN_FLASH_BYTES)),
+        None,
+    )
+    if target is None:
+        raise ValueError(
+            f'ESP32 firmware too large for QEMU: {len(fw_bytes)} bytes (max 16 MB)'
+        )
+    if len(fw_bytes) >= target:
+        return fw_bytes
+    return fw_bytes + b'\xff' * (target - len(fw_bytes))

backend/app/services/esp32_lib_bridge.py

@@ -163,7 +163,11 @@ class Esp32LibBridge:
 
     def start(self, firmware_b64: str, machine: str = 'esp32-picsimlab') -> None:
         """Decode firmware, init QEMU, start event loop in daemon thread."""
-        fw_bytes = base64.b64decode(firmware_b64)
+        from app.services.esp32_flash_image import pad_to_flash_size
+        # The compiler trims trailing 0xFF padding before serializing (issue
+        # #101 — full 4 MB images blew nginx buffers). Re-pad here so QEMU's
+        # MTD layer sees a valid power-of-2 flash size.
+        fw_bytes = pad_to_flash_size(base64.b64decode(firmware_b64))
         tmp = tempfile.NamedTemporaryFile(suffix='.bin', delete=False)
         tmp.write(fw_bytes)
         tmp.close()

backend/app/services/esp32_worker.py

@@ -314,7 +314,11 @@ def main() -> None:  # noqa: C901  (complexity OK for inline worker)
 
     # ── 3. Write firmware to a temp file ──────────────────────────────────────
     try:
-        fw_bytes = base64.b64decode(firmware_b64)
+        # The compiler trims trailing 0xFF padding before serializing (issue
+        # #101 — full 4 MB images blew nginx buffers). Re-pad here so QEMU's
+        # MTD layer sees a valid power-of-2 flash size.
+        from app.services.esp32_flash_image import pad_to_flash_size
+        fw_bytes = pad_to_flash_size(base64.b64decode(firmware_b64))
         tmp = tempfile.NamedTemporaryFile(suffix='.bin', delete=False)
         tmp.write(fw_bytes)
         tmp.close()

backend/app/services/espidf_compiler.py

@@ -771,6 +771,7 @@ class ESPIDFCompiler:
             missing = [k for k, v in files_found.items() if not v]
             raise FileNotFoundError(f'Missing binaries for merge: {missing}')
 
+        last_used = 0
         for offset, path in [
             (bootloader_offset, bootloader),
             (0x8000, partitions),
@@ -778,11 +779,24 @@ class ESPIDFCompiler:
         ]:
             data = path.read_bytes()
             flash[offset:offset + len(data)] = data
+            last_used = max(last_used, offset + len(data))
             logger.info(f'[espidf] Placed {path.name} at 0x{offset:04X} ({len(data)} bytes)')
 
+        # Trim the trailing 0xFF padding before serializing.
+        #
+        # Keeping the full 4 MB flash image here gives a ~5.5 MB base64 JSON
+        # response that nginx / Cloudflare can choke on (issue #101 — user
+        # saw "No response from server"). The frontend stores the trimmed
+        # bytes and the backend pads back to the QEMU flash size at the
+        # bridge layer right before mtd attach. Lossless: bytes after
+        # last_used are 0xFF by construction, so re-padding restores the
+        # original image byte-for-byte.
         merged_path = build_dir / 'merged_flash.bin'
-        merged_path.write_bytes(bytes(flash))
-        logger.info(f'[espidf] Merged flash image: {merged_path.stat().st_size} bytes')
+        merged_path.write_bytes(bytes(flash[:last_used]))
+        logger.info(
+            f'[espidf] Merged flash image (trimmed): {merged_path.stat().st_size} bytes '
+            f'(would have been {FLASH_SIZE} bytes unpadded)'
+        )
         return merged_path
 
     async def compile(self, files: list[dict], board_fqbn: str) -> dict: