diff --git a/backend/app/services/esp32_flash_image.py b/backend/app/services/esp32_flash_image.py new file mode 100644 index 00000000..24659521 --- /dev/null +++ b/backend/app/services/esp32_flash_image.py @@ -0,0 +1,44 @@ +"""Helpers for ESP32 flash images shared between the in-process bridge and +the subprocess worker. + +The ESP-IDF compiler in `espidf_compiler.py` builds a full 4 MB merged flash +image and then **trims the trailing 0xFF padding** before serializing the +binary into the JSON compile response (issue #101 — sending the full 4 MB +gave a ~5.5 MB base64 string that nginx / Cloudflare would buffer-fail with +"No response from server"). The frontend stores the trimmed bytes; this +module re-pads them on the receiving side just before QEMU attaches the +image as an MTD drive — QEMU rejects flash sizes that aren't a power-of-2 +megabyte (2 / 4 / 8 / 16 MB). + +This is a lossless round trip: bytes after `last_used` in the compiler's +merge are 0xFF by construction, so trim → pad reproduces the original +image byte-for-byte. +""" + +from __future__ import annotations + +# Sizes QEMU's esp32-picsimlab MTD layer accepts. ESP32 / S3 / C3 builds +# all default to 4 MB (CONFIG_ESPTOOLPY_FLASHSIZE_4MB). We only ever +# round UP — never down. +_VALID_FLASH_SIZES = [s * 1024 * 1024 for s in (2, 4, 8, 16)] +_MIN_FLASH_BYTES = 4 * 1024 * 1024 + + +def pad_to_flash_size(fw_bytes: bytes) -> bytes: + """Pad `fw_bytes` with 0xFF up to the next valid QEMU flash size. + + Returns the input unchanged when it's already at or above a valid size. + Raises `ValueError` if the firmware exceeds the largest size we accept + (16 MB) — at that point something is wrong with the upstream merge. + """ + target = next( + (s for s in _VALID_FLASH_SIZES if s >= max(len(fw_bytes), _MIN_FLASH_BYTES)), + None, + ) + if target is None: + raise ValueError( + f'ESP32 firmware too large for QEMU: {len(fw_bytes)} bytes (max 16 MB)' + ) + if len(fw_bytes) >= target: + return fw_bytes + return fw_bytes + b'\xff' * (target - len(fw_bytes)) diff --git a/backend/app/services/esp32_lib_bridge.py b/backend/app/services/esp32_lib_bridge.py index 73a96a1a..cd73365e 100644 --- a/backend/app/services/esp32_lib_bridge.py +++ b/backend/app/services/esp32_lib_bridge.py @@ -163,7 +163,11 @@ class Esp32LibBridge: def start(self, firmware_b64: str, machine: str = 'esp32-picsimlab') -> None: """Decode firmware, init QEMU, start event loop in daemon thread.""" - fw_bytes = base64.b64decode(firmware_b64) + from app.services.esp32_flash_image import pad_to_flash_size + # The compiler trims trailing 0xFF padding before serializing (issue + # #101 — full 4 MB images blew nginx buffers). Re-pad here so QEMU's + # MTD layer sees a valid power-of-2 flash size. + fw_bytes = pad_to_flash_size(base64.b64decode(firmware_b64)) tmp = tempfile.NamedTemporaryFile(suffix='.bin', delete=False) tmp.write(fw_bytes) tmp.close() diff --git a/backend/app/services/esp32_worker.py b/backend/app/services/esp32_worker.py index 918f7bec..d938f9b2 100644 --- a/backend/app/services/esp32_worker.py +++ b/backend/app/services/esp32_worker.py @@ -314,7 +314,11 @@ def main() -> None: # noqa: C901 (complexity OK for inline worker) # ── 3. Write firmware to a temp file ────────────────────────────────────── try: - fw_bytes = base64.b64decode(firmware_b64) + # The compiler trims trailing 0xFF padding before serializing (issue + # #101 — full 4 MB images blew nginx buffers). Re-pad here so QEMU's + # MTD layer sees a valid power-of-2 flash size. + from app.services.esp32_flash_image import pad_to_flash_size + fw_bytes = pad_to_flash_size(base64.b64decode(firmware_b64)) tmp = tempfile.NamedTemporaryFile(suffix='.bin', delete=False) tmp.write(fw_bytes) tmp.close() diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index af104e17..0633fe31 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -771,6 +771,7 @@ class ESPIDFCompiler: missing = [k for k, v in files_found.items() if not v] raise FileNotFoundError(f'Missing binaries for merge: {missing}') + last_used = 0 for offset, path in [ (bootloader_offset, bootloader), (0x8000, partitions), @@ -778,11 +779,24 @@ class ESPIDFCompiler: ]: data = path.read_bytes() flash[offset:offset + len(data)] = data + last_used = max(last_used, offset + len(data)) logger.info(f'[espidf] Placed {path.name} at 0x{offset:04X} ({len(data)} bytes)') + # Trim the trailing 0xFF padding before serializing. + # + # Keeping the full 4 MB flash image here gives a ~5.5 MB base64 JSON + # response that nginx / Cloudflare can choke on (issue #101 — user + # saw "No response from server"). The frontend stores the trimmed + # bytes and the backend pads back to the QEMU flash size at the + # bridge layer right before mtd attach. Lossless: bytes after + # last_used are 0xFF by construction, so re-padding restores the + # original image byte-for-byte. merged_path = build_dir / 'merged_flash.bin' - merged_path.write_bytes(bytes(flash)) - logger.info(f'[espidf] Merged flash image: {merged_path.stat().st_size} bytes') + merged_path.write_bytes(bytes(flash[:last_used])) + logger.info( + f'[espidf] Merged flash image (trimmed): {merged_path.stat().st_size} bytes ' + f'(would have been {FLASH_SIZE} bytes unpadded)' + ) return merged_path async def compile(self, files: list[dict], board_fqbn: str) -> dict: