From c2fe1af250d488dcb9fd06dd0ef309ebbe2014c3 Mon Sep 17 00:00:00 2001 From: davidmonterocrespo24 Date: Sat, 9 May 2026 08:33:11 +0200 Subject: [PATCH] perf(compile): dedup, concurrency limits, and persistent build dir for ESP-IDF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three coordinated fixes that together close the "ESP-IDF compile takes 5-7 min every time" gap and prevent the failure mode where a user clicking compile multiple times spawns six ninja processes that peel each other apart on a modest VPS. What was wrong - /compile/start generated a fresh uuid4 every call, so 6 clicks = 6 independent builds racing each other. Saw load average 30 on the prod VPS during a real BMP280 attempt today. - No concurrency limit anywhere; asyncio.create_task() fired without gating. - ccache was wired in last week (PR #149) but reported 18,350 cacheable calls and **0 hits** because the build dir was a fresh tempfile.TemporaryDirectory(prefix='espidf_') per compile. The random /tmp/espidf_/ path baked into -I and -fmacro-prefix-map flags → different command line every compile → ccache hash miss every time. What this PR does 1. Job deduplication (`backend/app/api/routes/compile.py`) - New `_job_key(files, board_fqbn)` returns SHA-256 of normalised file names + contents + board. Order-independent. - New `JOB_BY_KEY: dict[str, str]` indexes hash → job_id. - `compile_start` checks JOB_BY_KEY before spawning a new task; if a job for this exact content is already pending or running, returns the existing job_id (logs `[compile] dedup hit — reusing job `). - `_purge_expired_jobs` evicts both COMPILE_JOBS and JOB_BY_KEY, keeping the index consistent. Edge case where two jobs share a key (old finished, new running) is handled — only evict the key entry if it still points at the purged job. 2. Concurrency control (`backend/app/api/routes/compile.py`) - `_COMPILE_SEMAPHORE = asyncio.Semaphore(2)` global cap on simultaneous compiles. - `_target_lock(board_fqbn)` returns a per-target asyncio.Lock so concurrent compiles to the SAME board (sharing the persistent build dir) serialise. Different boards still run in parallel up to the semaphore cap. - `_compile_job` acquires sema → per-target lock → flips state to `running` → calls `_run_compile`. Pending state now accurately reflects "queued waiting for resources". 3. Persistent build dir (`backend/app/services/espidf_compiler.py`) - New `_prepare_persistent_project_dir(idf_target)` materialises `/var/lib/velxio-build//project/` from the template on first use; on subsequent compiles it wipes only `main/` and `user_libs/` (the per-compile parts) and leaves `build/` alone so ninja's incremental cache + ccache .o files survive. - Toolchain version sentinel (`.idf_version`) wipes the whole target dir if the ESP-IDF or arduino-esp32 version changes — cached objects from the old toolchain are no longer ABI-compatible. - `compile()` is now a thin dispatcher: persistent path or fallback to the legacy `tempfile.TemporaryDirectory()` flow. The actual build logic was extracted into `_compile_in_dir()` so both paths share one implementation, no duplication. - Escape hatch: `VELXIO_PERSISTENT_BUILD_DIR=0` env var falls back to the tempfile path without rebuilding the image. Critical for production safety. 4. ccache normalisation (`Dockerfile.standalone`) - + `ENV CCACHE_BASEDIR=/var/lib/velxio-build` makes ccache canonicalise absolute paths under that prefix when computing the cache key. Robustens hits against any future subdir rearrangement. 5. Docker compose (`docker-compose.yml`) - + named volume `velxio-build:/var/lib/velxio-build` so the persistent build dir survives `docker compose up -d --build`. - + env `VELXIO_PERSISTENT_BUILD_DIR=1` (default ON; users disable without rebuilding). Expected impact - Cold first compile per container per target: unchanged (~5-7 min). - Same sketch re-compiled: ~2-5 s (everything cached). - Different sketch, same target: ~5-30 s (only user code + new lib steps rebuild; ESP-IDF base hits cache). - Different sketch with new libraries: ~30-90 s (new lib component compiles; rest hits cache). - Concurrent clicks on same example: 1 build, others poll the same job_id. No more six-ninja meltdown. Tests - `test/backend/unit/test_compile_dedup.py` covers `_job_key` stability + variance and `_purge_expired_jobs` consistency (including the "two jobs share a key" edge case). Co-Authored-By: Claude Opus 4.7 (1M context) --- Dockerfile.standalone | 6 + backend/app/api/routes/compile.py | 99 +++- backend/app/services/espidf_compiler.py | 659 ++++++++++++++---------- docker-compose.yml | 8 + test/backend/unit/test_compile_dedup.py | 141 +++++ 5 files changed, 632 insertions(+), 281 deletions(-) create mode 100644 test/backend/unit/test_compile_dedup.py diff --git a/Dockerfile.standalone b/Dockerfile.standalone index e33a2cc3..991e3013 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -185,6 +185,12 @@ ENV ARDUINO_ESP32_PATH=/opt/arduino-esp32 # image rebuild — still better than no cache. ENV CCACHE_DIR=/var/cache/ccache ENV IDF_CCACHE_ENABLE=1 +# CCACHE_BASEDIR makes ccache treat absolute paths under this prefix as +# relative when computing the cache key. Combined with the persistent +# /var/lib/velxio-build// build dir we mount as a volume, this lets +# ccache hit across compiles even though some flags (-I, -fmacro-prefix-map) +# embed absolute paths into the command line. +ENV CCACHE_BASEDIR=/var/lib/velxio-build RUN mkdir -p /var/cache/ccache \ && ccache --max-size=2G \ && ccache --set-config=compression=true \ diff --git a/backend/app/api/routes/compile.py b/backend/app/api/routes/compile.py index 0e0bd112..acc24b4a 100644 --- a/backend/app/api/routes/compile.py +++ b/backend/app/api/routes/compile.py @@ -1,4 +1,5 @@ import asyncio +import hashlib import logging import time import uuid @@ -29,11 +30,53 @@ arduino_cli = ArduinoCLIService() # Single-instance only: if velxio ever scales to multiple FastAPI workers, this # needs to move to Redis or the sqlite database. For now one process is fine. COMPILE_JOBS: dict[str, dict[str, Any]] = {} +JOB_BY_KEY: dict[str, str] = {} # content_hash → job_id, for deduplication JOB_TTL_S = 1800 # purge results 30 min after completion +# ── Concurrency control ────────────────────────────────────────────────────── +# Cap simultaneous ESP-IDF compiles. The VPS is modest (saw load avg 30 with +# 6 ninja processes peeling each other apart). Two parallel compiles to +# different targets are fine; concurrent compiles to the SAME target would +# corrupt the persistent build dir, so we serialize those with a per-target +# lock layered on top. +_COMPILE_SEMAPHORE = asyncio.Semaphore(2) +_TARGET_LOCKS: dict[str, asyncio.Lock] = {} + + +def _target_lock(board_fqbn: str) -> asyncio.Lock: + """Lazy-initialised per-target lock so concurrent compiles to the same + board serialise. Different boards still run in parallel up to the + semaphore cap.""" + lock = _TARGET_LOCKS.get(board_fqbn) + if lock is None: + lock = asyncio.Lock() + _TARGET_LOCKS[board_fqbn] = lock + return lock + + +def _job_key(files: list[dict[str, str]], board_fqbn: str) -> str: + """Stable content hash of (files, board) used as deduplication key. + + Excludes project_id (analytics-only — different projects with identical + code should still dedup to one build). File order is normalised so the + same set of files in any order produces the same key. + """ + h = hashlib.sha256() + h.update(board_fqbn.encode()) + h.update(b"\0") + for f in sorted(files, key=lambda x: x["name"]): + h.update(f["name"].encode()) + h.update(b"\0") + h.update(f["content"].encode()) + h.update(b"\0") + return h.hexdigest() + def _purge_expired_jobs() -> None: - """Drop completed jobs older than JOB_TTL_S so the dict doesn't grow forever.""" + """Drop completed jobs older than JOB_TTL_S so the dict doesn't grow + forever. Also evicts the matching JOB_BY_KEY entry so the next request + with the same content schedules a fresh build instead of dedupping to + a stale job_id.""" now = time.time() stale = [ jid for jid, job in COMPILE_JOBS.items() @@ -41,7 +84,14 @@ def _purge_expired_jobs() -> None: and now - job.get("finished_at", now) > JOB_TTL_S ] for jid in stale: - COMPILE_JOBS.pop(jid, None) + job = COMPILE_JOBS.pop(jid, None) + if job is not None: + key = job.get("key") + # Only remove the JOB_BY_KEY entry if it still points at this job — + # a newer job with the same key may have replaced it after this one + # finished but before TTL elapsed. + if key and JOB_BY_KEY.get(key) == jid: + JOB_BY_KEY.pop(key, None) class SketchFile(BaseModel): @@ -185,16 +235,33 @@ async def _compile_job( files: list[dict[str, str]], user_id: int | None, ) -> None: - """Background worker: run the compile, store result in COMPILE_JOBS.""" + """Background worker: acquire global semaphore + per-target lock, run the + compile, store result in COMPILE_JOBS. + + `state=pending` while waiting on either gate; transitions to `running` + only once the actual build is about to start, so clients polling + /compile/status see an accurate snapshot of where their job is. + """ started = time.monotonic() + job = COMPILE_JOBS[job_id] + started_at = job["started_at"] + job_key = job.get("key") try: - COMPILE_JOBS[job_id]["state"] = "running" - response = await _run_compile(request, files) + async with _COMPILE_SEMAPHORE: + async with _target_lock(request.board_fqbn): + # Job may have been purged or replaced while we were queued. + # Re-fetch and bail out if so. + if COMPILE_JOBS.get(job_id) is None: + logger.info(f"[compile] job {job_id} purged before run; skipping") + return + COMPILE_JOBS[job_id]["state"] = "running" + response = await _run_compile(request, files) COMPILE_JOBS[job_id] = { "state": "done", - "started_at": COMPILE_JOBS[job_id]["started_at"], + "started_at": started_at, "finished_at": time.time(), "result": response.model_dump(), + "key": job_key, } error_kind = ( None if response.success @@ -213,9 +280,10 @@ async def _compile_job( logger.exception(f"[compile] async job {job_id} failed") COMPILE_JOBS[job_id] = { "state": "error", - "started_at": COMPILE_JOBS[job_id]["started_at"], + "started_at": started_at, "finished_at": time.time(), "error": str(exc)[:500], + "key": job_key, } await _record_async_metric( user_id=user_id, @@ -303,12 +371,27 @@ async def compile_start( `GET /compile/status/{job_id}` every couple of seconds until state is `done` or `error`. This sidesteps Cloudflare's 100s HTTP edge timeout — each individual request returns in milliseconds. + + Deduplication: identical (files, board_fqbn) submissions while a + matching job is still pending or running return the existing job_id + instead of spawning a new build. Prevents the "user clicks compile six + times → six concurrent ninja processes peeling each other apart" + failure mode. """ files = _resolve_files(request) _purge_expired_jobs() + key = _job_key(files, request.board_fqbn) + existing_id = JOB_BY_KEY.get(key) + if existing_id is not None: + existing = COMPILE_JOBS.get(existing_id) + if existing is not None and existing.get("state") in ("pending", "running"): + logger.info(f"[compile] dedup hit — reusing job {existing_id}") + return CompileStartResponse(job_id=existing_id) + job_id = uuid.uuid4().hex - COMPILE_JOBS[job_id] = {"state": "pending", "started_at": time.time()} + COMPILE_JOBS[job_id] = {"state": "pending", "started_at": time.time(), "key": key} + JOB_BY_KEY[key] = job_id asyncio.create_task( _compile_job( diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index be992b9d..9f6b6e60 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -29,6 +29,90 @@ logger = logging.getLogger(__name__) # Location of the ESP-IDF project template (relative to this file) _TEMPLATE_DIR = Path(__file__).parent / 'esp-idf-template' +# ── Persistent build dir ───────────────────────────────────────────────────── +# Cold ESP-IDF compiles rebuild ~1480 base objects (FreeRTOS, lwIP, esp_wifi, +# libsodium, …). The default tempfile.TemporaryDirectory flow gave each compile +# a fresh path under /tmp/espidf_/, which baked into -I and +# -fmacro-prefix-map flags and made ccache 0% effective (different cwd → hash +# miss every time). With the persistent dir, /var/lib/velxio-build// +# is a stable anchor: ninja's incremental cache + ccache hits combine to bring +# warm compiles down to ~5-30s. +# +# Concurrent compiles to the SAME target would corrupt the shared build dir; +# they're serialised by the per-target asyncio.Lock in routes/compile.py. +# Different targets get different subdirs and run in parallel. +# +# Set VELXIO_PERSISTENT_BUILD_DIR=0 to fall back to the legacy tempfile flow +# without rebuilding the image (escape hatch if the persistent dir misbehaves +# in production). +_BUILD_ROOT = Path(os.environ.get('VELXIO_BUILD_ROOT', '/var/lib/velxio-build')) +_USE_PERSISTENT_DIR = ( + os.environ.get('VELXIO_PERSISTENT_BUILD_DIR', '1') + not in ('0', 'false', 'False', '') +) + + +def _idf_version_signature() -> str: + """Snapshot of the ESP-IDF + arduino-esp32 toolchain version. Used to + invalidate persistent build dirs after an upstream submodule bump (the + cached object files on disk are no longer ABI-compatible).""" + parts = [] + idf_version_file = Path('/opt/esp-idf/version.txt') + if idf_version_file.exists(): + parts.append(idf_version_file.read_text(encoding='utf-8').strip()) + arduino_version_file = Path('/opt/arduino-esp32/version.txt') + if arduino_version_file.exists(): + parts.append(arduino_version_file.read_text(encoding='utf-8').strip()) + if not parts: + # Fall back to mtime of the IDF tree root — coarse but stable per + # image build. + try: + parts.append(str(int(Path('/opt/esp-idf').stat().st_mtime))) + except OSError: + parts.append('unknown') + return '|'.join(parts) + + +def _prepare_persistent_project_dir(idf_target: str) -> Path: + """Return the path to a per-target persistent project dir, materialising + it from the template on first use and resetting the per-compile parts + (main/, user_libs/) on every call so a previous sketch's files don't + leak into the next compile. + + Keeps the `build/` directory intact across calls — that's where ninja's + incremental cache + the .o files we want ccache to hit live. + """ + target_dir = _BUILD_ROOT / idf_target + target_dir.mkdir(parents=True, exist_ok=True) + + # Wipe the whole target dir if the toolchain version changed (the cached + # .o files are no longer compatible). + sentinel = target_dir / '.idf_version' + current_signature = _idf_version_signature() + if sentinel.exists() and sentinel.read_text(encoding='utf-8').strip() != current_signature: + logger.info( + f'[espidf] toolchain version changed; wiping persistent build dir {target_dir}' + ) + shutil.rmtree(target_dir) + target_dir.mkdir(parents=True, exist_ok=True) + + project_dir = target_dir / 'project' + + if not project_dir.exists(): + # First use of this target — full template copy. + shutil.copytree(_TEMPLATE_DIR, project_dir) + else: + # Subsequent compile — reset the per-compile parts only, keep build/. + # main/ is overwritten with the user's sketch + any extra files. + # user_libs/ is rebuilt by _resolve_library_components based on the + # sketch's #includes. + shutil.rmtree(project_dir / 'main', ignore_errors=True) + shutil.copytree(_TEMPLATE_DIR / 'main', project_dir / 'main') + shutil.rmtree(project_dir / 'user_libs', ignore_errors=True) + + sentinel.write_text(current_signature, encoding='utf-8') + return project_dir + # Static IP that matches slirp DHCP range (first client = x.x.x.15) _STATIC_IP = '192.168.4.15' _GATEWAY_IP = '192.168.4.2' @@ -805,6 +889,18 @@ class ESPIDFCompiler: Returns dict compatible with ArduinoCLIService.compile(): success, binary_content (base64), binary_type, stdout, stderr, error + + Build dir layout: + - With VELXIO_PERSISTENT_BUILD_DIR=1 (default): a per-target dir at + /var/lib/velxio-build//project/ is reused across compiles. + ninja's incremental cache + ccache hits combine to bring warm + compiles down to ~5-30s. + - With VELXIO_PERSISTENT_BUILD_DIR=0: legacy tempfile.TemporaryDirectory + flow. Every compile rebuilds from scratch. + + The caller (routes/compile.py:_compile_job) holds a per-target + asyncio.Lock for the duration of this call, so the persistent dir + is never accessed by two compiles at once. """ if not self.available: return { @@ -820,300 +916,317 @@ class ESPIDFCompiler: logger.info(f'[espidf] Compiling for {idf_target} (FQBN: {board_fqbn})') logger.info(f'[espidf] Files: {[f["name"] for f in files]}') + if _USE_PERSISTENT_DIR: + project_dir = _prepare_persistent_project_dir(idf_target) + logger.info(f'[espidf] Using persistent build dir: {project_dir}') + return await self._compile_in_dir(project_dir, files, idf_target, is_c3) + with tempfile.TemporaryDirectory(prefix='espidf_') as temp_dir: project_dir = Path(temp_dir) / 'project' - - # Copy template shutil.copytree(_TEMPLATE_DIR, project_dir) + logger.info(f'[espidf] Using ephemeral build dir: {project_dir}') + return await self._compile_in_dir(project_dir, files, idf_target, is_c3) - # Get sketch content - main_content = '' - for f in files: - if f['name'].endswith('.ino'): - main_content = f['content'] - break - if not main_content and files: - main_content = files[0]['content'] + async def _compile_in_dir( + self, + project_dir: Path, + files: list[dict], + idf_target: str, + is_c3: bool, + ) -> dict: + """Inner compile body: writes sketch + libs into `project_dir`, + runs cmake + ninja, merges binaries. Caller is responsible for + creating `project_dir` (with the template tree already copied in) + and for managing its lifecycle (persistent vs tempfile). + """ + # Get sketch content + main_content = '' + for f in files: + if f['name'].endswith('.ino'): + main_content = f['content'] + break + if not main_content and files: + main_content = files[0]['content'] - # ── QEMU WiFi compatibility ────────────────────────────────────── - # QEMU's WiFi AP broadcasts "Velxio-GUEST" on channel 6. - # We normalize ANY user SSID → "Velxio-GUEST", enforce channel 6, - # and use open auth (empty password) so the connection always works. - # Detect WiFi BEFORE normalization so the flag reflects the original sketch. - has_wifi = self._detect_wifi_usage(main_content) - main_content = self._normalize_wifi_for_qemu(main_content) + # ── QEMU WiFi compatibility ────────────────────────────────────── + # QEMU's WiFi AP broadcasts "Velxio-GUEST" on channel 6. + # We normalize ANY user SSID → "Velxio-GUEST", enforce channel 6, + # and use open auth (empty password) so the connection always works. + # Detect WiFi BEFORE normalization so the flag reflects the original sketch. + has_wifi = self._detect_wifi_usage(main_content) + main_content = self._normalize_wifi_for_qemu(main_content) - if self.has_arduino: - # Arduino-as-component mode: copy sketch as .cpp - sketch_cpp = project_dir / 'main' / 'sketch.ino.cpp' - # Prepend Arduino.h + velxio_compat.h if not already included. - # velxio_compat.h shims arduino-esp32 3.x APIs (ledcAttach, …) - # onto the 2.0.17 toolchain we currently pin. See - # esp-idf-template/main/velxio_compat.h. - if '#include' not in main_content or 'Arduino.h' not in main_content: - main_content = ( - '#include "Arduino.h"\n' - '#include "velxio_compat.h"\n' + main_content - ) - else: - main_content = main_content.replace( - '#include "Arduino.h"', - '#include "Arduino.h"\n#include "velxio_compat.h"', - 1, - ) - sketch_cpp.write_text(main_content, encoding='utf-8') - - # Copy additional files (.h, .cpp) - for f in files: - if not f['name'].endswith('.ino'): - (project_dir / 'main' / f['name']).write_text( - f['content'], encoding='utf-8' - ) - - # Remove the pure-C main to avoid conflict - main_c = project_dir / 'main' / 'main.c' - if main_c.exists(): - main_c.unlink() - sketch_translated = project_dir / 'main' / 'sketch_translated.c' - if sketch_translated.exists(): - sketch_translated.unlink() - - # ── Resolve external Arduino libraries as IDF components ────── - # arduino-cli installs libraries in ~/Arduino/libraries/ but the - # ESP-IDF build system does not scan that path. We create a - # user_libs/ directory where each external library becomes a - # proper ESP-IDF component with its own CMakeLists.txt and - # INCLUDE_DIRS. The root CMakeLists.txt (template) adds user_libs - # to EXTRA_COMPONENT_DIRS so ESP-IDF discovers them automatically. - ext_headers = self._detect_external_includes(main_content) - component_names: list[str] = [] - # arduino-esp32 component name (directory basename of ARDUINO_ESP32_PATH) - arduino_comp_name = Path(self.arduino_path).name if self.arduino_path else 'arduino-esp32' - - if ext_headers: - user_libs_dir = project_dir / 'user_libs' - user_libs_dir.mkdir(exist_ok=True) - - esp32_libs = Path(self.arduino_path) / 'libraries' if self.arduino_path else None - arduino_libs = self._find_arduino_libraries_dir() - - component_names, _ = self._resolve_library_components( - ext_headers, arduino_libs, esp32_libs, - arduino_comp_name, user_libs_dir, - ) - - # Patch main/CMakeLists.txt — REQUIRES and INCLUDE_DIRS for user_libs_all. - # The single merged component means one entry covers all external headers. - if component_names: # always ['user_libs_all'] when any lib was found - cmake_path = project_dir / 'main' / 'CMakeLists.txt' - cmake_text = cmake_path.read_text(encoding='utf-8') - - for old_req in [r'REQUIRES ${_arduino_comp_name}', f'REQUIRES {arduino_comp_name}']: - if old_req in cmake_text: - cmake_text = cmake_text.replace( - old_req, f'{old_req} user_libs_all' - ) - break - - cmake_text = cmake_text.replace( - 'INCLUDE_DIRS "."', - 'INCLUDE_DIRS "." "../user_libs/user_libs_all"', - ) - - cmake_path.write_text(cmake_text, encoding='utf-8') - logger.info('[espidf] Patched main CMakeLists: REQUIRES += user_libs_all, INCLUDE_DIRS += user_libs_all') + if self.has_arduino: + # Arduino-as-component mode: copy sketch as .cpp + sketch_cpp = project_dir / 'main' / 'sketch.ino.cpp' + # Prepend Arduino.h + velxio_compat.h if not already included. + # velxio_compat.h shims arduino-esp32 3.x APIs (ledcAttach, …) + # onto the 2.0.17 toolchain we currently pin. See + # esp-idf-template/main/velxio_compat.h. + if '#include' not in main_content or 'Arduino.h' not in main_content: + main_content = ( + '#include "Arduino.h"\n' + '#include "velxio_compat.h"\n' + main_content + ) else: - # Pure ESP-IDF mode: translate sketch - translated = self._translate_sketch_to_espidf(main_content) - (project_dir / 'main' / 'sketch_translated.c').write_text( - translated, encoding='utf-8' + main_content = main_content.replace( + '#include "Arduino.h"', + '#include "Arduino.h"\n#include "velxio_compat.h"', + 1, + ) + sketch_cpp.write_text(main_content, encoding='utf-8') + + # Copy additional files (.h, .cpp) + for f in files: + if not f['name'].endswith('.ino'): + (project_dir / 'main' / f['name']).write_text( + f['content'], encoding='utf-8' + ) + + # Remove the pure-C main to avoid conflict + main_c = project_dir / 'main' / 'main.c' + if main_c.exists(): + main_c.unlink() + sketch_translated = project_dir / 'main' / 'sketch_translated.c' + if sketch_translated.exists(): + sketch_translated.unlink() + + # ── Resolve external Arduino libraries as IDF components ────── + # arduino-cli installs libraries in ~/Arduino/libraries/ but the + # ESP-IDF build system does not scan that path. We create a + # user_libs/ directory where each external library becomes a + # proper ESP-IDF component with its own CMakeLists.txt and + # INCLUDE_DIRS. The root CMakeLists.txt (template) adds user_libs + # to EXTRA_COMPONENT_DIRS so ESP-IDF discovers them automatically. + ext_headers = self._detect_external_includes(main_content) + component_names: list[str] = [] + # arduino-esp32 component name (directory basename of ARDUINO_ESP32_PATH) + arduino_comp_name = Path(self.arduino_path).name if self.arduino_path else 'arduino-esp32' + + if ext_headers: + user_libs_dir = project_dir / 'user_libs' + user_libs_dir.mkdir(exist_ok=True) + + esp32_libs = Path(self.arduino_path) / 'libraries' if self.arduino_path else None + arduino_libs = self._find_arduino_libraries_dir() + + component_names, _ = self._resolve_library_components( + ext_headers, arduino_libs, esp32_libs, + arduino_comp_name, user_libs_dir, ) - # Remove Arduino main.cpp to avoid conflict - main_cpp = project_dir / 'main' / 'main.cpp' - if main_cpp.exists(): - main_cpp.unlink() + # Patch main/CMakeLists.txt — REQUIRES and INCLUDE_DIRS for user_libs_all. + # The single merged component means one entry covers all external headers. + if component_names: # always ['user_libs_all'] when any lib was found + cmake_path = project_dir / 'main' / 'CMakeLists.txt' + cmake_text = cmake_path.read_text(encoding='utf-8') - # Build using cmake + ninja (more portable than idf.py on Windows) - build_dir = project_dir / 'build' - build_dir.mkdir(exist_ok=True) - - env = self._build_env(idf_target) - - # Step 1: cmake configure - cmake_cmd = [ - 'cmake', - '-G', 'Ninja', - '-Wno-dev', - f'-DIDF_TARGET={idf_target}', - '-DCMAKE_BUILD_TYPE=Release', - f'-DSDKCONFIG_DEFAULTS={project_dir / "sdkconfig.defaults"}', - str(project_dir), - ] - - # ccache: ESP-IDF's tools/cmake/project.cmake enables ccache iff - # the CMake variable `CCACHE_ENABLE` is truthy. We don't go through - # `idf.py` (which would translate the env var for us), so wire it - # in here. Default ON; set IDF_CCACHE_ENABLE=0 in the env to - # bypass without rebuilding the image. - if os.environ.get('IDF_CCACHE_ENABLE', '1') not in ('0', 'false', 'False', ''): - cmake_cmd.append('-DCCACHE_ENABLE=1') - - logger.info(f'[espidf] cmake: {" ".join(cmake_cmd)}') - - def _run_cmake(): - return subprocess.run( - cmake_cmd, - cwd=str(build_dir), - capture_output=True, - text=True, - env=env, - timeout=120, - ) - - try: - cmake_result = await asyncio.to_thread(_run_cmake) - except subprocess.TimeoutExpired: - return { - 'success': False, - 'error': 'ESP-IDF cmake configure timed out (120s)', - 'stdout': '', - 'stderr': '', - } - - if cmake_result.returncode != 0: - logger.error(f'[espidf] cmake failed:\n{cmake_result.stderr}') - return { - 'success': False, - 'error': 'ESP-IDF cmake configure failed', - 'stdout': cmake_result.stdout, - 'stderr': cmake_result.stderr, - } - - # Step 2: ninja build - ninja_cmd = ['ninja'] - logger.info('[espidf] Building with ninja...') - - # Cold ESP-IDF builds with external Arduino libraries (e.g. Adafruit - # BMP280 + BusIO + Unified Sensor → ~1480 build steps) regularly take - # 5-7 minutes on modest hardware. 300s used to cut them off at 98%; - # bump to 600s so first-run cold compiles complete. Subsequent - # builds reuse ninja's cache and finish in seconds. - NINJA_TIMEOUT_S = 600 - - def _run_ninja(): - return subprocess.run( - ninja_cmd, - cwd=str(build_dir), - capture_output=True, - text=True, - env=env, - timeout=NINJA_TIMEOUT_S, - ) - - try: - ninja_result = await asyncio.to_thread(_run_ninja) - except subprocess.TimeoutExpired: - return { - 'success': False, - 'error': f'ESP-IDF build timed out ({NINJA_TIMEOUT_S}s)', - 'stdout': '', - 'stderr': '', - } - - all_stdout = cmake_result.stdout + '\n' + ninja_result.stdout - all_stderr = cmake_result.stderr + '\n' + ninja_result.stderr - - # Filter out expected but ugly warnings from stderr (e.g. absent git, cmake deprecation) - filtered_stderr_lines = [] - for line in all_stderr.splitlines(): - if 'fatal: not a git repository' in line: - continue - if 'CMake Deprecation Warning' in line: - continue - if 'Compatibility with CMake' in line: - continue - filtered_stderr_lines.append(line) - all_stderr = '\n'.join(filtered_stderr_lines) - - if ninja_result.returncode != 0: - # Extract the actual compiler errors from ninja's stdout. - # Ninja prints failed job blocks in stdout: - # FAILED: path/to/file.obj - # - # sketch.ino.cpp:5:10: fatal error: DHT.h: No such file or directory - # compilation terminated. - # ninja: build stopped: subcommand failed. - stdout_lines = ninja_result.stdout.split('\n') - error_lines: list[str] = [] - in_failed_block = False - for line in stdout_lines: - stripped = line.strip() - if stripped.startswith('FAILED:') or stripped == 'ninja: build stopped: subcommand failed.': - in_failed_block = True - error_lines.append(line) - continue - # Next [N/M] progress line ends the block - if in_failed_block and stripped.startswith('[') and '/' in stripped and ']' in stripped: - in_failed_block = False - if in_failed_block: - error_lines.append(line) - elif ': error:' in line or 'fatal error:' in line.lower(): - # Explicit compiler error outside a FAILED block - error_lines.append(line) - - extracted = '\n'.join(l for l in error_lines if l.strip()) - - # First non-FAILED, non-command error line → short summary for toolbar - summary = 'ESP-IDF build failed' - for l in error_lines: - s = l.strip() - if s and not s.startswith('FAILED:') and not s.startswith('ninja:') and not s.startswith('/') and 'error:' in s.lower(): - summary = s + for old_req in [r'REQUIRES ${_arduino_comp_name}', f'REQUIRES {arduino_comp_name}']: + if old_req in cmake_text: + cmake_text = cmake_text.replace( + old_req, f'{old_req} user_libs_all' + ) break - if summary == 'ESP-IDF build failed' and error_lines: - # Fall back to first non-empty error line - for l in error_lines: - if l.strip() and not l.strip().startswith('FAILED:'): - summary = l.strip() - break - # Put extracted errors in stderr so the console highlights them - combined_stderr = (extracted + '\n\n' + all_stderr).strip() if extracted else all_stderr + cmake_text = cmake_text.replace( + 'INCLUDE_DIRS "."', + 'INCLUDE_DIRS "." "../user_libs/user_libs_all"', + ) - logger.error(f'[espidf] ninja build failed (stdout):\n{ninja_result.stdout[-4000:]}') - logger.error(f'[espidf] ninja build failed (stderr):\n{ninja_result.stderr[-2000:]}') - return { - 'success': False, - 'error': summary, - 'stdout': all_stdout, - 'stderr': combined_stderr, - } + cmake_path.write_text(cmake_text, encoding='utf-8') + logger.info('[espidf] Patched main CMakeLists: REQUIRES += user_libs_all, INCLUDE_DIRS += user_libs_all') + else: + # Pure ESP-IDF mode: translate sketch + translated = self._translate_sketch_to_espidf(main_content) + (project_dir / 'main' / 'sketch_translated.c').write_text( + translated, encoding='utf-8' + ) - # Step 3: Merge binaries into flash image - try: - merged_path = self._merge_flash_image(build_dir, is_c3) - except FileNotFoundError as exc: - return { - 'success': False, - 'error': f'Binary merge failed: {exc}', - 'stdout': all_stdout, - 'stderr': all_stderr, - } + # Remove Arduino main.cpp to avoid conflict + main_cpp = project_dir / 'main' / 'main.cpp' + if main_cpp.exists(): + main_cpp.unlink() - binary_b64 = base64.b64encode(merged_path.read_bytes()).decode('ascii') - logger.info(f'[espidf] Compilation successful — {len(binary_b64) // 1024} KB (base64), has_wifi={has_wifi}') + # Build using cmake + ninja (more portable than idf.py on Windows) + build_dir = project_dir / 'build' + build_dir.mkdir(exist_ok=True) + env = self._build_env(idf_target) + + # Step 1: cmake configure + cmake_cmd = [ + 'cmake', + '-G', 'Ninja', + '-Wno-dev', + f'-DIDF_TARGET={idf_target}', + '-DCMAKE_BUILD_TYPE=Release', + f'-DSDKCONFIG_DEFAULTS={project_dir / "sdkconfig.defaults"}', + str(project_dir), + ] + + # ccache: ESP-IDF's tools/cmake/project.cmake enables ccache iff + # the CMake variable `CCACHE_ENABLE` is truthy. We don't go through + # `idf.py` (which would translate the env var for us), so wire it + # in here. Default ON; set IDF_CCACHE_ENABLE=0 in the env to + # bypass without rebuilding the image. + if os.environ.get('IDF_CCACHE_ENABLE', '1') not in ('0', 'false', 'False', ''): + cmake_cmd.append('-DCCACHE_ENABLE=1') + + logger.info(f'[espidf] cmake: {" ".join(cmake_cmd)}') + + def _run_cmake(): + return subprocess.run( + cmake_cmd, + cwd=str(build_dir), + capture_output=True, + text=True, + env=env, + timeout=120, + ) + + try: + cmake_result = await asyncio.to_thread(_run_cmake) + except subprocess.TimeoutExpired: return { - 'success': True, - 'hex_content': None, - 'binary_content': binary_b64, - 'binary_type': 'bin', - 'has_wifi': has_wifi, + 'success': False, + 'error': 'ESP-IDF cmake configure timed out (120s)', + 'stdout': '', + 'stderr': '', + } + + if cmake_result.returncode != 0: + logger.error(f'[espidf] cmake failed:\n{cmake_result.stderr}') + return { + 'success': False, + 'error': 'ESP-IDF cmake configure failed', + 'stdout': cmake_result.stdout, + 'stderr': cmake_result.stderr, + } + + # Step 2: ninja build + ninja_cmd = ['ninja'] + logger.info('[espidf] Building with ninja...') + + # Cold ESP-IDF builds with external Arduino libraries (e.g. Adafruit + # BMP280 + BusIO + Unified Sensor → ~1480 build steps) regularly take + # 5-7 minutes on modest hardware. 300s used to cut them off at 98%; + # bump to 600s so first-run cold compiles complete. Subsequent + # builds reuse ninja's cache and finish in seconds. + NINJA_TIMEOUT_S = 600 + + def _run_ninja(): + return subprocess.run( + ninja_cmd, + cwd=str(build_dir), + capture_output=True, + text=True, + env=env, + timeout=NINJA_TIMEOUT_S, + ) + + try: + ninja_result = await asyncio.to_thread(_run_ninja) + except subprocess.TimeoutExpired: + return { + 'success': False, + 'error': f'ESP-IDF build timed out ({NINJA_TIMEOUT_S}s)', + 'stdout': '', + 'stderr': '', + } + + all_stdout = cmake_result.stdout + '\n' + ninja_result.stdout + all_stderr = cmake_result.stderr + '\n' + ninja_result.stderr + + # Filter out expected but ugly warnings from stderr (e.g. absent git, cmake deprecation) + filtered_stderr_lines = [] + for line in all_stderr.splitlines(): + if 'fatal: not a git repository' in line: + continue + if 'CMake Deprecation Warning' in line: + continue + if 'Compatibility with CMake' in line: + continue + filtered_stderr_lines.append(line) + all_stderr = '\n'.join(filtered_stderr_lines) + + if ninja_result.returncode != 0: + # Extract the actual compiler errors from ninja's stdout. + # Ninja prints failed job blocks in stdout: + # FAILED: path/to/file.obj + # + # sketch.ino.cpp:5:10: fatal error: DHT.h: No such file or directory + # compilation terminated. + # ninja: build stopped: subcommand failed. + stdout_lines = ninja_result.stdout.split('\n') + error_lines: list[str] = [] + in_failed_block = False + for line in stdout_lines: + stripped = line.strip() + if stripped.startswith('FAILED:') or stripped == 'ninja: build stopped: subcommand failed.': + in_failed_block = True + error_lines.append(line) + continue + # Next [N/M] progress line ends the block + if in_failed_block and stripped.startswith('[') and '/' in stripped and ']' in stripped: + in_failed_block = False + if in_failed_block: + error_lines.append(line) + elif ': error:' in line or 'fatal error:' in line.lower(): + # Explicit compiler error outside a FAILED block + error_lines.append(line) + + extracted = '\n'.join(l for l in error_lines if l.strip()) + + # First non-FAILED, non-command error line → short summary for toolbar + summary = 'ESP-IDF build failed' + for l in error_lines: + s = l.strip() + if s and not s.startswith('FAILED:') and not s.startswith('ninja:') and not s.startswith('/') and 'error:' in s.lower(): + summary = s + break + if summary == 'ESP-IDF build failed' and error_lines: + # Fall back to first non-empty error line + for l in error_lines: + if l.strip() and not l.strip().startswith('FAILED:'): + summary = l.strip() + break + + # Put extracted errors in stderr so the console highlights them + combined_stderr = (extracted + '\n\n' + all_stderr).strip() if extracted else all_stderr + + logger.error(f'[espidf] ninja build failed (stdout):\n{ninja_result.stdout[-4000:]}') + logger.error(f'[espidf] ninja build failed (stderr):\n{ninja_result.stderr[-2000:]}') + return { + 'success': False, + 'error': summary, + 'stdout': all_stdout, + 'stderr': combined_stderr, + } + + # Step 3: Merge binaries into flash image + try: + merged_path = self._merge_flash_image(build_dir, is_c3) + except FileNotFoundError as exc: + return { + 'success': False, + 'error': f'Binary merge failed: {exc}', 'stdout': all_stdout, 'stderr': all_stderr, } + binary_b64 = base64.b64encode(merged_path.read_bytes()).decode('ascii') + logger.info(f'[espidf] Compilation successful — {len(binary_b64) // 1024} KB (base64), has_wifi={has_wifi}') + + return { + 'success': True, + 'hex_content': None, + 'binary_content': binary_b64, + 'binary_type': 'bin', + 'has_wifi': has_wifi, + 'stdout': all_stdout, + 'stderr': all_stderr, + } + # Singleton instance espidf_compiler = ESPIDFCompiler() diff --git a/docker-compose.yml b/docker-compose.yml index 38673a6a..7191c577 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,10 +28,17 @@ services: # to disable without changing the Dockerfile. - IDF_CCACHE_ENABLE=1 - CCACHE_DIR=/var/cache/ccache + # Persistent ESP-IDF build dir (one per board target). Lets ninja's + # incremental cache + ccache hits combine across compiles, taking warm + # builds from ~5-7 min to ~5-30s. Set VELXIO_PERSISTENT_BUILD_DIR=0 + # to disable without rebuilding the image (fallback to per-compile + # tempfile.TemporaryDirectory behaviour). + - VELXIO_PERSISTENT_BUILD_DIR=1 volumes: - ./data:/app/data - arduino-libs:/root/.arduino15 - ccache:/var/cache/ccache + - velxio-build:/var/lib/velxio-build healthcheck: test: ["CMD", "curl", "-f", "http://localhost/health"] interval: 30s @@ -42,3 +49,4 @@ services: volumes: arduino-libs: ccache: + velxio-build: diff --git a/test/backend/unit/test_compile_dedup.py b/test/backend/unit/test_compile_dedup.py new file mode 100644 index 00000000..01a1a84a --- /dev/null +++ b/test/backend/unit/test_compile_dedup.py @@ -0,0 +1,141 @@ +""" +Tests for the compile-job deduplication logic in routes/compile.py. + +Covers: +- _job_key() stability across invocations + variance with content/board +- _purge_expired_jobs() cleans both COMPILE_JOBS and JOB_BY_KEY consistently + +Does NOT exercise the full FastAPI route or the ESP-IDF toolchain — those are +covered by integration tests. This file is fast (no I/O, no toolchain). + +Run from the repo root: + python -m pytest test/backend/unit/test_compile_dedup.py -v +""" + +import sys +import time +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / 'backend')) + +from app.api.routes import compile as compile_module + + +class JobKeyTests(unittest.TestCase): + def setUp(self): + # Reset module-level state before each test so they don't leak. + compile_module.COMPILE_JOBS.clear() + compile_module.JOB_BY_KEY.clear() + + def test_key_stable_across_invocations(self): + files = [{'name': 'sketch.ino', 'content': 'void setup(){}'}] + k1 = compile_module._job_key(files, 'esp32:esp32:esp32') + k2 = compile_module._job_key(files, 'esp32:esp32:esp32') + self.assertEqual(k1, k2) + + def test_key_changes_with_content(self): + k1 = compile_module._job_key( + [{'name': 'sketch.ino', 'content': 'void setup(){}'}], + 'esp32:esp32:esp32', + ) + k2 = compile_module._job_key( + [{'name': 'sketch.ino', 'content': 'void loop(){}'}], + 'esp32:esp32:esp32', + ) + self.assertNotEqual(k1, k2) + + def test_key_changes_with_filename(self): + k1 = compile_module._job_key( + [{'name': 'sketch.ino', 'content': 'X'}], + 'esp32:esp32:esp32', + ) + k2 = compile_module._job_key( + [{'name': 'other.ino', 'content': 'X'}], + 'esp32:esp32:esp32', + ) + self.assertNotEqual(k1, k2) + + def test_key_changes_with_board(self): + files = [{'name': 'sketch.ino', 'content': 'X'}] + k1 = compile_module._job_key(files, 'esp32:esp32:esp32') + k2 = compile_module._job_key(files, 'esp32:esp32:esp32c3') + self.assertNotEqual(k1, k2) + + def test_key_independent_of_file_order(self): + files_a = [ + {'name': 'a.ino', 'content': 'X'}, + {'name': 'b.h', 'content': 'Y'}, + ] + files_b = [ + {'name': 'b.h', 'content': 'Y'}, + {'name': 'a.ino', 'content': 'X'}, + ] + k1 = compile_module._job_key(files_a, 'esp32:esp32:esp32') + k2 = compile_module._job_key(files_b, 'esp32:esp32:esp32') + self.assertEqual(k1, k2) + + +class PurgeExpiredJobsTests(unittest.TestCase): + def setUp(self): + compile_module.COMPILE_JOBS.clear() + compile_module.JOB_BY_KEY.clear() + + def test_purge_drops_done_jobs_past_ttl(self): + old_finished = time.time() - compile_module.JOB_TTL_S - 10 + compile_module.COMPILE_JOBS['old-id'] = { + 'state': 'done', + 'started_at': old_finished - 60, + 'finished_at': old_finished, + 'key': 'k1', + } + compile_module.JOB_BY_KEY['k1'] = 'old-id' + + compile_module._purge_expired_jobs() + + self.assertNotIn('old-id', compile_module.COMPILE_JOBS) + self.assertNotIn('k1', compile_module.JOB_BY_KEY) + + def test_purge_keeps_running_jobs(self): + # No finished_at; state=running. Should never be purged. + compile_module.COMPILE_JOBS['running-id'] = { + 'state': 'running', + 'started_at': time.time() - 10000, + 'key': 'k2', + } + compile_module.JOB_BY_KEY['k2'] = 'running-id' + + compile_module._purge_expired_jobs() + + self.assertIn('running-id', compile_module.COMPILE_JOBS) + self.assertIn('k2', compile_module.JOB_BY_KEY) + + def test_purge_does_not_evict_key_pointing_at_newer_job(self): + # Edge case: an old finished job and a newer running job share the + # same key. JOB_BY_KEY[key] points at the newer one. Purging the old + # job must NOT clear the key (it would orphan the running job from + # future dedup hits). + old_finished = time.time() - compile_module.JOB_TTL_S - 10 + compile_module.COMPILE_JOBS['old-id'] = { + 'state': 'done', + 'started_at': old_finished - 60, + 'finished_at': old_finished, + 'key': 'shared-key', + } + compile_module.COMPILE_JOBS['new-id'] = { + 'state': 'running', + 'started_at': time.time() - 5, + 'key': 'shared-key', + } + compile_module.JOB_BY_KEY['shared-key'] = 'new-id' + + compile_module._purge_expired_jobs() + + self.assertNotIn('old-id', compile_module.COMPILE_JOBS) + self.assertIn('new-id', compile_module.COMPILE_JOBS) + # Crucially, the key still points at the running job. + self.assertEqual(compile_module.JOB_BY_KEY.get('shared-key'), 'new-id') + + +if __name__ == '__main__': + unittest.main()