From d3f06265ac535adb99f8185527ae8db5a3f38962 Mon Sep 17 00:00:00 2001 From: David Montero Date: Sun, 7 Jun 2026 23:27:44 +0200 Subject: [PATCH] fix(P2.1f): isolate scoped library reads via ARDUINO_DIRECTORIES_USER, not --libraries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empirically, arduino-cli's --libraries ADDS to the search path — the global sketchbook is STILL scanned, so the prior commit did NOT isolate reads from the shared global volume. Point ARDUINO_DIRECTORIES_USER at the scratch sketchbook instead (its /libraries becomes the ONLY user-library dir); cores + board-manager URLs live in the DATA dir and are untouched. In-container functest (AVR uno): a cache-only lib compiles via the scope (never in global); a global-only lib NOT in the manifest is INVISIBLE to the scoped compile and only recovers via the scan-all retry (manifest_incomplete=True) — proving the global volume is no longer scanned for a manifest-scoped build. --- backend/app/api/routes/compile.py | 3 +- backend/app/services/arduino_cli.py | 63 ++++++++++++++++------------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/backend/app/api/routes/compile.py b/backend/app/api/routes/compile.py index 7d9c7a75..7fd4f3fe 100644 --- a/backend/app/api/routes/compile.py +++ b/backend/app/api/routes/compile.py @@ -317,7 +317,8 @@ async def _run_compile( # symmetry but currently ignored — those toolchains don't expose the # ESP32 partition / PSRAM knobs we're surfacing. P2.1f: the manifest scope # + owner now flow through so arduino-cli reads the content-addressed cache - # (via --libraries) instead of the shared global volume. + # (via a scoped ARDUINO_DIRECTORIES_USER sketchbook) instead of the shared + # global volume. result = await arduino_cli.compile( files, request.board_fqbn, board_options=request.board_options, allowed_libraries=allowed_libraries, owner_id=owner_id, diff --git a/backend/app/services/arduino_cli.py b/backend/app/services/arduino_cli.py index 9253cc67..8f94d900 100644 --- a/backend/app/services/arduino_cli.py +++ b/backend/app/services/arduino_cli.py @@ -4,6 +4,7 @@ import asyncio import base64 import shutil import re +import os from pathlib import Path from app.core.hooks import materialize_library_scope @@ -285,11 +286,12 @@ class ArduinoCLIService: `allowed_libraries` is the per-board manifest = library resolution SCOPE (P2.1f). When set, ONLY those libraries are made visible to arduino-cli - (via a throwaway --libraries dir of symlinks materialized by the pro - overlay from the content-addressed cache / owner store), instead of the - shared global volume. `owner_id` is the project OWNER's id so a shared / - embed compile resolves that owner's custom libraries. None/empty manifest - (or no overlay) -> arduino-cli's default scan-all (legacy parity). + (a throwaway scratch sketchbook of symlinks materialized by the pro + overlay from the content-addressed cache / owner store, pointed at via + ARDUINO_DIRECTORIES_USER), instead of the shared global volume. + `owner_id` is the project OWNER's id so a shared / embed compile resolves + that owner's custom libraries. None/empty manifest (or no overlay) -> + arduino-cli's default sketchbook -> scan-all (legacy parity). Returns: dict with keys: success, hex_content, stdout, stderr, error @@ -336,20 +338,27 @@ class ArduinoCLIService: # P2.1f — manifest-scoped library resolution. Symlink ONLY the # declared libraries (resolved owner-store -> content-addressed - # cache -> legacy global dir) into a throwaway dir and point - # arduino-cli at it with --libraries, instead of letting it scan - # the shared mutable global volume. None/empty manifest (or no pro - # overlay) -> no flag -> arduino-cli's default scan-all (legacy / - # OSS self-host parity). --libraries only overrides the USER library - # search path; cores + board-manager URLs live in the data dir, so - # RP2040 / ATTinyCore / AVR core resolution stays intact. + # cache -> legacy global dir) into a throwaway scratch sketchbook and + # point arduino-cli's USER directory at it, so it scans ONLY those + # libraries instead of the shared mutable global volume. None/empty + # manifest (or no pro overlay) -> no override -> arduino-cli's default + # sketchbook -> legacy global scan-all (parity). + # + # Mechanism: ARDUINO_DIRECTORIES_USER (the sketchbook), NOT the + # --libraries flag. Verified empirically that `--libraries` ADDS to + # the search path (the global sketchbook is STILL scanned, so it does + # not isolate), whereas pointing ARDUINO_DIRECTORIES_USER at the + # scratch root makes /libraries the ONLY user-library dir. + # scope_dir == /libraries, so its parent is the sketchbook + # root. Cores + board-manager URLs live in the DATA dir and are + # untouched, so RP2040 / ATTinyCore / AVR core resolution stays intact. scope_dir = None try: scope = materialize_library_scope(allowed_libraries, owner_id) scope_dir = scope[0] if scope else None - lib_args = ( - ["--libraries", str(scope_dir)] if scope_dir is not None else [] - ) + compile_env = dict(os.environ) + if scope_dir is not None: + compile_env["ARDUINO_DIRECTORIES_USER"] = str(scope_dir.parent) # Run compilation using subprocess.run in a thread (Windows compatible) # ESP32 lcgamboa emulator requires DIO flash mode and @@ -373,12 +382,10 @@ class ArduinoCLIService: # this define restores it as uint8_t (the type it was). "--build-property", "compiler.cpp.extra_flags=-DBitOrder=uint8_t", - *lib_args, "--output-dir", str(build_dir), str(sketch_dir)] else: cmd = [self.cli_path, "compile", "--fqbn", board_fqbn, - *lib_args, "--output-dir", str(build_dir), str(sketch_dir)] print(f"Running command: {' '.join(cmd)}") @@ -388,7 +395,8 @@ class ArduinoCLIService: return subprocess.run( cmd, capture_output=True, - text=True + text=True, + env=compile_env, ) result = await asyncio.to_thread(run_compile) @@ -542,15 +550,16 @@ class ArduinoCLIService: else: print("=== Compilation failed ===\n") # P2.1f graceful fallback (mirrors the ESP-IDF path): a - # manifest-scoped compile uses --libraries, which REPLACES - # the library search path. If the manifest omitted a needed - # library or a transitive dependency, a header goes missing - # and the build hard-fails where the legacy global scan-all - # would have found it. So when a scope was applied and the - # failure is a missing #include, retry ONCE without the - # scope (scan-all) and flag the manifest as incomplete. A - # genuine source error fails both attempts and returns the - # original scoped failure below. + # manifest-scoped compile points ARDUINO_DIRECTORIES_USER at + # a sketchbook holding ONLY the declared libraries, so the + # global volume is not scanned. If the manifest omitted a + # needed library or a transitive dependency, a header goes + # missing and the build hard-fails where the legacy global + # scan-all would have found it. So when a scope was applied + # and the failure is a missing #include, retry ONCE without + # the scope (global scan-all) and flag the manifest as + # incomplete. A genuine source error fails both attempts and + # returns the original scoped failure below. if scope_dir is not None and _looks_like_missing_header(result.stderr): print("=== Incomplete manifest — retrying scan-all ===\n") retry = await self.compile(