From 3fe1766dc8f71fd7d4c5965439a391ad57331d94 Mon Sep 17 00:00:00 2001 From: David Montero Date: Mon, 8 Jun 2026 06:02:55 +0200 Subject: [PATCH] feat(P2.1h): no-manifest + scan-all-retry resolve from the content-addressed cache, not the global volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last paths that read the shared global /root/Arduino/libraries: a compile with NO manifest (libraries=null — any from-scratch/anon sketch; the manifest is never auto-derived from #includes) and the incomplete-manifest scan-all retry (which re-enters compile unscoped). Both fell through to the global dir, bypassing the cache entirely (a cached lib still failed when global was gone). Now, when no scope is materialized, point the library search at the cache: the cache root is itself a valid Arduino libraries dir (each child is a library), exposed via env (pro overlay sets them): - arduino-cli: ARDUINO_DIRECTORIES_USER = VELXIO_FALLBACK_SKETCHBOOK (whose libraries/ -> cache root) when scope_dir is None. - ESP-IDF: _find_arduino_libraries_dir() prefers VELXIO_FALLBACK_LIBRARIES_DIR. Unset (OSS self-host) -> legacy global, unchanged. Also strip a trailing @version from manifest names (_bare_lib_names): norm_name fused 'ArduinoJson@6.21.5' -> 'arduinojson6215' (cache miss -> global scan-all); the per-board boards_json manifest is the path that still carries @version. --- backend/app/api/routes/compile.py | 17 +++++++++++++++-- backend/app/services/arduino_cli.py | 11 +++++++++++ backend/app/services/espidf_compiler.py | 15 +++++++++++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/backend/app/api/routes/compile.py b/backend/app/api/routes/compile.py index ba54ea5a..7440f449 100644 --- a/backend/app/api/routes/compile.py +++ b/backend/app/api/routes/compile.py @@ -216,6 +216,13 @@ def _resolve_files(request: CompileRequest) -> list[dict[str, str]]: ) +def _bare_lib_names(names) -> set[str] | None: + """Strip a trailing @version (or @wokwi:hash) from each manifest name and + drop empties. None if nothing remains. See _resolve_compile_scope (P2.1h).""" + out = {n.split("@", 1)[0].strip() for n in names if n and n.split("@", 1)[0].strip()} + return out or None + + async def _resolve_compile_scope( request: CompileRequest, requester_id: str | None ) -> tuple[set[str] | None, str | None]: @@ -248,13 +255,19 @@ async def _resolve_compile_scope( # owner-bytes gate; P2.2-sec). gated_owner = await resolve_compile_owner(request.project_id, requester_id) + # P2.1h: strip a trailing @version from each manifest name. norm_name fuses + # the version digits into the name otherwise ("ArduinoJson@6.21.5" -> + # "arduinojson6215"), which misses the cache entry ("arduinojson") and forces + # a global-dir scan-all. The per-board manifest (boards_json[].libraries, + # which the client sends in request.libraries) is the path that still carries + # @version. We don't version-pin today, so the bare name is what resolves. allowed_libraries: set[str] | None = None if request.libraries: - allowed_libraries = set(request.libraries) + allowed_libraries = _bare_lib_names(request.libraries) elif gated_owner is not None: project_libs = await get_project_libraries(request.project_id) if project_libs: - allowed_libraries = set(project_libs) + allowed_libraries = _bare_lib_names(project_libs) owner_id = gated_owner if gated_owner is not None else requester_id return allowed_libraries, owner_id diff --git a/backend/app/services/arduino_cli.py b/backend/app/services/arduino_cli.py index 8f94d900..e7d3b4cc 100644 --- a/backend/app/services/arduino_cli.py +++ b/backend/app/services/arduino_cli.py @@ -359,6 +359,17 @@ class ArduinoCLIService: compile_env = dict(os.environ) if scope_dir is not None: compile_env["ARDUINO_DIRECTORIES_USER"] = str(scope_dir.parent) + else: + # P2.1h: NO manifest -> point the default sketchbook at the + # content-addressed cache (VELXIO_FALLBACK_SKETCHBOOK, whose + # libraries/ is the cache root) instead of the shared global + # volume, so a from-scratch / no-manifest compile (and the + # scan-all retry, which re-enters here unscoped) resolves user + # libraries from the cache. Unset (OSS self-host) -> arduino- + # cli's default sketchbook (legacy global volume). + _fb = os.environ.get("VELXIO_FALLBACK_SKETCHBOOK") + if _fb: + compile_env["ARDUINO_DIRECTORIES_USER"] = _fb # Run compilation using subprocess.run in a thread (Windows compatible) # ESP32 lcgamboa emulator requires DIO flash mode and diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index 3be72ace..c116f052 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -562,8 +562,19 @@ class ESPIDFCompiler: return '\n'.join(lines) + '\n' def _find_arduino_libraries_dir(self) -> Path | None: - """Find the Arduino global user-libraries directory (installed via arduino-cli).""" - candidates = [ + """Find the Arduino global user-libraries directory (installed via arduino-cli). + + P2.1h: when the pro overlay sets VELXIO_FALLBACK_LIBRARIES_DIR (the + content-addressed cache root, itself a valid libraries dir whose children + are library folders), prefer it — so the no-manifest scan + the scan-all + retry resolve from the cache instead of the shared global volume, letting + the global volume be retired. Unset (OSS self-host) -> legacy global. + """ + candidates: list[Path] = [] + _fb = os.environ.get('VELXIO_FALLBACK_LIBRARIES_DIR') + if _fb: + candidates.append(Path(_fb)) + candidates += [ Path.home() / 'Arduino' / 'libraries', Path.home() / 'Documents' / 'Arduino' / 'libraries', Path('/root/Arduino/libraries'), # Docker / CI as root