a2nr
/
velxio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(espidf): per-compile library-scope materialization hook (P2.1e read repoint) 

A scoped ESP32 compile can now resolve libraries from a per-compile directory
provided by an overlay (the manifest's libs symlinked from a content-addressed
cache, with a legacy-dir fallback) instead of the single shared global volume.

- core/hooks.py: register_materialize_library_scope / materialize_library_scope
  (no-op default -> None, so the OSS image keeps its single scan-all dir).
- espidf_compiler: _attempt(allowed) calls the hook, folds the returned content
  token into the build-variant eff_hash (a content change gets a clean build
  dir), passes libraries_dir to _compile_in_dir (arduino_libs = libraries_dir or
  _find_arduino_libraries_dir()), and removes the throwaway dir after. The
  graceful scan-all fallback (allowed=None) keeps using the default dir, so the
  worst case of any materializer failure is fall-back-to-legacy (no break).
This commit is contained in:
David Montero 2026-06-07 16:37:37 +02:00
parent 96b8ca309e
commit 810d8e4b51
2 changed files with 85 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
backend/app/core/hooks.py
View File

@ -133,6 +133,42 @@ async def get_project_libraries(project_id: Optional[str]) -> Optional[list[str]
return None
# ── materialize_library_scope ─────────────────────────────────────────────────
# Given a compile's library manifest (the per-board allowed set), materialize a
# per-compile libraries directory the ESP-IDF resolver reads from — instead of
# the single shared global libraries volume. The overlay symlinks each declared
# library from the content-addressed cache (or, while the global volume is being
# retired, the legacy dir as a fallback) into a throwaway dir and returns
# (libraries_dir, content_token). The compiler folds the token into its build-
# variant hash (so a content change resets the build cache) and removes the dir
# after the attempt. Returns None when no overlay is loaded OR the manifest is
# empty -> the compiler uses its single default libraries dir (OSS self-host
# parity / scan-all). SYNC — pure filesystem (symlink creation).
MaterializeLibraryScopeHook = Callable[[set], Optional[tuple]]
_materialize_library_scope_hook: Optional[MaterializeLibraryScopeHook] = None
def register_materialize_library_scope(hook: MaterializeLibraryScopeHook) -> None:
"""Install the per-compile library-scope materializer. Called in register_pro."""
global _materialize_library_scope_hook
_materialize_library_scope_hook = hook
def materialize_library_scope(allowed_libraries: Optional[set]) -> Optional[tuple]:
"""Return (libraries_dir, content_token) for the manifest, or None to use the
compiler's default single libraries dir. Never raises (a failing materializer
degrades to the default dir)."""
if _materialize_library_scope_hook is None or not allowed_libraries:
return None
try:
return _materialize_library_scope_hook(allowed_libraries)
except Exception:
logger.exception("materialize_library_scope hook failed (using default libraries dir)")
return None
# ── lifespan startup ──────────────────────────────────────────────────────────
# Overlays that need to run async setup during FastAPI lifespan (DB init,
# table creation, legacy column migrations, etc.) register a coroutine here.

73
backend/app/services/espidf_compiler.py
View File

@ -30,6 +30,8 @@ from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Callable, Optional
from app.core.hooks import materialize_library_scope
logger = logging.getLogger(__name__)
# Location of the ESP-IDF project template (relative to this file)
@ -1793,34 +1795,51 @@ class ESPIDFCompiler:
))
async def _attempt(allowed: set[str] | None) -> dict:
# Fold the effective library set into the build-dir hash. A
# different manifest, or the scan-all fallback (allowed=None) after
# a scoped attempt, gets its own clean build dir — resetting at
# _prepare time (before any cmake), which is the well-tested wipe
# path. Same lib set across compiles -> same hash -> warm cache.
# P2.1e — materialize a per-compile library scope: the manifest's
# libs symlinked from the content-addressed cache (with a legacy-dir
# fallback for any not yet cached). A no-op overlay / scan-all
# fallback (allowed=None) returns None -> the compiler uses the
# single default libraries dir. The scope's content token is folded
# into the build-dir hash so a content change (cache vs legacy, or a
# cache update) gets its own clean build dir — and the throwaway
# scope dir is removed after the attempt (its files were already
# copied into the build's user_libs_all by _compile_in_dir).
scope = materialize_library_scope(allowed)
scope_dir = scope[0] if scope else None
scope_token = scope[1] if scope else ''
# Fold the effective library set + resolved content into the build-dir
# hash. A different manifest, the scan-all fallback (allowed=None), or
# changed lib CONTENT gets its own clean build dir — resetting at
# _prepare time (before any cmake), the well-tested wipe path.
_libs_token = (
'm:' + ','.join(sorted(allowed)) if allowed is not None else 'scanall'
('m:' + ','.join(sorted(allowed)) + ('|s:' + scope_token if scope_token else ''))
if allowed is not None else 'scanall'
)
eff_hash = hashlib.sha256(
(options_hash + '|' + _libs_token + '|i:' + _ext_inc_token).encode()
).hexdigest()[:12]
if _USE_PERSISTENT_DIR:
project_dir = _prepare_persistent_project_dir(idf_target, eff_hash)
logger.info(f'[espidf] Using persistent build dir: {project_dir}')
return await self._compile_in_dir(
project_dir, files, idf_target, is_c3,
progress_callback, normalized_opts, spiffs_files,
allowed_libraries=allowed,
)
with tempfile.TemporaryDirectory(prefix='espidf_') as temp_dir:
project_dir = Path(temp_dir) / 'project'
shutil.copytree(_TEMPLATE_DIR, project_dir)
logger.info(f'[espidf] Using ephemeral build dir: {project_dir}')
return await self._compile_in_dir(
project_dir, files, idf_target, is_c3,
progress_callback, normalized_opts, spiffs_files,
allowed_libraries=allowed,
)
try:
if _USE_PERSISTENT_DIR:
project_dir = _prepare_persistent_project_dir(idf_target, eff_hash)
logger.info(f'[espidf] Using persistent build dir: {project_dir}')
return await self._compile_in_dir(
project_dir, files, idf_target, is_c3,
progress_callback, normalized_opts, spiffs_files,
allowed_libraries=allowed, libraries_dir=scope_dir,
)
with tempfile.TemporaryDirectory(prefix='espidf_') as temp_dir:
project_dir = Path(temp_dir) / 'project'
shutil.copytree(_TEMPLATE_DIR, project_dir)
logger.info(f'[espidf] Using ephemeral build dir: {project_dir}')
return await self._compile_in_dir(
project_dir, files, idf_target, is_c3,
progress_callback, normalized_opts, spiffs_files,
allowed_libraries=allowed, libraries_dir=scope_dir,
)
finally:
if scope_dir is not None:
# rmtree unlinks the symlinks, never their cache/legacy targets.
shutil.rmtree(scope_dir.parent, ignore_errors=True)
async def _attempt_safe(allowed: set[str] | None) -> dict:
# Retry ONCE on a clearly-transient infrastructure failure (cmake /
@ -1872,6 +1891,7 @@ class ESPIDFCompiler:
board_options: dict | None = None,
spiffs_files: list[dict] | None = None,
allowed_libraries: set[str] | None = None,
libraries_dir: Path | None = None,
) -> dict:
"""Inner compile body: writes sketch + libs into `project_dir`,
runs cmake + ninja, merges binaries. Caller is responsible for
@ -1980,7 +2000,12 @@ class ESPIDFCompiler:
user_libs_dir.mkdir(exist_ok=True)
esp32_libs = Path(self.arduino_path) / 'libraries' if self.arduino_path else None
arduino_libs = self._find_arduino_libraries_dir()
# P2.1e — when a per-compile library scope was materialized (the
# manifest's libs symlinked from the content-addressed cache, with
# a legacy-dir fallback), resolve from THAT instead of the shared
# global volume. Falls back to the single global dir for scan-all
# / OSS self-host.
arduino_libs = libraries_dir or self._find_arduino_libraries_dir()
component_names, _ = self._resolve_library_components(
ext_headers, arduino_libs, esp32_libs,