From 5feb4d54ea87fab2e741e274a65afabf2411789a Mon Sep 17 00:00:00 2001 From: David Montero Date: Sat, 6 Jun 2026 18:09:49 +0200 Subject: [PATCH] feat(esp32): graceful fallback for incomplete library manifests (P2.3 safety) A manifest-scoped compile that fails because a header isn't in the manifest (an undeclared/transitive dependency) now retries once with scan-all, so a project with an incomplete manifest still compiles instead of regressing. The response reports manifest_incomplete=true and manifest_suggested_libraries={header: [candidate lib names]} so the manifest can be auto-completed (P2.4) or the user prompted to add the missing library. This de-risks turning on manifest sending (P2.3): an incomplete example/project manifest can never break a build that worked before. - compile(): _attempt(allowed) helper; retry scan-all on missing-lib failure. - _missing_library_headers / _suggest_libraries_for_headers helpers. - CompileResponse.manifest_incomplete + manifest_suggested_libraries. --- backend/app/api/routes/compile.py | 8 ++ backend/app/services/espidf_compiler.py | 102 ++++++++++++++++++++---- 2 files changed, 93 insertions(+), 17 deletions(-) diff --git a/backend/app/api/routes/compile.py b/backend/app/api/routes/compile.py index 873c7508..e652d669 100644 --- a/backend/app/api/routes/compile.py +++ b/backend/app/api/routes/compile.py @@ -160,6 +160,12 @@ class CompileResponse(BaseModel): stderr: str error: str | None = None core_install_log: str | None = None + # P2 — set when a manifest-scoped compile only succeeded after the + # scan-all fallback (i.e. the manifest is missing a dependency). The + # suggested map is {header: [candidate library names]} so the manifest + # can be auto-completed (P2.4) or the user prompted to add the lib. + manifest_incomplete: bool = False + manifest_suggested_libraries: dict | None = None def _classify_compile_error(stderr: str, error: str | None) -> str: @@ -225,6 +231,8 @@ async def _run_compile( stdout=result.get("stdout", ""), stderr=result.get("stderr", ""), error=result.get("error"), + manifest_incomplete=result.get("manifest_incomplete", False), + manifest_suggested_libraries=result.get("manifest_suggested_libraries"), ) # AVR, RP2040, and ESP32 fallback: use arduino-cli diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index 3cbe8856..df4882a8 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -737,6 +737,48 @@ class ESPIDFCompiler: return src_root return None + @staticmethod + def _missing_library_headers(result: dict) -> list[str]: + """Extract the header filenames a failed compile reported as missing + (`fatal error: X.h: No such file or directory`). De-duped, basename + only. Used to decide whether a manifest-scoped failure is a missing- + dependency case worth retrying with scan-all.""" + text = '\n'.join( + str(result.get(k) or '') for k in ('error', 'stderr', 'stdout') + ) + headers: list[str] = [] + for m in re.finditer( + r'fatal error:\s*([A-Za-z0-9_./+-]+\.h(?:pp)?)\s*:\s*No such file', + text, + ): + h = m.group(1).split('/')[-1] + if h not in headers: + headers.append(h) + return headers + + def _suggest_libraries_for_headers(self, headers: list[str]) -> dict: + """For each missing header, the installed libraries that provide it + (by Library Manager display name, else folder name). Returns + {header: [candidate names]} so a manifest can be completed.""" + arduino_libs = self._find_arduino_libraries_dir() + out: dict[str, list[str]] = {} + if not arduino_libs or not arduino_libs.is_dir(): + return out + for h in headers: + cands: list[str] = [] + for lib_dir in sorted(arduino_libs.iterdir()): + if not lib_dir.is_dir(): + continue + for src_root in (lib_dir, lib_dir / 'src'): + if (src_root / h).exists(): + name = self._parse_library_properties(lib_dir).get('name') or lib_dir.name + if name not in cands: + cands.append(name) + break + if cands: + out[h] = cands + return out + def _resolve_library_components( self, ext_headers: list[str], @@ -1700,24 +1742,50 @@ class ESPIDFCompiler: json.dumps(normalized_opts, sort_keys=True).encode() ).hexdigest()[:12] - if _USE_PERSISTENT_DIR: - project_dir = _prepare_persistent_project_dir(idf_target, options_hash) - logger.info(f'[espidf] Using persistent build dir: {project_dir}') - return await self._compile_in_dir( - project_dir, files, idf_target, is_c3, - progress_callback, normalized_opts, spiffs_files, - allowed_libraries=allowed_libraries, - ) + async def _attempt(allowed: set[str] | None) -> dict: + if _USE_PERSISTENT_DIR: + project_dir = _prepare_persistent_project_dir(idf_target, options_hash) + logger.info(f'[espidf] Using persistent build dir: {project_dir}') + return await self._compile_in_dir( + project_dir, files, idf_target, is_c3, + progress_callback, normalized_opts, spiffs_files, + allowed_libraries=allowed, + ) + with tempfile.TemporaryDirectory(prefix='espidf_') as temp_dir: + project_dir = Path(temp_dir) / 'project' + shutil.copytree(_TEMPLATE_DIR, project_dir) + logger.info(f'[espidf] Using ephemeral build dir: {project_dir}') + return await self._compile_in_dir( + project_dir, files, idf_target, is_c3, + progress_callback, normalized_opts, spiffs_files, + allowed_libraries=allowed, + ) - with tempfile.TemporaryDirectory(prefix='espidf_') as temp_dir: - project_dir = Path(temp_dir) / 'project' - shutil.copytree(_TEMPLATE_DIR, project_dir) - logger.info(f'[espidf] Using ephemeral build dir: {project_dir}') - return await self._compile_in_dir( - project_dir, files, idf_target, is_c3, - progress_callback, normalized_opts, spiffs_files, - allowed_libraries=allowed_libraries, - ) + result = await _attempt(allowed_libraries) + + # Graceful fallback (P2). A manifest-scoped compile that fails because a + # header isn't in the manifest (an undeclared / transitive dependency) + # retries once with scan-all, so a project with an incomplete manifest + # still compiles instead of regressing — and we report the gap so the + # manifest can be auto-completed (P2.4) or the user prompted to add the + # missing library. The caller holds the per-target lock for this whole + # method, so the retry safely reuses the same build dir. + if allowed_libraries is not None and not result.get('success'): + missing = self._missing_library_headers(result) + if missing: + logger.warning( + f'[espidf] scoped compile missing {missing} (not in manifest) — ' + f'retrying scan-all' + ) + retry = await _attempt(None) + if retry.get('success'): + retry['manifest_incomplete'] = True + retry['manifest_suggested_libraries'] = ( + self._suggest_libraries_for_headers(missing) + ) + return retry + # Both failed: the scoped error is the more informative one. + return result async def _compile_in_dir( self,