From 33fbc03429e99ba7022494e93ed62cbed986b968 Mon Sep 17 00:00:00 2001 From: David Montero Date: Sat, 6 Jun 2026 20:53:18 +0200 Subject: [PATCH] fix(esp32): reset build dir on library-set change via options hash, not mid-compile wipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Supersedes the mid-_compile_in_dir build/ wipe (161deb9): wiping build/ AFTER materializing libs but right before cmake left ESP-IDF's config half-regenerated during the fallback's scoped->scan-all double-compile, intermittently failing with 'sdkconfig.h: No such file'. Instead fold the effective library set (the manifest, else the sketch's non-core external includes) into the per-attempt build-dir hash, so a changed lib set — or the scan-all fallback after a scoped attempt — resets the persistent build/ at _prepare_persistent_project_dir time (before any cmake). That is the existing, well-tested early-wipe path, so the configure is always clean. Same lib set across compiles keeps the warm ccache/ninja cache; core-only sketches share one dir (core headers filtered out of the token) so they never trigger a spurious wipe. Fixes both the original cross-compile staleness (intermittent cmake-configure failures + stale-object false positives) and the fallback regression. --- backend/app/services/espidf_compiler.py | 77 +++++++++---------------- 1 file changed, 27 insertions(+), 50 deletions(-) diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index 46f6ea2b..4a4817b9 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -779,24 +779,6 @@ class ESPIDFCompiler: out[h] = cands return out - @staticmethod - def _fingerprint_dir(d: Path) -> str: - """Stable fingerprint of a directory's file set: sorted relative paths - + sizes. Captures libraries added / removed / version-changed without - reading file contents. Empty / missing dir -> a stable constant.""" - h = hashlib.sha256() - if d.is_dir(): - for f in sorted(d.rglob('*')): - if f.is_file(): - h.update(f.relative_to(d).as_posix().encode()) - h.update(b'\0') - try: - h.update(str(f.stat().st_size).encode()) - except OSError: - pass - h.update(b'\0') - return h.hexdigest() - def _resolve_library_components( self, ext_headers: list[str], @@ -1760,9 +1742,35 @@ class ESPIDFCompiler: json.dumps(normalized_opts, sort_keys=True).encode() ).hexdigest()[:12] + # External-include set of the sketch — a stable proxy for "which + # libraries this compile resolves". Folded into the per-attempt build + # hash below so the persistent build/ is reset (via the tested + # _prepare_persistent_project_dir wipe) whenever the library set + # changes between compiles. Without this, the shared build/ caches a + # cmake config + ninja graph + ccache objects for the PREVIOUS lib set, + # which causes intermittent "cmake configure failed" and stale-object + # false positives when a different project/manifest compiles next. + _sketch_text = '\n'.join(f.get('content', '') for f in files) + _core_hdrs = self._core_provided_headers() + _ext_inc_token = ','.join(sorted( + h for h in set(self._detect_external_includes(_sketch_text)) + if h not in _core_hdrs + )) + async def _attempt(allowed: set[str] | None) -> dict: + # Fold the effective library set into the build-dir hash. A + # different manifest, or the scan-all fallback (allowed=None) after + # a scoped attempt, gets its own clean build dir — resetting at + # _prepare time (before any cmake), which is the well-tested wipe + # path. Same lib set across compiles -> same hash -> warm cache. + _libs_token = ( + 'm:' + ','.join(sorted(allowed)) if allowed is not None else 'scanall' + ) + eff_hash = hashlib.sha256( + (options_hash + '|' + _libs_token + '|i:' + _ext_inc_token).encode() + ).hexdigest()[:12] if _USE_PERSISTENT_DIR: - project_dir = _prepare_persistent_project_dir(idf_target, options_hash) + project_dir = _prepare_persistent_project_dir(idf_target, eff_hash) logger.info(f'[espidf] Using persistent build dir: {project_dir}') return await self._compile_in_dir( project_dir, files, idf_target, is_c3, @@ -1963,37 +1971,6 @@ class ESPIDFCompiler: if main_cpp.exists(): main_cpp.unlink() - # ── Persistent-build-dir staleness guard ───────────────────────── - # The persistent build/ caches ESP-IDF's cmake configuration, ninja's - # incremental graph and (via ccache) compiled objects, all assuming a - # STABLE component set. When the resolved user_libs set changes between - # consecutive compiles on this dir (a different project/user, or a - # different library manifest) that cache is inconsistent: cmake - # reconfigure can fail ("cmake configure failed"), or ninja/ccache can - # reuse a previous compile's objects/headers and let a now-absent - # library slip through — a false-positive success against a lib the - # current sketch/manifest no longer includes. Force a clean configure - # by wiping build/ whenever the user_libs fingerprint changes. ccache - # (enabled) refills the objects so the rebuild stays cheap. No-op on - # the ephemeral path (fresh dir, no build/) and the first compile. - if _USE_PERSISTENT_DIR: - ul_fp = self._fingerprint_dir(project_dir / 'user_libs') - fp_sentinel = project_dir / '.user_libs_fingerprint' - prior_fp = ( - fp_sentinel.read_text(encoding='utf-8').strip() - if fp_sentinel.exists() else '' - ) - if ul_fp != prior_fp: - _bd = project_dir / 'build' - if _bd.exists(): - logger.info( - '[espidf] resolved library set changed (%s -> %s); ' - 'wiping build/ for a clean configure', - prior_fp[:12] or 'none', ul_fp[:12], - ) - shutil.rmtree(_bd, ignore_errors=True) - fp_sentinel.write_text(ul_fp, encoding='utf-8') - # Build using cmake + ninja (more portable than idf.py on Windows) build_dir = project_dir / 'build' build_dir.mkdir(exist_ok=True)