diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index 64e814a6..4ba24d31 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -630,6 +630,79 @@ class ESPIDFCompiler: 'WiFiServer.h', 'WiFiType.h', 'esp_wifi.h', }) + # arduino-esp32 uses a single library architecture id ("esp32") across + # every chip variant (esp32 / esp32c3 / esp32s3 ...). A library whose + # library.properties declares architectures= without "esp32" or "*" is + # built for another platform and must not be pulled into an ESP32 build. + _ESP32_LIB_ARCH = 'esp32' + + def _core_provided_headers(self) -> frozenset[str]: + """Header filenames provided by the arduino-esp32 core itself + (its `cores/` tree plus every bundled library under `libraries/`). + + These are compiled into the arduino-esp32 IDF component, so a user + library must NEVER shadow them — even when a lib installed in + ~/Arduino/libraries happens to ship a file by the same name. The + canonical break this guards against: `WiFiEspAT/src/WiFi.h` (an + ESP8266 AT-modem library) shadowing the core `WiFi.h`, which drags + `EspAtDrv.cpp` into the build where its `const char OK[]` / + `const char STATUS[]` collide with ESP-IDF's + `enum STATUS { ...OK... }` in rom/ets_sys.h and the compile fails. + + Computed once from the core tree and cached. Always unions the + static `_CORE_ESP32_HEADERS` fallback so the guard still holds even + when the core path is unknown (e.g. translation-only mode). + """ + cached = getattr(self, '_core_headers_cache', None) + if cached is not None: + return cached + headers: set[str] = set(self._CORE_ESP32_HEADERS) + root = Path(self.arduino_path) if self.arduino_path else None + if root and root.is_dir(): + for sub in ('cores', 'libraries'): + base = root / sub + if not base.is_dir(): + continue + for pattern in ('*.h', '*.hpp'): + for f in base.rglob(pattern): + headers.add(f.name) + result = frozenset(headers) + self._core_headers_cache = result + logger.info('[espidf] core-provided header set: %d headers', len(result)) + return result + + @staticmethod + def _parse_library_properties(lib_root: Path) -> dict[str, str]: + """Best-effort parse of an Arduino library.properties into a dict.""" + props: dict[str, str] = {} + try: + text = (lib_root / 'library.properties').read_text( + encoding='utf-8', errors='ignore' + ) + except OSError: + return props + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith('#') or '=' not in line: + continue + key, _, value = line.partition('=') + props[key.strip().lower()] = value.strip() + return props + + def _library_supports_esp32(self, lib_root: Path) -> bool: + """True if the library may be used on the ESP32 platform. + + A missing/empty `architectures` field means "all architectures" + (the Arduino default), so we allow it. Only libraries that + explicitly enumerate architectures WITHOUT esp32/* are rejected — + those are written for another platform and would not compile. + """ + arch = self._parse_library_properties(lib_root).get('architectures', '').strip() + if not arch: + return True + arches = {a.strip().lower() for a in arch.split(',') if a.strip()} + return '*' in arches or self._ESP32_LIB_ARCH in arches + def _resolve_library_components( self, ext_headers: list[str], @@ -678,12 +751,39 @@ class ESPIDFCompiler: continue resolved_headers.add(header) + # Core-first. arduino-esp32 core headers (WiFi.h, Wire.h, SPI.h, + # WebServer.h, HTTPClient.h, ...) are compiled into the + # arduino-esp32 component. They must NEVER resolve to a user + # library, even when an installed lib ships a same-named file + # (e.g. WiFiEspAT/src/WiFi.h). Resolving to it would merge that + # foreign library and break the build. Skip resolution entirely + # and let the core provide the header. + if header in self._core_provided_headers(): + logger.info( + f'[espidf] <{header}> is provided by the arduino-esp32 core ' + f'— never resolving against user libraries' + ) + continue + src_root = ( self._find_library_for_header(header, arduino_libs) if arduino_libs and arduino_libs.is_dir() else None ) + # Architecture guard. A user lib that resolves the header but + # whose library.properties declares architectures= without + # esp32/* is written for a different platform (AVR-only AT-modem + # shims, etc.) and would not compile against ESP-IDF. Drop it. + if src_root is not None: + _lib_root = src_root.parent if src_root.name == 'src' else src_root + if not self._library_supports_esp32(_lib_root): + logger.warning( + f'[espidf] <{header}> resolved to "{_lib_root.name}" but its ' + f'library.properties architectures exclude esp32 — skipping' + ) + src_root = None + # Tracks the "resolved to a core lib that's already compiled into # the arduino-esp32 component" case, so we don't fall through to # the scary "not found — build may fail" warning below for a diff --git a/docs/wiki/esp32-external-library-compilation.md b/docs/wiki/esp32-external-library-compilation.md index e4c0a783..7d6f0b5e 100644 --- a/docs/wiki/esp32-external-library-compilation.md +++ b/docs/wiki/esp32-external-library-compilation.md @@ -262,3 +262,29 @@ python test_espidf_compiler.py | `frontend/src/utils/compilationLogger.ts` | Ninja `FAILED:` block state machine; classifies lines as `'error'` | | `frontend/src/components/editor/CompilationConsole.tsx` | Auto-switches to Errors filter when new errors arrive | | `backend/test_espidf_compiler.py` | 25-test suite covering all library resolution logic | + +--- + +## Core-first resolution (2026-06 fix) + +The "Library Search Order" / `_detect_external_includes` description above +implied core headers like `WiFi.h` / `Wire.h` were skipped. They were not — +`_BUILTIN_HEADERS` never contained them, so a user library that shipped a +same-named header could shadow the arduino-esp32 core. `WiFiEspAT/src/WiFi.h` +(installed via the Library Manager) shadowed the core `WiFi.h`, dragging +`EspAtDrv.cpp` into the build, whose `const char OK[]` / `const char STATUS[]` +collide with ESP-IDF's `enum STATUS { ... OK ... }` in `rom/ets_sys.h`. Result: +every ESP32 sketch that `#include ` failed to compile. + +Fix in `_resolve_library_components`: + +1. **Core-first.** Before any user-lib lookup, a header is skipped if it is + provided by the arduino-esp32 core. The set is computed by + `_core_provided_headers()` scanning `$ARDUINO_ESP32_PATH/{cores,libraries}` + (cached), unioned with the static `_CORE_ESP32_HEADERS` fallback. A core + header can never resolve to a user library, regardless of install order. +2. **Architecture guard.** A user lib that resolves a header but whose + `library.properties` `architectures=` excludes `esp32`/`*` is skipped + (`_library_supports_esp32()`). + +Regression tests: `test/backend/unit/test_espidf_core_first.py`. diff --git a/test/backend/unit/test_espidf_core_first.py b/test/backend/unit/test_espidf_core_first.py new file mode 100644 index 00000000..2d06d878 --- /dev/null +++ b/test/backend/unit/test_espidf_core_first.py @@ -0,0 +1,92 @@ +"""Regression tests for ESP32 library-resolution core-first behaviour. + +A user library that ships a core-named header (e.g. WiFiEspAT/src/WiFi.h) +must never shadow the arduino-esp32 core. This guards the WiFi.h -> +WiFiEspAT -> EspAtDrv.cpp 'const char OK[]'/'STATUS[]' clash with ESP-IDF's +enum STATUS in rom/ets_sys.h, which broke every ESP32 sketch that +#include . + +No ESP-IDF toolchain required — pure resolution logic. +""" + +import sys +import tempfile +import shutil +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / 'backend')) + +from app.services.espidf_compiler import ESPIDFCompiler + + +def _mk(p: Path, content: str = "x") -> None: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + + +class TestCoreFirstResolution(unittest.TestCase): + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp()) + + def tearDown(self) -> None: + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_core_header_not_resolved_to_user_lib(self): + core = self.tmp / "arduino-esp32" + _mk(core / "cores" / "esp32" / "Arduino.h") + _mk(core / "libraries" / "WiFi" / "src" / "WiFi.h") + + ulibs = self.tmp / "Arduino" / "libraries" + _mk(ulibs / "WiFiEspAT" / "library.properties", + "name=WiFiEspAT\narchitectures=*\n") + _mk(ulibs / "WiFiEspAT" / "src" / "WiFi.h") + _mk(ulibs / "WiFiEspAT" / "src" / "utility" / "EspAtDrv.cpp", + "const char OK[];") + # a legit cross-platform lib (no architectures field) must still merge + _mk(ulibs / "DHT_sensor_library" / "DHT.h", "#include \n") + _mk(ulibs / "DHT_sensor_library" / "DHT.cpp") + + c = ESPIDFCompiler() + c.arduino_path = str(core) + c._core_headers_cache = None + out = self.tmp / "project" / "user_libs" + out.mkdir(parents=True) + + names, hdr2comp = c._resolve_library_components( + ["WiFi.h", "DHT.h"], + arduino_libs=ulibs, esp32_libs=None, + arduino_comp_name="arduino-esp32", user_libs_dir=out, + ) + + merged = out / "user_libs_all" + copied = [p.name for p in merged.rglob("*")] if merged.exists() else [] + self.assertNotIn("EspAtDrv.cpp", copied, + "WiFiEspAT was merged — WiFi.h shadowed the core") + self.assertNotIn("WiFi.h", hdr2comp) + self.assertIn("DHT.cpp", copied) + self.assertEqual(hdr2comp.get("DHT.h"), "user_libs_all") + + def test_arch_excluded_lib_skipped(self): + ulibs = self.tmp / "Arduino" / "libraries" + _mk(ulibs / "AvrOnlyLib" / "library.properties", + "name=AvrOnlyLib\narchitectures=avr\n") + _mk(ulibs / "AvrOnlyLib" / "Foo.h") + _mk(ulibs / "AvrOnlyLib" / "Foo.cpp") + + c = ESPIDFCompiler() + c.arduino_path = "" # no core path -> static fallback set + c._core_headers_cache = None + out = self.tmp / "project" / "user_libs" + out.mkdir(parents=True) + + names, hdr2comp = c._resolve_library_components( + ["Foo.h"], + arduino_libs=ulibs, esp32_libs=None, + arduino_comp_name="arduino-esp32", user_libs_dir=out, + ) + self.assertNotIn("Foo.h", hdr2comp) + + +if __name__ == "__main__": + unittest.main()