1339 lines
56 KiB
Python
1339 lines
56 KiB
Python
"""
|
|
ESP-IDF Compilation Service for ESP32 targets.
|
|
|
|
Replaces arduino-cli for ESP32/ESP32-C3 compilation. User Arduino sketches
|
|
are compiled using ESP-IDF (with optional Arduino-as-component) to produce
|
|
firmware that boots reliably in the lcgamboa QEMU fork.
|
|
|
|
The key difference vs arduino-cli: ESP-IDF gives control over bootloader,
|
|
sdkconfig, and flash mapping — all of which must be QEMU-compatible.
|
|
|
|
Two compilation modes:
|
|
1. Arduino-as-component: Full Arduino API (WiFi.h, WebServer.h, etc.)
|
|
compiled through idf.py. Requires ARDUINO_ESP32_PATH env var.
|
|
2. Pure ESP-IDF: Translates common Arduino patterns to ESP-IDF C APIs.
|
|
Fallback when Arduino component is not installed.
|
|
"""
|
|
import asyncio
|
|
import base64
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
from dataclasses import dataclass
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Callable, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Location of the ESP-IDF project template (relative to this file)
|
|
_TEMPLATE_DIR = Path(__file__).parent / 'esp-idf-template'
|
|
|
|
# ── Persistent build dir ─────────────────────────────────────────────────────
|
|
# Cold ESP-IDF compiles rebuild ~1480 base objects (FreeRTOS, lwIP, esp_wifi,
|
|
# libsodium, …). The default tempfile.TemporaryDirectory flow gave each compile
|
|
# a fresh path under /tmp/espidf_<random>/, which baked into -I and
|
|
# -fmacro-prefix-map flags and made ccache 0% effective (different cwd → hash
|
|
# miss every time). With the persistent dir, /var/lib/velxio-build/<target>/
|
|
# is a stable anchor: ninja's incremental cache + ccache hits combine to bring
|
|
# warm compiles down to ~5-30s.
|
|
#
|
|
# Concurrent compiles to the SAME target would corrupt the shared build dir;
|
|
# they're serialised by the per-target asyncio.Lock in routes/compile.py.
|
|
# Different targets get different subdirs and run in parallel.
|
|
#
|
|
# Set VELXIO_PERSISTENT_BUILD_DIR=0 to fall back to the legacy tempfile flow
|
|
# without rebuilding the image (escape hatch if the persistent dir misbehaves
|
|
# in production).
|
|
_BUILD_ROOT = Path(os.environ.get('VELXIO_BUILD_ROOT', '/var/lib/velxio-build'))
|
|
_USE_PERSISTENT_DIR = (
|
|
os.environ.get('VELXIO_PERSISTENT_BUILD_DIR', '1')
|
|
not in ('0', 'false', 'False', '')
|
|
)
|
|
|
|
|
|
def _idf_version_signature() -> str:
|
|
"""Snapshot of the ESP-IDF + arduino-esp32 toolchain version. Used to
|
|
invalidate persistent build dirs after an upstream submodule bump (the
|
|
cached object files on disk are no longer ABI-compatible)."""
|
|
parts = []
|
|
idf_version_file = Path('/opt/esp-idf/version.txt')
|
|
if idf_version_file.exists():
|
|
parts.append(idf_version_file.read_text(encoding='utf-8').strip())
|
|
arduino_version_file = Path('/opt/arduino-esp32/version.txt')
|
|
if arduino_version_file.exists():
|
|
parts.append(arduino_version_file.read_text(encoding='utf-8').strip())
|
|
if not parts:
|
|
# Fall back to mtime of the IDF tree root — coarse but stable per
|
|
# image build.
|
|
try:
|
|
parts.append(str(int(Path('/opt/esp-idf').stat().st_mtime)))
|
|
except OSError:
|
|
parts.append('unknown')
|
|
return '|'.join(parts)
|
|
|
|
|
|
# Type for live progress callback. Called from a worker thread for every
|
|
# stdout/stderr line as the build runs. Implementations should be cheap and
|
|
# thread-safe (callers commonly stash lines into a dict shared with the main
|
|
# event loop). Exceptions raised from the callback are swallowed so a faulty
|
|
# UI hook can never break the build.
|
|
ProgressCallback = Callable[[str], None]
|
|
|
|
|
|
@dataclass
|
|
class _RunResult:
|
|
"""Drop-in replacement for the fields we read off subprocess.CompletedProcess."""
|
|
returncode: int
|
|
stdout: str
|
|
stderr: str
|
|
|
|
|
|
def _run_with_streaming(
|
|
cmd: list[str],
|
|
*,
|
|
cwd: str,
|
|
env: dict[str, str],
|
|
timeout: float,
|
|
progress_callback: Optional[ProgressCallback],
|
|
) -> _RunResult:
|
|
"""Run `cmd` synchronously and stream stdout + stderr line-by-line.
|
|
|
|
Behaves like subprocess.run(capture_output=True, text=True) but invokes
|
|
`progress_callback(line)` for every line as it arrives. When
|
|
progress_callback is None this falls back to a single subprocess.run call
|
|
so we don't pay the threading cost on the unit-test path that doesn't
|
|
care about live output.
|
|
|
|
Raises subprocess.TimeoutExpired on timeout (matches the existing flow).
|
|
"""
|
|
if progress_callback is None:
|
|
cp = subprocess.run(
|
|
cmd, cwd=cwd, env=env, capture_output=True, text=True, timeout=timeout,
|
|
)
|
|
return _RunResult(returncode=cp.returncode, stdout=cp.stdout, stderr=cp.stderr)
|
|
|
|
proc = subprocess.Popen(
|
|
cmd,
|
|
cwd=cwd,
|
|
env=env,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
bufsize=1, # line-buffered
|
|
)
|
|
stdout_lines: list[str] = []
|
|
stderr_lines: list[str] = []
|
|
|
|
def _drain(stream, sink: list[str]) -> None:
|
|
try:
|
|
for line in iter(stream.readline, ''):
|
|
sink.append(line)
|
|
try:
|
|
progress_callback(line)
|
|
except Exception:
|
|
# A faulty progress sink must never break the build.
|
|
pass
|
|
finally:
|
|
try:
|
|
stream.close()
|
|
except Exception:
|
|
pass
|
|
|
|
t_out = threading.Thread(target=_drain, args=(proc.stdout, stdout_lines), daemon=True)
|
|
t_err = threading.Thread(target=_drain, args=(proc.stderr, stderr_lines), daemon=True)
|
|
t_out.start()
|
|
t_err.start()
|
|
|
|
try:
|
|
proc.wait(timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
# Give drain threads a chance to flush before we raise.
|
|
t_out.join(timeout=2)
|
|
t_err.join(timeout=2)
|
|
raise
|
|
|
|
t_out.join(timeout=5)
|
|
t_err.join(timeout=5)
|
|
return _RunResult(
|
|
returncode=proc.returncode,
|
|
stdout=''.join(stdout_lines),
|
|
stderr=''.join(stderr_lines),
|
|
)
|
|
|
|
|
|
def _prepare_persistent_project_dir(idf_target: str) -> Path:
|
|
"""Return the path to a per-target persistent project dir, materialising
|
|
it from the template on first use and resetting the per-compile parts
|
|
(main/, user_libs/) on every call so a previous sketch's files don't
|
|
leak into the next compile.
|
|
|
|
Keeps the `build/` directory intact across calls — that's where ninja's
|
|
incremental cache + the .o files we want ccache to hit live.
|
|
"""
|
|
target_dir = _BUILD_ROOT / idf_target
|
|
target_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Wipe the whole target dir if the toolchain version changed (the cached
|
|
# .o files are no longer compatible).
|
|
sentinel = target_dir / '.idf_version'
|
|
current_signature = _idf_version_signature()
|
|
if sentinel.exists() and sentinel.read_text(encoding='utf-8').strip() != current_signature:
|
|
logger.info(
|
|
f'[espidf] toolchain version changed; wiping persistent build dir {target_dir}'
|
|
)
|
|
shutil.rmtree(target_dir)
|
|
target_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
project_dir = target_dir / 'project'
|
|
|
|
if not project_dir.exists():
|
|
# First use of this target — full template copy.
|
|
shutil.copytree(_TEMPLATE_DIR, project_dir)
|
|
else:
|
|
# Subsequent compile — reset the per-compile parts only, keep build/.
|
|
# main/ is overwritten with the user's sketch + any extra files.
|
|
# user_libs/ is rebuilt by _resolve_library_components based on the
|
|
# sketch's #includes.
|
|
shutil.rmtree(project_dir / 'main', ignore_errors=True)
|
|
shutil.copytree(_TEMPLATE_DIR / 'main', project_dir / 'main')
|
|
shutil.rmtree(project_dir / 'user_libs', ignore_errors=True)
|
|
|
|
sentinel.write_text(current_signature, encoding='utf-8')
|
|
return project_dir
|
|
|
|
# Static IP that matches slirp DHCP range (first client = x.x.x.15)
|
|
_STATIC_IP = '192.168.4.15'
|
|
_GATEWAY_IP = '192.168.4.2'
|
|
_NETMASK = '255.255.255.0'
|
|
|
|
# SSID the QEMU WiFi AP broadcasts.
|
|
# Must match one of the access_point_info entries in esp32_wifi_ap.c
|
|
# (the lcgamboa QEMU fork). "Espressif" is on channel 5 in that array.
|
|
_QEMU_WIFI_SSID = 'Espressif'
|
|
_QEMU_WIFI_CHANNEL = 5
|
|
|
|
|
|
class ESPIDFCompiler:
|
|
"""Compile Arduino sketches using ESP-IDF for QEMU-compatible output."""
|
|
|
|
def __init__(self):
|
|
self.idf_path = os.environ.get('IDF_PATH', '')
|
|
self.arduino_path = os.environ.get('ARDUINO_ESP32_PATH', '')
|
|
self.has_arduino = bool(self.arduino_path) and os.path.isdir(self.arduino_path)
|
|
|
|
# Try common locations on Windows dev machines
|
|
if not self.idf_path:
|
|
for candidate in [
|
|
r'C:\Espressif\frameworks\esp-idf-v4.4.7',
|
|
r'C:\esp\esp-idf',
|
|
'/opt/esp-idf',
|
|
]:
|
|
if os.path.isdir(candidate):
|
|
self.idf_path = candidate
|
|
break
|
|
|
|
# Auto-detect Arduino-as-component if not explicitly set
|
|
if self.idf_path and not self.has_arduino:
|
|
for candidate in [
|
|
r'C:\Espressif\components\arduino-esp32',
|
|
os.path.join(self.idf_path, '..', 'components', 'arduino-esp32'),
|
|
'/opt/arduino-esp32',
|
|
]:
|
|
if os.path.isdir(candidate):
|
|
self.arduino_path = os.path.abspath(candidate)
|
|
self.has_arduino = True
|
|
break
|
|
|
|
if self.idf_path:
|
|
logger.info(f'[espidf] IDF_PATH={self.idf_path}')
|
|
if self.has_arduino:
|
|
logger.info(f'[espidf] Arduino component: yes ({self.arduino_path})')
|
|
else:
|
|
logger.info('[espidf] Arduino component: no (pure ESP-IDF fallback)')
|
|
else:
|
|
logger.warning('[espidf] IDF_PATH not set — ESP-IDF compilation unavailable')
|
|
|
|
@property
|
|
def available(self) -> bool:
|
|
"""Whether ESP-IDF toolchain is available."""
|
|
return bool(self.idf_path) and os.path.isdir(self.idf_path)
|
|
|
|
def _is_esp32c3(self, board_fqbn: str) -> bool:
|
|
"""Return True if FQBN targets ESP32-C3 (RISC-V)."""
|
|
return 'esp32c3' in board_fqbn or 'esp32-c3' in board_fqbn
|
|
|
|
def _idf_target(self, board_fqbn: str) -> str:
|
|
"""Map FQBN to IDF_TARGET."""
|
|
if self._is_esp32c3(board_fqbn):
|
|
return 'esp32c3'
|
|
# Default to esp32 (Xtensa) for all other ESP32 variants
|
|
return 'esp32'
|
|
|
|
def _detect_wifi_usage(self, code: str) -> bool:
|
|
"""Check if sketch uses WiFi."""
|
|
return bool(re.search(r'#include\s*[<"]WiFi\.h[">]|WiFi\.begin\(', code))
|
|
|
|
def _detect_webserver_usage(self, code: str) -> bool:
|
|
"""Check if sketch uses WebServer."""
|
|
return bool(re.search(
|
|
r'#include\s*[<"]WebServer\.h[">]|#include\s*[<"]ESP8266WebServer\.h[">]|WebServer\s+\w+',
|
|
code
|
|
))
|
|
|
|
def _normalize_wifi_for_qemu(self, code: str) -> str:
|
|
"""
|
|
Normalize WiFi SSID/password/channel in Arduino sketches for QEMU.
|
|
|
|
QEMU's WiFi AP broadcasts _QEMU_WIFI_SSID on _QEMU_WIFI_CHANNEL with open auth.
|
|
This method rewrites the user's sketch so that:
|
|
- Any SSID string literal → _QEMU_WIFI_SSID
|
|
- Password → "" (open auth)
|
|
- Channel → _QEMU_WIFI_CHANNEL
|
|
The user's editor still shows their original code; only the compiled
|
|
binary is modified.
|
|
"""
|
|
if not self._detect_wifi_usage(code):
|
|
return code
|
|
|
|
# 1) Replace SSID variable definitions:
|
|
# const char* ssid = "anything" → _QEMU_WIFI_SSID
|
|
# char ssid[] = "anything" → _QEMU_WIFI_SSID
|
|
# #define WIFI_SSID "anything" → _QEMU_WIFI_SSID
|
|
code = re.sub(
|
|
r'((?:const\s+)?char\s*\*?\s*ssid\s*\[?\]?\s*=\s*)"[^"]*"',
|
|
rf'\1"{_QEMU_WIFI_SSID}"',
|
|
code,
|
|
flags=re.IGNORECASE
|
|
)
|
|
code = re.sub(
|
|
r'(#define\s+\w*SSID\w*\s+)"[^"]*"',
|
|
rf'\1"{_QEMU_WIFI_SSID}"',
|
|
code,
|
|
flags=re.IGNORECASE
|
|
)
|
|
|
|
# 2) Normalize WiFi.begin() calls:
|
|
# WiFi.begin("X") → WiFi.begin(_QEMU_WIFI_SSID, "", _QEMU_WIFI_CHANNEL)
|
|
# WiFi.begin("X", "pass") → WiFi.begin(_QEMU_WIFI_SSID, "", _QEMU_WIFI_CHANNEL)
|
|
# WiFi.begin(ssid, pass, N) → WiFi.begin(ssid, "", _QEMU_WIFI_CHANNEL)
|
|
# WiFi.begin(ssid) → WiFi.begin(ssid, "", _QEMU_WIFI_CHANNEL)
|
|
|
|
def _rewrite_wifi_begin(m: re.Match) -> str:
|
|
args = m.group(1)
|
|
parts = [a.strip() for a in args.split(',')]
|
|
ssid_arg = parts[0]
|
|
# If SSID is a string literal, force to _QEMU_WIFI_SSID
|
|
if ssid_arg.startswith('"'):
|
|
ssid_arg = f'"{_QEMU_WIFI_SSID}"'
|
|
return f'WiFi.begin({ssid_arg}, "", {_QEMU_WIFI_CHANNEL})'
|
|
|
|
code = re.sub(
|
|
r'WiFi\.begin\s*\(([^)]+)\)',
|
|
_rewrite_wifi_begin,
|
|
code
|
|
)
|
|
|
|
logger.info('[espidf] WiFi normalized: SSID→%s, channel→%d, open auth', _QEMU_WIFI_SSID, _QEMU_WIFI_CHANNEL)
|
|
return code
|
|
|
|
def _translate_sketch_to_espidf(self, sketch_code: str) -> str:
|
|
"""
|
|
Translate an Arduino WiFi+WebServer sketch to pure ESP-IDF C code.
|
|
|
|
This handles the common pattern:
|
|
- WiFi.begin("ssid", "pass") → esp_wifi_start() with static IP
|
|
- WebServer server(80) + server.on("/", handler) → esp_http_server
|
|
- digitalWrite/pinMode → gpio_set_level/gpio_set_direction
|
|
|
|
Returns C source code for sketch_translated.c
|
|
"""
|
|
uses_wifi = self._detect_wifi_usage(sketch_code)
|
|
uses_webserver = self._detect_webserver_usage(sketch_code)
|
|
|
|
# Extract route handlers from server.on() calls
|
|
routes = []
|
|
handler_bodies = {}
|
|
if uses_webserver:
|
|
# Match: server.on("/path", handler_func)
|
|
# or: server.on("/path", HTTP_GET, handler_func)
|
|
for m in re.finditer(
|
|
r'server\.on\(\s*"([^"]+)"\s*,\s*(?:HTTP_\w+\s*,\s*)?(\w+)\s*\)',
|
|
sketch_code
|
|
):
|
|
routes.append((m.group(1), m.group(2)))
|
|
|
|
# Extract handler function bodies
|
|
# Match: void handler_name() { ... server.send(...) ... }
|
|
handler_bodies = {}
|
|
for m in re.finditer(
|
|
r'void\s+(\w+)\s*\(\s*\)\s*\{([^}]*(?:\{[^}]*\}[^}]*)*)\}',
|
|
sketch_code,
|
|
re.DOTALL
|
|
):
|
|
fname = m.group(1)
|
|
body = m.group(2)
|
|
# Extract server.send() content
|
|
send_match = re.search(
|
|
r'server\.send\s*\(\s*(\d+)\s*,\s*"([^"]+)"\s*,\s*"([^"]*)"',
|
|
body
|
|
)
|
|
if not send_match:
|
|
# Try multi-line string or variable
|
|
send_match = re.search(
|
|
r'server\.send\s*\(\s*(\d+)\s*,\s*"([^"]+)"\s*,\s*(\w+)',
|
|
body
|
|
)
|
|
if send_match:
|
|
handler_bodies[fname] = {
|
|
'status': send_match.group(1),
|
|
'content_type': send_match.group(2),
|
|
'content': send_match.group(3),
|
|
}
|
|
|
|
# Build the translated C source
|
|
lines = []
|
|
lines.append('/* Auto-translated from Arduino sketch to ESP-IDF */')
|
|
lines.append('')
|
|
|
|
if uses_wifi:
|
|
lines.append(f'#define WIFI_SSID "{_QEMU_WIFI_SSID}"')
|
|
lines.append('#define WIFI_PASS ""')
|
|
lines.append(f'#define STATIC_IP "{_STATIC_IP}"')
|
|
lines.append(f'#define GATEWAY_IP "{_GATEWAY_IP}"')
|
|
lines.append(f'#define NETMASK "{_NETMASK}"')
|
|
lines.append('')
|
|
|
|
# Generate HTML content variables from handler bodies
|
|
for fname, info in handler_bodies.items():
|
|
content = info['content']
|
|
if content.startswith('"') or content.startswith("'"):
|
|
content = content.strip('"').strip("'")
|
|
lines.append(f'static const char *{fname}_html = "{content}";')
|
|
lines.append('')
|
|
|
|
# Generate ESP-IDF HTTP handlers
|
|
if uses_webserver:
|
|
for path, handler_name in routes:
|
|
info = handler_bodies.get(handler_name, {})
|
|
ct = info.get('content_type', 'text/html')
|
|
lines.append(f'static esp_err_t {handler_name}_handler(httpd_req_t *req) {{')
|
|
lines.append(f' httpd_resp_set_type(req, "{ct}");')
|
|
if handler_name in handler_bodies:
|
|
lines.append(f' return httpd_resp_send(req, {handler_name}_html, HTTPD_RESP_USE_STRLEN);')
|
|
else:
|
|
lines.append(f' return httpd_resp_send(req, "OK", 2);')
|
|
lines.append('}')
|
|
lines.append('')
|
|
|
|
# Generate webserver start function
|
|
if uses_webserver:
|
|
lines.append('static void start_webserver(void) {')
|
|
lines.append(' httpd_config_t config = HTTPD_DEFAULT_CONFIG();')
|
|
lines.append(' httpd_handle_t server = NULL;')
|
|
lines.append(' if (httpd_start(&server, &config) == ESP_OK) {')
|
|
for path, handler_name in routes:
|
|
uri_var = handler_name + '_uri'
|
|
lines.append(f' httpd_uri_t {uri_var} = {{')
|
|
lines.append(f' .uri = "{path}",')
|
|
lines.append(f' .method = HTTP_GET,')
|
|
lines.append(f' .handler = {handler_name}_handler')
|
|
lines.append(f' }};')
|
|
lines.append(f' httpd_register_uri_handler(server, &{uri_var});')
|
|
lines.append(' }')
|
|
lines.append('}')
|
|
lines.append('')
|
|
|
|
# WiFi event handler + init
|
|
if uses_wifi:
|
|
lines.append('static EventGroupHandle_t s_wifi_event_group;')
|
|
lines.append('#define WIFI_CONNECTED_BIT BIT0')
|
|
lines.append('')
|
|
lines.append('static void wifi_event_handler(void *arg, esp_event_base_t base,')
|
|
lines.append(' int32_t id, void *data) {')
|
|
lines.append(' if (base == WIFI_EVENT && id == WIFI_EVENT_STA_START)')
|
|
lines.append(' esp_wifi_connect();')
|
|
lines.append(' else if (base == WIFI_EVENT && id == WIFI_EVENT_STA_DISCONNECTED)')
|
|
lines.append(' esp_wifi_connect();')
|
|
lines.append(' else if (base == IP_EVENT && id == IP_EVENT_STA_GOT_IP)')
|
|
lines.append(' xEventGroupSetBits(s_wifi_event_group, WIFI_CONNECTED_BIT);')
|
|
lines.append('}')
|
|
lines.append('')
|
|
lines.append('static void wifi_init_sta(void) {')
|
|
lines.append(' s_wifi_event_group = xEventGroupCreate();')
|
|
lines.append(' esp_netif_init();')
|
|
lines.append(' esp_event_loop_create_default();')
|
|
lines.append(' esp_netif_t *sta = esp_netif_create_default_wifi_sta();')
|
|
lines.append(' esp_netif_dhcpc_stop(sta);')
|
|
lines.append(' esp_netif_ip_info_t ip_info;')
|
|
lines.append(' ip_info.ip.addr = ipaddr_addr(STATIC_IP);')
|
|
lines.append(' ip_info.gw.addr = ipaddr_addr(GATEWAY_IP);')
|
|
lines.append(' ip_info.netmask.addr = ipaddr_addr(NETMASK);')
|
|
lines.append(' esp_netif_set_ip_info(sta, &ip_info);')
|
|
lines.append(' wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();')
|
|
lines.append(' esp_wifi_init(&cfg);')
|
|
lines.append(' esp_event_handler_instance_register(WIFI_EVENT, ESP_EVENT_ANY_ID,')
|
|
lines.append(' &wifi_event_handler, NULL, NULL);')
|
|
lines.append(' esp_event_handler_instance_register(IP_EVENT, IP_EVENT_STA_GOT_IP,')
|
|
lines.append(' &wifi_event_handler, NULL, NULL);')
|
|
lines.append(' wifi_config_t wifi_config = {')
|
|
lines.append(' .sta = {')
|
|
lines.append(' .ssid = WIFI_SSID,')
|
|
lines.append(' .password = WIFI_PASS,')
|
|
lines.append(' .threshold.authmode = WIFI_AUTH_OPEN,')
|
|
lines.append(' },')
|
|
lines.append(' };')
|
|
lines.append(' esp_wifi_set_mode(WIFI_MODE_STA);')
|
|
lines.append(' esp_wifi_set_config(WIFI_IF_STA, &wifi_config);')
|
|
lines.append(' esp_wifi_start();')
|
|
lines.append('}')
|
|
lines.append('')
|
|
|
|
# app_main
|
|
lines.append('void app_main(void) {')
|
|
if uses_wifi:
|
|
lines.append(' esp_err_t ret = nvs_flash_init();')
|
|
lines.append(' if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {')
|
|
lines.append(' nvs_flash_erase();')
|
|
lines.append(' nvs_flash_init();')
|
|
lines.append(' }')
|
|
lines.append(' wifi_init_sta();')
|
|
lines.append(' vTaskDelay(pdMS_TO_TICKS(3000));')
|
|
if uses_webserver:
|
|
lines.append(' start_webserver();')
|
|
lines.append(' while (1) {')
|
|
lines.append(' vTaskDelay(pdMS_TO_TICKS(1000));')
|
|
lines.append(' }')
|
|
lines.append('}')
|
|
|
|
return '\n'.join(lines) + '\n'
|
|
|
|
def _find_arduino_libraries_dir(self) -> Path | None:
|
|
"""Find the Arduino global user-libraries directory (installed via arduino-cli)."""
|
|
candidates = [
|
|
Path.home() / 'Arduino' / 'libraries',
|
|
Path.home() / 'Documents' / 'Arduino' / 'libraries',
|
|
Path('/root/Arduino/libraries'), # Docker / CI as root
|
|
Path('/home/user/Arduino/libraries'),
|
|
Path('/Arduino/libraries'),
|
|
]
|
|
# Also check arduino-cli's data directory
|
|
for base in [
|
|
Path.home() / '.arduino15',
|
|
Path('/root/.arduino15'),
|
|
Path('/home/user/.arduino15'),
|
|
]:
|
|
candidates.append(base / 'libraries')
|
|
|
|
for c in candidates:
|
|
if c.is_dir():
|
|
logger.info(f'[espidf] Arduino libraries dir: {c}')
|
|
return c
|
|
logger.warning('[espidf] Arduino libraries dir not found')
|
|
return None
|
|
|
|
# Headers that will NEVER appear in any Arduino library directory:
|
|
# - C/C++ standard library headers
|
|
# - Arduino core API types compiled directly into arduino-esp32 (not installable)
|
|
#
|
|
# Everything else (Wire.h, SPI.h, WiFi.h, Adafruit_GFX.h, …) is resolved
|
|
# dynamically: user-installed libs → IDF component; arduino-esp32 bundled
|
|
# libs → skip (already compiled in); not found → warning.
|
|
_BUILTIN_HEADERS = frozenset({
|
|
# C/C++ standard library
|
|
'math.h', 'stdint.h', 'stdio.h', 'stdlib.h', 'string.h', 'stdarg.h',
|
|
'stddef.h', 'stdbool.h', 'float.h', 'limits.h', 'assert.h',
|
|
# Arduino core types — part of arduino-esp32 source, not installable libraries
|
|
'Arduino.h', 'HardwareSerial.h', 'Stream.h', 'Print.h', 'WString.h',
|
|
'pgmspace.h', 'IPAddress.h',
|
|
})
|
|
|
|
# Core arduino-esp32 bundled libraries — already compiled into the IDF component,
|
|
# must NOT be duplicated as separate user_libs components.
|
|
_CORE_ESP32_LIBS: frozenset[str] = frozenset({
|
|
'Wire', 'SPI', 'WiFi', 'EEPROM', 'SD', 'FS',
|
|
'LittleFS', 'SPIFFS', 'WebServer', 'HTTPClient',
|
|
'WiFiClientSecure', 'BluetoothSerial', 'BLE',
|
|
'Preferences', 'Update', 'Ticker',
|
|
})
|
|
|
|
def _resolve_library_components(
|
|
self,
|
|
ext_headers: list[str],
|
|
arduino_libs: Path | None,
|
|
esp32_libs: Path | None,
|
|
arduino_comp_name: str,
|
|
user_libs_dir: Path,
|
|
) -> tuple[list[str], dict[str, str]]:
|
|
"""
|
|
BFS over ext_headers (and transitive includes) to discover all external
|
|
Arduino libraries and merge them into a single 'user_libs_all' IDF component.
|
|
|
|
All library files are copied flat into one directory, so every header is
|
|
visible to every other header and source file without any cross-component
|
|
REQUIRES propagation — which is unreliable in ESP-IDF 4.x for deeply
|
|
nested transitive dependencies.
|
|
|
|
Search priority per header:
|
|
1. arduino_libs (user-installed via Library Manager) → merge into component
|
|
2. esp32_libs (bundled with arduino-esp32) → skip core libs (Wire, SPI, …);
|
|
merge non-core libs (e.g. Adafruit libs shipped with arduino-esp32)
|
|
3. not found → warning only
|
|
|
|
Returns:
|
|
component_names — ['user_libs_all'] if any lib found, else []
|
|
header_to_comp — every resolved header → 'user_libs_all'
|
|
"""
|
|
logger.info(f'[espidf] ext_headers detected: {ext_headers}')
|
|
logger.info(f'[espidf] arduino_libs: {arduino_libs}')
|
|
logger.info(f'[espidf] esp32_libs: {esp32_libs}')
|
|
|
|
comp_dir = user_libs_dir / 'user_libs_all'
|
|
comp_dir.mkdir(exist_ok=True)
|
|
|
|
cpp_files: list[str] = []
|
|
seen_names: set[str] = set()
|
|
header_to_comp: dict[str, str] = {}
|
|
found_any = False
|
|
|
|
headers_to_resolve: list[str] = list(ext_headers)
|
|
resolved_headers: set[str] = set()
|
|
|
|
while headers_to_resolve:
|
|
header = headers_to_resolve.pop(0)
|
|
if header in resolved_headers:
|
|
continue
|
|
resolved_headers.add(header)
|
|
|
|
src_root = (
|
|
self._find_library_for_header(header, arduino_libs)
|
|
if arduino_libs and arduino_libs.is_dir()
|
|
else None
|
|
)
|
|
|
|
if src_root is None and esp32_libs and esp32_libs.is_dir():
|
|
esp32_root = self._find_library_for_header(header, esp32_libs)
|
|
if esp32_root:
|
|
lib_name = esp32_root.parent.name if esp32_root.name == 'src' else esp32_root.name
|
|
if lib_name in self._CORE_ESP32_LIBS:
|
|
logger.debug(f'[espidf] <{header}> is bundled core lib "{lib_name}", skipping')
|
|
else:
|
|
logger.info(f'[espidf] <{header}> found in esp32_libs as "{lib_name}", merging')
|
|
src_root = esp32_root
|
|
|
|
if src_root:
|
|
lib_dir_name = src_root.parent.name if src_root.name == 'src' else src_root.name
|
|
logger.info(f'[espidf] Merging "{lib_dir_name}" into user_libs_all for <{header}>')
|
|
found_any = True
|
|
header_to_comp[header] = 'user_libs_all'
|
|
|
|
# Preserve directory structure while merging libraries.
|
|
# Skip non-buildable directories like examples, tests, docs.
|
|
lib_root = src_root.parent if src_root.name == 'src' else src_root
|
|
has_src_layout = (lib_root / 'src').is_dir()
|
|
|
|
excluded_dirs = {
|
|
'.git', '.github', '.vscode', '__pycache__',
|
|
'docs', 'doc', 'example', 'examples', 'test', 'tests',
|
|
'extras', 'ci', 'fuzz', 'fuzzing', 'benchmark', 'benchmarks',
|
|
}
|
|
|
|
def _should_include(rel_path: Path) -> bool:
|
|
parts = rel_path.parts
|
|
if any(part.lower() in excluded_dirs for part in parts[:-1]):
|
|
return False
|
|
if rel_path.suffix not in ('.h', '.hpp', '.c', '.cpp'):
|
|
return False
|
|
if has_src_layout:
|
|
return parts[0] == 'src' or len(parts) == 1
|
|
# Non-src-layout libs (Adafruit_GFX, etc.) keep auxiliary
|
|
# headers in subdirs like Fonts/ or gfxfont/. Anything not
|
|
# already excluded (docs/examples/tests handled above) is
|
|
# presumed to be buildable source.
|
|
return True
|
|
|
|
for f in lib_root.rglob('*'):
|
|
if not f.is_file():
|
|
continue
|
|
rel_path = f.relative_to(lib_root)
|
|
if not _should_include(rel_path):
|
|
continue
|
|
# Track file by its relative path to preserve structure
|
|
file_key = str(rel_path).replace('\\', '/')
|
|
if file_key not in seen_names:
|
|
dest = comp_dir / rel_path
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(f, dest)
|
|
seen_names.add(file_key)
|
|
if f.suffix in ('.cpp', '.c') and file_key not in cpp_files:
|
|
cpp_files.append(file_key)
|
|
|
|
# Scan newly copied headers for transitive includes.
|
|
# Use rglob so libs with `src/` layout (e.g. GxEPD2, ArduinoJson)
|
|
# are scanned recursively — otherwise their headers live under
|
|
# `src/`/subdirs and we'd miss every transitive include.
|
|
for lib_file in comp_dir.rglob('*.h'):
|
|
try:
|
|
lib_content = lib_file.read_text(encoding='utf-8', errors='ignore')
|
|
for th in self._detect_external_includes(lib_content):
|
|
if th not in resolved_headers:
|
|
headers_to_resolve.append(th)
|
|
except OSError:
|
|
pass
|
|
else:
|
|
logger.warning(f'[espidf] Library for <{header}> not found — build may fail')
|
|
|
|
if not found_any:
|
|
return [], {}
|
|
|
|
srcs_line = 'SRCS ' + ' '.join(f'"{f}"' for f in sorted(cpp_files)) if cpp_files else ''
|
|
|
|
# Generate INCLUDE_DIRS from the directory structure of copied files.
|
|
# Use PurePosixPath so paths stay forward-slashed on Windows — CMake
|
|
# parses backslashes as string escapes (e.g. "src\bitmaps" → invalid \b).
|
|
include_dirs: set[str] = {'.'}
|
|
for file_key in seen_names:
|
|
parent = str(PurePosixPath(file_key).parent)
|
|
if parent and parent != '.':
|
|
include_dirs.add(parent)
|
|
|
|
include_dirs_line = 'INCLUDE_DIRS ' + ' '.join(f'"{d}"' for d in sorted(include_dirs))
|
|
|
|
cmake_content = (
|
|
'# Auto-generated by Velxio — all user libraries merged into one component.\n'
|
|
'# Directory structure preserved for libraries like ArduinoJson with src/ layout.\n'
|
|
'idf_component_register(\n'
|
|
f' {srcs_line}\n'
|
|
f' {include_dirs_line}\n'
|
|
f' REQUIRES {arduino_comp_name}\n'
|
|
')\n'
|
|
)
|
|
(comp_dir / 'CMakeLists.txt').write_text(cmake_content, encoding='utf-8')
|
|
logger.info(
|
|
f'[espidf] user_libs_all: {len(cpp_files)} source files, '
|
|
f'{len(header_to_comp)} resolved headers'
|
|
)
|
|
return ['user_libs_all'], header_to_comp
|
|
|
|
def _detect_external_includes(self, code: str) -> list[str]:
|
|
"""Return library header names that are likely from external libraries."""
|
|
headers = []
|
|
for m in re.finditer(r'#\s*include\s*<([^>]+)>', code):
|
|
h = m.group(1)
|
|
if h in self._BUILTIN_HEADERS:
|
|
continue
|
|
# Skip paths with / (esp-idf internal headers like freertos/FreeRTOS.h)
|
|
if '/' in h:
|
|
continue
|
|
# Skip headers that look like esp-idf internal (prefix pattern)
|
|
if re.match(r'^(esp_|driver/|soc/|hal/|nvs|rom/)', h):
|
|
continue
|
|
headers.append(h)
|
|
return headers
|
|
|
|
def _find_library_for_header(self, header: str, libs_dir: Path) -> Path | None:
|
|
"""
|
|
Search libs_dir for a library that provides `header`.
|
|
Returns the source root of the library (root or src/ subdirectory).
|
|
"""
|
|
for lib_dir in sorted(libs_dir.iterdir()):
|
|
if not lib_dir.is_dir():
|
|
continue
|
|
for src_root in [lib_dir, lib_dir / 'src']:
|
|
if (src_root / header).exists():
|
|
return src_root
|
|
return None
|
|
|
|
def _create_idf_component(
|
|
self,
|
|
header: str,
|
|
src_root: Path,
|
|
user_libs_dir: Path,
|
|
arduino_comp_name: str,
|
|
) -> str:
|
|
"""
|
|
Create a proper ESP-IDF component for a library in user_libs_dir.
|
|
|
|
Each library becomes user_libs/<comp_name>/ with its own CMakeLists.txt
|
|
that calls idf_component_register(). This is the correct ESP-IDF way to
|
|
include third-party code and properly handles include paths so that
|
|
internal library includes like #include "utility/xyz.h" work correctly.
|
|
|
|
Returns the component directory name (used in REQUIRES of main).
|
|
"""
|
|
# Sanitise name: use the library directory name, not the header name
|
|
# src_root may be the library root OR lib/src/ — handle both cases
|
|
lib_dir_name = src_root.parent.name if src_root.name == 'src' else src_root.name
|
|
safe_name = re.sub(r'[^A-Za-z0-9_]', '_', lib_dir_name)
|
|
comp_dir = user_libs_dir / safe_name
|
|
comp_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Preserve the original library layout for actual buildable library code
|
|
# while skipping repo-only content such as examples, tests, and CI files.
|
|
lib_root = src_root.parent if src_root.name == 'src' else src_root
|
|
include_dirs: set[str] = {'.'}
|
|
cpp_files: set[str] = set()
|
|
copied_any = False
|
|
|
|
has_src_layout = (lib_root / 'src').is_dir()
|
|
excluded_dirs = {
|
|
'.git',
|
|
'.github',
|
|
'.vscode',
|
|
'__pycache__',
|
|
'docs',
|
|
'doc',
|
|
'example',
|
|
'examples',
|
|
'test',
|
|
'tests',
|
|
'extras',
|
|
'ci',
|
|
'fuzz',
|
|
'fuzzing',
|
|
'benchmark',
|
|
'benchmarks',
|
|
}
|
|
|
|
def should_include(relative_path: Path) -> bool:
|
|
parts = relative_path.parts
|
|
if any(part.lower() in excluded_dirs for part in parts[:-1]):
|
|
return False
|
|
if relative_path.suffix not in ('.h', '.hpp', '.c', '.cpp'):
|
|
return False
|
|
|
|
if has_src_layout:
|
|
return parts[0] == 'src' or len(parts) == 1
|
|
|
|
return len(parts) == 1 or parts[0].lower() == 'utility'
|
|
|
|
for f in lib_root.rglob('*'):
|
|
if not f.is_file():
|
|
continue
|
|
rel_path = f.relative_to(lib_root)
|
|
if not should_include(rel_path):
|
|
continue
|
|
dest = comp_dir / rel_path
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(f, dest)
|
|
copied_any = True
|
|
include_dirs.add(str(rel_path.parent).replace('\\', '/'))
|
|
if f.suffix in ('.cpp', '.c'):
|
|
cpp_files.add(str(rel_path).replace('\\', '/'))
|
|
|
|
if not copied_any:
|
|
raise ValueError(f'No buildable source files found in library {lib_dir_name}')
|
|
|
|
# Generate CMakeLists.txt for this component
|
|
include_dirs.discard('.')
|
|
ordered_include_dirs = ['.'] + sorted(d for d in include_dirs if d and d != '.')
|
|
|
|
if cpp_files:
|
|
srcs_line = 'SRCS ' + ' '.join(f'"{f}"' for f in sorted(cpp_files))
|
|
else:
|
|
srcs_line = '# header-only library'
|
|
|
|
include_dirs_line = 'INCLUDE_DIRS ' + ' '.join(
|
|
f'"{include_dir}"' for include_dir in ordered_include_dirs
|
|
)
|
|
|
|
cmake_content = (
|
|
f'# Auto-generated by Velxio for library: {lib_dir_name}\n'
|
|
f'idf_component_register(\n'
|
|
f' {srcs_line}\n'
|
|
f' {include_dirs_line}\n'
|
|
f' REQUIRES {arduino_comp_name}\n'
|
|
f')\n'
|
|
)
|
|
(comp_dir / 'CMakeLists.txt').write_text(cmake_content, encoding='utf-8')
|
|
|
|
logger.info(
|
|
f'[espidf] Created IDF component "{safe_name}" for <{header}>'
|
|
f' ({len(cpp_files)} source file(s))'
|
|
)
|
|
return safe_name
|
|
|
|
def _build_env(self, idf_target: str) -> dict:
|
|
"""Build environment dict for ESP-IDF subprocess."""
|
|
env = os.environ.copy()
|
|
env['IDF_PATH'] = self.idf_path
|
|
env['IDF_TARGET'] = idf_target
|
|
|
|
if self.has_arduino:
|
|
env['ARDUINO_ESP32_PATH'] = self.arduino_path
|
|
|
|
# On Windows, ESP-IDF uses its own Python venv
|
|
if os.name == 'nt':
|
|
py_venv = os.path.join(
|
|
os.path.dirname(self.idf_path), '..',
|
|
'python_env', 'idf4.4_py3.10_env'
|
|
)
|
|
# Also try the standard Espressif location
|
|
if not os.path.isdir(py_venv):
|
|
py_venv = r'C:\Espressif\python_env\idf4.4_py3.10_env'
|
|
|
|
if os.path.isdir(py_venv):
|
|
py_scripts = os.path.join(py_venv, 'Scripts')
|
|
env['PATH'] = py_scripts + os.pathsep + env.get('PATH', '')
|
|
env['VIRTUAL_ENV'] = py_venv
|
|
|
|
# Add ESP-IDF tools to PATH
|
|
tools_path = os.environ.get('IDF_TOOLS_PATH', r'C:\Users\David\.espressif')
|
|
if os.path.isdir(tools_path):
|
|
# Add all tool bin dirs
|
|
for tool_dir in Path(tools_path).glob('tools/*/*/bin'):
|
|
env['PATH'] = str(tool_dir) + os.pathsep + env['PATH']
|
|
# Xtensa toolchain
|
|
for tc_dir in Path(tools_path).glob('tools/xtensa-esp32-elf/*/xtensa-esp32-elf/bin'):
|
|
env['PATH'] = str(tc_dir) + os.pathsep + env['PATH']
|
|
for tc_dir in Path(tools_path).glob('tools/riscv32-esp-elf/*/riscv32-esp-elf/bin'):
|
|
env['PATH'] = str(tc_dir) + os.pathsep + env['PATH']
|
|
else:
|
|
# Linux/Docker: explicitly add toolchain bin dirs to PATH so cmake
|
|
# can find the cross-compilers even when the process wasn't started
|
|
# with export.sh (e.g. after a uvicorn restart or in tests).
|
|
tools_path = os.environ.get('IDF_TOOLS_PATH', os.path.expanduser('~/.espressif'))
|
|
env['IDF_TOOLS_PATH'] = tools_path
|
|
if os.path.isdir(tools_path):
|
|
extra_paths: list[str] = []
|
|
# Xtensa toolchain (ESP32, ESP32-S3)
|
|
for tc_dir in Path(tools_path).glob('tools/xtensa-esp32-elf/*/xtensa-esp32-elf/bin'):
|
|
extra_paths.append(str(tc_dir))
|
|
for tc_dir in Path(tools_path).glob('tools/xtensa-esp-elf/*/xtensa-esp-elf/bin'):
|
|
extra_paths.append(str(tc_dir))
|
|
# RISC-V toolchain (ESP32-C3)
|
|
for tc_dir in Path(tools_path).glob('tools/riscv32-esp-elf/*/riscv32-esp-elf/bin'):
|
|
extra_paths.append(str(tc_dir))
|
|
# ESP-IDF host tools (esptool, partition_table, etc.)
|
|
for tool_dir in Path(tools_path).glob('tools/*/*/bin'):
|
|
extra_paths.append(str(tool_dir))
|
|
if extra_paths:
|
|
env['PATH'] = os.pathsep.join(extra_paths) + os.pathsep + env.get('PATH', '')
|
|
|
|
return env
|
|
|
|
def _merge_flash_image(self, build_dir: Path, is_c3: bool) -> Path:
|
|
"""Merge bootloader + partitions + app into 4MB flash image."""
|
|
FLASH_SIZE = 4 * 1024 * 1024
|
|
flash = bytearray(b'\xff' * FLASH_SIZE)
|
|
|
|
bootloader_offset = 0x0000 if is_c3 else 0x1000
|
|
|
|
# ESP-IDF build output paths
|
|
bootloader = build_dir / 'bootloader' / 'bootloader.bin'
|
|
partitions = build_dir / 'partition_table' / 'partition-table.bin'
|
|
app = build_dir / 'velxio-sketch.bin'
|
|
|
|
if not app.exists():
|
|
# Try alternate names
|
|
for pattern in ['*.bin']:
|
|
candidates = [f for f in build_dir.glob(pattern)
|
|
if 'bootloader' not in f.name and 'partition' not in f.name]
|
|
if candidates:
|
|
app = candidates[0]
|
|
break
|
|
|
|
files_found = {
|
|
'bootloader': bootloader.exists(),
|
|
'partitions': partitions.exists(),
|
|
'app': app.exists(),
|
|
}
|
|
logger.info(f'[espidf] Merge files: {files_found}')
|
|
|
|
if not all(files_found.values()):
|
|
missing = [k for k, v in files_found.items() if not v]
|
|
raise FileNotFoundError(f'Missing binaries for merge: {missing}')
|
|
|
|
last_used = 0
|
|
for offset, path in [
|
|
(bootloader_offset, bootloader),
|
|
(0x8000, partitions),
|
|
(0x10000, app),
|
|
]:
|
|
data = path.read_bytes()
|
|
flash[offset:offset + len(data)] = data
|
|
last_used = max(last_used, offset + len(data))
|
|
logger.info(f'[espidf] Placed {path.name} at 0x{offset:04X} ({len(data)} bytes)')
|
|
|
|
# Trim the trailing 0xFF padding before serializing.
|
|
#
|
|
# Keeping the full 4 MB flash image here gives a ~5.5 MB base64 JSON
|
|
# response that nginx / Cloudflare can choke on (issue #101 — user
|
|
# saw "No response from server"). The frontend stores the trimmed
|
|
# bytes and the backend pads back to the QEMU flash size at the
|
|
# bridge layer right before mtd attach. Lossless: bytes after
|
|
# last_used are 0xFF by construction, so re-padding restores the
|
|
# original image byte-for-byte.
|
|
merged_path = build_dir / 'merged_flash.bin'
|
|
merged_path.write_bytes(bytes(flash[:last_used]))
|
|
logger.info(
|
|
f'[espidf] Merged flash image (trimmed): {merged_path.stat().st_size} bytes '
|
|
f'(would have been {FLASH_SIZE} bytes unpadded)'
|
|
)
|
|
return merged_path
|
|
|
|
async def compile(
|
|
self,
|
|
files: list[dict],
|
|
board_fqbn: str,
|
|
progress_callback: Optional[ProgressCallback] = None,
|
|
) -> dict:
|
|
"""
|
|
Compile Arduino sketch using ESP-IDF.
|
|
|
|
Returns dict compatible with ArduinoCLIService.compile():
|
|
success, binary_content (base64), binary_type, stdout, stderr, error
|
|
|
|
Build dir layout:
|
|
- With VELXIO_PERSISTENT_BUILD_DIR=1 (default): a per-target dir at
|
|
/var/lib/velxio-build/<idf_target>/project/ is reused across compiles.
|
|
ninja's incremental cache + ccache hits combine to bring warm
|
|
compiles down to ~5-30s.
|
|
- With VELXIO_PERSISTENT_BUILD_DIR=0: legacy tempfile.TemporaryDirectory
|
|
flow. Every compile rebuilds from scratch.
|
|
|
|
The caller (routes/compile.py:_compile_job) holds a per-target
|
|
asyncio.Lock for the duration of this call, so the persistent dir
|
|
is never accessed by two compiles at once.
|
|
|
|
progress_callback (optional): if provided, called from a worker
|
|
thread for every stdout/stderr line as cmake and ninja run. Used
|
|
by the async compile path to expose live build output to clients
|
|
polling /api/compile/status/{job_id}.
|
|
"""
|
|
if not self.available:
|
|
return {
|
|
'success': False,
|
|
'error': 'ESP-IDF toolchain not found. Set IDF_PATH environment variable.',
|
|
'stdout': '',
|
|
'stderr': '',
|
|
}
|
|
|
|
idf_target = self._idf_target(board_fqbn)
|
|
is_c3 = self._is_esp32c3(board_fqbn)
|
|
|
|
logger.info(f'[espidf] Compiling for {idf_target} (FQBN: {board_fqbn})')
|
|
logger.info(f'[espidf] Files: {[f["name"] for f in files]}')
|
|
|
|
if _USE_PERSISTENT_DIR:
|
|
project_dir = _prepare_persistent_project_dir(idf_target)
|
|
logger.info(f'[espidf] Using persistent build dir: {project_dir}')
|
|
return await self._compile_in_dir(
|
|
project_dir, files, idf_target, is_c3, progress_callback,
|
|
)
|
|
|
|
with tempfile.TemporaryDirectory(prefix='espidf_') as temp_dir:
|
|
project_dir = Path(temp_dir) / 'project'
|
|
shutil.copytree(_TEMPLATE_DIR, project_dir)
|
|
logger.info(f'[espidf] Using ephemeral build dir: {project_dir}')
|
|
return await self._compile_in_dir(
|
|
project_dir, files, idf_target, is_c3, progress_callback,
|
|
)
|
|
|
|
async def _compile_in_dir(
|
|
self,
|
|
project_dir: Path,
|
|
files: list[dict],
|
|
idf_target: str,
|
|
is_c3: bool,
|
|
progress_callback: Optional[ProgressCallback] = None,
|
|
) -> dict:
|
|
"""Inner compile body: writes sketch + libs into `project_dir`,
|
|
runs cmake + ninja, merges binaries. Caller is responsible for
|
|
creating `project_dir` (with the template tree already copied in)
|
|
and for managing its lifecycle (persistent vs tempfile).
|
|
"""
|
|
# Get sketch content
|
|
main_content = ''
|
|
for f in files:
|
|
if f['name'].endswith('.ino'):
|
|
main_content = f['content']
|
|
break
|
|
if not main_content and files:
|
|
main_content = files[0]['content']
|
|
|
|
# ── QEMU WiFi compatibility ──────────────────────────────────────
|
|
# QEMU's WiFi AP broadcasts "Velxio-GUEST" on channel 6.
|
|
# We normalize ANY user SSID → "Velxio-GUEST", enforce channel 6,
|
|
# and use open auth (empty password) so the connection always works.
|
|
# Detect WiFi BEFORE normalization so the flag reflects the original sketch.
|
|
has_wifi = self._detect_wifi_usage(main_content)
|
|
main_content = self._normalize_wifi_for_qemu(main_content)
|
|
|
|
if self.has_arduino:
|
|
# Arduino-as-component mode: copy sketch as .cpp
|
|
sketch_cpp = project_dir / 'main' / 'sketch.ino.cpp'
|
|
# Prepend Arduino.h + velxio_compat.h if not already included.
|
|
# velxio_compat.h shims arduino-esp32 3.x APIs (ledcAttach, …)
|
|
# onto the 2.0.17 toolchain we currently pin. See
|
|
# esp-idf-template/main/velxio_compat.h.
|
|
if '#include' not in main_content or 'Arduino.h' not in main_content:
|
|
main_content = (
|
|
'#include "Arduino.h"\n'
|
|
'#include "velxio_compat.h"\n' + main_content
|
|
)
|
|
else:
|
|
main_content = main_content.replace(
|
|
'#include "Arduino.h"',
|
|
'#include "Arduino.h"\n#include "velxio_compat.h"',
|
|
1,
|
|
)
|
|
sketch_cpp.write_text(main_content, encoding='utf-8')
|
|
|
|
# Copy additional files (.h, .cpp)
|
|
for f in files:
|
|
if not f['name'].endswith('.ino'):
|
|
(project_dir / 'main' / f['name']).write_text(
|
|
f['content'], encoding='utf-8'
|
|
)
|
|
|
|
# Remove the pure-C main to avoid conflict
|
|
main_c = project_dir / 'main' / 'main.c'
|
|
if main_c.exists():
|
|
main_c.unlink()
|
|
sketch_translated = project_dir / 'main' / 'sketch_translated.c'
|
|
if sketch_translated.exists():
|
|
sketch_translated.unlink()
|
|
|
|
# ── Resolve external Arduino libraries as IDF components ──────
|
|
# arduino-cli installs libraries in ~/Arduino/libraries/ but the
|
|
# ESP-IDF build system does not scan that path. We create a
|
|
# user_libs/ directory where each external library becomes a
|
|
# proper ESP-IDF component with its own CMakeLists.txt and
|
|
# INCLUDE_DIRS. The root CMakeLists.txt (template) adds user_libs
|
|
# to EXTRA_COMPONENT_DIRS so ESP-IDF discovers them automatically.
|
|
ext_headers = self._detect_external_includes(main_content)
|
|
component_names: list[str] = []
|
|
# arduino-esp32 component name (directory basename of ARDUINO_ESP32_PATH)
|
|
arduino_comp_name = Path(self.arduino_path).name if self.arduino_path else 'arduino-esp32'
|
|
|
|
if ext_headers:
|
|
user_libs_dir = project_dir / 'user_libs'
|
|
user_libs_dir.mkdir(exist_ok=True)
|
|
|
|
esp32_libs = Path(self.arduino_path) / 'libraries' if self.arduino_path else None
|
|
arduino_libs = self._find_arduino_libraries_dir()
|
|
|
|
component_names, _ = self._resolve_library_components(
|
|
ext_headers, arduino_libs, esp32_libs,
|
|
arduino_comp_name, user_libs_dir,
|
|
)
|
|
|
|
# Patch main/CMakeLists.txt — REQUIRES and INCLUDE_DIRS for user_libs_all.
|
|
# The single merged component means one entry covers all external headers.
|
|
if component_names: # always ['user_libs_all'] when any lib was found
|
|
cmake_path = project_dir / 'main' / 'CMakeLists.txt'
|
|
cmake_text = cmake_path.read_text(encoding='utf-8')
|
|
|
|
for old_req in [r'REQUIRES ${_arduino_comp_name}', f'REQUIRES {arduino_comp_name}']:
|
|
if old_req in cmake_text:
|
|
cmake_text = cmake_text.replace(
|
|
old_req, f'{old_req} user_libs_all'
|
|
)
|
|
break
|
|
|
|
cmake_text = cmake_text.replace(
|
|
'INCLUDE_DIRS "."',
|
|
'INCLUDE_DIRS "." "../user_libs/user_libs_all"',
|
|
)
|
|
|
|
cmake_path.write_text(cmake_text, encoding='utf-8')
|
|
logger.info('[espidf] Patched main CMakeLists: REQUIRES += user_libs_all, INCLUDE_DIRS += user_libs_all')
|
|
else:
|
|
# Pure ESP-IDF mode: translate sketch
|
|
translated = self._translate_sketch_to_espidf(main_content)
|
|
(project_dir / 'main' / 'sketch_translated.c').write_text(
|
|
translated, encoding='utf-8'
|
|
)
|
|
|
|
# Remove Arduino main.cpp to avoid conflict
|
|
main_cpp = project_dir / 'main' / 'main.cpp'
|
|
if main_cpp.exists():
|
|
main_cpp.unlink()
|
|
|
|
# Build using cmake + ninja (more portable than idf.py on Windows)
|
|
build_dir = project_dir / 'build'
|
|
build_dir.mkdir(exist_ok=True)
|
|
|
|
env = self._build_env(idf_target)
|
|
|
|
# Step 1: cmake configure
|
|
cmake_cmd = [
|
|
'cmake',
|
|
'-G', 'Ninja',
|
|
'-Wno-dev',
|
|
f'-DIDF_TARGET={idf_target}',
|
|
'-DCMAKE_BUILD_TYPE=Release',
|
|
f'-DSDKCONFIG_DEFAULTS={project_dir / "sdkconfig.defaults"}',
|
|
str(project_dir),
|
|
]
|
|
|
|
# ccache: ESP-IDF's tools/cmake/project.cmake enables ccache iff
|
|
# the CMake variable `CCACHE_ENABLE` is truthy. We don't go through
|
|
# `idf.py` (which would translate the env var for us), so wire it
|
|
# in here. Default ON; set IDF_CCACHE_ENABLE=0 in the env to
|
|
# bypass without rebuilding the image.
|
|
if os.environ.get('IDF_CCACHE_ENABLE', '1') not in ('0', 'false', 'False', ''):
|
|
cmake_cmd.append('-DCCACHE_ENABLE=1')
|
|
|
|
logger.info(f'[espidf] cmake: {" ".join(cmake_cmd)}')
|
|
|
|
def _run_cmake():
|
|
return _run_with_streaming(
|
|
cmake_cmd,
|
|
cwd=str(build_dir),
|
|
env=env,
|
|
timeout=120,
|
|
progress_callback=progress_callback,
|
|
)
|
|
|
|
try:
|
|
cmake_result = await asyncio.to_thread(_run_cmake)
|
|
except subprocess.TimeoutExpired:
|
|
return {
|
|
'success': False,
|
|
'error': 'ESP-IDF cmake configure timed out (120s)',
|
|
'stdout': '',
|
|
'stderr': '',
|
|
}
|
|
|
|
if cmake_result.returncode != 0:
|
|
logger.error(f'[espidf] cmake failed:\n{cmake_result.stderr}')
|
|
return {
|
|
'success': False,
|
|
'error': 'ESP-IDF cmake configure failed',
|
|
'stdout': cmake_result.stdout,
|
|
'stderr': cmake_result.stderr,
|
|
}
|
|
|
|
# Step 2: ninja build
|
|
ninja_cmd = ['ninja']
|
|
logger.info('[espidf] Building with ninja...')
|
|
|
|
# Cold ESP-IDF builds with external Arduino libraries (e.g. Adafruit
|
|
# BMP280 + BusIO + Unified Sensor → ~1480 build steps) regularly take
|
|
# 5-7 minutes on modest hardware. 300s used to cut them off at 98%;
|
|
# bump to 600s so first-run cold compiles complete. Subsequent
|
|
# builds reuse ninja's cache and finish in seconds.
|
|
NINJA_TIMEOUT_S = 600
|
|
|
|
def _run_ninja():
|
|
return _run_with_streaming(
|
|
ninja_cmd,
|
|
cwd=str(build_dir),
|
|
env=env,
|
|
timeout=NINJA_TIMEOUT_S,
|
|
progress_callback=progress_callback,
|
|
)
|
|
|
|
try:
|
|
ninja_result = await asyncio.to_thread(_run_ninja)
|
|
except subprocess.TimeoutExpired:
|
|
return {
|
|
'success': False,
|
|
'error': f'ESP-IDF build timed out ({NINJA_TIMEOUT_S}s)',
|
|
'stdout': '',
|
|
'stderr': '',
|
|
}
|
|
|
|
all_stdout = cmake_result.stdout + '\n' + ninja_result.stdout
|
|
all_stderr = cmake_result.stderr + '\n' + ninja_result.stderr
|
|
|
|
# Filter out expected but ugly warnings from stderr (e.g. absent git, cmake deprecation)
|
|
filtered_stderr_lines = []
|
|
for line in all_stderr.splitlines():
|
|
if 'fatal: not a git repository' in line:
|
|
continue
|
|
if 'CMake Deprecation Warning' in line:
|
|
continue
|
|
if 'Compatibility with CMake' in line:
|
|
continue
|
|
filtered_stderr_lines.append(line)
|
|
all_stderr = '\n'.join(filtered_stderr_lines)
|
|
|
|
if ninja_result.returncode != 0:
|
|
# Extract the actual compiler errors from ninja's stdout.
|
|
# Ninja prints failed job blocks in stdout:
|
|
# FAILED: path/to/file.obj
|
|
# <compiler command>
|
|
# sketch.ino.cpp:5:10: fatal error: DHT.h: No such file or directory
|
|
# compilation terminated.
|
|
# ninja: build stopped: subcommand failed.
|
|
stdout_lines = ninja_result.stdout.split('\n')
|
|
error_lines: list[str] = []
|
|
in_failed_block = False
|
|
for line in stdout_lines:
|
|
stripped = line.strip()
|
|
if stripped.startswith('FAILED:') or stripped == 'ninja: build stopped: subcommand failed.':
|
|
in_failed_block = True
|
|
error_lines.append(line)
|
|
continue
|
|
# Next [N/M] progress line ends the block
|
|
if in_failed_block and stripped.startswith('[') and '/' in stripped and ']' in stripped:
|
|
in_failed_block = False
|
|
if in_failed_block:
|
|
error_lines.append(line)
|
|
elif ': error:' in line or 'fatal error:' in line.lower():
|
|
# Explicit compiler error outside a FAILED block
|
|
error_lines.append(line)
|
|
|
|
extracted = '\n'.join(l for l in error_lines if l.strip())
|
|
|
|
# First non-FAILED, non-command error line → short summary for toolbar
|
|
summary = 'ESP-IDF build failed'
|
|
for l in error_lines:
|
|
s = l.strip()
|
|
if s and not s.startswith('FAILED:') and not s.startswith('ninja:') and not s.startswith('/') and 'error:' in s.lower():
|
|
summary = s
|
|
break
|
|
if summary == 'ESP-IDF build failed' and error_lines:
|
|
# Fall back to first non-empty error line
|
|
for l in error_lines:
|
|
if l.strip() and not l.strip().startswith('FAILED:'):
|
|
summary = l.strip()
|
|
break
|
|
|
|
# Put extracted errors in stderr so the console highlights them
|
|
combined_stderr = (extracted + '\n\n' + all_stderr).strip() if extracted else all_stderr
|
|
|
|
logger.error(f'[espidf] ninja build failed (stdout):\n{ninja_result.stdout[-4000:]}')
|
|
logger.error(f'[espidf] ninja build failed (stderr):\n{ninja_result.stderr[-2000:]}')
|
|
return {
|
|
'success': False,
|
|
'error': summary,
|
|
'stdout': all_stdout,
|
|
'stderr': combined_stderr,
|
|
}
|
|
|
|
# Step 3: Merge binaries into flash image
|
|
try:
|
|
merged_path = self._merge_flash_image(build_dir, is_c3)
|
|
except FileNotFoundError as exc:
|
|
return {
|
|
'success': False,
|
|
'error': f'Binary merge failed: {exc}',
|
|
'stdout': all_stdout,
|
|
'stderr': all_stderr,
|
|
}
|
|
|
|
binary_b64 = base64.b64encode(merged_path.read_bytes()).decode('ascii')
|
|
logger.info(f'[espidf] Compilation successful — {len(binary_b64) // 1024} KB (base64), has_wifi={has_wifi}')
|
|
|
|
return {
|
|
'success': True,
|
|
'hex_content': None,
|
|
'binary_content': binary_b64,
|
|
'binary_type': 'bin',
|
|
'has_wifi': has_wifi,
|
|
'stdout': all_stdout,
|
|
'stderr': all_stderr,
|
|
}
|
|
|
|
|
|
# Singleton instance
|
|
espidf_compiler = ESPIDFCompiler()
|