517 lines
22 KiB
Python
517 lines
22 KiB
Python
"""
|
|
EspLibManager — ESP32 emulation via lcgamboa libqemu-xtensa (.dll/.so).
|
|
|
|
Each call to start_instance() launches a fresh esp32_worker.py subprocess that
|
|
loads the DLL in its own address space. This enables multiple simultaneous
|
|
ESP32 emulations without DLL global-state conflicts.
|
|
|
|
subprocess.Popen is used instead of asyncio.create_subprocess_exec because on
|
|
Windows with uvicorn --reload the asyncio ProactorEventLoop child watcher is
|
|
not available, causing asyncio subprocess creation to raise NotImplementedError
|
|
(which has an empty str() and therefore appears as a blank error message).
|
|
Background daemon threads read stdout/stderr and dispatch events back to the
|
|
asyncio event loop via asyncio.run_coroutine_threadsafe().
|
|
|
|
Public API is identical to the previous in-process version so simulation.py
|
|
requires no changes.
|
|
|
|
Activation: set environment variable QEMU_ESP32_LIB to the library path, or
|
|
place libqemu-xtensa.dll (Windows) / libqemu-xtensa.so (Linux) beside this
|
|
module.
|
|
|
|
Events emitted via callback(event_type, data):
|
|
system {event: 'booting'|'booted'|'crash'|'reboot'}
|
|
serial_output {data: str, uart: int} — UART 0/1/2 text
|
|
gpio_change {pin: int, state: int} — real GPIO number (0-39)
|
|
gpio_dir {pin: int, dir: int} — 0=input 1=output
|
|
i2c_event {bus: int, addr: int, event: int, response: int}
|
|
spi_event {bus: int, event: int, response: int}
|
|
rmt_event {channel: int, config0: int, value: int,
|
|
level0: int, dur0: int, level1: int, dur1: int}
|
|
ws2812_update {channel: int, pixels: list[{r,g,b}]}
|
|
ledc_update {channel: int, duty: int, duty_pct: float}
|
|
error {message: str}
|
|
"""
|
|
import asyncio
|
|
import base64
|
|
import dataclasses
|
|
import json
|
|
import logging
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from typing import Callable, Awaitable
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# ── Library path detection ────────────────────────────────────────────────────
|
|
_SERVICES_DIR = pathlib.Path(__file__).parent
|
|
|
|
# Xtensa library (ESP32, ESP32-S3)
|
|
_LIB_XTENSA_NAME = 'libqemu-xtensa.dll' if sys.platform == 'win32' else 'libqemu-xtensa.so'
|
|
_DEFAULT_LIB_XTENSA = str(_SERVICES_DIR / _LIB_XTENSA_NAME)
|
|
LIB_PATH: str = os.environ.get('QEMU_ESP32_LIB', '') or (
|
|
_DEFAULT_LIB_XTENSA if os.path.isfile(_DEFAULT_LIB_XTENSA) else ''
|
|
)
|
|
|
|
# RISC-V library (ESP32-C3)
|
|
_LIB_RISCV_NAME = 'libqemu-riscv32.dll' if sys.platform == 'win32' else 'libqemu-riscv32.so'
|
|
_DEFAULT_LIB_RISCV = str(_SERVICES_DIR / _LIB_RISCV_NAME)
|
|
LIB_RISCV_PATH: str = os.environ.get('QEMU_RISCV32_LIB', '') or (
|
|
_DEFAULT_LIB_RISCV if os.path.isfile(_DEFAULT_LIB_RISCV) else ''
|
|
)
|
|
|
|
_WORKER_SCRIPT = _SERVICES_DIR / 'esp32_worker.py'
|
|
|
|
EventCallback = Callable[[str, dict], Awaitable[None]]
|
|
|
|
# lcgamboa machine names and which DLL each board requires
|
|
_MACHINE: dict[str, str] = {
|
|
'esp32': 'esp32-picsimlab',
|
|
'esp32-s3': 'esp32s3-picsimlab',
|
|
'esp32-c3': 'esp32c3-picsimlab',
|
|
'xiao-esp32-c3': 'esp32c3-picsimlab',
|
|
'aitewinrobot-esp32c3-supermini': 'esp32c3-picsimlab',
|
|
}
|
|
|
|
# Board types that require the RISC-V library instead of the Xtensa one
|
|
_RISCV_BOARDS = {'esp32-c3', 'xiao-esp32-c3', 'aitewinrobot-esp32c3-supermini'}
|
|
|
|
|
|
# ── UART buffer ───────────────────────────────────────────────────────────────
|
|
|
|
class _UartBuffer:
|
|
"""Accumulate bytes per UART channel, flush on newline or size limit."""
|
|
|
|
def __init__(self, uart_id: int, flush_size: int = 256):
|
|
self.uart_id = uart_id
|
|
self.flush_size = flush_size
|
|
self._buf: bytearray = bytearray()
|
|
self._lock = threading.Lock()
|
|
|
|
def feed(self, byte_val: int) -> str | None:
|
|
"""Add one byte. Returns decoded string when a flush occurs, else None."""
|
|
with self._lock:
|
|
self._buf.append(byte_val)
|
|
# Flush on newline, carriage return, period, or max size
|
|
# This ensures progress dots '...' don't buffer endlessly.
|
|
if byte_val in (ord('\n'), ord('\r'), ord('.')) or len(self._buf) >= self.flush_size:
|
|
text = self._buf.decode('utf-8', errors='replace')
|
|
self._buf.clear()
|
|
return text
|
|
return None
|
|
|
|
def flush(self) -> str | None:
|
|
"""Force-flush any remaining bytes."""
|
|
with self._lock:
|
|
if self._buf:
|
|
text = self._buf.decode('utf-8', errors='replace')
|
|
self._buf.clear()
|
|
return text
|
|
return None
|
|
|
|
|
|
# ── Per-instance state ────────────────────────────────────────────────────────
|
|
|
|
@dataclasses.dataclass
|
|
class _WorkerInstance:
|
|
process: subprocess.Popen
|
|
stdin_lock: threading.Lock
|
|
callback: EventCallback
|
|
board_type: str
|
|
uart_bufs: dict[int, _UartBuffer]
|
|
threads: list[threading.Thread]
|
|
loop: asyncio.AbstractEventLoop
|
|
running: bool = True
|
|
wifi_enabled: bool = False
|
|
wifi_hostfwd_port: int = 0
|
|
|
|
|
|
# ── Manager ───────────────────────────────────────────────────────────────────
|
|
|
|
class EspLibManager:
|
|
"""
|
|
Manages ESP32 emulation — each instance is a separate Python subprocess
|
|
that loads libqemu-xtensa in its own address space.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self._instances: dict[str, _WorkerInstance] = {}
|
|
self._instances_lock = threading.Lock()
|
|
|
|
# ── Availability ──────────────────────────────────────────────────────────
|
|
|
|
@staticmethod
|
|
def is_available() -> bool:
|
|
"""Returns True if the Xtensa DLL is present (minimum for ESP32/ESP32-S3)."""
|
|
return (
|
|
bool(LIB_PATH)
|
|
and os.path.isfile(LIB_PATH)
|
|
and _WORKER_SCRIPT.exists()
|
|
)
|
|
|
|
@staticmethod
|
|
def is_riscv_available() -> bool:
|
|
"""Returns True if the RISC-V DLL is present (required for ESP32-C3)."""
|
|
return bool(LIB_RISCV_PATH) and os.path.isfile(LIB_RISCV_PATH)
|
|
|
|
# ── Public API ────────────────────────────────────────────────────────────
|
|
|
|
def get_instance(self, client_id: str) -> _WorkerInstance | None:
|
|
"""Return the worker instance for a client, or None."""
|
|
with self._instances_lock:
|
|
return self._instances.get(client_id)
|
|
|
|
async def start_instance(
|
|
self,
|
|
client_id: str,
|
|
board_type: str,
|
|
callback: EventCallback,
|
|
firmware_b64: str | None = None,
|
|
sensors: list | None = None,
|
|
wifi_enabled: bool = False,
|
|
wifi_hostfwd_port: int = 0,
|
|
) -> None:
|
|
# Stop any existing instance for this client_id first
|
|
if client_id in self._instances:
|
|
logger.info('start_instance: %s already running — stopping first', client_id)
|
|
await self.stop_instance(client_id)
|
|
|
|
if not firmware_b64:
|
|
logger.info('start_instance %s: no firmware — skipping worker launch', client_id)
|
|
return
|
|
|
|
machine = _MACHINE.get(board_type, 'esp32-picsimlab')
|
|
lib_path = LIB_RISCV_PATH if board_type in _RISCV_BOARDS else LIB_PATH
|
|
config = json.dumps({
|
|
'lib_path': lib_path,
|
|
'firmware_b64': firmware_b64,
|
|
'machine': machine,
|
|
'sensors': sensors or [],
|
|
'wifi_enabled': wifi_enabled,
|
|
'wifi_hostfwd_port': wifi_hostfwd_port,
|
|
})
|
|
|
|
logger.info('Launching esp32_worker for %s (machine=%s, script=%s, python=%s)',
|
|
client_id, machine, _WORKER_SCRIPT, sys.executable)
|
|
try:
|
|
await callback('system', {'event': 'booting'})
|
|
except Exception as exc:
|
|
logger.warning('start_instance %s: booting event delivery failed: %s', client_id, exc)
|
|
|
|
try:
|
|
proc = subprocess.Popen(
|
|
[sys.executable, str(_WORKER_SCRIPT)],
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
except Exception as exc:
|
|
logger.error('Failed to launch esp32_worker for %s: %r', client_id, exc, exc_info=True)
|
|
await callback('error', {'message': f'Worker launch failed: {type(exc).__name__}: {exc}'})
|
|
return
|
|
|
|
# Write config as the first stdin line
|
|
try:
|
|
assert proc.stdin is not None
|
|
assert proc.stdout is not None
|
|
assert proc.stderr is not None
|
|
proc.stdin.write((config + '\n').encode())
|
|
proc.stdin.flush()
|
|
except Exception as exc:
|
|
logger.error('Failed to write config to esp32_worker %s: %r', client_id, exc)
|
|
proc.kill()
|
|
return
|
|
|
|
loop = asyncio.get_running_loop()
|
|
inst = _WorkerInstance(
|
|
process = proc,
|
|
stdin_lock = threading.Lock(),
|
|
callback = callback,
|
|
board_type = board_type,
|
|
uart_bufs = {0: _UartBuffer(0), 1: _UartBuffer(1), 2: _UartBuffer(2)},
|
|
threads = [],
|
|
loop = loop,
|
|
wifi_enabled = wifi_enabled,
|
|
wifi_hostfwd_port = wifi_hostfwd_port,
|
|
)
|
|
|
|
with self._instances_lock:
|
|
self._instances[client_id] = inst
|
|
|
|
t_out = threading.Thread(
|
|
target=self._thread_read_stdout,
|
|
args=(inst, client_id),
|
|
daemon=True,
|
|
name=f'worker-stdout-{client_id[:8]}',
|
|
)
|
|
t_err = threading.Thread(
|
|
target=self._thread_read_stderr,
|
|
args=(inst, client_id),
|
|
daemon=True,
|
|
name=f'worker-stderr-{client_id[:8]}',
|
|
)
|
|
inst.threads = [t_out, t_err]
|
|
t_out.start()
|
|
t_err.start()
|
|
|
|
async def stop_instance(self, client_id: str) -> None:
|
|
with self._instances_lock:
|
|
inst = self._instances.pop(client_id, None)
|
|
if not inst:
|
|
return
|
|
inst.running = False
|
|
|
|
# Flush any remaining UART bytes
|
|
for buf in inst.uart_bufs.values():
|
|
text = buf.flush()
|
|
if text:
|
|
try:
|
|
await inst.callback('serial_output', {'data': text, 'uart': buf.uart_id})
|
|
except Exception:
|
|
pass
|
|
|
|
# Ask the worker to stop gracefully
|
|
self._write_cmd(inst, {'cmd': 'stop'})
|
|
|
|
# Wait for clean shutdown in a thread to avoid blocking the event loop
|
|
def _wait_and_kill():
|
|
try:
|
|
inst.process.wait(timeout=6.0)
|
|
except subprocess.TimeoutExpired:
|
|
logger.warning('Worker %s did not stop in 6 s — killing', client_id)
|
|
inst.process.kill()
|
|
inst.process.wait()
|
|
except Exception as exc:
|
|
logger.debug('stop_instance %s wait: %s', client_id, exc)
|
|
|
|
await asyncio.to_thread(_wait_and_kill)
|
|
logger.info('WorkerInstance %s shut down', client_id)
|
|
|
|
def load_firmware(self, client_id: str, firmware_b64: str) -> None:
|
|
"""Hot-reload firmware: stop the current worker and start a fresh one."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if not inst:
|
|
logger.warning('load_firmware: no instance for %s', client_id)
|
|
return
|
|
board_type = inst.board_type
|
|
callback = inst.callback
|
|
|
|
async def _restart() -> None:
|
|
await self.stop_instance(client_id)
|
|
await asyncio.sleep(0.1)
|
|
await self.start_instance(client_id, board_type, callback, firmware_b64)
|
|
|
|
asyncio.ensure_future(_restart())
|
|
|
|
# ── GPIO / ADC / UART control ─────────────────────────────────────────────
|
|
|
|
def set_pin_state(self, client_id: str, pin: int | str, state_val: int) -> None:
|
|
"""Drive a GPIO input pin (real GPIO number 0-39)."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if inst and inst.running and inst.process.returncode is None:
|
|
self._write_cmd(inst, {'cmd': 'set_pin', 'pin': int(pin), 'value': state_val})
|
|
|
|
async def send_serial_bytes(
|
|
self, client_id: str, data: bytes, uart_id: int = 0
|
|
) -> None:
|
|
"""Send bytes to ESP32 UART RX (uart_id 0/1/2)."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if inst and inst.running and inst.process.returncode is None:
|
|
self._write_cmd(inst, {
|
|
'cmd': 'uart_send',
|
|
'uart': uart_id,
|
|
'data': base64.b64encode(data).decode(),
|
|
})
|
|
|
|
def set_adc(self, client_id: str, channel: int, millivolts: int) -> None:
|
|
"""Set ADC channel voltage in millivolts (0-3300 mV)."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if inst and inst.running and inst.process.returncode is None:
|
|
self._write_cmd(inst, {'cmd': 'set_adc', 'channel': channel, 'millivolts': millivolts})
|
|
|
|
def set_adc_raw(self, client_id: str, channel: int, raw: int) -> None:
|
|
"""Set ADC channel with a 12-bit raw value (0-4095)."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if inst and inst.running and inst.process.returncode is None:
|
|
self._write_cmd(inst, {'cmd': 'set_adc_raw', 'channel': channel, 'raw': raw})
|
|
|
|
# ── I2C / SPI device simulation ───────────────────────────────────────────
|
|
|
|
def set_i2c_response(self, client_id: str, addr: int, response_byte: int) -> None:
|
|
"""Configure the byte returned when ESP32 reads from I2C address addr."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if inst and inst.running and inst.process.returncode is None:
|
|
self._write_cmd(inst, {'cmd': 'set_i2c_response', 'addr': addr,
|
|
'response': response_byte & 0xFF})
|
|
|
|
def set_spi_response(self, client_id: str, response_byte: int) -> None:
|
|
"""Configure the MISO byte returned during SPI transfers."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if inst and inst.running and inst.process.returncode is None:
|
|
self._write_cmd(inst, {'cmd': 'set_spi_response', 'response': response_byte & 0xFF})
|
|
|
|
# ── Generic sensor protocol offloading ──────────────────────────────────
|
|
|
|
def sensor_attach(self, client_id: str, sensor_type: str, pin: int,
|
|
properties: dict) -> None:
|
|
"""Register a sensor on a GPIO pin — the worker handles its protocol."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if inst and inst.running and inst.process.returncode is None:
|
|
self._write_cmd(inst, {
|
|
'cmd': 'sensor_attach', 'sensor_type': sensor_type,
|
|
'pin': pin, **{k: v for k, v in properties.items()
|
|
if k not in ('sensor_type', 'pin')},
|
|
})
|
|
|
|
def sensor_update(self, client_id: str, pin: int,
|
|
properties: dict) -> None:
|
|
"""Update a sensor's properties (temperature, humidity, distance…)."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if inst and inst.running and inst.process.returncode is None:
|
|
self._write_cmd(inst, {
|
|
'cmd': 'sensor_update', 'pin': pin,
|
|
**{k: v for k, v in properties.items() if k != 'pin'},
|
|
})
|
|
|
|
def sensor_detach(self, client_id: str, pin: int) -> None:
|
|
"""Remove a sensor from a GPIO pin."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if inst and inst.running and inst.process.returncode is None:
|
|
self._write_cmd(inst, {'cmd': 'sensor_detach', 'pin': pin})
|
|
|
|
# ── LEDC polling (no-op: worker polls automatically) ─────────────────────
|
|
|
|
async def poll_ledc(self, client_id: str) -> None:
|
|
"""No-op: LEDC polling runs inside the worker subprocess."""
|
|
|
|
# ── Status ────────────────────────────────────────────────────────────────
|
|
|
|
def get_status(self, client_id: str) -> dict:
|
|
"""Return runtime status for a client instance."""
|
|
with self._instances_lock:
|
|
inst = self._instances.get(client_id)
|
|
if not inst:
|
|
return {'running': False}
|
|
return {
|
|
'running': True,
|
|
'alive': inst.process.returncode is None,
|
|
'board': inst.board_type,
|
|
}
|
|
|
|
# ── Internal helpers ──────────────────────────────────────────────────────
|
|
|
|
def _write_cmd(self, inst: _WorkerInstance, cmd: dict) -> None:
|
|
"""Write one JSON command line to the worker's stdin (thread-safe)."""
|
|
try:
|
|
with inst.stdin_lock:
|
|
assert inst.process.stdin is not None
|
|
inst.process.stdin.write((json.dumps(cmd) + '\n').encode())
|
|
inst.process.stdin.flush()
|
|
except Exception as exc:
|
|
logger.debug('_write_cmd failed: %s', exc)
|
|
|
|
def _thread_read_stdout(self, inst: _WorkerInstance, client_id: str) -> None:
|
|
"""
|
|
Background daemon thread: reads JSON event lines from the worker's
|
|
stdout and dispatches them to the asyncio callback via
|
|
run_coroutine_threadsafe().
|
|
"""
|
|
try:
|
|
assert inst.process.stdout is not None
|
|
for raw in inst.process.stdout:
|
|
raw = raw.strip()
|
|
if not raw:
|
|
continue
|
|
# With -nographic, serial0 is connected to the stdio mux so
|
|
# qemu_chr_fe_write() writes the raw UART byte to fd 1 just
|
|
# before picsimlab_uart_tx_event emits the JSON line. Strip
|
|
# any prefix bytes before the JSON object marker.
|
|
idx = raw.find(b'{"type":')
|
|
if idx > 0:
|
|
raw = raw[idx:]
|
|
elif idx < 0:
|
|
logger.debug('[%s] ignoring non-JSON worker line: %s',
|
|
client_id, raw[:200])
|
|
continue
|
|
try:
|
|
event = json.loads(raw)
|
|
except Exception:
|
|
logger.debug('[%s] bad JSON from worker: %s', client_id, raw[:200])
|
|
continue
|
|
|
|
etype = event.pop('type', '')
|
|
|
|
if etype == 'uart_tx':
|
|
uart_id = event.get('uart', 0)
|
|
byte_val = event.get('byte', 0)
|
|
buf = inst.uart_bufs.get(uart_id)
|
|
if buf:
|
|
text = buf.feed(byte_val)
|
|
if text:
|
|
self._dispatch(inst, 'serial_output', {
|
|
'data': text, 'uart': uart_id,
|
|
})
|
|
# Parse WiFi/BLE status from UART0 output
|
|
if uart_id == 0 and inst.wifi_enabled:
|
|
from app.services.wifi_status_parser import parse_serial_text
|
|
wifi_evts, ble_evts = parse_serial_text(text)
|
|
for we in wifi_evts:
|
|
self._dispatch(inst, 'wifi_status', dict(we))
|
|
for be in ble_evts:
|
|
self._dispatch(inst, 'ble_status', dict(be))
|
|
elif etype:
|
|
self._dispatch(inst, etype, event)
|
|
|
|
except Exception as exc:
|
|
if inst.running:
|
|
logger.debug('[%s] _thread_read_stdout ended: %s', client_id, exc)
|
|
finally:
|
|
rc = inst.process.returncode
|
|
if rc is None:
|
|
# process stdout closed but process still running
|
|
inst.process.poll()
|
|
rc = inst.process.returncode
|
|
if inst.running and rc is not None:
|
|
logger.warning('[%s] worker exited unexpectedly (code %s)', client_id, rc)
|
|
self._dispatch(inst, 'system', {
|
|
'event': 'crash',
|
|
'reason': 'worker_exit',
|
|
'code': rc,
|
|
})
|
|
|
|
def _thread_read_stderr(self, inst: _WorkerInstance, client_id: str) -> None:
|
|
"""Forward worker stderr to backend logs at DEBUG level."""
|
|
try:
|
|
assert inst.process.stderr is not None
|
|
for line in inst.process.stderr:
|
|
logger.info('[worker:%s] %s', client_id,
|
|
line.decode(errors='replace').rstrip())
|
|
except Exception:
|
|
pass
|
|
|
|
def _dispatch(self, inst: _WorkerInstance, etype: str, data: dict) -> None:
|
|
"""Schedule inst.callback(etype, data) on the instance's event loop."""
|
|
try:
|
|
coro = inst.callback(etype, data)
|
|
# callback is always an async def, so coro is a Coroutine at runtime.
|
|
# run_coroutine_threadsafe typing requires Coroutine, not Awaitable.
|
|
asyncio.run_coroutine_threadsafe(coro, inst.loop) # type: ignore[arg-type]
|
|
except Exception as exc:
|
|
logger.debug('_dispatch %s failed: %s', etype, exc)
|
|
|
|
|
|
esp_lib_manager = EspLibManager()
|