diff --git a/backend/app/api/routes/compile.py b/backend/app/api/routes/compile.py index acc24b4a..4fae563f 100644 --- a/backend/app/api/routes/compile.py +++ b/backend/app/api/routes/compile.py @@ -152,11 +152,21 @@ def _resolve_files(request: CompileRequest) -> list[dict[str, str]]: async def _run_compile( request: CompileRequest, files: list[dict[str, str]], + progress_callback: Any = None, ) -> CompileResponse: - """Do the actual compile (ESP-IDF for esp32:*, arduino-cli otherwise).""" + """Do the actual compile (ESP-IDF for esp32:*, arduino-cli otherwise). + + `progress_callback`, if provided, receives every stdout/stderr line as + cmake + ninja run. Wired into the async compile path so the live build + output is exposed via /api/compile/status/{job_id}'s `stdout` field. + AVR / RP2040 builds via arduino-cli don't surface progress yet — those + typically finish in seconds anyway. + """ if request.board_fqbn.startswith("esp32:") and espidf_compiler.available: logger.info(f"[compile] Using ESP-IDF for {request.board_fqbn}") - result = await espidf_compiler.compile(files, request.board_fqbn) + result = await espidf_compiler.compile( + files, request.board_fqbn, progress_callback=progress_callback, + ) return CompileResponse( success=result["success"], hex_content=result.get("hex_content"), @@ -241,11 +251,34 @@ async def _compile_job( `state=pending` while waiting on either gate; transitions to `running` only once the actual build is about to start, so clients polling /compile/status see an accurate snapshot of where their job is. + + Live build output is appended to COMPILE_JOBS[job_id]['stdout_buffer'] + line-by-line as cmake + ninja emit it, so /compile/status responses + stream a growing log instead of returning everything at the end. """ started = time.monotonic() job = COMPILE_JOBS[job_id] started_at = job["started_at"] job_key = job.get("key") + + # Live stdout buffer — written from a worker thread (espidf_compiler + # drain threads). dict[str].update with a single str assignment is GIL- + # protected so we don't need an explicit lock; the polling endpoint + # reads the same field. + COMPILE_JOBS[job_id]["stdout_buffer"] = "" + + def on_progress_line(line: str) -> None: + # Cap buffer at 256 KB so a runaway build can't OOM the process. + # Keep the tail (most recent output) — that's what the user wants + # to see anyway. + current = COMPILE_JOBS.get(job_id) + if current is None: + return + new = (current.get("stdout_buffer", "") or "") + line + if len(new) > 262_144: + new = new[-262_144:] + current["stdout_buffer"] = new + try: async with _COMPILE_SEMAPHORE: async with _target_lock(request.board_fqbn): @@ -255,13 +288,19 @@ async def _compile_job( logger.info(f"[compile] job {job_id} purged before run; skipping") return COMPILE_JOBS[job_id]["state"] = "running" - response = await _run_compile(request, files) + response = await _run_compile( + request, files, progress_callback=on_progress_line, + ) COMPILE_JOBS[job_id] = { "state": "done", "started_at": started_at, "finished_at": time.time(), "result": response.model_dump(), "key": job_key, + # Preserve the streamed buffer post-completion so a late poll + # still has access to the live log (clients usually display + # result.stdout once state=done, but having both costs nothing). + "stdout_buffer": COMPILE_JOBS.get(job_id, {}).get("stdout_buffer", ""), } error_kind = ( None if response.success @@ -284,6 +323,7 @@ async def _compile_job( "finished_at": time.time(), "error": str(exc)[:500], "key": job_key, + "stdout_buffer": COMPILE_JOBS.get(job_id, {}).get("stdout_buffer", ""), } await _record_async_metric( user_id=user_id, @@ -355,6 +395,11 @@ class CompileStatusResponse(BaseModel): state: str # 'pending' | 'running' | 'done' | 'error' started_at: float finished_at: float | None = None + # Live build output. Grows line-by-line during state=running so the + # frontend can stream it into the compilation console instead of + # waiting for everything to land at the end. Capped at 256 KB + # (most recent tail kept). + stdout: str = "" result: CompileResponse | None = None error: str | None = None @@ -406,7 +451,14 @@ async def compile_start( @router.get("/status/{job_id}", response_model=CompileStatusResponse) async def compile_status(job_id: str): - """Poll the status of an async compile job submitted via /compile/start.""" + """Poll the status of an async compile job submitted via /compile/start. + + `stdout` carries live cmake + ninja output captured line-by-line as + the build runs. Clients should poll every 1-2s and re-render the + full string each time (or compute a length delta). Once state=done, + `result.stdout` carries the same content too — both are kept so a + late-arriving poll always has the log available. + """ job = COMPILE_JOBS.get(job_id) if not job: raise HTTPException(status_code=404, detail="job not found or expired") @@ -414,6 +466,7 @@ async def compile_status(job_id: str): state=job["state"], started_at=job["started_at"], finished_at=job.get("finished_at"), + stdout=job.get("stdout_buffer", "") or "", result=job.get("result"), error=job.get("error"), ) diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index 9f6b6e60..a313c985 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -22,7 +22,10 @@ import re import shutil import subprocess import tempfile +import threading +from dataclasses import dataclass from pathlib import Path, PurePosixPath +from typing import Callable, Optional logger = logging.getLogger(__name__) @@ -73,6 +76,96 @@ def _idf_version_signature() -> str: return '|'.join(parts) +# Type for live progress callback. Called from a worker thread for every +# stdout/stderr line as the build runs. Implementations should be cheap and +# thread-safe (callers commonly stash lines into a dict shared with the main +# event loop). Exceptions raised from the callback are swallowed so a faulty +# UI hook can never break the build. +ProgressCallback = Callable[[str], None] + + +@dataclass +class _RunResult: + """Drop-in replacement for the fields we read off subprocess.CompletedProcess.""" + returncode: int + stdout: str + stderr: str + + +def _run_with_streaming( + cmd: list[str], + *, + cwd: str, + env: dict[str, str], + timeout: float, + progress_callback: Optional[ProgressCallback], +) -> _RunResult: + """Run `cmd` synchronously and stream stdout + stderr line-by-line. + + Behaves like subprocess.run(capture_output=True, text=True) but invokes + `progress_callback(line)` for every line as it arrives. When + progress_callback is None this falls back to a single subprocess.run call + so we don't pay the threading cost on the unit-test path that doesn't + care about live output. + + Raises subprocess.TimeoutExpired on timeout (matches the existing flow). + """ + if progress_callback is None: + cp = subprocess.run( + cmd, cwd=cwd, env=env, capture_output=True, text=True, timeout=timeout, + ) + return _RunResult(returncode=cp.returncode, stdout=cp.stdout, stderr=cp.stderr) + + proc = subprocess.Popen( + cmd, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, # line-buffered + ) + stdout_lines: list[str] = [] + stderr_lines: list[str] = [] + + def _drain(stream, sink: list[str]) -> None: + try: + for line in iter(stream.readline, ''): + sink.append(line) + try: + progress_callback(line) + except Exception: + # A faulty progress sink must never break the build. + pass + finally: + try: + stream.close() + except Exception: + pass + + t_out = threading.Thread(target=_drain, args=(proc.stdout, stdout_lines), daemon=True) + t_err = threading.Thread(target=_drain, args=(proc.stderr, stderr_lines), daemon=True) + t_out.start() + t_err.start() + + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + # Give drain threads a chance to flush before we raise. + t_out.join(timeout=2) + t_err.join(timeout=2) + raise + + t_out.join(timeout=5) + t_err.join(timeout=5) + return _RunResult( + returncode=proc.returncode, + stdout=''.join(stdout_lines), + stderr=''.join(stderr_lines), + ) + + def _prepare_persistent_project_dir(idf_target: str) -> Path: """Return the path to a per-target persistent project dir, materialising it from the template on first use and resetting the per-compile parts @@ -883,7 +976,12 @@ class ESPIDFCompiler: ) return merged_path - async def compile(self, files: list[dict], board_fqbn: str) -> dict: + async def compile( + self, + files: list[dict], + board_fqbn: str, + progress_callback: Optional[ProgressCallback] = None, + ) -> dict: """ Compile Arduino sketch using ESP-IDF. @@ -901,6 +999,11 @@ class ESPIDFCompiler: The caller (routes/compile.py:_compile_job) holds a per-target asyncio.Lock for the duration of this call, so the persistent dir is never accessed by two compiles at once. + + progress_callback (optional): if provided, called from a worker + thread for every stdout/stderr line as cmake and ninja run. Used + by the async compile path to expose live build output to clients + polling /api/compile/status/{job_id}. """ if not self.available: return { @@ -919,13 +1022,17 @@ class ESPIDFCompiler: if _USE_PERSISTENT_DIR: project_dir = _prepare_persistent_project_dir(idf_target) logger.info(f'[espidf] Using persistent build dir: {project_dir}') - return await self._compile_in_dir(project_dir, files, idf_target, is_c3) + return await self._compile_in_dir( + project_dir, files, idf_target, is_c3, progress_callback, + ) with tempfile.TemporaryDirectory(prefix='espidf_') as temp_dir: project_dir = Path(temp_dir) / 'project' shutil.copytree(_TEMPLATE_DIR, project_dir) logger.info(f'[espidf] Using ephemeral build dir: {project_dir}') - return await self._compile_in_dir(project_dir, files, idf_target, is_c3) + return await self._compile_in_dir( + project_dir, files, idf_target, is_c3, progress_callback, + ) async def _compile_in_dir( self, @@ -933,6 +1040,7 @@ class ESPIDFCompiler: files: list[dict], idf_target: str, is_c3: bool, + progress_callback: Optional[ProgressCallback] = None, ) -> dict: """Inner compile body: writes sketch + libs into `project_dir`, runs cmake + ninja, merges binaries. Caller is responsible for @@ -1075,13 +1183,12 @@ class ESPIDFCompiler: logger.info(f'[espidf] cmake: {" ".join(cmake_cmd)}') def _run_cmake(): - return subprocess.run( + return _run_with_streaming( cmake_cmd, cwd=str(build_dir), - capture_output=True, - text=True, env=env, timeout=120, + progress_callback=progress_callback, ) try: @@ -1115,13 +1222,12 @@ class ESPIDFCompiler: NINJA_TIMEOUT_S = 600 def _run_ninja(): - return subprocess.run( + return _run_with_streaming( ninja_cmd, cwd=str(build_dir), - capture_output=True, - text=True, env=env, timeout=NINJA_TIMEOUT_S, + progress_callback=progress_callback, ) try: diff --git a/frontend/src/components/editor/EditorToolbar.tsx b/frontend/src/components/editor/EditorToolbar.tsx index 51954573..78d4333a 100644 --- a/frontend/src/components/editor/EditorToolbar.tsx +++ b/frontend/src/components/editor/EditorToolbar.tsx @@ -198,8 +198,38 @@ export const EditorToolbar = ({ name: f.name, content: f.content, })); - const result = await compileCode(sketchFiles, fqbn, currentProject?.id ?? null); + // Stream live cmake + ninja output into the compilation console as + // it arrives, instead of waiting for the whole build to finish. + // Each poll the backend returns the cumulative stdout buffer; we + // append only the delta since the previous call as 'info' lines. + let lastStreamedLen = 0; + const result = await compileCode( + sketchFiles, + fqbn, + currentProject?.id ?? null, + ({ stdout }) => { + if (stdout.length <= lastStreamedLen) return; + const delta = stdout.slice(lastStreamedLen); + lastStreamedLen = stdout.length; + const newLines = delta.split('\n').filter((s) => s.trim()); + if (!newLines.length) return; + const now = new Date(); + setCompileLogs((prev: CompilationLog[]) => [ + ...prev, + ...newLines.map((line) => ({ + timestamp: now, + type: 'info' as const, + message: line, + })), + ]); + }, + ); + + // After the build settles, append the structured analysis on top of + // the live stream — parseCompileResult highlights FAILED blocks and + // tags compiler errors with type='error', which the console uses for + // colour + the auto-switch-to-errors filter. const resultLogs = parseCompileResult(result, boardLabel); setCompileLogs((prev: CompilationLog[]) => [...prev, ...resultLogs]); @@ -456,7 +486,31 @@ export const EditorToolbar = ({ try { const groupFiles = useEditorStore.getState().getGroupFiles(board.activeFileGroupId); const sketchFiles = groupFiles.map((f) => ({ name: f.name, content: f.content })); - const result = await compileCode(sketchFiles, fqbn, currentProject?.id ?? null); + + // Stream live cmake + ninja output per-board (Compile-All flow). + let lastStreamedLen = 0; + const result = await compileCode( + sketchFiles, + fqbn, + currentProject?.id ?? null, + ({ stdout }) => { + if (stdout.length <= lastStreamedLen) return; + const delta = stdout.slice(lastStreamedLen); + lastStreamedLen = stdout.length; + const newLines = delta.split('\n').filter((s) => s.trim()); + if (!newLines.length) return; + const now = new Date(); + setCompileLogs((prev: CompilationLog[]) => [ + ...prev, + ...newLines.map((line) => ({ + timestamp: now, + type: 'info' as const, + message: `${label}: ${line}`, + })), + ]); + }, + ); + const resultLogs = parseCompileResult(result, label); setCompileLogs((prev: CompilationLog[]) => [...prev, ...resultLogs]); diff --git a/frontend/src/services/compilation.ts b/frontend/src/services/compilation.ts index 1828311c..ba3de6ae 100644 --- a/frontend/src/services/compilation.ts +++ b/frontend/src/services/compilation.ts @@ -27,10 +27,23 @@ interface CompileStatusResponse { state: 'pending' | 'running' | 'done' | 'error'; started_at: number; finished_at: number | null; + stdout: string; result: CompileResult | null; error: string | null; } +/** + * Live progress callback — called on every poll while state ∈ {pending, + * running}. `stdout` is the full live cmake + ninja output captured so far + * (cap of ~256 KB on the server side, tail kept). Caller can compute a + * delta against the previous call if it wants to append-only render. + */ +export type CompileProgress = (info: { + state: 'pending' | 'running'; + stdout: string; + elapsedSeconds: number; +}) => void; + const POLL_INTERVAL_MS = 2000; const MAX_POLL_DURATION_MS = 15 * 60 * 1000; // 15 minutes — covers cold ESP-IDF builds @@ -40,16 +53,21 @@ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); * Compile a sketch via the async job pipeline. * * POST /compile/start → { job_id } - * GET /compile/status/ (×N) → { state, result?, error? } + * GET /compile/status/ (×N) → { state, stdout, result?, error? } * * Each individual request returns in milliseconds, so Cloudflare's 100s edge * timeout never kicks in — even when the underlying ESP-IDF cold build runs * for 5-7 minutes. Falls back to throwing an Error after MAX_POLL_DURATION_MS. + * + * `onProgress` (optional): called every poll with the live cmake + ninja + * output so the editor can stream the compilation console instead of + * waiting for everything at the end. */ export async function compileCode( files: SketchFile[], board: string = 'arduino:avr:uno', projectId?: string | null, + onProgress?: CompileProgress, ): Promise { console.log('Sending compilation request to:', `${API_BASE}/compile/start`); console.log('Board:', board); @@ -119,12 +137,26 @@ export async function compileCode( console.error(`[compile] job ${jobId} errored:`, status.error); return { success: false, - stdout: '', + stdout: status.stdout || '', stderr: '', error: status.error || 'Compile failed', }; } + // state ∈ {pending, running} — surface live build output if requested + if (onProgress) { + try { + onProgress({ + state: status.state, + stdout: status.stdout || '', + elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), + }); + } catch (err) { + // A faulty UI hook must never break the polling loop. + console.warn('[compile] onProgress threw:', err); + } + } + await sleep(POLL_INTERVAL_MS); } }