diff --git a/backend/app/api/routes/compile.py b/backend/app/api/routes/compile.py index 6b5da6f2..0e0bd112 100644 --- a/backend/app/api/routes/compile.py +++ b/backend/app/api/routes/compile.py @@ -1,12 +1,15 @@ +import asyncio import logging import time +import uuid +from typing import Any from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession from app.core.dependencies import get_current_user -from app.database.session import get_db +from app.database.session import AsyncSessionLocal, get_db from app.models.user import User from app.services.arduino_cli import ArduinoCLIService from app.services.metrics import record_compile @@ -17,6 +20,29 @@ logger = logging.getLogger(__name__) router = APIRouter() arduino_cli = ArduinoCLIService() +# ── Async compile job registry ─────────────────────────────────────────────── +# In-process job dict for /compile/start + /compile/status/{job_id}. Cold ESP-IDF +# builds can take 5-7 minutes — far longer than Cloudflare's 100s edge timeout +# that hits any single HTTP request. The async path lets the client poll a +# short-lived status endpoint instead of holding one long-lived POST open. +# +# Single-instance only: if velxio ever scales to multiple FastAPI workers, this +# needs to move to Redis or the sqlite database. For now one process is fine. +COMPILE_JOBS: dict[str, dict[str, Any]] = {} +JOB_TTL_S = 1800 # purge results 30 min after completion + + +def _purge_expired_jobs() -> None: + """Drop completed jobs older than JOB_TTL_S so the dict doesn't grow forever.""" + now = time.time() + stale = [ + jid for jid, job in COMPILE_JOBS.items() + if job.get("state") in ("done", "error") + and now - job.get("finished_at", now) > JOB_TTL_S + ] + for jid in stale: + COMPILE_JOBS.pop(jid, None) + class SketchFile(BaseModel): name: str @@ -61,6 +87,147 @@ def _classify_compile_error(stderr: str, error: str | None) -> str: return "unknown" +def _resolve_files(request: CompileRequest) -> list[dict[str, str]]: + """Normalise the multi-file vs legacy single-file request bodies.""" + if request.files: + return [{"name": f.name, "content": f.content} for f in request.files] + if request.code is not None: + return [{"name": "sketch.ino", "content": request.code}] + raise HTTPException( + status_code=422, + detail="Provide either 'files' or 'code' in the request body.", + ) + + +async def _run_compile( + request: CompileRequest, + files: list[dict[str, str]], +) -> CompileResponse: + """Do the actual compile (ESP-IDF for esp32:*, arduino-cli otherwise).""" + if request.board_fqbn.startswith("esp32:") and espidf_compiler.available: + logger.info(f"[compile] Using ESP-IDF for {request.board_fqbn}") + result = await espidf_compiler.compile(files, request.board_fqbn) + return CompileResponse( + success=result["success"], + hex_content=result.get("hex_content"), + binary_content=result.get("binary_content"), + binary_type=result.get("binary_type"), + has_wifi=result.get("has_wifi", False), + stdout=result.get("stdout", ""), + stderr=result.get("stderr", ""), + error=result.get("error"), + ) + + # AVR, RP2040, and ESP32 fallback: use arduino-cli + core_status = await arduino_cli.ensure_core_for_board(request.board_fqbn) + core_log = core_status.get("log", "") + if core_status.get("needed") and not core_status.get("installed"): + return CompileResponse( + success=False, + stdout="", + stderr=core_log, + error=f"Failed to install required core: {core_status.get('core_id')}", + ) + + result = await arduino_cli.compile(files, request.board_fqbn) + return CompileResponse( + success=result["success"], + hex_content=result.get("hex_content"), + binary_content=result.get("binary_content"), + binary_type=result.get("binary_type"), + stdout=result.get("stdout", ""), + stderr=result.get("stderr", ""), + error=result.get("error"), + core_install_log=core_log if core_log else None, + ) + + +async def _record_async_metric( + *, + user_id: int | None, + project_id: str | None, + board_fqbn: str, + success: bool, + duration_ms: int, + error_kind: str | None, + extra: dict[str, Any], +) -> None: + """Open a fresh DB session and record one compile metric. + + Used by the async path: the request-scoped session is gone by the time + the background task finishes, so we open our own short-lived one. + Country/IP tagging is dropped on this path (the original Request is no + longer alive); user_id and success/duration still flow through. + """ + try: + async with AsyncSessionLocal() as session: + user = None + if user_id is not None: + user = await session.get(User, user_id) + await record_compile( + session, + user=user, + project_id=project_id, + board_fqbn=board_fqbn, + success=success, + duration_ms=duration_ms, + error_kind=error_kind, + extra=extra, + request=None, + ) + except Exception as exc: + logger.warning(f"[compile] async metric record failed: {exc}") + + +async def _compile_job( + job_id: str, + request: CompileRequest, + files: list[dict[str, str]], + user_id: int | None, +) -> None: + """Background worker: run the compile, store result in COMPILE_JOBS.""" + started = time.monotonic() + try: + COMPILE_JOBS[job_id]["state"] = "running" + response = await _run_compile(request, files) + COMPILE_JOBS[job_id] = { + "state": "done", + "started_at": COMPILE_JOBS[job_id]["started_at"], + "finished_at": time.time(), + "result": response.model_dump(), + } + error_kind = ( + None if response.success + else _classify_compile_error(response.stderr, response.error) + ) + await _record_async_metric( + user_id=user_id, + project_id=request.project_id, + board_fqbn=request.board_fqbn, + success=response.success, + duration_ms=int((time.monotonic() - started) * 1000), + error_kind=error_kind, + extra={"file_count": len(files), "has_wifi": response.has_wifi, "async": True}, + ) + except Exception as exc: + logger.exception(f"[compile] async job {job_id} failed") + COMPILE_JOBS[job_id] = { + "state": "error", + "started_at": COMPILE_JOBS[job_id]["started_at"], + "finished_at": time.time(), + "error": str(exc)[:500], + } + await _record_async_metric( + user_id=user_id, + project_id=request.project_id, + board_fqbn=request.board_fqbn, + success=False, + duration_ms=int((time.monotonic() - started) * 1000), + error_kind="exception", + extra={"file_count": len(files), "exception": str(exc)[:200], "async": True}, + ) + + @router.post("/", response_model=CompileResponse) async def compile_sketch( request: CompileRequest, @@ -69,64 +236,21 @@ async def compile_sketch( current_user: User | None = Depends(get_current_user), ): """ - Compile Arduino sketch and return hex/binary. + Compile Arduino sketch and return hex/binary in a single response. + + Synchronous path: held open until the build finishes. Works for AVR / + RP2040 builds (seconds), but ESP-IDF cold builds can run 5-7 minutes + and will hit Cloudflare's 100s edge timeout (HTTP 524). Use the async + path (`/compile/start` + `/compile/status/{job_id}`) for those. + Accepts either `files` (multi-file) or legacy `code` (single file). Auto-installs the required board core if not present. """ - # Resolve files list - if request.files: - files = [{"name": f.name, "content": f.content} for f in request.files] - elif request.code is not None: - files = [{"name": "sketch.ino", "content": request.code}] - else: - raise HTTPException( - status_code=422, - detail="Provide either 'files' or 'code' in the request body.", - ) - + files = _resolve_files(request) started = time.monotonic() - response: CompileResponse try: - # ESP32 targets: use ESP-IDF compiler for QEMU-compatible output - if request.board_fqbn.startswith("esp32:") and espidf_compiler.available: - logger.info(f"[compile] Using ESP-IDF for {request.board_fqbn}") - result = await espidf_compiler.compile(files, request.board_fqbn) - response = CompileResponse( - success=result["success"], - hex_content=result.get("hex_content"), - binary_content=result.get("binary_content"), - binary_type=result.get("binary_type"), - has_wifi=result.get("has_wifi", False), - stdout=result.get("stdout", ""), - stderr=result.get("stderr", ""), - error=result.get("error"), - ) - else: - # AVR, RP2040, and ESP32 fallback: use arduino-cli - core_status = await arduino_cli.ensure_core_for_board(request.board_fqbn) - core_log = core_status.get("log", "") - - if core_status.get("needed") and not core_status.get("installed"): - response = CompileResponse( - success=False, - stdout="", - stderr=core_log, - error=f"Failed to install required core: {core_status.get('core_id')}", - ) - else: - result = await arduino_cli.compile(files, request.board_fqbn) - response = CompileResponse( - success=result["success"], - hex_content=result.get("hex_content"), - binary_content=result.get("binary_content"), - binary_type=result.get("binary_type"), - stdout=result.get("stdout", ""), - stderr=result.get("stderr", ""), - error=result.get("error"), - core_install_log=core_log if core_log else None, - ) + response = await _run_compile(request, files) except Exception as e: - # Even on hard failure we want the metric. await record_compile( db, user=current_user, @@ -140,7 +264,6 @@ async def compile_sketch( ) raise HTTPException(status_code=500, detail=str(e)) - # Best-effort metrics — never blocks the response. duration_ms = int((time.monotonic() - started) * 1000) await record_compile( db, @@ -156,6 +279,63 @@ async def compile_sketch( return response +class CompileStartResponse(BaseModel): + job_id: str + + +class CompileStatusResponse(BaseModel): + state: str # 'pending' | 'running' | 'done' | 'error' + started_at: float + finished_at: float | None = None + result: CompileResponse | None = None + error: str | None = None + + +@router.post("/start", response_model=CompileStartResponse) +async def compile_start( + request: CompileRequest, + current_user: User | None = Depends(get_current_user), +): + """ + Queue a compile and return a `job_id` immediately. + + The actual compile runs in a background task; clients then poll + `GET /compile/status/{job_id}` every couple of seconds until state is + `done` or `error`. This sidesteps Cloudflare's 100s HTTP edge timeout — + each individual request returns in milliseconds. + """ + files = _resolve_files(request) + _purge_expired_jobs() + + job_id = uuid.uuid4().hex + COMPILE_JOBS[job_id] = {"state": "pending", "started_at": time.time()} + + asyncio.create_task( + _compile_job( + job_id=job_id, + request=request, + files=files, + user_id=current_user.id if current_user else None, + ), + ) + return CompileStartResponse(job_id=job_id) + + +@router.get("/status/{job_id}", response_model=CompileStatusResponse) +async def compile_status(job_id: str): + """Poll the status of an async compile job submitted via /compile/start.""" + job = COMPILE_JOBS.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="job not found or expired") + return CompileStatusResponse( + state=job["state"], + started_at=job["started_at"], + finished_at=job.get("finished_at"), + result=job.get("result"), + error=job.get("error"), + ) + + @router.get("/setup-status") async def setup_status(): return await arduino_cli.get_setup_status() diff --git a/frontend/src/services/compilation.ts b/frontend/src/services/compilation.ts index ee920552..1828311c 100644 --- a/frontend/src/services/compilation.ts +++ b/frontend/src/services/compilation.ts @@ -19,39 +19,112 @@ export interface CompileResult { core_install_log?: string; } +interface CompileStartResponse { + job_id: string; +} + +interface CompileStatusResponse { + state: 'pending' | 'running' | 'done' | 'error'; + started_at: number; + finished_at: number | null; + result: CompileResult | null; + error: string | null; +} + +const POLL_INTERVAL_MS = 2000; +const MAX_POLL_DURATION_MS = 15 * 60 * 1000; // 15 minutes — covers cold ESP-IDF builds + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** + * Compile a sketch via the async job pipeline. + * + * POST /compile/start → { job_id } + * GET /compile/status/ (×N) → { state, result?, error? } + * + * Each individual request returns in milliseconds, so Cloudflare's 100s edge + * timeout never kicks in — even when the underlying ESP-IDF cold build runs + * for 5-7 minutes. Falls back to throwing an Error after MAX_POLL_DURATION_MS. + */ export async function compileCode( files: SketchFile[], board: string = 'arduino:avr:uno', projectId?: string | null, ): Promise { + console.log('Sending compilation request to:', `${API_BASE}/compile/start`); + console.log('Board:', board); + console.log( + 'Files:', + files.map((f) => f.name), + ); + + let jobId: string; try { - console.log('Sending compilation request to:', `${API_BASE}/compile`); - console.log('Board:', board); - console.log( - 'Files:', - files.map((f) => f.name), - ); - - const response = await axios.post( - `${API_BASE}/compile/`, + const startResp = await axios.post( + `${API_BASE}/compile/start`, { files, board_fqbn: board, project_id: projectId ?? null }, - { withCredentials: true, timeout: 600000 }, + { withCredentials: true, timeout: 30000 }, ); - - console.log('Compilation response status:', response.status); - return response.data; + jobId = startResp.data.job_id; + console.log('[compile] queued job', jobId); } catch (error) { console.error('Compilation request failed:', error); + if (axios.isAxiosError(error) && error.response) { + // Server returned a structured error (422, 500, etc.) — surface as a + // failed CompileResult so the editor can show stderr/error. + return error.response.data as CompileResult; + } + throw error instanceof Error + ? error + : new Error('No response from server. Is the backend running?'); + } - if (axios.isAxiosError(error)) { - if (error.response) { - console.error('Error response data:', error.response.data); - return error.response.data; - } else if (error.request) { - throw new Error('No response from server. Is the backend running on port 8001?'); - } + const startedAt = Date.now(); + // Initial small delay so we don't hit /status before the background task + // has even moved past 'pending'. + await sleep(500); + + while (true) { + if (Date.now() - startedAt > MAX_POLL_DURATION_MS) { + throw new Error( + `Compile timed out client-side after ${Math.round(MAX_POLL_DURATION_MS / 1000)}s`, + ); } - throw error; + let status: CompileStatusResponse; + try { + const resp = await axios.get( + `${API_BASE}/compile/status/${jobId}`, + { withCredentials: true, timeout: 30000 }, + ); + status = resp.data; + } catch (error) { + // Transient poll error — log, wait, retry. Only abort on 404 (job + // expired or never existed). + if (axios.isAxiosError(error) && error.response?.status === 404) { + throw new Error(`Compile job ${jobId} not found (server may have restarted)`); + } + console.warn('[compile] status poll error, retrying:', error); + await sleep(POLL_INTERVAL_MS); + continue; + } + + if (status.state === 'done' && status.result) { + const elapsed = Math.round((Date.now() - startedAt) / 1000); + console.log(`[compile] job ${jobId} done in ${elapsed}s`); + return status.result; + } + + if (status.state === 'error') { + console.error(`[compile] job ${jobId} errored:`, status.error); + return { + success: false, + stdout: '', + stderr: '', + error: status.error || 'Compile failed', + }; + } + + await sleep(POLL_INTERVAL_MS); } }