525 lines
18 KiB
Python
525 lines
18 KiB
Python
"""
|
|
One-time backfill: populate ``projects.boards_json`` and reorganise on-disk
|
|
files into per-board subdirs for projects created before the multi-board
|
|
persistence change.
|
|
|
|
Pre-existing schema only stored ``board_type`` (a single string) plus flat
|
|
``components_json`` / ``wires_json`` columns. The frontend, however, has been
|
|
multi-board-capable for a while — wire endpoints reference board IDs literally
|
|
(e.g. ``componentId='esp32-cam'``). On reload, ``setBoardType`` mutated the
|
|
sole default board's ``boardKind`` but kept its ``id`` fixed at
|
|
``'arduino-uno'`` (the legacy ``INITIAL_BOARD_ID``), so any wire that pointed
|
|
to a non-Uno board id was orphaned and visually anchored to (0, 0).
|
|
|
|
Heuristic per project (idempotent — projects whose ``boards_json`` already
|
|
holds a non-empty array are skipped):
|
|
|
|
Case A — single-board legacy
|
|
Wires reference only ``'arduino-uno'`` AND ``board_type != 'arduino-uno'``.
|
|
The user really had ONE board of kind ``board_type``; the id is residual
|
|
from the ``INITIAL_BOARD_ID`` quirk. Rename the wire endpoints
|
|
``'arduino-uno'`` -> ``board_type`` and create one board with
|
|
``id == board_type``.
|
|
|
|
Case B — single-board normal
|
|
Wires reference at most one board id, and that id matches ``board_type``
|
|
(or there are no wires). Create one board verbatim.
|
|
|
|
Case C — multi-board
|
|
Wires reference more than one distinct board id. Create one BoardInstance
|
|
per distinct id, in order of first appearance, laid out left-to-right at
|
|
(50 + i*350, 50). The active board is the one matching ``board_type`` if
|
|
present, otherwise the first.
|
|
|
|
Refs whose kind cannot be inferred (e.g. typos like ``'esp'``) are left alone:
|
|
the wires stay orphaned and are reported in the summary for manual review.
|
|
|
|
Usage::
|
|
|
|
# Dry-run (default) — prints the plan, touches nothing
|
|
python -m backend.scripts.backfill_boards_2026_05
|
|
|
|
# Apply changes to DB and disk
|
|
python -m backend.scripts.backfill_boards_2026_05 --apply
|
|
|
|
# Run for a single project
|
|
python -m backend.scripts.backfill_boards_2026_05 --apply --project-id <uuid>
|
|
|
|
# Custom DB / data path (defaults to backend/.env's values)
|
|
python -m backend.scripts.backfill_boards_2026_05 --db data/velxio.db --data-dir data
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import dataclasses
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
# ── Canonical board kinds (mirrors frontend/src/types/board.ts) ────────────────
|
|
BOARD_KINDS: set[str] = {
|
|
"arduino-uno",
|
|
"arduino-nano",
|
|
"arduino-mega",
|
|
"attiny85",
|
|
"raspberry-pi-pico",
|
|
"pi-pico-w",
|
|
"raspberry-pi-3",
|
|
"esp32",
|
|
"esp32-devkit-c-v4",
|
|
"esp32-cam",
|
|
"wemos-lolin32-lite",
|
|
"esp32-s3",
|
|
"xiao-esp32-s3",
|
|
"arduino-nano-esp32",
|
|
"esp32-c3",
|
|
"xiao-esp32-c3",
|
|
"aitewinrobot-esp32c3-supermini",
|
|
}
|
|
|
|
# Suffix added by addBoard for additional instances of the same kind
|
|
INSTANCE_SUFFIX_RE = re.compile(r"^(?P<kind>.+?)-(?P<n>\d+)$")
|
|
|
|
DEFAULT_X = 50
|
|
DEFAULT_Y = 50
|
|
BOARD_X_STEP = 350
|
|
|
|
|
|
def infer_kind(ref: str) -> str | None:
|
|
"""Map a wire endpoint componentId to a BoardKind, or None if unknown.
|
|
|
|
'esp32' -> 'esp32', 'esp32-cam' -> 'esp32-cam', 'esp32-2' -> 'esp32',
|
|
'arduino-uno-3' -> 'arduino-uno', 'esp' -> None.
|
|
"""
|
|
if ref in BOARD_KINDS:
|
|
return ref
|
|
m = INSTANCE_SUFFIX_RE.match(ref)
|
|
if m and m.group("kind") in BOARD_KINDS:
|
|
return m.group("kind")
|
|
return None
|
|
|
|
|
|
def make_board(board_id: str, board_kind: str, x: int, y: int) -> dict:
|
|
"""A serialisable BoardInstance matching frontend/src/types/board.ts."""
|
|
return {
|
|
"id": board_id,
|
|
"boardKind": board_kind,
|
|
"x": x,
|
|
"y": y,
|
|
"running": False,
|
|
"compiledProgram": None,
|
|
"serialOutput": "",
|
|
"serialBaudRate": 0,
|
|
"serialMonitorOpen": False,
|
|
"activeFileGroupId": f"group-{board_id}",
|
|
"languageMode": "arduino",
|
|
}
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class PlanItem:
|
|
project_id: str
|
|
name: str
|
|
case: str # 'A' | 'B' | 'C' | 'skip-already-migrated' | 'skip-empty'
|
|
board_type: str
|
|
new_boards: list[dict]
|
|
wires_renamed: int
|
|
new_wires_json: str | None
|
|
new_disk_layout: dict | None # {"target_group": str, "files": list[dict]} or None
|
|
orphan_refs: list[str]
|
|
|
|
|
|
def plan_for_project(
|
|
project_id: str,
|
|
name: str,
|
|
board_type: str,
|
|
components_json: str,
|
|
wires_json: str,
|
|
boards_json: str,
|
|
project_files_dir: Path,
|
|
) -> PlanItem:
|
|
# Idempotent skip: already migrated.
|
|
try:
|
|
existing = json.loads(boards_json or "[]")
|
|
except (ValueError, TypeError):
|
|
existing = []
|
|
if isinstance(existing, list) and existing:
|
|
return PlanItem(
|
|
project_id=project_id,
|
|
name=name,
|
|
case="skip-already-migrated",
|
|
board_type=board_type,
|
|
new_boards=[],
|
|
wires_renamed=0,
|
|
new_wires_json=None,
|
|
new_disk_layout=None,
|
|
orphan_refs=[],
|
|
)
|
|
|
|
try:
|
|
components = json.loads(components_json or "[]")
|
|
except (ValueError, TypeError):
|
|
components = []
|
|
try:
|
|
wires = json.loads(wires_json or "[]")
|
|
except (ValueError, TypeError):
|
|
wires = []
|
|
|
|
comp_ids: set[str] = set()
|
|
for c in components:
|
|
if isinstance(c, dict):
|
|
cid = c.get("id")
|
|
if isinstance(cid, str) and cid:
|
|
comp_ids.add(cid)
|
|
|
|
# Collect unique wire-endpoint refs that are NOT components — these are
|
|
# board ids. Preserve order of first appearance.
|
|
seen: list[str] = []
|
|
seen_set: set[str] = set()
|
|
for w in wires:
|
|
if not isinstance(w, dict):
|
|
continue
|
|
for ep_key in ("start", "end"):
|
|
ep = w.get(ep_key)
|
|
if not isinstance(ep, dict):
|
|
continue
|
|
cid = ep.get("componentId")
|
|
if isinstance(cid, str) and cid and cid not in comp_ids and cid not in seen_set:
|
|
seen.append(cid)
|
|
seen_set.add(cid)
|
|
|
|
bt = board_type or "arduino-uno"
|
|
|
|
# ── Case selection ────────────────────────────────────────────────────────
|
|
new_boards: list[dict] = []
|
|
new_wires_json: str | None = None
|
|
wires_renamed = 0
|
|
orphans: list[str] = []
|
|
case: str
|
|
|
|
if not seen:
|
|
# No wires, or wires only reference components. Single board from board_type.
|
|
case = "B"
|
|
new_boards = [make_board(bt, bt, DEFAULT_X, DEFAULT_Y)]
|
|
elif seen == ["arduino-uno"] and bt != "arduino-uno":
|
|
# Case A: legacy id residue. Rename to board_type.
|
|
case = "A"
|
|
new_boards = [make_board(bt, bt, DEFAULT_X, DEFAULT_Y)]
|
|
# Rewrite wires
|
|
new_wires = []
|
|
for w in wires:
|
|
if not isinstance(w, dict):
|
|
new_wires.append(w)
|
|
continue
|
|
new_w = dict(w)
|
|
for ep_key in ("start", "end"):
|
|
ep = new_w.get(ep_key)
|
|
if isinstance(ep, dict) and ep.get("componentId") == "arduino-uno":
|
|
new_w[ep_key] = {**ep, "componentId": bt}
|
|
wires_renamed += 1
|
|
new_wires.append(new_w)
|
|
new_wires_json = json.dumps(new_wires)
|
|
elif len(seen) == 1:
|
|
# Case B: single-board, wire ids match (or board_type is arduino-uno
|
|
# and wires use 'arduino-uno', which is consistent).
|
|
case = "B"
|
|
only = seen[0]
|
|
kind = infer_kind(only)
|
|
if kind is None:
|
|
# Truly unknown ref — fall back to board_type, log orphan
|
|
orphans.append(only)
|
|
new_boards = [make_board(bt, bt, DEFAULT_X, DEFAULT_Y)]
|
|
else:
|
|
# If the single ref's kind matches board_type, board_id == ref.
|
|
# Otherwise the ref is some other id (rare, but possible) — keep
|
|
# the ref as the board id since wires reference it literally.
|
|
new_boards = [make_board(only, kind, DEFAULT_X, DEFAULT_Y)]
|
|
else:
|
|
# Case C: multi-board.
|
|
case = "C"
|
|
for i, ref in enumerate(seen):
|
|
kind = infer_kind(ref)
|
|
if kind is None:
|
|
orphans.append(ref)
|
|
continue
|
|
new_boards.append(
|
|
make_board(ref, kind, DEFAULT_X + i * BOARD_X_STEP, DEFAULT_Y)
|
|
)
|
|
|
|
# Active board sorting: ensure the board matching board_type sits first
|
|
# (the frontend treats boards[0] as the default visible board).
|
|
if new_boards:
|
|
for i, b in enumerate(new_boards):
|
|
if b["id"] == bt or b["boardKind"] == bt:
|
|
if i > 0:
|
|
new_boards.insert(0, new_boards.pop(i))
|
|
break
|
|
|
|
# ── Disk layout: promote flat files into a group dir ──────────────────────
|
|
new_disk_layout: dict | None = None
|
|
if new_boards and project_files_dir.exists():
|
|
# Collect any flat files at the project root (skip subdirs).
|
|
flat_files = []
|
|
for entry in project_files_dir.iterdir():
|
|
if entry.is_file():
|
|
try:
|
|
flat_files.append(
|
|
{
|
|
"name": entry.name,
|
|
"content": entry.read_text(encoding="utf-8"),
|
|
}
|
|
)
|
|
except (OSError, UnicodeDecodeError):
|
|
# Binary or unreadable file — skip but keep on disk
|
|
pass
|
|
if flat_files:
|
|
target_group = new_boards[0]["activeFileGroupId"]
|
|
new_disk_layout = {"target_group": target_group, "files": flat_files}
|
|
|
|
return PlanItem(
|
|
project_id=project_id,
|
|
name=name,
|
|
case=case,
|
|
board_type=bt,
|
|
new_boards=new_boards,
|
|
wires_renamed=wires_renamed,
|
|
new_wires_json=new_wires_json,
|
|
new_disk_layout=new_disk_layout,
|
|
orphan_refs=orphans,
|
|
)
|
|
|
|
|
|
def apply_plan(plan: PlanItem, conn: sqlite3.Connection, data_dir: Path) -> None:
|
|
"""Persist plan to DB and disk. Idempotent per row."""
|
|
if plan.case in ("skip-already-migrated", "skip-empty"):
|
|
return
|
|
|
|
boards_json = json.dumps(plan.new_boards)
|
|
if plan.new_wires_json is not None:
|
|
conn.execute(
|
|
"UPDATE projects SET boards_json = ?, wires_json = ?, "
|
|
"updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
(boards_json, plan.new_wires_json, plan.project_id),
|
|
)
|
|
else:
|
|
conn.execute(
|
|
"UPDATE projects SET boards_json = ?, "
|
|
"updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
(boards_json, plan.project_id),
|
|
)
|
|
|
|
# Disk layout: move flat files into the active group's subdir.
|
|
if plan.new_disk_layout:
|
|
proj_dir = data_dir / "projects" / plan.project_id
|
|
target_dir = proj_dir / plan.new_disk_layout["target_group"]
|
|
target_dir.mkdir(parents=True, exist_ok=True)
|
|
for f in plan.new_disk_layout["files"]:
|
|
(target_dir / f["name"]).write_text(f["content"], encoding="utf-8")
|
|
# Remove the flat copy
|
|
flat = proj_dir / f["name"]
|
|
if flat.is_file():
|
|
try:
|
|
flat.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
ap = argparse.ArgumentParser(
|
|
description=(__doc__ or "").split("\n\n", 1)[0]
|
|
)
|
|
ap.add_argument(
|
|
"--apply",
|
|
action="store_true",
|
|
help="Persist changes (default is dry-run).",
|
|
)
|
|
ap.add_argument(
|
|
"--db",
|
|
default=None,
|
|
help="Path to velxio.db (default: derived from DATABASE_URL or "
|
|
"data/velxio.db).",
|
|
)
|
|
ap.add_argument(
|
|
"--data-dir",
|
|
default=None,
|
|
help="Path to data directory containing 'projects/' (default: "
|
|
"DATA_DIR env var or 'data').",
|
|
)
|
|
ap.add_argument(
|
|
"--project-id",
|
|
default=None,
|
|
help="Run for a single project id (UUID).",
|
|
)
|
|
ap.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=None,
|
|
help="Stop after N projects (for spot-checking).",
|
|
)
|
|
return ap.parse_args()
|
|
|
|
|
|
def resolve_paths(args: argparse.Namespace) -> tuple[Path, Path]:
|
|
db_path = args.db
|
|
if not db_path:
|
|
# Try DATABASE_URL of the form sqlite+aiosqlite:///<path>
|
|
url = os.environ.get("DATABASE_URL", "")
|
|
if url.startswith("sqlite"):
|
|
after = url.split(":///", 1)[-1]
|
|
if after:
|
|
db_path = after
|
|
if not db_path:
|
|
# Fall back to the canonical local restored path
|
|
for candidate in ("data/velxio.db", "../data/velxio.db", "velxio.db"):
|
|
if Path(candidate).exists():
|
|
db_path = candidate
|
|
break
|
|
if not db_path:
|
|
raise SystemExit("Could not locate velxio.db. Pass --db <path>.")
|
|
|
|
data_dir = args.data_dir or os.environ.get("DATA_DIR")
|
|
if not data_dir:
|
|
# Same parent as the DB, sibling to the file
|
|
data_dir = str(Path(db_path).parent)
|
|
|
|
return Path(db_path), Path(data_dir)
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
db_path, data_dir = resolve_paths(args)
|
|
|
|
if not db_path.exists():
|
|
print(f"DB not found: {db_path}", file=sys.stderr)
|
|
return 1
|
|
|
|
print(f"DB: {db_path}")
|
|
print(f"DATA_DIR: {data_dir}")
|
|
print(f"Mode: {'APPLY' if args.apply else 'DRY-RUN'}")
|
|
print()
|
|
|
|
conn = sqlite3.connect(str(db_path))
|
|
conn.row_factory = sqlite3.Row
|
|
|
|
# Schema check: boards_json must exist (added by lifespan auto-migration).
|
|
cols = {r[1] for r in conn.execute("PRAGMA table_info(projects)").fetchall()}
|
|
if "boards_json" not in cols:
|
|
print(
|
|
"FATAL: projects.boards_json column does not exist. Start the "
|
|
"backend at least once so the lifespan auto-migration runs, or "
|
|
"execute manually:\n"
|
|
" ALTER TABLE projects ADD COLUMN boards_json TEXT NOT NULL "
|
|
"DEFAULT '[]';",
|
|
file=sys.stderr,
|
|
)
|
|
return 2
|
|
|
|
if args.project_id:
|
|
rows = conn.execute(
|
|
"SELECT id, name, board_type, components_json, wires_json, boards_json "
|
|
"FROM projects WHERE id = ?",
|
|
(args.project_id,),
|
|
).fetchall()
|
|
else:
|
|
rows = conn.execute(
|
|
"SELECT id, name, board_type, components_json, wires_json, boards_json "
|
|
"FROM projects ORDER BY created_at"
|
|
).fetchall()
|
|
if args.limit:
|
|
rows = rows[: args.limit]
|
|
|
|
case_counts: Counter[str] = Counter()
|
|
plans: list[PlanItem] = []
|
|
total_wires_renamed = 0
|
|
all_orphans: list[tuple[str, str, str]] = [] # (project_id, name, ref)
|
|
|
|
for r in rows:
|
|
proj_dir = data_dir / "projects" / r["id"]
|
|
plan = plan_for_project(
|
|
project_id=r["id"],
|
|
name=r["name"] or "",
|
|
board_type=r["board_type"] or "arduino-uno",
|
|
components_json=r["components_json"] or "[]",
|
|
wires_json=r["wires_json"] or "[]",
|
|
boards_json=r["boards_json"] or "[]",
|
|
project_files_dir=proj_dir,
|
|
)
|
|
plans.append(plan)
|
|
case_counts[plan.case] += 1
|
|
total_wires_renamed += plan.wires_renamed
|
|
for ref in plan.orphan_refs:
|
|
all_orphans.append((plan.project_id, plan.name, ref))
|
|
|
|
# ── Print per-project plan ────────────────────────────────────────────────
|
|
for p in plans:
|
|
if p.case == "skip-already-migrated":
|
|
continue
|
|
ids = [b["id"] for b in p.new_boards]
|
|
extras = []
|
|
if p.wires_renamed:
|
|
extras.append(f"renamed {p.wires_renamed} wire endpoints")
|
|
if p.new_disk_layout:
|
|
extras.append(
|
|
f"move {len(p.new_disk_layout['files'])} flat file(s) -> "
|
|
f"{p.new_disk_layout['target_group']}/"
|
|
)
|
|
if p.orphan_refs:
|
|
extras.append(f"orphan refs={p.orphan_refs}")
|
|
suffix = (" | " + "; ".join(extras)) if extras else ""
|
|
name_repr = (p.name[:35] + "...") if len(p.name) > 38 else p.name
|
|
try:
|
|
print(
|
|
f"[{p.case}] {p.project_id[:8]} {name_repr!r:40s} "
|
|
f"board_type={p.board_type!r:25s} -> boards={ids}{suffix}"
|
|
)
|
|
except UnicodeEncodeError:
|
|
# Windows cp1252 console can't render some emojis/glyphs
|
|
safe = name_repr.encode("ascii", "replace").decode("ascii")
|
|
print(
|
|
f"[{p.case}] {p.project_id[:8]} {safe!r:40s} "
|
|
f"board_type={p.board_type!r:25s} -> boards={ids}{suffix}"
|
|
)
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────
|
|
print()
|
|
print("=" * 70)
|
|
print(f"Total projects considered: {len(plans)}")
|
|
for case in ("A", "B", "C", "skip-already-migrated"):
|
|
n = case_counts.get(case, 0)
|
|
if n:
|
|
label = {
|
|
"A": "Case A — legacy id rename (single-board)",
|
|
"B": "Case B — single-board normal (no wire rename)",
|
|
"C": "Case C — multi-board recovered",
|
|
"skip-already-migrated": "Skipped (boards_json already populated)",
|
|
}[case]
|
|
print(f" {label}: {n}")
|
|
print(f"Wire endpoints renamed: {total_wires_renamed}")
|
|
if all_orphans:
|
|
print()
|
|
print(f"WARNING: {len(all_orphans)} orphan board refs (kind unknown):")
|
|
for pid, name, ref in all_orphans[:30]:
|
|
print(f" {pid[:8]} {name!r:30s} ref={ref!r}")
|
|
if len(all_orphans) > 30:
|
|
print(f" ... and {len(all_orphans) - 30} more")
|
|
print()
|
|
|
|
if args.apply:
|
|
for p in plans:
|
|
apply_plan(p, conn, data_dir)
|
|
conn.commit()
|
|
print("Changes committed.")
|
|
else:
|
|
print("Dry-run only. Re-run with --apply to commit.")
|
|
conn.close()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|