From b9d39c0bd7ba82a897dc92aa3c4e014711fa6aca Mon Sep 17 00:00:00 2001 From: davidmonterocrespo24 Date: Sat, 16 May 2026 06:23:03 +0200 Subject: [PATCH] fix(pi3): show kernel boot + autologin SD + sidecar cache invalidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User report: clicked Pi 3 board → nothing visible happens. Three defects, all on the same path: 1. The kernel cmdline carried over from the original pre-OSS-split code: `quiet init=/bin/sh`. Result: kernel boot messages suppressed, then dropped straight to bare /bin/sh with no PS1 so the user sees an empty serial. Removed both. The kernel cmdline is now just `console=ttyAMA0 root=/dev/mmcblk0p2 rootwait rw dwc_otg.lpm_enable=0`, which lets systemd start a real serial-getty@ttyAMA0.service. 2. Pi OS Trixie armhf since Bookworm ships without a default user (no more pi/raspberry). With cmdline #1 fixed, the user would land at a login prompt and be stuck. Fix: pre-bake a systemd drop-in at /etc/systemd/system/serial-getty@ttyAMA0.service.d/ autologin.conf that uses `agetty --autologin root` so the serial console drops to a root shell on first prompt. The browser canvas IS the authentication boundary; the SD image is mounted RO via a qcow2 overlay so per-session edits don't persist. Edit happens in velxio-prod/scripts/configure-pi3-autologin.sh (to follow in a separate commit). 3. Architectural: the original cache-hit probe was size-only. Today's SD image rebake produced a file with identical byte count but different SHA256 — the cache served stale content for every request even after a manifest bump. Fix: write a sidecar `.sha256` after every successful materialise and trust it on subsequent probes. Manifest SHA bumps invalidate the cache regardless of size. Two regression tests guard this: - test_provider_sidecar_invalidates_on_sha_mismatch - test_provider_missing_sidecar_treats_file_as_invalid Manifest bumped to version "2026-04-21+autologin" for the SD image (kernel + DTB unchanged, still 2026-04-21). --- .../app/services/boot_images/manifest.json | 8 +- backend/app/services/boot_images/provider.py | 132 +++++++++++------- backend/app/services/qemu_manager.py | 8 +- test/backend/unit/test_boot_images.py | 77 ++++++++++ 4 files changed, 172 insertions(+), 53 deletions(-) diff --git a/backend/app/services/boot_images/manifest.json b/backend/app/services/boot_images/manifest.json index b4a65535..2c688c27 100644 --- a/backend/app/services/boot_images/manifest.json +++ b/backend/app/services/boot_images/manifest.json @@ -22,13 +22,13 @@ { "name": "raspios-trixie-armhf.img", "asset_id": "raspios-trixie-armhf-zst", - "sha256": "eb94f66c11fac1212d6cceda54e44736f68099dd5dea33825f1f3a9312b776db", + "sha256": "09bec8e099451d751ee9741e903f694ad641a0c534862efac0caabfc8a2beaf1", "size_bytes": 5729419264, - "version": "2026-04-21", + "version": "2026-04-21+autologin", "compressed": { "encoding": "zstd", - "sha256": "390ea79bcd3e1165d8d17548d8db26103a10097a8047a4b1bdb314443f14f6b7", - "size_bytes": 1488002803 + "sha256": "e665edfc94d481f5a756da74f6d494af78917ea4114dc5ad258718dfee1fce5b", + "size_bytes": 1488004394 } } ] diff --git a/backend/app/services/boot_images/provider.py b/backend/app/services/boot_images/provider.py index b8b50776..a2cb3087 100644 --- a/backend/app/services/boot_images/provider.py +++ b/backend/app/services/boot_images/provider.py @@ -99,21 +99,18 @@ class BootImageProvider: def is_cached(self, set_id: str) -> bool: """Sync probe used by health/status endpoints. - Only checks file presence + size — does NOT re-hash on every - probe (a 3 GiB SHA256 every health check would be wasteful). - Full hash verification still happens on ``get()`` when a cache - slot is populated. + Uses the same sidecar-SHA check ``_is_valid_cached`` does so + a manifest bump correctly reports "not cached yet" until the + next ``get()`` re-materialises the file. """ try: spec = self._manifest.get(set_id) except BootImageError: return False set_dir = self._cache_dir / set_id - for img in spec.images: - target = set_dir / img.name - if not target.is_file() or target.stat().st_size != img.size_bytes: - return False - return True + return all( + self._is_valid_cached(set_dir / img.name, img) for img in spec.images + ) # ── Internals ───────────────────────────────────────────────────────── @@ -148,24 +145,49 @@ class BootImageProvider: return out @staticmethod - def _is_valid_cached(path: Path, spec: BootImageSpec) -> bool: - """Cheap cache-hit probe — file presence + size match only. + def _sidecar(target: Path) -> Path: + """Sidecar file recording the SHA256 of the cached payload. + + Written atomically (temp + rename) after a successful + download+verify, read on every cache-validity probe. Lets the + provider detect manifest SHA bumps without re-hashing + multi-GiB files on every container start. + """ + return target.parent / f"{target.name}.sha256" + + @classmethod + def _is_valid_cached(cls, path: Path, spec: BootImageSpec) -> bool: + """O(1) cache-hit probe — presence + size + sidecar SHA match. We deliberately do NOT re-hash the file on every probe. The 5.4 GiB Pi 3 SD image takes ~30 s to SHA256, and that cost would be paid on every container boot pre-warm AND every user - request that triggers ``provider.get()``. Cache contents only - change via this class's own atomic-rename ladder, which always - verifies SHA256 before promoting a file into the cache slot. - Operators who manually edit the cache directory get the - behaviour they deserve. + request that triggers ``provider.get()``. - If you suspect cache corruption, delete the cache directory - and the next ``get()`` re-downloads + re-verifies. + Instead, after a successful materialise we write a sidecar + ``.sha256`` containing the expected hash and trust it on + subsequent probes. A manifest SHA bump invalidates the sidecar + even if the size is unchanged (e.g. an in-place SD image edit + that ends up the exact same byte count), forcing a re-fetch. + + If the sidecar is missing (legacy cache from before this + change, or operator tampering) the file is treated as invalid + and re-fetched. Manual operators who want to inject a file can + write the sidecar themselves: ``sha256sum file | cut -d' ' -f1 + > file.sha256``. """ if not path.is_file(): return False - return path.stat().st_size == spec.size_bytes + if path.stat().st_size != spec.size_bytes: + return False + sidecar = cls._sidecar(path) + if not sidecar.is_file(): + return False + try: + recorded = sidecar.read_text(encoding="ascii").strip().lower() + except OSError: + return False + return recorded == spec.sha256.lower() async def _fetch_and_verify( self, img: BootImageSpec, target: Path, @@ -175,33 +197,47 @@ class BootImageProvider: await asyncio.to_thread( verify_sha256, target, img.sha256, label=img.name, ) - return - - # Compressed path: download → verify wire-format sha → decompress - # → verify decompressed sha → atomic rename to final cache slot. - with tempfile.TemporaryDirectory( - dir=target.parent, prefix=".staging-", - ) as staging: - staging_dir = Path(staging) - compressed_path = staging_dir / f"{img.name}.{img.compressed.encoding}" - await self._downloader.fetch(img.asset_id, compressed_path) - await asyncio.to_thread( - verify_sha256, - compressed_path, - img.compressed.sha256, - label=f"{img.name} (compressed)", - ) - decoded = staging_dir / img.name - if img.compressed.encoding == "zstd": - await asyncio.to_thread(decompress_zstd, compressed_path, decoded) - else: - raise BootImageError( - f"unsupported compression {img.compressed.encoding!r}" + else: + # Compressed path: download → verify wire-format sha → + # decompress → verify decompressed sha → atomic rename to + # final cache slot. + with tempfile.TemporaryDirectory( + dir=target.parent, prefix=".staging-", + ) as staging: + staging_dir = Path(staging) + compressed_path = ( + staging_dir / f"{img.name}.{img.compressed.encoding}" ) - await asyncio.to_thread( - verify_sha256, - decoded, - img.sha256, - label=f"{img.name} (decompressed)", - ) - await asyncio.to_thread(decoded.replace, target) + await self._downloader.fetch(img.asset_id, compressed_path) + await asyncio.to_thread( + verify_sha256, + compressed_path, + img.compressed.sha256, + label=f"{img.name} (compressed)", + ) + decoded = staging_dir / img.name + if img.compressed.encoding == "zstd": + await asyncio.to_thread(decompress_zstd, compressed_path, decoded) + else: + raise BootImageError( + f"unsupported compression {img.compressed.encoding!r}" + ) + await asyncio.to_thread( + verify_sha256, + decoded, + img.sha256, + label=f"{img.name} (decompressed)", + ) + await asyncio.to_thread(decoded.replace, target) + # Record the expected SHA next to the file so future cache + # probes can detect manifest bumps without re-hashing the + # whole file. Sidecar write is atomic (temp + rename) so a + # process crash mid-write can't leave a half-written hash. + await asyncio.to_thread(self._write_sidecar, target, img.sha256) + + @classmethod + def _write_sidecar(cls, target: Path, sha256: str) -> None: + sidecar = cls._sidecar(target) + tmp = sidecar.with_suffix(sidecar.suffix + ".tmp") + tmp.write_text(sha256.lower() + "\n", encoding="ascii") + tmp.replace(sidecar) diff --git a/backend/app/services/qemu_manager.py b/backend/app/services/qemu_manager.py index c91ed4a5..04ef8df4 100644 --- a/backend/app/services/qemu_manager.py +++ b/backend/app/services/qemu_manager.py @@ -184,8 +184,14 @@ class QemuManager: # ttyAMA1 → GPIO shim protocol '-serial', f'tcp:127.0.0.1:{inst.gpio_port},server,nowait', '-append', + # No `quiet`: surface kernel boot messages to ttyAMA0 so the + # user sees progress while the 5.4 GiB Pi OS rootfs comes up + # (~30-60 s wall on a QEMU-emulated Cortex-A53 quad). No + # `init=/bin/sh`: let systemd run normally so a real + # `serial-getty@ttyAMA0` lands the user at a login prompt + # instead of a silent non-interactive shell. 'console=ttyAMA0 root=/dev/mmcblk0p2 rootwait rw ' - 'dwc_otg.lpm_enable=0 quiet init=/bin/sh', + 'dwc_otg.lpm_enable=0', ] logger.info('Launching QEMU for %s: %s', inst.client_id, ' '.join(cmd)) diff --git a/test/backend/unit/test_boot_images.py b/test/backend/unit/test_boot_images.py index 210dcaec..e05b99d4 100644 --- a/test/backend/unit/test_boot_images.py +++ b/test/backend/unit/test_boot_images.py @@ -435,6 +435,83 @@ async def test_provider_is_cached_probe(tmp_path: Path) -> None: assert provider.is_cached("unknown-board") is False +@pytest.mark.asyncio +async def test_provider_sidecar_invalidates_on_sha_mismatch( + tmp_path: Path, +) -> None: + """Regression test for the exact bug that broke Pi 3 in May 2026. + + Background: the original cache-hit probe was size-only. A + re-baked SD image with the same byte count but different + contents (e.g. an in-place edit of /etc/systemd/system/) was + served stale from the cache after a deploy. The sidecar SHA + check now catches this — manifest SHA bump invalidates the + cache even when size is unchanged. + """ + payload_v1 = b"original-bytes" * 1024 # 14 KiB + # Same size, different content — exactly the bug pattern. + payload_v2 = b"modified-bytes" * 1024 + assert len(payload_v1) == len(payload_v2) + + spec_v1 = _spec("rootfs.img", "rootfs", payload_v1) + iset_v1 = ImageSetSpec(id="set-a", description="", images=(spec_v1,)) + provider_v1, dl_v1 = _build_provider(tmp_path, iset_v1, {"rootfs": payload_v1}) + + # First get with v1 manifest: download + verify + sidecar written. + await provider_v1.get("set-a") + assert len(dl_v1.calls) == 1 + + # Now simulate a redeploy with a manifest SHA bump (same size). + # Build a brand-new provider against the same cache dir but with the + # v2 spec. + spec_v2 = _spec("rootfs.img", "rootfs", payload_v2) + iset_v2 = ImageSetSpec(id="set-a", description="", images=(spec_v2,)) + dl_v2 = FakeDownloader({"rootfs": payload_v2}) + provider_v2 = BootImageProvider( + manifest=_manifest(iset_v2), + downloader=dl_v2, + cache_dir=tmp_path / "cache", # reuse v1's cache dir + ) + + # is_cached must report False even though the file at that path + # exists with the right size — the sidecar SHA mismatches v2. + assert provider_v2.is_cached("set-a") is False + + # And calling get() re-fetches and overwrites with v2 bytes. + result = await provider_v2.get("set-a") + assert result["rootfs.img"].read_bytes() == payload_v2 + assert len(dl_v2.calls) == 1 + + # Final state: v2 is cached, sidecar matches v2 SHA. + assert provider_v2.is_cached("set-a") is True + + +@pytest.mark.asyncio +async def test_provider_missing_sidecar_treats_file_as_invalid( + tmp_path: Path, +) -> None: + """A pre-existing file without a sidecar (legacy cache, or manual + drop-in) is treated as not cached so the provider re-materialises + it and writes the sidecar this time.""" + payload = b"hello" * 200 + spec = _spec("kernel8.img", "kernel-pi3", payload) + iset = ImageSetSpec(id="raspberry-pi-3", description="", images=(spec,)) + provider, dl = _build_provider(tmp_path, iset, {"kernel-pi3": payload}) + + # Hand-place the cache file WITHOUT a sidecar (legacy state). + cache_target = tmp_path / "cache" / "raspberry-pi-3" / "kernel8.img" + cache_target.parent.mkdir(parents=True, exist_ok=True) + cache_target.write_bytes(payload) + assert not (cache_target.parent / "kernel8.img.sha256").exists() + + # Provider must NOT trust the orphan file — it has no proof of + # integrity. is_cached → False, get() re-downloads. + assert provider.is_cached("raspberry-pi-3") is False + await provider.get("raspberry-pi-3") + assert len(dl.calls) == 1 + assert (cache_target.parent / "kernel8.img.sha256").exists() + + @pytest.mark.asyncio async def test_provider_unknown_set_raises_typed_error(tmp_path: Path) -> None: provider = BootImageProvider(