Add conformance suite; client: raw PPM screenshot + PPM parser 8ac19619e24b
- conformance/: platform-agnostic pytest asserting the contract, attach or launch mode; verified green (9/9) against the x16emu reference implementation - client: screenshot_ppm() (Pillow-free) and parse_ppm_header() helper Git-Commit: ee17521cd514d090023c5dd2ad3cd16002005134 Git-Author: Andy McCall <[email protected]>
- Author
- Andy McCall @andymccall <[email protected]>
- When
- · 2026-07-19 18:23 UTC
- Signature
- ◈ sealed by Andy McCall
- Hash
8ac19619e24b6fcb349832849eafdce5a7bee67d1b2d3de35b233ba629b39b1d- Depends on
- 961b3c610c66
- Depended on by
- 8fd33bc520f9 1b40a077e41f bed0f1e524fe 346ca3ee83f4 d69f9ef6bbac 4c0b050e9a25 674b8447022b d44319b1e712
- In bookmarks
- main perf-6
- Edits
- 7 atoms, 4 tree operations, 4 files
Browse the tree at this change
added conformance/README.md +48 −0
| 1 | + | # Conformance suite |
|
| 2 | + | ||
| 3 | + | pytest tests that assert an emulator correctly implements the control contract |
|
| 4 | + | (`../SPEC.md`, 0.1.x). Point them at any emulator — the reference x16emu, FAB, |
|
| 5 | + | the Neo6502 — and they check the same surface. Platform-agnostic: framebuffer |
|
| 6 | + | dimensions come from the PPM header, addresses use `$0000`, and `bank` is |
|
| 7 | + | exercised without requiring the platform to have banking. |
|
| 8 | + | ||
| 9 | + | ## Run |
|
| 10 | + | ||
| 11 | + | Attach to an already-running emulator: |
|
| 12 | + | ||
| 13 | + | ``` |
|
| 14 | + | RRDC_PORT=8080 pytest -v |
|
| 15 | + | ``` |
|
| 16 | + | ||
| 17 | + | Or let the suite launch one (`{port}` is substituted into the command): |
|
| 18 | + | ||
| 19 | + | ``` |
|
| 20 | + | RRDC_EMU_CMD="SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy \ |
|
| 21 | + | /path/to/x16emu -rom rom.bin -warp -controlport {port}" \ |
|
| 22 | + | RRDC_PORT=8391 pytest -v |
|
| 23 | + | ``` |
|
| 24 | + | ||
| 25 | + | If nothing is listening and no `RRDC_EMU_CMD` is set, the tests **skip** (a bare |
|
| 26 | + | `pytest` in CI without an emulator is a skip, not a failure). |
|
| 27 | + | ||
| 28 | + | ## What it checks |
|
| 29 | + | ||
| 30 | + | - `/status` — required keys present, contract MAJOR matches, sane types. |
|
| 31 | + | - `/regs` — a non-empty JSON object of integer registers. |
|
| 32 | + | - `/mem` — returns exactly `len` bytes; honours `bank`. |
|
| 33 | + | - `/screenshot` — a well-formed PPM whose body size matches its dimensions. |
|
| 34 | + | - `/step` — deterministic: equal steps advance the frame counter by equal, |
|
| 35 | + | positive amounts, and `/step` returns the halted counter. |
|
| 36 | + | - `/pause` + `/resume` — toggle the paused state. |
|
| 37 | + | - Errors — unknown path → 404; `GET /step` → 405. |
|
| 38 | + | ||
| 39 | + | ## Environment |
|
| 40 | + | ||
| 41 | + | | Var | Default | Meaning | |
|
| 42 | + | |---|---|---| |
|
| 43 | + | | `RRDC_PORT` | `8080` | control port to attach to / launch on | |
|
| 44 | + | | `RRDC_EMU_CMD` | — | shell command to launch an emulator (`{port}` substituted) | |
|
| 45 | + | | `RRDC_BOOT_TIMEOUT` | `20` | seconds to wait for `/status` | |
|
| 46 | + | ||
| 47 | + | Requires `pytest`. Nothing else — the client is stdlib-only (Pillow is optional |
|
| 48 | + | and unused here). |
added conformance/conftest.py +59 −0
| 1 | + | """Conformance harness fixtures. |
|
| 2 | + | ||
| 3 | + | Provides an `emu` fixture (an EmuControl client). Two modes: |
|
| 4 | + | ||
| 5 | + | - Attach: point RRDC_PORT at an already-running emulator's control port. |
|
| 6 | + | - Launch: set RRDC_EMU_CMD to a shell command that starts one (the literal |
|
| 7 | + | {port} is substituted). The suite spawns it, waits for /status, and tears it |
|
| 8 | + | down afterward. |
|
| 9 | + | ||
| 10 | + | If nothing is listening and no RRDC_EMU_CMD is set, the tests SKIP (so a bare |
|
| 11 | + | `pytest` in CI without an emulator is a skip, not a failure). |
|
| 12 | + | """ |
|
| 13 | + | import os |
|
| 14 | + | import signal |
|
| 15 | + | import subprocess |
|
| 16 | + | import sys |
|
| 17 | + | ||
| 18 | + | import pytest |
|
| 19 | + | ||
| 20 | + | HERE = os.path.dirname(os.path.abspath(__file__)) |
|
| 21 | + | sys.path.insert(0, os.path.join(HERE, "..", "client")) |
|
| 22 | + | ||
| 23 | + | from emu_control import EmuControl # noqa: E402 |
|
| 24 | + | ||
| 25 | + | ||
| 26 | + | def _kill(proc): |
|
| 27 | + | for sig in (signal.SIGTERM, signal.SIGKILL): |
|
| 28 | + | try: |
|
| 29 | + | os.killpg(os.getpgid(proc.pid), sig) |
|
| 30 | + | proc.wait(timeout=5) |
|
| 31 | + | return |
|
| 32 | + | except Exception: |
|
| 33 | + | continue |
|
| 34 | + | ||
| 35 | + | ||
| 36 | + | @pytest.fixture(scope="session") |
|
| 37 | + | def emu(): |
|
| 38 | + | port = int(os.environ.get("RRDC_PORT", "8080")) |
|
| 39 | + | cmd = os.environ.get("RRDC_EMU_CMD") |
|
| 40 | + | proc = None |
|
| 41 | + | if cmd: |
|
| 42 | + | proc = subprocess.Popen( |
|
| 43 | + | cmd.replace("{port}", str(port)), shell=True, start_new_session=True, |
|
| 44 | + | stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) |
|
| 45 | + | ||
| 46 | + | client = EmuControl(port) |
|
| 47 | + | try: |
|
| 48 | + | timeout = float(os.environ.get("RRDC_BOOT_TIMEOUT", "20")) |
|
| 49 | + | client.wait_ready(tries=max(1, int(timeout / 0.1)), delay=0.1) |
|
| 50 | + | except Exception as ex: |
|
| 51 | + | if proc is not None: |
|
| 52 | + | _kill(proc) |
|
| 53 | + | pytest.skip(f"no emulator on control port {port}: {ex} " |
|
| 54 | + | f"(set RRDC_EMU_CMD to launch one, or RRDC_PORT to attach)") |
|
| 55 | + | ||
| 56 | + | yield client |
|
| 57 | + | ||
| 58 | + | if proc is not None: |
|
| 59 | + | _kill(proc) |
added conformance/test_conformance.py +84 −0
| 1 | + | """Contract conformance tests (SPEC.md 0.1.x). Platform-agnostic: framebuffer |
|
| 2 | + | dimensions come from the PPM header, addresses use $0000, and `bank` is |
|
| 3 | + | exercised but banking is not required.""" |
|
| 4 | + | import urllib.error |
|
| 5 | + | import urllib.request |
|
| 6 | + | ||
| 7 | + | from emu_control import parse_ppm_header |
|
| 8 | + | ||
| 9 | + | ||
| 10 | + | def _raw(emu, method, path): |
|
| 11 | + | """Raw request returning (status_code, body) without the client's wrapping, |
|
| 12 | + | so we can assert on 4xx status codes.""" |
|
| 13 | + | req = urllib.request.Request(emu.base + path, method=method) |
|
| 14 | + | try: |
|
| 15 | + | with urllib.request.urlopen(req, timeout=emu.timeout) as r: |
|
| 16 | + | return r.status, r.read() |
|
| 17 | + | except urllib.error.HTTPError as e: |
|
| 18 | + | return e.code, e.read() |
|
| 19 | + | ||
| 20 | + | ||
| 21 | + | def test_status_contract(emu): |
|
| 22 | + | st = emu.status() |
|
| 23 | + | for k in ("contract", "emulator", "platform", "frame", "paused", "running"): |
|
| 24 | + | assert k in st, f"missing /status.{k}" |
|
| 25 | + | assert str(st["contract"]).split(".")[0] == "0", "unexpected contract MAJOR" |
|
| 26 | + | assert isinstance(st["platform"], str) and st["platform"] |
|
| 27 | + | assert isinstance(st["frame"], int) |
|
| 28 | + | assert isinstance(st["paused"], bool) |
|
| 29 | + | ||
| 30 | + | ||
| 31 | + | def test_regs_object(emu): |
|
| 32 | + | r = emu.regs() |
|
| 33 | + | assert isinstance(r, dict) and r, "regs must be a non-empty JSON object" |
|
| 34 | + | assert all(isinstance(v, int) for v in r.values()), "register values must be ints" |
|
| 35 | + | ||
| 36 | + | ||
| 37 | + | def test_mem_exact_length(emu): |
|
| 38 | + | assert len(emu.mem(0x0000, 32)) == 32 |
|
| 39 | + | assert len(emu.mem(0x0000, 256)) == 256 |
|
| 40 | + | assert len(emu.mem(0x0000, 0)) == 0 |
|
| 41 | + | ||
| 42 | + | ||
| 43 | + | def test_mem_bank_param_returns_length(emu): |
|
| 44 | + | # bank is optional (ignored on flat platforms) but must still honour len. |
|
| 45 | + | assert len(emu.mem(0x0000, 8, bank=0)) == 8 |
|
| 46 | + | ||
| 47 | + | ||
| 48 | + | def test_screenshot_ppm(emu): |
|
| 49 | + | data = emu.screenshot_ppm() |
|
| 50 | + | w, h, maxval, off = parse_ppm_header(data) |
|
| 51 | + | assert w > 0 and h > 0, f"bad dimensions {w}x{h}" |
|
| 52 | + | assert maxval == 255 |
|
| 53 | + | assert len(data) == off + 3 * w * h, "PPM body size mismatches dimensions" |
|
| 54 | + | ||
| 55 | + | ||
| 56 | + | def test_step_is_deterministic(emu): |
|
| 57 | + | emu.pause() |
|
| 58 | + | try: |
|
| 59 | + | a = emu.frame() |
|
| 60 | + | b = emu.step(2); c = emu.frame() |
|
| 61 | + | assert b == c, "/step must return the halted frame counter" |
|
| 62 | + | d = emu.step(2); e = emu.frame() |
|
| 63 | + | assert d == e |
|
| 64 | + | assert c - a > 0, "/step must advance the frame counter" |
|
| 65 | + | assert (c - a) == (e - c), "equal steps must advance by equal amounts" |
|
| 66 | + | finally: |
|
| 67 | + | emu.resume() |
|
| 68 | + | ||
| 69 | + | ||
| 70 | + | def test_pause_resume_toggle(emu): |
|
| 71 | + | emu.pause() |
|
| 72 | + | assert emu.status()["paused"] is True |
|
| 73 | + | emu.resume() |
|
| 74 | + | assert emu.status()["paused"] is False |
|
| 75 | + | ||
| 76 | + | ||
| 77 | + | def test_unknown_path_404(emu): |
|
| 78 | + | code, _ = _raw(emu, "GET", "/no-such-endpoint") |
|
| 79 | + | assert code == 404 |
|
| 80 | + | ||
| 81 | + | ||
| 82 | + | def test_step_requires_post(emu): |
|
| 83 | + | code, _ = _raw(emu, "GET", "/step?frames=1") |
|
| 84 | + | assert code == 405 |
modified client/emu_control.py +29 −1
| … | 28 unchanged lines |
||
| 29 | 29 | pass |
|
| 30 | 30 | ||
| 31 | 31 | ||
| 32 | + | def parse_ppm_header(data: bytes): |
|
| 33 | + | """Parse a binary PPM (P6) header. Returns (width, height, maxval, offset) |
|
| 34 | + | where offset is the index of the first pixel byte. Raises on malformed |
|
| 35 | + | input. Stdlib-only — no Pillow needed.""" |
|
| 36 | + | if data[:2] != b"P6": |
|
| 37 | + | raise EmuControlError(f"not a P6 PPM (magic {data[:2]!r})") |
|
| 38 | + | fields, i, n = [], 2, len(data) |
|
| 39 | + | while len(fields) < 3 and i < n: |
|
| 40 | + | while i < n and data[i:i + 1].isspace(): |
|
| 41 | + | i += 1 |
|
| 42 | + | if i < n and data[i:i + 1] == b"#": # comment to end of line |
|
| 43 | + | while i < n and data[i:i + 1] not in (b"\n", b"\r"): |
|
| 44 | + | i += 1 |
|
| 45 | + | continue |
|
| 46 | + | start = i |
|
| 47 | + | while i < n and not data[i:i + 1].isspace(): |
|
| 48 | + | i += 1 |
|
| 49 | + | fields.append(int(data[start:i])) |
|
| 50 | + | if len(fields) < 3: |
|
| 51 | + | raise EmuControlError("truncated PPM header") |
|
| 52 | + | return fields[0], fields[1], fields[2], i + 1 # +1: single whitespace |
|
| 53 | + | ||
| 54 | + | ||
| 32 | 55 | class EmuControl: |
|
| 33 | 56 | def __init__(self, port: int, host: str = "127.0.0.1", timeout: float = 5.0): |
|
| 34 | 57 | self.base = f"http://{host}:{port}" |
|
| … | 50 unchanged lines |
||
| 85 | 108 | ||
| 86 | 109 | def resume(self) -> None: |
|
| 87 | 110 | self._post("/resume") |
|
| 111 | + | ||
| 112 | + | def screenshot_ppm(self) -> bytes: |
|
| 113 | + | """Return the live screen as raw PPM (P6) bytes. Stdlib-only.""" |
|
| 114 | + | body, _ = self._get("/screenshot") |
|
| 115 | + | return body |
|
| 88 | 116 | ||
| 89 | 117 | def screenshot(self): |
|
| 90 | 118 | """Return the live screen as a PIL.Image (requires Pillow).""" |
|
| 91 | - | body, ctype = self._get("/screenshot") |
|
| 119 | + | body = self.screenshot_ppm() |
|
| 92 | 120 | try: |
|
| 93 | 121 | from PIL import Image |
|
| 94 | 122 | except ImportError as e: # pragma: no cover |
|
| … | 21 unchanged lines |