OXYGENHUB
◆ main retro-remote-debug-controller / conformance / test_conformance.py
14.3 KiB · Python · last changed feat: contract 0.6 — GET/POST /perf, the performance metrics (budget, current frame, windows, zones, raw frames since one, a platform section) and their settings, and /status.capabilities; backend callbacks get_perf and set_perf return allocated JSON (#6) 25 days ago source blame history raw
1
"""Contract conformance tests (SPEC.md 0.3.x). Platform-agnostic: framebuffer
2
dimensions come from the PPM header, read addresses use $0000, and `bank` is
3
exercised but banking is not required. The one address that must be *writable*
4
(the mem-write round-trip) is chosen per-platform via `_scratch_ram`, because
5
low memory is not RAM everywhere — on the eZ80 Agon $0000-$03FFFF is on-chip
6
flash (MOS ROM) and RAM only begins at $040000. Tests gated on the server's
7
advertised minor (input injection 0.2; memory write + audio 0.3) skip on older
8
servers."""
9
import urllib.error
10
import urllib.request
11
12
import pytest
13
14
from emu_control import parse_ppm_header
15
16
17
def _raw(emu, method, path, body=None):
18
    """Raw request returning (status_code, body) without the client's wrapping,
19
    so we can assert on 4xx status codes."""
20
    req = urllib.request.Request(emu.base + path, method=method, data=body)
21
    try:
22
        with urllib.request.urlopen(req, timeout=emu.timeout) as r:
23
            return r.status, r.read()
24
    except urllib.error.HTTPError as e:
25
        return e.code, e.read()
26
27
28
def test_status_contract(emu):
29
    st = emu.status()
30
    for k in ("contract", "emulator", "platform", "frame", "paused", "running"):
31
        assert k in st, f"missing /status.{k}"
32
    assert str(st["contract"]).split(".")[0] == "0", "unexpected contract MAJOR"
33
    assert isinstance(st["platform"], str) and st["platform"]
34
    assert isinstance(st["frame"], int)
35
    assert isinstance(st["paused"], bool)
36
37
38
def test_regs_object(emu):
39
    r = emu.regs()
40
    assert isinstance(r, dict) and r, "regs must be a non-empty JSON object"
41
    assert all(isinstance(v, int) for v in r.values()), "register values must be ints"
42
43
44
def test_mem_exact_length(emu):
45
    assert len(emu.mem(0x0000, 32)) == 32
46
    assert len(emu.mem(0x0000, 256)) == 256
47
    assert len(emu.mem(0x0000, 0)) == 0
48
49
50
def test_mem_bank_param_returns_length(emu):
51
    # bank is optional (ignored on flat platforms) but must still honour len.
52
    assert len(emu.mem(0x0000, 8, bank=0)) == 8
53
54
55
def test_screenshot_ppm(emu):
56
    data = emu.screenshot_ppm()
57
    w, h, maxval, off = parse_ppm_header(data)
58
    assert w > 0 and h > 0, f"bad dimensions {w}x{h}"
59
    assert maxval == 255
60
    assert len(data) == off + 3 * w * h, "PPM body size mismatches dimensions"
61
62
63
def test_step_is_deterministic(emu):
64
    emu.pause()
65
    try:
66
        a = emu.frame()
67
        b = emu.step(2); c = emu.frame()
68
        assert b == c, "/step must return the halted frame counter"
69
        d = emu.step(2); e = emu.frame()
70
        assert d == e
71
        assert c - a > 0, "/step must advance the frame counter"
72
        assert (c - a) == (e - c), "equal steps must advance by equal amounts"
73
    finally:
74
        emu.resume()
75
76
77
def test_pause_resume_toggle(emu):
78
    emu.pause()
79
    assert emu.status()["paused"] is True
80
    emu.resume()
81
    assert emu.status()["paused"] is False
82
83
84
def test_unknown_path_404(emu):
85
    code, _ = _raw(emu, "GET", "/no-such-endpoint")
86
    assert code == 404
87
88
89
def test_step_requires_post(emu):
90
    code, _ = _raw(emu, "GET", "/step?frames=1")
91
    assert code == 405
92
93
94
def _contract_minor(emu) -> int:
95
    return int(str(emu.status()["contract"]).split(".")[1])
96
97
98
# -- contract 0.2: input injection (skipped on 0.1 servers) ------------------
99
100
def test_key_injection(emu):
101
    """/key accepts text= and code= with an optional down=, and rejects a
102
    request with neither. The key's *effect* is platform/program-specific and
103
    tested per-port; here we assert only the contract surface."""
104
    if _contract_minor(emu) < 2:
105
        pytest.skip("contract < 0.2 (no input injection)")
106
    emu.key(text="a", down=True)
107
    emu.key(text="a", down=False)
108
    emu.key(text="a")                    # tap (press then release)
109
    emu.key(code=32, down=True)
110
    emu.key(code=32, down=False)
111
    code, _ = _raw(emu, "POST", "/key")  # neither text nor code
112
    assert code == 400
113
114
115
def test_key_requires_post(emu):
116
    if _contract_minor(emu) < 2:
117
        pytest.skip("contract < 0.2")
118
    code, _ = _raw(emu, "GET", "/key?text=a")
119
    assert code == 405
120
121
122
# -- contract 0.3: memory write + audio (skipped on < 0.3 servers) -----------
123
124
def _scratch_ram(emu):
125
    """A writable RAM address for the platform. Most cores are flat with RAM at
126
    $0000, but the eZ80 Agon maps $0000-$03FFFF to flash (MOS ROM), so a write
127
    there is a silent no-op and the read-back returns ROM. RAM starts at
128
    $040000 there; $050000 is well clear of the MOS load area."""
129
    plat = (emu.status().get("platform") or "").lower()
130
    return 0x050000 if "agon" in plat else 0x0400
131
132
133
def test_mem_write_roundtrip(emu):
134
    """POST /mem pokes bytes; GET /mem reads them back (contract 0.3)."""
135
    if _contract_minor(emu) < 3:
136
        pytest.skip("contract < 0.3 (no memory write)")
137
    emu.pause()               # freeze so the machine can't overwrite between write+read
138
    try:
139
        addr, payload = _scratch_ram(emu), bytes([0xA5, 0x5A, 0x00, 0xFF])
140
        assert emu.mem_write(addr, payload) == len(payload)
141
        assert emu.mem(addr, len(payload)) == payload
142
    finally:
143
        emu.resume()
144
145
146
def test_mem_write_requires_post(emu):
147
    if _contract_minor(emu) < 3:
148
        pytest.skip("contract < 0.3")
149
    # GET /mem is the read path (200), never a write — writing needs POST.
150
    code, _ = _raw(emu, "GET", "/mem?addr=0&len=1")
151
    assert code == 200
152
153
154
def test_audio_is_valid_wav(emu):
155
    """GET /audio returns a parseable PCM WAV (contract 0.3)."""
156
    if _contract_minor(emu) < 3:
157
        pytest.skip("contract < 0.3 (no audio)")
158
    import io
159
    import wave
160
    emu.pause(); emu.step(4)          # produce a little audio deterministically
161
    w = wave.open(io.BytesIO(emu.audio()), "rb")
162
    assert w.getsampwidth() == 2      # signed 16-bit
163
    assert w.getnchannels() in (1, 2)
164
    assert w.getframerate() > 0
165
    emu.resume()
166
167
168
# -- contract 0.4: pointer injection (skipped on < 0.4 servers) --------------
169
170
def test_pointer_injection(emu):
171
    """/pointer accepts absolute (x,y) and relative (dx,dy) with an optional
172
    buttons= bitmask, and rejects a request with neither. The pointer's *effect*
173
    is platform-specific and tested per-port; here we assert the contract
174
    surface only."""
175
    if _contract_minor(emu) < 4:
176
        pytest.skip("contract < 0.4 (no pointer injection)")
177
    emu.pointer(x=10, y=20)                 # absolute move, buttons unchanged
178
    emu.pointer(x=10, y=20, buttons=1)      # primary button down
179
    emu.pointer(x=10, y=20, buttons=0)      # buttons released
180
    emu.pointer(dx=4, dy=-2)                # relative move
181
    code, _ = _raw(emu, "POST", "/pointer")  # neither x/y nor dx/dy
182
    assert code == 400
183
184
185
def test_pointer_readback(emu):
186
    """GET /pointer reports {x, y, buttons} as ints (contract 0.4). Values are
187
    platform-specific; we assert only shape."""
188
    if _contract_minor(emu) < 4:
189
        pytest.skip("contract < 0.4")
190
    emu.pause()
191
    try:
192
        emu.pointer(x=42, y=24, buttons=0)
193
        emu.step(1)
194
        p = emu.pointer_get()
195
        assert {"x", "y", "buttons"} <= p.keys()
196
        assert all(isinstance(p[k], int) for k in ("x", "y", "buttons"))
197
    finally:
198
        emu.resume()
199
200
201
def test_pointer_get_is_get(emu):
202
    if _contract_minor(emu) < 4:
203
        pytest.skip("contract < 0.4")
204
    # GET /pointer is the readback path (200); POST is the inject path.
205
    code, _ = _raw(emu, "GET", "/pointer")
206
    assert code == 200
207
208
209
def test_reset(emu):
210
    """/reset responds and is POST-only. Runs last (it restarts the machine)."""
211
    if _contract_minor(emu) < 2:
212
        pytest.skip("contract < 0.2")
213
    code, _ = _raw(emu, "GET", "/reset")
214
    assert code == 405
215
    emu.reset()
216
217
218
# -- contract 0.5: pad injection (skipped on < 0.5 servers) ------------------
219
220
def test_pad_injection(emu):
221
    """/pad presents a virtual controller and sets what is held. The pad's
222
    *effect* is platform-specific and tested per-port; the contract surface is
223
    asserted here."""
224
    if _contract_minor(emu) < 5:
225
        pytest.skip("contract < 0.5 (no pad injection)")
226
    emu.pad(buttons=emu.PAD_A)                      # A held, pad plugged in
227
    emu.pad(buttons=emu.PAD_A | emu.PAD_LEFT)       # two at once
228
    emu.pad(buttons=0)                              # all released, still there
229
    emu.pad(index=1, buttons=emu.PAD_B)             # a second pad
230
    emu.pad(connected=False)                        # unplug pad 0
231
    emu.pad(connected=False, index=1)
232
233
234
def test_pad_readback(emu):
235
    """GET /pad reports {index, connected, buttons} (contract 0.5)."""
236
    if _contract_minor(emu) < 5:
237
        pytest.skip("contract < 0.5")
238
    emu.pause()
239
    try:
240
        emu.pad(buttons=emu.PAD_A | emu.PAD_RIGHT)
241
        emu.step(1)
242
        st = emu.pad_get()
243
        assert isinstance(st["buttons"], int)
244
        assert isinstance(st["connected"], bool)
245
        assert st["index"] == 0
246
        assert st["connected"] is True, "injected pad reports itself absent"
247
        # The mask must survive the round trip: this is the check that
248
        # separates "the server accepted it" from "the backend stored it".
249
        assert st["buttons"] == emu.PAD_A | emu.PAD_RIGHT, (
250
            f"held {st['buttons']:#x}, injected "
251
            f"{emu.PAD_A | emu.PAD_RIGHT:#x}")
252
    finally:
253
        emu.pad(buttons=0)
254
        emu.resume()
255
256
257
def test_pad_is_a_level_not_an_event(emu):
258
    """A pad HOLDS. Unlike a key tap, an injected press must survive stepping —
259
    a program that samples input once a frame has to still see it on frame 10.
260
    This is the difference that makes /pad worth having rather than reusing
261
    /key, so it is asserted rather than assumed."""
262
    if _contract_minor(emu) < 5:
263
        pytest.skip("contract < 0.5")
264
    emu.pause()
265
    try:
266
        emu.pad(buttons=emu.PAD_B)
267
        emu.step(10)
268
        assert emu.pad_get()["buttons"] == emu.PAD_B, (
269
            "the held mask did not survive 10 frames — /pad is behaving "
270
            "like a momentary key tap")
271
    finally:
272
        emu.pad(buttons=0)
273
        emu.resume()
274
275
276
def test_pad_unplug_is_observable(emu):
277
    """connected=0 must actually unplug. A program that offers two-player only
278
    when a second controller is present needs both answers to be reachable, so
279
    a backend that always reports 'connected' is not conformant."""
280
    if _contract_minor(emu) < 5:
281
        pytest.skip("contract < 0.5")
282
    emu.pause()
283
    try:
284
        emu.pad(connected=False)
285
        if emu.pad_get()["connected"]:
286
            pytest.skip(
287
                "a physical controller is present at index 0, so `connected` "
288
                "cannot go false — the SPEC says it reports anything present, "
289
                "real or virtual. Re-run with the controller unplugged.")
290
        emu.pad(buttons=0)                     # plugging in is what we assert
291
        assert emu.pad_get()["connected"] is True, (
292
            "injecting did not make the pad present")
293
        emu.pad(connected=False)
294
        assert emu.pad_get()["connected"] is False, (
295
            "the pad still reports connected after connected=0")
296
    finally:
297
        emu.pad(connected=False)
298
        emu.resume()
299
300
301
# -- contract 0.6: performance metrics (skipped on < 0.6 servers) ------------
302
303
def test_capabilities_list_the_groups(emu):
304
    """/status.capabilities names each group the server serves. Always the
305
    four 0.1 groups; at 0.6 also perf, and the groups below it the ladder
306
    implies."""
307
    if _contract_minor(emu) < 6:
308
        pytest.skip("contract < 0.6 (no capabilities)")
309
    caps = emu.capabilities()
310
    for c in ("mem", "regs", "screenshot", "step", "key", "mem-write", "audio", "pointer", "pad", "perf"):
311
        assert c in caps, (c, caps)
312
313
314
def _perf_on(emu):
315
    emu.perf_config(enabled=True)
316
317
318
def test_perf_shape(emu):
319
    """GET /perf's sections and their arithmetic: work = total - idle, main =
320
    work - irq, the windows asked for, raw frames oldest first. The numbers
321
    themselves are the platform's and the program's; only their relations are
322
    the contract's."""
323
    if _contract_minor(emu) < 6:
324
        pytest.skip("contract < 0.6")
325
    _perf_on(emu)
326
    emu.pause()
327
    try:
328
        emu.step(10)
329
        p = emu.perf(windows=[1, "session"], frames=5)
330
    finally:
331
        emu.resume()
332
    assert p["enabled"] is True
333
    b = p["budget"]
334
    for k in ("targetFps", "vsyncHz", "cyclesPerFrame", "scanlinesPerFrame", "cyclesPerScanline",
335
              "budgetCycles", "framesPerBudget", "historyFrames", "totalFrames", "idleMode"):
336
        assert k in b, k
337
    assert b["cyclesPerFrame"] > 0 and b["budgetCycles"] > 0
338
    c = p["current"]
339
    assert c["workCycles"] == c["totalCycles"] - c["idleCycles"], c
340
    assert c["mainCycles"] == c["workCycles"] - c["irqCycles"], c
341
    assert isinstance(c["overBudget"], bool)
342
    assert [w["name"] for w in p["windows"]] == ["1s", "session"], p["windows"]
343
    for w in p["windows"]:
344
        for series in ("work", "budgetWork", "total", "irq", "hostUs"):
345
            s = w[series]
346
            assert s["min"] <= s["p50"] <= s["p95"] <= s["p99"] <= s["max"], (series, s)
347
    assert isinstance(p["zones"], list) and isinstance(p["platform"], dict)
348
    frames = [f["frame"] for f in p["frames"]]
349
    assert 1 <= len(frames) <= 5 and frames == sorted(frames), frames
350
    assert frames[-1] == c["frame"], (frames, c["frame"])
351
352
353
def test_perf_since_misses_no_frame(emu):
354
    """since=<f> returns every completed frame after f, so a harness that
355
    polls with the last frame it saw gets each frame exactly once."""
356
    if _contract_minor(emu) < 6:
357
        pytest.skip("contract < 0.6")
358
    _perf_on(emu)
359
    emu.pause()
360
    try:
361
        emu.step(2)
362
        last = emu.perf(zones=False)["current"]["frame"]
363
        emu.step(5)
364
        got = [f["frame"] for f in emu.perf(zones=False, since=last)["frames"]]
365
    finally:
366
        emu.resume()
367
    assert got == list(range(last + 1, last + 6)), (last, got)
368
369
370
def test_perf_zones_name_a_marker_region(emu):
371
    """POST /perf's zones replace the list, and a {name, marker} entry names
372
    the guest's marker region; the zone is reported under that name whether
373
    or not the program has entered it yet."""
374
    if _contract_minor(emu) < 6:
375
        pytest.skip("contract < 0.6")
376
    try:
377
        emu.perf_config(enabled=True, zones=[{"name": "conformance", "marker": 63}])
378
        names = [z["name"] for z in emu.perf()["zones"]]
379
        assert "conformance" in names, names
380
    finally:
381
        emu.perf_config(zones=[])
382
383
384
def test_perf_rejects_a_bad_body(emu):
385
    if _contract_minor(emu) < 6:
386
        pytest.skip("contract < 0.6")
387
    code, _ = _raw(emu, "POST", "/perf", b"not json")
388
    assert code == 400