OXYGENHUB
◆ main enhanced-x16-emulator / tests / mutation_check.py
37.2 KiB · Python · last changed chore(): initial commit 25 days ago source blame history raw
1 
#!/usr/bin/env python3
2 
"""Check that the tests actually fail when the emulator is wrong.
3
4 
Every test here is written to pass against correct behaviour, which says
5 
nothing about whether it would notice incorrect behaviour. A test that asserts
6 
the wrong thing, or reads a value it never actually exercises, passes exactly
7 
as convincingly as one that works.
8
9 
So: break the emulator on purpose, one small change at a time, and check the
10 
test that should care goes red. A mutation that survives means either the test
11 
is not covering what it claims, or the code it changed does not matter.
12
13 
    python tests/mutation_check.py --build-dir build
14
15 
Every mutation is reverted afterwards, including on failure or interrupt. The
16 
source is restored from a copy taken before each edit, so an aborted run leaves
17 
the tree as it found it.
18 
"""
19
20 
import argparse
21 
import os
22 
import shutil
23 
import subprocess
24 
import sys
25 
import tempfile
26 
from pathlib import Path
27
28 
ROOT = Path(__file__).resolve().parent.parent
29
30 
CMAKE = os.environ.get("CMAKE", "cmake")
31 
CTEST = os.environ.get("CTEST", "ctest")
32
33 
# Each entry breaks one specific behaviour and names the tests that should
34 
# notice. `count` is how many places the pattern is expected to match, and a
35 
# mismatch is reported rather than applied -- a pattern that has quietly stopped
36 
# matching would otherwise look exactly like a mutation the tests caught.
37 
MUTATIONS = [
38 
    {
39 
        "name": "PCM FIFO accepts one byte too many",
40 
        "file": "src/vera_pcm.c",
41 
        # audio_fifo.v:27  assign full = (wridx_next == rdidx_r);
42 
        # The ring stores one fewer than its 4096 cells so that full and empty
43 
        # stay distinguishable. Off by one here makes a full FIFO read as empty.
44 
        "find": "if (fifo_cnt < sizeof(fifo) - 1) {",
45 
        "into": "if (fifo_cnt <= sizeof(fifo) - 1) {",
46 
        "count": 1,
47 
        "caught_by": ["vera_pcm"],
48 
    },
49 
    {
50 
        "name": "PCM almost-empty boundary is off by one",
51 
        "file": "src/vera_pcm.c",
52 
        # audio_fifo.v:28  assign almost_empty = fifo_count < 12'd1024;
53 
        # Drives the AFLOW interrupt: firing a byte late or early changes when
54 
        # a program is asked for more audio.
55 
        "find": "return fifo_cnt < 1024;",
56 
        "into": "return fifo_cnt <= 1024;",
57 
        "count": 1,
58 
        "caught_by": ["vera_pcm"],
59 
    },
60 
    # No mutation for the AUDIO_RATE fold. The test asserts the RTL value
61 
    # (top.v:480 stores write_data raw) and marks the emulator's fold as a
62 
    # divergence, so it deliberately does not pin `256 - val`. A mutation of
63 
    # that expression would survive, and correctly so. Add one once the fold is
64 
    # removed and the register stores what was written.
65 
    {
66 
        "name": "VERA debug reads report a constant",
67 
        "file": "src/video.c",
68 
        # The sweep in test_vera_debug_read.c records exactly which registers
69 
        # read differently to the debugger than to the machine. Breaking the
70 
        # debug path further changes that set, which the recorded count fails
71 
        # on -- so the sweep cannot quietly stop measuring anything.
72 
        "find": "if (debugOn) return video_get_dc_value(i);",
73 
        "into": "if (debugOn) return 0;",
74 
        "count": 1,
75 
        "caught_by": ["vera_debug_read"],
76 
    },
77 
    {
78 
        "name": "VERA DCSEL is truncated to four bits",
79 
        "file": "src/video.c",
80 
        # top.v:86 declares dc_select_r as [5:0] and top.v:336 fills it from
81 
        # write_data[6:1]. Narrowing it silently aliases the high banks onto
82 
        # the low ones.
83 
        "find": "io_dcsel = (value >> 1) & 0x3f;",
84 
        "into": "io_dcsel = (value >> 1) & 0x0f;",
85 
        "count": 1,
86 
        "caught_by": ["vera_regbank"],
87 
    },
88 
    {
89 
        "name": "VERA address increment uses a power-of-two step",
90 
        "file": "src/video.c",
91 
        # addr_data.v:193 gives 40 for code 0x0B. The non-power-of-two steps
92 
        # exist so a program can walk tile rows, and are exactly what an
93 
        # implementation "tidying" the series to 1 << n would lose.
94 
        "find": "\t40,  -40,",
95 
        "into": "\t48,  -48,",
96 
        "count": 1,
97 
        "caught_by": ["vera_video"],
98 
    },
99 
    {
100 
        "name": "VERA reads the increment from the wrong bits of ADDRx_H",
101 
        "file": "src/video.c",
102 
        # video.c:2796. Bits 7:3 are the decrement flag and increment code;
103 
        # shifting by one more drops the decrement bit entirely.
104 
        "find": "io_inc[io_addrsel]  = value >> 3;",
105 
        "into": "io_inc[io_addrsel]  = value >> 4;",
106 
        "count": 1,
107 
        "caught_by": ["vera_video"],
108 
    },
109 
    {
110 
        "name": "PSG sawtooth loses its pulse-width shaping",
111 
        "file": "src/vera_psg.c",
112 
        # Reverts the waveform to the R47 form. psg.v:163 (R48) XORs the saw
113 
        # with the inverted pulse width; R47 read the phase directly. This is
114 
        # the mutation that proves pinning psg.v to R48 does any work.
115 
        "find": "case WF_SAWTOOTH: v = (ch->phase >> 11) ^ ((ch->pw ^ 0x3f) & 0x3f); break;",
116 
        "into": "case WF_SAWTOOTH: v = (ch->phase >> 11); break;",
117 
        "count": 1,
118 
        "caught_by": ["vera_psg"],
119 
    },
120 
    {
121 
        "name": "PSG voices keep running while silenced",
122 
        "file": "src/vera_psg.c",
123 
        # psg.v:140 holds a voice at phase zero unless a side is enabled, so
124 
        # unmuting restarts the waveform rather than resuming it.
125 
        "find": "uint32_t new_phase = (ch->left || ch->right) ? ((ch->phase + ch->freq) & 0x1FFFF) : 0;",
126 
        "into": "uint32_t new_phase = ((ch->phase + ch->freq) & 0x1FFFF);",
127 
        "count": 1,
128 
        "caught_by": ["vera_psg"],
129 
    },
130 
    {
131 
        "name": "PSG noise LFSR loses a tap",
132 
        "file": "src/vera_psg.c",
133 
        # psg.v:128 taps 1, 2, 4 and 15. A wrong tap set still sounds like
134 
        # noise, so only the exact sequence catches it.
135 
        "find": "(((noise_state >> 1) ^ (noise_state >> 2) ^ (noise_state >> 4) ^ (noise_state >> 15)) & 1)",
136 
        "into": "(((noise_state >> 1) ^ (noise_state >> 2) ^ (noise_state >> 15)) & 1)",
137 
        "count": 1,
138 
        "caught_by": ["vera_psg"],
139 
    },
140 
    {
141 
        "name": "PSG waveform is decoded from the wrong bits",
142 
        "file": "src/vera_psg.c",
143 
        # psg.v:44 cur_waveform = cur_channel_attr_r[31:30], the top two bits
144 
        # of the fourth attribute byte.
145 
        "find": "channels[ch].waveform = val >> 6;",
146 
        "into": "channels[ch].waveform = (val >> 5) & 3;",
147 
        "count": 1,
148 
        "caught_by": ["vera_psg"],
149 
    },
150 
    {
151 
        "name": "VERA ISR clears on a written zero",
152 
        "file": "src/video.c",
153 
        # top.v:440-442 clear a latch when the written bit is one:
154 
        #   irq_status_vsync_next = irq_status_vsync_r & !write_data[0];
155 
        # Inverting that acknowledges every pending source whenever a program
156 
        # clears one, which loses interrupts rather than reporting them wrong.
157 
        "find": "isr &= value ^ 0xff;",
158 
        "into": "isr &= value;",
159 
        "count": 1,
160 
        "caught_by": ["vera_irq"],
161 
    },
162 
    {
163 
        "name": "VERA IEN stores a fifth enable bit",
164 
        "file": "src/video.c",
165 
        # top.v:175 reads bits 5:4 as 2'b0 and top.v:433-436 give them no write
166 
        # target, so a program cannot store anything there.
167 
        "find": "ien = value & 0xF;",
168 
        "into": "ien = value & 0x1F;",
169 
        "count": 1,
170 
        "caught_by": ["vera_irq"],
171 
    },
172 
    {
173 
        "name": "VERA IRQLINE bit 8 comes from the wrong IEN bit",
174 
        "file": "src/video.c",
175 
        # top.v:432  irq_line_next[8] = write_data[7];
176 
        # The compare value is split across $9F26 and $9F28; taking the high
177 
        # bit from bit 6 puts it where the scanline readback is instead.
178 
        "find": "irq_line = (irq_line & 0xFF) | ((value >> 7) << 8);",
179 
        "into": "irq_line = (irq_line & 0xFF) | (((value >> 6) & 1) << 8);",
180 
        "count": 1,
181 
        "caught_by": ["vera_irq"],
182 
    },
183 
    {
184 
        "name": "VERA ISR bit 3 latches instead of following the FIFO",
185 
        "file": "src/video.c",
186 
        # top.v:176 takes bit 3 from the audio_fifo_low wire, not a latch, so
187 
        # it reports the FIFO level at the instant of the read.
188 
        "find": "case 0x07: return isr | (pcm_is_fifo_almost_empty() ? 8 : 0);",
189 
        "into": "case 0x07: return isr;",
190 
        "count": 1,
191 
        "caught_by": ["vera_irq"],
192 
    },
193 
    {
194 
        "name": "VERA asserts IRQ regardless of the enables",
195 
        "file": "src/video.c",
196 
        # top.v:1200  assign extbus_irq_n = (irq_status & irq_enable) == 0;
197 
        # Dropping the mask interrupts on every VSYNC whether or not the
198 
        # program asked for one.
199 
        "find": "return (tmp_isr & ien) != 0;",
200 
        "into": "return tmp_isr != 0;",
201 
        "count": 1,
202 
        "caught_by": ["vera_irq"],
203 
    },
204 
    {
205 
        "name": "VERA IEN bit 6 reports a stored bit, not the scanline",
206 
        "file": "src/video.c",
207 
        # top.v:175 sources bit 6 from scanline[8], live, with nothing writing
208 
        # it. A debugger reading IEN would otherwise show a raster position
209 
        # that never moves.
210 
        "find": "((scanline & 0x100) >> 2)",
211 
        "into": "0",
212 
        "count": 1,
213 
        "caught_by": ["vera_irq"],
214 
    },
215 
    {
216 
        "name": "VERA collision nibble accumulates across frames",
217 
        "file": "src/video.c",
218 
        # sprite_renderer.v:402  frame_collision_mask_next = cur_collision_mask_r;
219 
        # Assigned unconditionally at frame_done, so the nibble describes the
220 
        # frame just ended. ORing instead makes a collision permanent.
221 
        "find": "isr = (isr & 0xf) | sprite_line_collisions;",
222 
        "into": "isr = isr | sprite_line_collisions;",
223 
        "count": 1,
224 
        "caught_by": ["vera_irq"],
225 
    },
226 
    {
227 
        "name": "VERA sprite collision mask is read from the wrong nibble",
228 
        "file": "src/video.c",
229 
        # sprite_renderer.v:82  wire [3:0] sprite_attr_collision_mask = sprite_attr[23:20];
230 
        # The top half of attribute byte 6; the bottom half is z-depth and the
231 
        # flip bits.
232 
        "find": "props->sprite_collision_mask = sprite_data[sprite][6] & 0xf0;",
233 
        "into": "props->sprite_collision_mask = (sprite_data[sprite][6] & 0x0f) << 4;",
234 
        "count": 1,
235 
        "caught_by": ["vera_irq"],
236 
    },
237 
    {
238 
        "name": "VERA sprites collide where their masks differ",
239 
        "file": "src/video.c",
240 
        # sprite_renderer.v:328 uses the mask directly:
241 
        #   ... ? (linebuf_rddata[15:12] & sprite_collision_mask_r) : 4'b0;
242 
        # It was inverted until 4f46cb32 (2023-01-26). This mutation restores
243 
        # that form, which is the one bug the RTL's own history records here.
244 
        "find": "sprite_line_collisions |= sprite_line_mask[line_x] & props->sprite_collision_mask;",
245 
        "into": "sprite_line_collisions |= sprite_line_mask[line_x] & ~props->sprite_collision_mask;",
246 
        "count": 1,
247 
        "caught_by": ["vera_irq"],
248 
    },
249 
    # No mutation for the ISR write clearing bits 7:4. The test records that as
250 
    # a divergence rather than asserting it, so mutating video.c:2910 towards
251 
    # the RTL would make the marker pass unexpectedly -- which is a failure by
252 
    # design, not a caught mutation. Add one once the write stops reaching the
253 
    # nibble.
254 
    {
255 
        "name": "direct-page cycle penalty is never cleared",
256 
        "file": "src/cpu/fake6502.c",
257 
        # step6502()'s copy, four-space indented. exec6502()'s is eight-space
258 
        # and is dead code -- mutating that proves nothing, which is exactly
259 
        # what an earlier version of this list accidentally did.
260 
        "find": "\n    penaltyd = 0;\n",
261 
        "into": "\n",
262 
        "count": 1,
263 
        "caught_by": ["cpu_smoke"],
264 
    },
265 
    {
266 
        "name": "page-cross penalty is dropped",
267 
        "file": "src/cpu/fake6502.c",
268 
        "find": "if (penaltyop && penaltyaddr) clockticks6502++;",
269 
        "into": "if (0) clockticks6502++;",
270 
        "count": 2,  # exec6502() and step6502() both have it
271 
        # Not klaus: its functional tests check results, not cycle counts, so
272 
        # they are blind to timing. Worth knowing.
273 
        "caught_by": ["cpu_loadstore"],
274 
    },
275 
    {
276 
        "name": "decimal mode costs no extra cycle",
277 
        "file": "src/cpu/instructions.h",
278 
        "find": "clockticks6502 += (uint32_t)(!regs.is65c816);",
279 
        "into": "clockticks6502 += (uint32_t)(0);",
280 
        "count": 2,  # adc() and sbc()
281 
        "caught_by": ["cpu_decimal"],
282 
    },
283 
    {
284 
        "name": "VIA is not told a read is for the debugger",
285 
        "file": "src/memory.c",
286 
        "find": "return via1_read(address & 0xf, debugOn);",
287 
        "into": "return via1_read(address & 0xf, false);",
288 
        "count": 1,
289 
        "caught_by": ["debugon_contract"],
290 
    },
291 
    {
292 
        "name": "a debug read of slow I/O costs cycles",
293 
        "file": "src/memory.c",
294 
        # The write path guards the same way, so this needs the read's own
295 
        # continuation to stay unique to it. The read's guard lives in
296 
        # io_read(), which is a function in its own right so that the value it
297 
        # returns can be traced -- hence one tab of indent here, not three.
298 
        "find": "if (!debugOn && address >= 0x9fa0) {\n\t\t// slow IO5-7 range\n\t\tclockticks6502 += 3;\n\t}\n\tif (address >= 0x9f00 && address < 0x9f10) {\n\t\treturn via1_read",
299 
        "into": "if (address >= 0x9fa0) {\n\t\t// slow IO5-7 range\n\t\tclockticks6502 += 3;\n\t}\n\tif (address >= 0x9f00 && address < 0x9f10) {\n\t\treturn via1_read",
300 
        "count": 1,
301 
        "caught_by": ["debugon_contract"],
302 
    },
303 
    {
304 
        "name": "a watchpoint swallows the store",
305 
        "file": "src/memory.c",
306 
        "find": "\t\tDEBUGBreakOnWatchpoint();\n\t}",
307 
        "into": "\t\tDEBUGBreakOnWatchpoint();\n\t\treturn;\n\t}",
308 
        "count": 1,
309 
        "caught_by": ["watchpoint_purity"],
310 
    },
311 
    {
312 
        "name": "the overflow flag is never set",
313 
        "file": "src/cpu/support.h",
314 
        # Masking with 0x00 rather than replacing the condition with 0: the
315 
        # macro parameters stay referenced, so this compiles under -Werror
316 
        # where an `if (0)` leaves them unused and GCC refuses.
317 
        "find": "if (((n) ^ (m)) & ((n) ^ (o)) & 0x80) setoverflow();",
318 
        "into": "if (((n) ^ (m)) & ((n) ^ (o)) & 0x00) setoverflow();",
319 
        "count": 1,
320 
        "caught_by": ["cpu_adc_sbc", "klaus"],
321 
    },
322 
    {
323 
        "name": "zero page indexing does not wrap inside page zero",
324 
        "file": "src/cpu/support.h",
325 
        "find": "((uint16_t) ((uint8_t) (regs.dp & 0x00FF)) + (offset & 0xFF))",
326 
        "into": "((uint16_t) ((uint8_t) (regs.dp & 0x00FF)) + offset)",
327 
        "count": 1,
328 
        "caught_by": ["cpu_loadstore", "klaus"],
329 
    },
330 
    # The three below exist to check ProcessorTests specifically: a baseline
331 
    # comparison is only worth anything if it moves when behaviour moves.
332 
    {
333 
        "name": "the negative flag is never set",
334 
        "file": "src/cpu/support.h",
335 
        "find": "else if ((n) & 0x0080) setsign();",
336 
        "into": "else if ((n) & 0x0000) setsign();",
337 
        "count": 1,
338 
        "requires": "tests/pt/wdc65c02.bin",
339 
        "caught_by": ["processor_tests", "klaus"],
340 
    },
341 
    {
342 
        "name": "the zero flag is set from the wrong width",
343 
        "file": "src/cpu/support.h",
344 
        "find": "else if ((n) & 0x00FF) clearzero();",
345 
        "into": "else if ((n) & 0x000F) clearzero();",
346 
        "count": 1,
347 
        "requires": "tests/pt/wdc65c02.bin",
348 
        "caught_by": ["processor_tests", "klaus"],
349 
    },
350 
    {
351 
        # Timing only: results stay correct, so klaus cannot see it.
352 
        "name": "a taken branch costs no extra cycle",
353 
        "file": "src/cpu/instructions.h",
354 
        "find": "clockticks6502++;",
355 
        "into": "clockticks6502 += 0;",
356 
        "count": 1,
357 
        "requires": "tests/pt/wdc65c02.bin",
358 
        "caught_by": ["processor_tests"],
359 
    },
360 
    # The 65C816 fixes below. Each is checked by the suite for the mode it
361 
    # applies to, so a revert shows up as a named opcode rather than a number.
362 
    {
363 
        "name": "indirect addressing ignores the data bank",
364 
        "file": "src/cpu/modes.h",
365 
        "find": "ea = addr_with_db((uint16_t)read6502(direct_page_add(eahelp), 0)",
366 
        "into": "ea = ((uint16_t)read6502(direct_page_add(eahelp), 0)",
367 
        "count": 2,  # ind0 and indx
368 
        "requires": "tests/pt/816-emu.bin",
369 
        "caught_by": ["processor_tests_816_emu"],
370 
    },
371 
    {
372 
        "name": "the new stack instructions wrap inside page one",
373 
        "file": "src/cpu/instructions.h",
374 
        "find": "regs.dp = pull16_long();",
375 
        "into": "regs.dp = pull16();",
376 
        "count": 1,
377 
        "requires": "tests/pt/816-emu.bin",
378 
        "caught_by": ["processor_tests_816_emu"],
379 
    },
380 
    {
381 
        "name": "a 16-bit operand costs no extra cycle",
382 
        "file": "src/cpu/support.h",
383 
        "find": "(penaltym = addressing_is_acc ? 0 : (uint8_t)(n))",
384 
        "into": "(penaltym = addressing_is_acc ? 0 : (uint8_t)((n) & 0x00))",
385 
        "count": 1,
386 
        "requires": "tests/pt/816-native.bin",
387 
        "caught_by": ["processor_tests_816_native"],
388 
    },
389 
    {
390 
        "name": "BIT reads the wrong half of a 16-bit operand",
391 
        "file": "src/cpu/instructions.h",
392 
        "find": "uint16_t top = memory_16bit() ? (uint16_t)(value >> 8) : value;",
393 
        "into": "uint16_t top = value;",
394 
        "count": 1,
395 
        "requires": "tests/pt/816-native.bin",
396 
        "caught_by": ["processor_tests_816_native"],
397 
    },
398 
    {
399 
        # The two CPUs correct decimal subtraction differently; using one
400 
        # algorithm for both was the fault this replaced.
401 
        "name": "the 65C02 decimal SBC algorithm is used on the 65C816",
402 
        "file": "src/cpu/instructions.h",
403 
        "find": "            int16_t adjusted;\n            if (regs.is65c816) {",
404 
        "into": "            int16_t adjusted;\n            if (0) {",
405 
        "count": 1,
406 
        "requires": "tests/pt/816-emu.bin",
407 
        "caught_by": ["processor_tests_816_emu"],
408 
    },
409 
    {
410 
        # WAI is one of the four opcodes ProcessorTests records in a form it
411 
        # cannot compare, so this is a table change no other test can see. It
412 
        # is the mutation that shows opcode_spec covers something new rather
413 
        # than repeating the traces.
414 
        "name": "a documented cycle count is changed",
415 
        "file": "src/cpu/65c02.opcodes",
416 
        "find": "wai imp 3 $cb",
417 
        "into": "wai imp 4 $cb",
418 
        "count": 1,
419 
        "caught_by": ["opcode_spec"],
420 
    },
421 
    {
422 
        "name": "VERA reset keeps the pending sprite collisions",
423 
        "file": "src/video.c",
424 
        # sprite_renderer.v:416  cur_collision_mask_r <= 0;
425 
        # The half of the sprite reset that is right. Dropping it carries
426 
        # collisions from before a reset into the frame after it.
427 
        "find": "\tsprite_line_collisions = 0;\n\n\tvga_scan_pos_x = 0;",
428 
        "into": "\tvga_scan_pos_x = 0;",
429 
        "count": 1,
430 
        "caught_by": ["vera_reset"],
431 
    },
432 
    {
433 
        "name": "VERA sprite z-depth is ignored",
434 
        "file": "src/video.c",
435 
        # sprite_renderer.v:81  wire [1:0] sprite_attr_z = sprite_attr[19:18];
436 
        # A z-depth of zero is not drawn, which is how a guest retires a
437 
        # sprite. test_vera_reset.c measures a reset by whether sprites stop,
438 
        # so this breaks the channel the rest of that file reads.
439 
        "find": "props->sprite_zdepth = (sprite_data[sprite][6] >> 2) & 3;",
440 
        "into": "props->sprite_zdepth = 1;",
441 
        "count": 1,
442 
        "caught_by": ["vera_reset"],
443 
    },
444 
    {
445 
        "name": "VERA layer map base is shifted one bit short",
446 
        "file": "src/video.c",
447 
        # layer_renderer.v:107  wire [14:0] map_addr = {map_baseaddr, 7'b0} + map_idx[15:1];
448 
        # The register names a byte address of register x 512. A shift of 8
449 
        # puts the tilemap at half the offset the guest asked for.
450 
        "find": "props->map_base       = reg_layer[layer][1] << 9;",
451 
        "into": "props->map_base       = reg_layer[layer][1] << 8;",
452 
        "count": 1,
453 
        "caught_by": ["vera_layer"],
454 
    },
455 
    {
456 
        "name": "VERA tile base takes in the two size bits",
457 
        "file": "src/video.c",
458 
        # top.v:383-384 forces tile_baseaddr[1:0] to zero on write; bits 1 and 0
459 
        # of the register are the tile height and width. Letting them into the
460 
        # address moves the tile data whenever the tile size changes.
461 
        "find": "props->tile_base      = (reg_layer[layer][2] & 0xFC) << 9;",
462 
        "into": "props->tile_base      = (reg_layer[layer][2] & 0xFF) << 9;",
463 
        "count": 1,
464 
        "caught_by": ["vera_layer"],
465 
    },
466 
    {
467 
        "name": "VERA map width starts at 16 tiles",
468 
        "file": "src/video.c",
469 
        # layer_renderer.v:100 masks the column index to 5 bits at map_width 0,
470 
        # so the narrowest map is 32 tiles. Starting an octave low halves every
471 
        # map and moves the end of the tilemap with it.
472 
        "find": "props->mapw_log2 = 5 + ((reg_layer[layer][0] >> 4) & 3);",
473 
        "into": "props->mapw_log2 = 4 + ((reg_layer[layer][0] >> 4) & 3);",
474 
        "count": 1,
475 
        "caught_by": ["vera_layer"],
476 
    },
477 
    {
478 
        "name": "VERA layer 1 registers are written to layer 0",
479 
        "file": "src/video.c",
480 
        # top.v:219-225 gives layer 1 its own registers at 5'h14-5'h1A. One
481 
        # bank behind both sets of addresses would leave the two layers sharing
482 
        # a configuration.
483 
        "find": "reg_layer[1][reg - 0x14] = value;",
484 
        "into": "reg_layer[0][reg - 0x14] = value;",
485 
        "count": 1,
486 
        "caught_by": ["vera_layer"],
487 
    },
488 
    # No mutation for the scroll high bytes reading back whole. That is
489 
    # recorded as a divergence rather than asserted, so mutating video.c
490 
    # towards the RTL would make the marker pass unexpectedly -- a failure by
491 
    # design, not a caught mutation.
492 
    {
493 
        "name": "VERA layer row does not wrap at the layer height",
494 
        "file": "src/video.c",
495 
        # layer_renderer.v:86-89 masks the tile row to the map height. Without
496 
        # the mask a tall scroll walks off the end of the tilemap instead of
497 
        # coming back round to the top.
498 
        "find": "return (y + props->vscroll) & (props->layerh_max);",
499 
        "into": "return y + props->vscroll;",
500 
        "count": 1,
501 
        "caught_by": ["vera_layer_rows"],
502 
    },
503 
    {
504 
        "name": "VERA map height starts at 16 tiles",
505 
        "file": "src/video.c",
506 
        # layer_renderer.v:86 masks to 5 bits at map_height 0, so the shortest
507 
        # map is 32 tiles. An octave low halves the wrap point.
508 
        "find": "props->maph_log2 = 5 + ((reg_layer[layer][0] >> 6) & 3);",
509 
        "into": "props->maph_log2 = 4 + ((reg_layer[layer][0] >> 6) & 3);",
510 
        "count": 1,
511 
        "caught_by": ["vera_layer_rows"],
512 
    },
513 
    {
514 
        "name": "VERA tile height is read from the tile width bit",
515 
        "file": "src/video.c",
516 
        # top.v:386-387 takes height from write_data[1] and width from
517 
        # write_data[0]. Swapping them changes which bits of the scrolled line
518 
        # index select the row, per layer_renderer.v:82.
519 
        "find": "props->tileh_log2 = 3 + ((reg_layer[layer][2] >> 1) & 1);",
520 
        "into": "props->tileh_log2 = 3 + (reg_layer[layer][2] & 1);",
521 
        "count": 1,
522 
        "caught_by": ["vera_layer_rows"],
523 
    },
524 
    {
525 
        "name": "VERA default palette entry is wrong",
526 
        "file": "src/video.c",
527 
        # palette_ram.mem is the table baked into the FPGA bitstream, so the
528 
        # default palette is hardware and not a ROM convention. One edited
529 
        # entry in a 256-entry table is how this actually goes wrong, and the
530 
        # only thing that notices is a comparison against the file.
531 
        "find": "0x08f,0xbbb,",
532 
        "into": "0x08f,0xbbc,",
533 
        "count": 1,
534 
        "caught_by": ["vera_palette"],
535 
    },
536 
    {
537 
        "name": "VIA IER reads back without bit 7",
538 
        "file": "src/via.c",
539 
        # W65C22S p27, Table 2-12 note 3: "If a read of this register is done,
540 
        # bit 7 will be Logic 1 and all other bits will reflect their
541 
        # enable/disable state."
542 
        "find": "return via->registers[14] | 0x80;",
543 
        "into": "return via->registers[14];",
544 
        "count": 1,
545 
        "caught_by": ["via_irq"],
546 
    },
547 
    {
548 
        "name": "VIA IER set and clear are the wrong way round",
549 
        "file": "src/via.c",
550 
        # W65C22S p26: bit 7 of the written value selects between setting and
551 
        # clearing the enables. Inverted, every attempt to disable an interrupt
552 
        # enables it instead.
553 
        "find": "if (value & 0x80) {",
554 
        "into": "if (!(value & 0x80)) {",
555 
        "count": 1,
556 
        "caught_by": ["via_irq"],
557 
    },
558 
    {
559 
        "name": "VIA IFR bit 7 ignores the enables",
560 
        "file": "src/via.c",
561 
        # W65C22S p26: IRQ = IFR6 & IER6 | IFR5 & IER5 | ... | IFR0 & IER0.
562 
        # Dropping the enable half reports an interrupt for a flag nobody asked
563 
        # to be told about.
564 
        "find": "irq = (ifr & via->registers[14]) != 0;",
565 
        "into": "irq = ifr != 0;",
566 
        "count": 1,
567 
        "caught_by": ["via_irq"],
568 
    },
569 
    {
570 
        "name": "VIA debug read of T1C-L acknowledges the timer",
571 
        "file": "src/via.c",
572 
        # W65C22S p27, Table 2-11: reading T1C-L low clears the T1 flag. That
573 
        # is a side effect the debugger must not perform, or opening a memory
574 
        # view on $9F14 acknowledges the guest's timer interrupt for it.
575 
        "find": "if (!debug) via->registers[13] &= ~0x40;",
576 
        "into": "via->registers[13] &= ~0x40;",
577 
        "count": 1,
578 
        "caught_by": ["via_irq"],
579 
    },
580 
    {
581 
        "name": "VIA T1 one-shot and free-run are swapped",
582 
        "file": "src/via.c",
583 
        # W65C22S p17-18: ACR bit 6 clear is one-shot, "a single Interrupt Flag
584 
        # each time the Timer is loaded"; set is free-run, where the flag is
585 
        # raised at every zero with no reload.
586 
        "find": "if (!(acr & 0x40)) via->timer_running[0] = false;",
587 
        "into": "if (acr & 0x40) via->timer_running[0] = false;",
588 
        "count": 1,
589 
        "caught_by": ["via_timers"],
590 
    },
591 
    {
592 
        "name": "VIA T1 loads its low byte from the write, not the latch",
593 
        "file": "src/via.c",
594 
        # W65C22S p17: "the microprocessor does not write directly into the T1
595 
        # low order counter. Instead, this half of the counter is loaded
596 
        # automatically from the low order register when the microprocessor
597 
        # writes into the high order register and counter."
598 
        "find": "via->timer_count[0] = ((unsigned)value << 8) | via->registers[6];",
599 
        "into": "via->timer_count[0] = ((unsigned)value << 8) | value;",
600 
        "count": 1,
601 
        "caught_by": ["via_timers"],
602 
    },
603 
    {
604 
        "name": "VIA T2 counts the clock in pulse mode",
605 
        "file": "src/via.c",
606 
        # W65C22S p19: T2 "operates in the One-Shot Mode only (as an interval
607 
        # timer), or as a pulse counter for counting negative pulses on PB6. A
608 
        # single control bit within ACR5 is used to select between these two
609 
        # modes."
610 
        "find": "tclk = (acr & 0x20) ? via->pb6_pulse_counts : clocks;",
611 
        "into": "tclk = clocks;",
612 
        "count": 1,
613 
        "caught_by": ["via_timers"],
614 
    },
615 
    {
616 
        "name": "VIA T2 re-arms itself after timing out",
617 
        "file": "src/via.c",
618 
        # W65C22S p19: T2 has no free-run mode, so its flag is raised once per
619 
        # load. Leaving it armed turns a one-shot into a repeating interrupt.
620 
        "find": "\t\t\tifr |= 0x20;\n\t\t\tvia->timer_running[1] = false;",
621 
        "into": "\t\t\tifr |= 0x20;",
622 
        "count": 1,
623 
        "caught_by": ["via_timers"],
624 
    },
625 
    {
626 
        "name": "banked RAM window is 4K instead of 8K",
627 
        "file": "src/memory.c",
628 
        # x16-docs Memory Map: "$A000-$BFFF Banked RAM (8 KB window into one of
629 
        # 256 banks for a total of 2 MB)". A short stride overlaps every bank
630 
        # with the top half of the one below it.
631 
        "find": "return BRAM[(ramBank << 13) + address - 0xa000];",
632 
        "into": "return BRAM[(ramBank << 12) + address - 0xa000];",
633 
        "count": 1,
634 
        "caught_by": ["memory_banking"],
635 
    },
636 
    {
637 
        "name": "banked RAM writes ignore the bank",
638 
        "file": "src/memory.c",
639 
        # The window is the point: every bank would otherwise be the same 8K,
640 
        # and a write through one bank would be visible through all of them.
641 
        "find": "BRAM[(memory_get_ram_bank() << 13) + address - 0xa000] = value;",
642 
        "into": "BRAM[address - 0xa000] = value;",
643 
        "count": 1,
644 
        "caught_by": ["memory_banking"],
645 
    },
646 
    {
647 
        "name": "the RAM bank register is masked to the installed banks",
648 
        "file": "src/memory.c",
649 
        # x16-docs Memory Map: "$0000 Current RAM bank (0-255)". Masking on the
650 
        # way in would make a bank beyond the installed RAM alias a real one
651 
        # instead of reaching nothing.
652 
        "find": "case 0:\n\t\t\tmemory_set_ram_bank(value);",
653 
        "into": "case 0:\n\t\t\tmemory_set_ram_bank(value & 0x3f);",
654 
        "count": 1,
655 
        "caught_by": ["memory_banking"],
656 
    },
657 
    {
658 
        "name": "an uninstalled RAM bank reads as zero",
659 
        "file": "src/memory.c",
660 
        # No oracle says what an uninstalled bank returns; the test records
661 
        # this emulator's answer as characterization. The mutation is here so
662 
        # that record cannot quietly stop describing anything.
663 
        "find": "\t\t\treturn (address >> 8) & 0xff; // open bus read\n\t\t}\n\t} else { // banked ROM",
664 
        "into": "\t\t\treturn 0;\n\t\t}\n\t} else { // banked ROM",
665 
        "count": 1,
666 
        "caught_by": ["memory_banking"],
667 
    },
668 
    {
669 
        "name": "a debug write charges the slow-I/O cycles",
670 
        "file": "src/memory.c",
671 
        # A debugger's edit updates the machine and does nothing else, so it
672 
        # spends no cycles -- the same reason a debug read spends none. Losing
673 
        # the guard makes using the debugger change program timing.
674 
        "find": "\t\t\t// slow IO2 range\n\t\t\tif (!debugOn) {\n\t\t\t\tclockticks6502 += 3;\n\t\t\t}\n\t\t\tif ((address & 0x01) == 0) {   // YM reg (partially decoded)",
675 
        "into": "\t\t\t// slow IO2 range\n\t\t\tclockticks6502 += 3;\n\t\t\tif ((address & 0x01) == 0) {   // YM reg (partially decoded)",
676 
        "count": 1,
677 
        "caught_by": ["debug_write_path"],
678 
    },
679 
    {
680 
        "name": "a program write to slow I/O is free",
681 
        "file": "src/memory.c",
682 
        # The other side of the same guard: a store the program executed still
683 
        # costs what it costs, and the debug case is only interesting against
684 
        # it.
685 
        "find": "\t\tif (!debugOn && address >= 0x9fa0) {\n\t\t\t// slow IO5-7 range\n\t\t\tclockticks6502 += 3;\n\t\t}\n\t\tif (address >= 0x9f00 && address < 0x9f10) {\n\t\t\tvia1_write",
686 
        "into": "\t\tif (false && address >= 0x9fa0) {\n\t\t\t// slow IO5-7 range\n\t\t\tclockticks6502 += 3;\n\t\t}\n\t\tif (address >= 0x9f00 && address < 0x9f10) {\n\t\t\tvia1_write",
687 
        "count": 1,
688 
        "caught_by": ["debug_write_path"],
689 
    },
690 
    {
691 
        "name": "a debug write reports a watchpoint",
692 
        "file": "src/memory.c",
693 
        # A watchpoint answers "when does the program touch this?". Breaking on
694 
        # the debugger's own edit answers a question nobody asked, and stops
695 
        # the machine while someone is typing into it.
696 
        "find": "if (!debugOn && debug_wp_count() > 0 && bank == 0 && debug_wp_check_write(address, value)) {",
697 
        "into": "if (debug_wp_count() > 0 && bank == 0 && debug_wp_check_write(address, value)) {",
698 
        "count": 1,
699 
        "caught_by": ["debug_write_path"],
700 
    },
701 
]
702
703
704 
def run(cmd, **kw):
705 
    return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, **kw)
706
707
708 
def build(build_dir, passes=1):
709 
    """Build the tests, optionally more than once.
710
711 
    A mutation to an .opcodes file regenerates src/cpu/tables.h during the
712 
    build, and the objects that include it are only recompiled on the following
713 
    pass: the header dependency comes from the compiler's depfile, which ninja
714 
    evaluates before the generator has run. One pass would leave the binaries
715 
    linked from the unmutated tables, so such a mutation would look like it
716 
    survived when nothing had actually been tested.
717 
    """
718 
    out = ""
719 
    for _ in range(passes):
720 
        r = run([CMAKE, "--build", build_dir, "--target", "unit_tests"])
721 
        out += r.stdout + r.stderr
722 
        if r.returncode != 0:
723 
            return False, out
724 
    return True, out
725
726
727 
def tests_fail(build_dir, names):
728 
    """True if every named test reports failure."""
729 
    for name in names:
730 
        r = run([CTEST, "--test-dir", build_dir, "-R", f"^{name}$"])
731 
        if r.returncode == 0:
732 
            return False, name
733 
    return True, None
734
735
736 
def main():
737 
    ap = argparse.ArgumentParser()
738 
    ap.add_argument("--build-dir", default="build")
739 
    ap.add_argument("--only", help="run just the mutation whose name contains this")
740 
    args = ap.parse_args()
741
742 
    ok, out = build(args.build_dir)
743 
    if not ok:
744 
        print("the tree does not build before any mutation:\n" + out[-2000:])
745 
        return 2
746
747 
    survived = []
748 
    not_applicable = []
749 
    for m in MUTATIONS:
750 
        if args.only and args.only not in m["name"]:
751 
            continue
752
753 
        # Without its fixture the test skips, passes, and looks like the
754 
        # mutation survived.
755 
        need = m.get("requires")
756 
        if need and not (ROOT / need).exists():
757 
            print(f"n/a   {m['name']}\n      needs {need}, which is not present")
758 
            not_applicable.append(m["name"])
759 
            continue
760
761 
        path = ROOT / m["file"]
762 
        # Universal newlines, so a pattern written with \n matches a file
763 
        # checked out with CRLF. The original bytes are restored afterwards.
764 
        original = path.read_text(encoding="utf-8")
765
766 
        want = m.get("count", 1)
767 
        count = original.count(m["find"])
768 
        if count != want:
769 
            print(f"SKIP  {m['name']}\n      pattern matches {count} times in "
770 
                  f"{m['file']}, expected {want} -- the source has moved on")
771 
            survived.append(m["name"] + " (pattern stale)")
772 
            continue
773
774 
        # Building an .opcodes mutation rewrites these, and they are tracked, so
775 
        # they have to be put back with everything else.
776 
        GENERATED = ("src/cpu/tables.h", "src/cpu/mnemonics.h")
777
778 
        backup = Path(tempfile.gettempdir()) / (path.name + ".mutation_backup")
779 
        shutil.copy2(path, backup)
780 
        gen_backups = {}
781 
        for rel in GENERATED:
782 
            gen = ROOT / rel
783 
            if gen.exists():
784 
                gen_backups[gen] = Path(tempfile.gettempdir()) / (gen.name + ".mutation_backup")
785 
                shutil.copy2(gen, gen_backups[gen])
786 
        try:
787 
            path.write_text(original.replace(m["find"], m["into"]),
788 
                            encoding="utf-8")
789
790 
            # Delete the generated headers rather than trusting the build to
791 
            # notice they are stale. The generator fires on a timestamp
792 
            # comparison against the .opcodes file, and restoring the previous
793 
            # mutation stamps these to "now" -- so when a whole mutate, build
794 
            # and restore cycle lands inside one filesystem mtime tick, the
795 
            # mutated .opcodes is not newer than tables.h, generation is
796 
            # skipped, and the tests read the PREVIOUS opcode tables. The
797 
            # mutation then looks survived when nothing was ever built from it.
798 
            #
799 
            # That is machine-speed dependent, so it shows up as an occasional
800 
            # red build on CI and never locally. Deleting them cannot be
801 
            # defeated by any timestamp.
802 
            if path.suffix == ".opcodes":
803 
                for gen in gen_backups:
804 
                    gen.unlink(missing_ok=True)
805
806 
            built, out = build(args.build_dir, passes=2)
807 
            if not built:
808 
                # A mutation that will not compile proves nothing either way,
809 
                # and is usually a mutation that needs rewriting rather than a
810 
                # problem with the tests. Show enough to fix it.
811 
                first = next((l for l in out.splitlines()
812 
                              if "error" in l.lower()), "")
813 
                print(f"SKIP  {m['name']}\n      does not compile: {first.strip()[:120]}")
814 
                survived.append(m["name"] + " (did not compile)")
815 
                continue
816
817 
            caught, missed = tests_fail(args.build_dir, m["caught_by"])
818 
            if caught:
819 
                print(f"caught  {m['name']}")
820 
            else:
821 
                print(f"SURVIVED  {m['name']}\n"
822 
                      f"          {missed} still passed with this broken")
823 
                survived.append(m["name"])
824 
        finally:
825 
            shutil.copy2(backup, path)
826 
            # copy2 preserves mtime, leaving the restored file older than the
827 
            # object built from the mutated version. Ninja and make skip the
828 
            # rebuild, so the mutation stays compiled in and every later one
829 
            # runs against a broken emulator and is reported caught.
830 
            os.utime(path, None)
831 
            backup.unlink(missing_ok=True)
832 
            for gen, saved in gen_backups.items():
833 
                shutil.copy2(saved, gen)
834 
                os.utime(gen, None)
835 
                saved.unlink(missing_ok=True)
836
837 
    # Leave the tree built from clean source.
838 
    ok, _ = build(args.build_dir, passes=2)
839 
    if not ok:
840 
        print("the tree does not build after restoring the sources")
841 
        return 2
842
843 
    total = len([m for m in MUTATIONS
844 
                 if not args.only or args.only in m["name"]])
845 
    total -= len(not_applicable)
846 
    print(f"\n{total - len(survived)}/{total} mutations caught")
847 
    if not_applicable:
848 
        print(f"{len(not_applicable)} not applicable here (missing fixtures)")
849 
    if survived:
850 
        print("survived:")
851 
        for s in survived:
852 
            print(f"  {s}")
853 
        return 1
854 
    return 0
855
856
857 
if __name__ == "__main__":
858 
    sys.exit(main())