#!/usr/bin/env python3

"""Fetch the reference documents the conformance tests are written against.

The tests assert hardware behaviour, so every non-obvious expectation should be

traceable to a document rather than to somebody's memory. This pulls those

documents down and turns them into greppable text with page markers, so a test

comment can cite a page and the next person can check it.

    python tests/refs/fetch_refs.py

Nothing fetched here is committed. The datasheets are copyrighted by Western

Design Center and the tutorials are Bruce Clark's work; this only automates

downloading them to a local, ignored directory. Run it once, and again whenever

you want a fresh copy.

Text extraction needs pypdf (pip install pypdf). Without it the PDFs are still

downloaded and the HTML references still work; only the .txt files are skipped,

and a test that cites a page can still be checked against the PDF by hand.

The CPU tests already cite pages of the two WDC CPU datasheets. Until this

script was committed those citations named a document nobody could fetch,

which is the failure mode the whole "quote your oracle" rule exists to stop.

"""

import re

import sys

import urllib.parse

import urllib.request

from pathlib import Path

DEST = Path(__file__).resolve().parent / "downloads"

UA = {"User-Agent": "Mozilla/5.0 (x16-emulator test reference fetcher)"}

PDFS = {

    # Cited by the CPU conformance tests.

    "w65c816s.pdf": "https://www.westerndesigncenter.com/wdc/documentation/w65c816s.pdf",

    "w65c02s.pdf": "https://www.westerndesigncenter.com/wdc/documentation/w65c02s.pdf",

    # The VIA. Two of these sit at $9F00 and $9F10; the X16 uses them for the

    # PS/2 ports, the SD card select, the I2C bus and the RAM/ROM bank latches.

    "w65c22s.pdf": "https://www.westerndesigncenter.com/wdc/documentation/w65c22s.pdf",

}

PAGES = {

    "6502org-65c816-opcodes.txt": "http://6502.org/tutorials/65c816opcodes.html",

    "6502org-65c02-opcodes.txt": "http://6502.org/tutorials/65c02opcodes.html",

}

# The X16's own documentation. Unlike VERA there is no hardware definition to

# consult: the banking is discrete logic on the board, with no schematic or

# programmable-logic source published anywhere, so this is the strongest source

# there is for the memory map -- and it is documentation, a step removed.

#

# Pinned to a commit rather than a branch, for the same reason the VERA RTL is:

# a citation of a line in a moving file is not checkable. The Memory Map

# chapter carries its own warning that "the layout of the banks may still

# change", so this pin will need moving one day.

DOCS_COMMIT = "79a8303bc33cda4ec3e85f3239f9b3d4c8c76260"

DOCS = {

    "x16-memory-map.md": "X16 Reference - 08 - Memory Map.md",

    "x16-hardware.md": "X16 Reference - 14 - Hardware.md",

}

def fetch(url, path):

    if path.exists():

        print(f"  have {path.name}")

        return True

    print(f"  get  {path.name}")

    try:

        req = urllib.request.Request(url, headers=UA)

        with urllib.request.urlopen(req, timeout=120) as r:

            path.write_bytes(r.read())

        return True

    except Exception as exc:

        print(f"  FAIL {path.name}: {exc}")

        return False

def html_to_text(path):

    raw = path.read_bytes().decode("utf-8", "replace")

    raw = re.sub(r"(?is)<(script|style).*?</\1>", "", raw)

    raw = re.sub(r"(?i)<br\s*/?>", "\n", raw)

    raw = re.sub(r"(?i)</(p|div|tr|h\d|li)>", "\n", raw)

    text = re.sub(r"(?s)<[^>]+>", "", raw)

    for a, b in (("&lt;", "<"), ("&gt;", ">"), ("&amp;", "&"), ("&nbsp;", " ")):

        text = text.replace(a, b)

    return re.sub(r"\n{3,}", "\n\n", text)

def pdf_to_text(pdf, out):

    try:

        import pypdf

    except ImportError:

        print(f"  skip {out.name} (pip install pypdf to enable)")

        return

    reader = pypdf.PdfReader(str(pdf))

    with out.open("w", encoding="utf-8") as fh:

        for i, page in enumerate(reader.pages, 1):

            # The page marker is the point of this: a citation is only useful if

            # the next person can find the line again.

            fh.write(f"\n===== page {i} =====\n")

            fh.write(page.extract_text() or "")

    print(f"  made {out.name} ({len(reader.pages)} pages)")

def main():

    DEST.mkdir(parents=True, exist_ok=True)

    ok = True

    print("WDC datasheets:")

    for name, url in PDFS.items():

        path = DEST / name

        if fetch(url, path):

            pdf_to_text(path, path.with_suffix(".txt"))

        else:

            ok = False

    print("6502.org tutorials:")

    for name, url in PAGES.items():

        raw = DEST / (name + ".html")

        if fetch(url, raw):

            (DEST / name).write_text(html_to_text(raw), encoding="utf-8")

            raw.unlink()

            print(f"  made {name}")

        else:

            ok = False

    print("X16 documentation:")

    base = f"https://raw.githubusercontent.com/X16Community/x16-docs/{DOCS_COMMIT}"

    for name, path in DOCS.items():

        quoted = urllib.parse.quote(path)

        if not fetch(f"{base}/{quoted}", DEST / name):

            ok = False

    print(f"\nreferences in {DEST}")

    return 0 if ok else 1

if __name__ == "__main__":

    sys.exit(main())