update

2026-08-18 09:20:44 +02:00
commit b826cd721a
522 changed files with 93730 additions and 0 deletions
@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""Independently verify B23 explicit-extent images and expectations."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+from pathlib import Path
+import struct
+
+
+SUPER = 1024
+MAGIC = 0xE0F5E1E2
+FEATURE_COMPAT_SB_CHKSUM = 0x00000001
+METABOX_NID_BIT = 1 << 63
+BLOCK_SIZE = 4096
+CRC32C_POLYNOMIAL = 0x82F63B78
+
+
+class OracleError(RuntimeError):
+    pass
+
+
+def sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def crc32c(data: bytes | bytearray) -> int:
+    checksum = 0xFFFFFFFF
+    for byte in data:
+        checksum ^= byte
+        for _ in range(8):
+            checksum = (checksum >> 1) ^ (
+                CRC32C_POLYNOMIAL if checksum & 1 else 0
+            )
+    return checksum & 0xFFFFFFFF
+
+
+def u16(data: bytes, offset: int) -> int:
+    return struct.unpack_from("<H", data, offset)[0]
+
+
+def u32(data: bytes, offset: int) -> int:
+    return struct.unpack_from("<I", data, offset)[0]
+
+
+def u64(data: bytes, offset: int) -> int:
+    return struct.unpack_from("<Q", data, offset)[0]
+
+
+def verify_super(data: bytes) -> tuple[int, int]:
+    if len(data) < SUPER + 144 or u32(data, SUPER) != MAGIC:
+        raise OracleError("invalid EROFS superblock")
+    block_bits = data[SUPER + 12]
+    if block_bits != 12:
+        raise OracleError("B23 image block size changed")
+    declared_size = u32(data, SUPER + 36) << block_bits
+    if declared_size > len(data):
+        raise OracleError("declared image exceeds provider")
+    if u32(data, SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM:
+        window = bytearray(data[SUPER : 1 << block_bits])
+        expected = u32(window, 4)
+        struct.pack_into("<I", window, 4, 0)
+        if crc32c(window) != expected:
+            raise OracleError("superblock checksum mismatch")
+    return 1 << block_bits, declared_size
+
+
+def expected_payload() -> bytes:
+    return bytes((index * 37 + 11) & 0xFF for index in range(BLOCK_SIZE))
+
+
+def changed_offsets(left: bytes, right: bytes) -> list[int]:
+    if len(left) != len(right):
+        raise OracleError("negative and baseline provider sizes differ")
+    return [index for index, pair in enumerate(zip(left, right)) if pair[0] != pair[1]]
+
+
+def verify_mutation(directory: Path, image: bytes, case: dict[str, object]) -> None:
+    mutation = case.get("mutation")
+    baseline_name = case.get("baseline")
+    if mutation is None:
+        if baseline_name is not None:
+            raise OracleError("positive case unexpectedly names a baseline")
+        return
+    if not isinstance(mutation, dict) or not isinstance(baseline_name, str):
+        raise OracleError("negative case lacks mutation evidence")
+    baseline = (directory / baseline_name).read_bytes()
+    differences = changed_offsets(image, baseline)
+    start = int(mutation["offset"])
+    end = start + int(mutation["size"])
+    if not differences or any(offset < start or offset >= end for offset in differences):
+        raise OracleError(f"mutation is not confined to {mutation['field']}")
+
+
+def record_values(
+    image: bytes, case: dict[str, object]
+) -> tuple[list[int], list[int], list[int]]:
+    record_size = int(case["record_size"])
+    record_offset = int(case["record_image_offset"])
+    stored_count = int(case["stored_count"])
+    payload_offset = int(case["payload_offset"])
+    plens: list[int] = []
+    pstarts: list[int] = []
+    lstarts: list[int] = []
+    physical = u64(image, int(case["header_image_offset"]) + 8) if record_size == 4 else 0
+    for index in range(stored_count):
+        offset = record_offset + index * record_size
+        if offset + record_size > len(image):
+            raise OracleError("stored record exceeds provider bytes")
+        plen = u32(image, offset)
+        if record_size == 4:
+            pstart = physical
+            physical += plen & ((2 * 1024 * 1024) - 1)
+            lstart = index * BLOCK_SIZE
+        else:
+            pstart = u32(image, offset + 4)
+            if record_size >= 16:
+                pstart |= u32(image, offset + 8) << 32
+                lstart = u32(image, offset + 12)
+                if record_size == 32:
+                    lstart |= u32(image, offset + 16) << 32
+            else:
+                lstart = index * BLOCK_SIZE
+        plens.append(plen)
+        pstarts.append(pstart)
+        lstarts.append(lstart)
+    if any(plen != BLOCK_SIZE for plen in plens):
+        raise OracleError("mapped shifted extent length changed")
+    if any(pstart != payload_offset for pstart in pstarts):
+        raise OracleError("mapped shifted extent physical offset changed")
+    return plens, pstarts, lstarts
+
+
+def verify_case(
+    directory: Path,
+    spec_case: dict[str, object],
+    case: dict[str, object],
+    chunk_size: int,
+) -> dict[str, object]:
+    image_path = directory / str(case["image"])
+    image = image_path.read_bytes()
+    block_size, declared_size = verify_super(image)
+    if sha256(image) != case["image_sha256"]:
+        raise OracleError(f"image hash mismatch: {image_path.name}")
+    if image[int(case["payload_offset"]) : int(case["payload_offset"]) + block_size] != expected_payload():
+        raise OracleError(f"payload block mismatch: {image_path.name}")
+    verify_mutation(directory, image, case)
+
+    if u64(image, int(case["target_dirent_offset"])) != int(case["target_nid"]):
+        raise OracleError("target dirent NID changed")
+    if case["backing"] == "metabox":
+        if not int(case["target_nid"]) & METABOX_NID_BIT:
+            raise OracleError("metabox target lacks bit-63 NID")
+    elif int(case["target_nid"]) & METABOX_NID_BIT:
+        raise OracleError("primary target unexpectedly has bit-63 NID")
+
+    inode = int(case["inode_image_offset"])
+    inode_format = u16(image, inode)
+    if inode_format != 3 or u16(image, inode + 4) != 0o100444:
+        raise OracleError("synthetic inode format or mode changed")
+    if u64(image, inode + 8) != int(case["file_size"]):
+        raise OracleError("synthetic inode size changed")
+    header = int(case["header_image_offset"])
+    advise = u16(image, header + 4)
+    observed_record_size = 4 << ((advise >> 1) & 3)
+    if not advise & 1 or observed_record_size != int(case["record_size"]):
+        raise OracleError("explicit extent advise changed")
+    declared_count = u32(image, header) | (u16(image, header + 6) << 32)
+    if int(case["record_size"]) >= 16 and declared_count != int(case["declared_count"]):
+        raise OracleError("explicit extent count changed")
+
+    _, _, observed_lstarts = record_values(image, case)
+    if observed_lstarts != case["lstarts"]:
+        raise OracleError("little-endian lstart decode differs from the manifest")
+    outcome = str(spec_case["outcome"])
+    violations = [
+        index for index in range(1, len(observed_lstarts))
+        if observed_lstarts[index] <= observed_lstarts[index - 1]
+    ]
+    if spec_case.get("mutation") == "descending-lstart":
+        if violations != [2]:
+            raise OracleError("ordering negative lacks its single violation")
+    elif violations:
+        raise OracleError("non-ordering case has unordered logical starts")
+
+    if outcome == "PASS":
+        if int(case["expected_errno"]) != 0:
+            raise OracleError("positive case has an errno")
+        if int(case["record_size"]) <= 8:
+            expected_calls = 1
+        else:
+            expected_calls = math.ceil(
+                int(case["stored_count"]) * int(case["record_size"]) / chunk_size
+            )
+        if int(case["expected_scan_calls"]) != expected_calls:
+            raise OracleError("bounded scan call count changed")
+        if int(case["stored_count"]) >= 4096 and expected_calls >= int(case["stored_count"]):
+            raise OracleError("large table regressed to per-record I/O")
+    elif int(case["expected_errno"]) != 97:
+        raise OracleError("corruption case does not use positive EINTEGRITY")
+
+    mutation = spec_case.get("mutation")
+    if mutation == "missing-tail":
+        requested_end = int(case["record_logical_offset"]) + 2 * int(case["record_size"])
+        if requested_end <= declared_size:
+            raise OracleError("primary missing-tail request remains provider-backed")
+    elif mutation == "tail-minus-one":
+        if int(case["carrier_size"]) != int(case["table_end_logical"]):
+            raise OracleError("metabox baseline does not end at the final record")
+        if u64(image, int(case["carrier_inode_offset"]) + 8) + 1 != int(case["table_end_logical"]):
+            raise OracleError("metabox tail is not exactly one byte short")
+    elif mutation == "huge-count":
+        if int(case["declared_count"]) <= (1 << 32):
+            raise OracleError("huge-count fixture does not exercise 48-bit count decode")
+
+    return {
+        "backing": case["backing"],
+        "expected_scan_calls": case["expected_scan_calls"],
+        "name": spec_case["name"],
+        "outcome": outcome,
+        "record_size": case["record_size"],
+        "stored_count": case["stored_count"],
+        "target_marker": case["target_marker"],
+    }
+
+
+def verify(spec_path: Path, directory: Path, report_path: Path) -> None:
+    spec = json.loads(spec_path.read_text(encoding="ascii"))
+    manifest = json.loads((directory / "fixture-manifest.json").read_text(encoding="ascii"))
+    if (
+        spec.get("schema") != 1
+        or spec.get("batch") != "B23"
+        or spec.get("test") != "TC171-explicit-extents"
+        or spec.get("candidates") != ["P15-060", "P15-085"]
+    ):
+        raise OracleError("B23 spec identity changed")
+    if (
+        manifest.get("schema") != 1
+        or manifest.get("batch") != "B23"
+        or manifest.get("test") != spec["test"]
+        or manifest.get("chunk_size") != spec["chunk_size"]
+    ):
+        raise OracleError("B23 manifest identity changed")
+    if spec.get("errno") != {
+        "corruption": 97,
+        "provider_io": "unchanged",
+        "sign": "positive",
+    }:
+        raise OracleError("B23 errno contract changed")
+
+    cases = manifest.get("cases")
+    if not isinstance(cases, dict) or set(cases) != {
+        str(case["name"]) for case in spec["cases"]
+    }:
+        raise OracleError("B23 case set changed")
+    decoded = []
+    markers = set()
+    for spec_case in spec["cases"]:
+        name = str(spec_case["name"])
+        case = cases[name]
+        if case["target_marker"] != spec_case["target_marker"]:
+            raise OracleError("target marker changed")
+        markers.add(str(case["target_marker"]))
+        decoded.append(
+            verify_case(directory, spec_case, case, int(spec["chunk_size"]))
+        )
+    if len(markers) != len(spec["cases"]):
+        raise OracleError("target markers are not unique")
+    if {int(case["record_size"]) for case in decoded} != {4, 8, 16, 32}:
+        raise OracleError("record-size coverage is incomplete")
+    if {str(case["backing"]) for case in decoded} != {"primary", "metabox"}:
+        raise OracleError("backing coverage is incomplete")
+
+    report = {
+        "batch": "B23",
+        "case_count": len(decoded),
+        "cases": decoded,
+        "chunk_size": spec["chunk_size"],
+        "large_scan_calls": {
+            str(case["name"]): case["expected_scan_calls"]
+            for case in decoded if int(case["stored_count"]) >= 4096
+        },
+        "status": "PASS",
+        "test": spec["test"],
+    }
+    report_path.write_text(
+        json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
+        encoding="ascii",
+    )
+    print(
+        f"B23 independent oracle PASS cases={len(decoded)} "
+        f"markers={len(markers)}"
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--spec", required=True, type=Path)
+    parser.add_argument("--fixtures", required=True, type=Path)
+    parser.add_argument("--report", required=True, type=Path)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    verify(args.spec, args.fixtures, args.report)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (OracleError, OSError, ValueError, KeyError, struct.error) as error:
+        raise SystemExit(f"B23 oracle failed: {error}") from error