317 lines
12 KiB
Python
317 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Independently verify B23 explicit-extent images and expectations."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import struct
|
|
|
|
|
|
SUPER = 1024
|
|
MAGIC = 0xE0F5E1E2
|
|
FEATURE_COMPAT_SB_CHKSUM = 0x00000001
|
|
METABOX_NID_BIT = 1 << 63
|
|
BLOCK_SIZE = 4096
|
|
CRC32C_POLYNOMIAL = 0x82F63B78
|
|
|
|
|
|
class OracleError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def sha256(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def crc32c(data: bytes | bytearray) -> int:
|
|
checksum = 0xFFFFFFFF
|
|
for byte in data:
|
|
checksum ^= byte
|
|
for _ in range(8):
|
|
checksum = (checksum >> 1) ^ (
|
|
CRC32C_POLYNOMIAL if checksum & 1 else 0
|
|
)
|
|
return checksum & 0xFFFFFFFF
|
|
|
|
|
|
def u16(data: bytes, offset: int) -> int:
|
|
return struct.unpack_from("<H", data, offset)[0]
|
|
|
|
|
|
def u32(data: bytes, offset: int) -> int:
|
|
return struct.unpack_from("<I", data, offset)[0]
|
|
|
|
|
|
def u64(data: bytes, offset: int) -> int:
|
|
return struct.unpack_from("<Q", data, offset)[0]
|
|
|
|
|
|
def verify_super(data: bytes) -> tuple[int, int]:
|
|
if len(data) < SUPER + 144 or u32(data, SUPER) != MAGIC:
|
|
raise OracleError("invalid EROFS superblock")
|
|
block_bits = data[SUPER + 12]
|
|
if block_bits != 12:
|
|
raise OracleError("B23 image block size changed")
|
|
declared_size = u32(data, SUPER + 36) << block_bits
|
|
if declared_size > len(data):
|
|
raise OracleError("declared image exceeds provider")
|
|
if u32(data, SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM:
|
|
window = bytearray(data[SUPER : 1 << block_bits])
|
|
expected = u32(window, 4)
|
|
struct.pack_into("<I", window, 4, 0)
|
|
if crc32c(window) != expected:
|
|
raise OracleError("superblock checksum mismatch")
|
|
return 1 << block_bits, declared_size
|
|
|
|
|
|
def expected_payload() -> bytes:
|
|
return bytes((index * 37 + 11) & 0xFF for index in range(BLOCK_SIZE))
|
|
|
|
|
|
def changed_offsets(left: bytes, right: bytes) -> list[int]:
|
|
if len(left) != len(right):
|
|
raise OracleError("negative and baseline provider sizes differ")
|
|
return [index for index, pair in enumerate(zip(left, right)) if pair[0] != pair[1]]
|
|
|
|
|
|
def verify_mutation(directory: Path, image: bytes, case: dict[str, object]) -> None:
|
|
mutation = case.get("mutation")
|
|
baseline_name = case.get("baseline")
|
|
if mutation is None:
|
|
if baseline_name is not None:
|
|
raise OracleError("positive case unexpectedly names a baseline")
|
|
return
|
|
if not isinstance(mutation, dict) or not isinstance(baseline_name, str):
|
|
raise OracleError("negative case lacks mutation evidence")
|
|
baseline = (directory / baseline_name).read_bytes()
|
|
differences = changed_offsets(image, baseline)
|
|
start = int(mutation["offset"])
|
|
end = start + int(mutation["size"])
|
|
if not differences or any(offset < start or offset >= end for offset in differences):
|
|
raise OracleError(f"mutation is not confined to {mutation['field']}")
|
|
|
|
|
|
def record_values(
|
|
image: bytes, case: dict[str, object]
|
|
) -> tuple[list[int], list[int], list[int]]:
|
|
record_size = int(case["record_size"])
|
|
record_offset = int(case["record_image_offset"])
|
|
stored_count = int(case["stored_count"])
|
|
payload_offset = int(case["payload_offset"])
|
|
plens: list[int] = []
|
|
pstarts: list[int] = []
|
|
lstarts: list[int] = []
|
|
physical = u64(image, int(case["header_image_offset"]) + 8) if record_size == 4 else 0
|
|
for index in range(stored_count):
|
|
offset = record_offset + index * record_size
|
|
if offset + record_size > len(image):
|
|
raise OracleError("stored record exceeds provider bytes")
|
|
plen = u32(image, offset)
|
|
if record_size == 4:
|
|
pstart = physical
|
|
physical += plen & ((2 * 1024 * 1024) - 1)
|
|
lstart = index * BLOCK_SIZE
|
|
else:
|
|
pstart = u32(image, offset + 4)
|
|
if record_size >= 16:
|
|
pstart |= u32(image, offset + 8) << 32
|
|
lstart = u32(image, offset + 12)
|
|
if record_size == 32:
|
|
lstart |= u32(image, offset + 16) << 32
|
|
else:
|
|
lstart = index * BLOCK_SIZE
|
|
plens.append(plen)
|
|
pstarts.append(pstart)
|
|
lstarts.append(lstart)
|
|
if any(plen != BLOCK_SIZE for plen in plens):
|
|
raise OracleError("mapped shifted extent length changed")
|
|
if any(pstart != payload_offset for pstart in pstarts):
|
|
raise OracleError("mapped shifted extent physical offset changed")
|
|
return plens, pstarts, lstarts
|
|
|
|
|
|
def verify_case(
|
|
directory: Path,
|
|
spec_case: dict[str, object],
|
|
case: dict[str, object],
|
|
chunk_size: int,
|
|
) -> dict[str, object]:
|
|
image_path = directory / str(case["image"])
|
|
image = image_path.read_bytes()
|
|
block_size, declared_size = verify_super(image)
|
|
if sha256(image) != case["image_sha256"]:
|
|
raise OracleError(f"image hash mismatch: {image_path.name}")
|
|
if image[int(case["payload_offset"]) : int(case["payload_offset"]) + block_size] != expected_payload():
|
|
raise OracleError(f"payload block mismatch: {image_path.name}")
|
|
verify_mutation(directory, image, case)
|
|
|
|
if u64(image, int(case["target_dirent_offset"])) != int(case["target_nid"]):
|
|
raise OracleError("target dirent NID changed")
|
|
if case["backing"] == "metabox":
|
|
if not int(case["target_nid"]) & METABOX_NID_BIT:
|
|
raise OracleError("metabox target lacks bit-63 NID")
|
|
elif int(case["target_nid"]) & METABOX_NID_BIT:
|
|
raise OracleError("primary target unexpectedly has bit-63 NID")
|
|
|
|
inode = int(case["inode_image_offset"])
|
|
inode_format = u16(image, inode)
|
|
if inode_format != 3 or u16(image, inode + 4) != 0o100444:
|
|
raise OracleError("synthetic inode format or mode changed")
|
|
if u64(image, inode + 8) != int(case["file_size"]):
|
|
raise OracleError("synthetic inode size changed")
|
|
header = int(case["header_image_offset"])
|
|
advise = u16(image, header + 4)
|
|
observed_record_size = 4 << ((advise >> 1) & 3)
|
|
if not advise & 1 or observed_record_size != int(case["record_size"]):
|
|
raise OracleError("explicit extent advise changed")
|
|
declared_count = u32(image, header) | (u16(image, header + 6) << 32)
|
|
if int(case["record_size"]) >= 16 and declared_count != int(case["declared_count"]):
|
|
raise OracleError("explicit extent count changed")
|
|
|
|
_, _, observed_lstarts = record_values(image, case)
|
|
if observed_lstarts != case["lstarts"]:
|
|
raise OracleError("little-endian lstart decode differs from the manifest")
|
|
outcome = str(spec_case["outcome"])
|
|
violations = [
|
|
index for index in range(1, len(observed_lstarts))
|
|
if observed_lstarts[index] <= observed_lstarts[index - 1]
|
|
]
|
|
if spec_case.get("mutation") == "descending-lstart":
|
|
if violations != [2]:
|
|
raise OracleError("ordering negative lacks its single violation")
|
|
elif violations:
|
|
raise OracleError("non-ordering case has unordered logical starts")
|
|
|
|
if outcome == "PASS":
|
|
if int(case["expected_errno"]) != 0:
|
|
raise OracleError("positive case has an errno")
|
|
if int(case["record_size"]) <= 8:
|
|
expected_calls = 1
|
|
else:
|
|
expected_calls = math.ceil(
|
|
int(case["stored_count"]) * int(case["record_size"]) / chunk_size
|
|
)
|
|
if int(case["expected_scan_calls"]) != expected_calls:
|
|
raise OracleError("bounded scan call count changed")
|
|
if int(case["stored_count"]) >= 4096 and expected_calls >= int(case["stored_count"]):
|
|
raise OracleError("large table regressed to per-record I/O")
|
|
elif int(case["expected_errno"]) != 97:
|
|
raise OracleError("corruption case does not use positive EINTEGRITY")
|
|
|
|
mutation = spec_case.get("mutation")
|
|
if mutation == "missing-tail":
|
|
requested_end = int(case["record_logical_offset"]) + 2 * int(case["record_size"])
|
|
if requested_end <= declared_size:
|
|
raise OracleError("primary missing-tail request remains provider-backed")
|
|
elif mutation == "tail-minus-one":
|
|
if int(case["carrier_size"]) != int(case["table_end_logical"]):
|
|
raise OracleError("metabox baseline does not end at the final record")
|
|
if u64(image, int(case["carrier_inode_offset"]) + 8) + 1 != int(case["table_end_logical"]):
|
|
raise OracleError("metabox tail is not exactly one byte short")
|
|
elif mutation == "huge-count":
|
|
if int(case["declared_count"]) <= (1 << 32):
|
|
raise OracleError("huge-count fixture does not exercise 48-bit count decode")
|
|
|
|
return {
|
|
"backing": case["backing"],
|
|
"expected_scan_calls": case["expected_scan_calls"],
|
|
"name": spec_case["name"],
|
|
"outcome": outcome,
|
|
"record_size": case["record_size"],
|
|
"stored_count": case["stored_count"],
|
|
"target_marker": case["target_marker"],
|
|
}
|
|
|
|
|
|
def verify(spec_path: Path, directory: Path, report_path: Path) -> None:
|
|
spec = json.loads(spec_path.read_text(encoding="ascii"))
|
|
manifest = json.loads((directory / "fixture-manifest.json").read_text(encoding="ascii"))
|
|
if (
|
|
spec.get("schema") != 1
|
|
or spec.get("batch") != "B23"
|
|
or spec.get("test") != "TC171-explicit-extents"
|
|
or spec.get("candidates") != ["P15-060", "P15-085"]
|
|
):
|
|
raise OracleError("B23 spec identity changed")
|
|
if (
|
|
manifest.get("schema") != 1
|
|
or manifest.get("batch") != "B23"
|
|
or manifest.get("test") != spec["test"]
|
|
or manifest.get("chunk_size") != spec["chunk_size"]
|
|
):
|
|
raise OracleError("B23 manifest identity changed")
|
|
if spec.get("errno") != {
|
|
"corruption": 97,
|
|
"provider_io": "unchanged",
|
|
"sign": "positive",
|
|
}:
|
|
raise OracleError("B23 errno contract changed")
|
|
|
|
cases = manifest.get("cases")
|
|
if not isinstance(cases, dict) or set(cases) != {
|
|
str(case["name"]) for case in spec["cases"]
|
|
}:
|
|
raise OracleError("B23 case set changed")
|
|
decoded = []
|
|
markers = set()
|
|
for spec_case in spec["cases"]:
|
|
name = str(spec_case["name"])
|
|
case = cases[name]
|
|
if case["target_marker"] != spec_case["target_marker"]:
|
|
raise OracleError("target marker changed")
|
|
markers.add(str(case["target_marker"]))
|
|
decoded.append(
|
|
verify_case(directory, spec_case, case, int(spec["chunk_size"]))
|
|
)
|
|
if len(markers) != len(spec["cases"]):
|
|
raise OracleError("target markers are not unique")
|
|
if {int(case["record_size"]) for case in decoded} != {4, 8, 16, 32}:
|
|
raise OracleError("record-size coverage is incomplete")
|
|
if {str(case["backing"]) for case in decoded} != {"primary", "metabox"}:
|
|
raise OracleError("backing coverage is incomplete")
|
|
|
|
report = {
|
|
"batch": "B23",
|
|
"case_count": len(decoded),
|
|
"cases": decoded,
|
|
"chunk_size": spec["chunk_size"],
|
|
"large_scan_calls": {
|
|
str(case["name"]): case["expected_scan_calls"]
|
|
for case in decoded if int(case["stored_count"]) >= 4096
|
|
},
|
|
"status": "PASS",
|
|
"test": spec["test"],
|
|
}
|
|
report_path.write_text(
|
|
json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
|
|
encoding="ascii",
|
|
)
|
|
print(
|
|
f"B23 independent oracle PASS cases={len(decoded)} "
|
|
f"markers={len(markers)}"
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--spec", required=True, type=Path)
|
|
parser.add_argument("--fixtures", required=True, type=Path)
|
|
parser.add_argument("--report", required=True, type=Path)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
verify(args.spec, args.fixtures, args.report)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (OracleError, OSError, ValueError, KeyError, struct.error) as error:
|
|
raise SystemExit(f"B23 oracle failed: {error}") from error
|