714 lines
31 KiB
Bash
Executable File
714 lines
31 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
|
input=$gate_dir/P15-062-input.json
|
|
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
|
|
base=
|
|
output=
|
|
|
|
while test "$#" -gt 0; do
|
|
case "$1" in
|
|
--base)
|
|
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
|
base=$2
|
|
shift 2
|
|
;;
|
|
--output)
|
|
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
|
output=$2
|
|
shift 2
|
|
;;
|
|
*)
|
|
printf 'unknown argument: %s\n' "$1" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
|
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
|
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
|
|
test -d "$freebsd_src/sys" || {
|
|
printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2
|
|
exit 2
|
|
}
|
|
for tool in cc git make pkg-config python3 sha256sum; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'missing required host tool: %s\n' "$tool" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
case "$output" in
|
|
/*) ;;
|
|
*) output=$PWD/$output ;;
|
|
esac
|
|
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
|
mkdir -p "$output"
|
|
|
|
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import ctypes
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(sys.argv[1])
|
|
INPUT = Path(sys.argv[2])
|
|
REQUESTED_BASE = sys.argv[3]
|
|
OUTPUT = Path(sys.argv[4])
|
|
FREEBSD_SRC = Path(sys.argv[5])
|
|
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
|
|
|
|
|
|
class GateFailure(Exception):
|
|
def __init__(self, status: str, reason: str):
|
|
super().__init__(reason)
|
|
self.status = status
|
|
self.reason = reason
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_path(path: Path) -> str:
|
|
return sha256_bytes(path.read_bytes())
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
|
|
|
|
|
def git(*args: str) -> str:
|
|
return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip()
|
|
|
|
|
|
def source_at(commit: str, path: str) -> str:
|
|
completed = subprocess.run(
|
|
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
)
|
|
if completed.returncode != 0:
|
|
raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}")
|
|
return completed.stdout
|
|
|
|
|
|
def run_logged(argv: list[str], cwd: Path, log: Path, timeout: int = 120) -> subprocess.CompletedProcess[str]:
|
|
completed = subprocess.run(
|
|
argv,
|
|
cwd=cwd,
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
log.parent.mkdir(parents=True, exist_ok=True)
|
|
log.write_text(
|
|
"$ " + " ".join(argv) + "\n" + completed.stdout + f"\n[exit {completed.returncode}]\n",
|
|
encoding="utf-8",
|
|
)
|
|
if completed.returncode != 0:
|
|
raise GateFailure("INFRA_BLOCKED", f"command failed ({completed.returncode}): {' '.join(argv)}")
|
|
return completed
|
|
|
|
|
|
def raw_lz4_decode(data: bytes, target: int | None = None) -> tuple[bytes, int]:
|
|
ip = 0
|
|
output = bytearray()
|
|
while ip < len(data):
|
|
token = data[ip]
|
|
ip += 1
|
|
literal_length = token >> 4
|
|
if literal_length == 15:
|
|
while True:
|
|
if ip >= len(data):
|
|
raise GateFailure("INFRA_BLOCKED", "oracle saw a truncated literal length")
|
|
value = data[ip]
|
|
ip += 1
|
|
literal_length += value
|
|
if value != 255:
|
|
break
|
|
if ip + literal_length > len(data):
|
|
raise GateFailure("INFRA_BLOCKED", "oracle saw truncated literals")
|
|
output.extend(data[ip : ip + literal_length])
|
|
ip += literal_length
|
|
if target is not None and len(output) >= target:
|
|
return bytes(output[:target]), ip
|
|
if ip == len(data):
|
|
return bytes(output), ip
|
|
if ip + 2 > len(data):
|
|
raise GateFailure("INFRA_BLOCKED", "oracle saw a truncated match offset")
|
|
offset = data[ip] | data[ip + 1] << 8
|
|
ip += 2
|
|
if offset == 0 or offset > len(output):
|
|
raise GateFailure("INFRA_BLOCKED", f"oracle saw invalid match offset {offset}")
|
|
match_length = token & 15
|
|
if match_length == 15:
|
|
while True:
|
|
if ip >= len(data):
|
|
raise GateFailure("INFRA_BLOCKED", "oracle saw a truncated match length")
|
|
value = data[ip]
|
|
ip += 1
|
|
match_length += value
|
|
if value != 255:
|
|
break
|
|
for _ in range(match_length + 4):
|
|
output.append(output[-offset])
|
|
if target is not None and len(output) >= target:
|
|
return bytes(output[:target]), ip
|
|
raise GateFailure("INFRA_BLOCKED", "oracle input ended before a final literal sequence")
|
|
|
|
|
|
def make_legacy_source(path: Path) -> bytes:
|
|
content = b"".join(
|
|
bytes([segment["byte"]]) * segment["length"]
|
|
for segment in SPEC["legacy_fixture"]["segments"]
|
|
)
|
|
if sha256_bytes(content) != SPEC["legacy_fixture"]["expected_source_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "legacy source generator hash drift")
|
|
path.mkdir(parents=True)
|
|
(path / "big.txt").write_bytes(content)
|
|
return content
|
|
|
|
|
|
def make_partial_source(path: Path) -> tuple[bytes, bytes]:
|
|
path.mkdir(parents=True)
|
|
data_a = b"".join(
|
|
bytes([segment["byte"]]) * segment["length"]
|
|
for segment in SPEC["legacy_fixture"]["segments"]
|
|
)
|
|
partial = SPEC["partial_fixture"]
|
|
pattern = partial["prefix_pattern"].encode("ascii")
|
|
repeats = (partial["prefix_size"] + len(pattern) - 1) // len(pattern)
|
|
prefix = (pattern * repeats)[: partial["prefix_size"]]
|
|
prefix = prefix.ljust(partial["prefix_size"], bytes([partial["prefix_pad_byte"]]))
|
|
data_b = prefix + data_a
|
|
if sha256_bytes(data_a) != partial["a_sha256"] or sha256_bytes(data_b) != partial["b_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "partial source generator hash drift")
|
|
(path / "a.dat").write_bytes(data_a)
|
|
(path / "b.dat").write_bytes(data_b)
|
|
return data_a, data_b
|
|
|
|
|
|
def load_liblz4() -> tuple[Any, Any]:
|
|
upstream = SPEC["upstream"]
|
|
soname = Path(upstream["liblz4_soname"])
|
|
version = subprocess.check_output(["pkg-config", "--modversion", "liblz4"], text=True).strip()
|
|
if version != upstream["liblz4_version"] or sha256_path(soname) != upstream["liblz4_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "liblz4 identity changed")
|
|
library = ctypes.CDLL(str(soname))
|
|
full = library.LZ4_decompress_safe
|
|
full.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
|
|
full.restype = ctypes.c_int
|
|
partial = library.LZ4_decompress_safe_partial
|
|
partial.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int]
|
|
partial.restype = ctypes.c_int
|
|
return full, partial
|
|
|
|
|
|
def liblz4_call(function: Any, source: bytes, output_size: int, target: int | None = None) -> tuple[int, bytes]:
|
|
source_buffer = ctypes.create_string_buffer(source, len(source))
|
|
output_buffer = ctypes.create_string_buffer(output_size)
|
|
if target is None:
|
|
result = function(source_buffer, output_buffer, len(source), output_size)
|
|
else:
|
|
result = function(source_buffer, output_buffer, len(source), target, output_size)
|
|
return result, output_buffer.raw
|
|
|
|
|
|
def compile_dut_decoder(temp: Path, source: str) -> Any:
|
|
source_dir = temp / "dut-decoder"
|
|
sys_dir = source_dir / "sys"
|
|
sys_dir.mkdir(parents=True)
|
|
(source_dir / "decompressor_lz4.c").write_text(source, encoding="utf-8")
|
|
(sys_dir / "param.h").write_text(
|
|
"#include <stdbool.h>\n#include <stddef.h>\n#include <stdint.h>\n#include <string.h>\n"
|
|
"#define EINTEGRITY 97\n#define MIN(a, b) ((a) < (b) ? (a) : (b))\n",
|
|
encoding="ascii",
|
|
)
|
|
(sys_dir / "endian.h").write_text(
|
|
"#include <stdint.h>\nstatic inline uint16_t le16dec(const void *p) { "
|
|
"const uint8_t *b = p; return (uint16_t)b[0] | (uint16_t)b[1] << 8; }\n",
|
|
encoding="ascii",
|
|
)
|
|
(sys_dir / "systm.h").write_text("#include <string.h>\n", encoding="ascii")
|
|
(source_dir / "compress.h").write_text(
|
|
"#include <stdbool.h>\n#include <stddef.h>\n"
|
|
"struct z_erofs_decompress_req { void *sbi; const void *map; const void *in; "
|
|
"size_t inputsize; void *out; size_t outputsize; bool partial_decoding; };\n",
|
|
encoding="ascii",
|
|
)
|
|
library = temp / "dut-lz4.so"
|
|
run_logged(
|
|
["cc", "-shared", "-fPIC", "-std=c11", "-Wall", "-Wextra", "-Werror", "-I", str(source_dir),
|
|
"-o", str(library), str(source_dir / "decompressor_lz4.c")],
|
|
temp,
|
|
OUTPUT / "logs/dut-decoder-build.log",
|
|
)
|
|
loaded = ctypes.CDLL(str(library))
|
|
|
|
class Request(ctypes.Structure):
|
|
_fields_ = [
|
|
("sbi", ctypes.c_void_p),
|
|
("map", ctypes.c_void_p),
|
|
("in", ctypes.c_void_p),
|
|
("inputsize", ctypes.c_size_t),
|
|
("out", ctypes.c_void_p),
|
|
("outputsize", ctypes.c_size_t),
|
|
("partial_decoding", ctypes.c_bool),
|
|
]
|
|
|
|
function = loaded.z_erofs_lz4_decompress
|
|
function.argtypes = [ctypes.POINTER(Request)]
|
|
function.restype = ctypes.c_int
|
|
return loaded, function, Request
|
|
|
|
|
|
def dut_call(function: Any, request_type: Any, source: bytes, output_size: int, partial: bool) -> tuple[int, bytes, bool]:
|
|
source_buffer = ctypes.create_string_buffer(source, len(source))
|
|
guard_size = 64
|
|
backing = bytearray(b"\xa5" * guard_size + b"\0" * output_size + b"\x5a" * guard_size)
|
|
output_buffer = (ctypes.c_ubyte * len(backing)).from_buffer(backing)
|
|
output_pointer = ctypes.cast(ctypes.byref(output_buffer, guard_size), ctypes.c_void_p)
|
|
request = request_type(
|
|
None,
|
|
None,
|
|
ctypes.cast(source_buffer, ctypes.c_void_p),
|
|
len(source),
|
|
output_pointer,
|
|
output_size,
|
|
partial,
|
|
)
|
|
result = function(ctypes.byref(request))
|
|
guards_ok = backing[:guard_size] == b"\xa5" * guard_size and backing[-guard_size:] == b"\x5a" * guard_size
|
|
return result, bytes(backing[guard_size : guard_size + output_size]), guards_ok
|
|
|
|
|
|
def verify_cleanup_source(zdata_source: str) -> dict[str, Any]:
|
|
anchors = [
|
|
"error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen,",
|
|
"erofs_put_metabuf(&buf);",
|
|
"erofs_brelse(compressed);",
|
|
"free(decoded, M_EROFS);",
|
|
"*bufp = decoded;",
|
|
]
|
|
missing = [anchor for anchor in anchors if anchor not in zdata_source]
|
|
if missing:
|
|
raise GateFailure("INFRA_BLOCKED", f"DUT cleanup source anchors changed: {missing}")
|
|
release_pos = min(zdata_source.index("erofs_put_metabuf(&buf);"), zdata_source.index("erofs_brelse(compressed);"))
|
|
error_pos = zdata_source.index("if (error != 0) {", release_pos)
|
|
free_pos = zdata_source.index("free(decoded, M_EROFS);", error_pos)
|
|
publish_pos = zdata_source.index("*bufp = decoded;", free_pos)
|
|
return {
|
|
"decoded_error_free_after_input_release": release_pos < error_pos < free_pos,
|
|
"decoded_success_publish_after_error_branch": free_pos < publish_pos,
|
|
"metadata_release_anchor": True,
|
|
"physical_release_anchor": True,
|
|
}
|
|
|
|
|
|
def build_versions(temp: Path, legacy_source: bytes) -> tuple[list[dict[str, Any]], dict[str, Path]]:
|
|
upstream = SPEC["upstream"]
|
|
clone = temp / "erofs-utils"
|
|
run_logged(
|
|
["git", "clone", "--no-checkout", upstream["url"], str(clone)],
|
|
temp,
|
|
OUTPUT / "logs/upstream-clone.log",
|
|
timeout=90,
|
|
)
|
|
records = []
|
|
worktrees: dict[str, Path] = {}
|
|
fixture = SPEC["legacy_fixture"]
|
|
source_dir = temp / "legacy-source"
|
|
make_legacy_source(source_dir)
|
|
for version in upstream["versions"]:
|
|
name = version["name"]
|
|
worktree = temp / f"erofs-{name}"
|
|
worktrees[name] = worktree
|
|
run_logged(
|
|
["git", "-C", str(clone), "worktree", "add", "--detach", str(worktree), version["commit"]],
|
|
temp,
|
|
OUTPUT / f"logs/{name}-worktree.log",
|
|
)
|
|
resolved = subprocess.check_output(["git", "-C", str(worktree), "rev-parse", "HEAD"], text=True).strip()
|
|
if resolved != version["commit"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"upstream commit mismatch for {name}")
|
|
run_logged(["./autogen.sh"], worktree, OUTPUT / f"logs/{name}-autogen.log")
|
|
run_logged(["./configure", "--disable-fuse"], worktree, OUTPUT / f"logs/{name}-configure.log")
|
|
run_logged(["make", "-s", "-j2"], worktree, OUTPUT / f"logs/{name}-make.log")
|
|
mkfs = worktree / "mkfs/mkfs.erofs"
|
|
fsck = worktree / "fsck/fsck.erofs"
|
|
dump = worktree / "dump/dump.erofs"
|
|
images = temp / f"images-{name}"
|
|
images.mkdir()
|
|
generated = []
|
|
for label in ("a", "b"):
|
|
image = images / f"legacy-{label}.erofs"
|
|
argv = [str(mkfs), *fixture["mkfs_args"], str(image), str(source_dir)]
|
|
run_logged(argv, worktree, OUTPUT / f"logs/{name}-mkfs-{label}.log")
|
|
generated.append(image)
|
|
hashes = [sha256_path(path) for path in generated]
|
|
if hashes[0] != hashes[1] or hashes[0] != version["expected_image_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"{name} image reproducibility/hash mismatch: {hashes}")
|
|
run_logged([str(fsck), "--extract", str(generated[0])], worktree, OUTPUT / f"logs/{name}-fsck.log")
|
|
superblock = run_logged([str(dump), "-s", str(generated[0])], worktree, OUTPUT / f"logs/{name}-super.log").stdout
|
|
feature_line = next((line for line in superblock.splitlines() if "features:" in line), "")
|
|
if "0padding" in feature_line:
|
|
raise GateFailure("INFRA_BLOCKED", f"{name} legacy fixture unexpectedly has 0padding")
|
|
image = generated[0].read_bytes()
|
|
if len(image) != fixture["expected_image_size"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"{name} image size changed")
|
|
extents = []
|
|
reconstructed = bytearray()
|
|
for expected in fixture["extents"]:
|
|
block = image[expected["physical_offset"] : expected["physical_offset"] + expected["physical_length"]]
|
|
decoded, consumed = raw_lz4_decode(block, expected["logical_length"])
|
|
tail = block[consumed:]
|
|
if (
|
|
consumed != expected["consumed_bytes"]
|
|
or sha256_bytes(decoded) != expected["decoded_sha256"]
|
|
or len(tail) != expected["zero_tail_bytes"]
|
|
or any(tail)
|
|
):
|
|
raise GateFailure("INFRA_BLOCKED", f"{name} independent extent oracle drift")
|
|
logical = legacy_source[expected["logical_offset"] : expected["logical_offset"] + expected["logical_length"]]
|
|
if decoded != logical:
|
|
raise GateFailure("INFRA_BLOCKED", f"{name} decoded extent differs from source")
|
|
reconstructed.extend(decoded)
|
|
extents.append({
|
|
**expected,
|
|
"input_bytes": len(block),
|
|
"tail_all_zero": not any(tail),
|
|
})
|
|
if bytes(reconstructed) != legacy_source:
|
|
raise GateFailure("INFRA_BLOCKED", f"{name} reconstructed file mismatch")
|
|
records.append({
|
|
"commit": resolved,
|
|
"fsck": "PASS",
|
|
"image_repeated_sha256": hashes,
|
|
"image_size": len(image),
|
|
"legacy_without_0padding": True,
|
|
"name": name,
|
|
"extents": extents,
|
|
})
|
|
return records, worktrees
|
|
|
|
|
|
def verify_partial_fixture(temp: Path, worktree: Path, full: Any, partial_codec: Any, dut: Any) -> dict[str, Any]:
|
|
partial_spec = SPEC["partial_fixture"]
|
|
source_dir = temp / "partial-source"
|
|
data_a, data_b = make_partial_source(source_dir)
|
|
images = temp / "partial-images"
|
|
images.mkdir()
|
|
mkfs = worktree / "mkfs/mkfs.erofs"
|
|
fsck = worktree / "fsck/fsck.erofs"
|
|
dump = worktree / "dump/dump.erofs"
|
|
generated = []
|
|
for label in ("a", "b"):
|
|
image = images / f"partial-{label}.erofs"
|
|
run_logged(
|
|
[str(mkfs), *partial_spec["mkfs_args"], str(image), str(source_dir)],
|
|
worktree,
|
|
OUTPUT / f"logs/partial-mkfs-{label}.log",
|
|
)
|
|
generated.append(image)
|
|
hashes = [sha256_path(path) for path in generated]
|
|
if hashes[0] != hashes[1] or hashes[0] != partial_spec["expected_image_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"partial fixture reproducibility/hash mismatch: {hashes}")
|
|
run_logged([str(fsck), "--extract", str(generated[0])], worktree, OUTPUT / "logs/partial-fsck.log")
|
|
dump_b = run_logged(
|
|
[str(dump), "--path=/b.dat", "-e", str(generated[0])],
|
|
worktree,
|
|
OUTPUT / "logs/partial-dump-b.log",
|
|
).stdout
|
|
match = re.search(r"^NID:\s+(\d+)", dump_b, re.MULTILINE)
|
|
if match is None or int(match.group(1)) != partial_spec["b_nid"]:
|
|
raise GateFailure("INFRA_BLOCKED", "partial fixture b.dat NID drift")
|
|
image = generated[0].read_bytes()
|
|
if len(image) != partial_spec["expected_image_size"]:
|
|
raise GateFailure("INFRA_BLOCKED", "partial fixture image size drift")
|
|
inode_offset = partial_spec["b_nid"] * 32
|
|
inode_format, xattr_count, _, _, inode_size = struct.unpack_from("<HHHHI", image, inode_offset)
|
|
inode_bytes = 64 if inode_format & 1 else 32
|
|
datalayout = (inode_format >> 1) & 7
|
|
if xattr_count != 0 or datalayout != 1 or inode_size != len(data_b):
|
|
raise GateFailure("INFRA_BLOCKED", "partial fixture inode contract drift")
|
|
index_start = ((inode_offset + inode_bytes + 7) & ~7) + 16
|
|
index_records = []
|
|
for expected in partial_spec["partial_index_records"]:
|
|
position = index_start + expected["lcn"] * 8
|
|
advise, cluster_offset, pblk = struct.unpack_from("<HHI", image, position)
|
|
record = {
|
|
"advise": advise,
|
|
"cluster_offset": cluster_offset,
|
|
"lcn": expected["lcn"],
|
|
"partial_ref": bool(advise & 0x8000),
|
|
"pblk": pblk,
|
|
"position": position,
|
|
}
|
|
if not record["partial_ref"] or (cluster_offset, pblk) != (expected["cluster_offset"], expected["pblk"]):
|
|
raise GateFailure("INFRA_BLOCKED", "partial-reference index oracle drift")
|
|
index_records.append(record)
|
|
block_offset = partial_spec["shared_pcluster_offset"]
|
|
block = image[block_offset : block_offset + partial_spec["block_size"]]
|
|
first_nonzero = next((index for index, value in enumerate(block) if value), len(block))
|
|
if first_nonzero != partial_spec["stream_start"]:
|
|
raise GateFailure("INFRA_BLOCKED", "partial pcluster leading padding drift")
|
|
stream = block[first_nonzero:]
|
|
full_decoded, full_consumed = raw_lz4_decode(stream, partial_spec["full_decoded_bytes"])
|
|
if full_consumed != partial_spec["full_stream_consumed_bytes"]:
|
|
raise GateFailure("INFRA_BLOCKED", "partial pcluster full consumption drift")
|
|
expected_tail = data_a[-partial_spec["full_decoded_bytes"] :]
|
|
if full_decoded != expected_tail:
|
|
raise GateFailure("INFRA_BLOCKED", "partial pcluster decoded bytes drift")
|
|
partial_records = []
|
|
for output_size in partial_spec["partial_output_bytes"]:
|
|
decoded, consumed = raw_lz4_decode(stream, output_size)
|
|
if consumed != partial_spec["partial_consumed_bytes"] or decoded != expected_tail[:output_size]:
|
|
raise GateFailure("INFRA_BLOCKED", "partial parser consumption/output drift")
|
|
result, output, guards = dut_call(dut[1], dut[2], stream, output_size, True)
|
|
if result != 0 or output != expected_tail[:output_size] or not guards:
|
|
raise GateFailure("INFRA_BLOCKED", "frozen DUT partial decode mismatch")
|
|
partial_records.append({"consumed_bytes": consumed, "decoded_bytes": output_size, "dut_errno": result, "guards": guards})
|
|
corrupted = bytearray(stream)
|
|
corrupted[partial_spec["partial_consumed_bytes"] :] = b"\0" * (
|
|
len(corrupted) - partial_spec["partial_consumed_bytes"]
|
|
)
|
|
partial_result, partial_output, partial_guards = dut_call(
|
|
dut[1], dut[2], bytes(corrupted), partial_spec["partial_output_bytes"][0], True
|
|
)
|
|
full_result, _, full_guards = dut_call(
|
|
dut[1], dut[2], bytes(corrupted), partial_spec["full_decoded_bytes"], False
|
|
)
|
|
lib_partial_result, lib_partial_output = liblz4_call(
|
|
partial_codec, bytes(corrupted), partial_spec["partial_output_bytes"][0], partial_spec["partial_output_bytes"][0]
|
|
)
|
|
lib_full_result, _ = liblz4_call(full, bytes(corrupted), partial_spec["full_decoded_bytes"])
|
|
if (
|
|
partial_result != 0
|
|
or partial_output != expected_tail[: partial_spec["partial_output_bytes"][0]]
|
|
or full_result != 97
|
|
or lib_partial_result != partial_spec["partial_output_bytes"][0]
|
|
or lib_partial_output != expected_tail[: partial_spec["partial_output_bytes"][0]]
|
|
or lib_full_result >= 0
|
|
or not partial_guards
|
|
or not full_guards
|
|
):
|
|
raise GateFailure("INFRA_BLOCKED", "range-after-corruption oracle mismatch")
|
|
return {
|
|
"fixture_repeated_sha256": hashes,
|
|
"fsck": "PASS",
|
|
"index_records": index_records,
|
|
"partial_records": partial_records,
|
|
"range_after_corruption": {
|
|
"corruption_starts_at_consumed_byte": partial_spec["partial_consumed_bytes"],
|
|
"dut_full_errno": full_result,
|
|
"dut_partial_errno": partial_result,
|
|
"full_guard_unchanged": full_guards,
|
|
"liblz4_full_result": lib_full_result,
|
|
"liblz4_partial_result": lib_partial_result,
|
|
"partial_guard_unchanged": partial_guards,
|
|
"partial_output_matches_full_slice": True,
|
|
},
|
|
"shared_stream": {
|
|
"consumed_bytes": full_consumed,
|
|
"decoded_bytes": len(full_decoded),
|
|
"leading_zero_bytes": first_nonzero,
|
|
"sha256": sha256_bytes(stream),
|
|
},
|
|
}
|
|
|
|
|
|
def finalize() -> None:
|
|
lines = []
|
|
for path in sorted(OUTPUT.rglob("*")):
|
|
if path.is_file() and path.name != "SHA256SUMS":
|
|
lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
|
|
(OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
|
|
|
|
|
|
result: dict[str, Any] | None = None
|
|
exit_code = 0
|
|
owned_temp: str | None = None
|
|
try:
|
|
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-062" or SPEC.get("gate") != "G04":
|
|
raise GateFailure("INFRA_BLOCKED", "invalid P15-062 input schema")
|
|
resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
|
|
if resolved != SPEC["required_base"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"P15-062 must replay {SPEC['required_base']}, got {resolved}")
|
|
sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]}
|
|
source_hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()}
|
|
if source_hashes != SPEC["source_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed")
|
|
freebsd_head = subprocess.check_output(["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True).strip()
|
|
freebsd_hashes = {path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"]}
|
|
if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed")
|
|
if "#define\tEINTEGRITY\t97" not in (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8"):
|
|
raise GateFailure("INFRA_BLOCKED", "FreeBSD EINTEGRITY positive errno changed")
|
|
lz4_source = sources["repo-pre-15/src/decompressor_lz4.c"]
|
|
if "while (ip < iend)" not in lz4_source or "if (*ip++ != 0)" not in lz4_source:
|
|
raise GateFailure("INFRA_BLOCKED", "frozen DUT no longer has the zero-tail full-decode rule")
|
|
if "m->map->m_plen = m->compressedblks << m->sbi->blkszbits;" not in sources["repo-pre-15/src/zmap.c"]:
|
|
raise GateFailure("INFRA_BLOCKED", "frozen DUT no longer maps legacy pcluster input by whole blocks")
|
|
if "ret = LZ4_decompress_safe(src + inputmargin, out," not in sources["src-linux/decompressor.c"]:
|
|
raise GateFailure("INFRA_BLOCKED", "Linux LZ4 comparison anchor changed")
|
|
write_json(OUTPUT / "source-sha256.json", source_hashes)
|
|
write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes})
|
|
cleanup_contract = verify_cleanup_source(sources["repo-pre-15/src/zdata.c"])
|
|
full_codec, partial_codec = load_liblz4()
|
|
with tempfile.TemporaryDirectory(prefix="p15-062-g04-") as temporary:
|
|
owned_temp = temporary
|
|
temp = Path(temporary)
|
|
legacy_source = make_legacy_source(temp / "legacy-source-check")
|
|
generator_records, worktrees = build_versions(temp, legacy_source)
|
|
dut = compile_dut_decoder(temp, lz4_source)
|
|
dut_cases = []
|
|
for record in generator_records:
|
|
image_path = temp / f"images-{record['name']}/legacy-a.erofs"
|
|
image = image_path.read_bytes()
|
|
expected = SPEC["legacy_fixture"]["extents"][1]
|
|
block = image[expected["physical_offset"] : expected["physical_offset"] + expected["physical_length"]]
|
|
stream = block[: expected["consumed_bytes"]]
|
|
zero_tail_result, zero_tail_output, zero_tail_guards = dut_call(
|
|
dut[1], dut[2], block, expected["logical_length"], False
|
|
)
|
|
exact_result, exact_output, exact_guards = dut_call(
|
|
dut[1], dut[2], stream, expected["logical_length"], False
|
|
)
|
|
nonzero = bytearray(block)
|
|
nonzero[expected["consumed_bytes"]] = 1
|
|
nonzero_result, _, nonzero_guards = dut_call(
|
|
dut[1], dut[2], bytes(nonzero), expected["logical_length"], False
|
|
)
|
|
truncated_result, _, truncated_guards = dut_call(
|
|
dut[1], dut[2], stream[:-1], expected["logical_length"], False
|
|
)
|
|
lib_full_result, _ = liblz4_call(full_codec, block, expected["logical_length"])
|
|
lib_exact_result, lib_exact_output = liblz4_call(full_codec, stream, expected["logical_length"])
|
|
expected_output = legacy_source[expected["logical_offset"] :]
|
|
if (
|
|
zero_tail_result != 0
|
|
or zero_tail_output != expected_output
|
|
or exact_result != 0
|
|
or exact_output != expected_output
|
|
or nonzero_result != 97
|
|
or truncated_result != 97
|
|
or lib_full_result >= 0
|
|
or lib_exact_result != expected["logical_length"]
|
|
or lib_exact_output != expected_output
|
|
or not all((zero_tail_guards, exact_guards, nonzero_guards, truncated_guards))
|
|
):
|
|
raise GateFailure("INFRA_BLOCKED", f"DUT/liblz4 legacy oracle mismatch for {record['name']}")
|
|
dut_cases.append({
|
|
"current_dut_exact_errno": exact_result,
|
|
"current_dut_nonzero_tail_errno": nonzero_result,
|
|
"current_dut_truncated_errno": truncated_result,
|
|
"current_dut_zero_tail_errno": zero_tail_result,
|
|
"exact_policy_would_accept": expected["consumed_bytes"] == expected["physical_length"],
|
|
"guards_unchanged": True,
|
|
"liblz4_exact_result": lib_exact_result,
|
|
"liblz4_physical_input_result": lib_full_result,
|
|
"name": record["name"],
|
|
"physical_input_bytes": expected["physical_length"],
|
|
"stream_consumed_bytes": expected["consumed_bytes"],
|
|
"zero_tail_bytes": expected["zero_tail_bytes"],
|
|
})
|
|
partial_record = verify_partial_fixture(temp, worktrees["v1.8.6"], full_codec, partial_codec, dut)
|
|
write_json(OUTPUT / "generator-ledger.json", generator_records)
|
|
write_json(OUTPUT / "dut-cases.json", dut_cases)
|
|
write_json(OUTPUT / "partial-oracle.json", partial_record)
|
|
write_json(OUTPUT / "cleanup-ledger.json", {
|
|
**cleanup_contract,
|
|
"decoder_allocations": 0,
|
|
"decoder_owned_buffers": 0,
|
|
"guards_unchanged_for_all_cases": True,
|
|
"input_buffer_owner": "z_erofs_read_extent caller",
|
|
"output_buffer_owner": "z_erofs_read_extent caller",
|
|
"typed_corruption_errno": 97,
|
|
})
|
|
incompatible = [case["name"] for case in dut_cases if not case["exact_policy_would_accept"]]
|
|
if incompatible != [version["name"] for version in SPEC["upstream"]["versions"]]:
|
|
raise GateFailure("INFRA_BLOCKED", f"unexpected incompatibility set: {incompatible}")
|
|
result = {
|
|
"b26": "STOP-NO-SOURCE",
|
|
"candidate": "P15-062",
|
|
"cleanup": "PASS",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gate": "G04",
|
|
"historical_generations": incompatible,
|
|
"oracle": "independent raw LZ4 parser + liblz4 1.10.0 + matching erofs-utils fsck",
|
|
"partial_reference": "PASS",
|
|
"qemu": "NOT_RUN",
|
|
"qemu_reason": "host oracle proves exact-full would reject reproducible legal legacy images",
|
|
"reason": "all three reproducible legacy mkfs generations store a short valid final raw LZ4 stream followed by zero bytes inside the block-sized m_plen",
|
|
"requested_base": REQUESTED_BASE,
|
|
"resolved_base": resolved,
|
|
"schema": 1,
|
|
"status": "STOP",
|
|
"typed_errno": "PASS",
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
exit_code = 1
|
|
cleanup_record = {
|
|
"owned_temp": owned_temp,
|
|
"owned_temp_removed": owned_temp is not None and not Path(owned_temp).exists(),
|
|
"protected_pid_touched": False,
|
|
"protected_port_touched": False,
|
|
"qemu_started": False,
|
|
}
|
|
if not cleanup_record["owned_temp_removed"]:
|
|
raise GateFailure("INFRA_BLOCKED", "owned gate temporary directory survived cleanup")
|
|
write_json(OUTPUT / "owned-cleanup.json", cleanup_record)
|
|
except GateFailure as failure:
|
|
result = {
|
|
"b26": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN",
|
|
"candidate": "P15-062",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gate": "G04",
|
|
"qemu": "NOT_RUN",
|
|
"reason": failure.reason,
|
|
"requested_base": REQUESTED_BASE,
|
|
"schema": 1,
|
|
"status": failure.status,
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
exit_code = 21 if failure.status == "INFRA_BLOCKED" else 1
|
|
except (OSError, subprocess.SubprocessError, ValueError) as failure:
|
|
result = {
|
|
"b26": "NOT_RUN",
|
|
"candidate": "P15-062",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gate": "G04",
|
|
"qemu": "NOT_RUN",
|
|
"reason": f"gate infrastructure failure: {failure}",
|
|
"requested_base": REQUESTED_BASE,
|
|
"schema": 1,
|
|
"status": "INFRA_BLOCKED",
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
exit_code = 21
|
|
finally:
|
|
finalize()
|
|
|
|
print(json.dumps(result, sort_keys=True))
|
|
raise SystemExit(exit_code)
|
|
PY
|