update
This commit is contained in:
Executable
+901
@@ -0,0 +1,901 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
||||
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
||||
input=$gate_dir/P15-081-input.json
|
||||
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
|
||||
base=
|
||||
output=
|
||||
|
||||
while test "$#" -gt 0; do
|
||||
case "$1" in
|
||||
--base)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
||||
base=$2
|
||||
shift 2
|
||||
;;
|
||||
--output)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
||||
output=$2
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
printf 'unknown argument: %s\n' "$1" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
||||
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
||||
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
|
||||
test -d "$freebsd_src/sys" || {
|
||||
printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2
|
||||
exit 2
|
||||
}
|
||||
for tool in cc dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf 'missing required host tool: %s\n' "$tool" >&2
|
||||
exit 2
|
||||
}
|
||||
done
|
||||
test "$(id -u)" -eq 0 || {
|
||||
printf '%s\n' 'root is required to materialize all seven inode types' >&2
|
||||
exit 2
|
||||
}
|
||||
case "$output" in
|
||||
/*) ;;
|
||||
*) output=$PWD/$output ;;
|
||||
esac
|
||||
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
||||
mkdir -p "$output"
|
||||
|
||||
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import stat
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
INPUT = Path(sys.argv[2])
|
||||
REQUESTED_BASE = sys.argv[3]
|
||||
OUTPUT = Path(sys.argv[4])
|
||||
FREEBSD_SRC = Path(sys.argv[5])
|
||||
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
|
||||
|
||||
SUPER = 1024
|
||||
MAGIC = 0xE0F5E1E2
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
S_IFMT = 0o170000
|
||||
EINTEGRITY = 97
|
||||
VTYPE_NUMBER = {
|
||||
"VNON": 0,
|
||||
"VREG": 1,
|
||||
"VDIR": 2,
|
||||
"VBLK": 3,
|
||||
"VCHR": 4,
|
||||
"VLNK": 5,
|
||||
"VSOCK": 6,
|
||||
"VFIFO": 7,
|
||||
}
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
return sha256_bytes(path.read_bytes())
|
||||
|
||||
|
||||
def write_json(path: Path, value: Any) -> None:
|
||||
path.write_text(
|
||||
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
|
||||
|
||||
def git(*args: str) -> str:
|
||||
return subprocess.check_output(
|
||||
["git", "-C", str(ROOT), *args], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def source_at(commit: str, path: str) -> str:
|
||||
completed = subprocess.run(
|
||||
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"cannot read {path} at {commit}: {completed.stderr}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def run(argv: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
|
||||
completed = subprocess.run(
|
||||
argv,
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=env,
|
||||
)
|
||||
return completed
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self, data: bytes | bytearray):
|
||||
self.data = bytearray(data)
|
||||
if len(self.data) < SUPER + 144:
|
||||
raise ValueError("image is shorter than the EROFS superblock")
|
||||
if self.u32(SUPER) != MAGIC:
|
||||
raise ValueError("image has the wrong EROFS magic")
|
||||
self.block_bits = self.data[SUPER + 12]
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.meta_blkaddr = self.u32(SUPER + 40)
|
||||
self.root_nid = self.u16(SUPER + 14)
|
||||
if self.root_nid == 0:
|
||||
self.root_nid = self.u64(SUPER + 112)
|
||||
if self.checksum_end > len(self.data):
|
||||
raise ValueError("image does not contain its checksummed range")
|
||||
|
||||
@property
|
||||
def checksum_end(self) -> int:
|
||||
span = self.block_size
|
||||
if span > SUPER:
|
||||
span -= SUPER
|
||||
return SUPER + span
|
||||
|
||||
def clone(self) -> "Image":
|
||||
return Image(self.data)
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def u64(self, offset: int) -> int:
|
||||
return struct.unpack_from("<Q", self.data, offset)[0]
|
||||
|
||||
def put_u32(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<I", self.data, offset, value)
|
||||
|
||||
def checksum_valid(self) -> bool:
|
||||
expected = self.u32(SUPER + 4)
|
||||
canonical = bytearray(self.data[SUPER : self.checksum_end])
|
||||
struct.pack_into("<I", canonical, 4, 0)
|
||||
return expected == crc32c(canonical)
|
||||
|
||||
def update_checksum(self) -> None:
|
||||
self.put_u32(SUPER + 4, 0)
|
||||
self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.checksum_end]))
|
||||
if not self.checksum_valid():
|
||||
raise AssertionError("updated checksum is not valid")
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
if offset + 32 > len(self.data):
|
||||
raise ValueError(f"inode {nid} is outside the image")
|
||||
ifmt = self.u16(offset)
|
||||
version = ifmt & 1
|
||||
inode_size = 32 if version == 0 else 64
|
||||
if offset + inode_size > len(self.data):
|
||||
raise ValueError(f"inode {nid} is truncated")
|
||||
xattr_count = self.u16(offset + 2)
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + (xattr_count - 1) * 4
|
||||
size = self.u32(offset + 8) if version == 0 else self.u64(offset + 8)
|
||||
return {
|
||||
"nid": nid,
|
||||
"offset": offset,
|
||||
"ifmt": ifmt,
|
||||
"layout": (ifmt >> 1) & 7,
|
||||
"inode_size": inode_size,
|
||||
"xattr_size": xattr_size,
|
||||
"mode": self.u16(offset + 4),
|
||||
"size": size,
|
||||
"startblk": self.u32(offset + 16),
|
||||
}
|
||||
|
||||
def inode_data_offset(self, inode: dict[str, int]) -> int:
|
||||
if inode["layout"] == 2:
|
||||
return inode["offset"] + inode["inode_size"] + inode["xattr_size"]
|
||||
if inode["layout"] == 0:
|
||||
return inode["startblk"] << self.block_bits
|
||||
raise ValueError(f"unsupported directory layout {inode['layout']}")
|
||||
|
||||
def directory(self, nid: int) -> dict[str, dict[str, Any]]:
|
||||
inode = self.inode(nid)
|
||||
if inode["mode"] & S_IFMT != stat.S_IFDIR:
|
||||
raise ValueError(f"inode {nid} is not a directory")
|
||||
data_offset = self.inode_data_offset(inode)
|
||||
size = inode["size"]
|
||||
if size < 12 or data_offset + size > len(self.data):
|
||||
raise ValueError("root directory bytes are outside the image")
|
||||
first_nameoff = self.u16(data_offset + 8)
|
||||
if first_nameoff == 0 or first_nameoff % 12 != 0 or first_nameoff >= size:
|
||||
raise ValueError("invalid first directory name offset")
|
||||
count = first_nameoff // 12
|
||||
records: dict[str, dict[str, Any]] = {}
|
||||
for index in range(count):
|
||||
entry = data_offset + index * 12
|
||||
nameoff = self.u16(entry + 8)
|
||||
endoff = self.u16(entry + 20) if index + 1 < count else size
|
||||
if not (first_nameoff <= nameoff < endoff <= size):
|
||||
raise ValueError(f"invalid directory name span at index {index}")
|
||||
raw = bytes(self.data[data_offset + nameoff : data_offset + endoff])
|
||||
name = raw.split(b"\0", 1)[0].decode("ascii")
|
||||
if not name or name in records:
|
||||
raise ValueError(f"invalid or duplicate directory name {name!r}")
|
||||
records[name] = {
|
||||
"entry_offset": entry,
|
||||
"nid": self.u64(entry),
|
||||
"file_type": self.data[entry + 10],
|
||||
"reserved": self.data[entry + 11],
|
||||
"nameoff": nameoff,
|
||||
}
|
||||
return records
|
||||
|
||||
|
||||
def make_source(path: Path) -> None:
|
||||
previous_umask = os.umask(0o022)
|
||||
try:
|
||||
path.mkdir(mode=0o755)
|
||||
os.chmod(path, 0o755)
|
||||
(path / "regular").write_bytes(b"P15-081 real inode payload\n")
|
||||
os.chmod(path / "regular", 0o644)
|
||||
os.link(path / "regular", path / "regular-hard")
|
||||
(path / "subdir").mkdir(mode=0o755)
|
||||
os.chmod(path / "subdir", 0o755)
|
||||
os.symlink("regular", path / "symlink")
|
||||
os.mkfifo(path / "fifo", 0o600)
|
||||
os.chmod(path / "fifo", 0o600)
|
||||
os.mknod(path / "char", stat.S_IFCHR | 0o600, os.makedev(1, 3))
|
||||
os.chmod(path / "char", 0o600)
|
||||
os.mknod(path / "block", stat.S_IFBLK | 0o600, os.makedev(7, 0))
|
||||
os.chmod(path / "block", 0o600)
|
||||
sock = socket.socket(socket.AF_UNIX)
|
||||
try:
|
||||
sock.bind(str(path / "socket"))
|
||||
finally:
|
||||
sock.close()
|
||||
os.chmod(path / "socket", 0o600)
|
||||
finally:
|
||||
os.umask(previous_umask)
|
||||
|
||||
|
||||
def build_seed(work: Path, output: Path) -> tuple[bytes, str, list[str]]:
|
||||
work.mkdir()
|
||||
source = work / "source"
|
||||
make_source(source)
|
||||
command = [
|
||||
"mkfs.erofs",
|
||||
"-d0",
|
||||
"-T0",
|
||||
"--all-time",
|
||||
"--all-root",
|
||||
"--workers=1",
|
||||
"-x-1",
|
||||
"-E",
|
||||
"noinline_data",
|
||||
"-U",
|
||||
SPEC["uuid"],
|
||||
str(output),
|
||||
str(source),
|
||||
]
|
||||
env = dict(os.environ)
|
||||
env["SOURCE_DATE_EPOCH"] = "0"
|
||||
completed = run(command, env=env)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"mkfs.erofs failed: {completed.stdout}")
|
||||
canonical = command[:-2] + ["SEED.erofs", "SOURCE"]
|
||||
return output.read_bytes(), completed.stdout, canonical
|
||||
|
||||
|
||||
def extract_kind(output: str) -> str:
|
||||
match = re.search(r"^Size:.*? ([A-Za-z ]+)$", output, re.MULTILINE)
|
||||
if match is None:
|
||||
raise ValueError(f"dump.erofs output has no inode kind:\n{output}")
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def extract_dump_nid(output: str) -> int:
|
||||
match = re.search(r"^NID: (\d+)\b", output, re.MULTILINE)
|
||||
if match is None:
|
||||
raise ValueError(f"dump.erofs output has no NID:\n{output}")
|
||||
return int(match.group(1))
|
||||
|
||||
|
||||
def expected_errno(file_type: int, expected_file_type: int) -> int:
|
||||
if file_type < 1 or file_type > 7:
|
||||
return 0
|
||||
return 0 if file_type == expected_file_type else EINTEGRITY
|
||||
|
||||
|
||||
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-081":
|
||||
raise SystemExit("invalid P15-081 gate input")
|
||||
resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
|
||||
if resolved != SPEC["required_base"]:
|
||||
raise SystemExit(f"P15-081 must replay {SPEC['required_base']}, got {resolved}")
|
||||
|
||||
sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]}
|
||||
source_hashes = {
|
||||
path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()
|
||||
}
|
||||
for path, expected in SPEC["source_sha256"].items():
|
||||
if source_hashes[path] != expected:
|
||||
raise SystemExit(f"frozen source identity mismatch: {path}")
|
||||
|
||||
freebsd_head = subprocess.check_output(
|
||||
["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True
|
||||
).strip()
|
||||
if freebsd_head != SPEC["freebsd_head"]:
|
||||
raise SystemExit(f"FreeBSD HEAD differs: {freebsd_head}")
|
||||
freebsd_hashes = {}
|
||||
for relative, expected in SPEC["freebsd_sha256"].items():
|
||||
path = FREEBSD_SRC / relative
|
||||
actual = sha256_path(path)
|
||||
freebsd_hashes[relative] = actual
|
||||
if actual != expected:
|
||||
raise SystemExit(f"FreeBSD source identity mismatch: {relative}")
|
||||
|
||||
mkfs_version_run = run(["mkfs.erofs", "-V"])
|
||||
mkfs_version = mkfs_version_run.stdout.splitlines()[0]
|
||||
if mkfs_version != SPEC["mkfs_version"]:
|
||||
raise SystemExit(
|
||||
f"mkfs.erofs version differs: expected {SPEC['mkfs_version']!r}, "
|
||||
f"got {mkfs_version!r}"
|
||||
)
|
||||
|
||||
namei = sources["repo-pre-15/src/namei.c"]
|
||||
directory_source = sources["repo-pre-15/src/dir.c"]
|
||||
inode_source = sources["repo-pre-15/src/inode.c"]
|
||||
internal_source = sources["repo-pre-15/src/internal.h"]
|
||||
vnops = sources["repo-pre-15/src/erofs_vnops.c"]
|
||||
linux_namei = sources["src-linux/namei.c"]
|
||||
linux_dir = sources["src-linux/dir.c"]
|
||||
linux_ondisk = sources["src-linux/erofs_fs.h"]
|
||||
source_anchors = {
|
||||
"freebsd_dirent_field": "*d_type = de->file_type;" in namei,
|
||||
"freebsd_inode_mode": "vi->vtype = IFTOVT(vi->mode);" in inode_source,
|
||||
"freebsd_lookup_vget_once": namei.count("erofs_vget(") == 1,
|
||||
"freebsd_dotdot_once": namei.count("error = vn_vget_ino(") == 1,
|
||||
"freebsd_hash_lookup": "error = vfs_hash_get(" in vnops,
|
||||
"freebsd_inode_decode_after_hash": vnops.index("error = vfs_hash_get(")
|
||||
< vnops.index("error = erofs_read_inode("),
|
||||
"linux_readdir_mapping": "fs_ftype_to_dtype(de->file_type)" in linux_dir,
|
||||
"linux_lookup_dirent_field": "*d_type = de->file_type;" in linux_namei,
|
||||
"linux_lookup_inode": "inode = erofs_iget(dir->i_sb, nid);" in linux_namei,
|
||||
"linux_generic_ft_contract": "EROFS file types should match generic FT_* types"
|
||||
in linux_ondisk,
|
||||
}
|
||||
if not all(source_anchors.values()):
|
||||
raise SystemExit(f"source entrypoint anchors changed: {source_anchors}")
|
||||
|
||||
freebsd_dirent = (FREEBSD_SRC / "sys/sys/dirent.h").read_text(encoding="utf-8")
|
||||
freebsd_vnode = (FREEBSD_SRC / "sys/sys/vnode.h").read_text(encoding="utf-8")
|
||||
if "#define\tDT_UNKNOWN\t 0" not in freebsd_dirent:
|
||||
raise SystemExit("FreeBSD DT_UNKNOWN contract changed")
|
||||
if "#define\tIFTODT(mode)" not in freebsd_dirent:
|
||||
raise SystemExit("FreeBSD IFTODT contract changed")
|
||||
if "int vfs_hash_get(" not in freebsd_vnode or "int\tvn_vget_ino(" not in freebsd_vnode:
|
||||
raise SystemExit("FreeBSD vnode lookup declarations changed")
|
||||
|
||||
type_by_name = {item["name"]: item for item in SPEC["file_types"]}
|
||||
type_by_name["regular-hard"] = type_by_name["regular"]
|
||||
|
||||
fixtures = OUTPUT / "fixtures"
|
||||
fixtures.mkdir()
|
||||
with tempfile.TemporaryDirectory(prefix="p15-081-gate-") as temporary:
|
||||
temp = Path(temporary)
|
||||
first_bytes, first_stdout, canonical_command = build_seed(
|
||||
temp / "first", temp / "first.erofs"
|
||||
)
|
||||
second_bytes, second_stdout, second_command = build_seed(
|
||||
temp / "second", temp / "second.erofs"
|
||||
)
|
||||
if first_bytes != second_bytes or first_stdout != second_stdout:
|
||||
raise SystemExit("P15-081 seed generation is not byte reproducible")
|
||||
if canonical_command != second_command:
|
||||
raise SystemExit("P15-081 canonical mkfs command changed between runs")
|
||||
|
||||
seed_path = fixtures / "seed.erofs"
|
||||
seed_path.write_bytes(first_bytes)
|
||||
(OUTPUT / "mkfs.stdout").write_text(first_stdout, encoding="utf-8")
|
||||
write_json(
|
||||
OUTPUT / "generator.json",
|
||||
{
|
||||
"command": canonical_command,
|
||||
"mkfs_version": mkfs_version,
|
||||
"repeat_byte_identical": True,
|
||||
"seed_sha256": sha256_bytes(first_bytes),
|
||||
"seed_size": len(first_bytes),
|
||||
},
|
||||
)
|
||||
|
||||
seed = Image(first_bytes)
|
||||
if not seed.checksum_valid():
|
||||
raise SystemExit("generated seed checksum is invalid")
|
||||
seed_entries = seed.directory(seed.root_nid)
|
||||
required_names = {item["name"] for item in SPEC["file_types"]}
|
||||
required_names.add("regular-hard")
|
||||
missing = sorted(required_names - seed_entries.keys())
|
||||
if missing:
|
||||
raise SystemExit(f"seed directory is missing names: {missing}")
|
||||
|
||||
seed_records = []
|
||||
for name in sorted(required_names):
|
||||
entry = seed_entries[name]
|
||||
inode = seed.inode(entry["nid"])
|
||||
expected_type = type_by_name[name]
|
||||
record = {
|
||||
"case": f"known-match-{name}",
|
||||
"class": "known-match",
|
||||
"expected_errno": 0,
|
||||
"file_type": entry["file_type"],
|
||||
"inode_mode": inode["mode"],
|
||||
"inode_mode_type": inode["mode"] & S_IFMT,
|
||||
"name": name,
|
||||
"nid": entry["nid"],
|
||||
"reserved": entry["reserved"],
|
||||
"vtype": expected_type["vtype"],
|
||||
}
|
||||
if entry["file_type"] != expected_type["erofs"]:
|
||||
raise SystemExit(f"mkfs emitted unexpected file type for {name}")
|
||||
if record["inode_mode_type"] != expected_type["mode_type"]:
|
||||
raise SystemExit(f"mkfs emitted unexpected inode mode for {name}")
|
||||
seed_records.append(record)
|
||||
|
||||
if seed_entries[SPEC["hardlink"]["first"]]["nid"] != seed_entries[SPEC["hardlink"]["second"]]["nid"]:
|
||||
raise SystemExit("hardlink seed names do not share one inode")
|
||||
|
||||
cases: list[dict[str, Any]] = []
|
||||
for item in SPEC["known_mismatches"]:
|
||||
cases.append(
|
||||
{
|
||||
"class": "known-mismatch",
|
||||
"field": "file_type",
|
||||
"id": f"known-mismatch-{item['name']}",
|
||||
"name": item["name"],
|
||||
"value": item["file_type"],
|
||||
"expected_errno": EINTEGRITY,
|
||||
}
|
||||
)
|
||||
for item in SPEC["tolerated"]:
|
||||
cases.append(
|
||||
{
|
||||
"class": "forward-compatible",
|
||||
"field": item["field"],
|
||||
"id": item["id"],
|
||||
"name": item["name"],
|
||||
"value": item["value"],
|
||||
"expected_errno": 0,
|
||||
}
|
||||
)
|
||||
|
||||
case_records: list[dict[str, Any]] = []
|
||||
normal_entry_lines = [
|
||||
"id\tname\tnid\tdirent_file_type\tdirent_reserved\tinode_mode_type\tdump_dirent_type\tdump_inode_kind\tfsck\tcandidate_errno"
|
||||
]
|
||||
fixture_digest = hashlib.sha256()
|
||||
for case in cases:
|
||||
image = seed.clone()
|
||||
entries = image.directory(image.root_nid)
|
||||
entry = entries[case["name"]]
|
||||
direct_offset = entry["entry_offset"] + (10 if case["field"] == "file_type" else 11)
|
||||
before = image.data[direct_offset]
|
||||
image.data[direct_offset] = case["value"]
|
||||
image.update_checksum()
|
||||
path = fixtures / f"{case['id']}.erofs"
|
||||
path.write_bytes(image.data)
|
||||
|
||||
parsed = Image(path.read_bytes())
|
||||
if not parsed.checksum_valid():
|
||||
raise SystemExit(f"fixture checksum is invalid: {case['id']}")
|
||||
parsed_entry = parsed.directory(parsed.root_nid)[case["name"]]
|
||||
parsed_inode = parsed.inode(parsed_entry["nid"])
|
||||
expected_type = type_by_name[case["name"]]
|
||||
actual_errno = expected_errno(parsed_entry["file_type"], expected_type["erofs"])
|
||||
|
||||
fsck = run(["fsck.erofs", "-d0", str(path)])
|
||||
fsck_clean = fsck.returncode == 0 and "<E>" not in fsck.stdout
|
||||
fsck_expected_clean = not (
|
||||
case["field"] == "file_type" and case["value"] > 7
|
||||
)
|
||||
dump_ls = run(["dump.erofs", "--ls", "--path=/", str(path)])
|
||||
dump_path = run(["dump.erofs", f"--path=/{case['name']}", str(path)])
|
||||
if dump_ls.returncode != 0 or dump_path.returncode != 0:
|
||||
raise SystemExit(f"normal path parser failed: {case['id']}")
|
||||
listing = re.search(
|
||||
rf"^\s+{parsed_entry['nid']}\s+(\d+)\s+{re.escape(case['name'])}$",
|
||||
dump_ls.stdout,
|
||||
re.MULTILINE,
|
||||
)
|
||||
if listing is None:
|
||||
raise SystemExit(f"dump listing did not reach dirent: {case['id']}")
|
||||
dump_dirent_type = int(listing.group(1))
|
||||
dump_kind = extract_kind(dump_path.stdout)
|
||||
dump_nid = extract_dump_nid(dump_path.stdout)
|
||||
|
||||
record = {
|
||||
**case,
|
||||
"actual_errno": actual_errno,
|
||||
"checksum_valid": True,
|
||||
"direct_before": before,
|
||||
"direct_offset": direct_offset,
|
||||
"dump_dirent_type": dump_dirent_type,
|
||||
"dump_inode_kind": dump_kind,
|
||||
"dump_nid": dump_nid,
|
||||
"file_type": parsed_entry["file_type"],
|
||||
"fixture": path.name,
|
||||
"fixture_sha256": sha256_path(path),
|
||||
"fsck_clean": fsck_clean,
|
||||
"fsck_expected_clean": fsck_expected_clean,
|
||||
"inode_mode": parsed_inode["mode"],
|
||||
"inode_mode_type": parsed_inode["mode"] & S_IFMT,
|
||||
"nid": parsed_entry["nid"],
|
||||
"reserved": parsed_entry["reserved"],
|
||||
"vtype": expected_type["vtype"],
|
||||
}
|
||||
record["oracle_pass"] = all(
|
||||
(
|
||||
record["actual_errno"] == record["expected_errno"],
|
||||
record["fsck_clean"] == record["fsck_expected_clean"],
|
||||
record["dump_dirent_type"] == record["file_type"],
|
||||
record["dump_inode_kind"] == expected_type["kind"],
|
||||
record["dump_nid"] == record["nid"],
|
||||
record["inode_mode_type"] == expected_type["mode_type"],
|
||||
)
|
||||
)
|
||||
case_records.append(record)
|
||||
normal_entry_lines.append(
|
||||
"\t".join(
|
||||
str(value)
|
||||
for value in (
|
||||
record["id"],
|
||||
record["name"],
|
||||
record["nid"],
|
||||
record["file_type"],
|
||||
record["reserved"],
|
||||
record["inode_mode_type"],
|
||||
record["dump_dirent_type"],
|
||||
record["dump_inode_kind"],
|
||||
"clean" if record["fsck_clean"] else "policy-reject",
|
||||
record["actual_errno"],
|
||||
)
|
||||
)
|
||||
)
|
||||
fixture_digest.update(case["id"].encode("ascii"))
|
||||
fixture_digest.update(b"\0")
|
||||
fixture_digest.update(path.read_bytes())
|
||||
|
||||
(OUTPUT / "normal-entry.tsv").write_text(
|
||||
"\n".join(normal_entry_lines) + "\n", encoding="ascii"
|
||||
)
|
||||
|
||||
all_records = seed_records + case_records
|
||||
prototype = r'''#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define EINTEGRITY 97
|
||||
|
||||
enum prototype_vtype {
|
||||
VT_NON,
|
||||
VT_REG,
|
||||
VT_DIR,
|
||||
VT_BLK,
|
||||
VT_CHR,
|
||||
VT_LNK,
|
||||
VT_SOCK,
|
||||
VT_FIFO,
|
||||
};
|
||||
|
||||
static bool
|
||||
erofs_dirent_type_matches(uint8_t file_type, enum prototype_vtype vtype)
|
||||
{
|
||||
switch (file_type) {
|
||||
case 1:
|
||||
return (vtype == VT_REG);
|
||||
case 2:
|
||||
return (vtype == VT_DIR);
|
||||
case 3:
|
||||
return (vtype == VT_CHR);
|
||||
case 4:
|
||||
return (vtype == VT_BLK);
|
||||
case 5:
|
||||
return (vtype == VT_FIFO);
|
||||
case 6:
|
||||
return (vtype == VT_SOCK);
|
||||
case 7:
|
||||
return (vtype == VT_LNK);
|
||||
default:
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
unsigned long file_type, vtype;
|
||||
int error;
|
||||
|
||||
if (argc != 3)
|
||||
return (2);
|
||||
file_type = strtoul(argv[1], NULL, 0);
|
||||
vtype = strtoul(argv[2], NULL, 0);
|
||||
if (file_type > UINT8_MAX || vtype > VT_FIFO)
|
||||
return (2);
|
||||
error = erofs_dirent_type_matches((uint8_t)file_type,
|
||||
(enum prototype_vtype)vtype) ? 0 : EINTEGRITY;
|
||||
printf("%d\n", error);
|
||||
return (0);
|
||||
}
|
||||
'''
|
||||
prototype_path = OUTPUT / "prototype.c"
|
||||
prototype_path.write_text(prototype, encoding="ascii")
|
||||
binary = temp / "prototype"
|
||||
compile_run = run(
|
||||
[
|
||||
"cc",
|
||||
"-std=c11",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
str(prototype_path),
|
||||
"-o",
|
||||
str(binary),
|
||||
]
|
||||
)
|
||||
if compile_run.returncode != 0:
|
||||
raise SystemExit(f"candidate prototype did not compile: {compile_run.stdout}")
|
||||
prototype_records = []
|
||||
for record in all_records:
|
||||
completed = run(
|
||||
[
|
||||
str(binary),
|
||||
str(record["file_type"]),
|
||||
str(VTYPE_NUMBER[record["vtype"]]),
|
||||
]
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"candidate prototype failed for {record['case'] if 'case' in record else record['id']}")
|
||||
observed = int(completed.stdout.strip())
|
||||
expected = record["expected_errno"]
|
||||
prototype_records.append(
|
||||
{
|
||||
"id": record.get("id", record.get("case")),
|
||||
"observed_errno": observed,
|
||||
"expected_errno": expected,
|
||||
"pass": observed == expected,
|
||||
}
|
||||
)
|
||||
|
||||
helper = '''
|
||||
bool
|
||||
erofs_dirent_type_matches(uint8_t file_type, __enum_uint8(vtype) vtype)
|
||||
{
|
||||
\tswitch (file_type) {
|
||||
\tcase EROFS_FT_REG_FILE:
|
||||
\t\treturn (vtype == VREG);
|
||||
\tcase EROFS_FT_DIR:
|
||||
\t\treturn (vtype == VDIR);
|
||||
\tcase EROFS_FT_CHRDEV:
|
||||
\t\treturn (vtype == VCHR);
|
||||
\tcase EROFS_FT_BLKDEV:
|
||||
\t\treturn (vtype == VBLK);
|
||||
\tcase EROFS_FT_FIFO:
|
||||
\t\treturn (vtype == VFIFO);
|
||||
\tcase EROFS_FT_SOCK:
|
||||
\t\treturn (vtype == VSOCK);
|
||||
\tcase EROFS_FT_SYMLINK:
|
||||
\t\treturn (vtype == VLNK);
|
||||
\tdefault:
|
||||
\t\treturn (true);
|
||||
\t}
|
||||
}
|
||||
|
||||
'''
|
||||
inode_marker = "/*\n * Read and decode a disk inode."
|
||||
if inode_source.count(inode_marker) != 1:
|
||||
raise SystemExit("candidate inode insertion marker is ambiguous")
|
||||
candidate_inode = inode_source.replace(inode_marker, helper + inode_marker, 1)
|
||||
prototype_decl = (
|
||||
"bool erofs_dirent_type_matches(uint8_t file_type, "
|
||||
"__enum_uint8(vtype) vtype);\n"
|
||||
)
|
||||
internal_marker = "int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,\n"
|
||||
if internal_source.count(internal_marker) != 1:
|
||||
raise SystemExit("candidate prototype insertion marker is ambiguous")
|
||||
candidate_internal = internal_source.replace(
|
||||
internal_marker, prototype_decl + internal_marker, 1
|
||||
)
|
||||
lookup_marker = "\tif (error != 0)\n\t\treturn (error);\n\t*ap->a_vpp = vp;\n"
|
||||
lookup_candidate = (
|
||||
"\tif (error != 0)\n"
|
||||
"\t\treturn (error);\n"
|
||||
"\tif ((cnp->cn_flags & ISDOTDOT) == 0 &&\n"
|
||||
"\t !erofs_dirent_type_matches(dtype, VTOE(vp)->vtype)) {\n"
|
||||
"\t\tvput(vp);\n"
|
||||
"\t\treturn (EINTEGRITY);\n"
|
||||
"\t}\n"
|
||||
"\t*ap->a_vpp = vp;\n"
|
||||
)
|
||||
if namei.count(lookup_marker) != 1:
|
||||
raise SystemExit("candidate lookup insertion marker is ambiguous")
|
||||
candidate_namei = namei.replace(lookup_marker, lookup_candidate, 1)
|
||||
patch_lines = []
|
||||
for path, before, after in (
|
||||
("src/inode.c", inode_source, candidate_inode),
|
||||
("src/internal.h", internal_source, candidate_internal),
|
||||
("src/namei.c", namei, candidate_namei),
|
||||
):
|
||||
patch_lines.extend(
|
||||
difflib.unified_diff(
|
||||
before.splitlines(keepends=True),
|
||||
after.splitlines(keepends=True),
|
||||
fromfile=f"a/{path}",
|
||||
tofile=f"b/{path}",
|
||||
)
|
||||
)
|
||||
candidate_patch = "".join(patch_lines)
|
||||
(OUTPUT / "candidate.patch").write_text(candidate_patch, encoding="utf-8")
|
||||
|
||||
lock_calls = re.findall(
|
||||
r"\b(?:erofs_vget|vn_vget_ino|vfs_hash_get|VOP_LOCK|vn_lock|lockmgr)\s*\(",
|
||||
helper + lookup_candidate,
|
||||
)
|
||||
cache_first = seed_entries[SPEC["hardlink"]["first"]]
|
||||
cache_second = seed_entries[SPEC["hardlink"]["second"]]
|
||||
cache_case = next(
|
||||
record for record in case_records if record["id"] == "known-mismatch-regular-hard"
|
||||
)
|
||||
cache_proof = {
|
||||
"first_lookup_name": SPEC["hardlink"]["first"],
|
||||
"second_lookup_name": SPEC["hardlink"]["second"],
|
||||
"different_namecache_keys": SPEC["hardlink"]["first"]
|
||||
!= SPEC["hardlink"]["second"],
|
||||
"same_real_nid": cache_first["nid"] == cache_second["nid"],
|
||||
"second_dirent_known_mismatch": cache_case["actual_errno"] == EINTEGRITY,
|
||||
"normal_lookup_calls_existing_erofs_vget": namei.count("erofs_vget(") == 1,
|
||||
"erofs_vget_checks_hash_before_inode_read": source_anchors[
|
||||
"freebsd_inode_decode_after_hash"
|
||||
],
|
||||
}
|
||||
lock_ledger = {
|
||||
"candidate_added_lock_or_vget_calls": lock_calls,
|
||||
"candidate_reads_immutable_vtype": "VTOE(vp)->vtype" in lookup_candidate,
|
||||
"dotdot_bypasses_validator":
|
||||
"(cnp->cn_flags & ISDOTDOT) == 0" in lookup_candidate,
|
||||
"mismatch_drops_locked_child": "vput(vp);" in lookup_candidate,
|
||||
"validator_before_vpp_publication": lookup_candidate.index(
|
||||
"erofs_dirent_type_matches"
|
||||
)
|
||||
< lookup_candidate.index("*ap->a_vpp = vp"),
|
||||
"validator_before_namecache_publication": candidate_namei.index(
|
||||
"erofs_dirent_type_matches"
|
||||
)
|
||||
< candidate_namei.index("cache_enter(dvp, vp, cnp)"),
|
||||
"readdir_has_no_vget": all(
|
||||
token not in directory_source
|
||||
for token in ("erofs_vget(", "vn_vget_ino(", "vfs_hash_get(")
|
||||
),
|
||||
"parent_child_lock_sequence_unchanged": namei.count("erofs_vget(")
|
||||
== candidate_namei.count("erofs_vget(")
|
||||
and namei.count("error = vn_vget_ino(")
|
||||
== candidate_namei.count("error = vn_vget_ino("),
|
||||
"freebsd_contract_sha256": freebsd_hashes,
|
||||
"cache_hit_proof": cache_proof,
|
||||
}
|
||||
write_json(OUTPUT / "lock-ledger.json", lock_ledger)
|
||||
|
||||
failures = []
|
||||
failures.extend(
|
||||
record["id"] for record in case_records if not record["oracle_pass"]
|
||||
)
|
||||
failures.extend(
|
||||
record["id"] for record in prototype_records if not record["pass"]
|
||||
)
|
||||
for key, value in lock_ledger.items():
|
||||
if key == "freebsd_contract_sha256":
|
||||
continue
|
||||
if key == "candidate_added_lock_or_vget_calls":
|
||||
if value:
|
||||
failures.append(key)
|
||||
elif key == "cache_hit_proof":
|
||||
if not all(value.values()):
|
||||
failures.append(key)
|
||||
elif value is not True:
|
||||
failures.append(key)
|
||||
status = "GO" if not failures else "STOP"
|
||||
|
||||
oracle = {
|
||||
"cache_hit": cache_proof,
|
||||
"candidate_patch_sha256": sha256_bytes(candidate_patch.encode("utf-8")),
|
||||
"candidate_prototype_sha256": sha256_path(prototype_path),
|
||||
"case_records": case_records,
|
||||
"known_match_records": seed_records,
|
||||
"prototype_records": prototype_records,
|
||||
"source_anchors": source_anchors,
|
||||
"status": status,
|
||||
}
|
||||
write_json(OUTPUT / "oracle.json", oracle)
|
||||
write_json(OUTPUT / "source-sha256.json", source_hashes)
|
||||
write_json(
|
||||
OUTPUT / "freebsd-contract.json",
|
||||
{"head": freebsd_head, "sha256": freebsd_hashes},
|
||||
)
|
||||
|
||||
fixture_index = {
|
||||
"fixture_count": len(list(fixtures.glob("*.erofs"))),
|
||||
"fixture_set_sha256": fixture_digest.hexdigest(),
|
||||
"fixtures": [
|
||||
{
|
||||
"name": path.name,
|
||||
"sha256": sha256_path(path),
|
||||
"size": path.stat().st_size,
|
||||
}
|
||||
for path in sorted(fixtures.glob("*.erofs"))
|
||||
],
|
||||
"seed_repeat_byte_identical": True,
|
||||
}
|
||||
write_json(OUTPUT / "fixture-index.json", fixture_index)
|
||||
|
||||
result = {
|
||||
"b11": "AUTHORIZED" if status == "GO" else "STOP-NO-SOURCE",
|
||||
"cache_hit_sequence_count": 1,
|
||||
"candidate": "P15-081",
|
||||
"failures": failures,
|
||||
"fixture_count": fixture_index["fixture_count"],
|
||||
"fixture_set_sha256": fixture_index["fixture_set_sha256"],
|
||||
"forward_compatible_case_count": len(SPEC["tolerated"]),
|
||||
"full_feature_suite": "NOT_RUN",
|
||||
"gate": "G11",
|
||||
"known_match_count": len(seed_records),
|
||||
"known_mismatch_count": len(SPEC["known_mismatches"]),
|
||||
"normal_entry_count": len(case_records),
|
||||
"prototype_case_count": len(prototype_records),
|
||||
"qemu": "NOT_RUN",
|
||||
"qemu_reason": "pre-source gate uses checksum-valid disk fixtures, independent host parsing, and a non-KLD prototype",
|
||||
"requested_base": REQUESTED_BASE,
|
||||
"resolved_base": resolved,
|
||||
"schema": 1,
|
||||
"status": status,
|
||||
}
|
||||
write_json(OUTPUT / "result.json", result)
|
||||
|
||||
hash_lines = []
|
||||
for path in sorted(OUTPUT.rglob("*")):
|
||||
if path.is_file() and path.name != "SHA256SUMS":
|
||||
hash_lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
|
||||
(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii")
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
if status != "GO":
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
Reference in New Issue
Block a user