902 lines
31 KiB
Bash
Executable File
902 lines
31 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
|
input=$gate_dir/P15-081-input.json
|
|
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
|
|
base=
|
|
output=
|
|
|
|
while test "$#" -gt 0; do
|
|
case "$1" in
|
|
--base)
|
|
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
|
base=$2
|
|
shift 2
|
|
;;
|
|
--output)
|
|
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
|
output=$2
|
|
shift 2
|
|
;;
|
|
*)
|
|
printf 'unknown argument: %s\n' "$1" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
|
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
|
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
|
|
test -d "$freebsd_src/sys" || {
|
|
printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2
|
|
exit 2
|
|
}
|
|
for tool in cc dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'missing required host tool: %s\n' "$tool" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
test "$(id -u)" -eq 0 || {
|
|
printf '%s\n' 'root is required to materialize all seven inode types' >&2
|
|
exit 2
|
|
}
|
|
case "$output" in
|
|
/*) ;;
|
|
*) output=$PWD/$output ;;
|
|
esac
|
|
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
|
mkdir -p "$output"
|
|
|
|
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import difflib
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import stat
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(sys.argv[1])
|
|
INPUT = Path(sys.argv[2])
|
|
REQUESTED_BASE = sys.argv[3]
|
|
OUTPUT = Path(sys.argv[4])
|
|
FREEBSD_SRC = Path(sys.argv[5])
|
|
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
|
|
|
|
SUPER = 1024
|
|
MAGIC = 0xE0F5E1E2
|
|
CRC32C_POLY = 0x82F63B78
|
|
S_IFMT = 0o170000
|
|
EINTEGRITY = 97
|
|
VTYPE_NUMBER = {
|
|
"VNON": 0,
|
|
"VREG": 1,
|
|
"VDIR": 2,
|
|
"VBLK": 3,
|
|
"VCHR": 4,
|
|
"VLNK": 5,
|
|
"VSOCK": 6,
|
|
"VFIFO": 7,
|
|
}
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_path(path: Path) -> str:
|
|
return sha256_bytes(path.read_bytes())
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.write_text(
|
|
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
|
)
|
|
|
|
|
|
def git(*args: str) -> str:
|
|
return subprocess.check_output(
|
|
["git", "-C", str(ROOT), *args], text=True
|
|
).strip()
|
|
|
|
|
|
def source_at(commit: str, path: str) -> str:
|
|
completed = subprocess.run(
|
|
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
if completed.returncode != 0:
|
|
raise SystemExit(f"cannot read {path} at {commit}: {completed.stderr}")
|
|
return completed.stdout
|
|
|
|
|
|
def run(argv: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
|
|
completed = subprocess.run(
|
|
argv,
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
env=env,
|
|
)
|
|
return completed
|
|
|
|
|
|
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
|
value = seed
|
|
for byte in data:
|
|
value ^= byte
|
|
for _ in range(8):
|
|
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
|
return value & 0xFFFFFFFF
|
|
|
|
|
|
class Image:
|
|
def __init__(self, data: bytes | bytearray):
|
|
self.data = bytearray(data)
|
|
if len(self.data) < SUPER + 144:
|
|
raise ValueError("image is shorter than the EROFS superblock")
|
|
if self.u32(SUPER) != MAGIC:
|
|
raise ValueError("image has the wrong EROFS magic")
|
|
self.block_bits = self.data[SUPER + 12]
|
|
self.block_size = 1 << self.block_bits
|
|
self.meta_blkaddr = self.u32(SUPER + 40)
|
|
self.root_nid = self.u16(SUPER + 14)
|
|
if self.root_nid == 0:
|
|
self.root_nid = self.u64(SUPER + 112)
|
|
if self.checksum_end > len(self.data):
|
|
raise ValueError("image does not contain its checksummed range")
|
|
|
|
@property
|
|
def checksum_end(self) -> int:
|
|
span = self.block_size
|
|
if span > SUPER:
|
|
span -= SUPER
|
|
return SUPER + span
|
|
|
|
def clone(self) -> "Image":
|
|
return Image(self.data)
|
|
|
|
def u16(self, offset: int) -> int:
|
|
return struct.unpack_from("<H", self.data, offset)[0]
|
|
|
|
def u32(self, offset: int) -> int:
|
|
return struct.unpack_from("<I", self.data, offset)[0]
|
|
|
|
def u64(self, offset: int) -> int:
|
|
return struct.unpack_from("<Q", self.data, offset)[0]
|
|
|
|
def put_u32(self, offset: int, value: int) -> None:
|
|
struct.pack_into("<I", self.data, offset, value)
|
|
|
|
def checksum_valid(self) -> bool:
|
|
expected = self.u32(SUPER + 4)
|
|
canonical = bytearray(self.data[SUPER : self.checksum_end])
|
|
struct.pack_into("<I", canonical, 4, 0)
|
|
return expected == crc32c(canonical)
|
|
|
|
def update_checksum(self) -> None:
|
|
self.put_u32(SUPER + 4, 0)
|
|
self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.checksum_end]))
|
|
if not self.checksum_valid():
|
|
raise AssertionError("updated checksum is not valid")
|
|
|
|
def inode(self, nid: int) -> dict[str, int]:
|
|
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
|
if offset + 32 > len(self.data):
|
|
raise ValueError(f"inode {nid} is outside the image")
|
|
ifmt = self.u16(offset)
|
|
version = ifmt & 1
|
|
inode_size = 32 if version == 0 else 64
|
|
if offset + inode_size > len(self.data):
|
|
raise ValueError(f"inode {nid} is truncated")
|
|
xattr_count = self.u16(offset + 2)
|
|
xattr_size = 0 if xattr_count == 0 else 12 + (xattr_count - 1) * 4
|
|
size = self.u32(offset + 8) if version == 0 else self.u64(offset + 8)
|
|
return {
|
|
"nid": nid,
|
|
"offset": offset,
|
|
"ifmt": ifmt,
|
|
"layout": (ifmt >> 1) & 7,
|
|
"inode_size": inode_size,
|
|
"xattr_size": xattr_size,
|
|
"mode": self.u16(offset + 4),
|
|
"size": size,
|
|
"startblk": self.u32(offset + 16),
|
|
}
|
|
|
|
def inode_data_offset(self, inode: dict[str, int]) -> int:
|
|
if inode["layout"] == 2:
|
|
return inode["offset"] + inode["inode_size"] + inode["xattr_size"]
|
|
if inode["layout"] == 0:
|
|
return inode["startblk"] << self.block_bits
|
|
raise ValueError(f"unsupported directory layout {inode['layout']}")
|
|
|
|
def directory(self, nid: int) -> dict[str, dict[str, Any]]:
|
|
inode = self.inode(nid)
|
|
if inode["mode"] & S_IFMT != stat.S_IFDIR:
|
|
raise ValueError(f"inode {nid} is not a directory")
|
|
data_offset = self.inode_data_offset(inode)
|
|
size = inode["size"]
|
|
if size < 12 or data_offset + size > len(self.data):
|
|
raise ValueError("root directory bytes are outside the image")
|
|
first_nameoff = self.u16(data_offset + 8)
|
|
if first_nameoff == 0 or first_nameoff % 12 != 0 or first_nameoff >= size:
|
|
raise ValueError("invalid first directory name offset")
|
|
count = first_nameoff // 12
|
|
records: dict[str, dict[str, Any]] = {}
|
|
for index in range(count):
|
|
entry = data_offset + index * 12
|
|
nameoff = self.u16(entry + 8)
|
|
endoff = self.u16(entry + 20) if index + 1 < count else size
|
|
if not (first_nameoff <= nameoff < endoff <= size):
|
|
raise ValueError(f"invalid directory name span at index {index}")
|
|
raw = bytes(self.data[data_offset + nameoff : data_offset + endoff])
|
|
name = raw.split(b"\0", 1)[0].decode("ascii")
|
|
if not name or name in records:
|
|
raise ValueError(f"invalid or duplicate directory name {name!r}")
|
|
records[name] = {
|
|
"entry_offset": entry,
|
|
"nid": self.u64(entry),
|
|
"file_type": self.data[entry + 10],
|
|
"reserved": self.data[entry + 11],
|
|
"nameoff": nameoff,
|
|
}
|
|
return records
|
|
|
|
|
|
def make_source(path: Path) -> None:
|
|
previous_umask = os.umask(0o022)
|
|
try:
|
|
path.mkdir(mode=0o755)
|
|
os.chmod(path, 0o755)
|
|
(path / "regular").write_bytes(b"P15-081 real inode payload\n")
|
|
os.chmod(path / "regular", 0o644)
|
|
os.link(path / "regular", path / "regular-hard")
|
|
(path / "subdir").mkdir(mode=0o755)
|
|
os.chmod(path / "subdir", 0o755)
|
|
os.symlink("regular", path / "symlink")
|
|
os.mkfifo(path / "fifo", 0o600)
|
|
os.chmod(path / "fifo", 0o600)
|
|
os.mknod(path / "char", stat.S_IFCHR | 0o600, os.makedev(1, 3))
|
|
os.chmod(path / "char", 0o600)
|
|
os.mknod(path / "block", stat.S_IFBLK | 0o600, os.makedev(7, 0))
|
|
os.chmod(path / "block", 0o600)
|
|
sock = socket.socket(socket.AF_UNIX)
|
|
try:
|
|
sock.bind(str(path / "socket"))
|
|
finally:
|
|
sock.close()
|
|
os.chmod(path / "socket", 0o600)
|
|
finally:
|
|
os.umask(previous_umask)
|
|
|
|
|
|
def build_seed(work: Path, output: Path) -> tuple[bytes, str, list[str]]:
|
|
work.mkdir()
|
|
source = work / "source"
|
|
make_source(source)
|
|
command = [
|
|
"mkfs.erofs",
|
|
"-d0",
|
|
"-T0",
|
|
"--all-time",
|
|
"--all-root",
|
|
"--workers=1",
|
|
"-x-1",
|
|
"-E",
|
|
"noinline_data",
|
|
"-U",
|
|
SPEC["uuid"],
|
|
str(output),
|
|
str(source),
|
|
]
|
|
env = dict(os.environ)
|
|
env["SOURCE_DATE_EPOCH"] = "0"
|
|
completed = run(command, env=env)
|
|
if completed.returncode != 0:
|
|
raise SystemExit(f"mkfs.erofs failed: {completed.stdout}")
|
|
canonical = command[:-2] + ["SEED.erofs", "SOURCE"]
|
|
return output.read_bytes(), completed.stdout, canonical
|
|
|
|
|
|
def extract_kind(output: str) -> str:
|
|
match = re.search(r"^Size:.*? ([A-Za-z ]+)$", output, re.MULTILINE)
|
|
if match is None:
|
|
raise ValueError(f"dump.erofs output has no inode kind:\n{output}")
|
|
return match.group(1)
|
|
|
|
|
|
def extract_dump_nid(output: str) -> int:
|
|
match = re.search(r"^NID: (\d+)\b", output, re.MULTILINE)
|
|
if match is None:
|
|
raise ValueError(f"dump.erofs output has no NID:\n{output}")
|
|
return int(match.group(1))
|
|
|
|
|
|
def expected_errno(file_type: int, expected_file_type: int) -> int:
|
|
if file_type < 1 or file_type > 7:
|
|
return 0
|
|
return 0 if file_type == expected_file_type else EINTEGRITY
|
|
|
|
|
|
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-081":
|
|
raise SystemExit("invalid P15-081 gate input")
|
|
resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
|
|
if resolved != SPEC["required_base"]:
|
|
raise SystemExit(f"P15-081 must replay {SPEC['required_base']}, got {resolved}")
|
|
|
|
sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]}
|
|
source_hashes = {
|
|
path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()
|
|
}
|
|
for path, expected in SPEC["source_sha256"].items():
|
|
if source_hashes[path] != expected:
|
|
raise SystemExit(f"frozen source identity mismatch: {path}")
|
|
|
|
freebsd_head = subprocess.check_output(
|
|
["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True
|
|
).strip()
|
|
if freebsd_head != SPEC["freebsd_head"]:
|
|
raise SystemExit(f"FreeBSD HEAD differs: {freebsd_head}")
|
|
freebsd_hashes = {}
|
|
for relative, expected in SPEC["freebsd_sha256"].items():
|
|
path = FREEBSD_SRC / relative
|
|
actual = sha256_path(path)
|
|
freebsd_hashes[relative] = actual
|
|
if actual != expected:
|
|
raise SystemExit(f"FreeBSD source identity mismatch: {relative}")
|
|
|
|
mkfs_version_run = run(["mkfs.erofs", "-V"])
|
|
mkfs_version = mkfs_version_run.stdout.splitlines()[0]
|
|
if mkfs_version != SPEC["mkfs_version"]:
|
|
raise SystemExit(
|
|
f"mkfs.erofs version differs: expected {SPEC['mkfs_version']!r}, "
|
|
f"got {mkfs_version!r}"
|
|
)
|
|
|
|
namei = sources["repo-pre-15/src/namei.c"]
|
|
directory_source = sources["repo-pre-15/src/dir.c"]
|
|
inode_source = sources["repo-pre-15/src/inode.c"]
|
|
internal_source = sources["repo-pre-15/src/internal.h"]
|
|
vnops = sources["repo-pre-15/src/erofs_vnops.c"]
|
|
linux_namei = sources["src-linux/namei.c"]
|
|
linux_dir = sources["src-linux/dir.c"]
|
|
linux_ondisk = sources["src-linux/erofs_fs.h"]
|
|
source_anchors = {
|
|
"freebsd_dirent_field": "*d_type = de->file_type;" in namei,
|
|
"freebsd_inode_mode": "vi->vtype = IFTOVT(vi->mode);" in inode_source,
|
|
"freebsd_lookup_vget_once": namei.count("erofs_vget(") == 1,
|
|
"freebsd_dotdot_once": namei.count("error = vn_vget_ino(") == 1,
|
|
"freebsd_hash_lookup": "error = vfs_hash_get(" in vnops,
|
|
"freebsd_inode_decode_after_hash": vnops.index("error = vfs_hash_get(")
|
|
< vnops.index("error = erofs_read_inode("),
|
|
"linux_readdir_mapping": "fs_ftype_to_dtype(de->file_type)" in linux_dir,
|
|
"linux_lookup_dirent_field": "*d_type = de->file_type;" in linux_namei,
|
|
"linux_lookup_inode": "inode = erofs_iget(dir->i_sb, nid);" in linux_namei,
|
|
"linux_generic_ft_contract": "EROFS file types should match generic FT_* types"
|
|
in linux_ondisk,
|
|
}
|
|
if not all(source_anchors.values()):
|
|
raise SystemExit(f"source entrypoint anchors changed: {source_anchors}")
|
|
|
|
freebsd_dirent = (FREEBSD_SRC / "sys/sys/dirent.h").read_text(encoding="utf-8")
|
|
freebsd_vnode = (FREEBSD_SRC / "sys/sys/vnode.h").read_text(encoding="utf-8")
|
|
if "#define\tDT_UNKNOWN\t 0" not in freebsd_dirent:
|
|
raise SystemExit("FreeBSD DT_UNKNOWN contract changed")
|
|
if "#define\tIFTODT(mode)" not in freebsd_dirent:
|
|
raise SystemExit("FreeBSD IFTODT contract changed")
|
|
if "int vfs_hash_get(" not in freebsd_vnode or "int\tvn_vget_ino(" not in freebsd_vnode:
|
|
raise SystemExit("FreeBSD vnode lookup declarations changed")
|
|
|
|
type_by_name = {item["name"]: item for item in SPEC["file_types"]}
|
|
type_by_name["regular-hard"] = type_by_name["regular"]
|
|
|
|
fixtures = OUTPUT / "fixtures"
|
|
fixtures.mkdir()
|
|
with tempfile.TemporaryDirectory(prefix="p15-081-gate-") as temporary:
|
|
temp = Path(temporary)
|
|
first_bytes, first_stdout, canonical_command = build_seed(
|
|
temp / "first", temp / "first.erofs"
|
|
)
|
|
second_bytes, second_stdout, second_command = build_seed(
|
|
temp / "second", temp / "second.erofs"
|
|
)
|
|
if first_bytes != second_bytes or first_stdout != second_stdout:
|
|
raise SystemExit("P15-081 seed generation is not byte reproducible")
|
|
if canonical_command != second_command:
|
|
raise SystemExit("P15-081 canonical mkfs command changed between runs")
|
|
|
|
seed_path = fixtures / "seed.erofs"
|
|
seed_path.write_bytes(first_bytes)
|
|
(OUTPUT / "mkfs.stdout").write_text(first_stdout, encoding="utf-8")
|
|
write_json(
|
|
OUTPUT / "generator.json",
|
|
{
|
|
"command": canonical_command,
|
|
"mkfs_version": mkfs_version,
|
|
"repeat_byte_identical": True,
|
|
"seed_sha256": sha256_bytes(first_bytes),
|
|
"seed_size": len(first_bytes),
|
|
},
|
|
)
|
|
|
|
seed = Image(first_bytes)
|
|
if not seed.checksum_valid():
|
|
raise SystemExit("generated seed checksum is invalid")
|
|
seed_entries = seed.directory(seed.root_nid)
|
|
required_names = {item["name"] for item in SPEC["file_types"]}
|
|
required_names.add("regular-hard")
|
|
missing = sorted(required_names - seed_entries.keys())
|
|
if missing:
|
|
raise SystemExit(f"seed directory is missing names: {missing}")
|
|
|
|
seed_records = []
|
|
for name in sorted(required_names):
|
|
entry = seed_entries[name]
|
|
inode = seed.inode(entry["nid"])
|
|
expected_type = type_by_name[name]
|
|
record = {
|
|
"case": f"known-match-{name}",
|
|
"class": "known-match",
|
|
"expected_errno": 0,
|
|
"file_type": entry["file_type"],
|
|
"inode_mode": inode["mode"],
|
|
"inode_mode_type": inode["mode"] & S_IFMT,
|
|
"name": name,
|
|
"nid": entry["nid"],
|
|
"reserved": entry["reserved"],
|
|
"vtype": expected_type["vtype"],
|
|
}
|
|
if entry["file_type"] != expected_type["erofs"]:
|
|
raise SystemExit(f"mkfs emitted unexpected file type for {name}")
|
|
if record["inode_mode_type"] != expected_type["mode_type"]:
|
|
raise SystemExit(f"mkfs emitted unexpected inode mode for {name}")
|
|
seed_records.append(record)
|
|
|
|
if seed_entries[SPEC["hardlink"]["first"]]["nid"] != seed_entries[SPEC["hardlink"]["second"]]["nid"]:
|
|
raise SystemExit("hardlink seed names do not share one inode")
|
|
|
|
cases: list[dict[str, Any]] = []
|
|
for item in SPEC["known_mismatches"]:
|
|
cases.append(
|
|
{
|
|
"class": "known-mismatch",
|
|
"field": "file_type",
|
|
"id": f"known-mismatch-{item['name']}",
|
|
"name": item["name"],
|
|
"value": item["file_type"],
|
|
"expected_errno": EINTEGRITY,
|
|
}
|
|
)
|
|
for item in SPEC["tolerated"]:
|
|
cases.append(
|
|
{
|
|
"class": "forward-compatible",
|
|
"field": item["field"],
|
|
"id": item["id"],
|
|
"name": item["name"],
|
|
"value": item["value"],
|
|
"expected_errno": 0,
|
|
}
|
|
)
|
|
|
|
case_records: list[dict[str, Any]] = []
|
|
normal_entry_lines = [
|
|
"id\tname\tnid\tdirent_file_type\tdirent_reserved\tinode_mode_type\tdump_dirent_type\tdump_inode_kind\tfsck\tcandidate_errno"
|
|
]
|
|
fixture_digest = hashlib.sha256()
|
|
for case in cases:
|
|
image = seed.clone()
|
|
entries = image.directory(image.root_nid)
|
|
entry = entries[case["name"]]
|
|
direct_offset = entry["entry_offset"] + (10 if case["field"] == "file_type" else 11)
|
|
before = image.data[direct_offset]
|
|
image.data[direct_offset] = case["value"]
|
|
image.update_checksum()
|
|
path = fixtures / f"{case['id']}.erofs"
|
|
path.write_bytes(image.data)
|
|
|
|
parsed = Image(path.read_bytes())
|
|
if not parsed.checksum_valid():
|
|
raise SystemExit(f"fixture checksum is invalid: {case['id']}")
|
|
parsed_entry = parsed.directory(parsed.root_nid)[case["name"]]
|
|
parsed_inode = parsed.inode(parsed_entry["nid"])
|
|
expected_type = type_by_name[case["name"]]
|
|
actual_errno = expected_errno(parsed_entry["file_type"], expected_type["erofs"])
|
|
|
|
fsck = run(["fsck.erofs", "-d0", str(path)])
|
|
fsck_clean = fsck.returncode == 0 and "<E>" not in fsck.stdout
|
|
fsck_expected_clean = not (
|
|
case["field"] == "file_type" and case["value"] > 7
|
|
)
|
|
dump_ls = run(["dump.erofs", "--ls", "--path=/", str(path)])
|
|
dump_path = run(["dump.erofs", f"--path=/{case['name']}", str(path)])
|
|
if dump_ls.returncode != 0 or dump_path.returncode != 0:
|
|
raise SystemExit(f"normal path parser failed: {case['id']}")
|
|
listing = re.search(
|
|
rf"^\s+{parsed_entry['nid']}\s+(\d+)\s+{re.escape(case['name'])}$",
|
|
dump_ls.stdout,
|
|
re.MULTILINE,
|
|
)
|
|
if listing is None:
|
|
raise SystemExit(f"dump listing did not reach dirent: {case['id']}")
|
|
dump_dirent_type = int(listing.group(1))
|
|
dump_kind = extract_kind(dump_path.stdout)
|
|
dump_nid = extract_dump_nid(dump_path.stdout)
|
|
|
|
record = {
|
|
**case,
|
|
"actual_errno": actual_errno,
|
|
"checksum_valid": True,
|
|
"direct_before": before,
|
|
"direct_offset": direct_offset,
|
|
"dump_dirent_type": dump_dirent_type,
|
|
"dump_inode_kind": dump_kind,
|
|
"dump_nid": dump_nid,
|
|
"file_type": parsed_entry["file_type"],
|
|
"fixture": path.name,
|
|
"fixture_sha256": sha256_path(path),
|
|
"fsck_clean": fsck_clean,
|
|
"fsck_expected_clean": fsck_expected_clean,
|
|
"inode_mode": parsed_inode["mode"],
|
|
"inode_mode_type": parsed_inode["mode"] & S_IFMT,
|
|
"nid": parsed_entry["nid"],
|
|
"reserved": parsed_entry["reserved"],
|
|
"vtype": expected_type["vtype"],
|
|
}
|
|
record["oracle_pass"] = all(
|
|
(
|
|
record["actual_errno"] == record["expected_errno"],
|
|
record["fsck_clean"] == record["fsck_expected_clean"],
|
|
record["dump_dirent_type"] == record["file_type"],
|
|
record["dump_inode_kind"] == expected_type["kind"],
|
|
record["dump_nid"] == record["nid"],
|
|
record["inode_mode_type"] == expected_type["mode_type"],
|
|
)
|
|
)
|
|
case_records.append(record)
|
|
normal_entry_lines.append(
|
|
"\t".join(
|
|
str(value)
|
|
for value in (
|
|
record["id"],
|
|
record["name"],
|
|
record["nid"],
|
|
record["file_type"],
|
|
record["reserved"],
|
|
record["inode_mode_type"],
|
|
record["dump_dirent_type"],
|
|
record["dump_inode_kind"],
|
|
"clean" if record["fsck_clean"] else "policy-reject",
|
|
record["actual_errno"],
|
|
)
|
|
)
|
|
)
|
|
fixture_digest.update(case["id"].encode("ascii"))
|
|
fixture_digest.update(b"\0")
|
|
fixture_digest.update(path.read_bytes())
|
|
|
|
(OUTPUT / "normal-entry.tsv").write_text(
|
|
"\n".join(normal_entry_lines) + "\n", encoding="ascii"
|
|
)
|
|
|
|
all_records = seed_records + case_records
|
|
prototype = r'''#include <stdbool.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
|
|
#define EINTEGRITY 97
|
|
|
|
enum prototype_vtype {
|
|
VT_NON,
|
|
VT_REG,
|
|
VT_DIR,
|
|
VT_BLK,
|
|
VT_CHR,
|
|
VT_LNK,
|
|
VT_SOCK,
|
|
VT_FIFO,
|
|
};
|
|
|
|
static bool
|
|
erofs_dirent_type_matches(uint8_t file_type, enum prototype_vtype vtype)
|
|
{
|
|
switch (file_type) {
|
|
case 1:
|
|
return (vtype == VT_REG);
|
|
case 2:
|
|
return (vtype == VT_DIR);
|
|
case 3:
|
|
return (vtype == VT_CHR);
|
|
case 4:
|
|
return (vtype == VT_BLK);
|
|
case 5:
|
|
return (vtype == VT_FIFO);
|
|
case 6:
|
|
return (vtype == VT_SOCK);
|
|
case 7:
|
|
return (vtype == VT_LNK);
|
|
default:
|
|
return (true);
|
|
}
|
|
}
|
|
|
|
int
|
|
main(int argc, char **argv)
|
|
{
|
|
unsigned long file_type, vtype;
|
|
int error;
|
|
|
|
if (argc != 3)
|
|
return (2);
|
|
file_type = strtoul(argv[1], NULL, 0);
|
|
vtype = strtoul(argv[2], NULL, 0);
|
|
if (file_type > UINT8_MAX || vtype > VT_FIFO)
|
|
return (2);
|
|
error = erofs_dirent_type_matches((uint8_t)file_type,
|
|
(enum prototype_vtype)vtype) ? 0 : EINTEGRITY;
|
|
printf("%d\n", error);
|
|
return (0);
|
|
}
|
|
'''
|
|
prototype_path = OUTPUT / "prototype.c"
|
|
prototype_path.write_text(prototype, encoding="ascii")
|
|
binary = temp / "prototype"
|
|
compile_run = run(
|
|
[
|
|
"cc",
|
|
"-std=c11",
|
|
"-Wall",
|
|
"-Wextra",
|
|
"-Werror",
|
|
str(prototype_path),
|
|
"-o",
|
|
str(binary),
|
|
]
|
|
)
|
|
if compile_run.returncode != 0:
|
|
raise SystemExit(f"candidate prototype did not compile: {compile_run.stdout}")
|
|
prototype_records = []
|
|
for record in all_records:
|
|
completed = run(
|
|
[
|
|
str(binary),
|
|
str(record["file_type"]),
|
|
str(VTYPE_NUMBER[record["vtype"]]),
|
|
]
|
|
)
|
|
if completed.returncode != 0:
|
|
raise SystemExit(f"candidate prototype failed for {record['case'] if 'case' in record else record['id']}")
|
|
observed = int(completed.stdout.strip())
|
|
expected = record["expected_errno"]
|
|
prototype_records.append(
|
|
{
|
|
"id": record.get("id", record.get("case")),
|
|
"observed_errno": observed,
|
|
"expected_errno": expected,
|
|
"pass": observed == expected,
|
|
}
|
|
)
|
|
|
|
helper = '''
|
|
bool
|
|
erofs_dirent_type_matches(uint8_t file_type, __enum_uint8(vtype) vtype)
|
|
{
|
|
\tswitch (file_type) {
|
|
\tcase EROFS_FT_REG_FILE:
|
|
\t\treturn (vtype == VREG);
|
|
\tcase EROFS_FT_DIR:
|
|
\t\treturn (vtype == VDIR);
|
|
\tcase EROFS_FT_CHRDEV:
|
|
\t\treturn (vtype == VCHR);
|
|
\tcase EROFS_FT_BLKDEV:
|
|
\t\treturn (vtype == VBLK);
|
|
\tcase EROFS_FT_FIFO:
|
|
\t\treturn (vtype == VFIFO);
|
|
\tcase EROFS_FT_SOCK:
|
|
\t\treturn (vtype == VSOCK);
|
|
\tcase EROFS_FT_SYMLINK:
|
|
\t\treturn (vtype == VLNK);
|
|
\tdefault:
|
|
\t\treturn (true);
|
|
\t}
|
|
}
|
|
|
|
'''
|
|
inode_marker = "/*\n * Read and decode a disk inode."
|
|
if inode_source.count(inode_marker) != 1:
|
|
raise SystemExit("candidate inode insertion marker is ambiguous")
|
|
candidate_inode = inode_source.replace(inode_marker, helper + inode_marker, 1)
|
|
prototype_decl = (
|
|
"bool erofs_dirent_type_matches(uint8_t file_type, "
|
|
"__enum_uint8(vtype) vtype);\n"
|
|
)
|
|
internal_marker = "int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,\n"
|
|
if internal_source.count(internal_marker) != 1:
|
|
raise SystemExit("candidate prototype insertion marker is ambiguous")
|
|
candidate_internal = internal_source.replace(
|
|
internal_marker, prototype_decl + internal_marker, 1
|
|
)
|
|
lookup_marker = "\tif (error != 0)\n\t\treturn (error);\n\t*ap->a_vpp = vp;\n"
|
|
lookup_candidate = (
|
|
"\tif (error != 0)\n"
|
|
"\t\treturn (error);\n"
|
|
"\tif ((cnp->cn_flags & ISDOTDOT) == 0 &&\n"
|
|
"\t !erofs_dirent_type_matches(dtype, VTOE(vp)->vtype)) {\n"
|
|
"\t\tvput(vp);\n"
|
|
"\t\treturn (EINTEGRITY);\n"
|
|
"\t}\n"
|
|
"\t*ap->a_vpp = vp;\n"
|
|
)
|
|
if namei.count(lookup_marker) != 1:
|
|
raise SystemExit("candidate lookup insertion marker is ambiguous")
|
|
candidate_namei = namei.replace(lookup_marker, lookup_candidate, 1)
|
|
patch_lines = []
|
|
for path, before, after in (
|
|
("src/inode.c", inode_source, candidate_inode),
|
|
("src/internal.h", internal_source, candidate_internal),
|
|
("src/namei.c", namei, candidate_namei),
|
|
):
|
|
patch_lines.extend(
|
|
difflib.unified_diff(
|
|
before.splitlines(keepends=True),
|
|
after.splitlines(keepends=True),
|
|
fromfile=f"a/{path}",
|
|
tofile=f"b/{path}",
|
|
)
|
|
)
|
|
candidate_patch = "".join(patch_lines)
|
|
(OUTPUT / "candidate.patch").write_text(candidate_patch, encoding="utf-8")
|
|
|
|
lock_calls = re.findall(
|
|
r"\b(?:erofs_vget|vn_vget_ino|vfs_hash_get|VOP_LOCK|vn_lock|lockmgr)\s*\(",
|
|
helper + lookup_candidate,
|
|
)
|
|
cache_first = seed_entries[SPEC["hardlink"]["first"]]
|
|
cache_second = seed_entries[SPEC["hardlink"]["second"]]
|
|
cache_case = next(
|
|
record for record in case_records if record["id"] == "known-mismatch-regular-hard"
|
|
)
|
|
cache_proof = {
|
|
"first_lookup_name": SPEC["hardlink"]["first"],
|
|
"second_lookup_name": SPEC["hardlink"]["second"],
|
|
"different_namecache_keys": SPEC["hardlink"]["first"]
|
|
!= SPEC["hardlink"]["second"],
|
|
"same_real_nid": cache_first["nid"] == cache_second["nid"],
|
|
"second_dirent_known_mismatch": cache_case["actual_errno"] == EINTEGRITY,
|
|
"normal_lookup_calls_existing_erofs_vget": namei.count("erofs_vget(") == 1,
|
|
"erofs_vget_checks_hash_before_inode_read": source_anchors[
|
|
"freebsd_inode_decode_after_hash"
|
|
],
|
|
}
|
|
lock_ledger = {
|
|
"candidate_added_lock_or_vget_calls": lock_calls,
|
|
"candidate_reads_immutable_vtype": "VTOE(vp)->vtype" in lookup_candidate,
|
|
"dotdot_bypasses_validator":
|
|
"(cnp->cn_flags & ISDOTDOT) == 0" in lookup_candidate,
|
|
"mismatch_drops_locked_child": "vput(vp);" in lookup_candidate,
|
|
"validator_before_vpp_publication": lookup_candidate.index(
|
|
"erofs_dirent_type_matches"
|
|
)
|
|
< lookup_candidate.index("*ap->a_vpp = vp"),
|
|
"validator_before_namecache_publication": candidate_namei.index(
|
|
"erofs_dirent_type_matches"
|
|
)
|
|
< candidate_namei.index("cache_enter(dvp, vp, cnp)"),
|
|
"readdir_has_no_vget": all(
|
|
token not in directory_source
|
|
for token in ("erofs_vget(", "vn_vget_ino(", "vfs_hash_get(")
|
|
),
|
|
"parent_child_lock_sequence_unchanged": namei.count("erofs_vget(")
|
|
== candidate_namei.count("erofs_vget(")
|
|
and namei.count("error = vn_vget_ino(")
|
|
== candidate_namei.count("error = vn_vget_ino("),
|
|
"freebsd_contract_sha256": freebsd_hashes,
|
|
"cache_hit_proof": cache_proof,
|
|
}
|
|
write_json(OUTPUT / "lock-ledger.json", lock_ledger)
|
|
|
|
failures = []
|
|
failures.extend(
|
|
record["id"] for record in case_records if not record["oracle_pass"]
|
|
)
|
|
failures.extend(
|
|
record["id"] for record in prototype_records if not record["pass"]
|
|
)
|
|
for key, value in lock_ledger.items():
|
|
if key == "freebsd_contract_sha256":
|
|
continue
|
|
if key == "candidate_added_lock_or_vget_calls":
|
|
if value:
|
|
failures.append(key)
|
|
elif key == "cache_hit_proof":
|
|
if not all(value.values()):
|
|
failures.append(key)
|
|
elif value is not True:
|
|
failures.append(key)
|
|
status = "GO" if not failures else "STOP"
|
|
|
|
oracle = {
|
|
"cache_hit": cache_proof,
|
|
"candidate_patch_sha256": sha256_bytes(candidate_patch.encode("utf-8")),
|
|
"candidate_prototype_sha256": sha256_path(prototype_path),
|
|
"case_records": case_records,
|
|
"known_match_records": seed_records,
|
|
"prototype_records": prototype_records,
|
|
"source_anchors": source_anchors,
|
|
"status": status,
|
|
}
|
|
write_json(OUTPUT / "oracle.json", oracle)
|
|
write_json(OUTPUT / "source-sha256.json", source_hashes)
|
|
write_json(
|
|
OUTPUT / "freebsd-contract.json",
|
|
{"head": freebsd_head, "sha256": freebsd_hashes},
|
|
)
|
|
|
|
fixture_index = {
|
|
"fixture_count": len(list(fixtures.glob("*.erofs"))),
|
|
"fixture_set_sha256": fixture_digest.hexdigest(),
|
|
"fixtures": [
|
|
{
|
|
"name": path.name,
|
|
"sha256": sha256_path(path),
|
|
"size": path.stat().st_size,
|
|
}
|
|
for path in sorted(fixtures.glob("*.erofs"))
|
|
],
|
|
"seed_repeat_byte_identical": True,
|
|
}
|
|
write_json(OUTPUT / "fixture-index.json", fixture_index)
|
|
|
|
result = {
|
|
"b11": "AUTHORIZED" if status == "GO" else "STOP-NO-SOURCE",
|
|
"cache_hit_sequence_count": 1,
|
|
"candidate": "P15-081",
|
|
"failures": failures,
|
|
"fixture_count": fixture_index["fixture_count"],
|
|
"fixture_set_sha256": fixture_index["fixture_set_sha256"],
|
|
"forward_compatible_case_count": len(SPEC["tolerated"]),
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gate": "G11",
|
|
"known_match_count": len(seed_records),
|
|
"known_mismatch_count": len(SPEC["known_mismatches"]),
|
|
"normal_entry_count": len(case_records),
|
|
"prototype_case_count": len(prototype_records),
|
|
"qemu": "NOT_RUN",
|
|
"qemu_reason": "pre-source gate uses checksum-valid disk fixtures, independent host parsing, and a non-KLD prototype",
|
|
"requested_base": REQUESTED_BASE,
|
|
"resolved_base": resolved,
|
|
"schema": 1,
|
|
"status": status,
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
|
|
hash_lines = []
|
|
for path in sorted(OUTPUT.rglob("*")):
|
|
if path.is_file() and path.name != "SHA256SUMS":
|
|
hash_lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
|
|
(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii")
|
|
print(json.dumps(result, sort_keys=True))
|
|
if status != "GO":
|
|
raise SystemExit(1)
|
|
PY
|