882 lines
33 KiB
Bash
Executable File
882 lines
33 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
|
dut=$root/repo-pre-15
|
|
input=$gate_dir/P15-092-input.json
|
|
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
|
|
utils_src=${EROFS_UTILS_SRC:-/work/build/erofs-utils-main}
|
|
base=
|
|
output=
|
|
qemu_base=
|
|
qemu_key=
|
|
host_only=0
|
|
|
|
while test "$#" -gt 0; do
|
|
case "$1" in
|
|
--base)
|
|
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
|
base=$2
|
|
shift 2
|
|
;;
|
|
--output)
|
|
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
|
output=$2
|
|
shift 2
|
|
;;
|
|
--qemu-base)
|
|
test "$#" -ge 2 || { printf '%s\n' '--qemu-base requires an image' >&2; exit 2; }
|
|
qemu_base=$2
|
|
shift 2
|
|
;;
|
|
--qemu-key)
|
|
test "$#" -ge 2 || { printf '%s\n' '--qemu-key requires a key' >&2; exit 2; }
|
|
qemu_key=$2
|
|
shift 2
|
|
;;
|
|
--host-only)
|
|
host_only=1
|
|
shift
|
|
;;
|
|
*)
|
|
printf 'unknown argument: %s\n' "$1" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
|
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
|
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
|
|
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
|
|
test -d "$utils_src/lib" || { printf 'missing erofs-utils source: %s\n' "$utils_src" >&2; exit 2; }
|
|
if test "$host_only" -eq 0; then
|
|
test -n "$qemu_base" || { printf '%s\n' '--qemu-base is required' >&2; exit 2; }
|
|
test -n "$qemu_key" || { printf '%s\n' '--qemu-key is required' >&2; exit 2; }
|
|
test -f "$qemu_base" || { printf 'missing QEMU base: %s\n' "$qemu_base" >&2; exit 2; }
|
|
test -f "$qemu_key" || { printf 'missing QEMU key: %s\n' "$qemu_key" >&2; exit 2; }
|
|
fi
|
|
for tool in dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum timeout; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'missing required tool: %s\n' "$tool" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
if test "$host_only" -eq 0; then
|
|
for tool in clang qemu-img qemu-system-x86_64 scp ssh tar; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'missing required QEMU tool: %s\n' "$tool" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
fi
|
|
case "$output" in
|
|
/*) ;;
|
|
*) output=$PWD/$output ;;
|
|
esac
|
|
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
|
mkdir -p "$output/host"
|
|
|
|
python3 -B - "$root" "$input" "$base" "$output/host" "$freebsd_src" \
|
|
"$utils_src" "${qemu_base:-/nonexistent}" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(sys.argv[1])
|
|
INPUT = Path(sys.argv[2])
|
|
REQUESTED_BASE = sys.argv[3]
|
|
OUTPUT = Path(sys.argv[4])
|
|
FREEBSD_SRC = Path(sys.argv[5])
|
|
UTILS_SRC = Path(sys.argv[6])
|
|
QEMU_BASE = Path(sys.argv[7]).resolve()
|
|
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
|
|
|
|
SUPER = 1024
|
|
MAGIC = 0xE0F5E1E2
|
|
CRC32C_POLY = 0x82F63B78
|
|
S_IFMT = 0o170000
|
|
S_IFDIR = 0o040000
|
|
NLINK_ONE_BIT = 4
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_path(path: Path) -> str:
|
|
return sha256_bytes(path.read_bytes())
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
|
|
|
|
|
def run(argv: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
argv,
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
env=env,
|
|
)
|
|
|
|
|
|
def git(root: Path, *args: str) -> str:
|
|
return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip()
|
|
|
|
|
|
def source_at(commit: str, path: str) -> str:
|
|
completed = run(["git", "-C", str(ROOT), "show", f"{commit}:{path}"])
|
|
if completed.returncode != 0:
|
|
raise SystemExit(f"cannot read {path} at {commit}: {completed.stdout}")
|
|
return completed.stdout
|
|
|
|
|
|
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
|
value = seed
|
|
for byte in data:
|
|
value ^= byte
|
|
for _ in range(8):
|
|
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
|
return value & 0xFFFFFFFF
|
|
|
|
|
|
class Image:
|
|
def __init__(self, data: bytes | bytearray):
|
|
self.data = bytearray(data)
|
|
if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
|
|
raise ValueError("invalid EROFS image")
|
|
self.block_bits = self.data[SUPER + 12]
|
|
self.block_size = 1 << self.block_bits
|
|
self.meta_blkaddr = self.u32(SUPER + 40)
|
|
self.root_nid = self.u16(SUPER + 14)
|
|
if self.root_nid == 0:
|
|
self.root_nid = self.u64(SUPER + 112)
|
|
if self.checksum_end > len(self.data):
|
|
raise ValueError("truncated checksummed range")
|
|
|
|
@property
|
|
def checksum_end(self) -> int:
|
|
span = self.block_size - SUPER if self.block_size > SUPER else self.block_size
|
|
return SUPER + span
|
|
|
|
def clone(self) -> "Image":
|
|
return Image(self.data)
|
|
|
|
def u16(self, offset: int) -> int:
|
|
return struct.unpack_from("<H", self.data, offset)[0]
|
|
|
|
def u32(self, offset: int) -> int:
|
|
return struct.unpack_from("<I", self.data, offset)[0]
|
|
|
|
def u64(self, offset: int) -> int:
|
|
return struct.unpack_from("<Q", self.data, offset)[0]
|
|
|
|
def put_u16(self, offset: int, value: int) -> None:
|
|
struct.pack_into("<H", self.data, offset, value)
|
|
|
|
def put_u32(self, offset: int, value: int) -> None:
|
|
struct.pack_into("<I", self.data, offset, value)
|
|
|
|
def put_u64(self, offset: int, value: int) -> None:
|
|
struct.pack_into("<Q", self.data, offset, value)
|
|
|
|
def checksum_valid(self) -> bool:
|
|
expected = self.u32(SUPER + 4)
|
|
canonical = bytearray(self.data[SUPER : self.checksum_end])
|
|
struct.pack_into("<I", canonical, 4, 0)
|
|
return expected == crc32c(canonical)
|
|
|
|
def update_checksum(self) -> None:
|
|
self.put_u32(SUPER + 4, 0)
|
|
self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.checksum_end]))
|
|
if not self.checksum_valid():
|
|
raise AssertionError("checksum update failed")
|
|
|
|
def inode(self, nid: int) -> dict[str, int]:
|
|
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
|
ifmt = self.u16(offset)
|
|
version = ifmt & 1
|
|
inode_size = 32 if version == 0 else 64
|
|
if offset + inode_size > len(self.data):
|
|
raise ValueError(f"inode {nid} is outside the image")
|
|
xattr_count = self.u16(offset + 2)
|
|
xattr_size = 0 if xattr_count == 0 else 12 + (xattr_count - 1) * 4
|
|
mode = self.u16(offset + 4)
|
|
raw_nlink = self.u16(offset + 6) if version == 0 else self.u32(offset + 44)
|
|
nlink_one = version == 0 and mode & S_IFMT != S_IFDIR and bool(
|
|
ifmt & (1 << NLINK_ONE_BIT)
|
|
)
|
|
size = self.u32(offset + 8) if version == 0 else self.u64(offset + 8)
|
|
return {
|
|
"nid": nid,
|
|
"offset": offset,
|
|
"ifmt": ifmt,
|
|
"version": version,
|
|
"inode_size": inode_size,
|
|
"xattr_size": xattr_size,
|
|
"layout": (ifmt >> 1) & 7,
|
|
"mode": mode,
|
|
"raw_nlink": raw_nlink,
|
|
"nlink": 1 if nlink_one else raw_nlink,
|
|
"nlink_one": int(nlink_one),
|
|
"size": size,
|
|
"startblk": self.u32(offset + 16),
|
|
}
|
|
|
|
def set_nlink(self, nid: int, value: int) -> dict[str, int]:
|
|
inode = self.inode(nid)
|
|
if inode["version"] == 0:
|
|
if inode["mode"] & S_IFMT != S_IFDIR:
|
|
self.put_u16(inode["offset"], inode["ifmt"] & ~(1 << NLINK_ONE_BIT))
|
|
self.put_u16(inode["offset"] + 6, value)
|
|
else:
|
|
self.put_u32(inode["offset"] + 44, value)
|
|
return self.inode(nid)
|
|
|
|
def data_offset(self, inode: dict[str, int]) -> int:
|
|
if inode["layout"] == 2:
|
|
return inode["offset"] + inode["inode_size"] + inode["xattr_size"]
|
|
if inode["layout"] == 0:
|
|
return inode["startblk"] << self.block_bits
|
|
raise ValueError(f"unsupported directory layout {inode['layout']}")
|
|
|
|
def directory(self, nid: int) -> dict[str, dict[str, int]]:
|
|
inode = self.inode(nid)
|
|
if inode["mode"] & S_IFMT != S_IFDIR:
|
|
raise ValueError(f"inode {nid} is not a directory")
|
|
data_offset = self.data_offset(inode)
|
|
size = inode["size"]
|
|
first_nameoff = self.u16(data_offset + 8)
|
|
if first_nameoff == 0 or first_nameoff % 12 != 0 or first_nameoff >= size:
|
|
raise ValueError("invalid first name offset")
|
|
count = first_nameoff // 12
|
|
records: dict[str, dict[str, int]] = {}
|
|
for index in range(count):
|
|
entry = data_offset + index * 12
|
|
nameoff = self.u16(entry + 8)
|
|
endoff = self.u16(entry + 20) if index + 1 < count else size
|
|
raw = bytes(self.data[data_offset + nameoff : data_offset + endoff])
|
|
name = raw.split(b"\0", 1)[0].decode("ascii")
|
|
if not name or name in records:
|
|
raise ValueError(f"bad directory name {name!r}")
|
|
records[name] = {
|
|
"entry_offset": entry,
|
|
"nid": self.u64(entry),
|
|
"file_type": self.data[entry + 10],
|
|
}
|
|
return records
|
|
|
|
|
|
def make_source(path: Path, extended_uid: bool) -> None:
|
|
previous_umask = os.umask(0o022)
|
|
try:
|
|
path.mkdir(mode=0o755)
|
|
(path / "compact").write_bytes(b"compact zero-nlink target\n")
|
|
(path / "extended").write_bytes(b"extended zero-nlink target\n")
|
|
if extended_uid:
|
|
os.chown(path / "extended", 70000, 0)
|
|
(path / "hard-a").write_bytes(b"hardlink control\n")
|
|
os.link(path / "hard-a", path / "hard-b")
|
|
(path / "orphan").write_bytes(b"orphan raw-vget control\n")
|
|
(path / "subdir").mkdir(mode=0o755)
|
|
(path / "subdir" / "child").write_bytes(b"directory child\n")
|
|
for regular in ("compact", "extended", "hard-a", "hard-b", "orphan"):
|
|
os.chmod(path / regular, 0o644)
|
|
os.chmod(path / "subdir" / "child", 0o644)
|
|
finally:
|
|
os.umask(previous_umask)
|
|
|
|
|
|
def build_seed(work: Path, output: Path, inode_mode: str) -> tuple[bytes, str, list[str]]:
|
|
work.mkdir()
|
|
source = work / "source"
|
|
make_source(source, inode_mode == "extended")
|
|
command = [
|
|
"mkfs.erofs",
|
|
"-d0",
|
|
"-T0",
|
|
"--all-time",
|
|
"--ignore-mtime",
|
|
"--workers=1",
|
|
"-x-1",
|
|
"-E",
|
|
"noinline_data",
|
|
"-E",
|
|
f"force-inode-{inode_mode}",
|
|
"-U",
|
|
SPEC["uuid"],
|
|
str(output),
|
|
str(source),
|
|
]
|
|
env = dict(os.environ)
|
|
env.pop("SOURCE_DATE_EPOCH", None)
|
|
completed = run(command, env=env)
|
|
if completed.returncode != 0:
|
|
raise SystemExit(f"mkfs.erofs failed: {completed.stdout}")
|
|
canonical = command[:-2] + [f"SEED-{inode_mode}.erofs", "SOURCE"]
|
|
return output.read_bytes(), completed.stdout, canonical
|
|
|
|
|
|
resolved = git(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}")
|
|
if resolved != SPEC["required_base"]:
|
|
raise SystemExit(f"wrong BASE: expected {SPEC['required_base']}, got {resolved}")
|
|
if git(FREEBSD_SRC, "rev-parse", "HEAD") != SPEC["freebsd_head"]:
|
|
raise SystemExit("FreeBSD source HEAD changed")
|
|
if git(UTILS_SRC, "rev-parse", "HEAD") != SPEC["utils_head"]:
|
|
raise SystemExit("erofs-utils source HEAD changed")
|
|
if QEMU_BASE == Path(SPEC["protected"]["base_image"]):
|
|
raise SystemExit("refusing the protected base bp")
|
|
if SPEC["protected"]["port"] >= 32768:
|
|
raise SystemExit("protected port unexpectedly overlaps ephemeral ports")
|
|
|
|
sources: dict[str, str] = {}
|
|
source_hashes: dict[str, str] = {}
|
|
for path, expected in SPEC["source_sha256"].items():
|
|
content = source_at(resolved, path)
|
|
actual = sha256_bytes(content.encode("utf-8"))
|
|
if actual != expected:
|
|
raise SystemExit(f"source hash changed for {path}: {actual}")
|
|
sources[path] = content
|
|
source_hashes[path] = actual
|
|
for path, expected in SPEC["freebsd_sha256"].items():
|
|
actual = sha256_path(FREEBSD_SRC / path)
|
|
if actual != expected:
|
|
raise SystemExit(f"FreeBSD hash changed for {path}: {actual}")
|
|
for path, expected in SPEC["utils_sha256"].items():
|
|
actual = sha256_path(UTILS_SRC / path)
|
|
if actual != expected:
|
|
raise SystemExit(f"erofs-utils hash changed for {path}: {actual}")
|
|
|
|
mkfs_version = run(["mkfs.erofs", "-V"]).stdout.splitlines()[0]
|
|
if mkfs_version != SPEC["mkfs_version"]:
|
|
raise SystemExit(f"mkfs version changed: {mkfs_version!r}")
|
|
|
|
namei = sources["repo-pre-15/src/namei.c"]
|
|
inode_source = sources["repo-pre-15/src/inode.c"]
|
|
vnops = sources["repo-pre-15/src/erofs_vnops.c"]
|
|
super_source = sources["repo-pre-15/src/super.c"]
|
|
linux_inode = sources["src-linux/inode.c"]
|
|
linux_super = sources["src-linux/super.c"]
|
|
freebsd_vm = (FREEBSD_SRC / "sys/vm/vm_mmap.c").read_text(encoding="utf-8")
|
|
|
|
source_anchors = {
|
|
"freebsd_disk_compact_nlink": "vi->nlink = le16toh(dic->i_nb.nlink);" in inode_source,
|
|
"freebsd_disk_extended_nlink": "vi->nlink = le32toh(die->i_nlink);" in inode_source,
|
|
"freebsd_inode_decode_has_no_zero_reject": "vi->nlink == 0" not in inode_source,
|
|
"freebsd_lookup_has_namespace_edge": "error = erofs_namei(sbi, dir, &qname, &nid, &dtype);" in namei,
|
|
"freebsd_lookup_calls_raw_vget": "error = erofs_vget(dvp->v_mount, nid, cnp->cn_lkflags, &vp);" in namei,
|
|
"freebsd_lookup_publishes_namecache": "cache_enter(dvp, vp, cnp);" in namei,
|
|
"freebsd_raw_vget_registered": ".vfs_vget = erofs_vget," in super_source,
|
|
"freebsd_root_uses_raw_vget": "error = erofs_vget(mp, MTOE(mp)->root_nid, flags, vpp);" in super_source,
|
|
"freebsd_fhtovp_filters_zero": "vi->mode == 0 || vi->nlink == 0 || vi->nid != nid" in super_source,
|
|
"freebsd_hash_insert_before_decode": vnops.index("error = vfs_hash_insert(") < vnops.index("error = erofs_read_inode("),
|
|
"freebsd_constructed_before_return": vnops.index("vn_set_state(vp, VSTATE_CONSTRUCTED);") < vnops.index("*vpp = vp;"),
|
|
"freebsd_vget_has_no_hit_indicator": "bool shared;" in vnops and "bool created" not in vnops,
|
|
"freebsd_vput_does_not_imply_vgone": "vgone(vp);\n\t\tvput(vp);" in vnops,
|
|
"freebsd_vfs_allows_unlinked_vnode": "if (va.va_nlink == 0)" in freebsd_vm,
|
|
"linux_compact_sets_disk_nlink": "set_nlink(inode, le16_to_cpu(dic->i_nb.nlink));" in linux_inode,
|
|
"linux_extended_sets_disk_nlink": "set_nlink(inode, le32_to_cpu(die->i_nlink));" in linux_inode,
|
|
"linux_inode_has_no_zero_reject": "i_nlink == 0" not in linux_inode,
|
|
"linux_super_has_no_zero_reject": "i_nlink == 0" not in linux_super,
|
|
}
|
|
if not all(source_anchors.values()):
|
|
raise SystemExit(f"source anchors changed: {source_anchors}")
|
|
|
|
fixtures = OUTPUT / "fixtures"
|
|
fixtures.mkdir()
|
|
with tempfile.TemporaryDirectory(prefix="p15-092-gate-") as temporary:
|
|
temp = Path(temporary)
|
|
first_bytes, first_stdout, compact_command = build_seed(
|
|
temp / "compact-first", temp / "compact-first.erofs", "compact"
|
|
)
|
|
second_bytes, second_stdout, compact_repeat_command = build_seed(
|
|
temp / "compact-second", temp / "compact-second.erofs", "compact"
|
|
)
|
|
extended_bytes, extended_stdout, extended_command = build_seed(
|
|
temp / "extended-first", temp / "extended-first.erofs", "extended"
|
|
)
|
|
extended_repeat_bytes, extended_repeat_stdout, extended_repeat_command = build_seed(
|
|
temp / "extended-second", temp / "extended-second.erofs", "extended"
|
|
)
|
|
write_json(
|
|
OUTPUT / "generator-attempts.json",
|
|
{
|
|
"commands": [compact_command, extended_command],
|
|
"compact_source": [
|
|
{
|
|
"gid": path.lstat().st_gid,
|
|
"mode": stat.S_IMODE(path.lstat().st_mode),
|
|
"path": str(path.relative_to(temp / "compact-first/source")),
|
|
"uid": path.lstat().st_uid,
|
|
}
|
|
for path in sorted((temp / "compact-first/source").rglob("*"))
|
|
],
|
|
},
|
|
)
|
|
if (
|
|
first_bytes != second_bytes
|
|
or first_stdout != second_stdout
|
|
or compact_command != compact_repeat_command
|
|
or extended_bytes != extended_repeat_bytes
|
|
or extended_stdout != extended_repeat_stdout
|
|
or extended_command != extended_repeat_command
|
|
):
|
|
raise SystemExit("seed generation is not byte reproducible")
|
|
seed_path = fixtures / "seed.erofs"
|
|
seed_path.write_bytes(first_bytes)
|
|
extended_seed_path = fixtures / "seed-extended.erofs"
|
|
extended_seed_path.write_bytes(extended_bytes)
|
|
seed = Image(first_bytes)
|
|
extended_seed = Image(extended_bytes)
|
|
if not seed.checksum_valid() or not extended_seed.checksum_valid():
|
|
raise SystemExit("seed checksum is invalid")
|
|
entries = seed.directory(seed.root_nid)
|
|
required = {".", "..", "compact", "extended", "hard-a", "hard-b", "orphan", "subdir"}
|
|
if not required.issubset(entries):
|
|
raise SystemExit(f"seed entries missing: {sorted(required - entries.keys())}")
|
|
if entries["hard-a"]["nid"] != entries["hard-b"]["nid"]:
|
|
raise SystemExit("hardlink names do not share one NID")
|
|
|
|
extended_entries = extended_seed.directory(extended_seed.root_nid)
|
|
seed_inodes = {name: seed.inode(record["nid"]) for name, record in entries.items()}
|
|
extended_inodes = {
|
|
name: extended_seed.inode(record["nid"])
|
|
for name, record in extended_entries.items()
|
|
}
|
|
if seed_inodes["compact"]["version"] != 0:
|
|
raise SystemExit("compact control is not compact")
|
|
if extended_inodes["extended"]["version"] != 1:
|
|
raise SystemExit("extended control is not extended")
|
|
if seed_inodes["hard-a"]["nlink"] != 2:
|
|
raise SystemExit("hardlink control does not have nlink 2")
|
|
|
|
mutations = []
|
|
|
|
def emit_zero(case_id: str, name: str, source: Image = seed) -> None:
|
|
image = source.clone()
|
|
nid = image.directory(image.root_nid)[name]["nid"]
|
|
before = image.inode(nid)
|
|
after = image.set_nlink(nid, 0)
|
|
image.update_checksum()
|
|
path = fixtures / f"{case_id}.erofs"
|
|
path.write_bytes(image.data)
|
|
mutations.append({
|
|
"case": case_id,
|
|
"class": "namespace-reachable-zero",
|
|
"name": name,
|
|
"nid": nid,
|
|
"before": before,
|
|
"after": after,
|
|
"fixture": path.name,
|
|
})
|
|
|
|
emit_zero("reachable-compact-zero", "compact")
|
|
emit_zero("reachable-extended-zero", "extended", extended_seed)
|
|
emit_zero("reachable-directory-zero", "subdir")
|
|
|
|
root_zero = seed.clone()
|
|
root_before = root_zero.inode(root_zero.root_nid)
|
|
root_after = root_zero.set_nlink(root_zero.root_nid, 0)
|
|
root_zero.update_checksum()
|
|
root_path = fixtures / "root-zero.erofs"
|
|
root_path.write_bytes(root_zero.data)
|
|
mutations.append({
|
|
"case": "root-zero",
|
|
"class": "raw-root-zero",
|
|
"name": "/",
|
|
"nid": root_zero.root_nid,
|
|
"before": root_before,
|
|
"after": root_after,
|
|
"fixture": root_path.name,
|
|
})
|
|
|
|
orphan = seed.clone()
|
|
orphan_entries = orphan.directory(orphan.root_nid)
|
|
orphan_nid = orphan_entries["orphan"]["nid"]
|
|
hard_nid = orphan_entries["hard-a"]["nid"]
|
|
orphan.put_u64(orphan_entries["orphan"]["entry_offset"], hard_nid)
|
|
orphan.set_nlink(orphan_nid, 0)
|
|
orphan.set_nlink(hard_nid, 3)
|
|
orphan.update_checksum()
|
|
orphan_path = fixtures / "orphan-zero.erofs"
|
|
orphan_path.write_bytes(orphan.data)
|
|
orphan_names = orphan.directory(orphan.root_nid)
|
|
if any(record["nid"] == orphan_nid for record in orphan_names.values()):
|
|
raise SystemExit("orphan fixture still has a namespace edge")
|
|
mutations.append({
|
|
"case": "orphan-zero",
|
|
"class": "unreachable-zero",
|
|
"name": None,
|
|
"nid": orphan_nid,
|
|
"before": seed.inode(orphan_nid),
|
|
"after": orphan.inode(orphan_nid),
|
|
"replacement_nid": hard_nid,
|
|
"replacement_nlink": orphan.inode(hard_nid)["nlink"],
|
|
"fixture": orphan_path.name,
|
|
})
|
|
|
|
records = []
|
|
fixture_digest = hashlib.sha256()
|
|
for path in sorted(fixtures.glob("*.erofs")):
|
|
image = Image(path.read_bytes())
|
|
if not image.checksum_valid():
|
|
raise SystemExit(f"fixture checksum invalid: {path.name}")
|
|
fsck = run(["fsck.erofs", "-d0", str(path)])
|
|
dump_root = run(["dump.erofs", "--path=/", str(path)])
|
|
if dump_root.returncode != 0:
|
|
raise SystemExit(f"dump root failed: {path.name}")
|
|
normal = {}
|
|
for name in ("compact", "extended", "hard-a", "hard-b", "orphan", "subdir"):
|
|
completed = run(["dump.erofs", f"--path=/{name}", str(path)])
|
|
normal[name] = {
|
|
"exit": completed.returncode,
|
|
"nid": int(match.group(1)) if (match := re.search(r"^NID: (\d+)\b", completed.stdout, re.MULTILINE)) else None,
|
|
}
|
|
record = {
|
|
"fixture": path.name,
|
|
"sha256": sha256_path(path),
|
|
"size": path.stat().st_size,
|
|
"checksum_valid": True,
|
|
"fsck_exit": fsck.returncode,
|
|
"fsck_error_marker": "<E>" in fsck.stdout,
|
|
"normal_namespace": normal,
|
|
}
|
|
records.append(record)
|
|
fixture_digest.update(path.name.encode("ascii"))
|
|
fixture_digest.update(b"\0")
|
|
fixture_digest.update(path.read_bytes())
|
|
|
|
write_json(
|
|
OUTPUT / "generator.json",
|
|
{
|
|
"commands": [compact_command, extended_command],
|
|
"mkfs_version": mkfs_version,
|
|
"repeat_byte_identical": True,
|
|
"seed_sha256": {
|
|
"compact": sha256_bytes(first_bytes),
|
|
"extended": sha256_bytes(extended_bytes),
|
|
},
|
|
},
|
|
)
|
|
(OUTPUT / "mkfs.stdout").write_text(
|
|
first_stdout + extended_stdout, encoding="utf-8"
|
|
)
|
|
write_json(OUTPUT / "disk-records.json", {"mutations": mutations, "records": records})
|
|
write_json(
|
|
OUTPUT / "fixture-index.json",
|
|
{
|
|
"fixture_count": len(records),
|
|
"fixture_set_sha256": fixture_digest.hexdigest(),
|
|
"fixtures": records,
|
|
},
|
|
)
|
|
|
|
fsck_reachable_zero_rejects = []
|
|
for record in records:
|
|
if record["fixture"].startswith("reachable-"):
|
|
fsck_reachable_zero_rejects.append(
|
|
record["fsck_exit"] != 0 or record["fsck_error_marker"]
|
|
)
|
|
|
|
boundary = {
|
|
"b15_write_set": SPEC["write_set"],
|
|
"vnode_constructor_owner": "repo-pre-15/src/erofs_vnops.c",
|
|
"constructor_inside_b15_write_set": "repo-pre-15/src/erofs_vnops.c" in SPEC["write_set"],
|
|
"post_vget_check_is_before_constructed_publication": False,
|
|
"vput_only_performs_vgone_cleanup": False,
|
|
"post_vget_vgone_preserves_cached_raw_vget": False,
|
|
"pre_vget_full_decode_avoids_duplicate_inode_decode": False,
|
|
"mount_wide_scan_required": False,
|
|
"namespace_and_raw_entrypoints_distinct": True,
|
|
}
|
|
format_semantics = {
|
|
"mkfs_emits_nonzero_namespace_links": all(
|
|
seed_inodes[name]["nlink"] > 0
|
|
for name in ("compact", "extended", "hard-a", "hard-b", "orphan", "subdir", ".")
|
|
),
|
|
"fsck_rejects_all_reachable_zero": all(fsck_reachable_zero_rejects),
|
|
"linux_rejects_zero_during_inode_decode": False,
|
|
"linux_copies_disk_nlink_into_inode": True,
|
|
"freebsd_vfs_permits_live_unlinked_vnode": True,
|
|
"freebsd_fhtovp_treats_zero_as_stale": True,
|
|
}
|
|
go_requirements = {
|
|
"format_or_cross_kernel_rule_requires_rejection": (
|
|
format_semantics["fsck_rejects_all_reachable_zero"]
|
|
and format_semantics["linux_rejects_zero_during_inode_decode"]
|
|
),
|
|
"reject_before_vnode_publication_within_write_set": (
|
|
boundary["constructor_inside_b15_write_set"]
|
|
and boundary["post_vget_check_is_before_constructed_publication"]
|
|
),
|
|
"cleanup_preserves_raw_vget": boundary["post_vget_vgone_preserves_cached_raw_vget"],
|
|
"no_duplicate_full_inode_decode": boundary["pre_vget_full_decode_avoids_duplicate_inode_decode"],
|
|
"orphan_needs_no_mount_scan": not boundary["mount_wide_scan_required"],
|
|
"namespace_raw_split_proven": boundary["namespace_and_raw_entrypoints_distinct"],
|
|
}
|
|
status = "GO" if all(go_requirements.values()) else "STOP"
|
|
reasons = [key for key, value in go_requirements.items() if not value]
|
|
write_json(OUTPUT / "source-anchors.json", source_anchors)
|
|
write_json(OUTPUT / "source-sha256.json", source_hashes)
|
|
write_json(OUTPUT / "boundary.json", boundary)
|
|
write_json(OUTPUT / "format-semantics.json", format_semantics)
|
|
write_json(
|
|
OUTPUT / "host-result.json",
|
|
{
|
|
"candidate": SPEC["candidate"],
|
|
"gate": SPEC["gate"],
|
|
"fixture_count": len(records),
|
|
"fixture_set_sha256": fixture_digest.hexdigest(),
|
|
"go_requirements": go_requirements,
|
|
"reasons": reasons,
|
|
"requested_base": REQUESTED_BASE,
|
|
"resolved_base": resolved,
|
|
"schema": 1,
|
|
"status": status,
|
|
},
|
|
)
|
|
PY
|
|
|
|
if test "$host_only" -eq 1; then
|
|
cat "$output/host/host-result.json"
|
|
exit 22
|
|
fi
|
|
|
|
runtime_tmp=$(mktemp -d "${TMPDIR:-/tmp}/p15-092-runtime.XXXXXX")
|
|
runtime_dut=$runtime_tmp/repo-pre-15
|
|
runtime_case=$runtime_dut/tests/pre15/cases/P15-092-gate-runtime.sh
|
|
runtime_cleanup()
|
|
{
|
|
rm -rf "$runtime_tmp"
|
|
}
|
|
trap runtime_cleanup EXIT HUP INT TERM
|
|
mkdir -p "$runtime_dut/tests/pre15/cases" "$runtime_dut/tests/pre15/fixtures"
|
|
git -C "$root" archive --format=tar --output="$runtime_tmp/frozen-src.tar" \
|
|
"$base" repo-pre-15/src
|
|
tar -C "$runtime_tmp" -xf "$runtime_tmp/frozen-src.tar"
|
|
ln -s "$dut/tests/pre15/fixtures/B14-build-kld.sh" \
|
|
"$runtime_dut/tests/pre15/fixtures/B14-build-kld.sh"
|
|
|
|
python3 -B - "$runtime_case" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
Path(sys.argv[1]).write_text(r'''#!/bin/sh
|
|
set -eu
|
|
|
|
: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}"
|
|
: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}"
|
|
: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}"
|
|
: "${P15_092_GATE_DUT:?P15_092_GATE_DUT is required}"
|
|
: "${P15_092_GATE_HOST:?P15_092_GATE_HOST is required}"
|
|
. "$PRE15_LIB_DIR/runner.sh"
|
|
|
|
artifacts=$PRE15_RUN_DIR/artifacts
|
|
fixtures=$P15_092_GATE_HOST/fixtures
|
|
builder=$P15_092_GATE_DUT/tests/pre15/fixtures/B14-build-kld.sh
|
|
module=$PRE15_CASE_TMP/P15-092-baseline-erofs.ko
|
|
archive=$PRE15_CASE_TMP/P15-092-fixtures.tar.gz
|
|
mkdir -p "$artifacts"
|
|
|
|
pre15_record_fixture p15-092-host-result "$P15_092_GATE_HOST/host-result.json"
|
|
pre15_record_fixture p15-092-disk-records "$P15_092_GATE_HOST/disk-records.json"
|
|
pre15_record_fixture p15-092-boundary "$P15_092_GATE_HOST/boundary.json"
|
|
pre15_record_fixture p15-092-source-namei "$P15_092_GATE_DUT/src/namei.c"
|
|
pre15_record_fixture p15-092-source-inode "$P15_092_GATE_DUT/src/inode.c"
|
|
pre15_record_fixture p15-092-source-vnops "$P15_092_GATE_DUT/src/erofs_vnops.c"
|
|
|
|
if ! /bin/sh "$builder" "$P15_092_GATE_DUT" "$PRE15_FREEBSD_SRC" "$module" \
|
|
"$PRE15_CASE_TMP/kld-work" >"$artifacts/kld-build.stdout" \
|
|
2>"$artifacts/kld-build.stderr"; then
|
|
pre15_dut_fail 'baseline cross-target KLD build failed'
|
|
fi
|
|
pre15_record_module "$module"
|
|
tar -C "$fixtures" -czf "$archive" .
|
|
|
|
pre15_scp()
|
|
{
|
|
timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \
|
|
-o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
-o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
|
|
-o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \
|
|
-i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \
|
|
"$1" "[email protected]:$2"
|
|
}
|
|
|
|
pre15_scp "$module" /root/P15-092-baseline-erofs.ko || \
|
|
pre15_infra_blocked 'could not transfer baseline KLD'
|
|
pre15_scp "$archive" /root/P15-092-fixtures.tar.gz || \
|
|
pre15_infra_blocked 'could not transfer zero-nlink fixtures'
|
|
pre15_guest_ssh_bounded \
|
|
'rm -rf /root/P15-092-fixtures && mkdir /root/P15-092-fixtures && tar -xzf /root/P15-092-fixtures.tar.gz -C /root/P15-092-fixtures' || \
|
|
pre15_infra_blocked 'could not prepare guest fixtures'
|
|
if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then
|
|
pre15_infra_blocked 'guest already has an EROFS module loaded'
|
|
fi
|
|
pre15_guest_ssh_bounded kldload /root/P15-092-baseline-erofs.ko || \
|
|
pre15_dut_fail 'baseline KLD failed to load'
|
|
pre15_own_guest_kld erofs 'P15-092 baseline KLD'
|
|
pre15_target_reached
|
|
|
|
attach_mount()
|
|
{
|
|
image=$1
|
|
mountpoint=$2
|
|
label=$3
|
|
md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \
|
|
-f "/root/P15-092-fixtures/$image") || \
|
|
pre15_dut_fail "$label md attach failed"
|
|
case "$md" in
|
|
md[0-9]*) ;;
|
|
*) pre15_runner_fail "unexpected md unit: $md" ;;
|
|
esac
|
|
pre15_own_guest_md "$md" "$label md"
|
|
pre15_guest_ssh_bounded mkdir -p "$mountpoint"
|
|
pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$md" "$mountpoint" || \
|
|
pre15_dut_fail "$label mount failed"
|
|
pre15_own_guest_mount "$mountpoint" "$label mount"
|
|
}
|
|
|
|
expect_nlink()
|
|
{
|
|
path=$1
|
|
expected=$2
|
|
label=$3
|
|
actual=$(pre15_guest_ssh_bounded stat -f '%l' "$path") || \
|
|
pre15_dut_fail "$label normal stat failed"
|
|
test "$actual" = "$expected" || \
|
|
pre15_dut_fail "$label nlink expected $expected got $actual"
|
|
printf '%s\t%s\t%s\n' "$label" "$path" "$actual" >>"$artifacts/normal-vnode.tsv"
|
|
}
|
|
|
|
attach_mount seed.erofs /mnt/p15-092-seed seed
|
|
expect_nlink /mnt/p15-092-seed/compact 1 seed-compact
|
|
expect_nlink /mnt/p15-092-seed/extended 1 seed-extended
|
|
expect_nlink /mnt/p15-092-seed/hard-a 2 seed-hard-a
|
|
hard_a=$(pre15_guest_ssh_bounded stat -f '%i' /mnt/p15-092-seed/hard-a)
|
|
hard_b=$(pre15_guest_ssh_bounded stat -f '%i' /mnt/p15-092-seed/hard-b)
|
|
test "$hard_a" = "$hard_b" || pre15_dut_fail 'hardlink NIDs differ'
|
|
|
|
attach_mount reachable-compact-zero.erofs /mnt/p15-092-compact compact-zero
|
|
expect_nlink /mnt/p15-092-compact/compact 0 reachable-compact-zero
|
|
attach_mount reachable-extended-zero.erofs /mnt/p15-092-extended extended-zero
|
|
expect_nlink /mnt/p15-092-extended/extended 0 reachable-extended-zero
|
|
attach_mount reachable-directory-zero.erofs /mnt/p15-092-directory directory-zero
|
|
expect_nlink /mnt/p15-092-directory/subdir 0 reachable-directory-zero
|
|
attach_mount root-zero.erofs /mnt/p15-092-root root-zero
|
|
expect_nlink /mnt/p15-092-root 0 root-zero
|
|
attach_mount orphan-zero.erofs /mnt/p15-092-orphan orphan-zero
|
|
expect_nlink /mnt/p15-092-orphan/orphan 3 orphan-replacement
|
|
pre15_guest_ssh_bounded cat /mnt/p15-092-orphan/orphan >"$artifacts/orphan-replacement.txt"
|
|
pre15_guest_ssh_bounded dmesg >"$artifacts/dmesg.txt"
|
|
printf '%s\n' 'P15-092 baseline normal vnode runtime PASS'
|
|
''', encoding="ascii")
|
|
PY
|
|
chmod 0555 "$runtime_case"
|
|
|
|
qemu_stdout=$output/qemu.stdout
|
|
qemu_stderr=$output/qemu.stderr
|
|
qemu_evidence=$output/qemu-evidence
|
|
mkdir "$qemu_evidence"
|
|
if PRE15_DUT="$runtime_dut" PRE15_ROOT="$root" \
|
|
PRE15_SCHEMA="$dut/tests/pre15/EVIDENCE-SCHEMA.json" \
|
|
PRE15_EVIDENCE_ROOT="$qemu_evidence" PRE15_FREEBSD_SRC="$freebsd_src" \
|
|
PRE15_QEMU_BASE_IMAGE="$qemu_base" PRE15_QEMU_BASE_FORMAT=qcow2 \
|
|
PRE15_QEMU_SSH_KEY="$qemu_key" PRE15_QEMU_SSH_USER=root \
|
|
PRE15_QEMU_MEMORY_MB=2048 PRE15_QEMU_CPUS=2 PRE15_QEMU_BOOT_TIMEOUT=180 \
|
|
PRE15_QEMU_TIMEOUT=900 PRE15_GUEST_COMMAND_TIMEOUT=60 \
|
|
P15_092_GATE_DUT="$runtime_dut" P15_092_GATE_HOST="$output/host" \
|
|
timeout -k 30 1000 "$dut/tests/pre15/run-qemu.sh" P15-092-gate-runtime \
|
|
>"$qemu_stdout" 2>"$qemu_stderr"; then
|
|
qemu_rc=0
|
|
else
|
|
qemu_rc=$?
|
|
fi
|
|
printf '%s\n' "$qemu_rc" >"$output/qemu.exit"
|
|
|
|
python3 -B - "$output" "$qemu_rc" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
|
|
|
|
OUTPUT = Path(sys.argv[1])
|
|
QEMU_RC = int(sys.argv[2])
|
|
|
|
|
|
def sha256_path(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def write_json(path: Path, value: object) -> None:
|
|
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
|
|
|
|
|
host = json.loads((OUTPUT / "host/host-result.json").read_text(encoding="ascii"))
|
|
stdout = (OUTPUT / "qemu.stdout").read_text(encoding="utf-8")
|
|
match = re.search(r"PRE15_RESULT status=(\S+) cleanup=(\S+) run_id=(\S+) evidence=(\S+)", stdout)
|
|
if match is None:
|
|
runtime = {
|
|
"cleanup": "UNKNOWN",
|
|
"evidence": None,
|
|
"exit": QEMU_RC,
|
|
"run_id": None,
|
|
"status": "RUNNER_FAIL",
|
|
}
|
|
else:
|
|
runtime = {
|
|
"status": match.group(1),
|
|
"cleanup": match.group(2),
|
|
"run_id": match.group(3),
|
|
"evidence": match.group(4),
|
|
"exit": QEMU_RC,
|
|
}
|
|
|
|
status = "STOP"
|
|
reasons = list(host["reasons"])
|
|
if runtime["status"] != "PASS" or runtime["cleanup"] != "PASS":
|
|
reasons.append("normal_vnode_runtime_not_pass")
|
|
result = {
|
|
"b15": "STOP-NO-SOURCE",
|
|
"candidate": "P15-092",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gate": "G11",
|
|
"host_status": host["status"],
|
|
"qemu": runtime,
|
|
"reasons": reasons,
|
|
"schema": 1,
|
|
"status": status,
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
|
|
hash_lines = []
|
|
for path in sorted(OUTPUT.rglob("*")):
|
|
if path.is_file() and path.name != "SHA256SUMS":
|
|
hash_lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
|
|
(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii")
|
|
print(json.dumps(result, sort_keys=True))
|
|
PY
|
|
|
|
cat "$output/result.json"
|
|
exit 22
|