update
This commit is contained in:
Executable
+881
@@ -0,0 +1,881 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
||||
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
||||
dut=$root/repo-pre-15
|
||||
input=$gate_dir/P15-092-input.json
|
||||
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
|
||||
utils_src=${EROFS_UTILS_SRC:-/work/build/erofs-utils-main}
|
||||
base=
|
||||
output=
|
||||
qemu_base=
|
||||
qemu_key=
|
||||
host_only=0
|
||||
|
||||
while test "$#" -gt 0; do
|
||||
case "$1" in
|
||||
--base)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
||||
base=$2
|
||||
shift 2
|
||||
;;
|
||||
--output)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
||||
output=$2
|
||||
shift 2
|
||||
;;
|
||||
--qemu-base)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--qemu-base requires an image' >&2; exit 2; }
|
||||
qemu_base=$2
|
||||
shift 2
|
||||
;;
|
||||
--qemu-key)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--qemu-key requires a key' >&2; exit 2; }
|
||||
qemu_key=$2
|
||||
shift 2
|
||||
;;
|
||||
--host-only)
|
||||
host_only=1
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
printf 'unknown argument: %s\n' "$1" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
||||
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
||||
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
|
||||
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
|
||||
test -d "$utils_src/lib" || { printf 'missing erofs-utils source: %s\n' "$utils_src" >&2; exit 2; }
|
||||
if test "$host_only" -eq 0; then
|
||||
test -n "$qemu_base" || { printf '%s\n' '--qemu-base is required' >&2; exit 2; }
|
||||
test -n "$qemu_key" || { printf '%s\n' '--qemu-key is required' >&2; exit 2; }
|
||||
test -f "$qemu_base" || { printf 'missing QEMU base: %s\n' "$qemu_base" >&2; exit 2; }
|
||||
test -f "$qemu_key" || { printf 'missing QEMU key: %s\n' "$qemu_key" >&2; exit 2; }
|
||||
fi
|
||||
for tool in dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum timeout; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf 'missing required tool: %s\n' "$tool" >&2
|
||||
exit 2
|
||||
}
|
||||
done
|
||||
if test "$host_only" -eq 0; then
|
||||
for tool in clang qemu-img qemu-system-x86_64 scp ssh tar; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf 'missing required QEMU tool: %s\n' "$tool" >&2
|
||||
exit 2
|
||||
}
|
||||
done
|
||||
fi
|
||||
case "$output" in
|
||||
/*) ;;
|
||||
*) output=$PWD/$output ;;
|
||||
esac
|
||||
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
||||
mkdir -p "$output/host"
|
||||
|
||||
python3 -B - "$root" "$input" "$base" "$output/host" "$freebsd_src" \
|
||||
"$utils_src" "${qemu_base:-/nonexistent}" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
INPUT = Path(sys.argv[2])
|
||||
REQUESTED_BASE = sys.argv[3]
|
||||
OUTPUT = Path(sys.argv[4])
|
||||
FREEBSD_SRC = Path(sys.argv[5])
|
||||
UTILS_SRC = Path(sys.argv[6])
|
||||
QEMU_BASE = Path(sys.argv[7]).resolve()
|
||||
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
|
||||
|
||||
SUPER = 1024
|
||||
MAGIC = 0xE0F5E1E2
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
S_IFMT = 0o170000
|
||||
S_IFDIR = 0o040000
|
||||
NLINK_ONE_BIT = 4
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
return sha256_bytes(path.read_bytes())
|
||||
|
||||
|
||||
def write_json(path: Path, value: Any) -> None:
|
||||
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
|
||||
|
||||
def run(argv: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
argv,
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=env,
|
||||
)
|
||||
|
||||
|
||||
def git(root: Path, *args: str) -> str:
|
||||
return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip()
|
||||
|
||||
|
||||
def source_at(commit: str, path: str) -> str:
|
||||
completed = run(["git", "-C", str(ROOT), "show", f"{commit}:{path}"])
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"cannot read {path} at {commit}: {completed.stdout}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self, data: bytes | bytearray):
|
||||
self.data = bytearray(data)
|
||||
if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
|
||||
raise ValueError("invalid EROFS image")
|
||||
self.block_bits = self.data[SUPER + 12]
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.meta_blkaddr = self.u32(SUPER + 40)
|
||||
self.root_nid = self.u16(SUPER + 14)
|
||||
if self.root_nid == 0:
|
||||
self.root_nid = self.u64(SUPER + 112)
|
||||
if self.checksum_end > len(self.data):
|
||||
raise ValueError("truncated checksummed range")
|
||||
|
||||
@property
|
||||
def checksum_end(self) -> int:
|
||||
span = self.block_size - SUPER if self.block_size > SUPER else self.block_size
|
||||
return SUPER + span
|
||||
|
||||
def clone(self) -> "Image":
|
||||
return Image(self.data)
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def u64(self, offset: int) -> int:
|
||||
return struct.unpack_from("<Q", self.data, offset)[0]
|
||||
|
||||
def put_u16(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<H", self.data, offset, value)
|
||||
|
||||
def put_u32(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<I", self.data, offset, value)
|
||||
|
||||
def put_u64(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<Q", self.data, offset, value)
|
||||
|
||||
def checksum_valid(self) -> bool:
|
||||
expected = self.u32(SUPER + 4)
|
||||
canonical = bytearray(self.data[SUPER : self.checksum_end])
|
||||
struct.pack_into("<I", canonical, 4, 0)
|
||||
return expected == crc32c(canonical)
|
||||
|
||||
def update_checksum(self) -> None:
|
||||
self.put_u32(SUPER + 4, 0)
|
||||
self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.checksum_end]))
|
||||
if not self.checksum_valid():
|
||||
raise AssertionError("checksum update failed")
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
ifmt = self.u16(offset)
|
||||
version = ifmt & 1
|
||||
inode_size = 32 if version == 0 else 64
|
||||
if offset + inode_size > len(self.data):
|
||||
raise ValueError(f"inode {nid} is outside the image")
|
||||
xattr_count = self.u16(offset + 2)
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + (xattr_count - 1) * 4
|
||||
mode = self.u16(offset + 4)
|
||||
raw_nlink = self.u16(offset + 6) if version == 0 else self.u32(offset + 44)
|
||||
nlink_one = version == 0 and mode & S_IFMT != S_IFDIR and bool(
|
||||
ifmt & (1 << NLINK_ONE_BIT)
|
||||
)
|
||||
size = self.u32(offset + 8) if version == 0 else self.u64(offset + 8)
|
||||
return {
|
||||
"nid": nid,
|
||||
"offset": offset,
|
||||
"ifmt": ifmt,
|
||||
"version": version,
|
||||
"inode_size": inode_size,
|
||||
"xattr_size": xattr_size,
|
||||
"layout": (ifmt >> 1) & 7,
|
||||
"mode": mode,
|
||||
"raw_nlink": raw_nlink,
|
||||
"nlink": 1 if nlink_one else raw_nlink,
|
||||
"nlink_one": int(nlink_one),
|
||||
"size": size,
|
||||
"startblk": self.u32(offset + 16),
|
||||
}
|
||||
|
||||
def set_nlink(self, nid: int, value: int) -> dict[str, int]:
|
||||
inode = self.inode(nid)
|
||||
if inode["version"] == 0:
|
||||
if inode["mode"] & S_IFMT != S_IFDIR:
|
||||
self.put_u16(inode["offset"], inode["ifmt"] & ~(1 << NLINK_ONE_BIT))
|
||||
self.put_u16(inode["offset"] + 6, value)
|
||||
else:
|
||||
self.put_u32(inode["offset"] + 44, value)
|
||||
return self.inode(nid)
|
||||
|
||||
def data_offset(self, inode: dict[str, int]) -> int:
|
||||
if inode["layout"] == 2:
|
||||
return inode["offset"] + inode["inode_size"] + inode["xattr_size"]
|
||||
if inode["layout"] == 0:
|
||||
return inode["startblk"] << self.block_bits
|
||||
raise ValueError(f"unsupported directory layout {inode['layout']}")
|
||||
|
||||
def directory(self, nid: int) -> dict[str, dict[str, int]]:
|
||||
inode = self.inode(nid)
|
||||
if inode["mode"] & S_IFMT != S_IFDIR:
|
||||
raise ValueError(f"inode {nid} is not a directory")
|
||||
data_offset = self.data_offset(inode)
|
||||
size = inode["size"]
|
||||
first_nameoff = self.u16(data_offset + 8)
|
||||
if first_nameoff == 0 or first_nameoff % 12 != 0 or first_nameoff >= size:
|
||||
raise ValueError("invalid first name offset")
|
||||
count = first_nameoff // 12
|
||||
records: dict[str, dict[str, int]] = {}
|
||||
for index in range(count):
|
||||
entry = data_offset + index * 12
|
||||
nameoff = self.u16(entry + 8)
|
||||
endoff = self.u16(entry + 20) if index + 1 < count else size
|
||||
raw = bytes(self.data[data_offset + nameoff : data_offset + endoff])
|
||||
name = raw.split(b"\0", 1)[0].decode("ascii")
|
||||
if not name or name in records:
|
||||
raise ValueError(f"bad directory name {name!r}")
|
||||
records[name] = {
|
||||
"entry_offset": entry,
|
||||
"nid": self.u64(entry),
|
||||
"file_type": self.data[entry + 10],
|
||||
}
|
||||
return records
|
||||
|
||||
|
||||
def make_source(path: Path, extended_uid: bool) -> None:
|
||||
previous_umask = os.umask(0o022)
|
||||
try:
|
||||
path.mkdir(mode=0o755)
|
||||
(path / "compact").write_bytes(b"compact zero-nlink target\n")
|
||||
(path / "extended").write_bytes(b"extended zero-nlink target\n")
|
||||
if extended_uid:
|
||||
os.chown(path / "extended", 70000, 0)
|
||||
(path / "hard-a").write_bytes(b"hardlink control\n")
|
||||
os.link(path / "hard-a", path / "hard-b")
|
||||
(path / "orphan").write_bytes(b"orphan raw-vget control\n")
|
||||
(path / "subdir").mkdir(mode=0o755)
|
||||
(path / "subdir" / "child").write_bytes(b"directory child\n")
|
||||
for regular in ("compact", "extended", "hard-a", "hard-b", "orphan"):
|
||||
os.chmod(path / regular, 0o644)
|
||||
os.chmod(path / "subdir" / "child", 0o644)
|
||||
finally:
|
||||
os.umask(previous_umask)
|
||||
|
||||
|
||||
def build_seed(work: Path, output: Path, inode_mode: str) -> tuple[bytes, str, list[str]]:
|
||||
work.mkdir()
|
||||
source = work / "source"
|
||||
make_source(source, inode_mode == "extended")
|
||||
command = [
|
||||
"mkfs.erofs",
|
||||
"-d0",
|
||||
"-T0",
|
||||
"--all-time",
|
||||
"--ignore-mtime",
|
||||
"--workers=1",
|
||||
"-x-1",
|
||||
"-E",
|
||||
"noinline_data",
|
||||
"-E",
|
||||
f"force-inode-{inode_mode}",
|
||||
"-U",
|
||||
SPEC["uuid"],
|
||||
str(output),
|
||||
str(source),
|
||||
]
|
||||
env = dict(os.environ)
|
||||
env.pop("SOURCE_DATE_EPOCH", None)
|
||||
completed = run(command, env=env)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"mkfs.erofs failed: {completed.stdout}")
|
||||
canonical = command[:-2] + [f"SEED-{inode_mode}.erofs", "SOURCE"]
|
||||
return output.read_bytes(), completed.stdout, canonical
|
||||
|
||||
|
||||
resolved = git(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}")
|
||||
if resolved != SPEC["required_base"]:
|
||||
raise SystemExit(f"wrong BASE: expected {SPEC['required_base']}, got {resolved}")
|
||||
if git(FREEBSD_SRC, "rev-parse", "HEAD") != SPEC["freebsd_head"]:
|
||||
raise SystemExit("FreeBSD source HEAD changed")
|
||||
if git(UTILS_SRC, "rev-parse", "HEAD") != SPEC["utils_head"]:
|
||||
raise SystemExit("erofs-utils source HEAD changed")
|
||||
if QEMU_BASE == Path(SPEC["protected"]["base_image"]):
|
||||
raise SystemExit("refusing the protected base bp")
|
||||
if SPEC["protected"]["port"] >= 32768:
|
||||
raise SystemExit("protected port unexpectedly overlaps ephemeral ports")
|
||||
|
||||
sources: dict[str, str] = {}
|
||||
source_hashes: dict[str, str] = {}
|
||||
for path, expected in SPEC["source_sha256"].items():
|
||||
content = source_at(resolved, path)
|
||||
actual = sha256_bytes(content.encode("utf-8"))
|
||||
if actual != expected:
|
||||
raise SystemExit(f"source hash changed for {path}: {actual}")
|
||||
sources[path] = content
|
||||
source_hashes[path] = actual
|
||||
for path, expected in SPEC["freebsd_sha256"].items():
|
||||
actual = sha256_path(FREEBSD_SRC / path)
|
||||
if actual != expected:
|
||||
raise SystemExit(f"FreeBSD hash changed for {path}: {actual}")
|
||||
for path, expected in SPEC["utils_sha256"].items():
|
||||
actual = sha256_path(UTILS_SRC / path)
|
||||
if actual != expected:
|
||||
raise SystemExit(f"erofs-utils hash changed for {path}: {actual}")
|
||||
|
||||
mkfs_version = run(["mkfs.erofs", "-V"]).stdout.splitlines()[0]
|
||||
if mkfs_version != SPEC["mkfs_version"]:
|
||||
raise SystemExit(f"mkfs version changed: {mkfs_version!r}")
|
||||
|
||||
namei = sources["repo-pre-15/src/namei.c"]
|
||||
inode_source = sources["repo-pre-15/src/inode.c"]
|
||||
vnops = sources["repo-pre-15/src/erofs_vnops.c"]
|
||||
super_source = sources["repo-pre-15/src/super.c"]
|
||||
linux_inode = sources["src-linux/inode.c"]
|
||||
linux_super = sources["src-linux/super.c"]
|
||||
freebsd_vm = (FREEBSD_SRC / "sys/vm/vm_mmap.c").read_text(encoding="utf-8")
|
||||
|
||||
source_anchors = {
|
||||
"freebsd_disk_compact_nlink": "vi->nlink = le16toh(dic->i_nb.nlink);" in inode_source,
|
||||
"freebsd_disk_extended_nlink": "vi->nlink = le32toh(die->i_nlink);" in inode_source,
|
||||
"freebsd_inode_decode_has_no_zero_reject": "vi->nlink == 0" not in inode_source,
|
||||
"freebsd_lookup_has_namespace_edge": "error = erofs_namei(sbi, dir, &qname, &nid, &dtype);" in namei,
|
||||
"freebsd_lookup_calls_raw_vget": "error = erofs_vget(dvp->v_mount, nid, cnp->cn_lkflags, &vp);" in namei,
|
||||
"freebsd_lookup_publishes_namecache": "cache_enter(dvp, vp, cnp);" in namei,
|
||||
"freebsd_raw_vget_registered": ".vfs_vget = erofs_vget," in super_source,
|
||||
"freebsd_root_uses_raw_vget": "error = erofs_vget(mp, MTOE(mp)->root_nid, flags, vpp);" in super_source,
|
||||
"freebsd_fhtovp_filters_zero": "vi->mode == 0 || vi->nlink == 0 || vi->nid != nid" in super_source,
|
||||
"freebsd_hash_insert_before_decode": vnops.index("error = vfs_hash_insert(") < vnops.index("error = erofs_read_inode("),
|
||||
"freebsd_constructed_before_return": vnops.index("vn_set_state(vp, VSTATE_CONSTRUCTED);") < vnops.index("*vpp = vp;"),
|
||||
"freebsd_vget_has_no_hit_indicator": "bool shared;" in vnops and "bool created" not in vnops,
|
||||
"freebsd_vput_does_not_imply_vgone": "vgone(vp);\n\t\tvput(vp);" in vnops,
|
||||
"freebsd_vfs_allows_unlinked_vnode": "if (va.va_nlink == 0)" in freebsd_vm,
|
||||
"linux_compact_sets_disk_nlink": "set_nlink(inode, le16_to_cpu(dic->i_nb.nlink));" in linux_inode,
|
||||
"linux_extended_sets_disk_nlink": "set_nlink(inode, le32_to_cpu(die->i_nlink));" in linux_inode,
|
||||
"linux_inode_has_no_zero_reject": "i_nlink == 0" not in linux_inode,
|
||||
"linux_super_has_no_zero_reject": "i_nlink == 0" not in linux_super,
|
||||
}
|
||||
if not all(source_anchors.values()):
|
||||
raise SystemExit(f"source anchors changed: {source_anchors}")
|
||||
|
||||
fixtures = OUTPUT / "fixtures"
|
||||
fixtures.mkdir()
|
||||
with tempfile.TemporaryDirectory(prefix="p15-092-gate-") as temporary:
|
||||
temp = Path(temporary)
|
||||
first_bytes, first_stdout, compact_command = build_seed(
|
||||
temp / "compact-first", temp / "compact-first.erofs", "compact"
|
||||
)
|
||||
second_bytes, second_stdout, compact_repeat_command = build_seed(
|
||||
temp / "compact-second", temp / "compact-second.erofs", "compact"
|
||||
)
|
||||
extended_bytes, extended_stdout, extended_command = build_seed(
|
||||
temp / "extended-first", temp / "extended-first.erofs", "extended"
|
||||
)
|
||||
extended_repeat_bytes, extended_repeat_stdout, extended_repeat_command = build_seed(
|
||||
temp / "extended-second", temp / "extended-second.erofs", "extended"
|
||||
)
|
||||
write_json(
|
||||
OUTPUT / "generator-attempts.json",
|
||||
{
|
||||
"commands": [compact_command, extended_command],
|
||||
"compact_source": [
|
||||
{
|
||||
"gid": path.lstat().st_gid,
|
||||
"mode": stat.S_IMODE(path.lstat().st_mode),
|
||||
"path": str(path.relative_to(temp / "compact-first/source")),
|
||||
"uid": path.lstat().st_uid,
|
||||
}
|
||||
for path in sorted((temp / "compact-first/source").rglob("*"))
|
||||
],
|
||||
},
|
||||
)
|
||||
if (
|
||||
first_bytes != second_bytes
|
||||
or first_stdout != second_stdout
|
||||
or compact_command != compact_repeat_command
|
||||
or extended_bytes != extended_repeat_bytes
|
||||
or extended_stdout != extended_repeat_stdout
|
||||
or extended_command != extended_repeat_command
|
||||
):
|
||||
raise SystemExit("seed generation is not byte reproducible")
|
||||
seed_path = fixtures / "seed.erofs"
|
||||
seed_path.write_bytes(first_bytes)
|
||||
extended_seed_path = fixtures / "seed-extended.erofs"
|
||||
extended_seed_path.write_bytes(extended_bytes)
|
||||
seed = Image(first_bytes)
|
||||
extended_seed = Image(extended_bytes)
|
||||
if not seed.checksum_valid() or not extended_seed.checksum_valid():
|
||||
raise SystemExit("seed checksum is invalid")
|
||||
entries = seed.directory(seed.root_nid)
|
||||
required = {".", "..", "compact", "extended", "hard-a", "hard-b", "orphan", "subdir"}
|
||||
if not required.issubset(entries):
|
||||
raise SystemExit(f"seed entries missing: {sorted(required - entries.keys())}")
|
||||
if entries["hard-a"]["nid"] != entries["hard-b"]["nid"]:
|
||||
raise SystemExit("hardlink names do not share one NID")
|
||||
|
||||
extended_entries = extended_seed.directory(extended_seed.root_nid)
|
||||
seed_inodes = {name: seed.inode(record["nid"]) for name, record in entries.items()}
|
||||
extended_inodes = {
|
||||
name: extended_seed.inode(record["nid"])
|
||||
for name, record in extended_entries.items()
|
||||
}
|
||||
if seed_inodes["compact"]["version"] != 0:
|
||||
raise SystemExit("compact control is not compact")
|
||||
if extended_inodes["extended"]["version"] != 1:
|
||||
raise SystemExit("extended control is not extended")
|
||||
if seed_inodes["hard-a"]["nlink"] != 2:
|
||||
raise SystemExit("hardlink control does not have nlink 2")
|
||||
|
||||
mutations = []
|
||||
|
||||
def emit_zero(case_id: str, name: str, source: Image = seed) -> None:
|
||||
image = source.clone()
|
||||
nid = image.directory(image.root_nid)[name]["nid"]
|
||||
before = image.inode(nid)
|
||||
after = image.set_nlink(nid, 0)
|
||||
image.update_checksum()
|
||||
path = fixtures / f"{case_id}.erofs"
|
||||
path.write_bytes(image.data)
|
||||
mutations.append({
|
||||
"case": case_id,
|
||||
"class": "namespace-reachable-zero",
|
||||
"name": name,
|
||||
"nid": nid,
|
||||
"before": before,
|
||||
"after": after,
|
||||
"fixture": path.name,
|
||||
})
|
||||
|
||||
emit_zero("reachable-compact-zero", "compact")
|
||||
emit_zero("reachable-extended-zero", "extended", extended_seed)
|
||||
emit_zero("reachable-directory-zero", "subdir")
|
||||
|
||||
root_zero = seed.clone()
|
||||
root_before = root_zero.inode(root_zero.root_nid)
|
||||
root_after = root_zero.set_nlink(root_zero.root_nid, 0)
|
||||
root_zero.update_checksum()
|
||||
root_path = fixtures / "root-zero.erofs"
|
||||
root_path.write_bytes(root_zero.data)
|
||||
mutations.append({
|
||||
"case": "root-zero",
|
||||
"class": "raw-root-zero",
|
||||
"name": "/",
|
||||
"nid": root_zero.root_nid,
|
||||
"before": root_before,
|
||||
"after": root_after,
|
||||
"fixture": root_path.name,
|
||||
})
|
||||
|
||||
orphan = seed.clone()
|
||||
orphan_entries = orphan.directory(orphan.root_nid)
|
||||
orphan_nid = orphan_entries["orphan"]["nid"]
|
||||
hard_nid = orphan_entries["hard-a"]["nid"]
|
||||
orphan.put_u64(orphan_entries["orphan"]["entry_offset"], hard_nid)
|
||||
orphan.set_nlink(orphan_nid, 0)
|
||||
orphan.set_nlink(hard_nid, 3)
|
||||
orphan.update_checksum()
|
||||
orphan_path = fixtures / "orphan-zero.erofs"
|
||||
orphan_path.write_bytes(orphan.data)
|
||||
orphan_names = orphan.directory(orphan.root_nid)
|
||||
if any(record["nid"] == orphan_nid for record in orphan_names.values()):
|
||||
raise SystemExit("orphan fixture still has a namespace edge")
|
||||
mutations.append({
|
||||
"case": "orphan-zero",
|
||||
"class": "unreachable-zero",
|
||||
"name": None,
|
||||
"nid": orphan_nid,
|
||||
"before": seed.inode(orphan_nid),
|
||||
"after": orphan.inode(orphan_nid),
|
||||
"replacement_nid": hard_nid,
|
||||
"replacement_nlink": orphan.inode(hard_nid)["nlink"],
|
||||
"fixture": orphan_path.name,
|
||||
})
|
||||
|
||||
records = []
|
||||
fixture_digest = hashlib.sha256()
|
||||
for path in sorted(fixtures.glob("*.erofs")):
|
||||
image = Image(path.read_bytes())
|
||||
if not image.checksum_valid():
|
||||
raise SystemExit(f"fixture checksum invalid: {path.name}")
|
||||
fsck = run(["fsck.erofs", "-d0", str(path)])
|
||||
dump_root = run(["dump.erofs", "--path=/", str(path)])
|
||||
if dump_root.returncode != 0:
|
||||
raise SystemExit(f"dump root failed: {path.name}")
|
||||
normal = {}
|
||||
for name in ("compact", "extended", "hard-a", "hard-b", "orphan", "subdir"):
|
||||
completed = run(["dump.erofs", f"--path=/{name}", str(path)])
|
||||
normal[name] = {
|
||||
"exit": completed.returncode,
|
||||
"nid": int(match.group(1)) if (match := re.search(r"^NID: (\d+)\b", completed.stdout, re.MULTILINE)) else None,
|
||||
}
|
||||
record = {
|
||||
"fixture": path.name,
|
||||
"sha256": sha256_path(path),
|
||||
"size": path.stat().st_size,
|
||||
"checksum_valid": True,
|
||||
"fsck_exit": fsck.returncode,
|
||||
"fsck_error_marker": "<E>" in fsck.stdout,
|
||||
"normal_namespace": normal,
|
||||
}
|
||||
records.append(record)
|
||||
fixture_digest.update(path.name.encode("ascii"))
|
||||
fixture_digest.update(b"\0")
|
||||
fixture_digest.update(path.read_bytes())
|
||||
|
||||
write_json(
|
||||
OUTPUT / "generator.json",
|
||||
{
|
||||
"commands": [compact_command, extended_command],
|
||||
"mkfs_version": mkfs_version,
|
||||
"repeat_byte_identical": True,
|
||||
"seed_sha256": {
|
||||
"compact": sha256_bytes(first_bytes),
|
||||
"extended": sha256_bytes(extended_bytes),
|
||||
},
|
||||
},
|
||||
)
|
||||
(OUTPUT / "mkfs.stdout").write_text(
|
||||
first_stdout + extended_stdout, encoding="utf-8"
|
||||
)
|
||||
write_json(OUTPUT / "disk-records.json", {"mutations": mutations, "records": records})
|
||||
write_json(
|
||||
OUTPUT / "fixture-index.json",
|
||||
{
|
||||
"fixture_count": len(records),
|
||||
"fixture_set_sha256": fixture_digest.hexdigest(),
|
||||
"fixtures": records,
|
||||
},
|
||||
)
|
||||
|
||||
fsck_reachable_zero_rejects = []
|
||||
for record in records:
|
||||
if record["fixture"].startswith("reachable-"):
|
||||
fsck_reachable_zero_rejects.append(
|
||||
record["fsck_exit"] != 0 or record["fsck_error_marker"]
|
||||
)
|
||||
|
||||
boundary = {
|
||||
"b15_write_set": SPEC["write_set"],
|
||||
"vnode_constructor_owner": "repo-pre-15/src/erofs_vnops.c",
|
||||
"constructor_inside_b15_write_set": "repo-pre-15/src/erofs_vnops.c" in SPEC["write_set"],
|
||||
"post_vget_check_is_before_constructed_publication": False,
|
||||
"vput_only_performs_vgone_cleanup": False,
|
||||
"post_vget_vgone_preserves_cached_raw_vget": False,
|
||||
"pre_vget_full_decode_avoids_duplicate_inode_decode": False,
|
||||
"mount_wide_scan_required": False,
|
||||
"namespace_and_raw_entrypoints_distinct": True,
|
||||
}
|
||||
format_semantics = {
|
||||
"mkfs_emits_nonzero_namespace_links": all(
|
||||
seed_inodes[name]["nlink"] > 0
|
||||
for name in ("compact", "extended", "hard-a", "hard-b", "orphan", "subdir", ".")
|
||||
),
|
||||
"fsck_rejects_all_reachable_zero": all(fsck_reachable_zero_rejects),
|
||||
"linux_rejects_zero_during_inode_decode": False,
|
||||
"linux_copies_disk_nlink_into_inode": True,
|
||||
"freebsd_vfs_permits_live_unlinked_vnode": True,
|
||||
"freebsd_fhtovp_treats_zero_as_stale": True,
|
||||
}
|
||||
go_requirements = {
|
||||
"format_or_cross_kernel_rule_requires_rejection": (
|
||||
format_semantics["fsck_rejects_all_reachable_zero"]
|
||||
and format_semantics["linux_rejects_zero_during_inode_decode"]
|
||||
),
|
||||
"reject_before_vnode_publication_within_write_set": (
|
||||
boundary["constructor_inside_b15_write_set"]
|
||||
and boundary["post_vget_check_is_before_constructed_publication"]
|
||||
),
|
||||
"cleanup_preserves_raw_vget": boundary["post_vget_vgone_preserves_cached_raw_vget"],
|
||||
"no_duplicate_full_inode_decode": boundary["pre_vget_full_decode_avoids_duplicate_inode_decode"],
|
||||
"orphan_needs_no_mount_scan": not boundary["mount_wide_scan_required"],
|
||||
"namespace_raw_split_proven": boundary["namespace_and_raw_entrypoints_distinct"],
|
||||
}
|
||||
status = "GO" if all(go_requirements.values()) else "STOP"
|
||||
reasons = [key for key, value in go_requirements.items() if not value]
|
||||
write_json(OUTPUT / "source-anchors.json", source_anchors)
|
||||
write_json(OUTPUT / "source-sha256.json", source_hashes)
|
||||
write_json(OUTPUT / "boundary.json", boundary)
|
||||
write_json(OUTPUT / "format-semantics.json", format_semantics)
|
||||
write_json(
|
||||
OUTPUT / "host-result.json",
|
||||
{
|
||||
"candidate": SPEC["candidate"],
|
||||
"gate": SPEC["gate"],
|
||||
"fixture_count": len(records),
|
||||
"fixture_set_sha256": fixture_digest.hexdigest(),
|
||||
"go_requirements": go_requirements,
|
||||
"reasons": reasons,
|
||||
"requested_base": REQUESTED_BASE,
|
||||
"resolved_base": resolved,
|
||||
"schema": 1,
|
||||
"status": status,
|
||||
},
|
||||
)
|
||||
PY
|
||||
|
||||
if test "$host_only" -eq 1; then
|
||||
cat "$output/host/host-result.json"
|
||||
exit 22
|
||||
fi
|
||||
|
||||
runtime_tmp=$(mktemp -d "${TMPDIR:-/tmp}/p15-092-runtime.XXXXXX")
|
||||
runtime_dut=$runtime_tmp/repo-pre-15
|
||||
runtime_case=$runtime_dut/tests/pre15/cases/P15-092-gate-runtime.sh
|
||||
runtime_cleanup()
|
||||
{
|
||||
rm -rf "$runtime_tmp"
|
||||
}
|
||||
trap runtime_cleanup EXIT HUP INT TERM
|
||||
mkdir -p "$runtime_dut/tests/pre15/cases" "$runtime_dut/tests/pre15/fixtures"
|
||||
git -C "$root" archive --format=tar --output="$runtime_tmp/frozen-src.tar" \
|
||||
"$base" repo-pre-15/src
|
||||
tar -C "$runtime_tmp" -xf "$runtime_tmp/frozen-src.tar"
|
||||
ln -s "$dut/tests/pre15/fixtures/B14-build-kld.sh" \
|
||||
"$runtime_dut/tests/pre15/fixtures/B14-build-kld.sh"
|
||||
|
||||
python3 -B - "$runtime_case" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
Path(sys.argv[1]).write_text(r'''#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}"
|
||||
: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}"
|
||||
: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}"
|
||||
: "${P15_092_GATE_DUT:?P15_092_GATE_DUT is required}"
|
||||
: "${P15_092_GATE_HOST:?P15_092_GATE_HOST is required}"
|
||||
. "$PRE15_LIB_DIR/runner.sh"
|
||||
|
||||
artifacts=$PRE15_RUN_DIR/artifacts
|
||||
fixtures=$P15_092_GATE_HOST/fixtures
|
||||
builder=$P15_092_GATE_DUT/tests/pre15/fixtures/B14-build-kld.sh
|
||||
module=$PRE15_CASE_TMP/P15-092-baseline-erofs.ko
|
||||
archive=$PRE15_CASE_TMP/P15-092-fixtures.tar.gz
|
||||
mkdir -p "$artifacts"
|
||||
|
||||
pre15_record_fixture p15-092-host-result "$P15_092_GATE_HOST/host-result.json"
|
||||
pre15_record_fixture p15-092-disk-records "$P15_092_GATE_HOST/disk-records.json"
|
||||
pre15_record_fixture p15-092-boundary "$P15_092_GATE_HOST/boundary.json"
|
||||
pre15_record_fixture p15-092-source-namei "$P15_092_GATE_DUT/src/namei.c"
|
||||
pre15_record_fixture p15-092-source-inode "$P15_092_GATE_DUT/src/inode.c"
|
||||
pre15_record_fixture p15-092-source-vnops "$P15_092_GATE_DUT/src/erofs_vnops.c"
|
||||
|
||||
if ! /bin/sh "$builder" "$P15_092_GATE_DUT" "$PRE15_FREEBSD_SRC" "$module" \
|
||||
"$PRE15_CASE_TMP/kld-work" >"$artifacts/kld-build.stdout" \
|
||||
2>"$artifacts/kld-build.stderr"; then
|
||||
pre15_dut_fail 'baseline cross-target KLD build failed'
|
||||
fi
|
||||
pre15_record_module "$module"
|
||||
tar -C "$fixtures" -czf "$archive" .
|
||||
|
||||
pre15_scp()
|
||||
{
|
||||
timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
|
||||
-o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \
|
||||
-i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \
|
||||
"$1" "[email protected]:$2"
|
||||
}
|
||||
|
||||
pre15_scp "$module" /root/P15-092-baseline-erofs.ko || \
|
||||
pre15_infra_blocked 'could not transfer baseline KLD'
|
||||
pre15_scp "$archive" /root/P15-092-fixtures.tar.gz || \
|
||||
pre15_infra_blocked 'could not transfer zero-nlink fixtures'
|
||||
pre15_guest_ssh_bounded \
|
||||
'rm -rf /root/P15-092-fixtures && mkdir /root/P15-092-fixtures && tar -xzf /root/P15-092-fixtures.tar.gz -C /root/P15-092-fixtures' || \
|
||||
pre15_infra_blocked 'could not prepare guest fixtures'
|
||||
if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then
|
||||
pre15_infra_blocked 'guest already has an EROFS module loaded'
|
||||
fi
|
||||
pre15_guest_ssh_bounded kldload /root/P15-092-baseline-erofs.ko || \
|
||||
pre15_dut_fail 'baseline KLD failed to load'
|
||||
pre15_own_guest_kld erofs 'P15-092 baseline KLD'
|
||||
pre15_target_reached
|
||||
|
||||
attach_mount()
|
||||
{
|
||||
image=$1
|
||||
mountpoint=$2
|
||||
label=$3
|
||||
md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \
|
||||
-f "/root/P15-092-fixtures/$image") || \
|
||||
pre15_dut_fail "$label md attach failed"
|
||||
case "$md" in
|
||||
md[0-9]*) ;;
|
||||
*) pre15_runner_fail "unexpected md unit: $md" ;;
|
||||
esac
|
||||
pre15_own_guest_md "$md" "$label md"
|
||||
pre15_guest_ssh_bounded mkdir -p "$mountpoint"
|
||||
pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$md" "$mountpoint" || \
|
||||
pre15_dut_fail "$label mount failed"
|
||||
pre15_own_guest_mount "$mountpoint" "$label mount"
|
||||
}
|
||||
|
||||
expect_nlink()
|
||||
{
|
||||
path=$1
|
||||
expected=$2
|
||||
label=$3
|
||||
actual=$(pre15_guest_ssh_bounded stat -f '%l' "$path") || \
|
||||
pre15_dut_fail "$label normal stat failed"
|
||||
test "$actual" = "$expected" || \
|
||||
pre15_dut_fail "$label nlink expected $expected got $actual"
|
||||
printf '%s\t%s\t%s\n' "$label" "$path" "$actual" >>"$artifacts/normal-vnode.tsv"
|
||||
}
|
||||
|
||||
attach_mount seed.erofs /mnt/p15-092-seed seed
|
||||
expect_nlink /mnt/p15-092-seed/compact 1 seed-compact
|
||||
expect_nlink /mnt/p15-092-seed/extended 1 seed-extended
|
||||
expect_nlink /mnt/p15-092-seed/hard-a 2 seed-hard-a
|
||||
hard_a=$(pre15_guest_ssh_bounded stat -f '%i' /mnt/p15-092-seed/hard-a)
|
||||
hard_b=$(pre15_guest_ssh_bounded stat -f '%i' /mnt/p15-092-seed/hard-b)
|
||||
test "$hard_a" = "$hard_b" || pre15_dut_fail 'hardlink NIDs differ'
|
||||
|
||||
attach_mount reachable-compact-zero.erofs /mnt/p15-092-compact compact-zero
|
||||
expect_nlink /mnt/p15-092-compact/compact 0 reachable-compact-zero
|
||||
attach_mount reachable-extended-zero.erofs /mnt/p15-092-extended extended-zero
|
||||
expect_nlink /mnt/p15-092-extended/extended 0 reachable-extended-zero
|
||||
attach_mount reachable-directory-zero.erofs /mnt/p15-092-directory directory-zero
|
||||
expect_nlink /mnt/p15-092-directory/subdir 0 reachable-directory-zero
|
||||
attach_mount root-zero.erofs /mnt/p15-092-root root-zero
|
||||
expect_nlink /mnt/p15-092-root 0 root-zero
|
||||
attach_mount orphan-zero.erofs /mnt/p15-092-orphan orphan-zero
|
||||
expect_nlink /mnt/p15-092-orphan/orphan 3 orphan-replacement
|
||||
pre15_guest_ssh_bounded cat /mnt/p15-092-orphan/orphan >"$artifacts/orphan-replacement.txt"
|
||||
pre15_guest_ssh_bounded dmesg >"$artifacts/dmesg.txt"
|
||||
printf '%s\n' 'P15-092 baseline normal vnode runtime PASS'
|
||||
''', encoding="ascii")
|
||||
PY
|
||||
chmod 0555 "$runtime_case"
|
||||
|
||||
qemu_stdout=$output/qemu.stdout
|
||||
qemu_stderr=$output/qemu.stderr
|
||||
qemu_evidence=$output/qemu-evidence
|
||||
mkdir "$qemu_evidence"
|
||||
if PRE15_DUT="$runtime_dut" PRE15_ROOT="$root" \
|
||||
PRE15_SCHEMA="$dut/tests/pre15/EVIDENCE-SCHEMA.json" \
|
||||
PRE15_EVIDENCE_ROOT="$qemu_evidence" PRE15_FREEBSD_SRC="$freebsd_src" \
|
||||
PRE15_QEMU_BASE_IMAGE="$qemu_base" PRE15_QEMU_BASE_FORMAT=qcow2 \
|
||||
PRE15_QEMU_SSH_KEY="$qemu_key" PRE15_QEMU_SSH_USER=root \
|
||||
PRE15_QEMU_MEMORY_MB=2048 PRE15_QEMU_CPUS=2 PRE15_QEMU_BOOT_TIMEOUT=180 \
|
||||
PRE15_QEMU_TIMEOUT=900 PRE15_GUEST_COMMAND_TIMEOUT=60 \
|
||||
P15_092_GATE_DUT="$runtime_dut" P15_092_GATE_HOST="$output/host" \
|
||||
timeout -k 30 1000 "$dut/tests/pre15/run-qemu.sh" P15-092-gate-runtime \
|
||||
>"$qemu_stdout" 2>"$qemu_stderr"; then
|
||||
qemu_rc=0
|
||||
else
|
||||
qemu_rc=$?
|
||||
fi
|
||||
printf '%s\n' "$qemu_rc" >"$output/qemu.exit"
|
||||
|
||||
python3 -B - "$output" "$qemu_rc" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
OUTPUT = Path(sys.argv[1])
|
||||
QEMU_RC = int(sys.argv[2])
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, value: object) -> None:
|
||||
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
|
||||
|
||||
host = json.loads((OUTPUT / "host/host-result.json").read_text(encoding="ascii"))
|
||||
stdout = (OUTPUT / "qemu.stdout").read_text(encoding="utf-8")
|
||||
match = re.search(r"PRE15_RESULT status=(\S+) cleanup=(\S+) run_id=(\S+) evidence=(\S+)", stdout)
|
||||
if match is None:
|
||||
runtime = {
|
||||
"cleanup": "UNKNOWN",
|
||||
"evidence": None,
|
||||
"exit": QEMU_RC,
|
||||
"run_id": None,
|
||||
"status": "RUNNER_FAIL",
|
||||
}
|
||||
else:
|
||||
runtime = {
|
||||
"status": match.group(1),
|
||||
"cleanup": match.group(2),
|
||||
"run_id": match.group(3),
|
||||
"evidence": match.group(4),
|
||||
"exit": QEMU_RC,
|
||||
}
|
||||
|
||||
status = "STOP"
|
||||
reasons = list(host["reasons"])
|
||||
if runtime["status"] != "PASS" or runtime["cleanup"] != "PASS":
|
||||
reasons.append("normal_vnode_runtime_not_pass")
|
||||
result = {
|
||||
"b15": "STOP-NO-SOURCE",
|
||||
"candidate": "P15-092",
|
||||
"full_feature_suite": "NOT_RUN",
|
||||
"gate": "G11",
|
||||
"host_status": host["status"],
|
||||
"qemu": runtime,
|
||||
"reasons": reasons,
|
||||
"schema": 1,
|
||||
"status": status,
|
||||
}
|
||||
write_json(OUTPUT / "result.json", result)
|
||||
|
||||
hash_lines = []
|
||||
for path in sorted(OUTPUT.rglob("*")):
|
||||
if path.is_file() and path.name != "SHA256SUMS":
|
||||
hash_lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
|
||||
(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii")
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
PY
|
||||
|
||||
cat "$output/result.json"
|
||||
exit 22
|
||||
Reference in New Issue
Block a user