update
This commit is contained in:
Executable
+482
@@ -0,0 +1,482 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independently parse B17 images and compare frozen FreeBSD expectations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
|
||||
SUPER_OFFSET = 1024
|
||||
FEATURE_COMPAT_XATTR_FILTER = 0x00000004
|
||||
FEATURE_COMPAT_PLAIN_XATTR_PFX = 0x00000010
|
||||
FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040
|
||||
ACL_FILTER_BITS = (1 << 21) | (1 << 30)
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
|
||||
|
||||
class Reject(Exception):
|
||||
def __init__(self, errno_name: str, point: str):
|
||||
super().__init__(f"{errno_name} at {point}")
|
||||
self.errno_name = errno_name
|
||||
self.point = point
|
||||
|
||||
|
||||
def crc32c(data: bytes, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
class Reader:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self.data = path.read_bytes()
|
||||
if len(self.data) < SUPER_OFFSET + 144:
|
||||
raise Reject("EINTEGRITY", "super.bounds")
|
||||
if self.u32(SUPER_OFFSET) != 0xE0F5E1E2:
|
||||
raise Reject("EINTEGRITY", "super.magic")
|
||||
self.block_bits = self.data[SUPER_OFFSET + 12]
|
||||
if self.block_bits < 9 or self.block_bits > 16:
|
||||
raise Reject("EINTEGRITY", "super.block-size")
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.blocks = self.u32(SUPER_OFFSET + 36)
|
||||
self.limit = self.blocks << self.block_bits
|
||||
if self.limit > len(self.data):
|
||||
raise Reject("EINTEGRITY", "super.image-bounds")
|
||||
self.feature_compat = self.u32(SUPER_OFFSET + 8)
|
||||
self.feature_incompat = self.u32(SUPER_OFFSET + 80)
|
||||
self.filter_reserved = self.data[SUPER_OFFSET + 104]
|
||||
self.meta_blkaddr = self.u32(SUPER_OFFSET + 40)
|
||||
self.xattr_blkaddr = self.u32(SUPER_OFFSET + 44)
|
||||
self.root_nid = self.u16(SUPER_OFFSET + 14)
|
||||
self.packed_nid = self.u64(SUPER_OFFSET + 96)
|
||||
self.prefix_count = self.data[SUPER_OFFSET + 91]
|
||||
self.prefix_start = self.u32(SUPER_OFFSET + 92)
|
||||
self.verify_checksum()
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 2 > len(self.data):
|
||||
raise Reject("EINTEGRITY", "raw.u16-bounds")
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 4 > len(self.data):
|
||||
raise Reject("EINTEGRITY", "raw.u32-bounds")
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def u64(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 8 > len(self.data):
|
||||
raise Reject("EINTEGRITY", "raw.u64-bounds")
|
||||
return struct.unpack_from("<Q", self.data, offset)[0]
|
||||
|
||||
def verify_checksum(self) -> None:
|
||||
if not self.feature_compat & 1:
|
||||
return
|
||||
end = self.block_size
|
||||
if end > len(self.data):
|
||||
raise Reject("EINTEGRITY", "super.checksum-bounds")
|
||||
expected = self.u32(SUPER_OFFSET + 4)
|
||||
block = bytearray(self.data[SUPER_OFFSET:end])
|
||||
block[4:8] = bytes(4)
|
||||
if crc32c(bytes(block)) != expected:
|
||||
raise Reject("EINTEGRITY", "super.checksum")
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
if offset + 32 > self.limit:
|
||||
raise Reject("EINTEGRITY", "inode.bounds")
|
||||
inode_format = self.u16(offset)
|
||||
inode_size = 64 if inode_format & 1 else 32
|
||||
if offset + inode_size > self.limit:
|
||||
raise Reject("EINTEGRITY", "inode.bounds")
|
||||
xattr_count = self.u16(offset + 2)
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
||||
size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8)
|
||||
return {
|
||||
"nid": nid,
|
||||
"offset": offset,
|
||||
"inode_size": inode_size,
|
||||
"xattr_count": xattr_count,
|
||||
"xattr_size": xattr_size,
|
||||
"layout": (inode_format >> 1) & 7,
|
||||
"size": size,
|
||||
"start_block": self.u32(offset + 16),
|
||||
}
|
||||
|
||||
def inode_range(self, inode: dict[str, int], logical: int, length: int, point: str) -> bytes:
|
||||
if logical > inode["size"] or length > inode["size"] - logical:
|
||||
raise Reject("EINTEGRITY", point)
|
||||
if inode["layout"] == 2:
|
||||
physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical
|
||||
elif inode["layout"] == 0:
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
else:
|
||||
raise Reject("EOPNOTSUPP", "inode.layout")
|
||||
if physical > self.limit or length > self.limit - physical:
|
||||
raise Reject("EINTEGRITY", point)
|
||||
return self.data[physical : physical + length]
|
||||
|
||||
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
|
||||
data = self.inode_range(inode, 0, inode["size"], "directory.bounds")
|
||||
if len(data) < 12:
|
||||
raise Reject("EINTEGRITY", "directory.header")
|
||||
first_name = struct.unpack_from("<H", data, 8)[0]
|
||||
if first_name == 0 or first_name % 12 != 0 or first_name > len(data):
|
||||
raise Reject("EINTEGRITY", "directory.name-offset")
|
||||
count = first_name // 12
|
||||
entries = []
|
||||
for index in range(count):
|
||||
entry = index * 12
|
||||
nid = struct.unpack_from("<Q", data, entry)[0]
|
||||
name_start = struct.unpack_from("<H", data, entry + 8)[0]
|
||||
name_end = (
|
||||
struct.unpack_from("<H", data, entry + 20)[0]
|
||||
if index + 1 < count
|
||||
else len(data)
|
||||
)
|
||||
if name_start > name_end or name_end > len(data):
|
||||
raise Reject("EINTEGRITY", "directory.name-bounds")
|
||||
name = data[name_start:name_end].split(b"\0", 1)[0]
|
||||
entries.append((name, nid))
|
||||
return entries
|
||||
|
||||
def resolve(self, path: str) -> dict[str, int]:
|
||||
inode = self.inode(self.root_nid)
|
||||
for component in path.strip("/").encode("ascii").split(b"/"):
|
||||
if not component:
|
||||
continue
|
||||
nid = next(
|
||||
(candidate for name, candidate in self.directory_entries(inode)
|
||||
if name == component),
|
||||
None,
|
||||
)
|
||||
if nid is None:
|
||||
raise Reject("ENOATTR", "path.lookup")
|
||||
inode = self.inode(nid)
|
||||
return inode
|
||||
|
||||
def load_prefixes(self) -> list[tuple[int, bytes]]:
|
||||
if not self.feature_incompat & FEATURE_INCOMPAT_XATTR_PREFIXES or self.prefix_count == 0:
|
||||
return []
|
||||
if self.feature_compat & FEATURE_COMPAT_PLAIN_XATTR_PFX:
|
||||
raise Reject("EOPNOTSUPP", "prefix.plain-unexpected")
|
||||
if self.packed_nid == 0:
|
||||
raise Reject("EINTEGRITY", "prefix.carrier")
|
||||
packed = self.inode(self.packed_nid)
|
||||
logical = self.prefix_start << 2
|
||||
prefixes = []
|
||||
for _ in range(self.prefix_count):
|
||||
while logical % 4:
|
||||
logical += 1
|
||||
header = self.inode_range(packed, logical, 2, "prefix.header-bounds")
|
||||
length = struct.unpack("<H", header)[0]
|
||||
if length < 1 or length > 256:
|
||||
raise Reject("EINTEGRITY", "prefix.length")
|
||||
payload = self.inode_range(
|
||||
packed, logical + 2, length, "prefix.payload-bounds"
|
||||
)
|
||||
base_index = payload[0]
|
||||
infix = payload[1:]
|
||||
if b"\0" in infix:
|
||||
raise Reject("EINTEGRITY", "prefix.infix-nul")
|
||||
prefixes.append((base_index, infix))
|
||||
logical += 2 + length
|
||||
return prefixes
|
||||
|
||||
def load_body(self, inode: dict[str, int]) -> tuple[int, int, list[int]]:
|
||||
size = inode["xattr_size"]
|
||||
if size == 0:
|
||||
return 0, 0, []
|
||||
body = inode["offset"] + inode["inode_size"]
|
||||
if body > self.limit or size > self.limit - body:
|
||||
raise Reject("EINTEGRITY", "ibody.bounds")
|
||||
if size < 12:
|
||||
raise Reject("EINTEGRITY", "ibody.header-bounds")
|
||||
if size == 12:
|
||||
raise Reject("EOPNOTSUPP", "ibody.header-only")
|
||||
shared_count = self.data[body + 4]
|
||||
header_size = 12 + shared_count * 4
|
||||
if header_size > size:
|
||||
raise Reject("EINTEGRITY", "ibody.shared-count")
|
||||
shared = [self.u32(body + 12 + index * 4) for index in range(shared_count)]
|
||||
return body, header_size, shared
|
||||
|
||||
def entry(self, offset: int, limit: int, kind: str) -> dict[str, object]:
|
||||
if offset > limit or 4 > limit - offset:
|
||||
raise Reject("EINTEGRITY", f"{kind}.entry-header")
|
||||
name_length = self.data[offset]
|
||||
name_index = self.data[offset + 1]
|
||||
value_length = self.u16(offset + 2)
|
||||
name_end = offset + 4 + name_length
|
||||
if name_end > limit:
|
||||
raise Reject("EINTEGRITY", f"{kind}.name-bounds")
|
||||
value_end = name_end + value_length
|
||||
if value_end > limit:
|
||||
raise Reject("EINTEGRITY", f"{kind}.value-bounds")
|
||||
aligned_end = (value_end + 3) & ~3
|
||||
if aligned_end > limit:
|
||||
raise Reject("EINTEGRITY", f"{kind}.padding-bounds")
|
||||
name = self.data[offset + 4 : name_end]
|
||||
if b"\0" in name:
|
||||
raise Reject("EINTEGRITY", f"{kind}.name-nul")
|
||||
return {
|
||||
"offset": offset,
|
||||
"next": aligned_end,
|
||||
"name_index": name_index,
|
||||
"name": name,
|
||||
"value": self.data[name_end:value_end],
|
||||
}
|
||||
|
||||
def entries(self, inode: dict[str, int]) -> tuple[list[dict[str, object]], int]:
|
||||
body, header_size, shared_ids = self.load_body(inode)
|
||||
if body == 0:
|
||||
return [], 0
|
||||
body_end = body + inode["xattr_size"]
|
||||
cursor = body + header_size
|
||||
entries = []
|
||||
while cursor < body_end:
|
||||
item = self.entry(cursor, body_end, "inline")
|
||||
entries.append(item)
|
||||
cursor = int(item["next"])
|
||||
for shared_id in shared_ids:
|
||||
offset = (self.xattr_blkaddr << self.block_bits) + shared_id * 4
|
||||
if offset > self.limit or 4 > self.limit - offset:
|
||||
raise Reject("EINTEGRITY", "shared.offset-bounds")
|
||||
entries.append(self.entry(offset, self.limit, "shared"))
|
||||
return entries, len(shared_ids)
|
||||
|
||||
def resolved_name(
|
||||
self, entry: dict[str, object], prefixes: list[tuple[int, bytes]]
|
||||
) -> tuple[str, bytes]:
|
||||
index = int(entry["name_index"])
|
||||
infix = b""
|
||||
from_prefix = False
|
||||
if index & 0x80:
|
||||
prefix_id = index & 0x7F
|
||||
if prefix_id >= len(prefixes):
|
||||
raise Reject("ENOATTR", "name.long-prefix-id")
|
||||
index, infix = prefixes[prefix_id]
|
||||
from_prefix = True
|
||||
mapping = {
|
||||
1: ("user", b""),
|
||||
2: ("system", b"posix_acl_access"),
|
||||
3: ("system", b"posix_acl_default"),
|
||||
4: ("system", b"trusted."),
|
||||
6: ("system", b"security."),
|
||||
}
|
||||
if index not in mapping:
|
||||
point = "name.prefix-base-index" if from_prefix else "name.short-index"
|
||||
raise Reject("ENOATTR", point)
|
||||
namespace, fixed = mapping[index]
|
||||
return namespace, fixed + infix + bytes(entry["name"])
|
||||
|
||||
def getxattr(self, inode: dict[str, int], namespace: str, name: bytes,
|
||||
prefixes: list[tuple[int, bytes]]) -> tuple[bytes, int]:
|
||||
entries, shared_count = self.entries(inode)
|
||||
deferred: Reject | None = None
|
||||
for entry in entries:
|
||||
try:
|
||||
actual_namespace, actual_name = self.resolved_name(entry, prefixes)
|
||||
except Reject as error:
|
||||
deferred = error
|
||||
continue
|
||||
if actual_namespace == namespace and actual_name == name:
|
||||
return bytes(entry["value"]), shared_count
|
||||
if deferred is not None:
|
||||
raise deferred
|
||||
raise Reject("ENOATTR", "name.not-found")
|
||||
|
||||
def listxattr(self, inode: dict[str, int], namespace: str,
|
||||
prefixes: list[tuple[int, bytes]]) -> tuple[list[bytes], int]:
|
||||
entries, shared_count = self.entries(inode)
|
||||
names = []
|
||||
deferred: Reject | None = None
|
||||
for entry in entries:
|
||||
try:
|
||||
actual_namespace, actual_name = self.resolved_name(entry, prefixes)
|
||||
except Reject as error:
|
||||
deferred = error
|
||||
continue
|
||||
if actual_namespace == namespace:
|
||||
names.append(actual_name)
|
||||
if deferred is not None and not names:
|
||||
raise deferred
|
||||
return names, shared_count
|
||||
|
||||
|
||||
def parse_acl(value: bytes) -> list[list[int]]:
|
||||
if len(value) < 4 or (len(value) - 4) % 8:
|
||||
raise Reject("EINTEGRITY", "acl.value-size")
|
||||
if struct.unpack_from("<I", value)[0] != 2:
|
||||
raise Reject("EINTEGRITY", "acl.version")
|
||||
entries = []
|
||||
for offset in range(4, len(value), 8):
|
||||
entries.append(list(struct.unpack_from("<HHI", value, offset)))
|
||||
return entries
|
||||
|
||||
|
||||
def execute(case: dict, image_path: Path) -> dict[str, object]:
|
||||
reader = Reader(image_path)
|
||||
prefixes = reader.load_prefixes()
|
||||
if "expected_feature_filter" in case:
|
||||
actual_feature = bool(reader.feature_compat & FEATURE_COMPAT_XATTR_FILTER)
|
||||
if actual_feature != case["expected_feature_filter"]:
|
||||
raise AssertionError("raw xattr filter feature differs")
|
||||
if reader.filter_reserved != case["expected_filter_reserved"]:
|
||||
raise AssertionError("raw xattr filter reserved byte differs")
|
||||
usable = actual_feature and reader.filter_reserved == 0
|
||||
if usable != case["expected_filter_usable"]:
|
||||
raise AssertionError("xattr filter use-site gate differs")
|
||||
operation = case["operation"]
|
||||
result: dict[str, object] = {}
|
||||
if operation == "mount":
|
||||
return result
|
||||
inode = reader.resolve(case["target"])
|
||||
if "expected_acl_filter_negative" in case:
|
||||
body, _, _ = reader.load_body(inode)
|
||||
actual_negative = (
|
||||
reader.u32(body) & ACL_FILTER_BITS
|
||||
) == ACL_FILTER_BITS
|
||||
if actual_negative != case["expected_acl_filter_negative"]:
|
||||
raise AssertionError("ACL name-filter declaration differs")
|
||||
if operation == "get":
|
||||
value, shared_count = reader.getxattr(
|
||||
inode, case["namespace"], case["name"].encode("ascii"), prefixes
|
||||
)
|
||||
result["value_hex"] = value.hex()
|
||||
result["shared_count"] = shared_count
|
||||
if "expected_value_hex" in case and value.hex() != case["expected_value_hex"]:
|
||||
raise AssertionError("xattr value differs")
|
||||
if "expected_shared_count" in case and shared_count != case["expected_shared_count"]:
|
||||
raise AssertionError("xattr shared count differs")
|
||||
elif operation == "list":
|
||||
names, shared_count = reader.listxattr(inode, case["namespace"], prefixes)
|
||||
result["names"] = [name.decode("ascii") for name in names]
|
||||
result["shared_count"] = shared_count
|
||||
elif operation == "acl":
|
||||
value, _ = reader.getxattr(
|
||||
inode, "system", b"posix_acl_access", prefixes
|
||||
)
|
||||
acl = parse_acl(value)
|
||||
result["acl"] = acl
|
||||
if acl != case["expected_acl"]:
|
||||
raise AssertionError("ACL empty-suffix value differs")
|
||||
else:
|
||||
raise AssertionError(f"unknown operation: {operation}")
|
||||
return result
|
||||
|
||||
|
||||
def fsck_accept(path: Path) -> dict[str, object]:
|
||||
fsck = shutil.which("fsck.erofs")
|
||||
if fsck is None:
|
||||
raise SystemExit("fsck.erofs is required for legal B17 fixture baseline")
|
||||
completed = subprocess.run(
|
||||
[fsck, "-d0", "--xattrs", str(path)], check=False, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
)
|
||||
return {"exit": completed.returncode, "stdout": completed.stdout}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--fixtures", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = json.loads(args.spec.read_text(encoding="ascii"))
|
||||
index = json.loads(
|
||||
(args.fixtures / "fixture-index.json").read_text(encoding="ascii")
|
||||
)
|
||||
indexed = {item["id"]: item for item in index["fixtures"]}
|
||||
errnos = spec["freebsd_errno"]
|
||||
results = []
|
||||
failures = []
|
||||
for case in spec["cases"]:
|
||||
image_path = args.fixtures / indexed[case["id"]]["path"]
|
||||
actual_errno = 0
|
||||
actual_errno_name = None
|
||||
actual_reject = "accepted"
|
||||
details: dict[str, object] = {}
|
||||
try:
|
||||
details = execute(case, image_path)
|
||||
except Reject as error:
|
||||
actual_errno_name = error.errno_name
|
||||
actual_errno = errnos[error.errno_name]
|
||||
actual_reject = error.point
|
||||
except Exception as error:
|
||||
failures.append({"id": case["id"], "reason": repr(error)})
|
||||
results.append({"id": case["id"], "status": "FAIL", "exception": repr(error)})
|
||||
continue
|
||||
expected_name = case.get("expected_errno_name")
|
||||
matched = (
|
||||
actual_errno == case["expected_errno"]
|
||||
and actual_errno_name == expected_name
|
||||
and actual_reject == case["expected_reject"]
|
||||
)
|
||||
fsck = None
|
||||
if case.get("fsck_accept"):
|
||||
fsck = fsck_accept(image_path)
|
||||
matched = matched and fsck["exit"] == 0
|
||||
result = {
|
||||
"id": case["id"],
|
||||
"class": case["class"],
|
||||
"status": "PASS" if matched else "FAIL",
|
||||
"expected_errno": case["expected_errno"],
|
||||
"actual_errno": actual_errno,
|
||||
"actual_errno_name": actual_errno_name,
|
||||
"expected_reject": case["expected_reject"],
|
||||
"actual_reject": actual_reject,
|
||||
"details": details,
|
||||
}
|
||||
if fsck is not None:
|
||||
result["fsck_exit"] = fsck["exit"]
|
||||
result["fsck_stdout_sha256"] = hashlib.sha256(
|
||||
fsck["stdout"].encode("utf-8")
|
||||
).hexdigest()
|
||||
results.append(result)
|
||||
if not matched:
|
||||
failures.append(result)
|
||||
report = {
|
||||
"schema": 1,
|
||||
"batch": "B17",
|
||||
"status": "PASS" if not failures else "FAIL",
|
||||
"fixture_count": len(results),
|
||||
"legal_passed": sum(
|
||||
item["status"] == "PASS" and item["class"] == "legal"
|
||||
for item in results
|
||||
),
|
||||
"damaged_passed": sum(
|
||||
item["status"] == "PASS" and item["class"] == "damaged"
|
||||
for item in results
|
||||
),
|
||||
"failures": failures,
|
||||
"results": results,
|
||||
}
|
||||
args.report.write_text(
|
||||
json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": report["status"],
|
||||
"fixtures": report["fixture_count"],
|
||||
"legal_passed": report["legal_passed"],
|
||||
"damaged_passed": report["damaged_passed"],
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user