imrcpan/erofs-freebsd-out-tree
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
imrcpan
2026-08-18 09:20:44 +02:00
commit b826cd721a
522 changed files with 93730 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/pre15/fixtures/B19b-xattr-oracle.py
Executable
+426
View File
@@ -0,0 +1,426 @@
#!/usr/bin/env python3
"""Independently verify B19b fixtures, Bloom semantics, and source scope."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import re
import statistics
import struct
import subprocess
SUPER = 1024
MAGIC = 0xE0F5E1E2
class Reject(RuntimeError):
def __init__(self, errno_name: str, point: str):
super().__init__(f"{errno_name} at {point}")
self.errno_name = errno_name
self.point = point
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def write_json(path: Path, value: object) -> None:
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
def rotl32(value: int, count: int) -> int:
return ((value << count) | (value >> (32 - count))) & 0xFFFFFFFF
def xxh32(data: bytes, seed: int) -> int:
prime1 = 2654435761
prime2 = 2246822519
prime3 = 3266489917
prime4 = 668265263
prime5 = 374761393
cursor = 0
def round32(accumulator: int, lane: int) -> int:
accumulator = (accumulator + lane * prime2) & 0xFFFFFFFF
return (rotl32(accumulator, 13) * prime1) & 0xFFFFFFFF
if len(data) >= 16:
accumulator1 = (seed + prime1 + prime2) & 0xFFFFFFFF
accumulator2 = (seed + prime2) & 0xFFFFFFFF
accumulator3 = seed & 0xFFFFFFFF
accumulator4 = (seed - prime1) & 0xFFFFFFFF
limit = len(data) - 16
while cursor <= limit:
accumulator1 = round32(accumulator1, int.from_bytes(data[cursor:cursor + 4], "little"))
accumulator2 = round32(accumulator2, int.from_bytes(data[cursor + 4:cursor + 8], "little"))
accumulator3 = round32(accumulator3, int.from_bytes(data[cursor + 8:cursor + 12], "little"))
accumulator4 = round32(accumulator4, int.from_bytes(data[cursor + 12:cursor + 16], "little"))
cursor += 16
value = (
rotl32(accumulator1, 1)
+ rotl32(accumulator2, 7)
+ rotl32(accumulator3, 12)
+ rotl32(accumulator4, 18)
) & 0xFFFFFFFF
else:
value = (seed + prime5) & 0xFFFFFFFF
value = (value + len(data)) & 0xFFFFFFFF
while cursor + 4 <= len(data):
value = (value + int.from_bytes(data[cursor:cursor + 4], "little") * prime3) & 0xFFFFFFFF
value = (rotl32(value, 17) * prime4) & 0xFFFFFFFF
cursor += 4
while cursor < len(data):
value = (value + data[cursor] * prime5) & 0xFFFFFFFF
value = (rotl32(value, 11) * prime1) & 0xFFFFFFFF
cursor += 1
value ^= value >> 15
value = (value * prime2) & 0xFFFFFFFF
value ^= value >> 13
value = (value * prime3) & 0xFFFFFFFF
value ^= value >> 16
return value & 0xFFFFFFFF
class Reader:
def __init__(self, path: Path, spec: dict):
self.data = path.read_bytes()
self.spec = spec
self.calls = 0
self.bytes = 0
self.blocks_read: set[int] = set()
header = self.read(SUPER, 144, "super")
if struct.unpack_from("<I", header)[0] != MAGIC:
raise Reject("EINTEGRITY", "super.magic")
self.feature_compat = struct.unpack_from("<I", header, 8)[0]
self.block_bits = header[12]
self.block_size = 1 << self.block_bits
self.root_nid = struct.unpack_from("<H", header, 14)[0]
self.blocks = struct.unpack_from("<I", header, 36)[0]
self.limit = self.blocks << self.block_bits
self.meta_blkaddr = struct.unpack_from("<I", header, 40)[0]
self.xattr_blkaddr = struct.unpack_from("<I", header, 44)[0]
self.filter_reserved = header[104]
if self.limit > len(self.data):
raise Reject("EINTEGRITY", "super.bounds")
def read(self, offset: int, length: int, point: str) -> bytes:
limit = self.limit if hasattr(self, "limit") else len(self.data)
if offset < 0 or length < 0 or offset > limit or length > limit - offset:
raise Reject("EINTEGRITY", f"{point}.bounds")
self.calls += 1
self.bytes += length
if length:
self.blocks_read.update(range(offset // 4096, (offset + length - 1) // 4096 + 1))
return self.data[offset:offset + length]
def reset_reads(self) -> None:
self.calls = 0
self.bytes = 0
self.blocks_read.clear()
def inode(self, nid: int) -> dict[str, int]:
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
raw = self.read(offset, 64, "inode")
inode_format, xattr_count = struct.unpack_from("<HH", raw)
inode_size = 64 if inode_format & 1 else 32
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
size = struct.unpack_from("<Q" if inode_size == 64 else "<I", raw, 8)[0]
return {
"offset": offset,
"inode_size": inode_size,
"xattr_size": xattr_size,
"layout": (inode_format >> 1) & 7,
"size": size,
"start_block": struct.unpack_from("<I", raw, 16)[0],
}
def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
if inode["layout"] == 2:
physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical
elif inode["layout"] == 0:
physical = (inode["start_block"] << self.block_bits) + logical
else:
raise Reject("EINTEGRITY", "directory.layout")
return self.read(physical, length, "directory.data")
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
entries = []
logical = 0
while logical < inode["size"]:
length = min(self.block_size, inode["size"] - logical)
data = self.inode_data(inode, logical, length)
if len(data) < 12:
raise Reject("EINTEGRITY", "directory.header")
first_name = struct.unpack_from("<H", data, 8)[0]
if first_name == 0 or first_name % 12 or first_name > len(data):
raise Reject("EINTEGRITY", "directory.name-offset")
count = first_name // 12
for index in range(count):
offset = index * 12
nid = struct.unpack_from("<Q", data, offset)[0]
start = struct.unpack_from("<H", data, offset + 8)[0]
end = struct.unpack_from("<H", data, offset + 20)[0] if index + 1 < count else len(data)
if start > end or end > len(data):
raise Reject("EINTEGRITY", "directory.name-bounds")
entries.append((data[start:end].split(b"\0", 1)[0], nid))
logical += length
return entries
def resolve(self, path: str) -> dict[str, int]:
inode = self.inode(self.root_nid)
for component in path.strip("/").encode().split(b"/"):
matches = [nid for name, nid in self.directory_entries(inode) if name == component]
if len(matches) != 1:
raise Reject("ENOATTR", "path.lookup")
inode = self.inode(matches[0])
return inode
def stats(self, scanned: bool) -> dict[str, int | bool]:
return {
"bytes": self.bytes,
"calls": self.calls,
"provider_blocks": len(self.blocks_read),
"scanned": scanned,
}
def lookup(self, inode: dict[str, int], name: bytes, candidate: bool) -> tuple[str, bytes | None, dict]:
self.reset_reads()
body_offset = inode["offset"] + inode["inode_size"]
usable = self.feature_compat & self.spec["format"]["feature_compat"] and self.filter_reserved == 0
if candidate and usable:
if inode["xattr_size"] < 12:
raise Reject("EINTEGRITY", "ibody.header")
if inode["xattr_size"] == 12:
raise Reject("EOPNOTSUPP", "ibody.header-only")
header = self.read(body_offset, 12, "ibody.header")
shared_count = header[4]
if 12 + shared_count * 4 > inode["xattr_size"]:
raise Reject("EINTEGRITY", "ibody.shared-count")
name_filter = struct.unpack_from("<I", header)[0]
bit = xxh32(name, self.spec["format"]["seed"] + 1) & 31
if name_filter & (1 << bit):
return "ENOATTR", None, self.stats(False)
body = self.read(body_offset, inode["xattr_size"], "ibody")
if len(body) < 12:
raise Reject("EINTEGRITY", "ibody.header")
if len(body) == 12:
raise Reject("EOPNOTSUPP", "ibody.header-only")
shared_count = body[4]
header_size = 12 + shared_count * 4
if header_size > len(body):
raise Reject("EINTEGRITY", "ibody.shared-count")
cursor = header_size
while cursor < len(body):
if cursor + 4 > len(body):
raise Reject("EINTEGRITY", "inline.header")
name_length, name_index, value_length = struct.unpack_from("<BBH", body, cursor)
total = (4 + name_length + value_length + 3) & ~3
if total > len(body) - cursor:
raise Reject("EINTEGRITY", "inline.bounds")
actual = body[cursor + 4:cursor + 4 + name_length]
if b"\0" in actual:
raise Reject("EINTEGRITY", "inline.name-nul")
if name_index == 1 and actual == name:
start = cursor + 4 + name_length
return "PASS", body[start:start + value_length], self.stats(True)
cursor += total
for index in range(shared_count):
shared_id = struct.unpack_from("<I", body, 12 + index * 4)[0]
offset = (self.xattr_blkaddr << self.block_bits) + shared_id * 4
header = self.read(offset, 4, "shared.header")
name_length, name_index, value_length = struct.unpack("<BBH", header)
total = (4 + name_length + value_length + 3) & ~3
raw = self.read(offset, total, "shared.entry")
actual = raw[4:4 + name_length]
if b"\0" in actual:
raise Reject("EINTEGRITY", "shared.name-nul")
if name_index == 1 and actual == name:
return "PASS", raw[4 + name_length:4 + name_length + value_length], self.stats(True)
return "ENOATTR", None, self.stats(True)
def lookup(path: Path, name: bytes, candidate: bool, spec: dict) -> dict:
try:
reader = Reader(path, spec)
inode = reader.resolve(spec["fixture"]["target"])
status, value, reads = reader.lookup(inode, name, candidate)
return {"errno": 0 if status == "PASS" else spec["errno"][status], "reads": reads, "status": status, "value_hex": None if value is None else value.hex()}
except Reject as error:
return {"errno": spec["errno"].get(error.errno_name, -1), "point": error.point, "status": error.errno_name, "value_hex": None}
def function(source: str, name: str) -> str:
match = re.search(rf"\n{name}\([^;]*?\n\{{", source, re.DOTALL)
if match is None:
raise SystemExit(f"missing source function: {name}")
start = match.start() + 1
brace = source.index("{", match.start())
depth = 0
for index in range(brace, len(source)):
if source[index] == "{":
depth += 1
elif source[index] == "}":
depth -= 1
if depth == 0:
return source[start:index + 1]
raise SystemExit(f"unterminated source function: {name}")
def audit_source(root: Path, dut: Path, spec: dict) -> dict:
header = (dut / "src/erofs_fs.h").read_text(encoding="utf-8")
internal = (dut / "src/internal.h").read_text(encoding="utf-8")
xattr = (dut / "src/xattr.c").read_text(encoding="utf-8")
linux_header = (root / "src-linux/erofs_fs.h").read_text(encoding="utf-8")
linux_xattr = (root / "src-linux/xattr.c").read_text(encoding="utf-8")
for marker in (
"#define EROFS_XATTR_FILTER_BITS",
"#define EROFS_XATTR_FILTER_DEFAULT",
"#define EROFS_XATTR_FILTER_SEED",
):
if marker not in header or marker not in linux_header:
raise SystemExit(f"Bloom format marker missing: {marker}")
if "erofs_sb_has_xattr_filter_v1" not in internal or "xattr_filter_reserved == 0" not in internal:
raise SystemExit("FreeBSD unknown-filter fallback helper is incomplete")
hash_body = function(xattr, "erofs_xxh32")
filter_name = function(xattr, "erofs_xattr_filter_name")
filter_negative = function(xattr, "erofs_xattr_filter_negative")
getxattr = function(xattr, "erofs_getxattr")
if "static uint32_t\nerofs_xxh32" not in xattr:
raise SystemExit("xxh32 is not file-local and namespaced")
for marker in ("le32dec(cursor)", "UINT32_C(2654435761)", "hash ^= hash >> 16"):
if marker not in hash_body:
raise SystemExit(f"xxh32 source marker missing: {marker}")
for marker in (
"EXTATTR_NAMESPACE_USER",
"EXTATTR_NAMESPACE_SYSTEM",
"EROFS_XATTR_INDEX_USER",
"EROFS_XATTR_INDEX_POSIX_ACL_ACCESS",
"EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT",
"EROFS_XATTR_INDEX_TRUSTED",
"EROFS_XATTR_INDEX_SECURITY",
):
if marker not in filter_name:
raise SystemExit(f"FreeBSD namespace mapping marker missing: {marker}")
for marker in (
"erofs_sb_has_xattr_filter_v1(sbi)",
"header_size > vi->xattr_isize",
"erofs_xxh32(filter_name, filter_name_len",
"erofs_put_metabuf(&buf)",
):
if marker not in filter_negative:
raise SystemExit(f"fast-negative integrity marker missing: {marker}")
if any(marker in filter_negative.lower() for marker in ("malloc", "mtx_", "cv_", "cache")):
raise SystemExit("B19b introduced cache/allocation/locking into the fast-negative")
if getxattr.index("erofs_xattr_filter_negative") > getxattr.index("erofs_xattr_load_body"):
raise SystemExit("fast-negative runs after the full xattr body read")
if "if (filter_negative)\n\t\treturn (ENOATTR);" not in getxattr:
raise SystemExit("getxattr does not restrict the shortcut to proven negatives")
if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", header + internal + xattr):
raise SystemExit("Linux negative errno entered the FreeBSD implementation")
for marker in (
"xxh32(name, strlen(name)",
"EROFS_XATTR_FILTER_SEED + index",
"vi->xattr_name_filter & (1U << hashbit)",
"!sbi->xattr_filter_reserved",
):
if marker not in linux_xattr:
raise SystemExit(f"Linux comparison anchor changed: {marker}")
changed = subprocess.run(
["git", "-C", str(root), "diff", "--name-only", spec["gate"]["commit"], "--", "repo-pre-15/src"],
check=True,
text=True,
stdout=subprocess.PIPE,
).stdout.splitlines()
expected = ["repo-pre-15/src/erofs_fs.h", "repo-pre-15/src/internal.h", "repo-pre-15/src/xattr.c"]
if sorted(changed) != expected:
raise SystemExit(f"B19b source write set differs: {changed}")
return {
"freebsd_errno": "positive",
"freebsd_namespace": "extattr user/system mapping retained",
"hash_symbol": "file-local erofs_xxh32",
"linux_semantics": "xxh32(name suffix, seed + index), inverse 32-bit filter",
"source_write_set": expected,
"status": "PASS",
}
def verify(fixtures: Path, spec: dict, root: Path, dut: Path) -> dict:
hashes = {path.name: sha256(path) for path in sorted(fixtures.glob("*.erofs"))}
if hashes != spec["fixture_sha256"]:
raise SystemExit(f"fixture hashes differ: {hashes}")
cases = [
("hit", "valid.erofs", b"attr00", "PASS", True),
("miss", "valid.erofs", spec["fixture"]["miss"].encode(), "ENOATTR", False),
("false-positive", "valid.erofs", spec["fixture"]["collision"].encode(), "ENOATTR", True),
("unknown-filter", "unknown-filter.erofs", spec["fixture"]["miss"].encode(), "ENOATTR", True),
("feature-off", "feature-off.erofs", spec["fixture"]["miss"].encode(), "ENOATTR", True),
("corrupt-shared-count", "corrupt-shared-count.erofs", spec["fixture"]["miss"].encode(), "EINTEGRITY", None),
("corrupt-shared-id", "corrupt-shared-id.erofs", b"attr00", "EINTEGRITY", None),
]
results = []
for identifier, image, name, expected_status, expected_scan in cases:
baseline = lookup(fixtures / image, name, False, spec)
candidate = lookup(fixtures / image, name, True, spec)
if baseline["status"] != expected_status or candidate["status"] != expected_status:
raise SystemExit(f"{identifier} status differs: {baseline}, {candidate}")
if baseline["errno"] < 0 or candidate["errno"] < 0:
raise SystemExit(f"{identifier} returned negative errno")
if expected_scan is not None and candidate["reads"]["scanned"] != expected_scan:
raise SystemExit(f"{identifier} scan decision differs")
results.append({"baseline": baseline, "candidate": candidate, "id": identifier, "name": name.decode()})
baseline_calls = []
candidate_calls = []
baseline_blocks = []
candidate_blocks = []
for _ in range(spec["thresholds"]["samples"]):
baseline = lookup(fixtures / "valid.erofs", spec["fixture"]["miss"].encode(), False, spec)["reads"]
candidate = lookup(fixtures / "valid.erofs", spec["fixture"]["miss"].encode(), True, spec)["reads"]
baseline_calls.append(baseline["calls"])
candidate_calls.append(candidate["calls"])
baseline_blocks.append(baseline["provider_blocks"])
candidate_blocks.append(candidate["provider_blocks"])
call_reduction = 100 * (statistics.median(baseline_calls) - statistics.median(candidate_calls)) / statistics.median(baseline_calls)
block_reduction = 100 * (statistics.median(baseline_blocks) - statistics.median(candidate_blocks)) / statistics.median(baseline_blocks)
threshold = spec["thresholds"]["minimum_provider_metadata_read_reduction_percent"]
if call_reduction < threshold or block_reduction < threshold:
raise SystemExit("B19b source model no longer meets the measured benefit gate")
return {
"benchmark": {
"baseline_calls": baseline_calls,
"baseline_provider_blocks": baseline_blocks,
"call_reduction_percent": call_reduction,
"candidate_calls": candidate_calls,
"candidate_provider_blocks": candidate_blocks,
"provider_block_reduction_percent": block_reduction,
"threshold_percent": threshold,
},
"cases": results,
"fixture_sha256": hashes,
"source": audit_source(root, dut, spec),
"status": "PASS",
"test": spec["test"],
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--dut", type=Path, required=True)
parser.add_argument("--fixtures", type=Path, required=True)
parser.add_argument("--report", type=Path, required=True)
parser.add_argument("--root", type=Path, required=True)
parser.add_argument("--spec", type=Path, required=True)
args = parser.parse_args()
spec = json.loads(args.spec.read_text(encoding="ascii"))
if spec.get("schema") != 1 or spec.get("batch") != "B19b":
raise SystemExit("invalid B19b oracle spec")
report = verify(args.fixtures, spec, args.root, args.dut)
write_json(args.report, report)
print(json.dumps({"case_count": len(report["cases"]), "status": "PASS", "test": report["test"]}, sort_keys=True))
if __name__ == "__main__":
main()