#!/usr/bin/env python3 from __future__ import annotations import argparse import hashlib import json import math from pathlib import Path import re import stat import struct import subprocess import sys from typing import Any SCRIPT_DIR = Path(__file__).resolve().parent TESTS_DIR = SCRIPT_DIR.parent.parent sys.path.insert(0, str(TESTS_DIR)) from erofs_fixture import ErofsImage, FEATURE_INCOMPAT_48BIT, SUPER BLOCK_SIZE = 4096 CHUNK_FORMAT_INDEXES = 0x0020 CHUNK_FORMAT_48BIT = 0x0040 CHUNK_FORMAT_ALL = 0x007F FEATURE_INCOMPAT_CHUNKED_FILE = 0x00000004 EROFS_I_DOT_OMITTED_BIT = 4 EROFS_I_NLINK_1_BIT = 5 EROFS_FT_REG_FILE = 1 EROFS_FT_DIR = 2 EROFS_FT_CHRDEV = 3 EROFS_FT_SYMLINK = 7 UUID = "67360174-0014-0000-0000-000000000174" TARGET = b"target.txt" TARGET_CONTENT = b"B14 target content\n" ENTRY_CONTENT = b"B14 directory entry\n" class FixtureError(RuntimeError): pass def run(command: list[str], *, capture: bool = False) -> str: result = subprocess.run( command, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) if result.returncode != 0: raise FixtureError( f"command failed ({result.returncode}): {' '.join(command)}\n{result.stdout}" ) return result.stdout if capture else "" def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for block in iter(lambda: stream.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def align_up(value: int, alignment: int) -> int: return (value + alignment - 1) & ~(alignment - 1) def source_bytes(label: str, size: int) -> bytes: chunks = [] for index in range(math.ceil(size / 32)): chunks.append(hashlib.sha256(f"B14:{label}:{index}".encode("ascii")).digest()) return b"".join(chunks)[:size] def make_source(root: Path) -> None: real_dir = root / "real-dir" real_dir.mkdir(parents=True) (real_dir / "entry.txt").write_bytes(ENTRY_CONTENT) (root / "target.txt").write_bytes(TARGET_CONTENT) (root / "chunk-dir").write_bytes(source_bytes("chunk-dir", 8192)) link_data = TARGET + source_bytes("chunk-link", 8192 - len(TARGET)) (root / "chunk-link").write_bytes(link_data) (root / "chunk-char").write_bytes(source_bytes("chunk-char", 8192)) def make_image(mkfs: str, primary: Path, blob: Path, source: Path) -> None: run( [ mkfs, "-T0", "--all-time", "--all-root", "--workers=1", f"-U{UUID}", "--chunksize=4096", f"--blobdev={blob}", str(primary), str(source), ] ) def root_entry(image: ErofsImage, path: str): _, entry = image.resolve_root_entry(path) return entry def inode_data(image: ErofsImage, inode) -> bytes: if inode.layout == 0: offset = inode.start_block << image.block_bits elif inode.layout == 2: offset = inode.offset + inode.inode_size + inode.xattr_size else: raise FixtureError(f"unsupported source data layout {inode.layout}") end = offset + inode.size if end > len(image.data): raise FixtureError("source inode data exceeds primary image") return bytes(image.data[offset:end]) def chunk_info(image: ErofsImage, path: str) -> dict[str, int]: entry = root_entry(image, path) inode = image.inode(entry.nid) if inode.layout != 4: raise FixtureError(f"{path}: expected chunk layout, got {inode.layout}") chunk_format, reserved = struct.unpack_from(" None: entry = root_entry(image, path) offset = entry.offset + 10 if image.data[offset] != EROFS_FT_REG_FILE: raise FixtureError(f"{path}: root entry is not a regular file") image.data[offset] = file_type def patch_mode(image: ErofsImage, path: str, mode_type: int) -> None: info = chunk_info(image, path) if stat.S_IFMT(info["mode"]) != stat.S_IFREG: raise FixtureError(f"{path}: carrier inode is not regular") image.put_u16(info["inode_offset"] + 4, mode_type | (info["mode"] & 0o7777)) def patch_size(image: ErofsImage, path: str, size: int) -> None: info = chunk_info(image, path) if info["inode_size"] == 32: image.put_u32(info["inode_offset"] + 8, size) else: image.put_u64(info["inode_offset"] + 8, size) def patch_directory(image: ErofsImage, blob: bytearray) -> None: real_entry = root_entry(image, "/real-dir") real_inode = image.inode(real_entry.nid) directory_data = bytearray(inode_data(image, real_inode)) info = chunk_info(image, "/chunk-dir") if info["device_id"] != 1 or info["high"] != 0: raise FixtureError("chunk directory carrier is not on external slot 1") blob_offset = info["low"] << image.block_bits if blob_offset + image.block_size > len(blob): raise FixtureError("chunk directory carrier exceeds blob") first_name_offset = struct.unpack_from(" None: patch_mode(image, "/chunk-link", stat.S_IFLNK) patch_size(image, "/chunk-link", len(TARGET)) patch_root_type(image, "/chunk-link", EROFS_FT_SYMLINK) def patch_character(image: ErofsImage) -> None: patch_mode(image, "/chunk-char", stat.S_IFCHR) patch_size(image, "/chunk-char", 0) patch_root_type(image, "/chunk-char", EROFS_FT_CHRDEV) def enable_48bit(image: ErofsImage) -> None: if image.feature_incompat & FEATURE_INCOMPAT_48BIT: raise FixtureError("48-bit feature already enabled") root_nid = image.root_nid image.put_u32(SUPER + 80, image.feature_incompat | FEATURE_INCOMPAT_48BIT) image.put_u16(SUPER + 14, 0) image.put_u64(SUPER + 112, root_nid) for path in ("/chunk-dir", "/chunk-link", "/chunk-char"): info = chunk_info(image, path) if info["entry_size"] != 8 or info["high"] != 0: raise FixtureError(f"{path}: cannot promote chunk index to 48-bit") image.put_u16(info["inode_offset"] + 16, info["format"] | CHUNK_FORMAT_48BIT) def save_image(image: ErofsImage, path: Path) -> None: if image.feature_compat & 1: image.update_checksum() image.validate_superblock() image.save(path) def patch_index_low(image: ErofsImage, path: str, value: int) -> None: info = chunk_info(image, path) if info["entry_size"] != 8: raise FixtureError("bad-index fixture requires an indexed chunk") image.put_u32(info["index_base"] + 4, value) def patch_index48(image: ErofsImage, path: str, high: int, low: int) -> None: info = chunk_info(image, path) if info["entry_size"] != 8 or not info["format"] & CHUNK_FORMAT_48BIT: raise FixtureError("bad-index48 fixture requires a 48-bit chunk index") image.put_u16(info["index_base"], high) image.put_u32(info["index_base"] + 4, low) def load_spec(path: Path) -> dict[str, Any]: spec = json.loads(path.read_text(encoding="ascii")) if spec.get("schema") != 1 or spec.get("batch") != "B14": raise FixtureError("invalid B14 fixture schema") if spec.get("candidate") != "P15-056" or spec.get("test") != "TC174-chunk-nonregular": raise FixtureError("invalid B14 fixture identity") cases = spec.get("cases") if not isinstance(cases, list) or len(cases) != 10: raise FixtureError("B14 fixture case denominator changed") identifiers = [case.get("id") for case in cases] if len(set(identifiers)) != len(identifiers) or not all(isinstance(value, str) for value in identifiers): raise FixtureError("invalid or duplicate B14 case ID") return spec def write_sums(output: Path, names: list[str]) -> dict[str, str]: hashes = {name: sha256(output / name) for name in sorted(names)} with (output / "SHA256SUMS").open("w", encoding="ascii") as stream: for name, digest in hashes.items(): stream.write(f"{digest} {name}\n") return hashes def generate(args: argparse.Namespace) -> None: output = args.output if output.exists(): raise FixtureError(f"output already exists: {output}") spec = load_spec(args.spec) output.mkdir(parents=True) source = output / "source" source.mkdir() make_source(source) raw_primary = output / "raw.erofs" blob_path = output / "B14-slot1.blob" blob_path.write_bytes(b"") make_image(args.mkfs, raw_primary, blob_path, source) image = ErofsImage.load(raw_primary) image.validate_superblock() if not image.feature_incompat & FEATURE_INCOMPAT_CHUNKED_FILE: raise FixtureError("mkfs image lacks CHUNKED_FILE") blob = bytearray(blob_path.read_bytes()) for path in ("/chunk-dir", "/chunk-link", "/chunk-char"): info = chunk_info(image, path) if info["entry_size"] != 8 or info["device_id"] != 1 or info["high"] != 0: raise FixtureError(f"{path}: unexpected mkfs chunk carrier") patch_directory(image, blob) patch_symlink(image) patch_character(image) blob_path.write_bytes(blob) good32 = image.clone() save_image(good32, output / "B14-good32.erofs") good48 = image.clone() enable_48bit(good48) save_image(good48, output / "B14-good48.erofs") blob_blocks = len(blob) // BLOCK_SIZE bad_index = good32.clone() patch_index_low(bad_index, "/chunk-link", blob_blocks + 1) save_image(bad_index, output / "B14-bad-index.erofs") bad_index48 = good48.clone() patch_index48(bad_index48, "/chunk-link", 0xFFFF, 0xFFFFFFFE) save_image(bad_index48, output / "B14-bad-index48.erofs") bad_reserved = good32.clone() directory = chunk_info(bad_reserved, "/chunk-dir") bad_reserved.put_u16(directory["inode_offset"] + 18, 1) save_image(bad_reserved, output / "B14-bad-reserved.erofs") bad_format = good32.clone() link = chunk_info(bad_format, "/chunk-link") bad_format.put_u16(link["inode_offset"] + 16, link["format"] | (CHUNK_FORMAT_ALL + 1)) save_image(bad_format, output / "B14-bad-format.erofs") bad_no_index = good48.clone() link48 = chunk_info(bad_no_index, "/chunk-link") bad_no_index.put_u16( link48["inode_offset"] + 16, (link48["format"] | CHUNK_FORMAT_48BIT) & ~CHUNK_FORMAT_INDEXES, ) save_image(bad_no_index, output / "B14-bad-48-no-index.erofs") raw_primary.unlink() names = [ "B14-slot1.blob", "B14-good32.erofs", "B14-good48.erofs", "B14-bad-index.erofs", "B14-bad-index48.erofs", "B14-bad-reserved.erofs", "B14-bad-format.erofs", "B14-bad-48-no-index.erofs", ] hashes = write_sums(output, names) manifest = { "schema": 1, "batch": "B14", "candidate": "P15-056", "test": "TC174-chunk-nonregular", "mkfs_version": run([args.mkfs, "-V"], capture=True).splitlines()[0], "hashes": hashes, "blob_blocks": blob_blocks, "target": TARGET.decode("ascii"), "qualification": { "good32": "fsck.erofs plus independent parser", "good48": "independent parser; erofs-utils 1.8.6 lacks 48-bit support", }, "cases": [case["id"] for case in spec["cases"]], "good32": { path: chunk_info(good32, path) for path in ("/chunk-dir", "/chunk-link", "/chunk-char") }, "good48": { path: chunk_info(good48, path) for path in ("/chunk-dir", "/chunk-link", "/chunk-char") }, } (output / "manifest.json").write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="ascii" ) def verify(args: argparse.Namespace) -> None: spec = load_spec(args.spec) output = args.output manifest = json.loads((output / "manifest.json").read_text(encoding="ascii")) if manifest.get("cases") != [case["id"] for case in spec["cases"]]: raise FixtureError("manifest case list differs from B14 spec") for name, expected in manifest.get("hashes", {}).items(): if sha256(output / name) != expected: raise FixtureError(f"fixture hash mismatch: {name}") good32 = ErofsImage.load(output / "B14-good32.erofs") good48 = ErofsImage.load(output / "B14-good48.erofs") blob = (output / "B14-slot1.blob").read_bytes() for image, width in ((good32, 32), (good48, 48)): image.validate_superblock() directory = chunk_info(image, "/chunk-dir") link = chunk_info(image, "/chunk-link") character = chunk_info(image, "/chunk-char") for info in (directory, link, character): if info["device_id"] != 1 or info["entry_size"] != 8: raise FixtureError(f"{width}-bit external chunk contract changed") has_48bit = bool(info["format"] & CHUNK_FORMAT_48BIT) if has_48bit != (width == 48) or info["high"] != 0: raise FixtureError(f"{width}-bit chunk format contract changed") if stat.S_IFMT(directory["mode"]) != stat.S_IFDIR: raise FixtureError("chunk directory mode changed") if stat.S_IFMT(link["mode"]) != stat.S_IFLNK or link["size"] != len(TARGET): raise FixtureError("chunk symlink mode or size changed") if stat.S_IFMT(character["mode"]) != stat.S_IFCHR or character["size"] != 0: raise FixtureError("chunk character mode or size changed") link_offset = link["low"] << image.block_bits if blob[link_offset : link_offset + len(TARGET)] != TARGET: raise FixtureError("chunk symlink payload changed") directory_offset = directory["low"] << image.block_bits directory_data = blob[directory_offset : directory_offset + directory["size"]] first_name_offset = struct.unpack_from(" str: match = re.search( r"^(?:[A-Za-z_][^\n;{}]*\s+)?" + re.escape(name) + r"\s*\(", source, re.MULTILINE, ) if not match: raise FixtureError(f"missing function: {name}") start = source.rfind("\n\n", 0, match.start()) + 2 brace = source.find("{", match.end()) depth = 0 state = "code" index = brace while index < len(source): char = source[index] following = source[index + 1] if index + 1 < len(source) else "" if state == "code": if char == "/" and following == "*": state = "block" index += 2 continue if char == "/" and following == "/": state = "line" index += 2 continue if char == '"': state = "string" elif char == "'": state = "character" elif char == "{": depth += 1 elif char == "}": depth -= 1 if depth == 0: return source[start : index + 1] elif state == "block" and char == "*" and following == "/": state = "code" index += 2 continue elif state == "line" and char == "\n": state = "code" elif state in {"string", "character"}: if char == "\\": index += 2 continue if (state == "string" and char == '"') or (state == "character" and char == "'"): state = "code" index += 1 raise FixtureError(f"unterminated function: {name}") def committed(root: Path, baseline: str, path: str) -> str: result = subprocess.run( ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) if result.returncode != 0: raise FixtureError(f"cannot read B14 baseline {path}: {result.stderr}") return result.stdout def audit(args: argparse.Namespace) -> None: load_spec(args.spec) root = args.root dut = args.dut inode = (dut / "src/inode.c").read_text(encoding="utf-8") data = (dut / "src/data.c").read_text(encoding="utf-8") vnops = (dut / "src/erofs_vnops.c").read_text(encoding="utf-8") linux_inode = (root / "src-linux/inode.c").read_text(encoding="utf-8") base_inode = committed(root, args.baseline, "src/inode.c") base_data = committed(root, args.baseline, "src/data.c") old_gate = "\t\tif (!erofs_sb_has_chunked_file(sbi) || vi->vtype != VREG) {" new_gate = "\t\tif (!erofs_sb_has_chunked_file(sbi)) {" if base_inode.count(old_gate) != 1: raise FixtureError("B14 baseline type gate changed") expected_inode = base_inode.replace(old_gate, new_gate, 1) if inode != expected_inode: raise FixtureError("inode.c differs from the single declared B14 transform") if data != base_data: raise FixtureError("data.c changed although the chunk mapper is already type-neutral") read_inode = extract_function(inode, "erofs_read_inode") chunk_mapper = extract_function(data, "erofs_map_blocks_chunk") map_device = extract_function(data, "erofs_map_dev") open_vnode = extract_function(vnops, "erofs_open") read_vnode = extract_function(vnops, "erofs_read") readdir_vnode = extract_function(vnops, "erofs_readdir") readlink_vnode = extract_function(vnops, "erofs_readlink") linux_read_inode = extract_function(linux_inode, "erofs_read_inode") required_inode = ( "!erofs_sb_has_chunked_file(sbi)", "le16toh(chunk_info.reserved) != 0", "vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL", "EROFS_CHUNK_FORMAT_48BIT", "EROFS_CHUNK_FORMAT_INDEXES", "vi->chunkbits >= 64", "case VREG:", "case VDIR:", "case VLNK:", ) for marker in required_inode: if marker not in read_inode: raise FixtureError(f"B14 removed inode invariant: {marker}") if "vi->vtype != VREG" in read_inode: raise FixtureError("nonregular chunk type gate remains") if read_inode.count("return (EINTEGRITY);") < 8 or "return (EOPNOTSUPP);" not in read_inode: raise FixtureError("FreeBSD inode errno contract changed") required_mapper = ( "vi->chunkbits", "vi->chunkformat", "vi->inode_off", "vi->inode_isize", "vi->xattr_isize", "map->m_deviceid = raw_device_id & sbi->device_id_mask", "map->m_flags |= EROFS_MAP_MAPPED", ) for marker in required_mapper: if marker not in chunk_mapper: raise FixtureError(f"chunk mapper contract changed: {marker}") if "vtype" in chunk_mapper or "VREG" in chunk_mapper or "VLNK" in chunk_mapper or "VDIR" in chunk_mapper: raise FixtureError("chunk mapper acquired an OS vnode type dependency") if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", chunk_mapper + map_device): raise FixtureError("negative Linux errno entered the FreeBSD map path") for marker in ("ENODEV", "dif->devvp", "dif->cp", "erofs_check_device_range"): if marker not in map_device: raise FixtureError(f"FreeBSD device/GEOM boundary changed: {marker}") if "VN_ISDEV(vp)" not in open_vnode or "return (EOPNOTSUPP);" not in open_vnode: raise FixtureError("special vnode open safety changed") if "case VREG:" not in read_vnode or "case VDIR:" not in read_vnode: raise FixtureError("FreeBSD VOP_READ type boundary changed") if "v_type != VDIR" not in readdir_vnode or "v_type != VLNK" not in readlink_vnode: raise FixtureError("FreeBSD readdir/readlink type boundary changed") for marker in ("case S_IFDIR:", "case S_IFREG:", "case S_IFLNK:"): if marker not in linux_read_inode: raise FixtureError(f"Linux common data-layout type marker changed: {marker}") linux_chunk = linux_read_inode[linux_read_inode.index("if (vi->datalayout == EROFS_INODE_CHUNK_BASED)") :] if "vi->chunkformat" not in linux_chunk or "S_ISREG" in linux_chunk.split("inode_set_atime", 1)[0]: raise FixtureError("Linux chunk summary is no longer type-neutral") report = { "schema": 1, "batch": "B14", "candidate": "P15-056", "baseline": args.baseline, "source_transform_count": 1, "data_c_unchanged": True, "linux_common_types": ["S_IFDIR", "S_IFREG", "S_IFLNK"], "freebsd_vnode_types": ["VDIR", "VREG", "VLNK"], "freebsd_boundaries": [ "positive errno", "explicit GEOM device slots", "special vnode open EOPNOTSUPP", "typed readdir/readlink VOPs", ], } args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command", required=True) generate_parser = subparsers.add_parser("generate") generate_parser.add_argument("--output", type=Path, required=True) generate_parser.add_argument("--spec", type=Path, required=True) generate_parser.add_argument("--mkfs", default="mkfs.erofs") verify_parser = subparsers.add_parser("verify") verify_parser.add_argument("--output", type=Path, required=True) verify_parser.add_argument("--spec", type=Path, required=True) verify_parser.add_argument("--fsck", default="fsck.erofs") audit_parser = subparsers.add_parser("audit") audit_parser.add_argument("--root", type=Path, required=True) audit_parser.add_argument("--dut", type=Path, required=True) audit_parser.add_argument("--baseline", required=True) audit_parser.add_argument("--spec", type=Path, required=True) audit_parser.add_argument("--report", type=Path, required=True) return parser.parse_args() def main() -> None: args = parse_args() if args.command == "generate": generate(args) elif args.command == "verify": verify(args) else: audit(args) if __name__ == "__main__": main()