#!/usr/bin/env python3 """Generate deterministic real EROFS fixtures for B19a.""" from __future__ import annotations import argparse import hashlib import json import os from pathlib import Path import shutil import struct import subprocess SUPER = 1024 MAGIC = 0xE0F5E1E2 CRC32C_POLY = 0x82F63B78 def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: value = seed for byte in data: value ^= byte for _ in range(8): value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) return value & 0xFFFFFFFF def deterministic_value(label: str, length: int) -> bytes: seed = (label + "|").encode("ascii") return (seed * (length // len(seed) + 1))[:length] def load_spec(path: Path) -> dict: spec = json.loads(path.read_text(encoding="ascii")) if ( spec.get("schema") != 1 or spec.get("batch") != "B19a" or spec.get("candidate") != "P15-022" ): raise SystemExit("invalid B19a fixture spec identity") return spec class Image: def __init__(self, data: bytes | bytearray): self.data = bytearray(data) if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC: raise ValueError("invalid EROFS image") self.block_bits = self.data[SUPER + 12] if self.block_bits < 9 or self.block_bits > 16: raise ValueError("invalid EROFS block size") self.block_size = 1 << self.block_bits self.blocks = self.u32(SUPER + 36) self.limit = self.blocks << self.block_bits self.meta_blkaddr = self.u32(SUPER + 40) self.xattr_blkaddr = self.u32(SUPER + 44) self.root_nid = self.u16(SUPER + 14) if self.limit > len(self.data): raise ValueError("EROFS image is truncated") @classmethod def load(cls, path: Path) -> "Image": return cls(path.read_bytes()) def u16(self, offset: int) -> int: if offset < 0 or offset + 2 > len(self.data): raise ValueError("u16 read is out of bounds") return struct.unpack_from(" int: if offset < 0 or offset + 4 > len(self.data): raise ValueError("u32 read is out of bounds") return struct.unpack_from(" dict[str, int]: offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) if offset > self.limit or 64 > self.limit - offset: raise ValueError("inode is out of bounds") inode_format = self.u16(offset) inode_size = 64 if inode_format & 1 else 32 xattr_count = self.u16(offset + 2) xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) size = struct.unpack_from( "> 1) & 7, "size": size, "start_block": self.u32(offset + 16), } def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes: if logical > inode["size"] or length > inode["size"] - logical: raise ValueError("inode data range is out of bounds") if inode["layout"] == 0: physical = (inode["start_block"] << self.block_bits) + logical elif inode["layout"] == 2: tail_start = ((inode["size"] + self.block_size - 1) // self.block_size - 1) * self.block_size if logical < tail_start: if logical + length > tail_start: raise ValueError("inode data read crosses inline tail") physical = (inode["start_block"] << self.block_bits) + logical else: physical = ( inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical - tail_start ) else: raise ValueError("unsupported directory layout") if physical > self.limit or length > self.limit - physical: raise ValueError("inode data is outside the image") return bytes(self.data[physical : physical + length]) @staticmethod def parse_dirblock(data: bytes) -> list[tuple[bytes, int]]: if len(data) < 12: raise ValueError("directory block is short") first_name = struct.unpack_from("= len(data) or first_name % 12: raise ValueError("directory first name offset is invalid") count = first_name // 12 entries = [] previous_offset = 0 previous_name: bytes | None = None for index in range(count): entry = index * 12 nid = struct.unpack_from("= len(data) or (index == 0 and start != first_name) or (index != 0 and start <= previous_offset) or end <= start or end > len(data) ): raise ValueError("directory name bounds are invalid") span = data[start:end] if index + 1 < count: if b"\0" in span: raise ValueError("non-trailing directory name contains NUL") name = span else: name = span.split(b"\0", 1)[0] if not name or len(name) > 255 or b"/" in name: raise ValueError("directory name is invalid") if previous_name is not None and previous_name >= name: raise ValueError("directory names are not strictly ordered") entries.append((name, nid)) previous_offset = start previous_name = name return entries def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]: result = [] previous_name: bytes | None = None logical = 0 while logical < inode["size"]: length = min(self.block_size, inode["size"] - logical) block = self.parse_dirblock(self.inode_data(inode, logical, length)) if previous_name is not None and previous_name >= block[0][0]: raise ValueError("directory block boundary is not ordered") result.extend(block) previous_name = block[-1][0] logical += length return result def resolve(self, path: str) -> dict[str, int]: inode = self.inode(self.root_nid) for component in path.strip("/").encode("ascii").split(b"/"): if not component: continue matches = [nid for name, nid in self.directory_entries(inode) if name == component] if len(matches) != 1: raise ValueError(f"fixture path is absent or ambiguous: {path}") inode = self.inode(matches[0]) return inode def body_offset(self, inode: dict[str, int]) -> int: return inode["offset"] + inode["inode_size"] def body_shape(self, inode: dict[str, int]) -> tuple[int, list[int], list[int]]: body = self.body_offset(inode) size = inode["xattr_size"] if size <= 12 or body > self.limit or size > self.limit - body: raise ValueError("target xattr body is invalid") shared_count = self.data[body + 4] header_size = 12 + shared_count * 4 if header_size > size: raise ValueError("target shared count is invalid") shared_ids = [self.u32(body + 12 + index * 4) for index in range(shared_count)] inline_entries = [] cursor = body + header_size end = body + size while cursor < end: if cursor + 4 > end: raise ValueError("target inline xattr header is truncated") name_length = self.data[cursor] value_length = self.u16(cursor + 2) total = (4 + name_length + value_length + 3) & ~3 if total > end - cursor: raise ValueError("target inline xattr entry is truncated") inline_entries.append(cursor) cursor += total return header_size, shared_ids, inline_entries def update_checksum(self) -> None: if self.u32(SUPER + 8) & 1: struct.pack_into(" None: path.write_bytes(self.data) def set_epoch(path: Path) -> None: for entry in sorted(path.rglob("*"), reverse=True): os.utime(entry, (0, 0), follow_symlinks=False) os.utime(path, (0, 0), follow_symlinks=False) def create_source(path: Path, spec: dict) -> None: path.mkdir(parents=True) path.chmod(0o755) shared_value = deterministic_value( "shared-value-p15-022", spec["fixture"]["shared_value_bytes"] ) inline_value = deterministic_value( "inline-value-p15-022", spec["fixture"]["inline_value_bytes"] ) files = [] for index in range(spec["fixture"]["peer_count"]): prefix = f"peer-{index:03d}-" suffix = "x" * (spec["fixture"]["peer_name_bytes"] - len(prefix) - 4) files.append(path / f"{prefix}{suffix}.bin") target = path / "target.bin" files.append(target) for index, entry in enumerate(files): entry.write_bytes(f"P15-022 file {index:03d}\n".encode("ascii")) entry.chmod(0o644) os.setxattr(entry, b"user.shared", shared_value) if entry == target: os.setxattr(entry, b"user.inline", inline_value) else: os.setxattr( entry, b"user.peer", deterministic_value(f"peer-{index:03d}", 31), ) budget = path / "budget" budget.mkdir() budget.chmod(0o755) for index in range(spec["fixture"]["budget_file_count"]): entry = budget / f"budget-{index:03d}.bin" entry.write_bytes(b"B19a budget\n") entry.chmod(0o644) value = bytearray( deterministic_value( f"budget-value-{index:03d}", spec["fixture"]["budget_value_bytes"], ) ) struct.pack_into(" list[str]: mkfs = shutil.which("mkfs.erofs") if mkfs is None: raise SystemExit("mkfs.erofs is required") command = [ mkfs, "-d0", "-T0", "--all-time", "--all-root", "--workers=1", "--sort=path", f"-U{spec['fixture']['uuid']}", "-x2", "-Eforce-inode-extended", str(output), str(source), ] completed = subprocess.run( command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) if completed.returncode != 0: raise SystemExit( "mkfs.erofs failed: " + completed.stdout.decode("utf-8", "replace") ) return command def mutate(valid: Path, output: Path, mutation: str, spec: dict) -> None: image = Image.load(valid) target = image.resolve(spec["fixture"]["target"]) body = image.body_offset(target) _, shared_ids, inline_entries = image.body_shape(target) if mutation == "corrupt-shared-count": image.data[body + 4] = 255 elif mutation == "corrupt-shared-id": if not shared_ids: raise SystemExit("valid fixture has no shared xattr ID") struct.pack_into(" dict: if output.exists() or work.exists(): raise SystemExit("B19a output and work paths must be absent") output.mkdir(parents=True) source = work / "source" create_source(source, spec) valid = output / "valid.erofs" command = build_image(source, valid, spec) image = Image.load(valid) root = image.inode(image.root_nid) root_entries = image.directory_entries(root) target = image.resolve(spec["fixture"]["target"]) _, shared_ids, inline_entries = image.body_shape(target) if root["size"] <= image.block_size: raise SystemExit("B19a root directory did not span multiple blocks") if not shared_ids or not inline_entries: raise SystemExit("B19a target lacks both shared and inline xattrs") target_positions = [ index for index, (name, _) in enumerate(root_entries) if name == b"target.bin" ] if target_positions != [len(root_entries) - 1]: raise SystemExit("B19a target is not uniquely sorted after all peers") for mutation in ( "corrupt-shared-count", "corrupt-shared-id", "corrupt-inline-name", ): mutate(valid, output / f"{mutation}.erofs", mutation, spec) fsck = shutil.which("fsck.erofs") if fsck is None: raise SystemExit("fsck.erofs is required") completed = subprocess.run( [fsck, str(valid)], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) if completed.returncode != 0: raise SystemExit( "fsck.erofs rejected B19a valid fixture: " + completed.stdout.decode("utf-8", "replace") ) hashes = {path.name: sha256(path) for path in sorted(output.glob("*.erofs"))} frozen = spec.get("fixture_sha256", {}) if frozen and hashes != frozen: raise SystemExit(f"B19a fixture hashes differ: {hashes}") report = { "block_size": image.block_size, "command": [*command[:-2], "OUTPUT/valid.erofs", "WORK/source"], "fixture_sha256": hashes, "inline_entries": len(inline_entries), "root_directory_blocks": (root["size"] + image.block_size - 1) // image.block_size, "root_entry_count": len(root_entries), "shared_count": len(shared_ids), "status": "PASS", "target_entry_index": target_positions[0], } (output / "manifest.json").write_text( json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii" ) return report def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--spec", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--work", type=Path, required=True) args = parser.parse_args() spec = load_spec(args.spec) try: report = generate(args.output, args.work, spec) finally: shutil.rmtree(args.work, ignore_errors=True) print(json.dumps(report, sort_keys=True)) return 0 if __name__ == "__main__": raise SystemExit(main())