#!/usr/bin/env python3 """Generate and verify assertion-driven G6 chunk/multidevice fixtures.""" from __future__ import annotations import argparse import hashlib import json import math import os import shutil import struct import subprocess import tempfile from dataclasses import dataclass from pathlib import Path from typing import Any, Callable from erofs_fixture import ErofsImage, SUPER FEATURE_INCOMPAT_CHUNKED_FILE = 0x00000004 FEATURE_INCOMPAT_DEVICE_TABLE = 0x00000008 FEATURE_INCOMPAT_48BIT = 0x00000080 CHUNK_FORMAT_BLOCK_BITS_MASK = 0x001F CHUNK_FORMAT_INDEXES = 0x0020 CHUNK_FORMAT_48BIT = 0x0040 DEVICE_SLOT_SIZE = 128 DEVICE_SLOT_FIELDS = 64 BLOCK_SIZE = 4096 CHUNK_UUID = "67360006-0093-0094-0095-000000000101" FRAGMENT_UUID = "67360098-0000-0000-0000-000000000098" PCLUSTER_UUID = "67360094-0101-0000-0000-000000000094" CHUNK_FILES = { "tc006.bin": 24, "cross-device.bin": 96, "indexed.bin": 40, "cold-slot2.bin": 32, "unified-address.bin": 32, } class FixtureError(RuntimeError): pass @dataclass(frozen=True) class ChunkEntry: offset: int start_block_high: int device_id: int start_block_low: int @property def start_block(self) -> int: return self.start_block_low | (self.start_block_high << 32) @dataclass(frozen=True) class ChunkLayout: path: str nid: int inode_offset: int format_offset: int chunk_format: int chunk_bits: int chunk_size: int entry_size: int index_base: int entries: tuple[ChunkEntry, ...] @dataclass(frozen=True) class DeviceSlot: blocks: int uniaddr: int @dataclass class MultiFixture: primary: ErofsImage providers: list[bytearray] slots: list[DeviceSlot] def align_up(value: int, alignment: int) -> int: return (value + alignment - 1) & ~(alignment - 1) def sha256_bytes(data: bytes | bytearray) -> str: return hashlib.sha256(data).hexdigest() def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def deterministic_block(path: str, block: int) -> bytes: seed = f"repo22-g6:{path}:{block}".encode("ascii") return b"".join( hashlib.sha256(seed + index.to_bytes(4, "little")).digest() for index in range(128) )[:BLOCK_SIZE] def run( command: list[str], *, stdout: bool = False, cwd: Path | None = None ) -> str: result = subprocess.run( command, check=True, cwd=cwd, stdout=subprocess.PIPE if stdout else subprocess.DEVNULL, stderr=subprocess.STDOUT if stdout else None, text=True, ) return result.stdout if stdout else "" def tool_version(tool: str) -> str: output = run([tool, "-V"], stdout=True).strip().splitlines() if not output: raise FixtureError(f"{tool} returned no version") return output[0] def write_chunk_sources(root: Path) -> None: root.mkdir(parents=True) for name, blocks in CHUNK_FILES.items(): with (root / name).open("wb") as stream: for block in range(blocks): stream.write(deterministic_block(name, block)) def write_fragment_source(root: Path) -> None: root.mkdir(parents=True) seed = deterministic_block("fragment.dat", 0) data = (seed * math.ceil(100000 / len(seed)))[:100000] (root / "fragment.dat").write_bytes(data) def write_pcluster_source(root: Path) -> None: root.mkdir(parents=True) seed = b"".join( hashlib.sha256(f"repo22-g6:pcluster:{index}".encode("ascii")).digest() for index in range(188) )[:6000] data = (seed * math.ceil(131072 / len(seed)))[:131072] (root / "external-pcluster.bin").write_bytes(data) def make_image( mkfs: str, output: Path, source: Path, uuid: str, options: list[str], ) -> None: run( [ mkfs, "-T0", "--all-time", "--all-root", "--workers=1", f"-U{uuid}", *options, str(output), str(source), ] ) def chunk_layout(image: ErofsImage, path: str) -> ChunkLayout: _, entry = image.resolve_root_entry(path) inode = image.inode(entry.nid) if inode.layout != 4: raise FixtureError(f"{path}: expected chunk layout, got {inode.layout}") chunk_format, reserved = struct.unpack_from( " image.blocks * image.block_size: raise FixtureError(f"{path}: chunk index array exceeds declared image") entries = [] for index in range(count): offset = index_base + index * entry_size if entry_size == 8: high, device_id, low = struct.unpack_from( " None: for path in paths: layout = chunk_layout(image, path) if layout.entry_size != 8: raise FixtureError(f"{path}: blob baseline has no 8-byte indexes") minimum = 3 if path in ("/cross-device.bin", "/indexed.bin") else 1 if len(layout.entries) < minimum: raise FixtureError( f"{path}: need at least {minimum} chunk indexes" ) for entry in layout.entries: if entry.start_block_high != 0 or entry.device_id != 1: raise FixtureError(f"{path}: unexpected mkfs chunk index {entry}") def patch_chunk_entry( image: ErofsImage, entry: ChunkEntry, *, expected: tuple[int, int, int], replacement: tuple[int, int, int], ) -> None: current = struct.unpack_from(" None: for path in paths: layout = chunk_layout(image, path) if layout.entry_size != 8: raise FixtureError(f"{path}: 48-bit conversion requires indexes") expected = layout.chunk_format if expected & CHUNK_FORMAT_48BIT: raise FixtureError(f"{path}: already has 48-bit indexes") current = image.u16(layout.format_offset) if current != expected: raise FixtureError(f"{path}: chunk format changed before patch") image.put_u16(layout.format_offset, expected | CHUNK_FORMAT_48BIT) def enable_48bit(image: ErofsImage) -> None: if image.feature_incompat & FEATURE_INCOMPAT_48BIT: raise FixtureError("image already has 48-bit feature") root_nid = image.root_nid image.put_u32( SUPER + 80, image.feature_incompat | FEATURE_INCOMPAT_48BIT ) image.put_u16(SUPER + 14, 0) image.put_u64(SUPER + 112, root_nid) def decode_slots(image: ErofsImage) -> list[DeviceSlot]: extra, slot_offset = struct.unpack_from(" len(image.data): raise FixtureError("device table lies outside provider bytes") slots = [] for index in range(extra): offset = table_offset + index * DEVICE_SLOT_SIZE + DEVICE_SLOT_FIELDS blocks_low, uniaddr_low, blocks_high, uniaddr_high = struct.unpack_from( " None: if not slots: raise FixtureError("device table needs at least one slot") if table_offset % DEVICE_SLOT_SIZE != 0: raise FixtureError("device table is not 128-byte aligned") required = max( primary_blocks * image.block_size, table_offset + len(slots) * DEVICE_SLOT_SIZE, ) if len(image.data) > required: raise FixtureError("primary metadata exceeds requested primary size") image.data.extend(bytes(required - len(image.data))) image.put_u32(SUPER + 36, primary_blocks) image.put_u32( SUPER + 80, image.feature_incompat | FEATURE_INCOMPAT_DEVICE_TABLE ) image.put_u16(SUPER + 86, len(slots)) image.put_u16(SUPER + 88, table_offset // DEVICE_SLOT_SIZE) image.data[ table_offset : table_offset + len(slots) * DEVICE_SLOT_SIZE ] = bytes(len(slots) * DEVICE_SLOT_SIZE) for index, slot in enumerate(slots): if slot.blocks <= 0 or slot.blocks >= 1 << 48: raise FixtureError(f"slot {index + 1}: invalid blocks {slot.blocks}") if slot.uniaddr < 0 or slot.uniaddr >= 1 << 48: raise FixtureError(f"slot {index + 1}: invalid uniaddr {slot.uniaddr}") offset = table_offset + index * DEVICE_SLOT_SIZE tag = f"repo22-g6-slot-{index + 1}".encode("ascii") image.data[offset : offset + len(tag)] = tag struct.pack_into( "> 32, slot.uniaddr >> 32, ) def finalize_image(image: ErofsImage, path: Path) -> None: image.update_checksum() image.validate_superblock() image.save(path) def build_single_indexed( baseline: ErofsImage, blob: bytes, paths: list[str] ) -> ErofsImage: image = baseline.clone() if len(image.data) != BLOCK_SIZE or len(blob) % BLOCK_SIZE != 0: raise FixtureError("single indexed folding requires aligned providers") for path in paths: layout = chunk_layout(image, path) for entry in layout.entries: patch_chunk_entry( image, entry, expected=(0, 1, entry.start_block_low), replacement=(0, 0, entry.start_block_low + 1), ) image.data.extend(blob) image.put_u32(SUPER + 36, len(image.data) // BLOCK_SIZE) image.put_u32( SUPER + 80, image.feature_incompat & ~FEATURE_INCOMPAT_DEVICE_TABLE ) image.put_u16(SUPER + 86, 0) image.put_u16(SUPER + 88, 0) return image def build_multislot( baseline: ErofsImage, blob: bytes, paths: list[str], slot_count: int, ) -> MultiFixture: image = baseline.clone() providers = [bytearray() for _ in range(slot_count)] for path in paths: layout = chunk_layout(image, path) blocks_per_chunk = layout.chunk_size // BLOCK_SIZE for index, entry in enumerate(layout.entries): if entry.start_block_high != 0 or entry.device_id != 1: raise FixtureError(f"{path}: unexpected source index {entry}") source_start = entry.start_block * BLOCK_SIZE source_end = source_start + blocks_per_chunk * BLOCK_SIZE if source_end > len(blob): raise FixtureError(f"{path}: source chunk exceeds blob provider") slot = 1 if path == "/cold-slot2.bin" and slot_count >= 2 else ( index % slot_count ) local_block = len(providers[slot]) // BLOCK_SIZE providers[slot].extend(blob[source_start:source_end]) patch_chunk_entry( image, entry, expected=(0, 1, entry.start_block_low), replacement=(0, slot + 1, local_block), ) for index, provider in enumerate(providers): provider.extend(bytes((index + 1) * BLOCK_SIZE)) slots = [] uniaddr = 2 for provider in providers: blocks = len(provider) // BLOCK_SIZE slots.append(DeviceSlot(blocks, uniaddr)) uniaddr += blocks write_device_table(image, slots) return MultiFixture(image, providers, slots) def clone_multifixture(fixture: MultiFixture) -> MultiFixture: return MultiFixture( fixture.primary.clone(), [bytearray(provider) for provider in fixture.providers], list(fixture.slots), ) def rewrite_slots( fixture: MultiFixture, slots: list[DeviceSlot], *, table_offset: int = BLOCK_SIZE, ) -> None: fixture.slots = slots write_device_table(fixture.primary, slots, table_offset=table_offset) def build_flatdev(fixture: MultiFixture) -> bytes: end_block = max( [fixture.primary.blocks] + [slot.uniaddr + slot.blocks for slot in fixture.slots] ) data = bytearray(end_block * BLOCK_SIZE) data[: len(fixture.primary.data)] = fixture.primary.data for slot, provider in zip(fixture.slots, fixture.providers, strict=True): if len(provider) != slot.blocks * BLOCK_SIZE: raise FixtureError("provider length does not match slot declaration") start = slot.uniaddr * BLOCK_SIZE data[start : start + len(provider)] = provider return bytes(data) def first_entry(image: ErofsImage, path: str, index: int = 0) -> ChunkEntry: layout = chunk_layout(image, path) try: return layout.entries[index] except IndexError as error: raise FixtureError(f"{path}: missing chunk index {index}") from error def patch_unified_entry( fixture: MultiFixture, path: str, *, index: int = 0, ) -> None: entry = first_entry(fixture.primary, path, index) if entry.device_id < 1 or entry.device_id > len(fixture.slots): raise FixtureError(f"{path}: source index has no external slot") slot = fixture.slots[entry.device_id - 1] global_block = slot.uniaddr + entry.start_block patch_chunk_entry( fixture.primary, entry, expected=(entry.start_block_high, entry.device_id, entry.start_block_low), replacement=(global_block >> 32, 0, global_block & 0xFFFFFFFF), ) def patch_table_field( image: ErofsImage, slot: int, field_offset: int, fmt: str, value: int, ) -> None: _, slot_offset = struct.unpack_from(" tuple[MultiFixture, int]: _, entry = lz4_image.resolve_root_entry("/external-pcluster.bin") inode = lz4_image.inode(entry.nid) if inode.layout != 1 or inode.size != 131072: raise FixtureError("LZ4 source is not a legacy full-index inode") header = align_up(inode.offset + inode.inode_size + inode.xattr_size, 8) index_offset = header + 16 advise, cluster_offset, pblk = struct.unpack_from( " MultiFixture: image = positive.primary.clone() providers = [ bytearray(positive.providers[0][:BLOCK_SIZE]), bytearray(positive.providers[0][BLOCK_SIZE:]), ] slots = [DeviceSlot(1, 2), DeviceSlot(1, 3)] write_device_table(image, slots) current = struct.unpack_from(" dict[str, Any]: extra, slot_offset = struct.unpack_from(" None: source_files = sorted( path.relative_to(source) for path in source.rglob("*") if path.is_file() ) extracted_files = sorted( path.relative_to(extracted) for path in extracted.rglob("*") if path.is_file() ) if source_files != extracted_files: raise FixtureError( f"extracted paths differ: {source_files} != {extracted_files}" ) for relative in source_files: if sha256_file(source / relative) != sha256_file(extracted / relative): raise FixtureError(f"extracted checksum differs: {relative}") def fsck_extract( fsck: str, primary: Path, devices: list[Path], source: Path, scratch_parent: Path, ) -> None: with tempfile.TemporaryDirectory(prefix="repo22-g6-fsck-", dir=scratch_parent) as tmp: extracted = Path(tmp) / "extracted" command = [fsck] command.extend(f"--device={device}" for device in devices) command.extend([f"--extract={extracted}", str(primary)]) run(command) compare_trees(source, extracted) def write_provider(path: Path, data: bytes | bytearray) -> None: if len(data) == 0 or len(data) % BLOCK_SIZE != 0: raise FixtureError(f"{path.name}: provider is not block aligned") path.write_bytes(data) def save_multifixture( output: Path, name: str, fixture: MultiFixture, paths: list[str], manifest: dict[str, Any], ) -> tuple[Path, list[Path]]: primary_path = output / "images" / f"{name}-primary.erofs" finalize_image(fixture.primary, primary_path) provider_paths = [] for index, provider in enumerate(fixture.providers, 1): provider_path = output / "images" / f"{name}-slot{index}.blob" write_provider(provider_path, provider) provider_paths.append(provider_path) manifest["fixtures"][name] = { "primary": str(primary_path.relative_to(output)), "providers": [str(path.relative_to(output)) for path in provider_paths], "image": image_description(fixture.primary, paths), } return primary_path, provider_paths def save_image_fixture( output: Path, name: str, image: ErofsImage, paths: list[str], manifest: dict[str, Any], ) -> Path: path = output / "images" / f"{name}.erofs" finalize_image(image, path) manifest["fixtures"][name] = { "primary": str(path.relative_to(output)), "providers": [], "image": image_description(image, paths), } return path def add_mutation( manifest: dict[str, Any], artifact: Path, output: Path, label: str, offset: int, fmt: str, value: int, ) -> None: manifest["mutations"].append( { "artifact": str(artifact.relative_to(output)), "label": label, "offset": offset, "format": fmt, "value": value, } ) def save_negative( output: Path, name: str, image: ErofsImage, manifest: dict[str, Any], ) -> Path: path = output / "images" / f"{name}.erofs" finalize_image(image, path) manifest["negative_images"][name] = str(path.relative_to(output)) return path def generate(args: argparse.Namespace) -> None: output: Path = args.output.resolve() if output.exists(): raise FixtureError(f"output already exists: {output}") output.mkdir(parents=True) (output / "images").mkdir() chunk_source = output / "sources" / "chunk" fragment_source = output / "sources" / "fragment" pcluster_source = output / "sources" / "pcluster" write_chunk_sources(chunk_source) write_fragment_source(fragment_source) write_pcluster_source(pcluster_source) mkfs_version = tool_version(args.mkfs) fsck_version = tool_version(args.fsck) if "1.8.6" not in mkfs_version or "1.8.6" not in fsck_version: raise FixtureError( f"G6 requires erofs-utils 1.8.6, got {mkfs_version!r}, {fsck_version!r}" ) base_primary_path = output / "images" / "mkfs-blob-primary.erofs" base_blob_path = output / "images" / "mkfs-blob-slot1.blob" base_blob_path.write_bytes(b"") make_image( args.mkfs, base_primary_path, chunk_source, CHUNK_UUID, ["--chunksize=4096", f"--blobdev={base_blob_path}"], ) baseline = ErofsImage.load(base_primary_path) baseline.validate_superblock() paths = [f"/{name}" for name in CHUNK_FILES] assert_chunk_baseline(baseline, paths) base_blob = base_blob_path.read_bytes() if len(baseline.data) != BLOCK_SIZE or len(base_blob) % BLOCK_SIZE != 0: raise FixtureError("mkfs blob baseline has unexpected provider sizes") manifest: dict[str, Any] = { "schema": 1, "generator": "tests/g6_multidev_fixtures.py", "erofs_utils": {"mkfs": mkfs_version, "fsck": fsck_version}, "fixtures": {}, "negative_images": {}, "mutations": [], "artifacts": {}, "host_qualification": { "flatdev": ( "kernel-only: erofs-utils 1.8.6 requires --device for " "nonzero chunk device IDs" ), "external_pcluster": ( "kernel-only after relocation: erofs-utils 1.8.6 does not " "apply the device-ID-0 unified mapping used by the kernel" ), }, } manifest["fixtures"]["mkfs-blob"] = { "primary": str(base_primary_path.relative_to(output)), "providers": [str(base_blob_path.relative_to(output))], "image": image_description(baseline, paths), } single = build_single_indexed(baseline, base_blob, paths) single_path = save_image_fixture( output, "single-indexed", single, paths, manifest ) multi2 = build_multislot(baseline, base_blob, paths, 2) multi2_primary, multi2_providers = save_multifixture( output, "multi2", multi2, paths, manifest ) flatdev_path = output / "images" / "multi2-flatdev.erofs" write_provider(flatdev_path, build_flatdev(multi2)) manifest["fixtures"]["multi2-flatdev"] = { "primary": str(flatdev_path.relative_to(output)), "providers": [], "image": image_description(ErofsImage.load(flatdev_path), paths), } multi3 = build_multislot(baseline, base_blob, paths, 3) multi3_primary, multi3_providers = save_multifixture( output, "multi3", multi3, paths, manifest ) slot_zero = clone_multifixture(multi2) zero_slots = [DeviceSlot(slot_zero.slots[0].blocks, 0), slot_zero.slots[1]] rewrite_slots(slot_zero, zero_slots) slot_zero_primary, slot_zero_providers = save_multifixture( output, "multi2-uniaddr0", slot_zero, paths, manifest ) table_zero = clone_multifixture(multi2) table_bytes = bytes( table_zero.primary.data[ BLOCK_SIZE : BLOCK_SIZE + 2 * DEVICE_SLOT_SIZE ] ) table_zero.primary.data[: 2 * DEVICE_SLOT_SIZE] = table_bytes table_zero.primary.put_u16(SUPER + 88, 0) table_zero_primary, table_zero_providers = save_multifixture( output, "multi2-devt0", table_zero, paths, manifest ) unified = clone_multifixture(multi2) patch_unified_entry(unified, "/unified-address.bin") unified_primary, unified_providers = save_multifixture( output, "multi2-unified", unified, paths, manifest ) unified_flatdev_path = output / "images" / "multi2-unified-flatdev.erofs" write_provider(unified_flatdev_path, build_flatdev(unified)) manifest["fixtures"]["multi2-unified-flatdev"] = { "primary": str(unified_flatdev_path.relative_to(output)), "providers": [], "image": image_description( ErofsImage.load(unified_flatdev_path), paths ), } unified48 = clone_multifixture(multi2) enable_48bit(unified48.primary) set_chunk_48bit(unified48.primary, paths) high_start = (1 << 32) + 2 high_slots = [] cursor = high_start for slot in unified48.slots: high_slots.append(DeviceSlot(slot.blocks, cursor)) cursor += slot.blocks rewrite_slots(unified48, high_slots) patch_unified_entry(unified48, "/unified-address.bin") unified48_primary, unified48_providers = save_multifixture( output, "multi2-unified48", unified48, paths, manifest ) unified48_oob = clone_multifixture(unified48) unified48_oob_entry = first_entry( unified48_oob.primary, "/unified-address.bin" ) patch_chunk_entry( unified48_oob.primary, unified48_oob_entry, expected=( unified48_oob_entry.start_block_high, 0, unified48_oob_entry.start_block_low, ), replacement=(0xFFFF, 0, 0xFFFFFFFE), ) unified48_oob_primary, unified48_oob_providers = save_multifixture( output, "bad-unified48-out-of-range", unified48_oob, paths, manifest ) add_mutation( manifest, unified48_oob_primary, output, "device-ID-0 chunk address is the largest non-NULL 48-bit block", unified48_oob_entry.offset, " tuple[int, str, int, str]: value = image.blocks * BLOCK_SIZE // DEVICE_SLOT_SIZE image.put_u16(SUPER + 88, value) return SUPER + 88, " tuple[int, str, int, str]: patch_table_field(image, 1, 0, " tuple[int, str, int, str]: patch_table_field(image, 1, 4, " tuple[int, str, int, str]: value = multi2.slots[0].uniaddr patch_table_field(image, 2, 4, " tuple[int, str, int, str]: patch_table_field(image, 1, 0, " tuple[int, str, int, str]: enable_48bit(image) patch_table_field(image, 1, 0, " Any: size = struct.calcsize(fmt) with path.open("rb") as stream: stream.seek(offset) data = stream.read(size) if len(data) != size: raise FixtureError(f"{path}: field at {offset} is truncated") values = struct.unpack(fmt, data) return values[0] if len(values) == 1 else list(values) def verify_image_description(path: Path, expected: dict[str, Any]) -> None: image = ErofsImage.load(path) paths = list(expected["chunks"]) actual = image_description(image, paths) if actual != expected: raise FixtureError(f"{path}: parsed image fields differ from manifest") def verify_output(output: Path) -> None: manifest_path = output / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="ascii")) if manifest.get("schema") != 1: raise FixtureError("unsupported G6 manifest schema") for relative, expected in manifest["artifacts"].items(): path = output / relative if not path.is_file(): raise FixtureError(f"missing artifact: {relative}") if path.stat().st_size != expected["bytes"]: raise FixtureError(f"size differs: {relative}") if sha256_file(path) != expected["sha256"]: raise FixtureError(f"SHA256 differs: {relative}") for fixture in manifest["fixtures"].values(): image = fixture.get("image") if image is not None: verify_image_description(output / fixture["primary"], image) for mutation in manifest["mutations"]: actual = unpack_assertion( output / mutation["artifact"], mutation["offset"], mutation["format"], ) if actual != mutation["value"]: raise FixtureError( f"{mutation['label']}: expected {mutation['value']}, got {actual}" ) run(["sha256sum", "-c", "SHA256SUMS"], stdout=True, cwd=output) print( f"verified={output} artifacts={len(manifest['artifacts'])} " f"field_assertions={len(manifest['mutations'])}" ) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) generate_parser = subparsers.add_parser("generate") generate_parser.add_argument("--output", required=True, type=Path) generate_parser.add_argument("--mkfs", default=shutil.which("mkfs.erofs")) generate_parser.add_argument("--fsck", default=shutil.which("fsck.erofs")) generate_parser.set_defaults(function=generate) verify_parser = subparsers.add_parser("verify") verify_parser.add_argument("output", type=Path) verify_parser.set_defaults(function=lambda args: verify_output(args.output.resolve())) return parser.parse_args() def main() -> None: args = parse_args() if args.command == "generate" and (args.mkfs is None or args.fsck is None): raise FixtureError("mkfs.erofs and fsck.erofs are required") args.function(args) if __name__ == "__main__": os.environ.setdefault("PYTHONDONTWRITEBYTECODE", "1") main()