#!/usr/bin/env python3 """Generate and verify deterministic repo22 G5 compression fixtures.""" from __future__ import annotations import argparse from collections import Counter import hashlib import json from pathlib import Path import re import struct import subprocess from typing import Any from erofs_fixture import ErofsImage, Inode, SUPER, sha256 BLOCK_SIZE = 4096 FIXED_UUID = "00000000-0000-0000-0000-000000000000" FEATURE_INCOMPAT_COMPR_HEAD2 = 0x00000008 LAYOUT_COMPRESSED_FULL = 1 LAYOUT_COMPRESSED_COMPACT = 3 Z_EROFS_ADVISE_COMPACTED_2B = 0x0001 Z_EROFS_ADVISE_BIG_PCLUSTER_1 = 0x0002 Z_EROFS_ADVISE_BIG_PCLUSTER_2 = 0x0004 Z_EROFS_ADVISE_INLINE_PCLUSTER = 0x0008 Z_EROFS_ADVISE_INTERLACED_PCLUSTER = 0x0010 Z_EROFS_LCLUSTER_TYPE_MASK = 0x0003 Z_EROFS_LCLUSTER_TYPE_PLAIN = 0 Z_EROFS_LCLUSTER_TYPE_HEAD1 = 1 Z_EROFS_LCLUSTER_TYPE_NONHEAD = 2 Z_EROFS_LCLUSTER_TYPE_HEAD2 = 3 Z_EROFS_LI_PARTIAL_REF = 0x8000 Z_EROFS_LI_D0_CBLKCNT = 0x0800 ALGORITHMS = { "lz4": 0, "lzma": 1, "deflate": 2, "zstd": 3, } class FixtureError(RuntimeError): pass def align(value: int, alignment: int) -> int: return (value + alignment - 1) & -alignment def relative(path: Path, root: Path) -> str: return str(path.relative_to(root)) def run( command: list[str], *, log: Path | None = None, check: bool = True, ) -> subprocess.CompletedProcess[str]: result = subprocess.run( command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) if log is not None: log.write_text( "$ " + " ".join(command) + "\n" + result.stdout, encoding="utf-8", ) if check and result.returncode != 0: raise FixtureError( f"command failed ({result.returncode}): {' '.join(command)}\n" f"{result.stdout}" ) return result def write_repeated(path: Path, size: int, label: str) -> None: token = (label.encode("ascii") + b"\n") * 64 chunk = (token * ((1024 * 1024 + len(token) - 1) // len(token)))[ : 1024 * 1024 ] remaining = size with path.open("wb") as output: while remaining: amount = min(remaining, len(chunk)) output.write(chunk[:amount]) remaining -= amount def mixed_chunk(seed: str, chunk_index: int, size: int) -> bytes: output = bytearray() block_index = 0 while len(output) < size: amount = min(BLOCK_SIZE, size - len(output)) if block_index % 4 == 0: material = hashlib.shake_256( f"{seed}:{chunk_index}:{block_index}".encode("ascii") ).digest(amount) else: token = ( f"repo22-g5:{seed}:{chunk_index % 7}:{block_index % 11}\n" ).encode("ascii") material = (token * ((amount + len(token) - 1) // len(token)))[ :amount ] output.extend(material) block_index += 1 return bytes(output) def write_mixed(path: Path, size: int, seed: str) -> None: chunk_size = 1024 * 1024 remaining = size index = 0 with path.open("wb") as output: while remaining: amount = min(remaining, chunk_size) output.write(mixed_chunk(seed, index, amount)) remaining -= amount index += 1 def write_interlaced(path: Path, size: int) -> None: remaining = size extent = 0 with path.open("wb") as output: while remaining: compressible_size = min(remaining, 16384) token = f"repo22-interlaced-{extent % 13:02d}\n".encode("ascii") output.write( (token * ((compressible_size + len(token) - 1) // len(token)))[ :compressible_size ] ) remaining -= compressible_size if not remaining: break random_size = min(remaining, BLOCK_SIZE) output.write( hashlib.shake_256( f"repo22-interlaced-random-{extent}".encode("ascii") ).digest(random_size) ) remaining -= random_size extent += 1 def create_sources(source_root: Path) -> None: directories = { "lz4": source_root / "lz4", "large": source_root / "large", "levels": source_root / "levels", "lzma-large": source_root / "lzma-large", "microlzma": source_root / "microlzma", "partial": source_root / "partial", "partial-deflate": source_root / "partial-deflate", "shape": source_root / "shape", "ztail": source_root / "ztail", } for directory in directories.values(): directory.mkdir(parents=True) write_mixed(directories["lz4"] / "compressed.bin", 8 * 1024 * 1024, "lz4") write_mixed( directories["large"] / "large.bin", 256 * 1024 * 1024, "large" ) write_mixed( directories["levels"] / "level.dat", 8 * 1024 * 1024, "levels" ) write_mixed( directories["lzma-large"] / "large.bin", 100 * 1024 * 1024 + 1, "lzma-large", ) (directories["microlzma"] / "one-byte.bin").write_bytes(b"G") write_repeated( directories["microlzma"] / "block-4k.bin", BLOCK_SIZE, "microlzma-4k" ) write_repeated( directories["microlzma"] / "boundary-16k.bin", 4 * BLOCK_SIZE, "microlzma-16k", ) write_repeated( directories["partial"] / "a.dat", 1024 * 1024, "partial-stream" ) source_a = directories["partial"] / "a.dat" source_b = directories["partial"] / "b.dat" with source_a.open("rb") as source, source_b.open("wb") as target: target.write(source.read(700000)) write_mixed( directories["partial"] / "control.bin", 32768, "partial-control", ) write_repeated( directories["partial-deflate"] / "a.dat", 1024 * 1024, "partial-stream", ) with (directories["partial-deflate"] / "a.dat").open("rb") as source, ( directories["partial-deflate"] / "b.dat" ).open("wb") as target: target.write(source.read(100000)) write_mixed( directories["partial-deflate"] / "control.bin", 32768, "partial-control", ) write_interlaced(directories["shape"] / "shape.dat", 1024 * 1024) write_repeated(directories["ztail"] / "inline.dat", 131071, "ztail-inline") write_mixed( directories["ztail"] / "exact-pcluster.dat", BLOCK_SIZE, "ztail-exact" ) write_repeated( directories["ztail"] / "one-byte-tail.dat", BLOCK_SIZE + 1, "ztail-one-byte", ) write_repeated( directories["ztail"] / "max-tail.dat", BLOCK_SIZE * 2 - 1, "ztail-max", ) (directories["ztail"] / "zero-tail.dat").write_bytes(b"") def source_inventory(source_root: Path) -> list[dict[str, Any]]: inventory = [] for path in sorted(item for item in source_root.rglob("*") if item.is_file()): inventory.append( { "path": relative(path, source_root.parent), "size": path.stat().st_size, "sha256": sha256(path), } ) return inventory def mkfs_image( *, output_root: Path, image_name: str, source: Path, options: list[str], command_log: list[dict[str, Any]], ) -> Path: image = output_root / "images" / image_name command = [ "mkfs.erofs", "--workers=1", "--sort=path", "-T0", "--all-time", f"-U{FIXED_UUID}", "--all-root", *options, str(image), str(source), ] log = output_root / "logs" / f"mkfs-{image.stem}.log" run(command, log=log) command_log.append( { "image": relative(image, output_root), "source": relative(source, output_root), "options": options, "log": relative(log, output_root), } ) parsed = ErofsImage.load(image) parsed.validate_superblock() return image def inode_for(image: ErofsImage, path: str) -> Inode: _, entry = image.resolve_root_entry(path) return image.inode(entry.nid) def map_header(image: ErofsImage, inode: Inode) -> dict[str, int]: if inode.layout not in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT): raise FixtureError(f"inode layout {inode.layout} is not compressed") offset = align(inode.offset + inode.inode_size + inode.xattr_size, 8) if offset > len(image.data) - 8: raise FixtureError("compressed map header exceeds image") raw0, advise, algorithm, clusterbits = struct.unpack_from( "> 4, "clusterbits_raw": clusterbits, "lcluster_bits": image.block_bits + (clusterbits & 0x07), "idata_size": raw0 >> 16, } def full_index_offset(image: ErofsImage, inode: Inode) -> int: header = map_header(image, inode) return align(header["offset"] + 8, 8) + 8 def full_record(image: ErofsImage, inode: Inode, lcn: int) -> dict[str, int]: if inode.layout != LAYOUT_COMPRESSED_FULL: raise FixtureError("full record requested from a non-full inode") offset = full_index_offset(image, inode) + lcn * 8 if offset > len(image.data) - 8: raise FixtureError("full index record exceeds image") advise, clusterofs, word = struct.unpack_from("> 16, } def full_index_summary(image: ErofsImage, inode: Inode) -> dict[str, Any]: header = map_header(image, inode) count = (inode.size + (1 << header["lcluster_bits"]) - 1) >> header[ "lcluster_bits" ] types: Counter[int] = Counter() heads = [] partial_heads = [] for lcn in range(count): record = full_record(image, inode, lcn) types[record["type"]] += 1 if record["type"] in ( Z_EROFS_LCLUSTER_TYPE_PLAIN, Z_EROFS_LCLUSTER_TYPE_HEAD1, Z_EROFS_LCLUSTER_TYPE_HEAD2, ): if len(heads) < 16: heads.append( { "lcn": lcn, "offset": record["offset"], "type": record["type"], "pblk": record["pblk"], "clusterofs": record["clusterofs"], } ) if record["partial_ref"]: partial_heads.append( { "lcn": lcn, "offset": record["offset"], "pblk": record["pblk"], } ) return { "record_count": count, "type_counts": {str(key): value for key, value in sorted(types.items())}, "first_heads": heads, "partial_heads": partial_heads, } EXTENT_RE = re.compile( r"^\s*\d+:\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:" r"\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$" ) def dump_extents(image: Path, path: str, log: Path | None = None) -> list[dict[str, int]]: result = run( ["dump.erofs", f"--path={path}", "-e", str(image)], log=log, ) extents = [] for line in result.stdout.splitlines(): match = EXTENT_RE.match(line) if match is None: continue logical_start, logical_end, logical_length, physical_start, physical_end, physical_length = ( int(value) for value in match.groups() ) if logical_end - logical_start != logical_length: raise FixtureError("dump.erofs reported an inconsistent logical extent") if physical_end - physical_start != physical_length: raise FixtureError("dump.erofs reported an inconsistent physical extent") extents.append( { "logical_start": logical_start, "logical_end": logical_end, "logical_length": logical_length, "physical_start": physical_start, "physical_end": physical_end, "physical_length": physical_length, } ) if not extents: raise FixtureError(f"dump.erofs reported no extents for {image}:{path}") return extents def extent_summary(extents: list[dict[str, int]]) -> dict[str, Any]: transitions = [] for index in range(1, len(extents)): previous = extents[index - 1] current = extents[index] previous_plain = previous["logical_length"] == previous["physical_length"] current_plain = current["logical_length"] == current["physical_length"] if previous_plain != current_plain: transitions.append( { "logical_offset": current["logical_start"], "previous_plain": previous_plain, "current_plain": current_plain, } ) return { "count": len(extents), "first": extents[:8], "last": extents[-1], "max_logical_length": max(item["logical_length"] for item in extents), "max_physical_length": max(item["physical_length"] for item in extents), "plain_count": sum( item["physical_length"] != 0 and item["logical_length"] == item["physical_length"] for item in extents ), "compressed_count": sum( item["physical_length"] != 0 and item["logical_length"] > item["physical_length"] for item in extents ), "hole_or_fragment_count": sum( item["physical_length"] == 0 for item in extents ), "transitions": transitions[:8], } def inspect_path( output_root: Path, image_path: Path, path: str, *, write_log: bool = True, ) -> dict[str, Any]: image = ErofsImage.load(image_path) image.validate_superblock() inode = inode_for(image, path) details: dict[str, Any] = { "path": path, "nid": inode.nid, "inode_offset": inode.offset, "inode_size": inode.inode_size, "xattr_size": inode.xattr_size, "layout": inode.layout, "size": inode.size, } if inode.layout in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT): details["map_header"] = map_header(image, inode) log = None if write_log: safe_path = path.removeprefix("/").replace("/", "-") log = output_root / "logs" / f"dump-{image_path.stem}-{safe_path}.log" details["dump_log"] = relative(log, output_root) details["extents"] = extent_summary(dump_extents(image_path, path, log)) if inode.layout == LAYOUT_COMPRESSED_FULL: details["full_index"] = full_index_summary(image, inode) return details def assert_compressed( details: dict[str, Any], *, algorithm: str, layout: int | None = None, ) -> None: if details["layout"] not in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT): raise FixtureError(f"{details['path']} was not compressed") if layout is not None and details["layout"] != layout: raise FixtureError( f"{details['path']} layout {details['layout']} != expected {layout}" ) actual = details["map_header"]["head1_algorithm"] if actual != ALGORITHMS[algorithm]: raise FixtureError( f"{details['path']} algorithm {actual} != {ALGORITHMS[algorithm]}" ) def patch_partial_reference( base: Path, output: Path, *, algorithm: str, ) -> dict[str, Any]: image = ErofsImage.load(base) inode_a = inode_for(image, "/a.dat") inode_b = inode_for(image, "/b.dat") if inode_a.layout != LAYOUT_COMPRESSED_FULL or inode_b.layout != LAYOUT_COMPRESSED_FULL: raise FixtureError("partial-reference transform requires full indexes") header_a = map_header(image, inode_a) header_b = map_header(image, inode_b) expected_algorithm = ALGORITHMS[algorithm] if ( header_a["head1_algorithm"] != expected_algorithm or header_b["head1_algorithm"] != expected_algorithm ): raise FixtureError("partial-reference source algorithm mismatch") record_a = full_record(image, inode_a, 0) record_b = full_record(image, inode_b, 0) record_a_next = full_record(image, inode_a, 1) record_b_next = full_record(image, inode_b, 1) if record_a["type"] != Z_EROFS_LCLUSTER_TYPE_HEAD1: raise FixtureError("a.dat first record is not HEAD1") if record_b["type"] != Z_EROFS_LCLUSTER_TYPE_HEAD1: raise FixtureError("b.dat first record is not HEAD1") if record_a["pblk"] == record_b["pblk"]: raise FixtureError("mkfs unexpectedly reused the partial pcluster") if record_b["advise"] & Z_EROFS_LI_PARTIAL_REF: raise FixtureError("b.dat was already marked partial") if inode_b.size >= inode_a.size: raise FixtureError("partial file is not shorter than the complete stream") if ( record_a_next["type"] != Z_EROFS_LCLUSTER_TYPE_NONHEAD or record_b_next["type"] != Z_EROFS_LCLUSTER_TYPE_NONHEAD or not record_a_next["delta0"] & Z_EROFS_LI_D0_CBLKCNT or not record_b_next["delta0"] & Z_EROFS_LI_D0_CBLKCNT ): raise FixtureError("partial source lacks a first-pcluster block count") struct.pack_into( " dict[str, Any]: image = ErofsImage.load(base) inode = inode_for(image, "/shape.dat") if inode.layout != LAYOUT_COMPRESSED_FULL: raise FixtureError("HEAD2 transform requires full indexes") header = map_header(image, inode) if header["head1_algorithm"] != ALGORITHMS["lz4"]: raise FixtureError("HEAD2 base is not LZ4") if not header["advise"] & Z_EROFS_ADVISE_BIG_PCLUSTER_1: raise FixtureError("HEAD2 base lacks HEAD1 big-pcluster advise") count = (inode.size + (1 << header["lcluster_bits"]) - 1) >> header[ "lcluster_bits" ] target_lcn = None target = None for lcn in range(count): record = full_record(image, inode, lcn) if record["type"] == Z_EROFS_LCLUSTER_TYPE_HEAD1: target_lcn = lcn target = record break if target_lcn is None or target is None: raise FixtureError("HEAD2 base has no HEAD1 record") old_incompat = image.feature_incompat old_advise = header["advise"] old_algorithm = header["algorithm_raw"] new_advise = old_advise | Z_EROFS_ADVISE_BIG_PCLUSTER_2 new_algorithm = (old_algorithm & 0x0F) | ((old_algorithm & 0x0F) << 4) new_record_advise = ( target["advise"] & ~Z_EROFS_LCLUSTER_TYPE_MASK ) | Z_EROFS_LCLUSTER_TYPE_HEAD2 image.put_u32(SUPER + 80, old_incompat | FEATURE_INCOMPAT_COMPR_HEAD2) struct.pack_into(" dict[str, Any]: source = ErofsImage.load(image_path) inode = inode_for(source, path) header = map_header(source, inode) if header["head1_algorithm"] != ALGORITHMS[algorithm]: raise FixtureError("corruption target algorithm mismatch") extents = dump_extents(image_path, path) target = next( ( extent for extent in extents if extent["physical_length"] > 0 and extent["logical_length"] > extent["physical_length"] ), None, ) if target is None: raise FixtureError("no compressed extent available for corruption") start = target["physical_start"] end = target["physical_end"] if start < source.checksum_end or end > len(source.data): raise FixtureError("target compressed extent is outside payload bounds") payload = source.data[start:end] nonzero = [index for index, value in enumerate(payload) if value != 0] if len(nonzero) < 16: raise FixtureError("target compressed extent has too little encoded data") patch_start = start + nonzero[0] patch_length = min(64, end - patch_start) old = bytes(source.data[patch_start : patch_start + patch_length]) for index in range(patch_start, patch_start + patch_length): source.data[index] ^= 0xA5 new = bytes(source.data[patch_start : patch_start + patch_length]) if old == new: raise AssertionError("compressed corruption did not change bytes") if not source.checksum_valid(): raise FixtureError("payload corruption invalidated the superblock checksum") source.save(output) reopened = ErofsImage.load(output) reopened.validate_superblock() return { "path": path, "algorithm": algorithm, "algorithm_id": ALGORITHMS[algorithm], "nid": inode.nid, "map_header_offset": header["offset"], "extent_logical_start": target["logical_start"], "extent_logical_length": target["logical_length"], "pcluster_start": start, "pcluster_length": target["physical_length"], "patch_offset": patch_start, "patch_length": patch_length, "old_sha256": hashlib.sha256(old).hexdigest(), "new_sha256": hashlib.sha256(new).hexdigest(), "superblock_checksum_valid": True, } def write_checksum_file(root: Path, paths: list[Path], name: str) -> None: with (root / name).open("w", encoding="ascii") as output: for path in sorted(paths): output.write(f"{sha256(path)} {relative(path, root)}\n") def explicit_probe( output_root: Path, attempt_image: Path, utils_source: Path | None, ) -> dict[str, Any]: attempt = inspect_path(output_root, attempt_image, "/shape.dat") header = attempt["map_header"] tokens = [ "Z_EROFS_ADVISE_EXTENTS", "z_erofs_extent_recsize", "struct z_erofs_extent {", ] matches: dict[str, list[str]] = {token: [] for token in tokens} if utils_source is not None: for base in (utils_source / "include", utils_source / "lib"): if not base.is_dir(): continue for path in sorted(item for item in base.rglob("*") if item.is_file()): try: content = path.read_text(encoding="utf-8", errors="ignore") except OSError: continue for token in tokens: if token in content: matches[token].append(str(path)) return { "status": "SHELVED", "attempt_image": relative(attempt_image, output_root), "attempt_layout": attempt["layout"], "attempt_map_header_offset": header["offset"], "attempt_h_advise": header["advise"], "attempt_explicit_bit": bool( attempt["layout"] == LAYOUT_COMPRESSED_FULL and header["advise"] & Z_EROFS_ADVISE_COMPACTED_2B ), "mapped_payload_generated": False, "erofs_utils_source": str(utils_source) if utils_source else None, "source_token_matches": matches, "structured_transform_attempt": ( "Refused: converting legacy lcluster indexes into variable-size " "extent records requires relocating subsequent metadata and payload; " "the helper cannot validate a mapped result with erofs-utils 1.8.6." ), } def build_fixtures(args: argparse.Namespace) -> None: output_root = args.output.resolve() if output_root.exists(): raise FixtureError(f"output already exists: {output_root}") (output_root / "images").mkdir(parents=True) (output_root / "logs").mkdir() (output_root / "sources").mkdir() (output_root / "base-images").mkdir() create_sources(output_root / "sources") version = run(["mkfs.erofs", "-V"]).stdout.strip() if "1.8.6" not in version: raise FixtureError(f"mkfs.erofs 1.8.6 is required, got: {version}") commands: list[dict[str, Any]] = [] images: dict[str, Path] = {} targets: dict[str, dict[str, Any]] = {} def generate( name: str, source_dir: str, options: list[str], path: str, algorithm: str, layout: int | None = None, ) -> dict[str, Any]: image = mkfs_image( output_root=output_root, image_name=name, source=output_root / "sources" / source_dir, options=options, command_log=commands, ) images[name] = image details = inspect_path(output_root, image, path) assert_compressed(details, algorithm=algorithm, layout=layout) targets[f"{name}:{path}"] = details return details compact_4k = generate( "lz4-compact-4k.erofs", "lz4", ["-zlz4", "-C4096"], "/compressed.bin", "lz4", LAYOUT_COMPRESSED_COMPACT, ) full_4k = generate( "lz4-full-4k.erofs", "lz4", ["-zlz4", "-C4096", "-Elegacy-compress"], "/compressed.bin", "lz4", LAYOUT_COMPRESSED_FULL, ) compact_64k = generate( "lz4-compact-64k.erofs", "lz4", ["-zlz4", "-C65536"], "/compressed.bin", "lz4", LAYOUT_COMPRESSED_COMPACT, ) compact_256k = generate( "lz4-compact-256k.erofs", "lz4", ["-zlz4", "-C262144"], "/compressed.bin", "lz4", LAYOUT_COMPRESSED_COMPACT, ) generate( "lz4-large.erofs", "large", ["-zlz4", "-C65536"], "/large.bin", "lz4", ) ztail = generate( "lz4-ztail.erofs", "ztail", ["-zlz4", "-C4096", "-Eztailpacking"], "/inline.dat", "lz4", ) for path in ( "/exact-pcluster.dat", "/one-byte-tail.dat", "/max-tail.dat", "/zero-tail.dat", ): details = inspect_path(output_root, images["lz4-ztail.erofs"], path) targets[f"lz4-ztail.erofs:{path}"] = details if not compact_4k["map_header"]["advise"] & Z_EROFS_ADVISE_COMPACTED_2B: raise FixtureError("4K compact image lacks compacted index advise") if full_4k["map_header"]["advise"] & Z_EROFS_ADVISE_COMPACTED_2B: raise FixtureError("full-index image advertises compacted indexes") for details, requested in ((compact_64k, 65536), (compact_256k, 262144)): header = details["map_header"] if not header["advise"] & Z_EROFS_ADVISE_BIG_PCLUSTER_1: raise FixtureError(f"{requested} image lacks big-pcluster advise") if details["extents"]["max_physical_length"] <= BLOCK_SIZE: raise FixtureError(f"{requested} image has no multi-block pcluster") if not ztail["map_header"]["advise"] & Z_EROFS_ADVISE_INLINE_PCLUSTER: raise FixtureError("ztailpacking target lacks inline-pcluster advise") if ztail["map_header"]["idata_size"] == 0: raise FixtureError("ztailpacking target has zero inline encoded size") for level in (1, 6, 9): generate( f"deflate-level{level}.erofs", "levels", [f"-zdeflate,level={level}", "-C65536", "-Elegacy-compress"], "/level.dat", "deflate", LAYOUT_COMPRESSED_FULL, ) for level in (1, 15, 22): generate( f"zstd-level{level}.erofs", "levels", [f"-zzstd,level={level}", "-C65536", "-Elegacy-compress"], "/level.dat", "zstd", LAYOUT_COMPRESSED_FULL, ) generate( "lzma-level6.erofs", "levels", ["-zlzma,level=6", "-C65536", "-Elegacy-compress"], "/level.dat", "lzma", LAYOUT_COMPRESSED_FULL, ) generate( "lzma-large.erofs", "lzma-large", ["-zlzma,level=6", "-C65536", "-Elegacy-compress"], "/large.bin", "lzma", LAYOUT_COMPRESSED_FULL, ) generate( "microlzma-edge.erofs", "microlzma", ["-zlzma,level=6", "-C4096", "-Elegacy-compress"], "/boundary-16k.bin", "lzma", LAYOUT_COMPRESSED_FULL, ) for path in ("/one-byte.bin", "/block-4k.bin"): details = inspect_path(output_root, images["microlzma-edge.erofs"], path) targets[f"microlzma-edge.erofs:{path}"] = details partials: dict[str, Any] = {} corruptions: dict[str, Any] = {} for algorithm, compressor in ( ("deflate", "-zdeflate,level=1"), ("zstd", "-zzstd,level=1"), ("lzma", "-zlzma,level=6"), ): base_name = f"{algorithm}-partial-base.erofs" base = mkfs_image( output_root=output_root, image_name=f"../base-images/{base_name}", source=output_root / "sources" / ("partial-deflate" if algorithm == "deflate" else "partial"), options=[compressor, "-C1048576", "-Elegacy-compress"], command_log=commands, ) valid = output_root / "images" / f"{algorithm}-partial-ref.erofs" partials[algorithm] = patch_partial_reference( base, valid, algorithm=algorithm ) images[valid.name] = valid for path in ("/a.dat", "/b.dat"): details = inspect_path(output_root, valid, path) assert_compressed( details, algorithm=algorithm, layout=LAYOUT_COMPRESSED_FULL, ) targets[f"{valid.name}:{path}"] = details corrupt = output_root / "images" / f"{algorithm}-partial-ref-corrupt.erofs" corruptions[algorithm] = corrupt_target_extent( valid, corrupt, path="/a.dat", algorithm=algorithm, ) images[corrupt.name] = corrupt head2_base = mkfs_image( output_root=output_root, image_name="../base-images/head2-base.erofs", source=output_root / "sources" / "shape", options=["-zlz4", "-C65536", "-Elegacy-compress"], command_log=commands, ) head2 = output_root / "images" / "head2.erofs" head2_transform = patch_head2(head2_base, head2) images[head2.name] = head2 head2_details = inspect_path(output_root, head2, "/shape.dat") assert_compressed( head2_details, algorithm="lz4", layout=LAYOUT_COMPRESSED_FULL, ) targets[f"{head2.name}:/shape.dat"] = head2_details head2_corrupt = output_root / "images" / "head2-corrupt.erofs" corruptions["head2"] = corrupt_target_extent( head2, head2_corrupt, path="/shape.dat", algorithm="lz4", ) images[head2_corrupt.name] = head2_corrupt interlaced = generate( "interlaced.erofs", "shape", ["-zlz4", "-C4096", "-Efragments"], "/shape.dat", "lz4", ) if not interlaced["map_header"]["advise"] & Z_EROFS_ADVISE_INTERLACED_PCLUSTER: raise FixtureError("interlaced target lacks interlaced advise") if not interlaced["extents"]["transitions"]: raise FixtureError("interlaced target has no compressed/plain transition") extent_attempt = generate( "extent-attempt.erofs", "shape", [ "-zlz4", "-C65536", "-Elegacy-compress", "--max-extent-bytes=65536", ], "/shape.dat", "lz4", LAYOUT_COMPRESSED_FULL, ) if extent_attempt["map_header"]["advise"] & Z_EROFS_ADVISE_COMPACTED_2B: raise FixtureError("extent attempt unexpectedly selected the explicit bit") image_paths = sorted(set(images.values())) source_paths = sorted( path for path in (output_root / "sources").rglob("*") if path.is_file() ) write_checksum_file(output_root, image_paths, "SHA256SUMS") write_checksum_file(output_root, source_paths, "SOURCE-SHA256SUMS") manifest = { "schema": 1, "mkfs_version": version, "fixed_uuid": FIXED_UUID, "source_inventory": source_inventory(output_root / "sources"), "commands": commands, "images": { relative(path, output_root): { "size": path.stat().st_size, "sha256": sha256(path), } for path in image_paths }, "targets": targets, "partial_references": partials, "corruptions": corruptions, "head2_transform": head2_transform, "explicit_extent": explicit_probe( output_root, images["extent-attempt.erofs"], args.erofs_utils_source.resolve() if args.erofs_utils_source is not None else None, ), } (output_root / "fixture-manifest.json").write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="ascii", ) verify_output(output_root) def verify_output(output_root: Path) -> None: output_root = output_root.resolve() manifest_path = output_root / "fixture-manifest.json" manifest = json.loads(manifest_path.read_text(encoding="ascii")) for path_text, expected in manifest["images"].items(): path = output_root / path_text if not path.is_file(): raise FixtureError(f"missing image: {path_text}") if path.stat().st_size != expected["size"]: raise FixtureError(f"image size changed: {path_text}") if sha256(path) != expected["sha256"]: raise FixtureError(f"image hash changed: {path_text}") ErofsImage.load(path).validate_superblock() for expected in manifest["source_inventory"]: path = output_root / expected["path"] if path.stat().st_size != expected["size"] or sha256(path) != expected["sha256"]: raise FixtureError(f"source changed: {expected['path']}") for key, expected in manifest["targets"].items(): image_name, path = key.split(":", 1) actual = inspect_path( output_root, output_root / "images" / image_name, path, write_log=False, ) actual.pop("dump_log", None) comparable = dict(expected) comparable.pop("dump_log", None) if actual != comparable: raise FixtureError(f"structured target fields changed: {key}") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) create = subparsers.add_parser("create") create.add_argument("--output", type=Path, required=True) create.add_argument("--erofs-utils-source", type=Path) verify = subparsers.add_parser("verify") verify.add_argument("--output", type=Path, required=True) return parser.parse_args() def main() -> None: args = parse_args() if args.command == "create": build_fixtures(args) else: verify_output(args.output) if __name__ == "__main__": main()