#!/usr/bin/env python3 """Generate, compare, and verify deterministic Pre15 G3 fixtures.""" from __future__ import annotations import argparse import hashlib import json import os from pathlib import Path import shutil import stat import struct import subprocess import sys TESTS_DIR = Path(__file__).resolve().parents[2] if str(TESTS_DIR) not in sys.path: sys.path.insert(0, str(TESTS_DIR)) from erofs_fixture import ErofsImage, MAGIC, SUPER FLAT_PLAIN = 0 HUGE_DIRECTORY_BLOCKS = (1 << 31) + 1 S_IFMT = 0o170000 FEATURE_COMPAT_SB_CHKSUM = 0x00000001 FEATURE_COMPAT_PLAIN_XATTR_PFX = 0x00000010 FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040 FEATURE_INCOMPAT_METABOX = 0x00000100 EROFS_NAME_LEN = 255 class FixtureError(RuntimeError): pass def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def write_json(path: Path, value: object) -> None: with path.open("x", encoding="ascii") as output: json.dump(value, output, ensure_ascii=True, indent=2, sort_keys=True) output.write("\n") def require_tools(*names: str) -> None: missing = [name for name in names if shutil.which(name) is None] if missing: raise FixtureError(f"missing tools: {', '.join(missing)}") def run(argv: list[str], *, capture: bool = False) -> str: result = subprocess.run( argv, check=True, text=True, stdout=subprocess.PIPE if capture else None, stderr=subprocess.STDOUT if capture else None, ) return result.stdout if capture else "" def set_epoch(path: Path) -> None: for entry in sorted(path.rglob("*"), reverse=True): os.utime(entry, (0, 0), follow_symlinks=False) os.utime(path, (0, 0), follow_symlinks=False) def deterministic_bytes(length: int) -> bytes: return bytes( ((index * 131 + 17) ^ (index >> 3)) & 0xFF for index in range(length) ) def put_u16(image: bytearray, offset: int, value: int) -> None: struct.pack_into(" None: struct.pack_into(" None: struct.pack_into(" int: return struct.unpack_from(" int: return struct.unpack_from(" int: return struct.unpack_from(" None: erofs = ErofsImage(image, Path("generated")) erofs.update_checksum() def write_image(path: Path, image: bytearray) -> None: update_superblock_checksum(image) path.write_bytes(image) def build_image( artifact_dir: Path, fixture_dir: Path, uuid: str, image: str, source: str, *options: str, ) -> list[str]: argv = [ "mkfs.erofs", "-d0", "-x-1", "-T0", "--all-time", "--all-root", "--workers=1", "-U", uuid, *options, str(artifact_dir / image), str(fixture_dir / source), ] run(argv) return argv def inode_offset(image: bytes | bytearray, nid: int) -> int: block_bits = image[SUPER + 12] meta_blkaddr = u32(image, SUPER + 40) return (meta_blkaddr << block_bits) + (nid << 5) def compact_inode(image: bytes | bytearray, nid: int) -> tuple[int, int, int]: offset = inode_offset(image, nid) inode_format = u16(image, offset) if inode_format & 1: raise FixtureError("expected compact inode") return offset, (inode_format >> 1) & 7, u32(image, offset + 8) def inline_dir_entries( image: bytes | bytearray, nid: int ) -> tuple[int, int, int, list[tuple[int, int, int, bytes]]]: inode = inode_offset(image, nid) inode_format = u16(image, inode) inode_size = 64 if inode_format & 1 else 32 layout = (inode_format >> 1) & 7 size = u64(image, inode + 8) if inode_size == 64 else u32(image, inode + 8) if layout != 2 or u16(image, inode + 2) != 0: raise FixtureError("expected inline directory without xattrs") data = inode + inode_size first_nameoff = u16(image, data + 8) if first_nameoff < 12 or first_nameoff % 12: raise FixtureError("invalid first directory name offset") count = first_nameoff // 12 entries = [] for index in range(count): entry = data + index * 12 child_nid, nameoff = struct.unpack_from(" tuple[int, int, int, int, int, int]: offset = inode_offset(image, nid) inode_format = u16(image, offset) inode_size = 64 if inode_format & 1 else 32 size = u64(image, offset + 8) if inode_size == 64 else u32(image, offset + 8) xattr_count = u16(image, offset + 2) xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) layout = (inode_format >> 1) & 7 return offset, inode_format, inode_size, xattr_size, layout, size def directory_entries(image: bytes | bytearray, nid: int) -> list[tuple[bytes, int]]: offset, _, inode_size, xattr_size, layout, size = inode_info(image, nid) block_size = 1 << image[SUPER + 12] if not 0 < size <= block_size: raise FixtureError("directory must fit one block") if layout == 2: data = offset + inode_size + xattr_size elif layout == 0: data = u32(image, offset + 16) << image[SUPER + 12] else: raise FixtureError(f"unsupported directory layout {layout}") first_nameoff = u16(image, data + 8) if first_nameoff < 12 or first_nameoff % 12: raise FixtureError("invalid first directory name offset") count = first_nameoff // 12 entries = [] for index in range(count): entry = data + index * 12 child = u64(image, entry) nameoff = u16(image, entry + 8) endoff = u16(image, entry + 20) if index + 1 < count else size name = bytes(image[data + nameoff : data + endoff]).split(b"\0", 1)[0] if not name: raise FixtureError("empty directory name") entries.append((name, child)) return entries def child_nid(image: bytes | bytearray, parent_nid: int, name: bytes) -> int: return next(nid for entry_name, nid in directory_entries(image, parent_nid) if entry_name == name) def convert_inline_directory_to_plain(image: bytearray, nid: int) -> tuple[int, int]: offset, inode_format, inode_size, xattr_size, layout, size = inode_info(image, nid) if layout != 2 or inode_size != 64 or size <= 0: raise FixtureError("expected nonempty extended inline directory") block_size = 1 << image[SUPER + 12] tail_size = size % block_size if tail_size == 0: raise FixtureError("expected inline directory tail") raw_block = u32(image, offset + 16) full_size = size - tail_size inline_offset = offset + inode_size + xattr_size directory_data = bytes( image[raw_block * block_size : raw_block * block_size + full_size] + image[inline_offset : inline_offset + tail_size] ) if len(directory_data) != size: raise FixtureError("directory data reconstruction failed") new_raw_block = (len(image) + block_size - 1) // block_size image.extend(b"\0" * (new_raw_block * block_size - len(image))) image.extend(directory_data) image.extend(b"\0" * (-len(image) % block_size)) put_u16(image, offset, inode_format & ~(7 << 1)) put_u32(image, offset + 16, new_raw_block) put_u32(image, SUPER + 36, len(image) // block_size) if inode_info(image, nid)[4] != FLAT_PLAIN: raise FixtureError("directory layout conversion failed") return new_raw_block, len(image) // block_size def write_checksums(directory: Path) -> None: with (directory / "SHA256SUMS").open("x", encoding="ascii") as sums: for path in sorted(directory.glob("*.erofs")): sums.write(f"{sha256(path)} {path.name}\n") def make_metadata(fixture_dir: Path, artifact_dir: Path) -> list[list[str]]: require_tools("mkfs.erofs", "fsck.erofs", "dump.erofs") if fixture_dir.exists() or artifact_dir.exists(): raise FixtureError("metadata source and artifact paths must be absent") for relative in ( "inline", "special", "namei/alpha/bravo/charlie", "namei/alpha/sibling", "namei/wide", "pager", "nfs/basic/subdir", "nlink/subdir", ): (fixture_dir / relative).mkdir(parents=True, exist_ok=True) artifact_dir.mkdir(parents=True) (fixture_dir / "inline/inline.txt").write_text( "inline-tail-payload-0123456789\n", encoding="ascii" ) (fixture_dir / "special/target").write_text("special target\n", encoding="ascii") os.symlink("target", fixture_dir / "special/link") os.mknod( fixture_dir / "special/char-large", stat.S_IFCHR | 0o666, os.makedev(2748, 344865), ) os.mknod( fixture_dir / "special/block-large", stat.S_IFBLK | 0o666, os.makedev(2748, 344865), ) os.mkfifo(fixture_dir / "special/fifo") (fixture_dir / "namei/alpha/bravo/charlie/payload.txt").write_text( "cold nested lookup payload\n", encoding="ascii" ) (fixture_dir / "namei/alpha/bravo/repeat.txt").write_text( "repeat lookup payload\n", encoding="ascii" ) (fixture_dir / "namei/alpha/sibling/marker.txt").write_text( "sibling marker\n", encoding="ascii" ) for index in range(320): name = f"entry-{index:03d}-abcdefghijklmnopqrstuvwxyz.txt" (fixture_dir / "namei/wide" / name).write_text( f"wide entry {index:03d}\n", encoding="ascii" ) (fixture_dir / "pager/pager.bin").write_bytes(deterministic_bytes(5 * 4096 + 731)) (fixture_dir / "nfs/basic/regular.txt").write_text( "stable file handle payload\n", encoding="ascii" ) (fixture_dir / "nfs/basic/subdir/child.txt").write_text("child\n", encoding="ascii") os.symlink("regular.txt", fixture_dir / "nfs/basic/link-to-regular") os.mkfifo(fixture_dir / "nfs/basic/test.fifo") (fixture_dir / "nlink/single.txt").write_text("single\n", encoding="ascii") (fixture_dir / "nlink/hard-a.txt").write_text("hardlinked\n", encoding="ascii") os.link(fixture_dir / "nlink/hard-a.txt", fixture_dir / "nlink/hard-b.txt") (fixture_dir / "nlink/subdir/child.txt").write_text("child\n", encoding="ascii") set_epoch(fixture_dir) commands = [ build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555551", "inline.erofs", "inline"), build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555552", "special-compact.erofs", "special", "-E", "force-inode-compact"), build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555553", "special-extended.erofs", "special", "-E", "force-inode-extended"), build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555554", "namei-base.erofs", "namei"), build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555555", "pager-plain.erofs", "pager", "-E", "noinline_data"), build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555556", "pager-lz4.erofs", "pager", "-z", "lz4"), build_image(artifact_dir, fixture_dir, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee1", "nfs-a.erofs", "nfs"), build_image(artifact_dir, fixture_dir, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee2", "nfs-b.erofs", "nfs"), build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555557", "nlink.erofs", "nlink", "-E", "force-inode-compact"), ] inline = bytearray((artifact_dir / "inline.erofs").read_bytes()) _, _, _, inline_root = inline_dir_entries(inline, u16(inline, SUPER + 14)) inline_nid = next(nid for nid, _, _, name in inline_root if name == b"inline.txt") inline_inode, layout, inline_size = compact_inode(inline, inline_nid) if layout != 2 or u16(inline, inline_inode + 2) != 0: raise FixtureError("inline fixture has unexpected layout") inline_cross_xattr_icount = 695 inline_xattr_size = 12 + 4 * (inline_cross_xattr_icount - 1) inline_data_off = inline_inode + 32 + inline_xattr_size block_size = 1 << inline[SUPER + 12] if inline_data_off % block_size + inline_size <= block_size: raise FixtureError("inline fixture does not cross the block boundary") inline_cross = bytearray(inline) put_u16(inline_cross, inline_inode + 2, inline_cross_xattr_icount) write_image(artifact_dir / "inline-cross-block.erofs", inline_cross) namei = bytearray((artifact_dir / "namei-base.erofs").read_bytes()) root_nid = u16(namei, SUPER + 14) root_inode, root_data, _, root_entries = inline_dir_entries(namei, root_nid) wide_nid = next(nid for nid, _, _, name in root_entries if name == b"wide") wide_inode, wide_layout, _ = compact_inode(namei, wide_nid) if wide_layout != 2: raise FixtureError("wide directory is not inline") block_bits = namei[SUPER + 12] wide_block = u32(namei, wide_inode + 16) << block_bits wide_first_nameoff = u16(namei, wide_block + 8) wide_count = wide_first_nameoff // 12 wide_last_nameoff = u16(namei, wide_block + (wide_count - 1) * 12 + 8) padding_nul = namei.index(0, wide_block + wide_last_nameoff, wide_block + (1 << block_bits)) nonzero_padding = bytearray(namei) nonzero_padding[padding_nul + 1 : padding_nul + 9] = b"PAD!ERO!" write_image(artifact_dir / "namei-padding-nonzero.erofs", nonzero_padding) short_block = bytearray(namei) put_u32(short_block, root_inode + 8, 8) write_image(artifact_dir / "namei-corrupt-short.erofs", short_block) bad_nameoff = bytearray(namei) first_nameoff = u16(bad_nameoff, root_data + 8) put_u16(bad_nameoff, root_data + 20, first_nameoff) write_image(artifact_dir / "namei-corrupt-nameoff.erofs", bad_nameoff) bad_name = bytearray(namei) slash_offset = wide_block + wide_last_nameoff + 5 if bad_name[slash_offset] in (0, ord("/")): raise FixtureError("name corruption target is unsuitable") bad_name[slash_offset] = ord("/") write_image(artifact_dir / "namei-corrupt-name.erofs", bad_name) nlink = bytearray((artifact_dir / "nlink.erofs").read_bytes()) nlink_root_nid = u16(nlink, SUPER + 14) _, _, _, nlink_entries = inline_dir_entries(nlink, nlink_root_nid) single_nid = next(nid for nid, _, _, name in nlink_entries if name == b"single.txt") single_inode, _, _ = compact_inode(nlink, single_nid) nlink1 = bytearray(nlink) put_u16(nlink1, single_inode, u16(nlink1, single_inode) | 0x10) put_u16(nlink1, single_inode + 6, 0x1234) write_image(artifact_dir / "nlink1-patched.erofs", nlink1) with (artifact_dir / "fixture-evidence.txt").open("x", encoding="ascii") as output: output.write( f"inline_nid={inline_nid} inode_off={inline_inode} " f"inode_blockoff={inline_inode % block_size} " f"xattr_icount=0->{inline_cross_xattr_icount} " f"inline_data_blockoff={inline_data_off % block_size} " f"inline_size={inline_size}\n" ) output.write( f"wide_nid={wide_nid} block={wide_block >> block_bits} " f"dirents={wide_count} last_nameoff={wide_last_nameoff} " f"padding_patch={padding_nul + 1 - wide_block}:" f"{padding_nul + 9 - wide_block}\n" ) output.write(f"nlink1_nid={single_nid} i_format_bit4=1 i_nb=0x1234\n") for image_name in ("special-compact.erofs", "special-extended.erofs"): image = bytearray((artifact_dir / image_name).read_bytes()) special_root = u16(image, SUPER + 14) _, _, _, entries = inline_dir_entries(image, special_root) output.write(image_name + "\n") for nid, _, _, name in entries: if name not in (b"char-large", b"block-large", b"fifo"): continue offset = inode_offset(image, nid) output.write( f" {name.decode()} nid={nid} inode_size=" f"{64 if u16(image, offset) & 1 else 32} " f"raw_u={u32(image, offset + 16):#010x}\n" ) write_checksums(artifact_dir) run(["fsck.erofs", "-d1", str(artifact_dir / "namei-padding-nonzero.erofs")]) run( [ "dump.erofs", "--cat", "--path=/wide/entry-079-abcdefghijklmnopqrstuvwxyz.txt", str(artifact_dir / "namei-padding-nonzero.erofs"), ] ) return commands def make_final(fixture_dir: Path, artifact_dir: Path) -> list[list[str]]: require_tools("mkfs.erofs") if fixture_dir.exists() or artifact_dir.exists(): raise FixtureError("final source and artifact paths must be absent") for relative in ("fallback", "oversize", "extent", "large/huge"): (fixture_dir / relative).mkdir(parents=True, exist_ok=True) artifact_dir.mkdir(parents=True) (fixture_dir / "fallback/root.txt").write_text("48-bit fallback root\n", encoding="ascii") (fixture_dir / "oversize/big.dat").write_bytes(b"x") (fixture_dir / "extent/hole.dat").write_bytes(b"\0" * (1024 * 1024)) (fixture_dir / "large/huge/anchor.txt").write_text( "large directory anchor\n", encoding="ascii" ) for index in range(400): (fixture_dir / "large/huge" / f"entry-{index:03d}-abcdefghijklmnopqrstuvwxyz.txt").write_text( f"entry {index:03d}\n", encoding="ascii" ) set_epoch(fixture_dir) commands = [ build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666650", "fallback-base.erofs", "fallback", "-E", "force-inode-compact"), build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666651", "oversize-base.erofs", "oversize", "-E", "force-inode-extended"), build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666652", "extent-base.erofs", "extent", "-E", "legacy-compress,force-inode-extended", "-z", "lz4"), build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666653", "large-dir-base.erofs", "large", "-E", "force-inode-extended"), ] feature_incompat = SUPER + 80 rootnid_2b = SUPER + 14 blocks_lo = SUPER + 36 rootnid_8b = SUPER + 112 evidence = [] fallback = bytearray((artifact_dir / "fallback-base.erofs").read_bytes()) fallback_root = u16(fallback, rootnid_2b) fallback_blocks = u32(fallback, blocks_lo) if fallback_root == 0 or u64(fallback, rootnid_8b) != 0: raise FixtureError("unexpected fallback base root") put_u32(fallback, feature_incompat, u32(fallback, feature_incompat) | 0x80) put_u64(fallback, rootnid_8b, 0) write_image(artifact_dir / "fallback-48bit-root2.erofs", fallback) evidence.append( f"fallback rootnid_2b={fallback_root} rootnid_8b=0 " f"blocks_lo={fallback_blocks} union=0x{u16(fallback, rootnid_2b):04x}" ) oversize = bytearray((artifact_dir / "oversize-base.erofs").read_bytes()) oversize_root = u16(oversize, rootnid_2b) oversize_nid = child_nid(oversize, oversize_root, b"big.dat") oversize_off, oversize_format, oversize_isize, _, _, _ = inode_info(oversize, oversize_nid) if not oversize_format & 1 or oversize_isize != 64: raise FixtureError("oversize inode is not extended") put_u64(oversize, oversize_off + 8, 1 << 63) write_image(artifact_dir / "extended-size-bit63.erofs", oversize) evidence.append( f"oversize nid={oversize_nid} inode_off={oversize_off} " f"i_size=0x{u64(oversize, oversize_off + 8):016x}" ) extent = bytearray((artifact_dir / "extent-base.erofs").read_bytes()) extent_root = u16(extent, rootnid_2b) extent_nid = child_nid(extent, extent_root, b"hole.dat") extent_off, extent_format, extent_isize, extent_xattr, extent_layout, _ = inode_info(extent, extent_nid) if not extent_format & 1 or extent_isize != 64 or extent_layout != 1: raise FixtureError("extent inode has unexpected format") header = (extent_off + extent_isize + extent_xattr + 7) & ~7 record = (header + 8 + 15) & ~15 root_off = inode_offset(extent, extent_root) if header < root_off + 64 and root_off < record + 16: raise FixtureError("extent metadata overlaps root inode") hole_size = 5 * 1024 * 1024 * 1024 + 4096 put_u64(extent, extent_off + 8, hole_size) struct.pack_into(" None: testdir = source / "testdir" (testdir / "subdir").mkdir(parents=True) for index in range(5): (testdir / f"dir-{index:02d}").mkdir() (source / "parent/child/grandchild").mkdir(parents=True) (testdir / "file.txt").write_text("repo22 G3 file payload\n", encoding="ascii") for index in range(12): (testdir / f"regular-{index:02d}.txt").write_text(f"regular {index:02d}\n", encoding="ascii") for name in ("File.txt", "FILE.TXT", "FiLe.TxT"): (testdir / name).write_text(f"case variant {name}\n", encoding="ascii") for index in range(1, 4): (testdir / f"file{index}.txt").write_text(f"cache file {index}\n", encoding="ascii") (testdir / "subdir/child.txt").write_text("child\n", encoding="ascii") (testdir / "script.sh").write_text("#!/bin/sh\nexit 0\n", encoding="ascii") (testdir / "rootonly.txt").write_text("restricted\n", encoding="ascii") (testdir / "writeonly.txt").write_text("write only mode\n", encoding="ascii") (testdir / "noexec.txt").write_text("not executable\n", encoding="ascii") (testdir / ("long-" + "x" * 250)).write_text("255 byte name\n", encoding="ascii") long_target = "subdir/" + "y" * 112 (testdir / "subdir" / ("y" * 112)).write_text("long target\n", encoding="ascii") os.symlink("file.txt", testdir / "shortlink") os.symlink("subdir/child.txt", testdir / "relative-link") os.symlink("missing-target", testdir / "broken-link") os.symlink(long_target, testdir / "long-link") (source / "parent/file.txt").write_text("parent file\n", encoding="ascii") (source / "parent/child/grandchild/marker.txt").write_text("grandchild\n", encoding="ascii") (source / "pager.bin").write_bytes(deterministic_bytes(5 * 4096 + 731)) os.chmod(testdir / "script.sh", 0o755) os.chmod(testdir / "rootonly.txt", 0o600) os.chmod(testdir / "writeonly.txt", 0o200) os.chmod(testdir / "noexec.txt", 0o644) set_epoch(source) def directory_data_offset(image: ErofsImage, inode: object) -> int: if inode.layout == 2: return inode.offset + inode.inode_size + inode.xattr_size if inode.layout == FLAT_PLAIN: return inode.start_block << image.block_bits raise FixtureError(f"unsupported directory layout {inode.layout}") def limited_directory_entries(image: ErofsImage, inode: object) -> list[object]: if inode.size == 0 or inode.size > image.block_size: raise FixtureError("G3 qualifier accepts only one-block path components") return image.directory_entries(inode) def resolve(image: ErofsImage, path: str) -> object: inode = image.inode(image.root_nid) for component in path.strip("/").split("/"): if not component: continue entry = next( ( item for item in limited_directory_entries(image, inode) if item.name.decode("ascii") == component ), None, ) if entry is None: raise FixtureError(f"path not found: {path}") inode = image.inode(entry.nid) return inode def validate_vfs_image(image_path: Path, source: Path) -> list[str]: image = ErofsImage.load(image_path) image.validate_superblock() evidence = [ f"image={image_path.name} provider_bytes={len(image.data)} " f"block_size={image.block_size} blocks={image.blocks} " f"root_nid={image.root_nid} checksum_valid={image.checksum_valid()}" ] paths = [ "/testdir", "/testdir/file.txt", "/testdir/script.sh", "/testdir/rootonly.txt", "/testdir/writeonly.txt", "/testdir/shortlink", "/testdir/long-link", "/pager.bin", "/parent/child/grandchild", ] for path in paths: source_path = source / path.lstrip("/") source_stat = source_path.lstat() inode = resolve(image, path) if inode.mode & S_IFMT != stat.S_IFMT(source_stat.st_mode): raise FixtureError(f"{path}: inode type differs from source") if inode.mode & 0o7777 != source_stat.st_mode & 0o7777: raise FixtureError(f"{path}: inode permissions differ from source") expected_size = None if source_path.is_symlink(): expected_size = len(os.readlink(source_path).encode("ascii")) elif source_path.is_file(): expected_size = source_stat.st_size if expected_size is not None and inode.size != expected_size: raise FixtureError(f"{path}: size {inode.size}, expected {expected_size}") evidence.append( f"path={path} nid={inode.nid} inode_offset={inode.offset} " f"layout={inode.layout} mode={inode.mode:#07o} size={inode.size} " f"start_block={inode.start_block}" ) source_names = sorted(item.name for item in (source / "testdir").iterdir()) testdir = resolve(image, "/testdir") image_names = sorted( entry.name.decode("ascii") for entry in limited_directory_entries(image, testdir) if entry.name not in (b".", b"..") ) if source_names != image_names: raise FixtureError("/testdir names differ from source") evidence.append( f"path=/testdir real_entries={len(image_names)} " f"directory_size={testdir.size} layout={testdir.layout}" ) return evidence def write_source_hashes(source: Path, output: Path) -> None: with output.open("x", encoding="ascii") as sums: for path in sorted(source.rglob("*")): if path.is_file() and not path.is_symlink(): sums.write(f"{sha256(path)} {path.relative_to(source)}\n") def make_vfs(output: Path) -> list[list[str]]: require_tools("mkfs.erofs") if output.exists(): raise FixtureError(f"output already exists: {output}") source = output / "source" output.mkdir(parents=True) make_vfs_source(source) plain = output / "vfs-plain.erofs" lz4 = output / "vfs-lz4.erofs" common = ["mkfs.erofs", "-d0", "-T0", "--all-time", "--all-root", "--workers=1"] commands = [ common + ["-x-1", "-E", "noinline_data", "-U", "33333333-4444-5555-6666-777777777771", str(plain), str(source)], common + ["-z", "lz4", "-U", "33333333-4444-5555-6666-777777777772", str(lz4), str(source)], ] for command in commands: run(command) evidence = validate_vfs_image(plain, source) + validate_vfs_image(lz4, source) (output / "fixture-evidence.txt").write_text("\n".join(evidence) + "\n", encoding="ascii") (output / "expected-testdir.txt").write_text( "\n".join(sorted(item.name for item in (source / "testdir").iterdir())) + "\n", encoding="ascii", ) write_source_hashes(source, output / "SOURCE-SHA256SUMS") with (output / "IMAGE-SHA256SUMS").open("x", encoding="ascii") as sums: for path in (lz4, plain): sums.write(f"{sha256(path)} {path.name}\n") return commands def make_large_prefix(source: Path, output: Path, evidence_path: Path) -> None: image = ErofsImage.load(source) image.validate_superblock() huge = resolve(image, "/huge") expected_size = HUGE_DIRECTORY_BLOCKS * image.block_size if huge.inode_size != 64 or huge.layout != FLAT_PLAIN: raise FixtureError("TC153 directory is not extended FLAT_PLAIN/Layout0") if huge.size != expected_size: raise FixtureError(f"TC153 directory size {huge.size}, expected {expected_size}") if huge.start_block == 0: raise FixtureError("TC153 directory has no plain-data start block") provider_blocks = huge.start_block + HUGE_DIRECTORY_BLOCKS if provider_blocks > 0xFFFFFFFF: raise FixtureError("TC153 sparse provider does not fit blocks_lo") image.put_u32(SUPER + 36, provider_blocks) image.update_checksum() image.validate_superblock() output.parent.mkdir(parents=True, exist_ok=True) image.save(output) checked = ErofsImage.load(output) checked_huge = resolve(checked, "/huge") if checked.blocks != provider_blocks or checked_huge.layout != FLAT_PLAIN or checked_huge.size != expected_size: raise FixtureError("TC153 sparse prefix self-check failed") provider_bytes = provider_blocks * checked.block_size evidence_path.write_text( "tc153-sparse-prefix " f"nid={checked_huge.nid} inode_offset={checked_huge.offset} " f"layout=flat-plain start_block={checked_huge.start_block} " f"directory_blocks={HUGE_DIRECTORY_BLOCKS} last_block={HUGE_DIRECTORY_BLOCKS - 1} " f"blocks_lo={provider_blocks} provider_bytes={provider_bytes} " f"prefix_bytes={len(checked.data)} checksum_valid={checked.checksum_valid()} " f"sha256={sha256(output)}\n", encoding="ascii", ) def path_type(mode: int) -> str: if stat.S_ISREG(mode): return "file" if stat.S_ISDIR(mode): return "directory" if stat.S_ISLNK(mode): return "symlink" if stat.S_ISFIFO(mode): return "fifo" if stat.S_ISCHR(mode): return "char" if stat.S_ISBLK(mode): return "block" raise FixtureError(f"unsupported source type: {mode:#o}") def tree_inventory(root: Path) -> dict[str, object]: if not root.is_dir(): raise FixtureError(f"inventory root is absent: {root}") paths = [root] + sorted(root.rglob("*")) hardlinks: dict[tuple[int, int], list[str]] = {} for path in paths: info = path.lstat() if stat.S_ISREG(info.st_mode) and info.st_nlink > 1: hardlinks.setdefault((info.st_dev, info.st_ino), []).append( "." if path == root else path.relative_to(root).as_posix() ) hardlink_names = { key: ",".join(sorted(names)) for key, names in hardlinks.items() } entries = [] for path in paths: info = path.lstat() relative = "." if path == root else path.relative_to(root).as_posix() item: dict[str, object] = { "path": relative, "type": path_type(info.st_mode), "mode": info.st_mode & 0o7777, } if stat.S_ISREG(info.st_mode): item.update(size=info.st_size, sha256=sha256(path)) if info.st_nlink > 1: item["hardlink_group"] = hardlink_names[(info.st_dev, info.st_ino)] elif stat.S_ISLNK(info.st_mode): item["target"] = os.readlink(path) elif stat.S_ISCHR(info.st_mode) or stat.S_ISBLK(info.st_mode): item.update(major=os.major(info.st_rdev), minor=os.minor(info.st_rdev)) entries.append(item) encoded = json.dumps(entries, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode("ascii") return {"sha256": hashlib.sha256(encoded).hexdigest(), "entries": entries} def compare_set( left_source: Path, left_artifacts: Path, right_source: Path, right_artifacts: Path, output: Path, ) -> None: inventories = { "left_source": tree_inventory(left_source), "left_artifacts": tree_inventory(left_artifacts), "right_source": tree_inventory(right_source), "right_artifacts": tree_inventory(right_artifacts), } comparison = { name: {"sha256": value["sha256"], "entries": len(value["entries"])} for name, value in inventories.items() } comparison["source_equal"] = inventories["left_source"] == inventories["right_source"] comparison["artifacts_equal"] = inventories["left_artifacts"] == inventories["right_artifacts"] comparison["status"] = ( "PASS" if comparison["source_equal"] and comparison["artifacts_equal"] else "RUNNER_FAIL" ) if comparison["status"] != "PASS": comparison["differences"] = { "source": sorted( set(json.dumps(item, sort_keys=True) for item in inventories["left_source"]["entries"]) ^ set(json.dumps(item, sort_keys=True) for item in inventories["right_source"]["entries"]) )[:40], "artifacts": sorted( set(json.dumps(item, sort_keys=True) for item in inventories["left_artifacts"]["entries"]) ^ set(json.dumps(item, sort_keys=True) for item in inventories["right_artifacts"]["entries"]) )[:40], } write_json(output, comparison) if comparison["status"] != "PASS": raise FixtureError("stable and archived fixture trees differ") def verify_checksum_file(directory: Path, filename: str) -> None: for line in (directory / filename).read_text(encoding="ascii").splitlines(): digest, relative = line.split(" ", 1) if sha256(directory / relative) != digest: raise FixtureError(f"checksum mismatch: {directory / relative}") def make_expected_wide(fixture_dir: Path, artifact_dir: Path) -> None: names = sorted(path.name for path in (fixture_dir / "namei/wide").iterdir() if path.is_file()) if len(names) != 320: raise FixtureError(f"wide directory contains {len(names)} files") (artifact_dir / "expected-wide.txt").write_text("\n".join(names) + "\n", encoding="ascii") def make_all(output: Path) -> None: if output.exists(): raise FixtureError(f"output already exists: {output}") output.mkdir(parents=True) commands = { "vfs": make_vfs(output / "vfs"), "metadata": make_metadata(output / "metadata-source", output / "metadata"), "final": make_final(output / "final-source", output / "final"), } make_expected_wide(output / "metadata-source", output / "metadata") make_large_prefix( output / "final/large-dir-intmax.erofs", output / "final/large-dir-intmax-sparse-prefix.erofs", output / "final/tc153-sparse-evidence.txt", ) (output / "final/TC153-SPARSE-SHA256").write_text( f"{sha256(output / 'final/large-dir-intmax-sparse-prefix.erofs')} " "large-dir-intmax-sparse-prefix.erofs\n", encoding="ascii", ) verify_checksum_file(output / "vfs", "IMAGE-SHA256SUMS") verify_checksum_file(output / "vfs/source", "../SOURCE-SHA256SUMS") verify_checksum_file(output / "metadata", "SHA256SUMS") verify_checksum_file(output / "final", "SHA256SUMS") output_text = str(output) canonical_commands = { name: [ [ "$OUTPUT" + argument[len(output_text) :] if argument == output_text or argument.startswith(output_text + os.sep) else argument for argument in argv ] for argv in command_set ] for name, command_set in commands.items() } manifest = { "schema_version": 1, "generator": "tests/pre15/fixtures/g3.py", "generator_sha256": sha256(Path(__file__)), "mkfs_version": run(["mkfs.erofs", "-V"], capture=True).splitlines()[0], "commands": canonical_commands, "sets": { "vfs": tree_inventory(output / "vfs"), "metadata_source": tree_inventory(output / "metadata-source"), "metadata": tree_inventory(output / "metadata"), "final_source": tree_inventory(output / "final-source"), "final": tree_inventory(output / "final"), }, } write_json(output / "G3-MANIFEST.json", manifest) def verify_all(output: Path) -> None: manifest = json.loads((output / "G3-MANIFEST.json").read_text(encoding="ascii")) roots = { "vfs": output / "vfs", "metadata_source": output / "metadata-source", "metadata": output / "metadata", "final_source": output / "final-source", "final": output / "final", } for name, root in roots.items(): if tree_inventory(root) != manifest["sets"][name]: raise FixtureError(f"G3 manifest mismatch: {name}") verify_checksum_file(output / "vfs", "IMAGE-SHA256SUMS") verify_checksum_file(output / "vfs/source", "../SOURCE-SHA256SUMS") verify_checksum_file(output / "metadata", "SHA256SUMS") verify_checksum_file(output / "final", "SHA256SUMS") def source_constant(path: Path, name: str) -> int: for line in path.read_text(encoding="utf-8").splitlines(): fields = line.split() if len(fields) >= 3 and fields[0] == "#define" and fields[1] == name: return int(fields[2], 0) raise FixtureError(f"constant not found: {path}:{name}") def validate_xattr_sources(freebsd_root: Path, linux_root: Path) -> dict[str, str]: constants = ( "EROFS_FEATURE_COMPAT_SB_CHKSUM", "EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX", "EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES", "EROFS_FEATURE_INCOMPAT_METABOX", ) for name in constants: freebsd = source_constant(freebsd_root / "erofs_fs.h", name) linux = source_constant(linux_root / "erofs_fs.h", name) if freebsd != linux: raise FixtureError(f"FreeBSD/Linux constant mismatch: {name}") return { "freebsd_erofs_fs_sha256": sha256(freebsd_root / "erofs_fs.h"), "freebsd_xattr_sha256": sha256(freebsd_root / "xattr.c"), "linux_erofs_fs_sha256": sha256(linux_root / "erofs_fs.h"), "linux_xattr_sha256": sha256(linux_root / "xattr.c"), } def make_contract_image( path: Path, feature_compat: int, feature_incompat: int, prefix_start: int, prefix_count: int, packed_nid: int, metabox_nid: int, primary: bytes, ) -> None: data = bytearray(4096) put_u32(data, SUPER, MAGIC) put_u32(data, SUPER + 8, feature_compat | FEATURE_COMPAT_SB_CHKSUM) data[SUPER + 12] = 12 put_u16(data, SUPER + 14, 1) put_u32(data, SUPER + 36, 1) put_u32(data, SUPER + 40, 0) put_u32(data, SUPER + 80, feature_incompat) data[SUPER + 91] = prefix_count put_u32(data, SUPER + 92, prefix_start // 4) put_u64(data, SUPER + 96, packed_nid) put_u64(data, SUPER + 128, metabox_nid) if prefix_start + len(primary) <= len(data): data[prefix_start : prefix_start + len(primary)] = primary write_image(path, data) def select_prefix_backing(image: ErofsImage, carriers: dict[str, bytes]) -> tuple[str, bytes]: plain = bool(image.feature_compat & FEATURE_COMPAT_PLAIN_XATTR_PFX) metabox = bool(image.feature_incompat & FEATURE_INCOMPAT_METABOX) packed_nid = image.u64(SUPER + 96) if plain: return "primary", bytes(image.data) if metabox: return "metabox", carriers["metabox"] if packed_nid: return "packed", carriers["packed"] return "primary-legacy", bytes(image.data) def parse_prefix(backing: bytes, offset: int) -> tuple[int, bytes]: aligned = (offset + 3) & ~3 if aligned > len(backing) - 2: raise FixtureError("EINTEGRITY: prefix length is outside backing") raw_length = struct.unpack_from(" EROFS_NAME_LEN + 1: raise FixtureError("EINTEGRITY: invalid prefix length") if aligned + 2 + length > len(backing): raise FixtureError("EINTEGRITY: prefix payload is outside backing") payload = backing[aligned + 2 : aligned + 2 + length] return payload[0], payload[1:] def make_xattr_legacy( spec_path: Path, output: Path, freebsd_root: Path, linux_root: Path ) -> None: if output.exists(): raise FixtureError(f"output already exists: {output}") spec = json.loads(spec_path.read_text(encoding="ascii")) output.mkdir(parents=True) source_hashes = validate_xattr_sources(freebsd_root, linux_root) prefix_start = spec["prefix_start"] payload = bytes([spec["base_index"]]) + spec["infix"].encode("ascii") record = struct.pack(" argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) make_all_parser = subparsers.add_parser("make-all") make_all_parser.add_argument("--output", type=Path, required=True) verify_parser = subparsers.add_parser("verify") verify_parser.add_argument("--output", type=Path, required=True) vfs_parser = subparsers.add_parser("make-vfs") vfs_parser.add_argument("--output", type=Path, required=True) metadata_parser = subparsers.add_parser("make-metadata") metadata_parser.add_argument("--source", type=Path, required=True) metadata_parser.add_argument("--output", type=Path, required=True) final_parser = subparsers.add_parser("make-final") final_parser.add_argument("--source", type=Path, required=True) final_parser.add_argument("--output", type=Path, required=True) large_parser = subparsers.add_parser("make-large-prefix") large_parser.add_argument("--source", type=Path, required=True) large_parser.add_argument("--output", type=Path, required=True) large_parser.add_argument("--evidence", type=Path, required=True) compare_parser = subparsers.add_parser("compare-set") compare_parser.add_argument("--left-source", type=Path, required=True) compare_parser.add_argument("--left-artifacts", type=Path, required=True) compare_parser.add_argument("--right-source", type=Path, required=True) compare_parser.add_argument("--right-artifacts", type=Path, required=True) compare_parser.add_argument("--output", type=Path, required=True) xattr_parser = subparsers.add_parser("make-xattr-legacy") xattr_parser.add_argument("--spec", type=Path, required=True) xattr_parser.add_argument("--output", type=Path, required=True) xattr_parser.add_argument("--freebsd-root", type=Path, required=True) xattr_parser.add_argument("--linux-root", type=Path, required=True) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> None: os.umask(0o022) args = parse_args(argv) if args.command == "make-all": make_all(args.output) elif args.command == "verify": verify_all(args.output) elif args.command == "make-vfs": make_vfs(args.output) elif args.command == "make-metadata": make_metadata(args.source, args.output) make_expected_wide(args.source, args.output) elif args.command == "make-final": make_final(args.source, args.output) elif args.command == "make-large-prefix": make_large_prefix(args.source, args.output, args.evidence) elif args.command == "compare-set": compare_set( args.left_source, args.left_artifacts, args.right_source, args.right_artifacts, args.output, ) else: make_xattr_legacy(args.spec, args.output, args.freebsd_root, args.linux_root) if __name__ == "__main__": try: main() except (FixtureError, OSError, subprocess.CalledProcessError, ValueError) as error: print(f"fixture error: {error}", file=sys.stderr) raise SystemExit(1) from error