1143 lines
48 KiB
Python
Executable File
1143 lines
48 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Generate, compare, and verify deterministic Pre15 G3 fixtures."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import stat
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
TESTS_DIR = Path(__file__).resolve().parents[2]
|
|
if str(TESTS_DIR) not in sys.path:
|
|
sys.path.insert(0, str(TESTS_DIR))
|
|
|
|
from erofs_fixture import ErofsImage, MAGIC, SUPER
|
|
|
|
|
|
FLAT_PLAIN = 0
|
|
HUGE_DIRECTORY_BLOCKS = (1 << 31) + 1
|
|
S_IFMT = 0o170000
|
|
FEATURE_COMPAT_SB_CHKSUM = 0x00000001
|
|
FEATURE_COMPAT_PLAIN_XATTR_PFX = 0x00000010
|
|
FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040
|
|
FEATURE_INCOMPAT_METABOX = 0x00000100
|
|
EROFS_NAME_LEN = 255
|
|
|
|
|
|
class FixtureError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def write_json(path: Path, value: object) -> None:
|
|
with path.open("x", encoding="ascii") as output:
|
|
json.dump(value, output, ensure_ascii=True, indent=2, sort_keys=True)
|
|
output.write("\n")
|
|
|
|
|
|
def require_tools(*names: str) -> None:
|
|
missing = [name for name in names if shutil.which(name) is None]
|
|
if missing:
|
|
raise FixtureError(f"missing tools: {', '.join(missing)}")
|
|
|
|
|
|
def run(argv: list[str], *, capture: bool = False) -> str:
|
|
result = subprocess.run(
|
|
argv,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE if capture else None,
|
|
stderr=subprocess.STDOUT if capture else None,
|
|
)
|
|
return result.stdout if capture else ""
|
|
|
|
|
|
def set_epoch(path: Path) -> None:
|
|
for entry in sorted(path.rglob("*"), reverse=True):
|
|
os.utime(entry, (0, 0), follow_symlinks=False)
|
|
os.utime(path, (0, 0), follow_symlinks=False)
|
|
|
|
|
|
def deterministic_bytes(length: int) -> bytes:
|
|
return bytes(
|
|
((index * 131 + 17) ^ (index >> 3)) & 0xFF
|
|
for index in range(length)
|
|
)
|
|
|
|
|
|
def put_u16(image: bytearray, offset: int, value: int) -> None:
|
|
struct.pack_into("<H", image, offset, value)
|
|
|
|
|
|
def put_u32(image: bytearray, offset: int, value: int) -> None:
|
|
struct.pack_into("<I", image, offset, value)
|
|
|
|
|
|
def put_u64(image: bytearray, offset: int, value: int) -> None:
|
|
struct.pack_into("<Q", image, offset, value)
|
|
|
|
|
|
def u16(image: bytes | bytearray, offset: int) -> int:
|
|
return struct.unpack_from("<H", image, offset)[0]
|
|
|
|
|
|
def u32(image: bytes | bytearray, offset: int) -> int:
|
|
return struct.unpack_from("<I", image, offset)[0]
|
|
|
|
|
|
def u64(image: bytes | bytearray, offset: int) -> int:
|
|
return struct.unpack_from("<Q", image, offset)[0]
|
|
|
|
|
|
def update_superblock_checksum(image: bytearray) -> None:
|
|
erofs = ErofsImage(image, Path("generated"))
|
|
erofs.update_checksum()
|
|
|
|
|
|
def write_image(path: Path, image: bytearray) -> None:
|
|
update_superblock_checksum(image)
|
|
path.write_bytes(image)
|
|
|
|
|
|
def build_image(
|
|
artifact_dir: Path,
|
|
fixture_dir: Path,
|
|
uuid: str,
|
|
image: str,
|
|
source: str,
|
|
*options: str,
|
|
) -> list[str]:
|
|
argv = [
|
|
"mkfs.erofs",
|
|
"-d0",
|
|
"-x-1",
|
|
"-T0",
|
|
"--all-time",
|
|
"--all-root",
|
|
"--workers=1",
|
|
"-U",
|
|
uuid,
|
|
*options,
|
|
str(artifact_dir / image),
|
|
str(fixture_dir / source),
|
|
]
|
|
run(argv)
|
|
return argv
|
|
|
|
|
|
def inode_offset(image: bytes | bytearray, nid: int) -> int:
|
|
block_bits = image[SUPER + 12]
|
|
meta_blkaddr = u32(image, SUPER + 40)
|
|
return (meta_blkaddr << block_bits) + (nid << 5)
|
|
|
|
|
|
def compact_inode(image: bytes | bytearray, nid: int) -> tuple[int, int, int]:
|
|
offset = inode_offset(image, nid)
|
|
inode_format = u16(image, offset)
|
|
if inode_format & 1:
|
|
raise FixtureError("expected compact inode")
|
|
return offset, (inode_format >> 1) & 7, u32(image, offset + 8)
|
|
|
|
|
|
def inline_dir_entries(
|
|
image: bytes | bytearray, nid: int
|
|
) -> tuple[int, int, int, list[tuple[int, int, int, bytes]]]:
|
|
inode = inode_offset(image, nid)
|
|
inode_format = u16(image, inode)
|
|
inode_size = 64 if inode_format & 1 else 32
|
|
layout = (inode_format >> 1) & 7
|
|
size = u64(image, inode + 8) if inode_size == 64 else u32(image, inode + 8)
|
|
if layout != 2 or u16(image, inode + 2) != 0:
|
|
raise FixtureError("expected inline directory without xattrs")
|
|
data = inode + inode_size
|
|
first_nameoff = u16(image, data + 8)
|
|
if first_nameoff < 12 or first_nameoff % 12:
|
|
raise FixtureError("invalid first directory name offset")
|
|
count = first_nameoff // 12
|
|
entries = []
|
|
for index in range(count):
|
|
entry = data + index * 12
|
|
child_nid, nameoff = struct.unpack_from("<QH", image, entry)
|
|
endoff = u16(image, data + (index + 1) * 12 + 8) if index + 1 < count else size
|
|
name = bytes(image[data + nameoff : data + endoff]).split(b"\0", 1)[0]
|
|
if not name:
|
|
raise FixtureError("empty directory name")
|
|
entries.append((child_nid, nameoff, endoff, name))
|
|
return inode, data, size, entries
|
|
|
|
|
|
def inode_info(
|
|
image: bytes | bytearray, nid: int
|
|
) -> tuple[int, int, int, int, int, int]:
|
|
offset = inode_offset(image, nid)
|
|
inode_format = u16(image, offset)
|
|
inode_size = 64 if inode_format & 1 else 32
|
|
size = u64(image, offset + 8) if inode_size == 64 else u32(image, offset + 8)
|
|
xattr_count = u16(image, offset + 2)
|
|
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
|
layout = (inode_format >> 1) & 7
|
|
return offset, inode_format, inode_size, xattr_size, layout, size
|
|
|
|
|
|
def directory_entries(image: bytes | bytearray, nid: int) -> list[tuple[bytes, int]]:
|
|
offset, _, inode_size, xattr_size, layout, size = inode_info(image, nid)
|
|
block_size = 1 << image[SUPER + 12]
|
|
if not 0 < size <= block_size:
|
|
raise FixtureError("directory must fit one block")
|
|
if layout == 2:
|
|
data = offset + inode_size + xattr_size
|
|
elif layout == 0:
|
|
data = u32(image, offset + 16) << image[SUPER + 12]
|
|
else:
|
|
raise FixtureError(f"unsupported directory layout {layout}")
|
|
first_nameoff = u16(image, data + 8)
|
|
if first_nameoff < 12 or first_nameoff % 12:
|
|
raise FixtureError("invalid first directory name offset")
|
|
count = first_nameoff // 12
|
|
entries = []
|
|
for index in range(count):
|
|
entry = data + index * 12
|
|
child = u64(image, entry)
|
|
nameoff = u16(image, entry + 8)
|
|
endoff = u16(image, entry + 20) if index + 1 < count else size
|
|
name = bytes(image[data + nameoff : data + endoff]).split(b"\0", 1)[0]
|
|
if not name:
|
|
raise FixtureError("empty directory name")
|
|
entries.append((name, child))
|
|
return entries
|
|
|
|
|
|
def child_nid(image: bytes | bytearray, parent_nid: int, name: bytes) -> int:
|
|
return next(nid for entry_name, nid in directory_entries(image, parent_nid) if entry_name == name)
|
|
|
|
|
|
def convert_inline_directory_to_plain(image: bytearray, nid: int) -> tuple[int, int]:
|
|
offset, inode_format, inode_size, xattr_size, layout, size = inode_info(image, nid)
|
|
if layout != 2 or inode_size != 64 or size <= 0:
|
|
raise FixtureError("expected nonempty extended inline directory")
|
|
block_size = 1 << image[SUPER + 12]
|
|
tail_size = size % block_size
|
|
if tail_size == 0:
|
|
raise FixtureError("expected inline directory tail")
|
|
raw_block = u32(image, offset + 16)
|
|
full_size = size - tail_size
|
|
inline_offset = offset + inode_size + xattr_size
|
|
directory_data = bytes(
|
|
image[raw_block * block_size : raw_block * block_size + full_size]
|
|
+ image[inline_offset : inline_offset + tail_size]
|
|
)
|
|
if len(directory_data) != size:
|
|
raise FixtureError("directory data reconstruction failed")
|
|
new_raw_block = (len(image) + block_size - 1) // block_size
|
|
image.extend(b"\0" * (new_raw_block * block_size - len(image)))
|
|
image.extend(directory_data)
|
|
image.extend(b"\0" * (-len(image) % block_size))
|
|
put_u16(image, offset, inode_format & ~(7 << 1))
|
|
put_u32(image, offset + 16, new_raw_block)
|
|
put_u32(image, SUPER + 36, len(image) // block_size)
|
|
if inode_info(image, nid)[4] != FLAT_PLAIN:
|
|
raise FixtureError("directory layout conversion failed")
|
|
return new_raw_block, len(image) // block_size
|
|
|
|
|
|
def write_checksums(directory: Path) -> None:
|
|
with (directory / "SHA256SUMS").open("x", encoding="ascii") as sums:
|
|
for path in sorted(directory.glob("*.erofs")):
|
|
sums.write(f"{sha256(path)} {path.name}\n")
|
|
|
|
|
|
def make_metadata(fixture_dir: Path, artifact_dir: Path) -> list[list[str]]:
|
|
require_tools("mkfs.erofs", "fsck.erofs", "dump.erofs")
|
|
if fixture_dir.exists() or artifact_dir.exists():
|
|
raise FixtureError("metadata source and artifact paths must be absent")
|
|
for relative in (
|
|
"inline",
|
|
"special",
|
|
"namei/alpha/bravo/charlie",
|
|
"namei/alpha/sibling",
|
|
"namei/wide",
|
|
"pager",
|
|
"nfs/basic/subdir",
|
|
"nlink/subdir",
|
|
):
|
|
(fixture_dir / relative).mkdir(parents=True, exist_ok=True)
|
|
artifact_dir.mkdir(parents=True)
|
|
(fixture_dir / "inline/inline.txt").write_text(
|
|
"inline-tail-payload-0123456789\n", encoding="ascii"
|
|
)
|
|
(fixture_dir / "special/target").write_text("special target\n", encoding="ascii")
|
|
os.symlink("target", fixture_dir / "special/link")
|
|
os.mknod(
|
|
fixture_dir / "special/char-large",
|
|
stat.S_IFCHR | 0o666,
|
|
os.makedev(2748, 344865),
|
|
)
|
|
os.mknod(
|
|
fixture_dir / "special/block-large",
|
|
stat.S_IFBLK | 0o666,
|
|
os.makedev(2748, 344865),
|
|
)
|
|
os.mkfifo(fixture_dir / "special/fifo")
|
|
(fixture_dir / "namei/alpha/bravo/charlie/payload.txt").write_text(
|
|
"cold nested lookup payload\n", encoding="ascii"
|
|
)
|
|
(fixture_dir / "namei/alpha/bravo/repeat.txt").write_text(
|
|
"repeat lookup payload\n", encoding="ascii"
|
|
)
|
|
(fixture_dir / "namei/alpha/sibling/marker.txt").write_text(
|
|
"sibling marker\n", encoding="ascii"
|
|
)
|
|
for index in range(320):
|
|
name = f"entry-{index:03d}-abcdefghijklmnopqrstuvwxyz.txt"
|
|
(fixture_dir / "namei/wide" / name).write_text(
|
|
f"wide entry {index:03d}\n", encoding="ascii"
|
|
)
|
|
(fixture_dir / "pager/pager.bin").write_bytes(deterministic_bytes(5 * 4096 + 731))
|
|
(fixture_dir / "nfs/basic/regular.txt").write_text(
|
|
"stable file handle payload\n", encoding="ascii"
|
|
)
|
|
(fixture_dir / "nfs/basic/subdir/child.txt").write_text("child\n", encoding="ascii")
|
|
os.symlink("regular.txt", fixture_dir / "nfs/basic/link-to-regular")
|
|
os.mkfifo(fixture_dir / "nfs/basic/test.fifo")
|
|
(fixture_dir / "nlink/single.txt").write_text("single\n", encoding="ascii")
|
|
(fixture_dir / "nlink/hard-a.txt").write_text("hardlinked\n", encoding="ascii")
|
|
os.link(fixture_dir / "nlink/hard-a.txt", fixture_dir / "nlink/hard-b.txt")
|
|
(fixture_dir / "nlink/subdir/child.txt").write_text("child\n", encoding="ascii")
|
|
set_epoch(fixture_dir)
|
|
|
|
commands = [
|
|
build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555551", "inline.erofs", "inline"),
|
|
build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555552", "special-compact.erofs", "special", "-E", "force-inode-compact"),
|
|
build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555553", "special-extended.erofs", "special", "-E", "force-inode-extended"),
|
|
build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555554", "namei-base.erofs", "namei"),
|
|
build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555555", "pager-plain.erofs", "pager", "-E", "noinline_data"),
|
|
build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555556", "pager-lz4.erofs", "pager", "-z", "lz4"),
|
|
build_image(artifact_dir, fixture_dir, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee1", "nfs-a.erofs", "nfs"),
|
|
build_image(artifact_dir, fixture_dir, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee2", "nfs-b.erofs", "nfs"),
|
|
build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555557", "nlink.erofs", "nlink", "-E", "force-inode-compact"),
|
|
]
|
|
|
|
inline = bytearray((artifact_dir / "inline.erofs").read_bytes())
|
|
_, _, _, inline_root = inline_dir_entries(inline, u16(inline, SUPER + 14))
|
|
inline_nid = next(nid for nid, _, _, name in inline_root if name == b"inline.txt")
|
|
inline_inode, layout, inline_size = compact_inode(inline, inline_nid)
|
|
if layout != 2 or u16(inline, inline_inode + 2) != 0:
|
|
raise FixtureError("inline fixture has unexpected layout")
|
|
inline_cross_xattr_icount = 695
|
|
inline_xattr_size = 12 + 4 * (inline_cross_xattr_icount - 1)
|
|
inline_data_off = inline_inode + 32 + inline_xattr_size
|
|
block_size = 1 << inline[SUPER + 12]
|
|
if inline_data_off % block_size + inline_size <= block_size:
|
|
raise FixtureError("inline fixture does not cross the block boundary")
|
|
inline_cross = bytearray(inline)
|
|
put_u16(inline_cross, inline_inode + 2, inline_cross_xattr_icount)
|
|
write_image(artifact_dir / "inline-cross-block.erofs", inline_cross)
|
|
|
|
namei = bytearray((artifact_dir / "namei-base.erofs").read_bytes())
|
|
root_nid = u16(namei, SUPER + 14)
|
|
root_inode, root_data, _, root_entries = inline_dir_entries(namei, root_nid)
|
|
wide_nid = next(nid for nid, _, _, name in root_entries if name == b"wide")
|
|
wide_inode, wide_layout, _ = compact_inode(namei, wide_nid)
|
|
if wide_layout != 2:
|
|
raise FixtureError("wide directory is not inline")
|
|
block_bits = namei[SUPER + 12]
|
|
wide_block = u32(namei, wide_inode + 16) << block_bits
|
|
wide_first_nameoff = u16(namei, wide_block + 8)
|
|
wide_count = wide_first_nameoff // 12
|
|
wide_last_nameoff = u16(namei, wide_block + (wide_count - 1) * 12 + 8)
|
|
padding_nul = namei.index(0, wide_block + wide_last_nameoff, wide_block + (1 << block_bits))
|
|
nonzero_padding = bytearray(namei)
|
|
nonzero_padding[padding_nul + 1 : padding_nul + 9] = b"PAD!ERO!"
|
|
write_image(artifact_dir / "namei-padding-nonzero.erofs", nonzero_padding)
|
|
short_block = bytearray(namei)
|
|
put_u32(short_block, root_inode + 8, 8)
|
|
write_image(artifact_dir / "namei-corrupt-short.erofs", short_block)
|
|
bad_nameoff = bytearray(namei)
|
|
first_nameoff = u16(bad_nameoff, root_data + 8)
|
|
put_u16(bad_nameoff, root_data + 20, first_nameoff)
|
|
write_image(artifact_dir / "namei-corrupt-nameoff.erofs", bad_nameoff)
|
|
bad_name = bytearray(namei)
|
|
slash_offset = wide_block + wide_last_nameoff + 5
|
|
if bad_name[slash_offset] in (0, ord("/")):
|
|
raise FixtureError("name corruption target is unsuitable")
|
|
bad_name[slash_offset] = ord("/")
|
|
write_image(artifact_dir / "namei-corrupt-name.erofs", bad_name)
|
|
|
|
nlink = bytearray((artifact_dir / "nlink.erofs").read_bytes())
|
|
nlink_root_nid = u16(nlink, SUPER + 14)
|
|
_, _, _, nlink_entries = inline_dir_entries(nlink, nlink_root_nid)
|
|
single_nid = next(nid for nid, _, _, name in nlink_entries if name == b"single.txt")
|
|
single_inode, _, _ = compact_inode(nlink, single_nid)
|
|
nlink1 = bytearray(nlink)
|
|
put_u16(nlink1, single_inode, u16(nlink1, single_inode) | 0x10)
|
|
put_u16(nlink1, single_inode + 6, 0x1234)
|
|
write_image(artifact_dir / "nlink1-patched.erofs", nlink1)
|
|
|
|
with (artifact_dir / "fixture-evidence.txt").open("x", encoding="ascii") as output:
|
|
output.write(
|
|
f"inline_nid={inline_nid} inode_off={inline_inode} "
|
|
f"inode_blockoff={inline_inode % block_size} "
|
|
f"xattr_icount=0->{inline_cross_xattr_icount} "
|
|
f"inline_data_blockoff={inline_data_off % block_size} "
|
|
f"inline_size={inline_size}\n"
|
|
)
|
|
output.write(
|
|
f"wide_nid={wide_nid} block={wide_block >> block_bits} "
|
|
f"dirents={wide_count} last_nameoff={wide_last_nameoff} "
|
|
f"padding_patch={padding_nul + 1 - wide_block}:"
|
|
f"{padding_nul + 9 - wide_block}\n"
|
|
)
|
|
output.write(f"nlink1_nid={single_nid} i_format_bit4=1 i_nb=0x1234\n")
|
|
for image_name in ("special-compact.erofs", "special-extended.erofs"):
|
|
image = bytearray((artifact_dir / image_name).read_bytes())
|
|
special_root = u16(image, SUPER + 14)
|
|
_, _, _, entries = inline_dir_entries(image, special_root)
|
|
output.write(image_name + "\n")
|
|
for nid, _, _, name in entries:
|
|
if name not in (b"char-large", b"block-large", b"fifo"):
|
|
continue
|
|
offset = inode_offset(image, nid)
|
|
output.write(
|
|
f" {name.decode()} nid={nid} inode_size="
|
|
f"{64 if u16(image, offset) & 1 else 32} "
|
|
f"raw_u={u32(image, offset + 16):#010x}\n"
|
|
)
|
|
write_checksums(artifact_dir)
|
|
run(["fsck.erofs", "-d1", str(artifact_dir / "namei-padding-nonzero.erofs")])
|
|
run(
|
|
[
|
|
"dump.erofs",
|
|
"--cat",
|
|
"--path=/wide/entry-079-abcdefghijklmnopqrstuvwxyz.txt",
|
|
str(artifact_dir / "namei-padding-nonzero.erofs"),
|
|
]
|
|
)
|
|
return commands
|
|
|
|
|
|
def make_final(fixture_dir: Path, artifact_dir: Path) -> list[list[str]]:
|
|
require_tools("mkfs.erofs")
|
|
if fixture_dir.exists() or artifact_dir.exists():
|
|
raise FixtureError("final source and artifact paths must be absent")
|
|
for relative in ("fallback", "oversize", "extent", "large/huge"):
|
|
(fixture_dir / relative).mkdir(parents=True, exist_ok=True)
|
|
artifact_dir.mkdir(parents=True)
|
|
(fixture_dir / "fallback/root.txt").write_text("48-bit fallback root\n", encoding="ascii")
|
|
(fixture_dir / "oversize/big.dat").write_bytes(b"x")
|
|
(fixture_dir / "extent/hole.dat").write_bytes(b"\0" * (1024 * 1024))
|
|
(fixture_dir / "large/huge/anchor.txt").write_text(
|
|
"large directory anchor\n", encoding="ascii"
|
|
)
|
|
for index in range(400):
|
|
(fixture_dir / "large/huge" / f"entry-{index:03d}-abcdefghijklmnopqrstuvwxyz.txt").write_text(
|
|
f"entry {index:03d}\n", encoding="ascii"
|
|
)
|
|
set_epoch(fixture_dir)
|
|
commands = [
|
|
build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666650", "fallback-base.erofs", "fallback", "-E", "force-inode-compact"),
|
|
build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666651", "oversize-base.erofs", "oversize", "-E", "force-inode-extended"),
|
|
build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666652", "extent-base.erofs", "extent", "-E", "legacy-compress,force-inode-extended", "-z", "lz4"),
|
|
build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666653", "large-dir-base.erofs", "large", "-E", "force-inode-extended"),
|
|
]
|
|
feature_incompat = SUPER + 80
|
|
rootnid_2b = SUPER + 14
|
|
blocks_lo = SUPER + 36
|
|
rootnid_8b = SUPER + 112
|
|
evidence = []
|
|
fallback = bytearray((artifact_dir / "fallback-base.erofs").read_bytes())
|
|
fallback_root = u16(fallback, rootnid_2b)
|
|
fallback_blocks = u32(fallback, blocks_lo)
|
|
if fallback_root == 0 or u64(fallback, rootnid_8b) != 0:
|
|
raise FixtureError("unexpected fallback base root")
|
|
put_u32(fallback, feature_incompat, u32(fallback, feature_incompat) | 0x80)
|
|
put_u64(fallback, rootnid_8b, 0)
|
|
write_image(artifact_dir / "fallback-48bit-root2.erofs", fallback)
|
|
evidence.append(
|
|
f"fallback rootnid_2b={fallback_root} rootnid_8b=0 "
|
|
f"blocks_lo={fallback_blocks} union=0x{u16(fallback, rootnid_2b):04x}"
|
|
)
|
|
oversize = bytearray((artifact_dir / "oversize-base.erofs").read_bytes())
|
|
oversize_root = u16(oversize, rootnid_2b)
|
|
oversize_nid = child_nid(oversize, oversize_root, b"big.dat")
|
|
oversize_off, oversize_format, oversize_isize, _, _, _ = inode_info(oversize, oversize_nid)
|
|
if not oversize_format & 1 or oversize_isize != 64:
|
|
raise FixtureError("oversize inode is not extended")
|
|
put_u64(oversize, oversize_off + 8, 1 << 63)
|
|
write_image(artifact_dir / "extended-size-bit63.erofs", oversize)
|
|
evidence.append(
|
|
f"oversize nid={oversize_nid} inode_off={oversize_off} "
|
|
f"i_size=0x{u64(oversize, oversize_off + 8):016x}"
|
|
)
|
|
extent = bytearray((artifact_dir / "extent-base.erofs").read_bytes())
|
|
extent_root = u16(extent, rootnid_2b)
|
|
extent_nid = child_nid(extent, extent_root, b"hole.dat")
|
|
extent_off, extent_format, extent_isize, extent_xattr, extent_layout, _ = inode_info(extent, extent_nid)
|
|
if not extent_format & 1 or extent_isize != 64 or extent_layout != 1:
|
|
raise FixtureError("extent inode has unexpected format")
|
|
header = (extent_off + extent_isize + extent_xattr + 7) & ~7
|
|
record = (header + 8 + 15) & ~15
|
|
root_off = inode_offset(extent, extent_root)
|
|
if header < root_off + 64 and root_off < record + 16:
|
|
raise FixtureError("extent metadata overlaps root inode")
|
|
hole_size = 5 * 1024 * 1024 * 1024 + 4096
|
|
put_u64(extent, extent_off + 8, hole_size)
|
|
struct.pack_into("<IHH", extent, header, 1, 0x1 | 0x4, 0)
|
|
struct.pack_into("<IIII", extent, record, 0, 0, 0, 0)
|
|
write_image(artifact_dir / "extent-hole-5g.erofs", extent)
|
|
evidence.append(
|
|
f"extent-hole nid={extent_nid} inode_off={extent_off} size={hole_size} "
|
|
f"header={header} record={record} extents=1 recsize=16 plen=0 lstart=0"
|
|
)
|
|
large = bytearray((artifact_dir / "large-dir-base.erofs").read_bytes())
|
|
large_root = u16(large, rootnid_2b)
|
|
large_nid = child_nid(large, large_root, b"huge")
|
|
large_off, large_format, large_isize, _, _, _ = inode_info(large, large_nid)
|
|
if not large_format & 1 or large_isize != 64:
|
|
raise FixtureError("large directory inode is not extended")
|
|
large_raw_block, large_image_blocks = convert_inline_directory_to_plain(large, large_nid)
|
|
large_size = HUGE_DIRECTORY_BLOCKS * (1 << large[SUPER + 12])
|
|
put_u64(large, large_off + 8, large_size)
|
|
write_image(artifact_dir / "large-dir-intmax.erofs", large)
|
|
evidence.append(
|
|
f"large-dir nid={large_nid} inode_off={large_off} size={large_size} "
|
|
f"blocks={HUGE_DIRECTORY_BLOCKS} last_block={HUGE_DIRECTORY_BLOCKS - 1} "
|
|
f"layout=flat-plain raw_block={large_raw_block} image_blocks={large_image_blocks}"
|
|
)
|
|
(artifact_dir / "fixture-evidence.txt").write_text(
|
|
"\n".join(evidence) + "\n", encoding="ascii"
|
|
)
|
|
write_checksums(artifact_dir)
|
|
return commands
|
|
|
|
|
|
def make_vfs_source(source: Path) -> None:
|
|
testdir = source / "testdir"
|
|
(testdir / "subdir").mkdir(parents=True)
|
|
for index in range(5):
|
|
(testdir / f"dir-{index:02d}").mkdir()
|
|
(source / "parent/child/grandchild").mkdir(parents=True)
|
|
(testdir / "file.txt").write_text("repo22 G3 file payload\n", encoding="ascii")
|
|
for index in range(12):
|
|
(testdir / f"regular-{index:02d}.txt").write_text(f"regular {index:02d}\n", encoding="ascii")
|
|
for name in ("File.txt", "FILE.TXT", "FiLe.TxT"):
|
|
(testdir / name).write_text(f"case variant {name}\n", encoding="ascii")
|
|
for index in range(1, 4):
|
|
(testdir / f"file{index}.txt").write_text(f"cache file {index}\n", encoding="ascii")
|
|
(testdir / "subdir/child.txt").write_text("child\n", encoding="ascii")
|
|
(testdir / "script.sh").write_text("#!/bin/sh\nexit 0\n", encoding="ascii")
|
|
(testdir / "rootonly.txt").write_text("restricted\n", encoding="ascii")
|
|
(testdir / "writeonly.txt").write_text("write only mode\n", encoding="ascii")
|
|
(testdir / "noexec.txt").write_text("not executable\n", encoding="ascii")
|
|
(testdir / ("long-" + "x" * 250)).write_text("255 byte name\n", encoding="ascii")
|
|
long_target = "subdir/" + "y" * 112
|
|
(testdir / "subdir" / ("y" * 112)).write_text("long target\n", encoding="ascii")
|
|
os.symlink("file.txt", testdir / "shortlink")
|
|
os.symlink("subdir/child.txt", testdir / "relative-link")
|
|
os.symlink("missing-target", testdir / "broken-link")
|
|
os.symlink(long_target, testdir / "long-link")
|
|
(source / "parent/file.txt").write_text("parent file\n", encoding="ascii")
|
|
(source / "parent/child/grandchild/marker.txt").write_text("grandchild\n", encoding="ascii")
|
|
(source / "pager.bin").write_bytes(deterministic_bytes(5 * 4096 + 731))
|
|
os.chmod(testdir / "script.sh", 0o755)
|
|
os.chmod(testdir / "rootonly.txt", 0o600)
|
|
os.chmod(testdir / "writeonly.txt", 0o200)
|
|
os.chmod(testdir / "noexec.txt", 0o644)
|
|
set_epoch(source)
|
|
|
|
|
|
def directory_data_offset(image: ErofsImage, inode: object) -> int:
|
|
if inode.layout == 2:
|
|
return inode.offset + inode.inode_size + inode.xattr_size
|
|
if inode.layout == FLAT_PLAIN:
|
|
return inode.start_block << image.block_bits
|
|
raise FixtureError(f"unsupported directory layout {inode.layout}")
|
|
|
|
|
|
def limited_directory_entries(image: ErofsImage, inode: object) -> list[object]:
|
|
if inode.size == 0 or inode.size > image.block_size:
|
|
raise FixtureError("G3 qualifier accepts only one-block path components")
|
|
return image.directory_entries(inode)
|
|
|
|
|
|
def resolve(image: ErofsImage, path: str) -> object:
|
|
inode = image.inode(image.root_nid)
|
|
for component in path.strip("/").split("/"):
|
|
if not component:
|
|
continue
|
|
entry = next(
|
|
(
|
|
item
|
|
for item in limited_directory_entries(image, inode)
|
|
if item.name.decode("ascii") == component
|
|
),
|
|
None,
|
|
)
|
|
if entry is None:
|
|
raise FixtureError(f"path not found: {path}")
|
|
inode = image.inode(entry.nid)
|
|
return inode
|
|
|
|
|
|
def validate_vfs_image(image_path: Path, source: Path) -> list[str]:
|
|
image = ErofsImage.load(image_path)
|
|
image.validate_superblock()
|
|
evidence = [
|
|
f"image={image_path.name} provider_bytes={len(image.data)} "
|
|
f"block_size={image.block_size} blocks={image.blocks} "
|
|
f"root_nid={image.root_nid} checksum_valid={image.checksum_valid()}"
|
|
]
|
|
paths = [
|
|
"/testdir",
|
|
"/testdir/file.txt",
|
|
"/testdir/script.sh",
|
|
"/testdir/rootonly.txt",
|
|
"/testdir/writeonly.txt",
|
|
"/testdir/shortlink",
|
|
"/testdir/long-link",
|
|
"/pager.bin",
|
|
"/parent/child/grandchild",
|
|
]
|
|
for path in paths:
|
|
source_path = source / path.lstrip("/")
|
|
source_stat = source_path.lstat()
|
|
inode = resolve(image, path)
|
|
if inode.mode & S_IFMT != stat.S_IFMT(source_stat.st_mode):
|
|
raise FixtureError(f"{path}: inode type differs from source")
|
|
if inode.mode & 0o7777 != source_stat.st_mode & 0o7777:
|
|
raise FixtureError(f"{path}: inode permissions differ from source")
|
|
expected_size = None
|
|
if source_path.is_symlink():
|
|
expected_size = len(os.readlink(source_path).encode("ascii"))
|
|
elif source_path.is_file():
|
|
expected_size = source_stat.st_size
|
|
if expected_size is not None and inode.size != expected_size:
|
|
raise FixtureError(f"{path}: size {inode.size}, expected {expected_size}")
|
|
evidence.append(
|
|
f"path={path} nid={inode.nid} inode_offset={inode.offset} "
|
|
f"layout={inode.layout} mode={inode.mode:#07o} size={inode.size} "
|
|
f"start_block={inode.start_block}"
|
|
)
|
|
source_names = sorted(item.name for item in (source / "testdir").iterdir())
|
|
testdir = resolve(image, "/testdir")
|
|
image_names = sorted(
|
|
entry.name.decode("ascii")
|
|
for entry in limited_directory_entries(image, testdir)
|
|
if entry.name not in (b".", b"..")
|
|
)
|
|
if source_names != image_names:
|
|
raise FixtureError("/testdir names differ from source")
|
|
evidence.append(
|
|
f"path=/testdir real_entries={len(image_names)} "
|
|
f"directory_size={testdir.size} layout={testdir.layout}"
|
|
)
|
|
return evidence
|
|
|
|
|
|
def write_source_hashes(source: Path, output: Path) -> None:
|
|
with output.open("x", encoding="ascii") as sums:
|
|
for path in sorted(source.rglob("*")):
|
|
if path.is_file() and not path.is_symlink():
|
|
sums.write(f"{sha256(path)} {path.relative_to(source)}\n")
|
|
|
|
|
|
def make_vfs(output: Path) -> list[list[str]]:
|
|
require_tools("mkfs.erofs")
|
|
if output.exists():
|
|
raise FixtureError(f"output already exists: {output}")
|
|
source = output / "source"
|
|
output.mkdir(parents=True)
|
|
make_vfs_source(source)
|
|
plain = output / "vfs-plain.erofs"
|
|
lz4 = output / "vfs-lz4.erofs"
|
|
common = ["mkfs.erofs", "-d0", "-T0", "--all-time", "--all-root", "--workers=1"]
|
|
commands = [
|
|
common + ["-x-1", "-E", "noinline_data", "-U", "33333333-4444-5555-6666-777777777771", str(plain), str(source)],
|
|
common + ["-z", "lz4", "-U", "33333333-4444-5555-6666-777777777772", str(lz4), str(source)],
|
|
]
|
|
for command in commands:
|
|
run(command)
|
|
evidence = validate_vfs_image(plain, source) + validate_vfs_image(lz4, source)
|
|
(output / "fixture-evidence.txt").write_text("\n".join(evidence) + "\n", encoding="ascii")
|
|
(output / "expected-testdir.txt").write_text(
|
|
"\n".join(sorted(item.name for item in (source / "testdir").iterdir())) + "\n",
|
|
encoding="ascii",
|
|
)
|
|
write_source_hashes(source, output / "SOURCE-SHA256SUMS")
|
|
with (output / "IMAGE-SHA256SUMS").open("x", encoding="ascii") as sums:
|
|
for path in (lz4, plain):
|
|
sums.write(f"{sha256(path)} {path.name}\n")
|
|
return commands
|
|
|
|
|
|
def make_large_prefix(source: Path, output: Path, evidence_path: Path) -> None:
|
|
image = ErofsImage.load(source)
|
|
image.validate_superblock()
|
|
huge = resolve(image, "/huge")
|
|
expected_size = HUGE_DIRECTORY_BLOCKS * image.block_size
|
|
if huge.inode_size != 64 or huge.layout != FLAT_PLAIN:
|
|
raise FixtureError("TC153 directory is not extended FLAT_PLAIN/Layout0")
|
|
if huge.size != expected_size:
|
|
raise FixtureError(f"TC153 directory size {huge.size}, expected {expected_size}")
|
|
if huge.start_block == 0:
|
|
raise FixtureError("TC153 directory has no plain-data start block")
|
|
provider_blocks = huge.start_block + HUGE_DIRECTORY_BLOCKS
|
|
if provider_blocks > 0xFFFFFFFF:
|
|
raise FixtureError("TC153 sparse provider does not fit blocks_lo")
|
|
image.put_u32(SUPER + 36, provider_blocks)
|
|
image.update_checksum()
|
|
image.validate_superblock()
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
image.save(output)
|
|
checked = ErofsImage.load(output)
|
|
checked_huge = resolve(checked, "/huge")
|
|
if checked.blocks != provider_blocks or checked_huge.layout != FLAT_PLAIN or checked_huge.size != expected_size:
|
|
raise FixtureError("TC153 sparse prefix self-check failed")
|
|
provider_bytes = provider_blocks * checked.block_size
|
|
evidence_path.write_text(
|
|
"tc153-sparse-prefix "
|
|
f"nid={checked_huge.nid} inode_offset={checked_huge.offset} "
|
|
f"layout=flat-plain start_block={checked_huge.start_block} "
|
|
f"directory_blocks={HUGE_DIRECTORY_BLOCKS} last_block={HUGE_DIRECTORY_BLOCKS - 1} "
|
|
f"blocks_lo={provider_blocks} provider_bytes={provider_bytes} "
|
|
f"prefix_bytes={len(checked.data)} checksum_valid={checked.checksum_valid()} "
|
|
f"sha256={sha256(output)}\n",
|
|
encoding="ascii",
|
|
)
|
|
|
|
|
|
def path_type(mode: int) -> str:
|
|
if stat.S_ISREG(mode):
|
|
return "file"
|
|
if stat.S_ISDIR(mode):
|
|
return "directory"
|
|
if stat.S_ISLNK(mode):
|
|
return "symlink"
|
|
if stat.S_ISFIFO(mode):
|
|
return "fifo"
|
|
if stat.S_ISCHR(mode):
|
|
return "char"
|
|
if stat.S_ISBLK(mode):
|
|
return "block"
|
|
raise FixtureError(f"unsupported source type: {mode:#o}")
|
|
|
|
|
|
def tree_inventory(root: Path) -> dict[str, object]:
|
|
if not root.is_dir():
|
|
raise FixtureError(f"inventory root is absent: {root}")
|
|
paths = [root] + sorted(root.rglob("*"))
|
|
hardlinks: dict[tuple[int, int], list[str]] = {}
|
|
for path in paths:
|
|
info = path.lstat()
|
|
if stat.S_ISREG(info.st_mode) and info.st_nlink > 1:
|
|
hardlinks.setdefault((info.st_dev, info.st_ino), []).append(
|
|
"." if path == root else path.relative_to(root).as_posix()
|
|
)
|
|
hardlink_names = {
|
|
key: ",".join(sorted(names)) for key, names in hardlinks.items()
|
|
}
|
|
entries = []
|
|
for path in paths:
|
|
info = path.lstat()
|
|
relative = "." if path == root else path.relative_to(root).as_posix()
|
|
item: dict[str, object] = {
|
|
"path": relative,
|
|
"type": path_type(info.st_mode),
|
|
"mode": info.st_mode & 0o7777,
|
|
}
|
|
if stat.S_ISREG(info.st_mode):
|
|
item.update(size=info.st_size, sha256=sha256(path))
|
|
if info.st_nlink > 1:
|
|
item["hardlink_group"] = hardlink_names[(info.st_dev, info.st_ino)]
|
|
elif stat.S_ISLNK(info.st_mode):
|
|
item["target"] = os.readlink(path)
|
|
elif stat.S_ISCHR(info.st_mode) or stat.S_ISBLK(info.st_mode):
|
|
item.update(major=os.major(info.st_rdev), minor=os.minor(info.st_rdev))
|
|
entries.append(item)
|
|
encoded = json.dumps(entries, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode("ascii")
|
|
return {"sha256": hashlib.sha256(encoded).hexdigest(), "entries": entries}
|
|
|
|
|
|
def compare_set(
|
|
left_source: Path,
|
|
left_artifacts: Path,
|
|
right_source: Path,
|
|
right_artifacts: Path,
|
|
output: Path,
|
|
) -> None:
|
|
inventories = {
|
|
"left_source": tree_inventory(left_source),
|
|
"left_artifacts": tree_inventory(left_artifacts),
|
|
"right_source": tree_inventory(right_source),
|
|
"right_artifacts": tree_inventory(right_artifacts),
|
|
}
|
|
comparison = {
|
|
name: {"sha256": value["sha256"], "entries": len(value["entries"])}
|
|
for name, value in inventories.items()
|
|
}
|
|
comparison["source_equal"] = inventories["left_source"] == inventories["right_source"]
|
|
comparison["artifacts_equal"] = inventories["left_artifacts"] == inventories["right_artifacts"]
|
|
comparison["status"] = (
|
|
"PASS" if comparison["source_equal"] and comparison["artifacts_equal"] else "RUNNER_FAIL"
|
|
)
|
|
if comparison["status"] != "PASS":
|
|
comparison["differences"] = {
|
|
"source": sorted(
|
|
set(json.dumps(item, sort_keys=True) for item in inventories["left_source"]["entries"])
|
|
^ set(json.dumps(item, sort_keys=True) for item in inventories["right_source"]["entries"])
|
|
)[:40],
|
|
"artifacts": sorted(
|
|
set(json.dumps(item, sort_keys=True) for item in inventories["left_artifacts"]["entries"])
|
|
^ set(json.dumps(item, sort_keys=True) for item in inventories["right_artifacts"]["entries"])
|
|
)[:40],
|
|
}
|
|
write_json(output, comparison)
|
|
if comparison["status"] != "PASS":
|
|
raise FixtureError("stable and archived fixture trees differ")
|
|
|
|
|
|
def verify_checksum_file(directory: Path, filename: str) -> None:
|
|
for line in (directory / filename).read_text(encoding="ascii").splitlines():
|
|
digest, relative = line.split(" ", 1)
|
|
if sha256(directory / relative) != digest:
|
|
raise FixtureError(f"checksum mismatch: {directory / relative}")
|
|
|
|
|
|
def make_expected_wide(fixture_dir: Path, artifact_dir: Path) -> None:
|
|
names = sorted(path.name for path in (fixture_dir / "namei/wide").iterdir() if path.is_file())
|
|
if len(names) != 320:
|
|
raise FixtureError(f"wide directory contains {len(names)} files")
|
|
(artifact_dir / "expected-wide.txt").write_text("\n".join(names) + "\n", encoding="ascii")
|
|
|
|
|
|
def make_all(output: Path) -> None:
|
|
if output.exists():
|
|
raise FixtureError(f"output already exists: {output}")
|
|
output.mkdir(parents=True)
|
|
commands = {
|
|
"vfs": make_vfs(output / "vfs"),
|
|
"metadata": make_metadata(output / "metadata-source", output / "metadata"),
|
|
"final": make_final(output / "final-source", output / "final"),
|
|
}
|
|
make_expected_wide(output / "metadata-source", output / "metadata")
|
|
make_large_prefix(
|
|
output / "final/large-dir-intmax.erofs",
|
|
output / "final/large-dir-intmax-sparse-prefix.erofs",
|
|
output / "final/tc153-sparse-evidence.txt",
|
|
)
|
|
(output / "final/TC153-SPARSE-SHA256").write_text(
|
|
f"{sha256(output / 'final/large-dir-intmax-sparse-prefix.erofs')} "
|
|
"large-dir-intmax-sparse-prefix.erofs\n",
|
|
encoding="ascii",
|
|
)
|
|
verify_checksum_file(output / "vfs", "IMAGE-SHA256SUMS")
|
|
verify_checksum_file(output / "vfs/source", "../SOURCE-SHA256SUMS")
|
|
verify_checksum_file(output / "metadata", "SHA256SUMS")
|
|
verify_checksum_file(output / "final", "SHA256SUMS")
|
|
output_text = str(output)
|
|
canonical_commands = {
|
|
name: [
|
|
[
|
|
"$OUTPUT" + argument[len(output_text) :]
|
|
if argument == output_text or argument.startswith(output_text + os.sep)
|
|
else argument
|
|
for argument in argv
|
|
]
|
|
for argv in command_set
|
|
]
|
|
for name, command_set in commands.items()
|
|
}
|
|
manifest = {
|
|
"schema_version": 1,
|
|
"generator": "tests/pre15/fixtures/g3.py",
|
|
"generator_sha256": sha256(Path(__file__)),
|
|
"mkfs_version": run(["mkfs.erofs", "-V"], capture=True).splitlines()[0],
|
|
"commands": canonical_commands,
|
|
"sets": {
|
|
"vfs": tree_inventory(output / "vfs"),
|
|
"metadata_source": tree_inventory(output / "metadata-source"),
|
|
"metadata": tree_inventory(output / "metadata"),
|
|
"final_source": tree_inventory(output / "final-source"),
|
|
"final": tree_inventory(output / "final"),
|
|
},
|
|
}
|
|
write_json(output / "G3-MANIFEST.json", manifest)
|
|
|
|
|
|
def verify_all(output: Path) -> None:
|
|
manifest = json.loads((output / "G3-MANIFEST.json").read_text(encoding="ascii"))
|
|
roots = {
|
|
"vfs": output / "vfs",
|
|
"metadata_source": output / "metadata-source",
|
|
"metadata": output / "metadata",
|
|
"final_source": output / "final-source",
|
|
"final": output / "final",
|
|
}
|
|
for name, root in roots.items():
|
|
if tree_inventory(root) != manifest["sets"][name]:
|
|
raise FixtureError(f"G3 manifest mismatch: {name}")
|
|
verify_checksum_file(output / "vfs", "IMAGE-SHA256SUMS")
|
|
verify_checksum_file(output / "vfs/source", "../SOURCE-SHA256SUMS")
|
|
verify_checksum_file(output / "metadata", "SHA256SUMS")
|
|
verify_checksum_file(output / "final", "SHA256SUMS")
|
|
|
|
|
|
def source_constant(path: Path, name: str) -> int:
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
fields = line.split()
|
|
if len(fields) >= 3 and fields[0] == "#define" and fields[1] == name:
|
|
return int(fields[2], 0)
|
|
raise FixtureError(f"constant not found: {path}:{name}")
|
|
|
|
|
|
def validate_xattr_sources(freebsd_root: Path, linux_root: Path) -> dict[str, str]:
|
|
constants = (
|
|
"EROFS_FEATURE_COMPAT_SB_CHKSUM",
|
|
"EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX",
|
|
"EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES",
|
|
"EROFS_FEATURE_INCOMPAT_METABOX",
|
|
)
|
|
for name in constants:
|
|
freebsd = source_constant(freebsd_root / "erofs_fs.h", name)
|
|
linux = source_constant(linux_root / "erofs_fs.h", name)
|
|
if freebsd != linux:
|
|
raise FixtureError(f"FreeBSD/Linux constant mismatch: {name}")
|
|
return {
|
|
"freebsd_erofs_fs_sha256": sha256(freebsd_root / "erofs_fs.h"),
|
|
"freebsd_xattr_sha256": sha256(freebsd_root / "xattr.c"),
|
|
"linux_erofs_fs_sha256": sha256(linux_root / "erofs_fs.h"),
|
|
"linux_xattr_sha256": sha256(linux_root / "xattr.c"),
|
|
}
|
|
|
|
|
|
def make_contract_image(
|
|
path: Path,
|
|
feature_compat: int,
|
|
feature_incompat: int,
|
|
prefix_start: int,
|
|
prefix_count: int,
|
|
packed_nid: int,
|
|
metabox_nid: int,
|
|
primary: bytes,
|
|
) -> None:
|
|
data = bytearray(4096)
|
|
put_u32(data, SUPER, MAGIC)
|
|
put_u32(data, SUPER + 8, feature_compat | FEATURE_COMPAT_SB_CHKSUM)
|
|
data[SUPER + 12] = 12
|
|
put_u16(data, SUPER + 14, 1)
|
|
put_u32(data, SUPER + 36, 1)
|
|
put_u32(data, SUPER + 40, 0)
|
|
put_u32(data, SUPER + 80, feature_incompat)
|
|
data[SUPER + 91] = prefix_count
|
|
put_u32(data, SUPER + 92, prefix_start // 4)
|
|
put_u64(data, SUPER + 96, packed_nid)
|
|
put_u64(data, SUPER + 128, metabox_nid)
|
|
if prefix_start + len(primary) <= len(data):
|
|
data[prefix_start : prefix_start + len(primary)] = primary
|
|
write_image(path, data)
|
|
|
|
|
|
def select_prefix_backing(image: ErofsImage, carriers: dict[str, bytes]) -> tuple[str, bytes]:
|
|
plain = bool(image.feature_compat & FEATURE_COMPAT_PLAIN_XATTR_PFX)
|
|
metabox = bool(image.feature_incompat & FEATURE_INCOMPAT_METABOX)
|
|
packed_nid = image.u64(SUPER + 96)
|
|
if plain:
|
|
return "primary", bytes(image.data)
|
|
if metabox:
|
|
return "metabox", carriers["metabox"]
|
|
if packed_nid:
|
|
return "packed", carriers["packed"]
|
|
return "primary-legacy", bytes(image.data)
|
|
|
|
|
|
def parse_prefix(backing: bytes, offset: int) -> tuple[int, bytes]:
|
|
aligned = (offset + 3) & ~3
|
|
if aligned > len(backing) - 2:
|
|
raise FixtureError("EINTEGRITY: prefix length is outside backing")
|
|
raw_length = struct.unpack_from("<H", backing, aligned)[0]
|
|
length = 65536 if raw_length == 0 else raw_length
|
|
if length < 1 or length > EROFS_NAME_LEN + 1:
|
|
raise FixtureError("EINTEGRITY: invalid prefix length")
|
|
if aligned + 2 + length > len(backing):
|
|
raise FixtureError("EINTEGRITY: prefix payload is outside backing")
|
|
payload = backing[aligned + 2 : aligned + 2 + length]
|
|
return payload[0], payload[1:]
|
|
|
|
|
|
def make_xattr_legacy(
|
|
spec_path: Path, output: Path, freebsd_root: Path, linux_root: Path
|
|
) -> None:
|
|
if output.exists():
|
|
raise FixtureError(f"output already exists: {output}")
|
|
spec = json.loads(spec_path.read_text(encoding="ascii"))
|
|
output.mkdir(parents=True)
|
|
source_hashes = validate_xattr_sources(freebsd_root, linux_root)
|
|
prefix_start = spec["prefix_start"]
|
|
payload = bytes([spec["base_index"]]) + spec["infix"].encode("ascii")
|
|
record = struct.pack("<H", len(payload)) + payload
|
|
carrier = bytearray(prefix_start + len(record))
|
|
carrier[prefix_start : prefix_start + len(record)] = record
|
|
carriers = {"packed": bytes(carrier), "metabox": bytes(carrier)}
|
|
(output / "packed.bin").write_bytes(carriers["packed"])
|
|
(output / "metabox.bin").write_bytes(carriers["metabox"])
|
|
results = {}
|
|
for item in spec["positive"]:
|
|
compat = FEATURE_COMPAT_PLAIN_XATTR_PFX if item["plain"] else 0
|
|
incompat = FEATURE_INCOMPAT_XATTR_PREFIXES
|
|
if item["carrier"] == "metabox":
|
|
incompat |= FEATURE_INCOMPAT_METABOX
|
|
packed_nid = 7 if item["carrier"] == "packed" else 0
|
|
metabox_nid = 9 if item["carrier"] == "metabox" else 0
|
|
primary = record if item["carrier"] in ("primary", "primary-legacy") else b""
|
|
image_path = output / f"{item['name']}.erofs"
|
|
make_contract_image(
|
|
image_path,
|
|
compat,
|
|
incompat,
|
|
prefix_start,
|
|
1,
|
|
packed_nid,
|
|
metabox_nid,
|
|
primary,
|
|
)
|
|
image = ErofsImage.load(image_path)
|
|
image.validate_superblock()
|
|
selected, backing = select_prefix_backing(image, carriers)
|
|
base_index, infix = parse_prefix(backing, prefix_start)
|
|
if selected != item["carrier"] or base_index != spec["base_index"] or infix.decode("ascii") != spec["infix"]:
|
|
raise FixtureError(f"positive xattr oracle failed: {item['name']}")
|
|
results[item["name"]] = {
|
|
"carrier": selected,
|
|
"base_index": base_index,
|
|
"infix": infix.decode("ascii"),
|
|
"checksum_valid": image.checksum_valid(),
|
|
"image_sha256": sha256(image_path),
|
|
}
|
|
for item in spec["negative"]:
|
|
negative_record = record
|
|
negative_start = prefix_start
|
|
if item["field"] == "record_length":
|
|
negative_record = struct.pack("<H", item["value"]) + payload
|
|
elif item["field"] == "prefix_start":
|
|
negative_start = item["value"]
|
|
else:
|
|
raise FixtureError(f"unknown negative field: {item['field']}")
|
|
image_path = output / f"{item['name']}.erofs"
|
|
make_contract_image(
|
|
image_path,
|
|
0,
|
|
FEATURE_INCOMPAT_XATTR_PREFIXES,
|
|
negative_start,
|
|
1,
|
|
0,
|
|
0,
|
|
negative_record,
|
|
)
|
|
image = ErofsImage.load(image_path)
|
|
image.validate_superblock()
|
|
try:
|
|
_, backing = select_prefix_backing(image, carriers)
|
|
parse_prefix(backing, negative_start)
|
|
except FixtureError as error:
|
|
errno_name = str(error).split(":", 1)[0]
|
|
else:
|
|
raise FixtureError(f"negative xattr oracle unexpectedly passed: {item['name']}")
|
|
if errno_name != item["expected_errno"]:
|
|
raise FixtureError(
|
|
f"{item['name']}: errno {errno_name}, expected {item['expected_errno']}"
|
|
)
|
|
results[item["name"]] = {
|
|
"field": item["field"],
|
|
"expected_errno": errno_name,
|
|
"checksum_valid": image.checksum_valid(),
|
|
"image_sha256": sha256(image_path),
|
|
}
|
|
manifest = {
|
|
"schema_version": 1,
|
|
"generator": "tests/pre15/fixtures/g3.py",
|
|
"generator_sha256": sha256(Path(__file__)),
|
|
"spec_sha256": sha256(spec_path),
|
|
"source_hashes": source_hashes,
|
|
"results": results,
|
|
}
|
|
write_json(output / "fixture-manifest.json", manifest)
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
make_all_parser = subparsers.add_parser("make-all")
|
|
make_all_parser.add_argument("--output", type=Path, required=True)
|
|
verify_parser = subparsers.add_parser("verify")
|
|
verify_parser.add_argument("--output", type=Path, required=True)
|
|
vfs_parser = subparsers.add_parser("make-vfs")
|
|
vfs_parser.add_argument("--output", type=Path, required=True)
|
|
metadata_parser = subparsers.add_parser("make-metadata")
|
|
metadata_parser.add_argument("--source", type=Path, required=True)
|
|
metadata_parser.add_argument("--output", type=Path, required=True)
|
|
final_parser = subparsers.add_parser("make-final")
|
|
final_parser.add_argument("--source", type=Path, required=True)
|
|
final_parser.add_argument("--output", type=Path, required=True)
|
|
large_parser = subparsers.add_parser("make-large-prefix")
|
|
large_parser.add_argument("--source", type=Path, required=True)
|
|
large_parser.add_argument("--output", type=Path, required=True)
|
|
large_parser.add_argument("--evidence", type=Path, required=True)
|
|
compare_parser = subparsers.add_parser("compare-set")
|
|
compare_parser.add_argument("--left-source", type=Path, required=True)
|
|
compare_parser.add_argument("--left-artifacts", type=Path, required=True)
|
|
compare_parser.add_argument("--right-source", type=Path, required=True)
|
|
compare_parser.add_argument("--right-artifacts", type=Path, required=True)
|
|
compare_parser.add_argument("--output", type=Path, required=True)
|
|
xattr_parser = subparsers.add_parser("make-xattr-legacy")
|
|
xattr_parser.add_argument("--spec", type=Path, required=True)
|
|
xattr_parser.add_argument("--output", type=Path, required=True)
|
|
xattr_parser.add_argument("--freebsd-root", type=Path, required=True)
|
|
xattr_parser.add_argument("--linux-root", type=Path, required=True)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> None:
|
|
os.umask(0o022)
|
|
args = parse_args(argv)
|
|
if args.command == "make-all":
|
|
make_all(args.output)
|
|
elif args.command == "verify":
|
|
verify_all(args.output)
|
|
elif args.command == "make-vfs":
|
|
make_vfs(args.output)
|
|
elif args.command == "make-metadata":
|
|
make_metadata(args.source, args.output)
|
|
make_expected_wide(args.source, args.output)
|
|
elif args.command == "make-final":
|
|
make_final(args.source, args.output)
|
|
elif args.command == "make-large-prefix":
|
|
make_large_prefix(args.source, args.output, args.evidence)
|
|
elif args.command == "compare-set":
|
|
compare_set(
|
|
args.left_source,
|
|
args.left_artifacts,
|
|
args.right_source,
|
|
args.right_artifacts,
|
|
args.output,
|
|
)
|
|
else:
|
|
make_xattr_legacy(args.spec, args.output, args.freebsd_root, args.linux_root)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (FixtureError, OSError, subprocess.CalledProcessError, ValueError) as error:
|
|
print(f"fixture error: {error}", file=sys.stderr)
|
|
raise SystemExit(1) from error
|