1110 lines
38 KiB
Python
1110 lines
38 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate and verify deterministic repo22 G5 compression fixtures."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import struct
|
|
import subprocess
|
|
from typing import Any
|
|
|
|
from erofs_fixture import ErofsImage, Inode, SUPER, sha256
|
|
|
|
|
|
BLOCK_SIZE = 4096
|
|
FIXED_UUID = "00000000-0000-0000-0000-000000000000"
|
|
FEATURE_INCOMPAT_COMPR_HEAD2 = 0x00000008
|
|
LAYOUT_COMPRESSED_FULL = 1
|
|
LAYOUT_COMPRESSED_COMPACT = 3
|
|
Z_EROFS_ADVISE_COMPACTED_2B = 0x0001
|
|
Z_EROFS_ADVISE_BIG_PCLUSTER_1 = 0x0002
|
|
Z_EROFS_ADVISE_BIG_PCLUSTER_2 = 0x0004
|
|
Z_EROFS_ADVISE_INLINE_PCLUSTER = 0x0008
|
|
Z_EROFS_ADVISE_INTERLACED_PCLUSTER = 0x0010
|
|
Z_EROFS_LCLUSTER_TYPE_MASK = 0x0003
|
|
Z_EROFS_LCLUSTER_TYPE_PLAIN = 0
|
|
Z_EROFS_LCLUSTER_TYPE_HEAD1 = 1
|
|
Z_EROFS_LCLUSTER_TYPE_NONHEAD = 2
|
|
Z_EROFS_LCLUSTER_TYPE_HEAD2 = 3
|
|
Z_EROFS_LI_PARTIAL_REF = 0x8000
|
|
Z_EROFS_LI_D0_CBLKCNT = 0x0800
|
|
ALGORITHMS = {
|
|
"lz4": 0,
|
|
"lzma": 1,
|
|
"deflate": 2,
|
|
"zstd": 3,
|
|
}
|
|
|
|
|
|
class FixtureError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def align(value: int, alignment: int) -> int:
|
|
return (value + alignment - 1) & -alignment
|
|
|
|
|
|
def relative(path: Path, root: Path) -> str:
|
|
return str(path.relative_to(root))
|
|
|
|
|
|
def run(
|
|
command: list[str],
|
|
*,
|
|
log: Path | None = None,
|
|
check: bool = True,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
result = subprocess.run(
|
|
command,
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
)
|
|
if log is not None:
|
|
log.write_text(
|
|
"$ " + " ".join(command) + "\n" + result.stdout,
|
|
encoding="utf-8",
|
|
)
|
|
if check and result.returncode != 0:
|
|
raise FixtureError(
|
|
f"command failed ({result.returncode}): {' '.join(command)}\n"
|
|
f"{result.stdout}"
|
|
)
|
|
return result
|
|
|
|
|
|
def write_repeated(path: Path, size: int, label: str) -> None:
|
|
token = (label.encode("ascii") + b"\n") * 64
|
|
chunk = (token * ((1024 * 1024 + len(token) - 1) // len(token)))[
|
|
: 1024 * 1024
|
|
]
|
|
remaining = size
|
|
with path.open("wb") as output:
|
|
while remaining:
|
|
amount = min(remaining, len(chunk))
|
|
output.write(chunk[:amount])
|
|
remaining -= amount
|
|
|
|
|
|
def mixed_chunk(seed: str, chunk_index: int, size: int) -> bytes:
|
|
output = bytearray()
|
|
block_index = 0
|
|
while len(output) < size:
|
|
amount = min(BLOCK_SIZE, size - len(output))
|
|
if block_index % 4 == 0:
|
|
material = hashlib.shake_256(
|
|
f"{seed}:{chunk_index}:{block_index}".encode("ascii")
|
|
).digest(amount)
|
|
else:
|
|
token = (
|
|
f"repo22-g5:{seed}:{chunk_index % 7}:{block_index % 11}\n"
|
|
).encode("ascii")
|
|
material = (token * ((amount + len(token) - 1) // len(token)))[
|
|
:amount
|
|
]
|
|
output.extend(material)
|
|
block_index += 1
|
|
return bytes(output)
|
|
|
|
|
|
def write_mixed(path: Path, size: int, seed: str) -> None:
|
|
chunk_size = 1024 * 1024
|
|
remaining = size
|
|
index = 0
|
|
with path.open("wb") as output:
|
|
while remaining:
|
|
amount = min(remaining, chunk_size)
|
|
output.write(mixed_chunk(seed, index, amount))
|
|
remaining -= amount
|
|
index += 1
|
|
|
|
|
|
def write_interlaced(path: Path, size: int) -> None:
|
|
remaining = size
|
|
extent = 0
|
|
with path.open("wb") as output:
|
|
while remaining:
|
|
compressible_size = min(remaining, 16384)
|
|
token = f"repo22-interlaced-{extent % 13:02d}\n".encode("ascii")
|
|
output.write(
|
|
(token * ((compressible_size + len(token) - 1) // len(token)))[
|
|
:compressible_size
|
|
]
|
|
)
|
|
remaining -= compressible_size
|
|
if not remaining:
|
|
break
|
|
random_size = min(remaining, BLOCK_SIZE)
|
|
output.write(
|
|
hashlib.shake_256(
|
|
f"repo22-interlaced-random-{extent}".encode("ascii")
|
|
).digest(random_size)
|
|
)
|
|
remaining -= random_size
|
|
extent += 1
|
|
|
|
|
|
def create_sources(source_root: Path) -> None:
|
|
directories = {
|
|
"lz4": source_root / "lz4",
|
|
"large": source_root / "large",
|
|
"levels": source_root / "levels",
|
|
"lzma-large": source_root / "lzma-large",
|
|
"microlzma": source_root / "microlzma",
|
|
"partial": source_root / "partial",
|
|
"partial-deflate": source_root / "partial-deflate",
|
|
"shape": source_root / "shape",
|
|
"ztail": source_root / "ztail",
|
|
}
|
|
for directory in directories.values():
|
|
directory.mkdir(parents=True)
|
|
|
|
write_mixed(directories["lz4"] / "compressed.bin", 8 * 1024 * 1024, "lz4")
|
|
write_mixed(
|
|
directories["large"] / "large.bin", 256 * 1024 * 1024, "large"
|
|
)
|
|
write_mixed(
|
|
directories["levels"] / "level.dat", 8 * 1024 * 1024, "levels"
|
|
)
|
|
write_mixed(
|
|
directories["lzma-large"] / "large.bin",
|
|
100 * 1024 * 1024 + 1,
|
|
"lzma-large",
|
|
)
|
|
(directories["microlzma"] / "one-byte.bin").write_bytes(b"G")
|
|
write_repeated(
|
|
directories["microlzma"] / "block-4k.bin", BLOCK_SIZE, "microlzma-4k"
|
|
)
|
|
write_repeated(
|
|
directories["microlzma"] / "boundary-16k.bin",
|
|
4 * BLOCK_SIZE,
|
|
"microlzma-16k",
|
|
)
|
|
write_repeated(
|
|
directories["partial"] / "a.dat", 1024 * 1024, "partial-stream"
|
|
)
|
|
source_a = directories["partial"] / "a.dat"
|
|
source_b = directories["partial"] / "b.dat"
|
|
with source_a.open("rb") as source, source_b.open("wb") as target:
|
|
target.write(source.read(700000))
|
|
write_mixed(
|
|
directories["partial"] / "control.bin",
|
|
32768,
|
|
"partial-control",
|
|
)
|
|
write_repeated(
|
|
directories["partial-deflate"] / "a.dat",
|
|
1024 * 1024,
|
|
"partial-stream",
|
|
)
|
|
with (directories["partial-deflate"] / "a.dat").open("rb") as source, (
|
|
directories["partial-deflate"] / "b.dat"
|
|
).open("wb") as target:
|
|
target.write(source.read(100000))
|
|
write_mixed(
|
|
directories["partial-deflate"] / "control.bin",
|
|
32768,
|
|
"partial-control",
|
|
)
|
|
write_interlaced(directories["shape"] / "shape.dat", 1024 * 1024)
|
|
write_repeated(directories["ztail"] / "inline.dat", 131071, "ztail-inline")
|
|
write_mixed(
|
|
directories["ztail"] / "exact-pcluster.dat", BLOCK_SIZE, "ztail-exact"
|
|
)
|
|
write_repeated(
|
|
directories["ztail"] / "one-byte-tail.dat",
|
|
BLOCK_SIZE + 1,
|
|
"ztail-one-byte",
|
|
)
|
|
write_repeated(
|
|
directories["ztail"] / "max-tail.dat",
|
|
BLOCK_SIZE * 2 - 1,
|
|
"ztail-max",
|
|
)
|
|
(directories["ztail"] / "zero-tail.dat").write_bytes(b"")
|
|
|
|
|
|
def source_inventory(source_root: Path) -> list[dict[str, Any]]:
|
|
inventory = []
|
|
for path in sorted(item for item in source_root.rglob("*") if item.is_file()):
|
|
inventory.append(
|
|
{
|
|
"path": relative(path, source_root.parent),
|
|
"size": path.stat().st_size,
|
|
"sha256": sha256(path),
|
|
}
|
|
)
|
|
return inventory
|
|
|
|
|
|
def mkfs_image(
|
|
*,
|
|
output_root: Path,
|
|
image_name: str,
|
|
source: Path,
|
|
options: list[str],
|
|
command_log: list[dict[str, Any]],
|
|
) -> Path:
|
|
image = output_root / "images" / image_name
|
|
command = [
|
|
"mkfs.erofs",
|
|
"--workers=1",
|
|
"--sort=path",
|
|
"-T0",
|
|
"--all-time",
|
|
f"-U{FIXED_UUID}",
|
|
"--all-root",
|
|
*options,
|
|
str(image),
|
|
str(source),
|
|
]
|
|
log = output_root / "logs" / f"mkfs-{image.stem}.log"
|
|
run(command, log=log)
|
|
command_log.append(
|
|
{
|
|
"image": relative(image, output_root),
|
|
"source": relative(source, output_root),
|
|
"options": options,
|
|
"log": relative(log, output_root),
|
|
}
|
|
)
|
|
parsed = ErofsImage.load(image)
|
|
parsed.validate_superblock()
|
|
return image
|
|
|
|
|
|
def inode_for(image: ErofsImage, path: str) -> Inode:
|
|
_, entry = image.resolve_root_entry(path)
|
|
return image.inode(entry.nid)
|
|
|
|
|
|
def map_header(image: ErofsImage, inode: Inode) -> dict[str, int]:
|
|
if inode.layout not in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT):
|
|
raise FixtureError(f"inode layout {inode.layout} is not compressed")
|
|
offset = align(inode.offset + inode.inode_size + inode.xattr_size, 8)
|
|
if offset > len(image.data) - 8:
|
|
raise FixtureError("compressed map header exceeds image")
|
|
raw0, advise, algorithm, clusterbits = struct.unpack_from(
|
|
"<IHBB", image.data, offset
|
|
)
|
|
return {
|
|
"offset": offset,
|
|
"raw0": raw0,
|
|
"advise": advise,
|
|
"algorithm_raw": algorithm,
|
|
"head1_algorithm": algorithm & 0x0F,
|
|
"head2_algorithm": algorithm >> 4,
|
|
"clusterbits_raw": clusterbits,
|
|
"lcluster_bits": image.block_bits + (clusterbits & 0x07),
|
|
"idata_size": raw0 >> 16,
|
|
}
|
|
|
|
|
|
def full_index_offset(image: ErofsImage, inode: Inode) -> int:
|
|
header = map_header(image, inode)
|
|
return align(header["offset"] + 8, 8) + 8
|
|
|
|
|
|
def full_record(image: ErofsImage, inode: Inode, lcn: int) -> dict[str, int]:
|
|
if inode.layout != LAYOUT_COMPRESSED_FULL:
|
|
raise FixtureError("full record requested from a non-full inode")
|
|
offset = full_index_offset(image, inode) + lcn * 8
|
|
if offset > len(image.data) - 8:
|
|
raise FixtureError("full index record exceeds image")
|
|
advise, clusterofs, word = struct.unpack_from("<HHI", image.data, offset)
|
|
return {
|
|
"offset": offset,
|
|
"advise": advise,
|
|
"type": advise & Z_EROFS_LCLUSTER_TYPE_MASK,
|
|
"partial_ref": int(bool(advise & Z_EROFS_LI_PARTIAL_REF)),
|
|
"clusterofs": clusterofs,
|
|
"word": word,
|
|
"pblk": word,
|
|
"delta0": word & 0xFFFF,
|
|
"delta1": word >> 16,
|
|
}
|
|
|
|
|
|
def full_index_summary(image: ErofsImage, inode: Inode) -> dict[str, Any]:
|
|
header = map_header(image, inode)
|
|
count = (inode.size + (1 << header["lcluster_bits"]) - 1) >> header[
|
|
"lcluster_bits"
|
|
]
|
|
types: Counter[int] = Counter()
|
|
heads = []
|
|
partial_heads = []
|
|
for lcn in range(count):
|
|
record = full_record(image, inode, lcn)
|
|
types[record["type"]] += 1
|
|
if record["type"] in (
|
|
Z_EROFS_LCLUSTER_TYPE_PLAIN,
|
|
Z_EROFS_LCLUSTER_TYPE_HEAD1,
|
|
Z_EROFS_LCLUSTER_TYPE_HEAD2,
|
|
):
|
|
if len(heads) < 16:
|
|
heads.append(
|
|
{
|
|
"lcn": lcn,
|
|
"offset": record["offset"],
|
|
"type": record["type"],
|
|
"pblk": record["pblk"],
|
|
"clusterofs": record["clusterofs"],
|
|
}
|
|
)
|
|
if record["partial_ref"]:
|
|
partial_heads.append(
|
|
{
|
|
"lcn": lcn,
|
|
"offset": record["offset"],
|
|
"pblk": record["pblk"],
|
|
}
|
|
)
|
|
return {
|
|
"record_count": count,
|
|
"type_counts": {str(key): value for key, value in sorted(types.items())},
|
|
"first_heads": heads,
|
|
"partial_heads": partial_heads,
|
|
}
|
|
|
|
|
|
EXTENT_RE = re.compile(
|
|
r"^\s*\d+:\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:"
|
|
r"\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$"
|
|
)
|
|
|
|
|
|
def dump_extents(image: Path, path: str, log: Path | None = None) -> list[dict[str, int]]:
|
|
result = run(
|
|
["dump.erofs", f"--path={path}", "-e", str(image)],
|
|
log=log,
|
|
)
|
|
extents = []
|
|
for line in result.stdout.splitlines():
|
|
match = EXTENT_RE.match(line)
|
|
if match is None:
|
|
continue
|
|
logical_start, logical_end, logical_length, physical_start, physical_end, physical_length = (
|
|
int(value) for value in match.groups()
|
|
)
|
|
if logical_end - logical_start != logical_length:
|
|
raise FixtureError("dump.erofs reported an inconsistent logical extent")
|
|
if physical_end - physical_start != physical_length:
|
|
raise FixtureError("dump.erofs reported an inconsistent physical extent")
|
|
extents.append(
|
|
{
|
|
"logical_start": logical_start,
|
|
"logical_end": logical_end,
|
|
"logical_length": logical_length,
|
|
"physical_start": physical_start,
|
|
"physical_end": physical_end,
|
|
"physical_length": physical_length,
|
|
}
|
|
)
|
|
if not extents:
|
|
raise FixtureError(f"dump.erofs reported no extents for {image}:{path}")
|
|
return extents
|
|
|
|
|
|
def extent_summary(extents: list[dict[str, int]]) -> dict[str, Any]:
|
|
transitions = []
|
|
for index in range(1, len(extents)):
|
|
previous = extents[index - 1]
|
|
current = extents[index]
|
|
previous_plain = previous["logical_length"] == previous["physical_length"]
|
|
current_plain = current["logical_length"] == current["physical_length"]
|
|
if previous_plain != current_plain:
|
|
transitions.append(
|
|
{
|
|
"logical_offset": current["logical_start"],
|
|
"previous_plain": previous_plain,
|
|
"current_plain": current_plain,
|
|
}
|
|
)
|
|
return {
|
|
"count": len(extents),
|
|
"first": extents[:8],
|
|
"last": extents[-1],
|
|
"max_logical_length": max(item["logical_length"] for item in extents),
|
|
"max_physical_length": max(item["physical_length"] for item in extents),
|
|
"plain_count": sum(
|
|
item["physical_length"] != 0
|
|
and item["logical_length"] == item["physical_length"]
|
|
for item in extents
|
|
),
|
|
"compressed_count": sum(
|
|
item["physical_length"] != 0
|
|
and item["logical_length"] > item["physical_length"]
|
|
for item in extents
|
|
),
|
|
"hole_or_fragment_count": sum(
|
|
item["physical_length"] == 0 for item in extents
|
|
),
|
|
"transitions": transitions[:8],
|
|
}
|
|
|
|
|
|
def inspect_path(
|
|
output_root: Path,
|
|
image_path: Path,
|
|
path: str,
|
|
*,
|
|
write_log: bool = True,
|
|
) -> dict[str, Any]:
|
|
image = ErofsImage.load(image_path)
|
|
image.validate_superblock()
|
|
inode = inode_for(image, path)
|
|
details: dict[str, Any] = {
|
|
"path": path,
|
|
"nid": inode.nid,
|
|
"inode_offset": inode.offset,
|
|
"inode_size": inode.inode_size,
|
|
"xattr_size": inode.xattr_size,
|
|
"layout": inode.layout,
|
|
"size": inode.size,
|
|
}
|
|
if inode.layout in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT):
|
|
details["map_header"] = map_header(image, inode)
|
|
log = None
|
|
if write_log:
|
|
safe_path = path.removeprefix("/").replace("/", "-")
|
|
log = output_root / "logs" / f"dump-{image_path.stem}-{safe_path}.log"
|
|
details["dump_log"] = relative(log, output_root)
|
|
details["extents"] = extent_summary(dump_extents(image_path, path, log))
|
|
if inode.layout == LAYOUT_COMPRESSED_FULL:
|
|
details["full_index"] = full_index_summary(image, inode)
|
|
return details
|
|
|
|
|
|
def assert_compressed(
|
|
details: dict[str, Any],
|
|
*,
|
|
algorithm: str,
|
|
layout: int | None = None,
|
|
) -> None:
|
|
if details["layout"] not in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT):
|
|
raise FixtureError(f"{details['path']} was not compressed")
|
|
if layout is not None and details["layout"] != layout:
|
|
raise FixtureError(
|
|
f"{details['path']} layout {details['layout']} != expected {layout}"
|
|
)
|
|
actual = details["map_header"]["head1_algorithm"]
|
|
if actual != ALGORITHMS[algorithm]:
|
|
raise FixtureError(
|
|
f"{details['path']} algorithm {actual} != {ALGORITHMS[algorithm]}"
|
|
)
|
|
|
|
|
|
def patch_partial_reference(
|
|
base: Path,
|
|
output: Path,
|
|
*,
|
|
algorithm: str,
|
|
) -> dict[str, Any]:
|
|
image = ErofsImage.load(base)
|
|
inode_a = inode_for(image, "/a.dat")
|
|
inode_b = inode_for(image, "/b.dat")
|
|
if inode_a.layout != LAYOUT_COMPRESSED_FULL or inode_b.layout != LAYOUT_COMPRESSED_FULL:
|
|
raise FixtureError("partial-reference transform requires full indexes")
|
|
header_a = map_header(image, inode_a)
|
|
header_b = map_header(image, inode_b)
|
|
expected_algorithm = ALGORITHMS[algorithm]
|
|
if (
|
|
header_a["head1_algorithm"] != expected_algorithm
|
|
or header_b["head1_algorithm"] != expected_algorithm
|
|
):
|
|
raise FixtureError("partial-reference source algorithm mismatch")
|
|
record_a = full_record(image, inode_a, 0)
|
|
record_b = full_record(image, inode_b, 0)
|
|
record_a_next = full_record(image, inode_a, 1)
|
|
record_b_next = full_record(image, inode_b, 1)
|
|
if record_a["type"] != Z_EROFS_LCLUSTER_TYPE_HEAD1:
|
|
raise FixtureError("a.dat first record is not HEAD1")
|
|
if record_b["type"] != Z_EROFS_LCLUSTER_TYPE_HEAD1:
|
|
raise FixtureError("b.dat first record is not HEAD1")
|
|
if record_a["pblk"] == record_b["pblk"]:
|
|
raise FixtureError("mkfs unexpectedly reused the partial pcluster")
|
|
if record_b["advise"] & Z_EROFS_LI_PARTIAL_REF:
|
|
raise FixtureError("b.dat was already marked partial")
|
|
if inode_b.size >= inode_a.size:
|
|
raise FixtureError("partial file is not shorter than the complete stream")
|
|
if (
|
|
record_a_next["type"] != Z_EROFS_LCLUSTER_TYPE_NONHEAD
|
|
or record_b_next["type"] != Z_EROFS_LCLUSTER_TYPE_NONHEAD
|
|
or not record_a_next["delta0"] & Z_EROFS_LI_D0_CBLKCNT
|
|
or not record_b_next["delta0"] & Z_EROFS_LI_D0_CBLKCNT
|
|
):
|
|
raise FixtureError("partial source lacks a first-pcluster block count")
|
|
|
|
struct.pack_into(
|
|
"<H", image.data, record_b["offset"], record_b["advise"] | Z_EROFS_LI_PARTIAL_REF
|
|
)
|
|
struct.pack_into("<I", image.data, record_b["offset"] + 4, record_a["pblk"])
|
|
struct.pack_into(
|
|
"<H", image.data, record_b_next["offset"] + 4, record_a_next["delta0"]
|
|
)
|
|
image.update_checksum()
|
|
image.save(output)
|
|
|
|
reopened = ErofsImage.load(output)
|
|
reopened.validate_superblock()
|
|
new_a = full_record(reopened, inode_for(reopened, "/a.dat"), 0)
|
|
new_b = full_record(reopened, inode_for(reopened, "/b.dat"), 0)
|
|
new_b_next = full_record(reopened, inode_for(reopened, "/b.dat"), 1)
|
|
if not new_b["partial_ref"] or new_b["pblk"] != new_a["pblk"]:
|
|
raise FixtureError("partial-reference transform did not survive reopen")
|
|
if new_b_next["delta0"] != record_a_next["delta0"]:
|
|
raise FixtureError("partial-reference pcluster length did not survive reopen")
|
|
return {
|
|
"algorithm": algorithm,
|
|
"a_nid": inode_a.nid,
|
|
"a_size": inode_a.size,
|
|
"a_index_offset": record_a["offset"],
|
|
"a_pblk": record_a["pblk"],
|
|
"b_nid": inode_b.nid,
|
|
"b_size": inode_b.size,
|
|
"b_index_offset": record_b["offset"],
|
|
"b_old_advise": record_b["advise"],
|
|
"b_new_advise": new_b["advise"],
|
|
"b_old_pblk": record_b["pblk"],
|
|
"b_new_pblk": new_b["pblk"],
|
|
"b_old_compressed_blocks": record_b_next["delta0"]
|
|
& ~Z_EROFS_LI_D0_CBLKCNT,
|
|
"b_new_compressed_blocks": new_b_next["delta0"]
|
|
& ~Z_EROFS_LI_D0_CBLKCNT,
|
|
"source_stream_reused": True,
|
|
"partial_ref": True,
|
|
}
|
|
|
|
|
|
def patch_head2(base: Path, output: Path) -> dict[str, Any]:
|
|
image = ErofsImage.load(base)
|
|
inode = inode_for(image, "/shape.dat")
|
|
if inode.layout != LAYOUT_COMPRESSED_FULL:
|
|
raise FixtureError("HEAD2 transform requires full indexes")
|
|
header = map_header(image, inode)
|
|
if header["head1_algorithm"] != ALGORITHMS["lz4"]:
|
|
raise FixtureError("HEAD2 base is not LZ4")
|
|
if not header["advise"] & Z_EROFS_ADVISE_BIG_PCLUSTER_1:
|
|
raise FixtureError("HEAD2 base lacks HEAD1 big-pcluster advise")
|
|
|
|
count = (inode.size + (1 << header["lcluster_bits"]) - 1) >> header[
|
|
"lcluster_bits"
|
|
]
|
|
target_lcn = None
|
|
target = None
|
|
for lcn in range(count):
|
|
record = full_record(image, inode, lcn)
|
|
if record["type"] == Z_EROFS_LCLUSTER_TYPE_HEAD1:
|
|
target_lcn = lcn
|
|
target = record
|
|
break
|
|
if target_lcn is None or target is None:
|
|
raise FixtureError("HEAD2 base has no HEAD1 record")
|
|
|
|
old_incompat = image.feature_incompat
|
|
old_advise = header["advise"]
|
|
old_algorithm = header["algorithm_raw"]
|
|
new_advise = old_advise | Z_EROFS_ADVISE_BIG_PCLUSTER_2
|
|
new_algorithm = (old_algorithm & 0x0F) | ((old_algorithm & 0x0F) << 4)
|
|
new_record_advise = (
|
|
target["advise"] & ~Z_EROFS_LCLUSTER_TYPE_MASK
|
|
) | Z_EROFS_LCLUSTER_TYPE_HEAD2
|
|
image.put_u32(SUPER + 80, old_incompat | FEATURE_INCOMPAT_COMPR_HEAD2)
|
|
struct.pack_into("<H", image.data, header["offset"] + 4, new_advise)
|
|
image.data[header["offset"] + 6] = new_algorithm
|
|
struct.pack_into("<H", image.data, target["offset"], new_record_advise)
|
|
image.update_checksum()
|
|
image.save(output)
|
|
|
|
reopened = ErofsImage.load(output)
|
|
reopened.validate_superblock()
|
|
new_inode = inode_for(reopened, "/shape.dat")
|
|
new_header = map_header(reopened, new_inode)
|
|
new_record = full_record(reopened, new_inode, target_lcn)
|
|
if not reopened.feature_incompat & FEATURE_INCOMPAT_COMPR_HEAD2:
|
|
raise FixtureError("HEAD2 incompat feature did not survive reopen")
|
|
if not new_header["advise"] & Z_EROFS_ADVISE_BIG_PCLUSTER_2:
|
|
raise FixtureError("HEAD2 big-pcluster advise did not survive reopen")
|
|
if new_header["head2_algorithm"] != ALGORITHMS["lz4"]:
|
|
raise FixtureError("HEAD2 algorithm nibble did not survive reopen")
|
|
if new_record["type"] != Z_EROFS_LCLUSTER_TYPE_HEAD2:
|
|
raise FixtureError("HEAD2 index type did not survive reopen")
|
|
return {
|
|
"nid": inode.nid,
|
|
"header_offset": header["offset"],
|
|
"index_offset": target["offset"],
|
|
"target_lcn": target_lcn,
|
|
"pblk": target["pblk"],
|
|
"feature_incompat_before": old_incompat,
|
|
"feature_incompat_after": reopened.feature_incompat,
|
|
"header_advise_before": old_advise,
|
|
"header_advise_after": new_header["advise"],
|
|
"algorithm_raw_before": old_algorithm,
|
|
"algorithm_raw_after": new_header["algorithm_raw"],
|
|
"record_advise_before": target["advise"],
|
|
"record_advise_after": new_record["advise"],
|
|
}
|
|
|
|
|
|
def corrupt_target_extent(
|
|
image_path: Path,
|
|
output: Path,
|
|
*,
|
|
path: str,
|
|
algorithm: str,
|
|
) -> dict[str, Any]:
|
|
source = ErofsImage.load(image_path)
|
|
inode = inode_for(source, path)
|
|
header = map_header(source, inode)
|
|
if header["head1_algorithm"] != ALGORITHMS[algorithm]:
|
|
raise FixtureError("corruption target algorithm mismatch")
|
|
extents = dump_extents(image_path, path)
|
|
target = next(
|
|
(
|
|
extent
|
|
for extent in extents
|
|
if extent["physical_length"] > 0
|
|
and extent["logical_length"] > extent["physical_length"]
|
|
),
|
|
None,
|
|
)
|
|
if target is None:
|
|
raise FixtureError("no compressed extent available for corruption")
|
|
start = target["physical_start"]
|
|
end = target["physical_end"]
|
|
if start < source.checksum_end or end > len(source.data):
|
|
raise FixtureError("target compressed extent is outside payload bounds")
|
|
payload = source.data[start:end]
|
|
nonzero = [index for index, value in enumerate(payload) if value != 0]
|
|
if len(nonzero) < 16:
|
|
raise FixtureError("target compressed extent has too little encoded data")
|
|
patch_start = start + nonzero[0]
|
|
patch_length = min(64, end - patch_start)
|
|
old = bytes(source.data[patch_start : patch_start + patch_length])
|
|
for index in range(patch_start, patch_start + patch_length):
|
|
source.data[index] ^= 0xA5
|
|
new = bytes(source.data[patch_start : patch_start + patch_length])
|
|
if old == new:
|
|
raise AssertionError("compressed corruption did not change bytes")
|
|
if not source.checksum_valid():
|
|
raise FixtureError("payload corruption invalidated the superblock checksum")
|
|
source.save(output)
|
|
reopened = ErofsImage.load(output)
|
|
reopened.validate_superblock()
|
|
return {
|
|
"path": path,
|
|
"algorithm": algorithm,
|
|
"algorithm_id": ALGORITHMS[algorithm],
|
|
"nid": inode.nid,
|
|
"map_header_offset": header["offset"],
|
|
"extent_logical_start": target["logical_start"],
|
|
"extent_logical_length": target["logical_length"],
|
|
"pcluster_start": start,
|
|
"pcluster_length": target["physical_length"],
|
|
"patch_offset": patch_start,
|
|
"patch_length": patch_length,
|
|
"old_sha256": hashlib.sha256(old).hexdigest(),
|
|
"new_sha256": hashlib.sha256(new).hexdigest(),
|
|
"superblock_checksum_valid": True,
|
|
}
|
|
|
|
|
|
def write_checksum_file(root: Path, paths: list[Path], name: str) -> None:
|
|
with (root / name).open("w", encoding="ascii") as output:
|
|
for path in sorted(paths):
|
|
output.write(f"{sha256(path)} {relative(path, root)}\n")
|
|
|
|
|
|
def explicit_probe(
|
|
output_root: Path,
|
|
attempt_image: Path,
|
|
utils_source: Path | None,
|
|
) -> dict[str, Any]:
|
|
attempt = inspect_path(output_root, attempt_image, "/shape.dat")
|
|
header = attempt["map_header"]
|
|
tokens = [
|
|
"Z_EROFS_ADVISE_EXTENTS",
|
|
"z_erofs_extent_recsize",
|
|
"struct z_erofs_extent {",
|
|
]
|
|
matches: dict[str, list[str]] = {token: [] for token in tokens}
|
|
if utils_source is not None:
|
|
for base in (utils_source / "include", utils_source / "lib"):
|
|
if not base.is_dir():
|
|
continue
|
|
for path in sorted(item for item in base.rglob("*") if item.is_file()):
|
|
try:
|
|
content = path.read_text(encoding="utf-8", errors="ignore")
|
|
except OSError:
|
|
continue
|
|
for token in tokens:
|
|
if token in content:
|
|
matches[token].append(str(path))
|
|
return {
|
|
"status": "SHELVED",
|
|
"attempt_image": relative(attempt_image, output_root),
|
|
"attempt_layout": attempt["layout"],
|
|
"attempt_map_header_offset": header["offset"],
|
|
"attempt_h_advise": header["advise"],
|
|
"attempt_explicit_bit": bool(
|
|
attempt["layout"] == LAYOUT_COMPRESSED_FULL
|
|
and header["advise"] & Z_EROFS_ADVISE_COMPACTED_2B
|
|
),
|
|
"mapped_payload_generated": False,
|
|
"erofs_utils_source": str(utils_source) if utils_source else None,
|
|
"source_token_matches": matches,
|
|
"structured_transform_attempt": (
|
|
"Refused: converting legacy lcluster indexes into variable-size "
|
|
"extent records requires relocating subsequent metadata and payload; "
|
|
"the helper cannot validate a mapped result with erofs-utils 1.8.6."
|
|
),
|
|
}
|
|
|
|
|
|
def build_fixtures(args: argparse.Namespace) -> None:
|
|
output_root = args.output.resolve()
|
|
if output_root.exists():
|
|
raise FixtureError(f"output already exists: {output_root}")
|
|
(output_root / "images").mkdir(parents=True)
|
|
(output_root / "logs").mkdir()
|
|
(output_root / "sources").mkdir()
|
|
(output_root / "base-images").mkdir()
|
|
create_sources(output_root / "sources")
|
|
|
|
version = run(["mkfs.erofs", "-V"]).stdout.strip()
|
|
if "1.8.6" not in version:
|
|
raise FixtureError(f"mkfs.erofs 1.8.6 is required, got: {version}")
|
|
|
|
commands: list[dict[str, Any]] = []
|
|
images: dict[str, Path] = {}
|
|
targets: dict[str, dict[str, Any]] = {}
|
|
|
|
def generate(
|
|
name: str,
|
|
source_dir: str,
|
|
options: list[str],
|
|
path: str,
|
|
algorithm: str,
|
|
layout: int | None = None,
|
|
) -> dict[str, Any]:
|
|
image = mkfs_image(
|
|
output_root=output_root,
|
|
image_name=name,
|
|
source=output_root / "sources" / source_dir,
|
|
options=options,
|
|
command_log=commands,
|
|
)
|
|
images[name] = image
|
|
details = inspect_path(output_root, image, path)
|
|
assert_compressed(details, algorithm=algorithm, layout=layout)
|
|
targets[f"{name}:{path}"] = details
|
|
return details
|
|
|
|
compact_4k = generate(
|
|
"lz4-compact-4k.erofs",
|
|
"lz4",
|
|
["-zlz4", "-C4096"],
|
|
"/compressed.bin",
|
|
"lz4",
|
|
LAYOUT_COMPRESSED_COMPACT,
|
|
)
|
|
full_4k = generate(
|
|
"lz4-full-4k.erofs",
|
|
"lz4",
|
|
["-zlz4", "-C4096", "-Elegacy-compress"],
|
|
"/compressed.bin",
|
|
"lz4",
|
|
LAYOUT_COMPRESSED_FULL,
|
|
)
|
|
compact_64k = generate(
|
|
"lz4-compact-64k.erofs",
|
|
"lz4",
|
|
["-zlz4", "-C65536"],
|
|
"/compressed.bin",
|
|
"lz4",
|
|
LAYOUT_COMPRESSED_COMPACT,
|
|
)
|
|
compact_256k = generate(
|
|
"lz4-compact-256k.erofs",
|
|
"lz4",
|
|
["-zlz4", "-C262144"],
|
|
"/compressed.bin",
|
|
"lz4",
|
|
LAYOUT_COMPRESSED_COMPACT,
|
|
)
|
|
generate(
|
|
"lz4-large.erofs",
|
|
"large",
|
|
["-zlz4", "-C65536"],
|
|
"/large.bin",
|
|
"lz4",
|
|
)
|
|
ztail = generate(
|
|
"lz4-ztail.erofs",
|
|
"ztail",
|
|
["-zlz4", "-C4096", "-Eztailpacking"],
|
|
"/inline.dat",
|
|
"lz4",
|
|
)
|
|
for path in (
|
|
"/exact-pcluster.dat",
|
|
"/one-byte-tail.dat",
|
|
"/max-tail.dat",
|
|
"/zero-tail.dat",
|
|
):
|
|
details = inspect_path(output_root, images["lz4-ztail.erofs"], path)
|
|
targets[f"lz4-ztail.erofs:{path}"] = details
|
|
|
|
if not compact_4k["map_header"]["advise"] & Z_EROFS_ADVISE_COMPACTED_2B:
|
|
raise FixtureError("4K compact image lacks compacted index advise")
|
|
if full_4k["map_header"]["advise"] & Z_EROFS_ADVISE_COMPACTED_2B:
|
|
raise FixtureError("full-index image advertises compacted indexes")
|
|
for details, requested in ((compact_64k, 65536), (compact_256k, 262144)):
|
|
header = details["map_header"]
|
|
if not header["advise"] & Z_EROFS_ADVISE_BIG_PCLUSTER_1:
|
|
raise FixtureError(f"{requested} image lacks big-pcluster advise")
|
|
if details["extents"]["max_physical_length"] <= BLOCK_SIZE:
|
|
raise FixtureError(f"{requested} image has no multi-block pcluster")
|
|
if not ztail["map_header"]["advise"] & Z_EROFS_ADVISE_INLINE_PCLUSTER:
|
|
raise FixtureError("ztailpacking target lacks inline-pcluster advise")
|
|
if ztail["map_header"]["idata_size"] == 0:
|
|
raise FixtureError("ztailpacking target has zero inline encoded size")
|
|
|
|
for level in (1, 6, 9):
|
|
generate(
|
|
f"deflate-level{level}.erofs",
|
|
"levels",
|
|
[f"-zdeflate,level={level}", "-C65536", "-Elegacy-compress"],
|
|
"/level.dat",
|
|
"deflate",
|
|
LAYOUT_COMPRESSED_FULL,
|
|
)
|
|
for level in (1, 15, 22):
|
|
generate(
|
|
f"zstd-level{level}.erofs",
|
|
"levels",
|
|
[f"-zzstd,level={level}", "-C65536", "-Elegacy-compress"],
|
|
"/level.dat",
|
|
"zstd",
|
|
LAYOUT_COMPRESSED_FULL,
|
|
)
|
|
generate(
|
|
"lzma-level6.erofs",
|
|
"levels",
|
|
["-zlzma,level=6", "-C65536", "-Elegacy-compress"],
|
|
"/level.dat",
|
|
"lzma",
|
|
LAYOUT_COMPRESSED_FULL,
|
|
)
|
|
generate(
|
|
"lzma-large.erofs",
|
|
"lzma-large",
|
|
["-zlzma,level=6", "-C65536", "-Elegacy-compress"],
|
|
"/large.bin",
|
|
"lzma",
|
|
LAYOUT_COMPRESSED_FULL,
|
|
)
|
|
generate(
|
|
"microlzma-edge.erofs",
|
|
"microlzma",
|
|
["-zlzma,level=6", "-C4096", "-Elegacy-compress"],
|
|
"/boundary-16k.bin",
|
|
"lzma",
|
|
LAYOUT_COMPRESSED_FULL,
|
|
)
|
|
for path in ("/one-byte.bin", "/block-4k.bin"):
|
|
details = inspect_path(output_root, images["microlzma-edge.erofs"], path)
|
|
targets[f"microlzma-edge.erofs:{path}"] = details
|
|
|
|
partials: dict[str, Any] = {}
|
|
corruptions: dict[str, Any] = {}
|
|
for algorithm, compressor in (
|
|
("deflate", "-zdeflate,level=1"),
|
|
("zstd", "-zzstd,level=1"),
|
|
("lzma", "-zlzma,level=6"),
|
|
):
|
|
base_name = f"{algorithm}-partial-base.erofs"
|
|
base = mkfs_image(
|
|
output_root=output_root,
|
|
image_name=f"../base-images/{base_name}",
|
|
source=output_root
|
|
/ "sources"
|
|
/ ("partial-deflate" if algorithm == "deflate" else "partial"),
|
|
options=[compressor, "-C1048576", "-Elegacy-compress"],
|
|
command_log=commands,
|
|
)
|
|
valid = output_root / "images" / f"{algorithm}-partial-ref.erofs"
|
|
partials[algorithm] = patch_partial_reference(
|
|
base, valid, algorithm=algorithm
|
|
)
|
|
images[valid.name] = valid
|
|
for path in ("/a.dat", "/b.dat"):
|
|
details = inspect_path(output_root, valid, path)
|
|
assert_compressed(
|
|
details,
|
|
algorithm=algorithm,
|
|
layout=LAYOUT_COMPRESSED_FULL,
|
|
)
|
|
targets[f"{valid.name}:{path}"] = details
|
|
corrupt = output_root / "images" / f"{algorithm}-partial-ref-corrupt.erofs"
|
|
corruptions[algorithm] = corrupt_target_extent(
|
|
valid,
|
|
corrupt,
|
|
path="/a.dat",
|
|
algorithm=algorithm,
|
|
)
|
|
images[corrupt.name] = corrupt
|
|
|
|
head2_base = mkfs_image(
|
|
output_root=output_root,
|
|
image_name="../base-images/head2-base.erofs",
|
|
source=output_root / "sources" / "shape",
|
|
options=["-zlz4", "-C65536", "-Elegacy-compress"],
|
|
command_log=commands,
|
|
)
|
|
head2 = output_root / "images" / "head2.erofs"
|
|
head2_transform = patch_head2(head2_base, head2)
|
|
images[head2.name] = head2
|
|
head2_details = inspect_path(output_root, head2, "/shape.dat")
|
|
assert_compressed(
|
|
head2_details,
|
|
algorithm="lz4",
|
|
layout=LAYOUT_COMPRESSED_FULL,
|
|
)
|
|
targets[f"{head2.name}:/shape.dat"] = head2_details
|
|
head2_corrupt = output_root / "images" / "head2-corrupt.erofs"
|
|
corruptions["head2"] = corrupt_target_extent(
|
|
head2,
|
|
head2_corrupt,
|
|
path="/shape.dat",
|
|
algorithm="lz4",
|
|
)
|
|
images[head2_corrupt.name] = head2_corrupt
|
|
|
|
interlaced = generate(
|
|
"interlaced.erofs",
|
|
"shape",
|
|
["-zlz4", "-C4096", "-Efragments"],
|
|
"/shape.dat",
|
|
"lz4",
|
|
)
|
|
if not interlaced["map_header"]["advise"] & Z_EROFS_ADVISE_INTERLACED_PCLUSTER:
|
|
raise FixtureError("interlaced target lacks interlaced advise")
|
|
if not interlaced["extents"]["transitions"]:
|
|
raise FixtureError("interlaced target has no compressed/plain transition")
|
|
|
|
extent_attempt = generate(
|
|
"extent-attempt.erofs",
|
|
"shape",
|
|
[
|
|
"-zlz4",
|
|
"-C65536",
|
|
"-Elegacy-compress",
|
|
"--max-extent-bytes=65536",
|
|
],
|
|
"/shape.dat",
|
|
"lz4",
|
|
LAYOUT_COMPRESSED_FULL,
|
|
)
|
|
if extent_attempt["map_header"]["advise"] & Z_EROFS_ADVISE_COMPACTED_2B:
|
|
raise FixtureError("extent attempt unexpectedly selected the explicit bit")
|
|
|
|
image_paths = sorted(set(images.values()))
|
|
source_paths = sorted(
|
|
path for path in (output_root / "sources").rglob("*") if path.is_file()
|
|
)
|
|
write_checksum_file(output_root, image_paths, "SHA256SUMS")
|
|
write_checksum_file(output_root, source_paths, "SOURCE-SHA256SUMS")
|
|
|
|
manifest = {
|
|
"schema": 1,
|
|
"mkfs_version": version,
|
|
"fixed_uuid": FIXED_UUID,
|
|
"source_inventory": source_inventory(output_root / "sources"),
|
|
"commands": commands,
|
|
"images": {
|
|
relative(path, output_root): {
|
|
"size": path.stat().st_size,
|
|
"sha256": sha256(path),
|
|
}
|
|
for path in image_paths
|
|
},
|
|
"targets": targets,
|
|
"partial_references": partials,
|
|
"corruptions": corruptions,
|
|
"head2_transform": head2_transform,
|
|
"explicit_extent": explicit_probe(
|
|
output_root,
|
|
images["extent-attempt.erofs"],
|
|
args.erofs_utils_source.resolve()
|
|
if args.erofs_utils_source is not None
|
|
else None,
|
|
),
|
|
}
|
|
(output_root / "fixture-manifest.json").write_text(
|
|
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
|
encoding="ascii",
|
|
)
|
|
verify_output(output_root)
|
|
|
|
|
|
def verify_output(output_root: Path) -> None:
|
|
output_root = output_root.resolve()
|
|
manifest_path = output_root / "fixture-manifest.json"
|
|
manifest = json.loads(manifest_path.read_text(encoding="ascii"))
|
|
for path_text, expected in manifest["images"].items():
|
|
path = output_root / path_text
|
|
if not path.is_file():
|
|
raise FixtureError(f"missing image: {path_text}")
|
|
if path.stat().st_size != expected["size"]:
|
|
raise FixtureError(f"image size changed: {path_text}")
|
|
if sha256(path) != expected["sha256"]:
|
|
raise FixtureError(f"image hash changed: {path_text}")
|
|
ErofsImage.load(path).validate_superblock()
|
|
for expected in manifest["source_inventory"]:
|
|
path = output_root / expected["path"]
|
|
if path.stat().st_size != expected["size"] or sha256(path) != expected["sha256"]:
|
|
raise FixtureError(f"source changed: {expected['path']}")
|
|
for key, expected in manifest["targets"].items():
|
|
image_name, path = key.split(":", 1)
|
|
actual = inspect_path(
|
|
output_root,
|
|
output_root / "images" / image_name,
|
|
path,
|
|
write_log=False,
|
|
)
|
|
actual.pop("dump_log", None)
|
|
comparable = dict(expected)
|
|
comparable.pop("dump_log", None)
|
|
if actual != comparable:
|
|
raise FixtureError(f"structured target fields changed: {key}")
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
create = subparsers.add_parser("create")
|
|
create.add_argument("--output", type=Path, required=True)
|
|
create.add_argument("--erofs-utils-source", type=Path)
|
|
verify = subparsers.add_parser("verify")
|
|
verify.add_argument("--output", type=Path, required=True)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
if args.command == "create":
|
|
build_fixtures(args)
|
|
else:
|
|
verify_output(args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|