419 lines
16 KiB
Python
Executable File
419 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Generate deterministic real EROFS fixtures for B19a."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
|
|
|
|
SUPER = 1024
|
|
MAGIC = 0xE0F5E1E2
|
|
CRC32C_POLY = 0x82F63B78
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
|
value = seed
|
|
for byte in data:
|
|
value ^= byte
|
|
for _ in range(8):
|
|
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
|
return value & 0xFFFFFFFF
|
|
|
|
|
|
def deterministic_value(label: str, length: int) -> bytes:
|
|
seed = (label + "|").encode("ascii")
|
|
return (seed * (length // len(seed) + 1))[:length]
|
|
|
|
|
|
def load_spec(path: Path) -> dict:
|
|
spec = json.loads(path.read_text(encoding="ascii"))
|
|
if (
|
|
spec.get("schema") != 1
|
|
or spec.get("batch") != "B19a"
|
|
or spec.get("candidate") != "P15-022"
|
|
):
|
|
raise SystemExit("invalid B19a fixture spec identity")
|
|
return spec
|
|
|
|
|
|
class Image:
|
|
def __init__(self, data: bytes | bytearray):
|
|
self.data = bytearray(data)
|
|
if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
|
|
raise ValueError("invalid EROFS image")
|
|
self.block_bits = self.data[SUPER + 12]
|
|
if self.block_bits < 9 or self.block_bits > 16:
|
|
raise ValueError("invalid EROFS block size")
|
|
self.block_size = 1 << self.block_bits
|
|
self.blocks = self.u32(SUPER + 36)
|
|
self.limit = self.blocks << self.block_bits
|
|
self.meta_blkaddr = self.u32(SUPER + 40)
|
|
self.xattr_blkaddr = self.u32(SUPER + 44)
|
|
self.root_nid = self.u16(SUPER + 14)
|
|
if self.limit > len(self.data):
|
|
raise ValueError("EROFS image is truncated")
|
|
|
|
@classmethod
|
|
def load(cls, path: Path) -> "Image":
|
|
return cls(path.read_bytes())
|
|
|
|
def u16(self, offset: int) -> int:
|
|
if offset < 0 or offset + 2 > len(self.data):
|
|
raise ValueError("u16 read is out of bounds")
|
|
return struct.unpack_from("<H", self.data, offset)[0]
|
|
|
|
def u32(self, offset: int) -> int:
|
|
if offset < 0 or offset + 4 > len(self.data):
|
|
raise ValueError("u32 read is out of bounds")
|
|
return struct.unpack_from("<I", self.data, offset)[0]
|
|
|
|
def inode(self, nid: int) -> dict[str, int]:
|
|
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
|
if offset > self.limit or 64 > self.limit - offset:
|
|
raise ValueError("inode is out of bounds")
|
|
inode_format = self.u16(offset)
|
|
inode_size = 64 if inode_format & 1 else 32
|
|
xattr_count = self.u16(offset + 2)
|
|
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
|
size = struct.unpack_from(
|
|
"<Q" if inode_size == 64 else "<I", self.data, offset + 8
|
|
)[0]
|
|
return {
|
|
"nid": nid,
|
|
"offset": offset,
|
|
"inode_size": inode_size,
|
|
"xattr_size": xattr_size,
|
|
"layout": (inode_format >> 1) & 7,
|
|
"size": size,
|
|
"start_block": self.u32(offset + 16),
|
|
}
|
|
|
|
def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
|
|
if logical > inode["size"] or length > inode["size"] - logical:
|
|
raise ValueError("inode data range is out of bounds")
|
|
if inode["layout"] == 0:
|
|
physical = (inode["start_block"] << self.block_bits) + logical
|
|
elif inode["layout"] == 2:
|
|
tail_start = ((inode["size"] + self.block_size - 1) // self.block_size - 1) * self.block_size
|
|
if logical < tail_start:
|
|
if logical + length > tail_start:
|
|
raise ValueError("inode data read crosses inline tail")
|
|
physical = (inode["start_block"] << self.block_bits) + logical
|
|
else:
|
|
physical = (
|
|
inode["offset"]
|
|
+ inode["inode_size"]
|
|
+ inode["xattr_size"]
|
|
+ logical
|
|
- tail_start
|
|
)
|
|
else:
|
|
raise ValueError("unsupported directory layout")
|
|
if physical > self.limit or length > self.limit - physical:
|
|
raise ValueError("inode data is outside the image")
|
|
return bytes(self.data[physical : physical + length])
|
|
|
|
@staticmethod
|
|
def parse_dirblock(data: bytes) -> list[tuple[bytes, int]]:
|
|
if len(data) < 12:
|
|
raise ValueError("directory block is short")
|
|
first_name = struct.unpack_from("<H", data, 8)[0]
|
|
if first_name < 12 or first_name >= len(data) or first_name % 12:
|
|
raise ValueError("directory first name offset is invalid")
|
|
count = first_name // 12
|
|
entries = []
|
|
previous_offset = 0
|
|
previous_name: bytes | None = None
|
|
for index in range(count):
|
|
entry = index * 12
|
|
nid = struct.unpack_from("<Q", data, entry)[0]
|
|
start = struct.unpack_from("<H", data, entry + 8)[0]
|
|
end = (
|
|
struct.unpack_from("<H", data, entry + 20)[0]
|
|
if index + 1 < count
|
|
else len(data)
|
|
)
|
|
if (
|
|
start < first_name
|
|
or start >= len(data)
|
|
or (index == 0 and start != first_name)
|
|
or (index != 0 and start <= previous_offset)
|
|
or end <= start
|
|
or end > len(data)
|
|
):
|
|
raise ValueError("directory name bounds are invalid")
|
|
span = data[start:end]
|
|
if index + 1 < count:
|
|
if b"\0" in span:
|
|
raise ValueError("non-trailing directory name contains NUL")
|
|
name = span
|
|
else:
|
|
name = span.split(b"\0", 1)[0]
|
|
if not name or len(name) > 255 or b"/" in name:
|
|
raise ValueError("directory name is invalid")
|
|
if previous_name is not None and previous_name >= name:
|
|
raise ValueError("directory names are not strictly ordered")
|
|
entries.append((name, nid))
|
|
previous_offset = start
|
|
previous_name = name
|
|
return entries
|
|
|
|
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
|
|
result = []
|
|
previous_name: bytes | None = None
|
|
logical = 0
|
|
while logical < inode["size"]:
|
|
length = min(self.block_size, inode["size"] - logical)
|
|
block = self.parse_dirblock(self.inode_data(inode, logical, length))
|
|
if previous_name is not None and previous_name >= block[0][0]:
|
|
raise ValueError("directory block boundary is not ordered")
|
|
result.extend(block)
|
|
previous_name = block[-1][0]
|
|
logical += length
|
|
return result
|
|
|
|
def resolve(self, path: str) -> dict[str, int]:
|
|
inode = self.inode(self.root_nid)
|
|
for component in path.strip("/").encode("ascii").split(b"/"):
|
|
if not component:
|
|
continue
|
|
matches = [nid for name, nid in self.directory_entries(inode) if name == component]
|
|
if len(matches) != 1:
|
|
raise ValueError(f"fixture path is absent or ambiguous: {path}")
|
|
inode = self.inode(matches[0])
|
|
return inode
|
|
|
|
def body_offset(self, inode: dict[str, int]) -> int:
|
|
return inode["offset"] + inode["inode_size"]
|
|
|
|
def body_shape(self, inode: dict[str, int]) -> tuple[int, list[int], list[int]]:
|
|
body = self.body_offset(inode)
|
|
size = inode["xattr_size"]
|
|
if size <= 12 or body > self.limit or size > self.limit - body:
|
|
raise ValueError("target xattr body is invalid")
|
|
shared_count = self.data[body + 4]
|
|
header_size = 12 + shared_count * 4
|
|
if header_size > size:
|
|
raise ValueError("target shared count is invalid")
|
|
shared_ids = [self.u32(body + 12 + index * 4) for index in range(shared_count)]
|
|
inline_entries = []
|
|
cursor = body + header_size
|
|
end = body + size
|
|
while cursor < end:
|
|
if cursor + 4 > end:
|
|
raise ValueError("target inline xattr header is truncated")
|
|
name_length = self.data[cursor]
|
|
value_length = self.u16(cursor + 2)
|
|
total = (4 + name_length + value_length + 3) & ~3
|
|
if total > end - cursor:
|
|
raise ValueError("target inline xattr entry is truncated")
|
|
inline_entries.append(cursor)
|
|
cursor += total
|
|
return header_size, shared_ids, inline_entries
|
|
|
|
def update_checksum(self) -> None:
|
|
if self.u32(SUPER + 8) & 1:
|
|
struct.pack_into("<I", self.data, SUPER + 4, 0)
|
|
struct.pack_into(
|
|
"<I", self.data, SUPER + 4, crc32c(self.data[SUPER : self.block_size])
|
|
)
|
|
|
|
def write(self, path: Path) -> None:
|
|
path.write_bytes(self.data)
|
|
|
|
|
|
def set_epoch(path: Path) -> None:
|
|
for entry in sorted(path.rglob("*"), reverse=True):
|
|
os.utime(entry, (0, 0), follow_symlinks=False)
|
|
os.utime(path, (0, 0), follow_symlinks=False)
|
|
|
|
|
|
def create_source(path: Path, spec: dict) -> None:
|
|
path.mkdir(parents=True)
|
|
path.chmod(0o755)
|
|
shared_value = deterministic_value(
|
|
"shared-value-p15-022", spec["fixture"]["shared_value_bytes"]
|
|
)
|
|
inline_value = deterministic_value(
|
|
"inline-value-p15-022", spec["fixture"]["inline_value_bytes"]
|
|
)
|
|
files = []
|
|
for index in range(spec["fixture"]["peer_count"]):
|
|
prefix = f"peer-{index:03d}-"
|
|
suffix = "x" * (spec["fixture"]["peer_name_bytes"] - len(prefix) - 4)
|
|
files.append(path / f"{prefix}{suffix}.bin")
|
|
target = path / "target.bin"
|
|
files.append(target)
|
|
for index, entry in enumerate(files):
|
|
entry.write_bytes(f"P15-022 file {index:03d}\n".encode("ascii"))
|
|
entry.chmod(0o644)
|
|
os.setxattr(entry, b"user.shared", shared_value)
|
|
if entry == target:
|
|
os.setxattr(entry, b"user.inline", inline_value)
|
|
else:
|
|
os.setxattr(
|
|
entry,
|
|
b"user.peer",
|
|
deterministic_value(f"peer-{index:03d}", 31),
|
|
)
|
|
|
|
budget = path / "budget"
|
|
budget.mkdir()
|
|
budget.chmod(0o755)
|
|
for index in range(spec["fixture"]["budget_file_count"]):
|
|
entry = budget / f"budget-{index:03d}.bin"
|
|
entry.write_bytes(b"B19a budget\n")
|
|
entry.chmod(0o644)
|
|
value = bytearray(
|
|
deterministic_value(
|
|
f"budget-value-{index:03d}",
|
|
spec["fixture"]["budget_value_bytes"],
|
|
)
|
|
)
|
|
struct.pack_into("<I", value, 0, index)
|
|
os.setxattr(entry, b"user.budget", bytes(value))
|
|
set_epoch(path)
|
|
|
|
|
|
def build_image(source: Path, output: Path, spec: dict) -> list[str]:
|
|
mkfs = shutil.which("mkfs.erofs")
|
|
if mkfs is None:
|
|
raise SystemExit("mkfs.erofs is required")
|
|
command = [
|
|
mkfs,
|
|
"-d0",
|
|
"-T0",
|
|
"--all-time",
|
|
"--all-root",
|
|
"--workers=1",
|
|
"--sort=path",
|
|
f"-U{spec['fixture']['uuid']}",
|
|
"-x2",
|
|
"-Eforce-inode-extended",
|
|
str(output),
|
|
str(source),
|
|
]
|
|
completed = subprocess.run(
|
|
command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
|
)
|
|
if completed.returncode != 0:
|
|
raise SystemExit(
|
|
"mkfs.erofs failed: " + completed.stdout.decode("utf-8", "replace")
|
|
)
|
|
return command
|
|
|
|
|
|
def mutate(valid: Path, output: Path, mutation: str, spec: dict) -> None:
|
|
image = Image.load(valid)
|
|
target = image.resolve(spec["fixture"]["target"])
|
|
body = image.body_offset(target)
|
|
_, shared_ids, inline_entries = image.body_shape(target)
|
|
if mutation == "corrupt-shared-count":
|
|
image.data[body + 4] = 255
|
|
elif mutation == "corrupt-shared-id":
|
|
if not shared_ids:
|
|
raise SystemExit("valid fixture has no shared xattr ID")
|
|
struct.pack_into("<I", image.data, body + 12, 0xFFFFFFFF)
|
|
elif mutation == "corrupt-inline-name":
|
|
if not inline_entries or image.data[inline_entries[0]] == 0:
|
|
raise SystemExit("valid fixture has no inline xattr entry")
|
|
image.data[inline_entries[0] + 4] = 0
|
|
else:
|
|
raise SystemExit(f"unknown B19a mutation: {mutation}")
|
|
image.update_checksum()
|
|
image.write(output)
|
|
|
|
|
|
def generate(output: Path, work: Path, spec: dict) -> dict:
|
|
if output.exists() or work.exists():
|
|
raise SystemExit("B19a output and work paths must be absent")
|
|
output.mkdir(parents=True)
|
|
source = work / "source"
|
|
create_source(source, spec)
|
|
valid = output / "valid.erofs"
|
|
command = build_image(source, valid, spec)
|
|
image = Image.load(valid)
|
|
root = image.inode(image.root_nid)
|
|
root_entries = image.directory_entries(root)
|
|
target = image.resolve(spec["fixture"]["target"])
|
|
_, shared_ids, inline_entries = image.body_shape(target)
|
|
if root["size"] <= image.block_size:
|
|
raise SystemExit("B19a root directory did not span multiple blocks")
|
|
if not shared_ids or not inline_entries:
|
|
raise SystemExit("B19a target lacks both shared and inline xattrs")
|
|
target_positions = [
|
|
index for index, (name, _) in enumerate(root_entries) if name == b"target.bin"
|
|
]
|
|
if target_positions != [len(root_entries) - 1]:
|
|
raise SystemExit("B19a target is not uniquely sorted after all peers")
|
|
for mutation in (
|
|
"corrupt-shared-count",
|
|
"corrupt-shared-id",
|
|
"corrupt-inline-name",
|
|
):
|
|
mutate(valid, output / f"{mutation}.erofs", mutation, spec)
|
|
fsck = shutil.which("fsck.erofs")
|
|
if fsck is None:
|
|
raise SystemExit("fsck.erofs is required")
|
|
completed = subprocess.run(
|
|
[fsck, str(valid)], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
|
)
|
|
if completed.returncode != 0:
|
|
raise SystemExit(
|
|
"fsck.erofs rejected B19a valid fixture: "
|
|
+ completed.stdout.decode("utf-8", "replace")
|
|
)
|
|
hashes = {path.name: sha256(path) for path in sorted(output.glob("*.erofs"))}
|
|
frozen = spec.get("fixture_sha256", {})
|
|
if frozen and hashes != frozen:
|
|
raise SystemExit(f"B19a fixture hashes differ: {hashes}")
|
|
report = {
|
|
"block_size": image.block_size,
|
|
"command": [*command[:-2], "OUTPUT/valid.erofs", "WORK/source"],
|
|
"fixture_sha256": hashes,
|
|
"inline_entries": len(inline_entries),
|
|
"root_directory_blocks": (root["size"] + image.block_size - 1) // image.block_size,
|
|
"root_entry_count": len(root_entries),
|
|
"shared_count": len(shared_ids),
|
|
"status": "PASS",
|
|
"target_entry_index": target_positions[0],
|
|
}
|
|
(output / "manifest.json").write_text(
|
|
json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
|
)
|
|
return report
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--spec", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--work", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
spec = load_spec(args.spec)
|
|
try:
|
|
report = generate(args.output, args.work, spec)
|
|
finally:
|
|
shutil.rmtree(args.work, ignore_errors=True)
|
|
print(json.dumps(report, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|