update
This commit is contained in:
Executable
+418
@@ -0,0 +1,418 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate deterministic real EROFS fixtures for B19a."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
|
||||
SUPER = 1024
|
||||
MAGIC = 0xE0F5E1E2
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
def deterministic_value(label: str, length: int) -> bytes:
|
||||
seed = (label + "|").encode("ascii")
|
||||
return (seed * (length // len(seed) + 1))[:length]
|
||||
|
||||
|
||||
def load_spec(path: Path) -> dict:
|
||||
spec = json.loads(path.read_text(encoding="ascii"))
|
||||
if (
|
||||
spec.get("schema") != 1
|
||||
or spec.get("batch") != "B19a"
|
||||
or spec.get("candidate") != "P15-022"
|
||||
):
|
||||
raise SystemExit("invalid B19a fixture spec identity")
|
||||
return spec
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self, data: bytes | bytearray):
|
||||
self.data = bytearray(data)
|
||||
if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
|
||||
raise ValueError("invalid EROFS image")
|
||||
self.block_bits = self.data[SUPER + 12]
|
||||
if self.block_bits < 9 or self.block_bits > 16:
|
||||
raise ValueError("invalid EROFS block size")
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.blocks = self.u32(SUPER + 36)
|
||||
self.limit = self.blocks << self.block_bits
|
||||
self.meta_blkaddr = self.u32(SUPER + 40)
|
||||
self.xattr_blkaddr = self.u32(SUPER + 44)
|
||||
self.root_nid = self.u16(SUPER + 14)
|
||||
if self.limit > len(self.data):
|
||||
raise ValueError("EROFS image is truncated")
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "Image":
|
||||
return cls(path.read_bytes())
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 2 > len(self.data):
|
||||
raise ValueError("u16 read is out of bounds")
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 4 > len(self.data):
|
||||
raise ValueError("u32 read is out of bounds")
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
if offset > self.limit or 64 > self.limit - offset:
|
||||
raise ValueError("inode is out of bounds")
|
||||
inode_format = self.u16(offset)
|
||||
inode_size = 64 if inode_format & 1 else 32
|
||||
xattr_count = self.u16(offset + 2)
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
||||
size = struct.unpack_from(
|
||||
"<Q" if inode_size == 64 else "<I", self.data, offset + 8
|
||||
)[0]
|
||||
return {
|
||||
"nid": nid,
|
||||
"offset": offset,
|
||||
"inode_size": inode_size,
|
||||
"xattr_size": xattr_size,
|
||||
"layout": (inode_format >> 1) & 7,
|
||||
"size": size,
|
||||
"start_block": self.u32(offset + 16),
|
||||
}
|
||||
|
||||
def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
|
||||
if logical > inode["size"] or length > inode["size"] - logical:
|
||||
raise ValueError("inode data range is out of bounds")
|
||||
if inode["layout"] == 0:
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
elif inode["layout"] == 2:
|
||||
tail_start = ((inode["size"] + self.block_size - 1) // self.block_size - 1) * self.block_size
|
||||
if logical < tail_start:
|
||||
if logical + length > tail_start:
|
||||
raise ValueError("inode data read crosses inline tail")
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
else:
|
||||
physical = (
|
||||
inode["offset"]
|
||||
+ inode["inode_size"]
|
||||
+ inode["xattr_size"]
|
||||
+ logical
|
||||
- tail_start
|
||||
)
|
||||
else:
|
||||
raise ValueError("unsupported directory layout")
|
||||
if physical > self.limit or length > self.limit - physical:
|
||||
raise ValueError("inode data is outside the image")
|
||||
return bytes(self.data[physical : physical + length])
|
||||
|
||||
@staticmethod
|
||||
def parse_dirblock(data: bytes) -> list[tuple[bytes, int]]:
|
||||
if len(data) < 12:
|
||||
raise ValueError("directory block is short")
|
||||
first_name = struct.unpack_from("<H", data, 8)[0]
|
||||
if first_name < 12 or first_name >= len(data) or first_name % 12:
|
||||
raise ValueError("directory first name offset is invalid")
|
||||
count = first_name // 12
|
||||
entries = []
|
||||
previous_offset = 0
|
||||
previous_name: bytes | None = None
|
||||
for index in range(count):
|
||||
entry = index * 12
|
||||
nid = struct.unpack_from("<Q", data, entry)[0]
|
||||
start = struct.unpack_from("<H", data, entry + 8)[0]
|
||||
end = (
|
||||
struct.unpack_from("<H", data, entry + 20)[0]
|
||||
if index + 1 < count
|
||||
else len(data)
|
||||
)
|
||||
if (
|
||||
start < first_name
|
||||
or start >= len(data)
|
||||
or (index == 0 and start != first_name)
|
||||
or (index != 0 and start <= previous_offset)
|
||||
or end <= start
|
||||
or end > len(data)
|
||||
):
|
||||
raise ValueError("directory name bounds are invalid")
|
||||
span = data[start:end]
|
||||
if index + 1 < count:
|
||||
if b"\0" in span:
|
||||
raise ValueError("non-trailing directory name contains NUL")
|
||||
name = span
|
||||
else:
|
||||
name = span.split(b"\0", 1)[0]
|
||||
if not name or len(name) > 255 or b"/" in name:
|
||||
raise ValueError("directory name is invalid")
|
||||
if previous_name is not None and previous_name >= name:
|
||||
raise ValueError("directory names are not strictly ordered")
|
||||
entries.append((name, nid))
|
||||
previous_offset = start
|
||||
previous_name = name
|
||||
return entries
|
||||
|
||||
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
|
||||
result = []
|
||||
previous_name: bytes | None = None
|
||||
logical = 0
|
||||
while logical < inode["size"]:
|
||||
length = min(self.block_size, inode["size"] - logical)
|
||||
block = self.parse_dirblock(self.inode_data(inode, logical, length))
|
||||
if previous_name is not None and previous_name >= block[0][0]:
|
||||
raise ValueError("directory block boundary is not ordered")
|
||||
result.extend(block)
|
||||
previous_name = block[-1][0]
|
||||
logical += length
|
||||
return result
|
||||
|
||||
def resolve(self, path: str) -> dict[str, int]:
|
||||
inode = self.inode(self.root_nid)
|
||||
for component in path.strip("/").encode("ascii").split(b"/"):
|
||||
if not component:
|
||||
continue
|
||||
matches = [nid for name, nid in self.directory_entries(inode) if name == component]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"fixture path is absent or ambiguous: {path}")
|
||||
inode = self.inode(matches[0])
|
||||
return inode
|
||||
|
||||
def body_offset(self, inode: dict[str, int]) -> int:
|
||||
return inode["offset"] + inode["inode_size"]
|
||||
|
||||
def body_shape(self, inode: dict[str, int]) -> tuple[int, list[int], list[int]]:
|
||||
body = self.body_offset(inode)
|
||||
size = inode["xattr_size"]
|
||||
if size <= 12 or body > self.limit or size > self.limit - body:
|
||||
raise ValueError("target xattr body is invalid")
|
||||
shared_count = self.data[body + 4]
|
||||
header_size = 12 + shared_count * 4
|
||||
if header_size > size:
|
||||
raise ValueError("target shared count is invalid")
|
||||
shared_ids = [self.u32(body + 12 + index * 4) for index in range(shared_count)]
|
||||
inline_entries = []
|
||||
cursor = body + header_size
|
||||
end = body + size
|
||||
while cursor < end:
|
||||
if cursor + 4 > end:
|
||||
raise ValueError("target inline xattr header is truncated")
|
||||
name_length = self.data[cursor]
|
||||
value_length = self.u16(cursor + 2)
|
||||
total = (4 + name_length + value_length + 3) & ~3
|
||||
if total > end - cursor:
|
||||
raise ValueError("target inline xattr entry is truncated")
|
||||
inline_entries.append(cursor)
|
||||
cursor += total
|
||||
return header_size, shared_ids, inline_entries
|
||||
|
||||
def update_checksum(self) -> None:
|
||||
if self.u32(SUPER + 8) & 1:
|
||||
struct.pack_into("<I", self.data, SUPER + 4, 0)
|
||||
struct.pack_into(
|
||||
"<I", self.data, SUPER + 4, crc32c(self.data[SUPER : self.block_size])
|
||||
)
|
||||
|
||||
def write(self, path: Path) -> None:
|
||||
path.write_bytes(self.data)
|
||||
|
||||
|
||||
def set_epoch(path: Path) -> None:
|
||||
for entry in sorted(path.rglob("*"), reverse=True):
|
||||
os.utime(entry, (0, 0), follow_symlinks=False)
|
||||
os.utime(path, (0, 0), follow_symlinks=False)
|
||||
|
||||
|
||||
def create_source(path: Path, spec: dict) -> None:
|
||||
path.mkdir(parents=True)
|
||||
path.chmod(0o755)
|
||||
shared_value = deterministic_value(
|
||||
"shared-value-p15-022", spec["fixture"]["shared_value_bytes"]
|
||||
)
|
||||
inline_value = deterministic_value(
|
||||
"inline-value-p15-022", spec["fixture"]["inline_value_bytes"]
|
||||
)
|
||||
files = []
|
||||
for index in range(spec["fixture"]["peer_count"]):
|
||||
prefix = f"peer-{index:03d}-"
|
||||
suffix = "x" * (spec["fixture"]["peer_name_bytes"] - len(prefix) - 4)
|
||||
files.append(path / f"{prefix}{suffix}.bin")
|
||||
target = path / "target.bin"
|
||||
files.append(target)
|
||||
for index, entry in enumerate(files):
|
||||
entry.write_bytes(f"P15-022 file {index:03d}\n".encode("ascii"))
|
||||
entry.chmod(0o644)
|
||||
os.setxattr(entry, b"user.shared", shared_value)
|
||||
if entry == target:
|
||||
os.setxattr(entry, b"user.inline", inline_value)
|
||||
else:
|
||||
os.setxattr(
|
||||
entry,
|
||||
b"user.peer",
|
||||
deterministic_value(f"peer-{index:03d}", 31),
|
||||
)
|
||||
|
||||
budget = path / "budget"
|
||||
budget.mkdir()
|
||||
budget.chmod(0o755)
|
||||
for index in range(spec["fixture"]["budget_file_count"]):
|
||||
entry = budget / f"budget-{index:03d}.bin"
|
||||
entry.write_bytes(b"B19a budget\n")
|
||||
entry.chmod(0o644)
|
||||
value = bytearray(
|
||||
deterministic_value(
|
||||
f"budget-value-{index:03d}",
|
||||
spec["fixture"]["budget_value_bytes"],
|
||||
)
|
||||
)
|
||||
struct.pack_into("<I", value, 0, index)
|
||||
os.setxattr(entry, b"user.budget", bytes(value))
|
||||
set_epoch(path)
|
||||
|
||||
|
||||
def build_image(source: Path, output: Path, spec: dict) -> list[str]:
|
||||
mkfs = shutil.which("mkfs.erofs")
|
||||
if mkfs is None:
|
||||
raise SystemExit("mkfs.erofs is required")
|
||||
command = [
|
||||
mkfs,
|
||||
"-d0",
|
||||
"-T0",
|
||||
"--all-time",
|
||||
"--all-root",
|
||||
"--workers=1",
|
||||
"--sort=path",
|
||||
f"-U{spec['fixture']['uuid']}",
|
||||
"-x2",
|
||||
"-Eforce-inode-extended",
|
||||
str(output),
|
||||
str(source),
|
||||
]
|
||||
completed = subprocess.run(
|
||||
command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(
|
||||
"mkfs.erofs failed: " + completed.stdout.decode("utf-8", "replace")
|
||||
)
|
||||
return command
|
||||
|
||||
|
||||
def mutate(valid: Path, output: Path, mutation: str, spec: dict) -> None:
|
||||
image = Image.load(valid)
|
||||
target = image.resolve(spec["fixture"]["target"])
|
||||
body = image.body_offset(target)
|
||||
_, shared_ids, inline_entries = image.body_shape(target)
|
||||
if mutation == "corrupt-shared-count":
|
||||
image.data[body + 4] = 255
|
||||
elif mutation == "corrupt-shared-id":
|
||||
if not shared_ids:
|
||||
raise SystemExit("valid fixture has no shared xattr ID")
|
||||
struct.pack_into("<I", image.data, body + 12, 0xFFFFFFFF)
|
||||
elif mutation == "corrupt-inline-name":
|
||||
if not inline_entries or image.data[inline_entries[0]] == 0:
|
||||
raise SystemExit("valid fixture has no inline xattr entry")
|
||||
image.data[inline_entries[0] + 4] = 0
|
||||
else:
|
||||
raise SystemExit(f"unknown B19a mutation: {mutation}")
|
||||
image.update_checksum()
|
||||
image.write(output)
|
||||
|
||||
|
||||
def generate(output: Path, work: Path, spec: dict) -> dict:
|
||||
if output.exists() or work.exists():
|
||||
raise SystemExit("B19a output and work paths must be absent")
|
||||
output.mkdir(parents=True)
|
||||
source = work / "source"
|
||||
create_source(source, spec)
|
||||
valid = output / "valid.erofs"
|
||||
command = build_image(source, valid, spec)
|
||||
image = Image.load(valid)
|
||||
root = image.inode(image.root_nid)
|
||||
root_entries = image.directory_entries(root)
|
||||
target = image.resolve(spec["fixture"]["target"])
|
||||
_, shared_ids, inline_entries = image.body_shape(target)
|
||||
if root["size"] <= image.block_size:
|
||||
raise SystemExit("B19a root directory did not span multiple blocks")
|
||||
if not shared_ids or not inline_entries:
|
||||
raise SystemExit("B19a target lacks both shared and inline xattrs")
|
||||
target_positions = [
|
||||
index for index, (name, _) in enumerate(root_entries) if name == b"target.bin"
|
||||
]
|
||||
if target_positions != [len(root_entries) - 1]:
|
||||
raise SystemExit("B19a target is not uniquely sorted after all peers")
|
||||
for mutation in (
|
||||
"corrupt-shared-count",
|
||||
"corrupt-shared-id",
|
||||
"corrupt-inline-name",
|
||||
):
|
||||
mutate(valid, output / f"{mutation}.erofs", mutation, spec)
|
||||
fsck = shutil.which("fsck.erofs")
|
||||
if fsck is None:
|
||||
raise SystemExit("fsck.erofs is required")
|
||||
completed = subprocess.run(
|
||||
[fsck, str(valid)], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(
|
||||
"fsck.erofs rejected B19a valid fixture: "
|
||||
+ completed.stdout.decode("utf-8", "replace")
|
||||
)
|
||||
hashes = {path.name: sha256(path) for path in sorted(output.glob("*.erofs"))}
|
||||
frozen = spec.get("fixture_sha256", {})
|
||||
if frozen and hashes != frozen:
|
||||
raise SystemExit(f"B19a fixture hashes differ: {hashes}")
|
||||
report = {
|
||||
"block_size": image.block_size,
|
||||
"command": [*command[:-2], "OUTPUT/valid.erofs", "WORK/source"],
|
||||
"fixture_sha256": hashes,
|
||||
"inline_entries": len(inline_entries),
|
||||
"root_directory_blocks": (root["size"] + image.block_size - 1) // image.block_size,
|
||||
"root_entry_count": len(root_entries),
|
||||
"shared_count": len(shared_ids),
|
||||
"status": "PASS",
|
||||
"target_entry_index": target_positions[0],
|
||||
}
|
||||
(output / "manifest.json").write_text(
|
||||
json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--work", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = load_spec(args.spec)
|
||||
try:
|
||||
report = generate(args.output, args.work, spec)
|
||||
finally:
|
||||
shutil.rmtree(args.work, ignore_errors=True)
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user