254 lines
9.7 KiB
Python
Executable File
254 lines
9.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Generate deterministic real EROFS Bloom fixtures for B19b."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
|
|
|
|
SUPER = 1024
|
|
MAGIC = 0xE0F5E1E2
|
|
CRC32C_POLY = 0x82F63B78
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def write_json(path: Path, value: object) -> None:
|
|
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
|
|
|
|
|
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
|
value = seed
|
|
for byte in data:
|
|
value ^= byte
|
|
for _ in range(8):
|
|
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
|
return value & 0xFFFFFFFF
|
|
|
|
|
|
def load_spec(path: Path) -> dict:
|
|
spec = json.loads(path.read_text(encoding="ascii"))
|
|
if spec.get("schema") != 1 or spec.get("batch") != "B19b" or spec.get("candidate") != "P15-021":
|
|
raise SystemExit("invalid B19b fixture spec identity")
|
|
return spec
|
|
|
|
|
|
class Image:
|
|
def __init__(self, data: bytes | bytearray):
|
|
self.data = bytearray(data)
|
|
if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
|
|
raise ValueError("invalid EROFS image")
|
|
self.block_bits = self.data[SUPER + 12]
|
|
self.block_size = 1 << self.block_bits
|
|
self.blocks = self.u32(SUPER + 36)
|
|
self.limit = self.blocks << self.block_bits
|
|
self.meta_blkaddr = self.u32(SUPER + 40)
|
|
self.root_nid = self.u16(SUPER + 14)
|
|
if self.limit > len(self.data):
|
|
raise ValueError("EROFS image is truncated")
|
|
|
|
@classmethod
|
|
def load(cls, path: Path) -> "Image":
|
|
return cls(path.read_bytes())
|
|
|
|
def u16(self, offset: int) -> int:
|
|
return struct.unpack_from("<H", self.data, offset)[0]
|
|
|
|
def u32(self, offset: int) -> int:
|
|
return struct.unpack_from("<I", self.data, offset)[0]
|
|
|
|
def put_u32(self, offset: int, value: int) -> None:
|
|
struct.pack_into("<I", self.data, offset, value)
|
|
|
|
def inode(self, nid: int) -> dict[str, int]:
|
|
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
|
inode_format = self.u16(offset)
|
|
inode_size = 64 if inode_format & 1 else 32
|
|
xattr_count = self.u16(offset + 2)
|
|
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
|
size = struct.unpack_from("<Q" if inode_size == 64 else "<I", self.data, offset + 8)[0]
|
|
return {
|
|
"nid": nid,
|
|
"offset": offset,
|
|
"inode_size": inode_size,
|
|
"xattr_size": xattr_size,
|
|
"layout": (inode_format >> 1) & 7,
|
|
"size": size,
|
|
"start_block": self.u32(offset + 16),
|
|
}
|
|
|
|
def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
|
|
if inode["layout"] == 2:
|
|
physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical
|
|
elif inode["layout"] == 0:
|
|
physical = (inode["start_block"] << self.block_bits) + logical
|
|
else:
|
|
raise ValueError("fixture directory has an unsupported layout")
|
|
if physical > self.limit or length > self.limit - physical:
|
|
raise ValueError("fixture directory is out of bounds")
|
|
return bytes(self.data[physical:physical + length])
|
|
|
|
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
|
|
result = []
|
|
logical = 0
|
|
while logical < inode["size"]:
|
|
length = min(self.block_size, inode["size"] - logical)
|
|
data = self.inode_data(inode, logical, length)
|
|
if len(data) < 12:
|
|
raise ValueError("fixture directory block is short")
|
|
first_name = struct.unpack_from("<H", data, 8)[0]
|
|
if first_name == 0 or first_name % 12 or first_name > len(data):
|
|
raise ValueError("fixture directory name offset is invalid")
|
|
count = first_name // 12
|
|
for index in range(count):
|
|
entry = index * 12
|
|
nid = struct.unpack_from("<Q", data, entry)[0]
|
|
start = struct.unpack_from("<H", data, entry + 8)[0]
|
|
end = struct.unpack_from("<H", data, entry + 20)[0] if index + 1 < count else len(data)
|
|
if start > end or end > len(data):
|
|
raise ValueError("fixture directory name is out of bounds")
|
|
result.append((data[start:end].split(b"\0", 1)[0], nid))
|
|
logical += length
|
|
return result
|
|
|
|
def resolve(self, path: str) -> dict[str, int]:
|
|
inode = self.inode(self.root_nid)
|
|
for component in path.strip("/").encode().split(b"/"):
|
|
matches = [nid for name, nid in self.directory_entries(inode) if name == component]
|
|
if len(matches) != 1:
|
|
raise ValueError(f"fixture path is absent or ambiguous: {path}")
|
|
inode = self.inode(matches[0])
|
|
return inode
|
|
|
|
def update_checksum(self) -> None:
|
|
compat = self.u32(SUPER + 8)
|
|
if compat & 1:
|
|
self.put_u32(SUPER + 4, 0)
|
|
self.put_u32(SUPER + 4, crc32c(self.data[SUPER:self.block_size]))
|
|
|
|
def write(self, path: Path) -> None:
|
|
path.write_bytes(self.data)
|
|
|
|
|
|
def create_source(path: Path, spec: dict) -> None:
|
|
path.mkdir(parents=True)
|
|
path.chmod(0o755)
|
|
files = [path / f"peer-{index:03d}.bin" for index in range(spec["fixture"]["peer_count"])]
|
|
files.append(path / "target.bin")
|
|
for entry in files:
|
|
entry.write_bytes(b"P15-021\n")
|
|
entry.chmod(0o644)
|
|
for index in range(spec["fixture"]["attribute_count"]):
|
|
name = f"user.attr{index:02d}".encode()
|
|
prefix = f"value-{index:02d}-".encode()
|
|
length = spec["fixture"]["attribute_value_bytes"]
|
|
value = (prefix * (length // len(prefix) + 1))[:length]
|
|
os.setxattr(entry, name, value)
|
|
os.utime(entry, (0, 0), follow_symlinks=False)
|
|
os.utime(path, (0, 0), follow_symlinks=False)
|
|
|
|
|
|
def build_valid(output: Path, source: Path, spec: dict) -> list[str]:
|
|
mkfs = shutil.which("mkfs.erofs")
|
|
if mkfs is None:
|
|
raise SystemExit("mkfs.erofs is required")
|
|
command = [
|
|
mkfs,
|
|
"-d0",
|
|
"-T0",
|
|
"--all-time",
|
|
"--all-root",
|
|
"--workers=1",
|
|
"--sort=path",
|
|
f"-U{spec['fixture']['uuid']}",
|
|
"-x2",
|
|
"-Exattr-name-filter,force-inode-extended",
|
|
str(output),
|
|
str(source),
|
|
]
|
|
completed = subprocess.run(command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
if completed.returncode != 0:
|
|
raise SystemExit(f"mkfs.erofs failed: {completed.stdout.decode('utf-8', 'replace')}")
|
|
return command
|
|
|
|
|
|
def mutate(valid: Path, output: Path, mutation: str, spec: dict) -> None:
|
|
image = Image.load(valid)
|
|
target = image.resolve(spec["fixture"]["target"])
|
|
body = target["offset"] + target["inode_size"]
|
|
if mutation == "unknown-filter":
|
|
image.data[SUPER + 104] = 1
|
|
elif mutation == "feature-off":
|
|
image.put_u32(SUPER + 8, image.u32(SUPER + 8) & ~spec["format"]["feature_compat"])
|
|
elif mutation == "corrupt-shared-count":
|
|
image.data[body + 4] = 255
|
|
elif mutation == "corrupt-shared-id":
|
|
if image.data[body + 4] == 0:
|
|
raise SystemExit("valid fixture has no shared IDs")
|
|
image.put_u32(body + 12, 0xFFFFFFFF)
|
|
else:
|
|
raise SystemExit(f"unknown mutation: {mutation}")
|
|
image.update_checksum()
|
|
image.write(output)
|
|
|
|
|
|
def generate(output: Path, work: Path, spec: dict) -> None:
|
|
if output.exists():
|
|
raise SystemExit(f"refusing existing output: {output}")
|
|
if work.exists():
|
|
raise SystemExit(f"refusing existing work directory: {work}")
|
|
output.mkdir(parents=True)
|
|
work.mkdir(parents=True)
|
|
source = work / "source"
|
|
create_source(source, spec)
|
|
valid = output / "valid.erofs"
|
|
command = build_valid(valid, source, spec)
|
|
for mutation in ("unknown-filter", "feature-off", "corrupt-shared-count", "corrupt-shared-id"):
|
|
mutate(valid, output / f"{mutation}.erofs", mutation, spec)
|
|
hashes = {path.name: sha256(path) for path in sorted(output.glob("*.erofs"))}
|
|
if hashes != spec["fixture_sha256"]:
|
|
raise SystemExit(f"B19b generated fixture hashes differ: {hashes}")
|
|
fsck = shutil.which("fsck.erofs")
|
|
if fsck is None:
|
|
raise SystemExit("fsck.erofs is required")
|
|
for name in ("valid.erofs", "unknown-filter.erofs", "feature-off.erofs"):
|
|
completed = subprocess.run([fsck, str(output / name)], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
if completed.returncode != 0:
|
|
raise SystemExit(f"fsck.erofs rejected legal fixture {name}")
|
|
sums = "".join(f"{digest} {name}\n" for name, digest in sorted(hashes.items()))
|
|
(output / "SHA256SUMS").write_text(sums, encoding="ascii")
|
|
write_json(output / "manifest.json", {
|
|
"command": [*command[:-2], "OUTPUT/valid.erofs", "WORK/source"],
|
|
"fixture_sha256": hashes,
|
|
"schema": 1,
|
|
"status": "PASS",
|
|
})
|
|
print(json.dumps({"fixture_sha256": hashes, "status": "PASS"}, sort_keys=True))
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("generate", choices=("generate",))
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--work", type=Path, required=True)
|
|
parser.add_argument("--spec", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
generate(args.output, args.work, load_spec(args.spec))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|