update

tests/pre15/fixtures/B19b-xattr-generate.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+"""Generate deterministic real EROFS Bloom fixtures for B19b."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import struct
+import subprocess
+
+
+SUPER = 1024
+MAGIC = 0xE0F5E1E2
+CRC32C_POLY = 0x82F63B78
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def write_json(path: Path, value: object) -> None:
+    path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
+
+
+def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
+    value = seed
+    for byte in data:
+        value ^= byte
+        for _ in range(8):
+            value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
+    return value & 0xFFFFFFFF
+
+
+def load_spec(path: Path) -> dict:
+    spec = json.loads(path.read_text(encoding="ascii"))
+    if spec.get("schema") != 1 or spec.get("batch") != "B19b" or spec.get("candidate") != "P15-021":
+        raise SystemExit("invalid B19b fixture spec identity")
+    return spec
+
+
+class Image:
+    def __init__(self, data: bytes | bytearray):
+        self.data = bytearray(data)
+        if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
+            raise ValueError("invalid EROFS image")
+        self.block_bits = self.data[SUPER + 12]
+        self.block_size = 1 << self.block_bits
+        self.blocks = self.u32(SUPER + 36)
+        self.limit = self.blocks << self.block_bits
+        self.meta_blkaddr = self.u32(SUPER + 40)
+        self.root_nid = self.u16(SUPER + 14)
+        if self.limit > len(self.data):
+            raise ValueError("EROFS image is truncated")
+
+    @classmethod
+    def load(cls, path: Path) -> "Image":
+        return cls(path.read_bytes())
+
+    def u16(self, offset: int) -> int:
+        return struct.unpack_from("<H", self.data, offset)[0]
+
+    def u32(self, offset: int) -> int:
+        return struct.unpack_from("<I", self.data, offset)[0]
+
+    def put_u32(self, offset: int, value: int) -> None:
+        struct.pack_into("<I", self.data, offset, value)
+
+    def inode(self, nid: int) -> dict[str, int]:
+        offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
+        inode_format = self.u16(offset)
+        inode_size = 64 if inode_format & 1 else 32
+        xattr_count = self.u16(offset + 2)
+        xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
+        size = struct.unpack_from("<Q" if inode_size == 64 else "<I", self.data, offset + 8)[0]
+        return {
+            "nid": nid,
+            "offset": offset,
+            "inode_size": inode_size,
+            "xattr_size": xattr_size,
+            "layout": (inode_format >> 1) & 7,
+            "size": size,
+            "start_block": self.u32(offset + 16),
+        }
+
+    def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
+        if inode["layout"] == 2:
+            physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical
+        elif inode["layout"] == 0:
+            physical = (inode["start_block"] << self.block_bits) + logical
+        else:
+            raise ValueError("fixture directory has an unsupported layout")
+        if physical > self.limit or length > self.limit - physical:
+            raise ValueError("fixture directory is out of bounds")
+        return bytes(self.data[physical:physical + length])
+
+    def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
+        result = []
+        logical = 0
+        while logical < inode["size"]:
+            length = min(self.block_size, inode["size"] - logical)
+            data = self.inode_data(inode, logical, length)
+            if len(data) < 12:
+                raise ValueError("fixture directory block is short")
+            first_name = struct.unpack_from("<H", data, 8)[0]
+            if first_name == 0 or first_name % 12 or first_name > len(data):
+                raise ValueError("fixture directory name offset is invalid")
+            count = first_name // 12
+            for index in range(count):
+                entry = index * 12
+                nid = struct.unpack_from("<Q", data, entry)[0]
+                start = struct.unpack_from("<H", data, entry + 8)[0]
+                end = struct.unpack_from("<H", data, entry + 20)[0] if index + 1 < count else len(data)
+                if start > end or end > len(data):
+                    raise ValueError("fixture directory name is out of bounds")
+                result.append((data[start:end].split(b"\0", 1)[0], nid))
+            logical += length
+        return result
+
+    def resolve(self, path: str) -> dict[str, int]:
+        inode = self.inode(self.root_nid)
+        for component in path.strip("/").encode().split(b"/"):
+            matches = [nid for name, nid in self.directory_entries(inode) if name == component]
+            if len(matches) != 1:
+                raise ValueError(f"fixture path is absent or ambiguous: {path}")
+            inode = self.inode(matches[0])
+        return inode
+
+    def update_checksum(self) -> None:
+        compat = self.u32(SUPER + 8)
+        if compat & 1:
+            self.put_u32(SUPER + 4, 0)
+            self.put_u32(SUPER + 4, crc32c(self.data[SUPER:self.block_size]))
+
+    def write(self, path: Path) -> None:
+        path.write_bytes(self.data)
+
+
+def create_source(path: Path, spec: dict) -> None:
+    path.mkdir(parents=True)
+    path.chmod(0o755)
+    files = [path / f"peer-{index:03d}.bin" for index in range(spec["fixture"]["peer_count"])]
+    files.append(path / "target.bin")
+    for entry in files:
+        entry.write_bytes(b"P15-021\n")
+        entry.chmod(0o644)
+        for index in range(spec["fixture"]["attribute_count"]):
+            name = f"user.attr{index:02d}".encode()
+            prefix = f"value-{index:02d}-".encode()
+            length = spec["fixture"]["attribute_value_bytes"]
+            value = (prefix * (length // len(prefix) + 1))[:length]
+            os.setxattr(entry, name, value)
+        os.utime(entry, (0, 0), follow_symlinks=False)
+    os.utime(path, (0, 0), follow_symlinks=False)
+
+
+def build_valid(output: Path, source: Path, spec: dict) -> list[str]:
+    mkfs = shutil.which("mkfs.erofs")
+    if mkfs is None:
+        raise SystemExit("mkfs.erofs is required")
+    command = [
+        mkfs,
+        "-d0",
+        "-T0",
+        "--all-time",
+        "--all-root",
+        "--workers=1",
+        "--sort=path",
+        f"-U{spec['fixture']['uuid']}",
+        "-x2",
+        "-Exattr-name-filter,force-inode-extended",
+        str(output),
+        str(source),
+    ]
+    completed = subprocess.run(command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    if completed.returncode != 0:
+        raise SystemExit(f"mkfs.erofs failed: {completed.stdout.decode('utf-8', 'replace')}")
+    return command
+
+
+def mutate(valid: Path, output: Path, mutation: str, spec: dict) -> None:
+    image = Image.load(valid)
+    target = image.resolve(spec["fixture"]["target"])
+    body = target["offset"] + target["inode_size"]
+    if mutation == "unknown-filter":
+        image.data[SUPER + 104] = 1
+    elif mutation == "feature-off":
+        image.put_u32(SUPER + 8, image.u32(SUPER + 8) & ~spec["format"]["feature_compat"])
+    elif mutation == "corrupt-shared-count":
+        image.data[body + 4] = 255
+    elif mutation == "corrupt-shared-id":
+        if image.data[body + 4] == 0:
+            raise SystemExit("valid fixture has no shared IDs")
+        image.put_u32(body + 12, 0xFFFFFFFF)
+    else:
+        raise SystemExit(f"unknown mutation: {mutation}")
+    image.update_checksum()
+    image.write(output)
+
+
+def generate(output: Path, work: Path, spec: dict) -> None:
+    if output.exists():
+        raise SystemExit(f"refusing existing output: {output}")
+    if work.exists():
+        raise SystemExit(f"refusing existing work directory: {work}")
+    output.mkdir(parents=True)
+    work.mkdir(parents=True)
+    source = work / "source"
+    create_source(source, spec)
+    valid = output / "valid.erofs"
+    command = build_valid(valid, source, spec)
+    for mutation in ("unknown-filter", "feature-off", "corrupt-shared-count", "corrupt-shared-id"):
+        mutate(valid, output / f"{mutation}.erofs", mutation, spec)
+    hashes = {path.name: sha256(path) for path in sorted(output.glob("*.erofs"))}
+    if hashes != spec["fixture_sha256"]:
+        raise SystemExit(f"B19b generated fixture hashes differ: {hashes}")
+    fsck = shutil.which("fsck.erofs")
+    if fsck is None:
+        raise SystemExit("fsck.erofs is required")
+    for name in ("valid.erofs", "unknown-filter.erofs", "feature-off.erofs"):
+        completed = subprocess.run([fsck, str(output / name)], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        if completed.returncode != 0:
+            raise SystemExit(f"fsck.erofs rejected legal fixture {name}")
+    sums = "".join(f"{digest}  {name}\n" for name, digest in sorted(hashes.items()))
+    (output / "SHA256SUMS").write_text(sums, encoding="ascii")
+    write_json(output / "manifest.json", {
+        "command": [*command[:-2], "OUTPUT/valid.erofs", "WORK/source"],
+        "fixture_sha256": hashes,
+        "schema": 1,
+        "status": "PASS",
+    })
+    print(json.dumps({"fixture_sha256": hashes, "status": "PASS"}, sort_keys=True))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("generate", choices=("generate",))
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--work", type=Path, required=True)
+    parser.add_argument("--spec", type=Path, required=True)
+    args = parser.parse_args()
+    generate(args.output, args.work, load_spec(args.spec))
+
+
+if __name__ == "__main__":
+    main()