965 lines
37 KiB
Bash
Executable File
965 lines
37 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
umask 022
|
|
|
|
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
|
input=$gate_dir/P15-019-input.json
|
|
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
|
|
utils_src=${EROFS_UTILS_SRC:-/work/build/erofs-utils-main}
|
|
base=
|
|
output=
|
|
|
|
while test "$#" -gt 0; do
|
|
case "$1" in
|
|
--base)
|
|
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
|
base=$2
|
|
shift 2
|
|
;;
|
|
--output)
|
|
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
|
output=$2
|
|
shift 2
|
|
;;
|
|
*)
|
|
printf 'unknown argument: %s\n' "$1" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
|
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
|
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
|
|
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
|
|
test -d "$utils_src/lib" || { printf 'missing erofs-utils source: %s\n' "$utils_src" >&2; exit 2; }
|
|
for tool in dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum timeout; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'missing required host tool: %s\n' "$tool" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
case "$output" in
|
|
/*) ;;
|
|
*) output=$PWD/$output ;;
|
|
esac
|
|
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
|
mkdir -p "$output/host"
|
|
|
|
python3 -B - "$root" "$input" "$base" "$output/host" "$freebsd_src" "$utils_src" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(sys.argv[1])
|
|
INPUT = Path(sys.argv[2])
|
|
REQUESTED_BASE = sys.argv[3]
|
|
OUTPUT = Path(sys.argv[4])
|
|
FREEBSD_SRC = Path(sys.argv[5])
|
|
UTILS_SRC = Path(sys.argv[6])
|
|
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
|
|
DUT = ROOT / "repo-pre-15"
|
|
|
|
SUPER = 1024
|
|
MAGIC = 0xE0F5E1E2
|
|
CRC32C_POLY = 0x82F63B78
|
|
FEATURE_COMPAT_SB_CHKSUM = 0x00000001
|
|
FEATURE_INCOMPAT_CHUNKED_FILE = 0x00000004
|
|
EROFS_FT_REG_FILE = 1
|
|
EROFS_FT_SYMLINK = 7
|
|
S_IFMT = 0o170000
|
|
S_IFREG = 0o100000
|
|
S_IFLNK = 0o120000
|
|
CHUNK_FORMAT_INDEXES = 0x0020
|
|
LAYOUT_NAMES = {
|
|
0: "plain",
|
|
1: "compressed-full",
|
|
2: "inline",
|
|
3: "compressed-compact",
|
|
4: "chunk",
|
|
}
|
|
|
|
|
|
class GateStop(Exception):
|
|
pass
|
|
|
|
|
|
class InfraBlocked(Exception):
|
|
pass
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_path(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
|
|
|
|
|
def git(path: Path, *args: str) -> str:
|
|
completed = subprocess.run(
|
|
["git", "-C", str(path), *args],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
if completed.returncode != 0:
|
|
raise InfraBlocked(f"git {' '.join(args)} failed: {completed.stdout.strip()}")
|
|
return completed.stdout.strip()
|
|
|
|
|
|
COMMANDS: list[dict[str, Any]] = []
|
|
|
|
|
|
def display_arg(value: str, work: Path) -> str:
|
|
replacements = (
|
|
(str(work), "OWNED_WORK"),
|
|
(str(OUTPUT), "GATE_OUTPUT"),
|
|
(str(ROOT), "ROOT"),
|
|
)
|
|
for old, new in replacements:
|
|
value = value.replace(old, new)
|
|
return value
|
|
|
|
|
|
def run(argv: list[str], work: Path, timeout: int = 30) -> subprocess.CompletedProcess[str]:
|
|
try:
|
|
completed = subprocess.run(
|
|
argv,
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
timeout=timeout,
|
|
)
|
|
except subprocess.TimeoutExpired as error:
|
|
raise InfraBlocked(f"command timed out: {' '.join(argv)}") from error
|
|
COMMANDS.append(
|
|
{
|
|
"argv": [display_arg(item, work) for item in argv],
|
|
"exit": completed.returncode,
|
|
"stdout_sha256": sha256_bytes(completed.stdout.encode("utf-8")),
|
|
}
|
|
)
|
|
return completed
|
|
|
|
|
|
def run_required(argv: list[str], work: Path, timeout: int = 30) -> str:
|
|
completed = run(argv, work, timeout)
|
|
if completed.returncode != 0:
|
|
raise InfraBlocked(
|
|
f"command failed ({completed.returncode}): {' '.join(argv)}: "
|
|
f"{completed.stdout.strip()}"
|
|
)
|
|
return completed.stdout
|
|
|
|
|
|
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
|
value = seed
|
|
for byte in data:
|
|
value ^= byte
|
|
for _ in range(8):
|
|
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
|
return value & 0xFFFFFFFF
|
|
|
|
|
|
class Inode:
|
|
def __init__(
|
|
self,
|
|
nid: int,
|
|
offset: int,
|
|
inode_format: int,
|
|
inode_size: int,
|
|
xattr_size: int,
|
|
layout: int,
|
|
mode: int,
|
|
size: int,
|
|
start_block: int,
|
|
):
|
|
self.nid = nid
|
|
self.offset = offset
|
|
self.inode_format = inode_format
|
|
self.inode_size = inode_size
|
|
self.xattr_size = xattr_size
|
|
self.layout = layout
|
|
self.mode = mode
|
|
self.size = size
|
|
self.start_block = start_block
|
|
|
|
|
|
class DirectoryEntry:
|
|
def __init__(self, nid: int, offset: int, file_type: int, name: bytes):
|
|
self.nid = nid
|
|
self.offset = offset
|
|
self.file_type = file_type
|
|
self.name = name
|
|
|
|
|
|
class Image:
|
|
def __init__(self, data: bytes | bytearray):
|
|
self.data = bytearray(data)
|
|
if len(self.data) < SUPER + 128 or self.u32(SUPER) != MAGIC:
|
|
raise GateStop("fixture is not an EROFS image")
|
|
self.block_bits = self.data[SUPER + 12]
|
|
if self.block_bits < 9 or self.block_bits > 16:
|
|
raise GateStop("fixture has invalid block bits")
|
|
self.block_size = 1 << self.block_bits
|
|
self.meta_blkaddr = self.u32(SUPER + 40)
|
|
self.feature_compat = self.u32(SUPER + 8)
|
|
self.feature_incompat = self.u32(SUPER + 80)
|
|
self.blocks = self.u32(SUPER + 36)
|
|
self.root_nid = self.u16(SUPER + 14)
|
|
self.packed_nid = self.u64(SUPER + 96)
|
|
if self.blocks == 0 or self.blocks << self.block_bits > len(self.data):
|
|
raise GateStop("fixture primary image bounds are invalid")
|
|
self.verify_checksum()
|
|
|
|
@classmethod
|
|
def load(cls, path: Path) -> "Image":
|
|
return cls(path.read_bytes())
|
|
|
|
def u16(self, offset: int) -> int:
|
|
if offset < 0 or offset + 2 > len(self.data):
|
|
raise GateStop("u16 read outside image")
|
|
return struct.unpack_from("<H", self.data, offset)[0]
|
|
|
|
def u32(self, offset: int) -> int:
|
|
if offset < 0 or offset + 4 > len(self.data):
|
|
raise GateStop("u32 read outside image")
|
|
return struct.unpack_from("<I", self.data, offset)[0]
|
|
|
|
def u64(self, offset: int) -> int:
|
|
if offset < 0 or offset + 8 > len(self.data):
|
|
raise GateStop("u64 read outside image")
|
|
return struct.unpack_from("<Q", self.data, offset)[0]
|
|
|
|
def put_u16(self, offset: int, value: int) -> None:
|
|
struct.pack_into("<H", self.data, offset, value)
|
|
|
|
def put_u32(self, offset: int, value: int) -> None:
|
|
struct.pack_into("<I", self.data, offset, value)
|
|
|
|
def put_u64(self, offset: int, value: int) -> None:
|
|
struct.pack_into("<Q", self.data, offset, value)
|
|
|
|
@property
|
|
def checksum_end(self) -> int:
|
|
span = self.block_size - SUPER if self.block_size > SUPER else self.block_size
|
|
end = SUPER + span
|
|
if end > len(self.data):
|
|
raise GateStop("checksum span exceeds image")
|
|
return end
|
|
|
|
def calculated_checksum(self) -> int:
|
|
block = bytearray(self.data[SUPER : self.checksum_end])
|
|
block[4:8] = bytes(4)
|
|
return crc32c(block)
|
|
|
|
def verify_checksum(self) -> None:
|
|
if self.feature_compat & FEATURE_COMPAT_SB_CHKSUM:
|
|
if self.u32(SUPER + 4) != self.calculated_checksum():
|
|
raise GateStop("fixture checksum is invalid")
|
|
|
|
def update_checksum(self) -> None:
|
|
if not self.feature_compat & FEATURE_COMPAT_SB_CHKSUM:
|
|
raise GateStop("fixture does not advertise a checksum")
|
|
self.put_u32(SUPER + 4, 0)
|
|
self.put_u32(SUPER + 4, self.calculated_checksum())
|
|
self.verify_checksum()
|
|
|
|
def inode(self, nid: int) -> Inode:
|
|
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
|
if offset > len(self.data) - 32:
|
|
raise GateStop(f"nid {nid} lies outside the primary image")
|
|
inode_format = self.u16(offset)
|
|
inode_size = 64 if inode_format & 1 else 32
|
|
if offset > len(self.data) - inode_size:
|
|
raise GateStop(f"nid {nid} is truncated")
|
|
xattr_count = self.u16(offset + 2)
|
|
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
|
size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8)
|
|
return Inode(
|
|
nid=nid,
|
|
offset=offset,
|
|
inode_format=inode_format,
|
|
inode_size=inode_size,
|
|
xattr_size=xattr_size,
|
|
layout=(inode_format >> 1) & 7,
|
|
mode=self.u16(offset + 4),
|
|
size=size,
|
|
start_block=self.u32(offset + 16),
|
|
)
|
|
|
|
def uncompressed_data(self, inode: Inode, blob: bytes | None = None) -> bytes:
|
|
if inode.layout == 2:
|
|
offset = inode.offset + inode.inode_size + inode.xattr_size
|
|
if (offset & (self.block_size - 1)) + inode.size > self.block_size:
|
|
raise GateStop("inline fixture crosses its metadata block")
|
|
source = self.data
|
|
elif inode.layout == 0:
|
|
offset = inode.start_block << self.block_bits
|
|
source = self.data
|
|
elif inode.layout == 4:
|
|
if blob is None:
|
|
raise GateStop("chunk fixture has no external blob")
|
|
chunk_format = self.u16(inode.offset + 16)
|
|
entry_size = 8 if chunk_format & CHUNK_FORMAT_INDEXES else 4
|
|
index_base = (inode.offset + inode.inode_size + inode.xattr_size + entry_size - 1) & ~(entry_size - 1)
|
|
output = bytearray()
|
|
logical = 0
|
|
chunk_size = 1 << (self.block_bits + (chunk_format & 0x1F))
|
|
while logical < inode.size:
|
|
index = logical // chunk_size
|
|
position = index_base + index * entry_size
|
|
if entry_size == 8:
|
|
high, device_id, low = struct.unpack_from("<HHI", self.data, position)
|
|
block = low | high << 32
|
|
if device_id != 1:
|
|
raise GateStop(f"chunk fixture device id is {device_id}, not 1")
|
|
source = blob
|
|
else:
|
|
block = self.u32(position)
|
|
source = self.data
|
|
count = min(chunk_size, inode.size - logical)
|
|
offset = block << self.block_bits
|
|
if offset > len(source) or count > len(source) - offset:
|
|
raise GateStop("chunk payload exceeds its device")
|
|
output.extend(source[offset : offset + count])
|
|
logical += count
|
|
return bytes(output)
|
|
else:
|
|
raise GateStop(f"layout {inode.layout} is not uncompressed")
|
|
if offset > len(source) or inode.size > len(source) - offset:
|
|
raise GateStop("uncompressed payload exceeds its image")
|
|
return bytes(source[offset : offset + inode.size])
|
|
|
|
def directory_entries(self, inode: Inode) -> list[DirectoryEntry]:
|
|
data = self.uncompressed_data(inode)
|
|
if len(data) < 12:
|
|
raise GateStop("root directory is too short")
|
|
first_name = struct.unpack_from("<H", data, 8)[0]
|
|
if first_name < 12 or first_name % 12 != 0 or first_name >= len(data):
|
|
raise GateStop("root directory has an invalid first name offset")
|
|
count = first_name // 12
|
|
entries = []
|
|
for index in range(count):
|
|
slot = index * 12
|
|
name_start = struct.unpack_from("<H", data, slot + 8)[0]
|
|
name_end = (
|
|
struct.unpack_from("<H", data, slot + 20)[0]
|
|
if index + 1 < count
|
|
else len(data)
|
|
)
|
|
if name_start >= name_end or name_end > len(data):
|
|
raise GateStop("root directory name bounds are invalid")
|
|
name = data[name_start:name_end].split(b"\0", 1)[0]
|
|
data_offset = (
|
|
inode.offset + inode.inode_size + inode.xattr_size
|
|
if inode.layout == 2
|
|
else inode.start_block << self.block_bits
|
|
)
|
|
entries.append(
|
|
DirectoryEntry(
|
|
nid=struct.unpack_from("<Q", data, slot)[0],
|
|
offset=data_offset + slot,
|
|
file_type=data[slot + 10],
|
|
name=name,
|
|
)
|
|
)
|
|
return entries
|
|
|
|
def resolve(self, name: bytes = b"link") -> tuple[Inode, DirectoryEntry, Inode]:
|
|
root = self.inode(self.root_nid)
|
|
entry = next((item for item in self.directory_entries(root) if item.name == name), None)
|
|
if entry is None:
|
|
raise GateStop("fixture root has no /link entry")
|
|
return root, entry, self.inode(entry.nid)
|
|
|
|
|
|
def lz4_decode_exact(source: bytes, output_size: int) -> bytes:
|
|
ip = 0
|
|
output = bytearray()
|
|
while ip < len(source):
|
|
token = source[ip]
|
|
ip += 1
|
|
literal_length = token >> 4
|
|
if literal_length == 15:
|
|
while True:
|
|
if ip >= len(source):
|
|
raise ValueError("truncated literal length")
|
|
value = source[ip]
|
|
ip += 1
|
|
literal_length += value
|
|
if value != 255:
|
|
break
|
|
if ip + literal_length > len(source):
|
|
raise ValueError("truncated literals")
|
|
output.extend(source[ip : ip + literal_length])
|
|
ip += literal_length
|
|
if ip == len(source):
|
|
break
|
|
if ip + 2 > len(source):
|
|
raise ValueError("truncated match offset")
|
|
offset = source[ip] | source[ip + 1] << 8
|
|
ip += 2
|
|
if offset == 0 or offset > len(output):
|
|
raise ValueError("invalid match offset")
|
|
match_length = token & 15
|
|
if match_length == 15:
|
|
while True:
|
|
if ip >= len(source):
|
|
raise ValueError("truncated match length")
|
|
value = source[ip]
|
|
ip += 1
|
|
match_length += value
|
|
if value != 255:
|
|
break
|
|
for _ in range(match_length + 4):
|
|
output.append(output[-offset])
|
|
if len(output) > output_size:
|
|
raise ValueError("decoded output exceeds inode size")
|
|
if ip != len(source) or len(output) != output_size:
|
|
raise ValueError("raw LZ4 size mismatch")
|
|
return bytes(output)
|
|
|
|
|
|
def read_compressed_inline(image: Image, inode: Inode) -> tuple[bytes, dict[str, int]]:
|
|
header = (inode.offset + inode.inode_size + inode.xattr_size + 7) & ~7
|
|
if header > len(image.data) - 8:
|
|
raise GateStop("compressed map header exceeds image")
|
|
raw0, advise, algorithm, clusterbits = struct.unpack_from("<IHBB", image.data, header)
|
|
idata_size = raw0 >> 16
|
|
if inode.layout != 3 or not advise & 0x0008 or idata_size == 0:
|
|
raise GateStop("compressed fixture is not a compact inline-pcluster inode")
|
|
block_end = min((inode.offset | (image.block_size - 1)) + 1, len(image.data))
|
|
matches = []
|
|
for position in range(header + 8, block_end - idata_size + 1):
|
|
encoded = bytes(image.data[position : position + idata_size])
|
|
try:
|
|
decoded = lz4_decode_exact(encoded, inode.size)
|
|
except ValueError:
|
|
continue
|
|
matches.append((position, decoded))
|
|
if len(matches) != 1:
|
|
raise GateStop(f"compressed fixture has {len(matches)} independent decode candidates")
|
|
return matches[0][1], {
|
|
"map_header": header,
|
|
"advise": advise,
|
|
"algorithm": algorithm & 0x0F,
|
|
"clusterbits": clusterbits,
|
|
"idata_size": idata_size,
|
|
"encoded_offset": matches[0][0],
|
|
}
|
|
|
|
|
|
def read_fragment(image: Image, inode: Inode) -> tuple[bytes, dict[str, int]]:
|
|
header = (inode.offset + inode.inode_size + inode.xattr_size + 7) & ~7
|
|
raw = image.u64(header)
|
|
if inode.layout != 1 or raw & (1 << 63) == 0:
|
|
raise GateStop("fragment fixture lacks the whole-fragment header")
|
|
if image.packed_nid == 0 or image.packed_nid == inode.nid:
|
|
raise GateStop("fragment fixture has an invalid packed nid")
|
|
fragment_offset = raw ^ (1 << 63)
|
|
packed = image.inode(image.packed_nid)
|
|
packed_data = image.uncompressed_data(packed)
|
|
if fragment_offset > len(packed_data) or inode.size > len(packed_data) - fragment_offset:
|
|
raise GateStop("fragment target exceeds packed inode")
|
|
return bytes(packed_data[fragment_offset : fragment_offset + inode.size]), {
|
|
"map_header": header,
|
|
"fragment_offset": fragment_offset,
|
|
"packed_nid": image.packed_nid,
|
|
"packed_layout": packed.layout,
|
|
"packed_size": packed.size,
|
|
}
|
|
|
|
|
|
def target_bytes(length: int, nul_position: str | None = None) -> bytes:
|
|
pattern = b"p15-019-safe-target/"
|
|
target = bytearray((pattern * ((length + len(pattern) - 1) // len(pattern)))[:length])
|
|
if nul_position == "first":
|
|
target[0] = 0
|
|
elif nul_position == "middle":
|
|
target[length // 2] = 0
|
|
elif nul_position == "last":
|
|
target[-1] = 0
|
|
return bytes(target)
|
|
|
|
|
|
def protected_state() -> dict[str, Any]:
|
|
protected = SPEC["protected"]
|
|
base = Path(protected["base_image"])
|
|
base_stat = base.stat() if base.exists() else None
|
|
pid_path = Path("/proc") / str(protected["pid"])
|
|
command = None
|
|
if (pid_path / "cmdline").exists():
|
|
command = (pid_path / "cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace").strip()
|
|
port_hex = f"{protected['port']:04X}"
|
|
listeners = 0
|
|
for table in (Path("/proc/net/tcp"), Path("/proc/net/tcp6")):
|
|
if not table.exists():
|
|
continue
|
|
for line in table.read_text(encoding="ascii").splitlines()[1:]:
|
|
fields = line.split()
|
|
if len(fields) >= 4 and fields[1].endswith(f":{port_hex}") and fields[3] == "0A":
|
|
listeners += 1
|
|
return {
|
|
"base": None if base_stat is None else {
|
|
"device": base_stat.st_dev,
|
|
"inode": base_stat.st_ino,
|
|
"mtime_ns": base_stat.st_mtime_ns,
|
|
"size": base_stat.st_size,
|
|
},
|
|
"pid": protected["pid"],
|
|
"pid_command_sha256": None if command is None else sha256_bytes(command.encode("utf-8")),
|
|
"port": protected["port"],
|
|
"port_listener_count": listeners,
|
|
}
|
|
|
|
|
|
def verify_identities() -> dict[str, Any]:
|
|
required = SPEC["required_base"]
|
|
resolved = git(ROOT, "rev-parse", REQUESTED_BASE)
|
|
if resolved != required:
|
|
raise GateStop(f"requested base {resolved} is not required base {required}")
|
|
source_dirty = git(ROOT, "diff", "--name-only", required, "--", "repo-pre-15/src")
|
|
if source_dirty:
|
|
raise GateStop(f"production source differs from frozen base: {source_dirty}")
|
|
source_hashes = {}
|
|
for relative, expected in SPEC["source_sha256"].items():
|
|
content = subprocess.run(
|
|
["git", "-C", str(ROOT), "show", f"{required}:{relative}"],
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
if content.returncode != 0:
|
|
raise InfraBlocked(f"cannot read frozen source {relative}")
|
|
actual = sha256_bytes(content.stdout)
|
|
if actual != expected:
|
|
raise GateStop(f"frozen source hash differs for {relative}")
|
|
source_hashes[relative] = actual
|
|
if git(FREEBSD_SRC, "rev-parse", "HEAD") != SPEC["freebsd"]["head"]:
|
|
raise GateStop("FreeBSD source HEAD differs")
|
|
for relative, expected in SPEC["freebsd"]["sha256"].items():
|
|
if sha256_path(FREEBSD_SRC / relative) != expected:
|
|
raise GateStop(f"FreeBSD source hash differs for {relative}")
|
|
if git(UTILS_SRC, "rev-parse", "HEAD") != SPEC["utils"]["head"]:
|
|
raise GateStop("erofs-utils source HEAD differs")
|
|
for relative, expected in SPEC["utils"]["sha256"].items():
|
|
actual = sha256_path(UTILS_SRC / relative)
|
|
if actual != expected:
|
|
raise GateStop(f"erofs-utils source hash differs for {relative}")
|
|
for name, tool in SPEC["tools"].items():
|
|
path = Path(tool["path"])
|
|
if sha256_path(path) != tool["sha256"]:
|
|
raise GateStop(f"tool hash differs for {name}")
|
|
version = subprocess.run(
|
|
[SPEC["tools"]["mkfs.erofs"]["path"], "-V"],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
).stdout.splitlines()[0]
|
|
if version != SPEC["tools"]["mkfs.erofs"]["version"]:
|
|
raise GateStop(f"mkfs version differs: {version}")
|
|
return {"base": resolved, "source_sha256": source_hashes, "mkfs_version": version}
|
|
|
|
|
|
def verify_semantics() -> dict[str, Any]:
|
|
linux = (ROOT / "src-linux/inode.c").read_text(encoding="utf-8")
|
|
freebsd_inode = (DUT / "src/inode.c").read_text(encoding="utf-8")
|
|
freebsd_data = (DUT / "src/data.c").read_text(encoding="utf-8")
|
|
freebsd_vnops = (DUT / "src/erofs_vnops.c").read_text(encoding="utf-8")
|
|
lookup = (FREEBSD_SRC / "sys/kern/vfs_lookup.c").read_text(encoding="utf-8")
|
|
errno_h = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8")
|
|
syslimits = (FREEBSD_SRC / "sys/sys/syslimits.h").read_text(encoding="utf-8")
|
|
linux_markers = (
|
|
"vi->datalayout == EROFS_INODE_FLAT_INLINE",
|
|
"kmemdup_nul(bptr + ofs, inode->i_size, GFP_KERNEL)",
|
|
"!inode->i_size || strlen(link) != inode->i_size",
|
|
"return -EFSCORRUPTED",
|
|
".get_link = page_get_link",
|
|
".get_link = simple_get_link",
|
|
)
|
|
freebsd_markers = (
|
|
"return (erofs_read_uio(MTOE(vp->v_mount), VTOE(vp), uio));",
|
|
"return (erofs_readlink_target(ap->a_vp, ap->a_uio));",
|
|
".vop_readlink = erofs_readlink",
|
|
)
|
|
if not all(marker in linux for marker in linux_markers):
|
|
raise GateStop("Linux fast/page symlink path anchors changed")
|
|
if not all(marker in freebsd_data + freebsd_vnops for marker in freebsd_markers):
|
|
raise GateStop("FreeBSD vnode/readlink path anchors changed")
|
|
if "erofs_validate_symlink" in freebsd_inode + freebsd_data + freebsd_vnops:
|
|
raise GateStop("P15-019 production implementation already exists at the gate base")
|
|
if "aiov.iov_len = MAXPATHLEN;" not in lookup or "linklen == 0" not in lookup:
|
|
raise GateStop("FreeBSD namei readlink boundary anchors changed")
|
|
expected_errno = SPEC["errno"]
|
|
for name, value in expected_errno.items():
|
|
if re.search(rf"#define\s+{name}\s+{value}\b", errno_h) is None:
|
|
raise GateStop(f"FreeBSD errno {name} is not {value}")
|
|
maxpath = SPEC["freebsd"]["maxpathlen"]
|
|
if re.search(rf"#define\s+PATH_MAX\s+{maxpath}\b", syslimits) is None:
|
|
raise GateStop("FreeBSD PATH_MAX differs")
|
|
if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", freebsd_inode + freebsd_data + freebsd_vnops):
|
|
raise GateStop("FreeBSD EROFS source contains Linux negative errno returns")
|
|
return {
|
|
"linux": {
|
|
"fast_inline_validation": True,
|
|
"noninline_get_link": "page_get_link",
|
|
"errno_convention": "negative Linux errno",
|
|
},
|
|
"freebsd": {
|
|
"entry": "VOP_READLINK",
|
|
"path": "vnode -> erofs_readlink_target -> map/read -> GEOM or compressed backing",
|
|
"maxpathlen": maxpath,
|
|
"empty_namei_baseline": "ENOENT after zero-byte VOP_READLINK success",
|
|
"errno_convention": "positive FreeBSD errno",
|
|
},
|
|
"candidate": {
|
|
"empty": "EINTEGRITY",
|
|
"embedded_nul": "EINTEGRITY before uiomove",
|
|
"oversize": "ENAMETOOLONG before target I/O",
|
|
"normal": "exact byte target; no trailing NUL required",
|
|
},
|
|
}
|
|
|
|
|
|
def make_source(source: Path, payload: bytes, native_symlink: bool) -> None:
|
|
source.mkdir()
|
|
source.chmod(0o755)
|
|
link = source / "link"
|
|
if native_symlink:
|
|
os.symlink(payload.decode("ascii"), link)
|
|
else:
|
|
link.write_bytes(payload)
|
|
link.chmod(0o644)
|
|
os.utime(link, (0, 0), follow_symlinks=False)
|
|
os.utime(source, (0, 0), follow_symlinks=False)
|
|
|
|
|
|
def mkfs_case(
|
|
layout: str,
|
|
case_id: str,
|
|
payload: bytes,
|
|
native_symlink: bool,
|
|
work: Path,
|
|
fixtures: Path,
|
|
) -> tuple[Path, Path | None, list[int]]:
|
|
source = work / f"source-{layout}-{case_id}"
|
|
make_source(source, payload, native_symlink)
|
|
image_path = fixtures / f"{layout}-{case_id}.erofs"
|
|
blob_path = fixtures / f"{layout}-{case_id}.blob" if layout == "chunk" else None
|
|
options: list[str] = []
|
|
if layout == "plain":
|
|
options = ["-E^inline_data"]
|
|
elif layout == "chunk":
|
|
assert blob_path is not None
|
|
blob_path.write_bytes(b"")
|
|
options = ["--chunksize=4096", f"--blobdev={blob_path}"]
|
|
elif layout == "compressed":
|
|
options = ["-zlz4", "-C4096", "-Eztailpacking"]
|
|
elif layout == "fragment":
|
|
options = ["-zlz4", "-C4096", "-Eall-fragments"]
|
|
command = [
|
|
SPEC["tools"]["mkfs.erofs"]["path"],
|
|
"-T0",
|
|
"--all-time",
|
|
"--all-root",
|
|
"--workers=1",
|
|
"--sort=path",
|
|
"-x-1",
|
|
f"-U{SPEC['uuid']}",
|
|
*options,
|
|
str(image_path),
|
|
str(source),
|
|
]
|
|
run_required(command, work)
|
|
before = image_path.read_bytes()
|
|
image = Image(before)
|
|
_, entry, inode = image.resolve()
|
|
changed = []
|
|
if stat.S_IFMT(inode.mode) == S_IFREG:
|
|
image.put_u16(inode.offset + 4, S_IFLNK | 0o777)
|
|
image.data[entry.offset + 10] = EROFS_FT_SYMLINK
|
|
elif stat.S_IFMT(inode.mode) != S_IFLNK or entry.file_type != EROFS_FT_SYMLINK:
|
|
raise GateStop(f"{layout}/{case_id}: source inode type is unexpected")
|
|
if case_id == "empty":
|
|
if inode.inode_size == 64:
|
|
image.put_u64(inode.offset + 8, 0)
|
|
else:
|
|
image.put_u32(inode.offset + 8, 0)
|
|
image.update_checksum()
|
|
after = bytes(image.data)
|
|
for offset, (old, new) in enumerate(zip(before, after)):
|
|
if old != new:
|
|
changed.append(offset)
|
|
allowed = set(range(SUPER + 4, SUPER + 8))
|
|
allowed.update(range(inode.offset + 4, inode.offset + 6))
|
|
allowed.add(entry.offset + 10)
|
|
if case_id == "empty":
|
|
allowed.update(range(inode.offset + 8, inode.offset + (16 if inode.inode_size == 64 else 12)))
|
|
unexpected = sorted(set(changed) - allowed)
|
|
if unexpected:
|
|
raise GateStop(f"{layout}/{case_id}: transform changed unexpected offsets {unexpected}")
|
|
image_path.write_bytes(after)
|
|
return image_path, blob_path, changed
|
|
|
|
|
|
def expected_case(layout: str, case_id: str) -> tuple[int, bytes, str | None, str]:
|
|
short = SPEC["layouts"][layout]["short_length"]
|
|
if case_id == "normal-short":
|
|
return short, target_bytes(short), None, "PASS"
|
|
if case_id == "normal-max":
|
|
limit = SPEC["freebsd"]["maxpathlen"]
|
|
return limit, target_bytes(limit), None, "PASS"
|
|
if case_id == "empty":
|
|
return 0, target_bytes(short), None, "EINTEGRITY"
|
|
if case_id.startswith("nul-"):
|
|
position = case_id.removeprefix("nul-")
|
|
limit = SPEC["freebsd"]["maxpathlen"]
|
|
return limit, target_bytes(limit, position), position, "EINTEGRITY"
|
|
if case_id == "too-long":
|
|
length = SPEC["freebsd"]["maxpathlen"] + 1
|
|
return length, target_bytes(length), None, "ENAMETOOLONG"
|
|
raise AssertionError(case_id)
|
|
|
|
|
|
def independent_target(image: Image, inode: Inode, blob: bytes | None) -> tuple[bytes, dict[str, Any]]:
|
|
if inode.layout in (0, 2, 4):
|
|
return image.uncompressed_data(inode, blob), {}
|
|
if inode.layout == 3:
|
|
return read_compressed_inline(image, inode)
|
|
if inode.layout == 1:
|
|
return read_fragment(image, inode)
|
|
raise GateStop(f"unsupported target layout {inode.layout}")
|
|
|
|
|
|
def oracle(image: Image, inode: Inode, blob: bytes | None) -> tuple[str, bytes | None, dict[str, Any]]:
|
|
if inode.size == 0:
|
|
return "EINTEGRITY", None, {"point": "symlink.empty"}
|
|
if inode.size > SPEC["freebsd"]["maxpathlen"]:
|
|
return "ENAMETOOLONG", None, {"point": "symlink.maxpathlen"}
|
|
target, details = independent_target(image, inode, blob)
|
|
nul = target.find(b"\0")
|
|
if nul >= 0:
|
|
return "EINTEGRITY", target, {"point": "symlink.embedded-nul", "nul_offset": nul, **details}
|
|
return "PASS", target, {"point": "symlink.valid", **details}
|
|
|
|
|
|
def fixture_set_hash(fixtures: Path) -> tuple[str, dict[str, str]]:
|
|
hashes = {}
|
|
for path in sorted(item for item in fixtures.iterdir() if item.is_file()):
|
|
if path.name == "SHA256SUMS":
|
|
continue
|
|
hashes[path.name] = sha256_path(path)
|
|
lines = "".join(f"{digest} {name}\n" for name, digest in hashes.items())
|
|
(fixtures / "SHA256SUMS").write_text(lines, encoding="ascii")
|
|
return sha256_bytes(lines.encode("ascii")), hashes
|
|
|
|
|
|
def execute() -> dict[str, Any]:
|
|
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-019" or SPEC.get("gate") != "G03":
|
|
raise InfraBlocked("input identity is invalid")
|
|
identity = verify_identities()
|
|
semantics = verify_semantics()
|
|
write_json(OUTPUT / "semantics.json", semantics)
|
|
fixtures = OUTPUT / "fixtures"
|
|
fixtures.mkdir()
|
|
cases = (
|
|
"normal-short",
|
|
"normal-max",
|
|
"empty",
|
|
"nul-first",
|
|
"nul-middle",
|
|
"nul-last",
|
|
"too-long",
|
|
)
|
|
records = []
|
|
with tempfile.TemporaryDirectory(prefix=".p15-019-work-", dir=OUTPUT) as raw_work:
|
|
work = Path(raw_work)
|
|
for layout, layout_spec in sorted(SPEC["layouts"].items()):
|
|
for case_id in cases:
|
|
expected_size, payload, nul_position, expected = expected_case(layout, case_id)
|
|
native = layout == "inline" and nul_position is None
|
|
image_path, blob_path, changed = mkfs_case(
|
|
layout, case_id, payload, native, work, fixtures
|
|
)
|
|
image = Image.load(image_path)
|
|
_, entry, inode = image.resolve()
|
|
if inode.layout != layout_spec["expected_layout"]:
|
|
raise GateStop(
|
|
f"{layout}/{case_id}: layout {inode.layout} != {layout_spec['expected_layout']}"
|
|
)
|
|
if stat.S_IFMT(inode.mode) != S_IFLNK or entry.file_type != EROFS_FT_SYMLINK:
|
|
raise GateStop(f"{layout}/{case_id}: transformed inode is not a symlink")
|
|
if inode.size != expected_size:
|
|
raise GateStop(f"{layout}/{case_id}: inode size {inode.size} != {expected_size}")
|
|
blob = blob_path.read_bytes() if blob_path is not None else None
|
|
observed, target, details = oracle(image, inode, blob)
|
|
if observed != expected:
|
|
raise GateStop(f"{layout}/{case_id}: oracle {observed} != {expected}")
|
|
if expected != "EINTEGRITY" or case_id == "empty":
|
|
if case_id != "empty" and target is not None and target != payload:
|
|
raise GateStop(f"{layout}/{case_id}: target bytes differ from source")
|
|
if case_id.startswith("nul-"):
|
|
expected_nul = {"first": 0, "middle": len(payload) // 2, "last": len(payload) - 1}[nul_position]
|
|
if details.get("nul_offset") != expected_nul:
|
|
raise GateStop(f"{layout}/{case_id}: NUL offset differs")
|
|
dump = run(
|
|
[SPEC["tools"]["dump.erofs"]["path"], "--path=/link", str(image_path)],
|
|
work,
|
|
)
|
|
dump_layout = re.search(r"Layout:\s+(\d+)", dump.stdout)
|
|
dump_mode = "symlink file" in dump.stdout
|
|
if dump.returncode != 0 or dump_layout is None or int(dump_layout.group(1)) != inode.layout or not dump_mode:
|
|
raise GateStop(f"{layout}/{case_id}: dump.erofs cross-check failed")
|
|
fsck_argv = [SPEC["tools"]["fsck.erofs"]["path"], "-d0"]
|
|
if blob_path is not None:
|
|
fsck_argv.append(f"--device={blob_path}")
|
|
fsck_argv.append(str(image_path))
|
|
fsck = run(fsck_argv, work)
|
|
extracted = False
|
|
if expected == "PASS":
|
|
extract = work / f"extract-{layout}-{case_id}"
|
|
extract.mkdir()
|
|
extract_argv = [
|
|
SPEC["tools"]["fsck.erofs"]["path"],
|
|
f"--extract={extract}",
|
|
"--no-preserve",
|
|
]
|
|
if blob_path is not None:
|
|
extract_argv.append(f"--device={blob_path}")
|
|
extract_argv.append(str(image_path))
|
|
run_required(extract_argv, work)
|
|
extracted_link = extract / "link"
|
|
if not extracted_link.is_symlink() or os.readlink(extracted_link).encode("ascii") != payload:
|
|
raise GateStop(f"{layout}/{case_id}: fsck extraction target differs")
|
|
extracted = True
|
|
records.append(
|
|
{
|
|
"case": case_id,
|
|
"changed_offsets": changed,
|
|
"dump_cross_check": "PASS",
|
|
"expected": expected,
|
|
"fsck_check_exit": fsck.returncode,
|
|
"fsck_extract": extracted,
|
|
"image": image_path.name,
|
|
"inode": {
|
|
"layout": inode.layout,
|
|
"layout_name": LAYOUT_NAMES[inode.layout],
|
|
"mode": oct(inode.mode),
|
|
"nid": inode.nid,
|
|
"size": inode.size,
|
|
},
|
|
"layout": layout,
|
|
"oracle": details,
|
|
"source_payload_sha256": sha256_bytes(payload),
|
|
"target_sha256": None if target is None else sha256_bytes(target),
|
|
}
|
|
)
|
|
fixture_sha256, hashes = fixture_set_hash(fixtures)
|
|
expected_fixture = SPEC.get("expected_fixture_set_sha256", "")
|
|
if expected_fixture and fixture_sha256 != expected_fixture:
|
|
raise GateStop(
|
|
f"fixture set hash {fixture_sha256} != frozen {expected_fixture}"
|
|
)
|
|
write_json(OUTPUT / "cases.json", records)
|
|
write_json(OUTPUT / "commands.json", COMMANDS)
|
|
by_layout = {
|
|
layout: {
|
|
record["case"]: record["expected"]
|
|
for record in records
|
|
if record["layout"] == layout
|
|
}
|
|
for layout in sorted(SPEC["layouts"])
|
|
}
|
|
required = set(cases)
|
|
if any(set(result) != required for result in by_layout.values()):
|
|
raise GateStop("one or more layouts lack a complete oracle")
|
|
return {
|
|
"candidate": "P15-019",
|
|
"case_count": len(records),
|
|
"decision": "GO",
|
|
"fixture_file_count": len(hashes),
|
|
"fixture_set_sha256": fixture_sha256,
|
|
"gate": "G03",
|
|
"identity": identity,
|
|
"layout_results": by_layout,
|
|
"qemu": "NOT_RUN: Stage0 host fixture/oracle is complete",
|
|
"source_modified": False,
|
|
}
|
|
|
|
|
|
before_protected = protected_state()
|
|
status = "INFRA_BLOCKED"
|
|
exit_code = 2
|
|
result: dict[str, Any]
|
|
try:
|
|
result = execute()
|
|
status = "GO"
|
|
exit_code = 0
|
|
except GateStop as error:
|
|
status = "STOP"
|
|
exit_code = 1
|
|
result = {"candidate": "P15-019", "decision": status, "reason": str(error)}
|
|
except (InfraBlocked, OSError, ValueError, KeyError, json.JSONDecodeError) as error:
|
|
result = {"candidate": "P15-019", "decision": status, "reason": str(error)}
|
|
except Exception as error:
|
|
result = {
|
|
"candidate": "P15-019",
|
|
"decision": status,
|
|
"reason": f"unexpected gate error: {error}",
|
|
"traceback": traceback.format_exc(),
|
|
}
|
|
after_protected = protected_state()
|
|
cleanup = {
|
|
"owned_temp_remaining": sorted(path.name for path in OUTPUT.glob(".p15-019-work-*")),
|
|
"protected_before": before_protected,
|
|
"protected_after": after_protected,
|
|
"protected_unchanged": before_protected == after_protected,
|
|
}
|
|
write_json(OUTPUT / "cleanup.json", cleanup)
|
|
if cleanup["owned_temp_remaining"] or not cleanup["protected_unchanged"]:
|
|
status = "INFRA_BLOCKED"
|
|
exit_code = 2
|
|
result = {
|
|
"candidate": "P15-019",
|
|
"decision": status,
|
|
"reason": "cleanup or protected-resource identity changed",
|
|
}
|
|
result["cleanup_sha256"] = sha256_path(OUTPUT / "cleanup.json")
|
|
result["input_sha256"] = sha256_path(INPUT)
|
|
write_json(OUTPUT / "result.json", result)
|
|
print(f"P15-019 G03: {status}")
|
|
if "fixture_set_sha256" in result:
|
|
print(f"fixture_set_sha256={result['fixture_set_sha256']}")
|
|
if "reason" in result:
|
|
print(f"reason={result['reason']}")
|
|
sys.exit(exit_code)
|
|
PY
|