861 lines
31 KiB
Bash
Executable File
861 lines
31 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
|
input=$gate_dir/P15-083-input.json
|
|
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
|
|
base=
|
|
output=
|
|
|
|
while test "$#" -gt 0; do
|
|
case "$1" in
|
|
--base)
|
|
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
|
base=$2
|
|
shift 2
|
|
;;
|
|
--output)
|
|
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
|
output=$2
|
|
shift 2
|
|
;;
|
|
*)
|
|
printf 'unknown argument: %s\n' "$1" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
|
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
|
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
|
|
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
|
|
for tool in cc dump.erofs fsck.erofs git mkfs.erofs pkg-config python3 sha256sum; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'missing required host tool: %s\n' "$tool" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
case "$output" in
|
|
/*) ;;
|
|
*) output=$PWD/$output ;;
|
|
esac
|
|
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
|
mkdir -p "$output"
|
|
|
|
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(sys.argv[1])
|
|
INPUT = Path(sys.argv[2])
|
|
REQUESTED_BASE = sys.argv[3]
|
|
OUTPUT = Path(sys.argv[4])
|
|
FREEBSD_SRC = Path(sys.argv[5])
|
|
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
|
|
EINTEGRITY = 97
|
|
|
|
|
|
class GateFailure(Exception):
|
|
def __init__(self, status: str, reason: str):
|
|
super().__init__(reason)
|
|
self.status = status
|
|
self.reason = reason
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_path(path: Path) -> str:
|
|
return sha256_bytes(path.read_bytes())
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
|
|
|
|
|
def git(*args: str) -> str:
|
|
return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip()
|
|
|
|
|
|
def source_at(commit: str, path: str) -> str:
|
|
completed = subprocess.run(
|
|
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
)
|
|
if completed.returncode != 0:
|
|
raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}")
|
|
return completed.stdout
|
|
|
|
|
|
def run_logged(
|
|
argv: list[str], cwd: Path, log: Path, timeout: int = 120, expected: set[int] | None = None
|
|
) -> subprocess.CompletedProcess[str]:
|
|
completed = subprocess.run(
|
|
argv,
|
|
cwd=cwd,
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
log.parent.mkdir(parents=True, exist_ok=True)
|
|
log.write_text(
|
|
"$ " + " ".join(argv) + "\n" + completed.stdout + f"\n[exit {completed.returncode}]\n",
|
|
encoding="utf-8",
|
|
)
|
|
allowed = {0} if expected is None else expected
|
|
if completed.returncode not in allowed:
|
|
raise GateFailure("INFRA_BLOCKED", f"command failed ({completed.returncode}): {' '.join(argv)}")
|
|
return completed
|
|
|
|
|
|
def make_source(path: Path) -> bytes:
|
|
path.mkdir(parents=True)
|
|
content = b"".join(
|
|
f"P15-083-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode("ascii")
|
|
for index in range(SPEC["fixture"]["line_count"])
|
|
)
|
|
payload = path / "payload.bin"
|
|
payload.write_bytes(content)
|
|
os.utime(payload, (0, 0))
|
|
os.utime(path, (0, 0))
|
|
if len(content) != SPEC["fixture"]["source_size"] or sha256_bytes(content) != SPEC["fixture"]["source_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "deterministic source identity changed")
|
|
return content
|
|
|
|
|
|
EXTENT_RE = re.compile(
|
|
r"^\s*(\d+):\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:\s*"
|
|
r"(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$",
|
|
re.MULTILINE,
|
|
)
|
|
|
|
|
|
def parse_extents(text: str) -> list[dict[str, int]]:
|
|
records = []
|
|
for match in EXTENT_RE.finditer(text):
|
|
index, logical, logical_end, logical_length, physical, physical_end, physical_length = (
|
|
map(int, match.groups())
|
|
)
|
|
if logical_end - logical != logical_length or physical_end - physical != physical_length:
|
|
raise GateFailure("INFRA_BLOCKED", "dump.erofs extent arithmetic changed")
|
|
records.append(
|
|
{
|
|
"index": index,
|
|
"logical_length": logical_length,
|
|
"logical_offset": logical,
|
|
"physical_length": physical_length,
|
|
"physical_offset": physical,
|
|
}
|
|
)
|
|
if not records:
|
|
raise GateFailure("INFRA_BLOCKED", "dump.erofs returned no extents")
|
|
return records
|
|
|
|
|
|
ORACLE_SOURCE = r'''
|
|
#define _POSIX_C_SOURCE 200809L
|
|
#include <errno.h>
|
|
#include <fcntl.h>
|
|
#include <inttypes.h>
|
|
#include <lzma.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/mman.h>
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
#include <zlib.h>
|
|
#include <zstd.h>
|
|
|
|
static void
|
|
fail(const char *message)
|
|
{
|
|
fprintf(stderr, "%s\n", message);
|
|
exit(2);
|
|
}
|
|
|
|
static unsigned char *
|
|
read_file(const char *path, size_t *sizep)
|
|
{
|
|
struct stat st;
|
|
unsigned char *data;
|
|
ssize_t done, amount;
|
|
int fd;
|
|
|
|
fd = open(path, O_RDONLY);
|
|
if (fd < 0 || fstat(fd, &st) != 0 || st.st_size <= 0)
|
|
fail("cannot open oracle input");
|
|
*sizep = (size_t)st.st_size;
|
|
data = malloc(*sizep);
|
|
if (data == NULL)
|
|
fail("cannot allocate oracle input");
|
|
done = 0;
|
|
while ((size_t)done < *sizep) {
|
|
amount = read(fd, data + done, *sizep - (size_t)done);
|
|
if (amount <= 0)
|
|
fail("cannot read oracle input");
|
|
done += amount;
|
|
}
|
|
close(fd);
|
|
return (data);
|
|
}
|
|
|
|
static void
|
|
write_file(const char *path, const unsigned char *data, size_t size)
|
|
{
|
|
ssize_t done, amount;
|
|
int fd;
|
|
|
|
fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
|
if (fd < 0)
|
|
fail("cannot create oracle output");
|
|
done = 0;
|
|
while ((size_t)done < size) {
|
|
amount = write(fd, data + done, size - (size_t)done);
|
|
if (amount <= 0)
|
|
fail("cannot write oracle output");
|
|
done += amount;
|
|
}
|
|
close(fd);
|
|
}
|
|
|
|
int
|
|
main(int argc, char **argv)
|
|
{
|
|
unsigned char *input, *mapping, *output;
|
|
size_t input_size, output_size, usable, page_size;
|
|
size_t consumed = 0, produced = 0, library_status = 0;
|
|
uint32_t dict_size;
|
|
int cleanup = 0, codec_error = 0, full, guards = 1, stream_end = 0;
|
|
|
|
if (argc != 7)
|
|
fail("usage: oracle CODEC INPUT OUTPUT OUTPUT_SIZE FULL DICT_SIZE");
|
|
output_size = (size_t)strtoull(argv[4], NULL, 10);
|
|
full = atoi(argv[5]);
|
|
dict_size = (uint32_t)strtoul(argv[6], NULL, 10);
|
|
if (output_size == 0)
|
|
fail("zero output size");
|
|
input = read_file(argv[2], &input_size);
|
|
page_size = (size_t)sysconf(_SC_PAGESIZE);
|
|
usable = (output_size + page_size - 1) & ~(page_size - 1);
|
|
mapping = mmap(NULL, usable + 2 * page_size, PROT_READ | PROT_WRITE,
|
|
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
|
if (mapping == MAP_FAILED)
|
|
fail("cannot allocate guarded output");
|
|
if (mprotect(mapping, page_size, PROT_NONE) != 0 ||
|
|
mprotect(mapping + page_size + usable, page_size, PROT_NONE) != 0)
|
|
fail("cannot protect output guards");
|
|
output = mapping + page_size;
|
|
memset(output, 0xa5, usable);
|
|
|
|
if (strcmp(argv[1], "deflate") == 0) {
|
|
z_stream stream;
|
|
int ret = Z_OK, endret;
|
|
|
|
memset(&stream, 0, sizeof(stream));
|
|
stream.next_in = input;
|
|
stream.avail_in = (uInt)input_size;
|
|
stream.next_out = output;
|
|
stream.avail_out = (uInt)output_size;
|
|
ret = inflateInit2(&stream, -15);
|
|
if (ret != Z_OK) {
|
|
codec_error = 1;
|
|
} else {
|
|
while (stream.avail_out != 0) {
|
|
uInt in_before = stream.avail_in;
|
|
uInt out_before = stream.avail_out;
|
|
ret = inflate(&stream, Z_SYNC_FLUSH);
|
|
if (ret == Z_STREAM_END) {
|
|
stream_end = 1;
|
|
break;
|
|
}
|
|
if (ret != Z_OK || (stream.avail_in == in_before &&
|
|
stream.avail_out == out_before)) {
|
|
codec_error = 1;
|
|
break;
|
|
}
|
|
}
|
|
consumed = input_size - stream.avail_in;
|
|
produced = output_size - stream.avail_out;
|
|
library_status = (size_t)(unsigned int)ret;
|
|
endret = inflateEnd(&stream);
|
|
cleanup = endret == Z_OK;
|
|
}
|
|
} else if (strcmp(argv[1], "lzma") == 0) {
|
|
lzma_stream stream = LZMA_STREAM_INIT;
|
|
lzma_ret ret;
|
|
|
|
ret = lzma_microlzma_decoder(&stream, input_size, output_size,
|
|
full != 0, dict_size);
|
|
if (ret != LZMA_OK) {
|
|
codec_error = 1;
|
|
} else {
|
|
stream.next_in = input;
|
|
stream.avail_in = input_size;
|
|
stream.next_out = output;
|
|
stream.avail_out = output_size;
|
|
while (stream.avail_out != 0) {
|
|
size_t in_before = stream.avail_in;
|
|
size_t out_before = stream.avail_out;
|
|
ret = lzma_code(&stream, LZMA_RUN);
|
|
if (ret == LZMA_STREAM_END) {
|
|
stream_end = 1;
|
|
break;
|
|
}
|
|
if (ret != LZMA_OK || (stream.avail_in == in_before &&
|
|
stream.avail_out == out_before)) {
|
|
codec_error = 1;
|
|
break;
|
|
}
|
|
}
|
|
consumed = input_size - stream.avail_in;
|
|
produced = output_size - stream.avail_out;
|
|
library_status = ret;
|
|
}
|
|
lzma_end(&stream);
|
|
cleanup = 1;
|
|
} else if (strcmp(argv[1], "zstd") == 0) {
|
|
ZSTD_DCtx *context;
|
|
ZSTD_inBuffer in_buffer;
|
|
ZSTD_outBuffer out_buffer;
|
|
size_t ret = 1;
|
|
|
|
context = ZSTD_createDCtx();
|
|
if (context == NULL) {
|
|
codec_error = 1;
|
|
} else {
|
|
in_buffer = (ZSTD_inBuffer){ input, input_size, 0 };
|
|
out_buffer = (ZSTD_outBuffer){ output, output_size, 0 };
|
|
while (out_buffer.pos != out_buffer.size) {
|
|
size_t in_before = in_buffer.pos;
|
|
size_t out_before = out_buffer.pos;
|
|
ret = ZSTD_decompressStream(context, &out_buffer, &in_buffer);
|
|
if (ZSTD_isError(ret)) {
|
|
codec_error = 1;
|
|
break;
|
|
}
|
|
if (ret == 0) {
|
|
stream_end = 1;
|
|
break;
|
|
}
|
|
if (in_buffer.pos == in_before && out_buffer.pos == out_before) {
|
|
codec_error = 1;
|
|
break;
|
|
}
|
|
}
|
|
consumed = in_buffer.pos;
|
|
produced = out_buffer.pos;
|
|
library_status = ret;
|
|
cleanup = !ZSTD_isError(ZSTD_freeDCtx(context));
|
|
}
|
|
} else {
|
|
fail("unknown codec");
|
|
}
|
|
|
|
for (size_t index = output_size; index < usable; ++index) {
|
|
if (output[index] != 0xa5) {
|
|
guards = 0;
|
|
break;
|
|
}
|
|
}
|
|
write_file(argv[3], output, produced);
|
|
printf("codec_error=%d cleanup=%d consumed=%zu guards=%d library_status=%zu "
|
|
"produced=%zu stream_end=%d\n", codec_error, cleanup, consumed, guards,
|
|
library_status, produced, stream_end);
|
|
munmap(mapping, usable + 2 * page_size);
|
|
free(input);
|
|
return (0);
|
|
}
|
|
'''
|
|
|
|
|
|
def compile_oracle(temp: Path) -> tuple[Path, dict[str, str]]:
|
|
source = temp / "p15-083-oracle.c"
|
|
binary = temp / "p15-083-oracle"
|
|
source.write_text(ORACLE_SOURCE, encoding="ascii")
|
|
completed = run_logged(
|
|
[
|
|
"cc",
|
|
"-O2",
|
|
"-std=c17",
|
|
"-Wall",
|
|
"-Wextra",
|
|
"-Werror",
|
|
str(source),
|
|
"-o",
|
|
str(binary),
|
|
"-llzma",
|
|
"-lz",
|
|
"-lzstd",
|
|
],
|
|
temp,
|
|
OUTPUT / "logs/oracle-build.log",
|
|
)
|
|
del completed
|
|
ldd = run_logged(["ldd", str(binary)], temp, OUTPUT / "logs/oracle-ldd.log")
|
|
versions = {
|
|
"liblzma": subprocess.check_output(["pkg-config", "--modversion", "liblzma"], text=True).strip(),
|
|
"zlib": subprocess.check_output(["pkg-config", "--modversion", "zlib"], text=True).strip(),
|
|
"libzstd": subprocess.check_output(["pkg-config", "--modversion", "libzstd"], text=True).strip(),
|
|
}
|
|
if versions != SPEC["libraries"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"independent library versions changed: {versions}")
|
|
return binary, {"binary_sha256": sha256_path(binary), "ldd": ldd.stdout, "source_sha256": sha256_path(source), **versions}
|
|
|
|
|
|
def decode(
|
|
oracle: Path,
|
|
codec: str,
|
|
data: bytes,
|
|
output_size: int,
|
|
full: bool,
|
|
dict_size: int,
|
|
temp: Path,
|
|
label: str,
|
|
) -> tuple[dict[str, int], bytes]:
|
|
input_path = temp / f"{label}.input"
|
|
output_path = temp / f"{label}.output"
|
|
input_path.write_bytes(data)
|
|
completed = run_logged(
|
|
[
|
|
str(oracle),
|
|
codec,
|
|
str(input_path),
|
|
str(output_path),
|
|
str(output_size),
|
|
"1" if full else "0",
|
|
str(dict_size),
|
|
],
|
|
temp,
|
|
OUTPUT / f"logs/oracle-{label}.log",
|
|
)
|
|
record = {}
|
|
for field in completed.stdout.strip().split():
|
|
key, value = field.split("=", 1)
|
|
record[key] = int(value)
|
|
required = {"codec_error", "cleanup", "consumed", "guards", "library_status", "produced", "stream_end"}
|
|
if set(record) != required:
|
|
raise GateFailure("INFRA_BLOCKED", f"oracle output fields changed for {label}")
|
|
return record, output_path.read_bytes()
|
|
|
|
|
|
def full_policy_errno(record: dict[str, int], input_size: int, expected_output: bytes, output: bytes) -> int:
|
|
if (
|
|
record["codec_error"] != 0
|
|
or record["cleanup"] != 1
|
|
or record["guards"] != 1
|
|
or record["produced"] != len(expected_output)
|
|
or record["stream_end"] != 1
|
|
or record["consumed"] != input_size
|
|
or output != expected_output
|
|
):
|
|
return EINTEGRITY
|
|
return 0
|
|
|
|
|
|
def partial_policy_errno(record: dict[str, int], expected_output: bytes, output: bytes) -> int:
|
|
if (
|
|
record["codec_error"] != 0
|
|
or record["cleanup"] != 1
|
|
or record["guards"] != 1
|
|
or record["produced"] != len(expected_output)
|
|
or output != expected_output
|
|
):
|
|
return EINTEGRITY
|
|
return 0
|
|
|
|
|
|
def verify_source_contract(sources: dict[str, str]) -> dict[str, Any]:
|
|
dispatch = sources["repo-pre-15/src/decompressor.c"]
|
|
if not all(
|
|
anchor in dispatch
|
|
for anchor in (
|
|
"map->m_algorithmformat != Z_EROFS_COMPRESSION_LZ4",
|
|
"src[padding] == 0",
|
|
"rq.inputsize = srclen;",
|
|
"return (decompressor->decompress(&rq));",
|
|
)
|
|
):
|
|
raise GateFailure("INFRA_BLOCKED", "frozen DUT leading-padding dispatch changed")
|
|
checks = {
|
|
"deflate": "(ret != Z_STREAM_END || strm.avail_in != 0)",
|
|
"lzma": "buffer.in_pos == rq->inputsize",
|
|
"zstd": "(ret != 0 || input.pos != input.size)",
|
|
}
|
|
for codec, anchor in checks.items():
|
|
if anchor not in sources[f"repo-pre-15/src/decompressor_{codec}.c"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"frozen DUT {codec} full-tail check changed")
|
|
zdata = sources["repo-pre-15/src/zdata.c"]
|
|
read_extent_start = zdata.index("z_erofs_read_extent(")
|
|
read_extent_end = zdata.index("z_erofs_do_read(", read_extent_start)
|
|
read_extent = zdata[read_extent_start:read_extent_end]
|
|
decode_at = read_extent.index("error = z_erofs_decompress")
|
|
meta_release_at = read_extent.index("erofs_put_metabuf(&buf)", decode_at)
|
|
physical_release_at = read_extent.index("erofs_brelse(compressed)", decode_at)
|
|
error_at = read_extent.index("if (error != 0)", decode_at)
|
|
free_at = read_extent.index("free(decoded, M_EROFS)", error_at)
|
|
publish_at = read_extent.index("*bufp = decoded;", free_at)
|
|
if not (
|
|
decode_at < meta_release_at < error_at < free_at < publish_at
|
|
and decode_at < physical_release_at < error_at
|
|
):
|
|
raise GateFailure("INFRA_BLOCKED", "frozen DUT buffer cleanup ordering changed")
|
|
linux_common = sources["src-linux/decompressor.c"]
|
|
if "For others, zero_padding is enabled all the time." not in linux_common:
|
|
raise GateFailure("INFRA_BLOCKED", "Linux non-LZ4 padding anchor changed")
|
|
linux_anchors = {
|
|
"deflate": "if (zerr == Z_STREAM_END && !rq->outputsize)",
|
|
"lzma": "xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize",
|
|
"zstd": "zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);",
|
|
}
|
|
for codec, anchor in linux_anchors.items():
|
|
if anchor not in sources[f"src-linux/decompressor_{codec}.c"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"Linux {codec} comparison anchor changed")
|
|
return {
|
|
"freebsd_positive_eintegrity": True,
|
|
"input_release_after_decode": True,
|
|
"linux_leading_zero_padding": True,
|
|
"output_freed_on_error": True,
|
|
"output_published_only_on_success": True,
|
|
"per_codec_full_checks": checks,
|
|
}
|
|
|
|
|
|
def verify_tools() -> dict[str, Any]:
|
|
records = {}
|
|
for name, expected in SPEC["tools"].items():
|
|
path = Path(expected["path"])
|
|
if not path.is_file() or sha256_path(path) != expected["sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"tool identity changed: {name}")
|
|
records[name] = {**expected}
|
|
mkfs_version = subprocess.check_output([SPEC["tools"]["mkfs.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True)
|
|
fsck_version = subprocess.check_output([SPEC["tools"]["fsck.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True)
|
|
for codec in SPEC["codecs"]:
|
|
if codec not in mkfs_version or codec not in fsck_version:
|
|
raise GateFailure("STOP", f"{codec} lacks a real mkfs/fsck codec path")
|
|
records["mkfs_version"] = mkfs_version.strip()
|
|
records["fsck_version"] = fsck_version.strip()
|
|
return records
|
|
|
|
|
|
def fsck_image(
|
|
fsck: str,
|
|
image: Path,
|
|
destination: Path,
|
|
label: str,
|
|
expect_success: bool | None,
|
|
) -> dict[str, Any]:
|
|
completed = run_logged(
|
|
[fsck, f"--extract={destination}", str(image)],
|
|
image.parent,
|
|
OUTPUT / f"logs/fsck-{label}.log",
|
|
expected=set(range(0, 256)),
|
|
)
|
|
success = completed.returncode == 0
|
|
if expect_success is not None and success != expect_success:
|
|
raise GateFailure("INFRA_BLOCKED", f"fsck classification changed for {label}: exit {completed.returncode}")
|
|
return {"exit": completed.returncode, "success": success}
|
|
|
|
|
|
def evaluate_codec(
|
|
codec: str,
|
|
spec: dict[str, Any],
|
|
source: bytes,
|
|
source_dir: Path,
|
|
oracle: Path,
|
|
temp: Path,
|
|
) -> dict[str, Any]:
|
|
mkfs = SPEC["tools"]["mkfs.erofs"]["path"]
|
|
fsck = SPEC["tools"]["fsck.erofs"]["path"]
|
|
dump = SPEC["tools"]["dump.erofs"]["path"]
|
|
images = []
|
|
for pass_name in ("a", "b"):
|
|
image = temp / f"{codec}-{pass_name}.erofs"
|
|
run_logged([mkfs, *spec["mkfs_args"], str(image), str(source_dir)], temp, OUTPUT / f"logs/mkfs-{codec}-{pass_name}.log")
|
|
images.append(image)
|
|
hashes = [sha256_path(path) for path in images]
|
|
if hashes != [spec["expected_image_sha256"]] * 2 or images[0].stat().st_size != spec["expected_image_size"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"{codec} image reproducibility changed: {hashes}")
|
|
valid_fsck = fsck_image(fsck, images[0], temp / f"extract-{codec}-valid", f"{codec}-valid", True)
|
|
extracted = (temp / f"extract-{codec}-valid/payload.bin").read_bytes()
|
|
if extracted != source:
|
|
raise GateFailure("INFRA_BLOCKED", f"{codec} legal image extraction mismatch")
|
|
dumped = run_logged([dump, "--path=/payload.bin", "-e", str(images[0])], temp, OUTPUT / f"logs/dump-{codec}.log")
|
|
extents = parse_extents(dumped.stdout)
|
|
expected_extent = spec["extent"]
|
|
selected = next((record for record in extents if record["index"] == expected_extent["index"]), None)
|
|
if selected is None or selected != {key: expected_extent[key] for key in selected}:
|
|
raise GateFailure("INFRA_BLOCKED", f"{codec} selected extent changed")
|
|
image_bytes = images[0].read_bytes()
|
|
block_start = selected["physical_offset"]
|
|
block_end = block_start + selected["physical_length"]
|
|
block = image_bytes[block_start:block_end]
|
|
leading = next((index for index, value in enumerate(block) if value), len(block))
|
|
stream = block[leading:]
|
|
if leading != expected_extent["leading_zero_bytes"] or len(stream) != expected_extent["stream_bytes"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"{codec} leading padding or stream length changed")
|
|
logical = source[selected["logical_offset"] : selected["logical_offset"] + selected["logical_length"]]
|
|
full_record, full_output = decode(oracle, codec, stream, len(logical), True, spec["dict_size"], temp, f"{codec}-full")
|
|
if full_policy_errno(full_record, len(stream), logical, full_output) != 0:
|
|
raise GateFailure("STOP", f"{codec} legal mkfs extent is not exact after EROFS leading padding")
|
|
|
|
tail = bytes.fromhex(SPEC["fixture"]["tail_bytes_hex"])
|
|
if leading <= len(tail):
|
|
raise GateFailure("INFRA_BLOCKED", f"{codec} selected extent lacks mutation room")
|
|
tail_stream = stream + tail
|
|
tail_record, tail_output = decode(oracle, codec, tail_stream, len(logical), True, spec["dict_size"], temp, f"{codec}-tail")
|
|
tail_errno = full_policy_errno(tail_record, len(tail_stream), logical, tail_output)
|
|
if tail_errno != EINTEGRITY:
|
|
raise GateFailure("STOP", f"{codec} cannot distinguish nonzero trailing garbage from a complete EROFS stream")
|
|
tail_image_bytes = bytearray(image_bytes)
|
|
shifted_start = block_start + leading - len(tail)
|
|
tail_image_bytes[shifted_start:block_end] = tail_stream
|
|
tail_image = temp / f"{codec}-tail.erofs"
|
|
tail_image.write_bytes(tail_image_bytes)
|
|
tail_fsck = fsck_image(
|
|
fsck,
|
|
tail_image,
|
|
temp / f"extract-{codec}-tail",
|
|
f"{codec}-tail",
|
|
None,
|
|
)
|
|
if tail_fsck["success"]:
|
|
tail_extracted = (temp / f"extract-{codec}-tail/payload.bin").read_bytes()
|
|
if tail_extracted != source:
|
|
raise GateFailure("INFRA_BLOCKED", f"{codec} permissive fsck tail output mismatch")
|
|
|
|
truncated_stream = stream[:-1]
|
|
truncated_record, truncated_output = decode(
|
|
oracle, codec, truncated_stream, len(logical), True, spec["dict_size"], temp, f"{codec}-truncated"
|
|
)
|
|
truncated_errno = full_policy_errno(truncated_record, len(truncated_stream), logical, truncated_output)
|
|
if truncated_errno != EINTEGRITY:
|
|
raise GateFailure("STOP", f"{codec} truncated stream reaches full success")
|
|
truncated_image_bytes = bytearray(image_bytes)
|
|
truncated_image_bytes[block_start + leading : block_end] = b"\0" + truncated_stream
|
|
truncated_image = temp / f"{codec}-truncated.erofs"
|
|
truncated_image.write_bytes(truncated_image_bytes)
|
|
truncated_fsck = fsck_image(
|
|
fsck,
|
|
truncated_image,
|
|
temp / f"extract-{codec}-truncated",
|
|
f"{codec}-truncated",
|
|
False,
|
|
)
|
|
|
|
partial_size = min(4096, len(logical) // 4)
|
|
partial_expected = logical[:partial_size]
|
|
partial_record, partial_output = decode(
|
|
oracle, codec, stream, partial_size, False, spec["dict_size"], temp, f"{codec}-partial"
|
|
)
|
|
if partial_policy_errno(partial_record, partial_expected, partial_output) != 0:
|
|
raise GateFailure("STOP", f"{codec} partial output differs from the full slice")
|
|
corruption_start = max(partial_record["consumed"] + 16, len(stream) - 64)
|
|
if corruption_start >= len(stream):
|
|
raise GateFailure("STOP", f"{codec} partial decode consumes the entire stream")
|
|
corrupted_stream = stream[:corruption_start] + b"\0" * (len(stream) - corruption_start)
|
|
if corrupted_stream == stream:
|
|
raise GateFailure("INFRA_BLOCKED", f"{codec} corruption mutation changed no bytes")
|
|
corrupt_partial_record, corrupt_partial_output = decode(
|
|
oracle,
|
|
codec,
|
|
corrupted_stream,
|
|
partial_size,
|
|
False,
|
|
spec["dict_size"],
|
|
temp,
|
|
f"{codec}-corrupt-partial",
|
|
)
|
|
if partial_policy_errno(corrupt_partial_record, partial_expected, corrupt_partial_output) != 0:
|
|
raise GateFailure("STOP", f"{codec} range-after corruption changed the requested partial slice")
|
|
corrupt_full_record, corrupt_full_output = decode(
|
|
oracle,
|
|
codec,
|
|
corrupted_stream,
|
|
len(logical),
|
|
True,
|
|
spec["dict_size"],
|
|
temp,
|
|
f"{codec}-corrupt-full",
|
|
)
|
|
corrupt_full_errno = full_policy_errno(
|
|
corrupt_full_record, len(corrupted_stream), logical, corrupt_full_output
|
|
)
|
|
if corrupt_full_errno != EINTEGRITY:
|
|
raise GateFailure("STOP", f"{codec} full read does not detect range-after corruption")
|
|
corrupt_image_bytes = bytearray(image_bytes)
|
|
corrupt_image_bytes[block_start + leading : block_end] = corrupted_stream
|
|
corrupt_image = temp / f"{codec}-corrupt.erofs"
|
|
corrupt_image.write_bytes(corrupt_image_bytes)
|
|
corrupt_fsck = fsck_image(
|
|
fsck,
|
|
corrupt_image,
|
|
temp / f"extract-{codec}-corrupt",
|
|
f"{codec}-corrupt",
|
|
False,
|
|
)
|
|
|
|
return {
|
|
"codec": codec,
|
|
"corruption": {
|
|
"full_errno": corrupt_full_errno,
|
|
"fsck": corrupt_fsck,
|
|
"starts_after_partial_consumed": corruption_start > partial_record["consumed"],
|
|
"starts_at_stream_byte": corruption_start,
|
|
},
|
|
"extent": {**selected, "leading_zero_bytes": leading, "stream_bytes": len(stream)},
|
|
"full": {**full_record, "policy_errno": 0, "output_sha256": sha256_bytes(full_output)},
|
|
"image_repeated_sha256": hashes,
|
|
"legal_fsck": valid_fsck,
|
|
"partial": {
|
|
**partial_record,
|
|
"corrupt_policy_errno": 0,
|
|
"output_matches_full_slice": True,
|
|
"policy_errno": 0,
|
|
"requested_bytes": partial_size,
|
|
},
|
|
"tail": {
|
|
**tail_record,
|
|
"bytes": len(tail),
|
|
"fsck": tail_fsck,
|
|
"nonzero": True,
|
|
"policy_errno": tail_errno,
|
|
},
|
|
"truncated": {**truncated_record, "fsck": truncated_fsck, "policy_errno": truncated_errno},
|
|
}
|
|
|
|
|
|
def finalize() -> None:
|
|
lines = []
|
|
for path in sorted(OUTPUT.rglob("*")):
|
|
if path.is_file() and path.name != "SHA256SUMS":
|
|
lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
|
|
(OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
|
|
|
|
|
|
result: dict[str, Any] | None = None
|
|
exit_code = 0
|
|
owned_temp: str | None = None
|
|
try:
|
|
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-083" or SPEC.get("gate") != "G04":
|
|
raise GateFailure("INFRA_BLOCKED", "invalid P15-083 input schema")
|
|
resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
|
|
if resolved != SPEC["required_base"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"P15-083 must replay {SPEC['required_base']}, got {resolved}")
|
|
sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]}
|
|
hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()}
|
|
if hashes != SPEC["source_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed")
|
|
write_json(OUTPUT / "source-sha256.json", hashes)
|
|
freebsd_head = subprocess.check_output(["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True).strip()
|
|
freebsd_hashes = {path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"]}
|
|
if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed")
|
|
errno_source = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8")
|
|
if "#define\tEINTEGRITY\t97" not in errno_source:
|
|
raise GateFailure("INFRA_BLOCKED", "FreeBSD positive EINTEGRITY changed")
|
|
write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes})
|
|
write_json(OUTPUT / "toolchain.json", verify_tools())
|
|
write_json(OUTPUT / "source-contract.json", verify_source_contract(sources))
|
|
with tempfile.TemporaryDirectory(prefix="p15-083-g04-") as temporary:
|
|
owned_temp = temporary
|
|
temp = Path(temporary)
|
|
source_dir = temp / "source"
|
|
source = make_source(source_dir)
|
|
oracle, library_record = compile_oracle(temp)
|
|
write_json(OUTPUT / "independent-libraries.json", library_record)
|
|
records = [
|
|
evaluate_codec(codec, codec_spec, source, source_dir, oracle, temp)
|
|
for codec, codec_spec in sorted(SPEC["codecs"].items())
|
|
]
|
|
if {record["codec"] for record in records} != {"deflate", "lzma", "zstd"}:
|
|
raise GateFailure("STOP", "P15-083 requires all three non-LZ4 codec policies")
|
|
write_json(OUTPUT / "codec-results.json", records)
|
|
result = {
|
|
"b27": "AUTHORIZED",
|
|
"candidate": "P15-083",
|
|
"cleanup": "PASS",
|
|
"codecs": {record["codec"]: "GO" for record in records},
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gate": "G04",
|
|
"oracle": "real erofs-utils 1.8.6 images + liblzma/zlib/libzstd consumed-byte oracle",
|
|
"policy": {
|
|
"deflate": "strip EROFS leading zero padding; full raw stream must reach Z_STREAM_END with no unread bytes",
|
|
"lzma": "strip EROFS leading zero padding; MicroLZMA compressed size is exact and all bytes must be consumed",
|
|
"zstd": "strip EROFS leading zero padding; one frame must complete with no unread bytes",
|
|
},
|
|
"qemu": "NOT_RUN",
|
|
"qemu_reason": "Stage0 policy and compatibility oracle is complete on real host fixtures; B27 acceptance owns TC176 QEMU",
|
|
"requested_base": REQUESTED_BASE,
|
|
"resolved_base": resolved,
|
|
"schema": 1,
|
|
"status": "GO",
|
|
"typed_errno": "PASS",
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
cleanup_record = {
|
|
"owned_temp": owned_temp,
|
|
"owned_temp_removed": owned_temp is not None and not Path(owned_temp).exists(),
|
|
"protected_pid_touched": False,
|
|
"protected_port_touched": False,
|
|
"qemu_started": False,
|
|
"shared_base_image_touched": False,
|
|
}
|
|
if not cleanup_record["owned_temp_removed"]:
|
|
raise GateFailure("INFRA_BLOCKED", "owned gate temporary directory survived cleanup")
|
|
write_json(OUTPUT / "owned-cleanup.json", cleanup_record)
|
|
except GateFailure as failure:
|
|
result = {
|
|
"b27": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN",
|
|
"candidate": "P15-083",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gate": "G04",
|
|
"qemu": "NOT_RUN",
|
|
"reason": failure.reason,
|
|
"requested_base": REQUESTED_BASE,
|
|
"schema": 1,
|
|
"status": failure.status,
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
exit_code = 1 if failure.status == "STOP" else 21
|
|
except (OSError, subprocess.SubprocessError, ValueError) as failure:
|
|
result = {
|
|
"b27": "NOT_RUN",
|
|
"candidate": "P15-083",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gate": "G04",
|
|
"qemu": "NOT_RUN",
|
|
"reason": f"gate infrastructure failure: {failure}",
|
|
"requested_base": REQUESTED_BASE,
|
|
"schema": 1,
|
|
"status": "INFRA_BLOCKED",
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
exit_code = 21
|
|
finally:
|
|
finalize()
|
|
|
|
print(json.dumps(result, sort_keys=True))
|
|
raise SystemExit(exit_code)
|
|
PY
|