update
This commit is contained in:
Executable
+860
@@ -0,0 +1,860 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
||||
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
||||
input=$gate_dir/P15-083-input.json
|
||||
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
|
||||
base=
|
||||
output=
|
||||
|
||||
while test "$#" -gt 0; do
|
||||
case "$1" in
|
||||
--base)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
||||
base=$2
|
||||
shift 2
|
||||
;;
|
||||
--output)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
||||
output=$2
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
printf 'unknown argument: %s\n' "$1" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
||||
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
||||
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
|
||||
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
|
||||
for tool in cc dump.erofs fsck.erofs git mkfs.erofs pkg-config python3 sha256sum; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf 'missing required host tool: %s\n' "$tool" >&2
|
||||
exit 2
|
||||
}
|
||||
done
|
||||
case "$output" in
|
||||
/*) ;;
|
||||
*) output=$PWD/$output ;;
|
||||
esac
|
||||
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
||||
mkdir -p "$output"
|
||||
|
||||
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
INPUT = Path(sys.argv[2])
|
||||
REQUESTED_BASE = sys.argv[3]
|
||||
OUTPUT = Path(sys.argv[4])
|
||||
FREEBSD_SRC = Path(sys.argv[5])
|
||||
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
|
||||
EINTEGRITY = 97
|
||||
|
||||
|
||||
class GateFailure(Exception):
|
||||
def __init__(self, status: str, reason: str):
|
||||
super().__init__(reason)
|
||||
self.status = status
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
return sha256_bytes(path.read_bytes())
|
||||
|
||||
|
||||
def write_json(path: Path, value: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
|
||||
|
||||
def git(*args: str) -> str:
|
||||
return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip()
|
||||
|
||||
|
||||
def source_at(commit: str, path: str) -> str:
|
||||
completed = subprocess.run(
|
||||
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def run_logged(
|
||||
argv: list[str], cwd: Path, log: Path, timeout: int = 120, expected: set[int] | None = None
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
completed = subprocess.run(
|
||||
argv,
|
||||
cwd=cwd,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
log.parent.mkdir(parents=True, exist_ok=True)
|
||||
log.write_text(
|
||||
"$ " + " ".join(argv) + "\n" + completed.stdout + f"\n[exit {completed.returncode}]\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
allowed = {0} if expected is None else expected
|
||||
if completed.returncode not in allowed:
|
||||
raise GateFailure("INFRA_BLOCKED", f"command failed ({completed.returncode}): {' '.join(argv)}")
|
||||
return completed
|
||||
|
||||
|
||||
def make_source(path: Path) -> bytes:
|
||||
path.mkdir(parents=True)
|
||||
content = b"".join(
|
||||
f"P15-083-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode("ascii")
|
||||
for index in range(SPEC["fixture"]["line_count"])
|
||||
)
|
||||
payload = path / "payload.bin"
|
||||
payload.write_bytes(content)
|
||||
os.utime(payload, (0, 0))
|
||||
os.utime(path, (0, 0))
|
||||
if len(content) != SPEC["fixture"]["source_size"] or sha256_bytes(content) != SPEC["fixture"]["source_sha256"]:
|
||||
raise GateFailure("INFRA_BLOCKED", "deterministic source identity changed")
|
||||
return content
|
||||
|
||||
|
||||
EXTENT_RE = re.compile(
|
||||
r"^\s*(\d+):\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:\s*"
|
||||
r"(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def parse_extents(text: str) -> list[dict[str, int]]:
|
||||
records = []
|
||||
for match in EXTENT_RE.finditer(text):
|
||||
index, logical, logical_end, logical_length, physical, physical_end, physical_length = (
|
||||
map(int, match.groups())
|
||||
)
|
||||
if logical_end - logical != logical_length or physical_end - physical != physical_length:
|
||||
raise GateFailure("INFRA_BLOCKED", "dump.erofs extent arithmetic changed")
|
||||
records.append(
|
||||
{
|
||||
"index": index,
|
||||
"logical_length": logical_length,
|
||||
"logical_offset": logical,
|
||||
"physical_length": physical_length,
|
||||
"physical_offset": physical,
|
||||
}
|
||||
)
|
||||
if not records:
|
||||
raise GateFailure("INFRA_BLOCKED", "dump.erofs returned no extents")
|
||||
return records
|
||||
|
||||
|
||||
ORACLE_SOURCE = r'''
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <lzma.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <zlib.h>
|
||||
#include <zstd.h>
|
||||
|
||||
static void
|
||||
fail(const char *message)
|
||||
{
|
||||
fprintf(stderr, "%s\n", message);
|
||||
exit(2);
|
||||
}
|
||||
|
||||
static unsigned char *
|
||||
read_file(const char *path, size_t *sizep)
|
||||
{
|
||||
struct stat st;
|
||||
unsigned char *data;
|
||||
ssize_t done, amount;
|
||||
int fd;
|
||||
|
||||
fd = open(path, O_RDONLY);
|
||||
if (fd < 0 || fstat(fd, &st) != 0 || st.st_size <= 0)
|
||||
fail("cannot open oracle input");
|
||||
*sizep = (size_t)st.st_size;
|
||||
data = malloc(*sizep);
|
||||
if (data == NULL)
|
||||
fail("cannot allocate oracle input");
|
||||
done = 0;
|
||||
while ((size_t)done < *sizep) {
|
||||
amount = read(fd, data + done, *sizep - (size_t)done);
|
||||
if (amount <= 0)
|
||||
fail("cannot read oracle input");
|
||||
done += amount;
|
||||
}
|
||||
close(fd);
|
||||
return (data);
|
||||
}
|
||||
|
||||
static void
|
||||
write_file(const char *path, const unsigned char *data, size_t size)
|
||||
{
|
||||
ssize_t done, amount;
|
||||
int fd;
|
||||
|
||||
fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
if (fd < 0)
|
||||
fail("cannot create oracle output");
|
||||
done = 0;
|
||||
while ((size_t)done < size) {
|
||||
amount = write(fd, data + done, size - (size_t)done);
|
||||
if (amount <= 0)
|
||||
fail("cannot write oracle output");
|
||||
done += amount;
|
||||
}
|
||||
close(fd);
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
unsigned char *input, *mapping, *output;
|
||||
size_t input_size, output_size, usable, page_size;
|
||||
size_t consumed = 0, produced = 0, library_status = 0;
|
||||
uint32_t dict_size;
|
||||
int cleanup = 0, codec_error = 0, full, guards = 1, stream_end = 0;
|
||||
|
||||
if (argc != 7)
|
||||
fail("usage: oracle CODEC INPUT OUTPUT OUTPUT_SIZE FULL DICT_SIZE");
|
||||
output_size = (size_t)strtoull(argv[4], NULL, 10);
|
||||
full = atoi(argv[5]);
|
||||
dict_size = (uint32_t)strtoul(argv[6], NULL, 10);
|
||||
if (output_size == 0)
|
||||
fail("zero output size");
|
||||
input = read_file(argv[2], &input_size);
|
||||
page_size = (size_t)sysconf(_SC_PAGESIZE);
|
||||
usable = (output_size + page_size - 1) & ~(page_size - 1);
|
||||
mapping = mmap(NULL, usable + 2 * page_size, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (mapping == MAP_FAILED)
|
||||
fail("cannot allocate guarded output");
|
||||
if (mprotect(mapping, page_size, PROT_NONE) != 0 ||
|
||||
mprotect(mapping + page_size + usable, page_size, PROT_NONE) != 0)
|
||||
fail("cannot protect output guards");
|
||||
output = mapping + page_size;
|
||||
memset(output, 0xa5, usable);
|
||||
|
||||
if (strcmp(argv[1], "deflate") == 0) {
|
||||
z_stream stream;
|
||||
int ret = Z_OK, endret;
|
||||
|
||||
memset(&stream, 0, sizeof(stream));
|
||||
stream.next_in = input;
|
||||
stream.avail_in = (uInt)input_size;
|
||||
stream.next_out = output;
|
||||
stream.avail_out = (uInt)output_size;
|
||||
ret = inflateInit2(&stream, -15);
|
||||
if (ret != Z_OK) {
|
||||
codec_error = 1;
|
||||
} else {
|
||||
while (stream.avail_out != 0) {
|
||||
uInt in_before = stream.avail_in;
|
||||
uInt out_before = stream.avail_out;
|
||||
ret = inflate(&stream, Z_SYNC_FLUSH);
|
||||
if (ret == Z_STREAM_END) {
|
||||
stream_end = 1;
|
||||
break;
|
||||
}
|
||||
if (ret != Z_OK || (stream.avail_in == in_before &&
|
||||
stream.avail_out == out_before)) {
|
||||
codec_error = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
consumed = input_size - stream.avail_in;
|
||||
produced = output_size - stream.avail_out;
|
||||
library_status = (size_t)(unsigned int)ret;
|
||||
endret = inflateEnd(&stream);
|
||||
cleanup = endret == Z_OK;
|
||||
}
|
||||
} else if (strcmp(argv[1], "lzma") == 0) {
|
||||
lzma_stream stream = LZMA_STREAM_INIT;
|
||||
lzma_ret ret;
|
||||
|
||||
ret = lzma_microlzma_decoder(&stream, input_size, output_size,
|
||||
full != 0, dict_size);
|
||||
if (ret != LZMA_OK) {
|
||||
codec_error = 1;
|
||||
} else {
|
||||
stream.next_in = input;
|
||||
stream.avail_in = input_size;
|
||||
stream.next_out = output;
|
||||
stream.avail_out = output_size;
|
||||
while (stream.avail_out != 0) {
|
||||
size_t in_before = stream.avail_in;
|
||||
size_t out_before = stream.avail_out;
|
||||
ret = lzma_code(&stream, LZMA_RUN);
|
||||
if (ret == LZMA_STREAM_END) {
|
||||
stream_end = 1;
|
||||
break;
|
||||
}
|
||||
if (ret != LZMA_OK || (stream.avail_in == in_before &&
|
||||
stream.avail_out == out_before)) {
|
||||
codec_error = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
consumed = input_size - stream.avail_in;
|
||||
produced = output_size - stream.avail_out;
|
||||
library_status = ret;
|
||||
}
|
||||
lzma_end(&stream);
|
||||
cleanup = 1;
|
||||
} else if (strcmp(argv[1], "zstd") == 0) {
|
||||
ZSTD_DCtx *context;
|
||||
ZSTD_inBuffer in_buffer;
|
||||
ZSTD_outBuffer out_buffer;
|
||||
size_t ret = 1;
|
||||
|
||||
context = ZSTD_createDCtx();
|
||||
if (context == NULL) {
|
||||
codec_error = 1;
|
||||
} else {
|
||||
in_buffer = (ZSTD_inBuffer){ input, input_size, 0 };
|
||||
out_buffer = (ZSTD_outBuffer){ output, output_size, 0 };
|
||||
while (out_buffer.pos != out_buffer.size) {
|
||||
size_t in_before = in_buffer.pos;
|
||||
size_t out_before = out_buffer.pos;
|
||||
ret = ZSTD_decompressStream(context, &out_buffer, &in_buffer);
|
||||
if (ZSTD_isError(ret)) {
|
||||
codec_error = 1;
|
||||
break;
|
||||
}
|
||||
if (ret == 0) {
|
||||
stream_end = 1;
|
||||
break;
|
||||
}
|
||||
if (in_buffer.pos == in_before && out_buffer.pos == out_before) {
|
||||
codec_error = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
consumed = in_buffer.pos;
|
||||
produced = out_buffer.pos;
|
||||
library_status = ret;
|
||||
cleanup = !ZSTD_isError(ZSTD_freeDCtx(context));
|
||||
}
|
||||
} else {
|
||||
fail("unknown codec");
|
||||
}
|
||||
|
||||
for (size_t index = output_size; index < usable; ++index) {
|
||||
if (output[index] != 0xa5) {
|
||||
guards = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
write_file(argv[3], output, produced);
|
||||
printf("codec_error=%d cleanup=%d consumed=%zu guards=%d library_status=%zu "
|
||||
"produced=%zu stream_end=%d\n", codec_error, cleanup, consumed, guards,
|
||||
library_status, produced, stream_end);
|
||||
munmap(mapping, usable + 2 * page_size);
|
||||
free(input);
|
||||
return (0);
|
||||
}
|
||||
'''
|
||||
|
||||
|
||||
def compile_oracle(temp: Path) -> tuple[Path, dict[str, str]]:
|
||||
source = temp / "p15-083-oracle.c"
|
||||
binary = temp / "p15-083-oracle"
|
||||
source.write_text(ORACLE_SOURCE, encoding="ascii")
|
||||
completed = run_logged(
|
||||
[
|
||||
"cc",
|
||||
"-O2",
|
||||
"-std=c17",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
str(source),
|
||||
"-o",
|
||||
str(binary),
|
||||
"-llzma",
|
||||
"-lz",
|
||||
"-lzstd",
|
||||
],
|
||||
temp,
|
||||
OUTPUT / "logs/oracle-build.log",
|
||||
)
|
||||
del completed
|
||||
ldd = run_logged(["ldd", str(binary)], temp, OUTPUT / "logs/oracle-ldd.log")
|
||||
versions = {
|
||||
"liblzma": subprocess.check_output(["pkg-config", "--modversion", "liblzma"], text=True).strip(),
|
||||
"zlib": subprocess.check_output(["pkg-config", "--modversion", "zlib"], text=True).strip(),
|
||||
"libzstd": subprocess.check_output(["pkg-config", "--modversion", "libzstd"], text=True).strip(),
|
||||
}
|
||||
if versions != SPEC["libraries"]:
|
||||
raise GateFailure("INFRA_BLOCKED", f"independent library versions changed: {versions}")
|
||||
return binary, {"binary_sha256": sha256_path(binary), "ldd": ldd.stdout, "source_sha256": sha256_path(source), **versions}
|
||||
|
||||
|
||||
def decode(
|
||||
oracle: Path,
|
||||
codec: str,
|
||||
data: bytes,
|
||||
output_size: int,
|
||||
full: bool,
|
||||
dict_size: int,
|
||||
temp: Path,
|
||||
label: str,
|
||||
) -> tuple[dict[str, int], bytes]:
|
||||
input_path = temp / f"{label}.input"
|
||||
output_path = temp / f"{label}.output"
|
||||
input_path.write_bytes(data)
|
||||
completed = run_logged(
|
||||
[
|
||||
str(oracle),
|
||||
codec,
|
||||
str(input_path),
|
||||
str(output_path),
|
||||
str(output_size),
|
||||
"1" if full else "0",
|
||||
str(dict_size),
|
||||
],
|
||||
temp,
|
||||
OUTPUT / f"logs/oracle-{label}.log",
|
||||
)
|
||||
record = {}
|
||||
for field in completed.stdout.strip().split():
|
||||
key, value = field.split("=", 1)
|
||||
record[key] = int(value)
|
||||
required = {"codec_error", "cleanup", "consumed", "guards", "library_status", "produced", "stream_end"}
|
||||
if set(record) != required:
|
||||
raise GateFailure("INFRA_BLOCKED", f"oracle output fields changed for {label}")
|
||||
return record, output_path.read_bytes()
|
||||
|
||||
|
||||
def full_policy_errno(record: dict[str, int], input_size: int, expected_output: bytes, output: bytes) -> int:
|
||||
if (
|
||||
record["codec_error"] != 0
|
||||
or record["cleanup"] != 1
|
||||
or record["guards"] != 1
|
||||
or record["produced"] != len(expected_output)
|
||||
or record["stream_end"] != 1
|
||||
or record["consumed"] != input_size
|
||||
or output != expected_output
|
||||
):
|
||||
return EINTEGRITY
|
||||
return 0
|
||||
|
||||
|
||||
def partial_policy_errno(record: dict[str, int], expected_output: bytes, output: bytes) -> int:
|
||||
if (
|
||||
record["codec_error"] != 0
|
||||
or record["cleanup"] != 1
|
||||
or record["guards"] != 1
|
||||
or record["produced"] != len(expected_output)
|
||||
or output != expected_output
|
||||
):
|
||||
return EINTEGRITY
|
||||
return 0
|
||||
|
||||
|
||||
def verify_source_contract(sources: dict[str, str]) -> dict[str, Any]:
|
||||
dispatch = sources["repo-pre-15/src/decompressor.c"]
|
||||
if not all(
|
||||
anchor in dispatch
|
||||
for anchor in (
|
||||
"map->m_algorithmformat != Z_EROFS_COMPRESSION_LZ4",
|
||||
"src[padding] == 0",
|
||||
"rq.inputsize = srclen;",
|
||||
"return (decompressor->decompress(&rq));",
|
||||
)
|
||||
):
|
||||
raise GateFailure("INFRA_BLOCKED", "frozen DUT leading-padding dispatch changed")
|
||||
checks = {
|
||||
"deflate": "(ret != Z_STREAM_END || strm.avail_in != 0)",
|
||||
"lzma": "buffer.in_pos == rq->inputsize",
|
||||
"zstd": "(ret != 0 || input.pos != input.size)",
|
||||
}
|
||||
for codec, anchor in checks.items():
|
||||
if anchor not in sources[f"repo-pre-15/src/decompressor_{codec}.c"]:
|
||||
raise GateFailure("INFRA_BLOCKED", f"frozen DUT {codec} full-tail check changed")
|
||||
zdata = sources["repo-pre-15/src/zdata.c"]
|
||||
read_extent_start = zdata.index("z_erofs_read_extent(")
|
||||
read_extent_end = zdata.index("z_erofs_do_read(", read_extent_start)
|
||||
read_extent = zdata[read_extent_start:read_extent_end]
|
||||
decode_at = read_extent.index("error = z_erofs_decompress")
|
||||
meta_release_at = read_extent.index("erofs_put_metabuf(&buf)", decode_at)
|
||||
physical_release_at = read_extent.index("erofs_brelse(compressed)", decode_at)
|
||||
error_at = read_extent.index("if (error != 0)", decode_at)
|
||||
free_at = read_extent.index("free(decoded, M_EROFS)", error_at)
|
||||
publish_at = read_extent.index("*bufp = decoded;", free_at)
|
||||
if not (
|
||||
decode_at < meta_release_at < error_at < free_at < publish_at
|
||||
and decode_at < physical_release_at < error_at
|
||||
):
|
||||
raise GateFailure("INFRA_BLOCKED", "frozen DUT buffer cleanup ordering changed")
|
||||
linux_common = sources["src-linux/decompressor.c"]
|
||||
if "For others, zero_padding is enabled all the time." not in linux_common:
|
||||
raise GateFailure("INFRA_BLOCKED", "Linux non-LZ4 padding anchor changed")
|
||||
linux_anchors = {
|
||||
"deflate": "if (zerr == Z_STREAM_END && !rq->outputsize)",
|
||||
"lzma": "xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize",
|
||||
"zstd": "zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);",
|
||||
}
|
||||
for codec, anchor in linux_anchors.items():
|
||||
if anchor not in sources[f"src-linux/decompressor_{codec}.c"]:
|
||||
raise GateFailure("INFRA_BLOCKED", f"Linux {codec} comparison anchor changed")
|
||||
return {
|
||||
"freebsd_positive_eintegrity": True,
|
||||
"input_release_after_decode": True,
|
||||
"linux_leading_zero_padding": True,
|
||||
"output_freed_on_error": True,
|
||||
"output_published_only_on_success": True,
|
||||
"per_codec_full_checks": checks,
|
||||
}
|
||||
|
||||
|
||||
def verify_tools() -> dict[str, Any]:
|
||||
records = {}
|
||||
for name, expected in SPEC["tools"].items():
|
||||
path = Path(expected["path"])
|
||||
if not path.is_file() or sha256_path(path) != expected["sha256"]:
|
||||
raise GateFailure("INFRA_BLOCKED", f"tool identity changed: {name}")
|
||||
records[name] = {**expected}
|
||||
mkfs_version = subprocess.check_output([SPEC["tools"]["mkfs.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True)
|
||||
fsck_version = subprocess.check_output([SPEC["tools"]["fsck.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True)
|
||||
for codec in SPEC["codecs"]:
|
||||
if codec not in mkfs_version or codec not in fsck_version:
|
||||
raise GateFailure("STOP", f"{codec} lacks a real mkfs/fsck codec path")
|
||||
records["mkfs_version"] = mkfs_version.strip()
|
||||
records["fsck_version"] = fsck_version.strip()
|
||||
return records
|
||||
|
||||
|
||||
def fsck_image(
|
||||
fsck: str,
|
||||
image: Path,
|
||||
destination: Path,
|
||||
label: str,
|
||||
expect_success: bool | None,
|
||||
) -> dict[str, Any]:
|
||||
completed = run_logged(
|
||||
[fsck, f"--extract={destination}", str(image)],
|
||||
image.parent,
|
||||
OUTPUT / f"logs/fsck-{label}.log",
|
||||
expected=set(range(0, 256)),
|
||||
)
|
||||
success = completed.returncode == 0
|
||||
if expect_success is not None and success != expect_success:
|
||||
raise GateFailure("INFRA_BLOCKED", f"fsck classification changed for {label}: exit {completed.returncode}")
|
||||
return {"exit": completed.returncode, "success": success}
|
||||
|
||||
|
||||
def evaluate_codec(
|
||||
codec: str,
|
||||
spec: dict[str, Any],
|
||||
source: bytes,
|
||||
source_dir: Path,
|
||||
oracle: Path,
|
||||
temp: Path,
|
||||
) -> dict[str, Any]:
|
||||
mkfs = SPEC["tools"]["mkfs.erofs"]["path"]
|
||||
fsck = SPEC["tools"]["fsck.erofs"]["path"]
|
||||
dump = SPEC["tools"]["dump.erofs"]["path"]
|
||||
images = []
|
||||
for pass_name in ("a", "b"):
|
||||
image = temp / f"{codec}-{pass_name}.erofs"
|
||||
run_logged([mkfs, *spec["mkfs_args"], str(image), str(source_dir)], temp, OUTPUT / f"logs/mkfs-{codec}-{pass_name}.log")
|
||||
images.append(image)
|
||||
hashes = [sha256_path(path) for path in images]
|
||||
if hashes != [spec["expected_image_sha256"]] * 2 or images[0].stat().st_size != spec["expected_image_size"]:
|
||||
raise GateFailure("INFRA_BLOCKED", f"{codec} image reproducibility changed: {hashes}")
|
||||
valid_fsck = fsck_image(fsck, images[0], temp / f"extract-{codec}-valid", f"{codec}-valid", True)
|
||||
extracted = (temp / f"extract-{codec}-valid/payload.bin").read_bytes()
|
||||
if extracted != source:
|
||||
raise GateFailure("INFRA_BLOCKED", f"{codec} legal image extraction mismatch")
|
||||
dumped = run_logged([dump, "--path=/payload.bin", "-e", str(images[0])], temp, OUTPUT / f"logs/dump-{codec}.log")
|
||||
extents = parse_extents(dumped.stdout)
|
||||
expected_extent = spec["extent"]
|
||||
selected = next((record for record in extents if record["index"] == expected_extent["index"]), None)
|
||||
if selected is None or selected != {key: expected_extent[key] for key in selected}:
|
||||
raise GateFailure("INFRA_BLOCKED", f"{codec} selected extent changed")
|
||||
image_bytes = images[0].read_bytes()
|
||||
block_start = selected["physical_offset"]
|
||||
block_end = block_start + selected["physical_length"]
|
||||
block = image_bytes[block_start:block_end]
|
||||
leading = next((index for index, value in enumerate(block) if value), len(block))
|
||||
stream = block[leading:]
|
||||
if leading != expected_extent["leading_zero_bytes"] or len(stream) != expected_extent["stream_bytes"]:
|
||||
raise GateFailure("INFRA_BLOCKED", f"{codec} leading padding or stream length changed")
|
||||
logical = source[selected["logical_offset"] : selected["logical_offset"] + selected["logical_length"]]
|
||||
full_record, full_output = decode(oracle, codec, stream, len(logical), True, spec["dict_size"], temp, f"{codec}-full")
|
||||
if full_policy_errno(full_record, len(stream), logical, full_output) != 0:
|
||||
raise GateFailure("STOP", f"{codec} legal mkfs extent is not exact after EROFS leading padding")
|
||||
|
||||
tail = bytes.fromhex(SPEC["fixture"]["tail_bytes_hex"])
|
||||
if leading <= len(tail):
|
||||
raise GateFailure("INFRA_BLOCKED", f"{codec} selected extent lacks mutation room")
|
||||
tail_stream = stream + tail
|
||||
tail_record, tail_output = decode(oracle, codec, tail_stream, len(logical), True, spec["dict_size"], temp, f"{codec}-tail")
|
||||
tail_errno = full_policy_errno(tail_record, len(tail_stream), logical, tail_output)
|
||||
if tail_errno != EINTEGRITY:
|
||||
raise GateFailure("STOP", f"{codec} cannot distinguish nonzero trailing garbage from a complete EROFS stream")
|
||||
tail_image_bytes = bytearray(image_bytes)
|
||||
shifted_start = block_start + leading - len(tail)
|
||||
tail_image_bytes[shifted_start:block_end] = tail_stream
|
||||
tail_image = temp / f"{codec}-tail.erofs"
|
||||
tail_image.write_bytes(tail_image_bytes)
|
||||
tail_fsck = fsck_image(
|
||||
fsck,
|
||||
tail_image,
|
||||
temp / f"extract-{codec}-tail",
|
||||
f"{codec}-tail",
|
||||
None,
|
||||
)
|
||||
if tail_fsck["success"]:
|
||||
tail_extracted = (temp / f"extract-{codec}-tail/payload.bin").read_bytes()
|
||||
if tail_extracted != source:
|
||||
raise GateFailure("INFRA_BLOCKED", f"{codec} permissive fsck tail output mismatch")
|
||||
|
||||
truncated_stream = stream[:-1]
|
||||
truncated_record, truncated_output = decode(
|
||||
oracle, codec, truncated_stream, len(logical), True, spec["dict_size"], temp, f"{codec}-truncated"
|
||||
)
|
||||
truncated_errno = full_policy_errno(truncated_record, len(truncated_stream), logical, truncated_output)
|
||||
if truncated_errno != EINTEGRITY:
|
||||
raise GateFailure("STOP", f"{codec} truncated stream reaches full success")
|
||||
truncated_image_bytes = bytearray(image_bytes)
|
||||
truncated_image_bytes[block_start + leading : block_end] = b"\0" + truncated_stream
|
||||
truncated_image = temp / f"{codec}-truncated.erofs"
|
||||
truncated_image.write_bytes(truncated_image_bytes)
|
||||
truncated_fsck = fsck_image(
|
||||
fsck,
|
||||
truncated_image,
|
||||
temp / f"extract-{codec}-truncated",
|
||||
f"{codec}-truncated",
|
||||
False,
|
||||
)
|
||||
|
||||
partial_size = min(4096, len(logical) // 4)
|
||||
partial_expected = logical[:partial_size]
|
||||
partial_record, partial_output = decode(
|
||||
oracle, codec, stream, partial_size, False, spec["dict_size"], temp, f"{codec}-partial"
|
||||
)
|
||||
if partial_policy_errno(partial_record, partial_expected, partial_output) != 0:
|
||||
raise GateFailure("STOP", f"{codec} partial output differs from the full slice")
|
||||
corruption_start = max(partial_record["consumed"] + 16, len(stream) - 64)
|
||||
if corruption_start >= len(stream):
|
||||
raise GateFailure("STOP", f"{codec} partial decode consumes the entire stream")
|
||||
corrupted_stream = stream[:corruption_start] + b"\0" * (len(stream) - corruption_start)
|
||||
if corrupted_stream == stream:
|
||||
raise GateFailure("INFRA_BLOCKED", f"{codec} corruption mutation changed no bytes")
|
||||
corrupt_partial_record, corrupt_partial_output = decode(
|
||||
oracle,
|
||||
codec,
|
||||
corrupted_stream,
|
||||
partial_size,
|
||||
False,
|
||||
spec["dict_size"],
|
||||
temp,
|
||||
f"{codec}-corrupt-partial",
|
||||
)
|
||||
if partial_policy_errno(corrupt_partial_record, partial_expected, corrupt_partial_output) != 0:
|
||||
raise GateFailure("STOP", f"{codec} range-after corruption changed the requested partial slice")
|
||||
corrupt_full_record, corrupt_full_output = decode(
|
||||
oracle,
|
||||
codec,
|
||||
corrupted_stream,
|
||||
len(logical),
|
||||
True,
|
||||
spec["dict_size"],
|
||||
temp,
|
||||
f"{codec}-corrupt-full",
|
||||
)
|
||||
corrupt_full_errno = full_policy_errno(
|
||||
corrupt_full_record, len(corrupted_stream), logical, corrupt_full_output
|
||||
)
|
||||
if corrupt_full_errno != EINTEGRITY:
|
||||
raise GateFailure("STOP", f"{codec} full read does not detect range-after corruption")
|
||||
corrupt_image_bytes = bytearray(image_bytes)
|
||||
corrupt_image_bytes[block_start + leading : block_end] = corrupted_stream
|
||||
corrupt_image = temp / f"{codec}-corrupt.erofs"
|
||||
corrupt_image.write_bytes(corrupt_image_bytes)
|
||||
corrupt_fsck = fsck_image(
|
||||
fsck,
|
||||
corrupt_image,
|
||||
temp / f"extract-{codec}-corrupt",
|
||||
f"{codec}-corrupt",
|
||||
False,
|
||||
)
|
||||
|
||||
return {
|
||||
"codec": codec,
|
||||
"corruption": {
|
||||
"full_errno": corrupt_full_errno,
|
||||
"fsck": corrupt_fsck,
|
||||
"starts_after_partial_consumed": corruption_start > partial_record["consumed"],
|
||||
"starts_at_stream_byte": corruption_start,
|
||||
},
|
||||
"extent": {**selected, "leading_zero_bytes": leading, "stream_bytes": len(stream)},
|
||||
"full": {**full_record, "policy_errno": 0, "output_sha256": sha256_bytes(full_output)},
|
||||
"image_repeated_sha256": hashes,
|
||||
"legal_fsck": valid_fsck,
|
||||
"partial": {
|
||||
**partial_record,
|
||||
"corrupt_policy_errno": 0,
|
||||
"output_matches_full_slice": True,
|
||||
"policy_errno": 0,
|
||||
"requested_bytes": partial_size,
|
||||
},
|
||||
"tail": {
|
||||
**tail_record,
|
||||
"bytes": len(tail),
|
||||
"fsck": tail_fsck,
|
||||
"nonzero": True,
|
||||
"policy_errno": tail_errno,
|
||||
},
|
||||
"truncated": {**truncated_record, "fsck": truncated_fsck, "policy_errno": truncated_errno},
|
||||
}
|
||||
|
||||
|
||||
def finalize() -> None:
|
||||
lines = []
|
||||
for path in sorted(OUTPUT.rglob("*")):
|
||||
if path.is_file() and path.name != "SHA256SUMS":
|
||||
lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
|
||||
(OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
|
||||
|
||||
|
||||
result: dict[str, Any] | None = None
|
||||
exit_code = 0
|
||||
owned_temp: str | None = None
|
||||
try:
|
||||
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-083" or SPEC.get("gate") != "G04":
|
||||
raise GateFailure("INFRA_BLOCKED", "invalid P15-083 input schema")
|
||||
resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
|
||||
if resolved != SPEC["required_base"]:
|
||||
raise GateFailure("INFRA_BLOCKED", f"P15-083 must replay {SPEC['required_base']}, got {resolved}")
|
||||
sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]}
|
||||
hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()}
|
||||
if hashes != SPEC["source_sha256"]:
|
||||
raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed")
|
||||
write_json(OUTPUT / "source-sha256.json", hashes)
|
||||
freebsd_head = subprocess.check_output(["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True).strip()
|
||||
freebsd_hashes = {path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"]}
|
||||
if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]:
|
||||
raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed")
|
||||
errno_source = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8")
|
||||
if "#define\tEINTEGRITY\t97" not in errno_source:
|
||||
raise GateFailure("INFRA_BLOCKED", "FreeBSD positive EINTEGRITY changed")
|
||||
write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes})
|
||||
write_json(OUTPUT / "toolchain.json", verify_tools())
|
||||
write_json(OUTPUT / "source-contract.json", verify_source_contract(sources))
|
||||
with tempfile.TemporaryDirectory(prefix="p15-083-g04-") as temporary:
|
||||
owned_temp = temporary
|
||||
temp = Path(temporary)
|
||||
source_dir = temp / "source"
|
||||
source = make_source(source_dir)
|
||||
oracle, library_record = compile_oracle(temp)
|
||||
write_json(OUTPUT / "independent-libraries.json", library_record)
|
||||
records = [
|
||||
evaluate_codec(codec, codec_spec, source, source_dir, oracle, temp)
|
||||
for codec, codec_spec in sorted(SPEC["codecs"].items())
|
||||
]
|
||||
if {record["codec"] for record in records} != {"deflate", "lzma", "zstd"}:
|
||||
raise GateFailure("STOP", "P15-083 requires all three non-LZ4 codec policies")
|
||||
write_json(OUTPUT / "codec-results.json", records)
|
||||
result = {
|
||||
"b27": "AUTHORIZED",
|
||||
"candidate": "P15-083",
|
||||
"cleanup": "PASS",
|
||||
"codecs": {record["codec"]: "GO" for record in records},
|
||||
"full_feature_suite": "NOT_RUN",
|
||||
"gate": "G04",
|
||||
"oracle": "real erofs-utils 1.8.6 images + liblzma/zlib/libzstd consumed-byte oracle",
|
||||
"policy": {
|
||||
"deflate": "strip EROFS leading zero padding; full raw stream must reach Z_STREAM_END with no unread bytes",
|
||||
"lzma": "strip EROFS leading zero padding; MicroLZMA compressed size is exact and all bytes must be consumed",
|
||||
"zstd": "strip EROFS leading zero padding; one frame must complete with no unread bytes",
|
||||
},
|
||||
"qemu": "NOT_RUN",
|
||||
"qemu_reason": "Stage0 policy and compatibility oracle is complete on real host fixtures; B27 acceptance owns TC176 QEMU",
|
||||
"requested_base": REQUESTED_BASE,
|
||||
"resolved_base": resolved,
|
||||
"schema": 1,
|
||||
"status": "GO",
|
||||
"typed_errno": "PASS",
|
||||
}
|
||||
write_json(OUTPUT / "result.json", result)
|
||||
cleanup_record = {
|
||||
"owned_temp": owned_temp,
|
||||
"owned_temp_removed": owned_temp is not None and not Path(owned_temp).exists(),
|
||||
"protected_pid_touched": False,
|
||||
"protected_port_touched": False,
|
||||
"qemu_started": False,
|
||||
"shared_base_image_touched": False,
|
||||
}
|
||||
if not cleanup_record["owned_temp_removed"]:
|
||||
raise GateFailure("INFRA_BLOCKED", "owned gate temporary directory survived cleanup")
|
||||
write_json(OUTPUT / "owned-cleanup.json", cleanup_record)
|
||||
except GateFailure as failure:
|
||||
result = {
|
||||
"b27": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN",
|
||||
"candidate": "P15-083",
|
||||
"full_feature_suite": "NOT_RUN",
|
||||
"gate": "G04",
|
||||
"qemu": "NOT_RUN",
|
||||
"reason": failure.reason,
|
||||
"requested_base": REQUESTED_BASE,
|
||||
"schema": 1,
|
||||
"status": failure.status,
|
||||
}
|
||||
write_json(OUTPUT / "result.json", result)
|
||||
exit_code = 1 if failure.status == "STOP" else 21
|
||||
except (OSError, subprocess.SubprocessError, ValueError) as failure:
|
||||
result = {
|
||||
"b27": "NOT_RUN",
|
||||
"candidate": "P15-083",
|
||||
"full_feature_suite": "NOT_RUN",
|
||||
"gate": "G04",
|
||||
"qemu": "NOT_RUN",
|
||||
"reason": f"gate infrastructure failure: {failure}",
|
||||
"requested_base": REQUESTED_BASE,
|
||||
"schema": 1,
|
||||
"status": "INFRA_BLOCKED",
|
||||
}
|
||||
write_json(OUTPUT / "result.json", result)
|
||||
exit_code = 21
|
||||
finally:
|
||||
finalize()
|
||||
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
raise SystemExit(exit_code)
|
||||
PY
|
||||
Reference in New Issue
Block a user