1446 lines
53 KiB
Bash
Executable File
1446 lines
53 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
|
input=$gate_dir/P15-086-input.json
|
|
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
|
|
base=
|
|
output=
|
|
|
|
while test "$#" -gt 0; do
|
|
case "$1" in
|
|
--base)
|
|
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
|
base=$2
|
|
shift 2
|
|
;;
|
|
--output)
|
|
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
|
output=$2
|
|
shift 2
|
|
;;
|
|
*)
|
|
printf 'unknown argument: %s\n' "$1" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
|
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
|
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
|
|
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
|
|
for tool in autoreconf cc dump.erofs fsck.erofs git make mkfs.erofs pkg-config python3 sha256sum; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'missing required host tool: %s\n' "$tool" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
case "$output" in
|
|
/*) ;;
|
|
*) output=$PWD/$output ;;
|
|
esac
|
|
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
|
mkdir -p "$output"
|
|
|
|
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(sys.argv[1])
|
|
INPUT = Path(sys.argv[2])
|
|
REQUESTED_BASE = sys.argv[3]
|
|
OUTPUT = Path(sys.argv[4])
|
|
FREEBSD_SRC = Path(sys.argv[5])
|
|
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
|
|
EINTEGRITY = 97
|
|
EOVERFLOW = 84
|
|
|
|
|
|
class GateFailure(Exception):
|
|
def __init__(self, status: str, reason: str):
|
|
super().__init__(reason)
|
|
self.status = status
|
|
self.reason = reason
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_path(path: Path) -> str:
|
|
return sha256_bytes(path.read_bytes())
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
|
|
|
|
|
def git(*args: str) -> str:
|
|
return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip()
|
|
|
|
|
|
def source_at(commit: str, path: str) -> str:
|
|
completed = subprocess.run(
|
|
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
)
|
|
if completed.returncode != 0:
|
|
raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}")
|
|
return completed.stdout
|
|
|
|
|
|
def run_logged(
|
|
argv: list[str], cwd: Path, log: Path, timeout: int = 120,
|
|
expected: set[int] | None = None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
try:
|
|
completed = subprocess.run(
|
|
argv,
|
|
cwd=cwd,
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
except subprocess.TimeoutExpired as failure:
|
|
log.parent.mkdir(parents=True, exist_ok=True)
|
|
log.write_text(
|
|
"$ " + " ".join(argv) + f"\n[TIMEOUT after {timeout}s]\n",
|
|
encoding="utf-8",
|
|
)
|
|
raise GateFailure("INFRA_BLOCKED", f"command timed out: {' '.join(argv)}") from failure
|
|
log.parent.mkdir(parents=True, exist_ok=True)
|
|
log.write_text(
|
|
"$ " + " ".join(argv) + "\n" + completed.stdout +
|
|
f"\n[exit {completed.returncode}]\n",
|
|
encoding="utf-8",
|
|
)
|
|
allowed = {0} if expected is None else expected
|
|
if completed.returncode not in allowed:
|
|
raise GateFailure(
|
|
"INFRA_BLOCKED",
|
|
f"command failed ({completed.returncode}): {' '.join(argv)}",
|
|
)
|
|
return completed
|
|
|
|
|
|
EXTENT_RE = re.compile(
|
|
r"^\s*(\d+):\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:\s*"
|
|
r"(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$",
|
|
re.MULTILINE,
|
|
)
|
|
|
|
|
|
def parse_extents(text: str) -> list[dict[str, int]]:
|
|
records = []
|
|
for match in EXTENT_RE.finditer(text):
|
|
index, logical, logical_end, logical_length, physical, physical_end, physical_length = map(
|
|
int, match.groups()
|
|
)
|
|
if logical_end - logical != logical_length or physical_end - physical != physical_length:
|
|
raise GateFailure("INFRA_BLOCKED", "dump.erofs extent arithmetic changed")
|
|
records.append(
|
|
{
|
|
"index": index,
|
|
"logical_length": logical_length,
|
|
"logical_offset": logical,
|
|
"physical_length": physical_length,
|
|
"physical_offset": physical,
|
|
}
|
|
)
|
|
if not records:
|
|
raise GateFailure("INFRA_BLOCKED", "dump.erofs returned no extents")
|
|
return records
|
|
|
|
|
|
def make_stream_source(path: Path) -> bytes:
|
|
path.mkdir(parents=True)
|
|
content = b"".join(
|
|
f"P15-086-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode("ascii")
|
|
for index in range(SPEC["stream"]["source"]["line_count"])
|
|
)
|
|
expected = SPEC["stream"]["source"]
|
|
if len(content) != expected["size"] or sha256_bytes(content) != expected["sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "stream source identity changed")
|
|
payload = path / "payload.bin"
|
|
payload.write_bytes(content)
|
|
os.utime(payload, (0, 0))
|
|
os.utime(path, (0, 0))
|
|
return content
|
|
|
|
|
|
def make_lz4_source(path: Path) -> bytes:
|
|
path.mkdir(parents=True)
|
|
content = b"".join(
|
|
bytes([segment["byte"]]) * segment["length"]
|
|
for segment in SPEC["lz4"]["source"]["segments"]
|
|
)
|
|
expected = SPEC["lz4"]["source"]
|
|
if len(content) != expected["size"] or sha256_bytes(content) != expected["sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "LZ4 source identity changed")
|
|
payload = path / "payload.bin"
|
|
payload.write_bytes(content)
|
|
os.utime(payload, (0, 0))
|
|
os.utime(path, (0, 0))
|
|
return content
|
|
|
|
|
|
def raw_lz4_decode(data: bytes, target: int) -> tuple[bytes, int]:
|
|
ip = 0
|
|
output = bytearray()
|
|
while ip < len(data):
|
|
token = data[ip]
|
|
ip += 1
|
|
literal_length = token >> 4
|
|
if literal_length == 15:
|
|
while True:
|
|
if ip >= len(data):
|
|
raise GateFailure("STOP", "independent LZ4 parser saw truncated literal length")
|
|
value = data[ip]
|
|
ip += 1
|
|
literal_length += value
|
|
if value != 255:
|
|
break
|
|
if ip + literal_length > len(data):
|
|
raise GateFailure("STOP", "independent LZ4 parser saw truncated literals")
|
|
output.extend(data[ip : ip + literal_length])
|
|
ip += literal_length
|
|
if len(output) >= target:
|
|
return bytes(output[:target]), ip
|
|
if ip == len(data):
|
|
break
|
|
if ip + 2 > len(data):
|
|
raise GateFailure("STOP", "independent LZ4 parser saw truncated match offset")
|
|
offset = data[ip] | data[ip + 1] << 8
|
|
ip += 2
|
|
if offset == 0 or offset > len(output):
|
|
raise GateFailure("STOP", "independent LZ4 parser saw invalid match offset")
|
|
match_length = token & 15
|
|
if match_length == 15:
|
|
while True:
|
|
if ip >= len(data):
|
|
raise GateFailure("STOP", "independent LZ4 parser saw truncated match length")
|
|
value = data[ip]
|
|
ip += 1
|
|
match_length += value
|
|
if value != 255:
|
|
break
|
|
for _ in range(match_length + 4):
|
|
output.append(output[-offset])
|
|
if len(output) >= target:
|
|
return bytes(output[:target]), ip
|
|
raise GateFailure("STOP", "independent LZ4 parser ended before requested output")
|
|
|
|
|
|
ORACLE_SOURCE = r'''
|
|
#define _POSIX_C_SOURCE 200809L
|
|
#define ZSTD_STATIC_LINKING_ONLY
|
|
#include <errno.h>
|
|
#include <fcntl.h>
|
|
#include <inttypes.h>
|
|
#include <lz4.h>
|
|
#include <lzma.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/mman.h>
|
|
#include <sys/stat.h>
|
|
#include <time.h>
|
|
#include <unistd.h>
|
|
#include <zlib.h>
|
|
#include <zstd.h>
|
|
|
|
struct allocation {
|
|
size_t size;
|
|
};
|
|
|
|
struct tracker {
|
|
size_t current;
|
|
size_t peak;
|
|
};
|
|
|
|
struct result {
|
|
size_t consumed;
|
|
size_t produced;
|
|
size_t status;
|
|
size_t workspace_peak;
|
|
int cleanup;
|
|
int codec_error;
|
|
int stream_end;
|
|
};
|
|
|
|
static void
|
|
fail(const char *message)
|
|
{
|
|
fprintf(stderr, "%s\n", message);
|
|
exit(2);
|
|
}
|
|
|
|
static void *
|
|
tracked_alloc(struct tracker *tracker, size_t size)
|
|
{
|
|
struct allocation *allocation;
|
|
|
|
if (size > SIZE_MAX - sizeof(*allocation))
|
|
return NULL;
|
|
allocation = malloc(sizeof(*allocation) + size);
|
|
if (allocation == NULL)
|
|
return NULL;
|
|
allocation->size = size;
|
|
tracker->current += size;
|
|
if (tracker->current > tracker->peak)
|
|
tracker->peak = tracker->current;
|
|
return allocation + 1;
|
|
}
|
|
|
|
static void
|
|
tracked_free(struct tracker *tracker, void *address)
|
|
{
|
|
struct allocation *allocation;
|
|
|
|
if (address == NULL)
|
|
return;
|
|
allocation = (struct allocation *)address - 1;
|
|
if (allocation->size > tracker->current)
|
|
fail("allocation tracker underflow");
|
|
tracker->current -= allocation->size;
|
|
free(allocation);
|
|
}
|
|
|
|
static voidpf
|
|
zalloc_tracked(voidpf opaque, uInt items, uInt size)
|
|
{
|
|
if (items != 0 && size > SIZE_MAX / items)
|
|
return NULL;
|
|
return tracked_alloc(opaque, (size_t)items * size);
|
|
}
|
|
|
|
static void
|
|
zfree_tracked(voidpf opaque, voidpf address)
|
|
{
|
|
tracked_free(opaque, address);
|
|
}
|
|
|
|
static void *
|
|
lzma_alloc_tracked(void *opaque, size_t items, size_t size)
|
|
{
|
|
if (items != 0 && size > SIZE_MAX / items)
|
|
return NULL;
|
|
return tracked_alloc(opaque, items * size);
|
|
}
|
|
|
|
static void
|
|
lzma_free_tracked(void *opaque, void *address)
|
|
{
|
|
tracked_free(opaque, address);
|
|
}
|
|
|
|
static void *
|
|
zstd_alloc_tracked(void *opaque, size_t size)
|
|
{
|
|
return tracked_alloc(opaque, size);
|
|
}
|
|
|
|
static void
|
|
zstd_free_tracked(void *opaque, void *address)
|
|
{
|
|
tracked_free(opaque, address);
|
|
}
|
|
|
|
static unsigned char *
|
|
read_file(const char *path, size_t *sizep)
|
|
{
|
|
struct stat st;
|
|
unsigned char *data;
|
|
ssize_t amount;
|
|
size_t done;
|
|
int fd;
|
|
|
|
fd = open(path, O_RDONLY);
|
|
if (fd < 0 || fstat(fd, &st) != 0 || st.st_size <= 0)
|
|
fail("cannot open oracle input");
|
|
*sizep = (size_t)st.st_size;
|
|
data = malloc(*sizep);
|
|
if (data == NULL)
|
|
fail("cannot allocate oracle input");
|
|
done = 0;
|
|
while (done < *sizep) {
|
|
amount = read(fd, data + done, *sizep - done);
|
|
if (amount <= 0)
|
|
fail("cannot read oracle input");
|
|
done += (size_t)amount;
|
|
}
|
|
close(fd);
|
|
return data;
|
|
}
|
|
|
|
static void
|
|
write_file(const char *path, const unsigned char *data, size_t size)
|
|
{
|
|
ssize_t amount;
|
|
size_t done;
|
|
int fd;
|
|
|
|
fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
|
if (fd < 0)
|
|
fail("cannot create oracle output");
|
|
done = 0;
|
|
while (done < size) {
|
|
amount = write(fd, data + done, size - done);
|
|
if (amount <= 0)
|
|
fail("cannot write oracle output");
|
|
done += (size_t)amount;
|
|
}
|
|
close(fd);
|
|
}
|
|
|
|
static struct result
|
|
decode_once(const char *codec, const unsigned char *input, size_t input_size,
|
|
unsigned char *output, size_t output_size, int full, uint32_t dict_size)
|
|
{
|
|
struct result result = { 0 };
|
|
struct tracker tracker = { 0 };
|
|
|
|
if (strcmp(codec, "lz4") == 0) {
|
|
int ret;
|
|
|
|
if (full)
|
|
ret = LZ4_decompress_safe((const char *)input, (char *)output,
|
|
(int)input_size, (int)output_size);
|
|
else
|
|
ret = LZ4_decompress_safe_partial((const char *)input,
|
|
(char *)output, (int)input_size, (int)output_size,
|
|
(int)output_size);
|
|
if (ret < 0) {
|
|
result.codec_error = 1;
|
|
} else {
|
|
result.produced = (size_t)ret;
|
|
result.stream_end = full && result.produced == output_size;
|
|
}
|
|
result.consumed = full && !result.codec_error ? input_size : 0;
|
|
result.status = ret < 0 ? (size_t)-ret : 0;
|
|
result.cleanup = 1;
|
|
} else if (strcmp(codec, "deflate") == 0) {
|
|
z_stream stream;
|
|
int ret = Z_OK, endret;
|
|
|
|
memset(&stream, 0, sizeof(stream));
|
|
stream.zalloc = zalloc_tracked;
|
|
stream.zfree = zfree_tracked;
|
|
stream.opaque = &tracker;
|
|
stream.next_in = (Bytef *)(uintptr_t)input;
|
|
stream.avail_in = (uInt)input_size;
|
|
stream.next_out = output;
|
|
stream.avail_out = (uInt)output_size;
|
|
ret = inflateInit2(&stream, -15);
|
|
if (ret != Z_OK) {
|
|
result.codec_error = 1;
|
|
} else {
|
|
while (stream.avail_out != 0) {
|
|
uInt in_before = stream.avail_in;
|
|
uInt out_before = stream.avail_out;
|
|
ret = inflate(&stream, Z_SYNC_FLUSH);
|
|
if (ret == Z_STREAM_END) {
|
|
result.stream_end = 1;
|
|
break;
|
|
}
|
|
if (ret != Z_OK || (stream.avail_in == in_before &&
|
|
stream.avail_out == out_before)) {
|
|
result.codec_error = 1;
|
|
break;
|
|
}
|
|
}
|
|
result.consumed = input_size - stream.avail_in;
|
|
result.produced = output_size - stream.avail_out;
|
|
result.status = (size_t)(unsigned int)ret;
|
|
endret = inflateEnd(&stream);
|
|
result.cleanup = endret == Z_OK;
|
|
}
|
|
} else if (strcmp(codec, "lzma") == 0) {
|
|
lzma_stream stream = LZMA_STREAM_INIT;
|
|
lzma_allocator allocator = {
|
|
.alloc = lzma_alloc_tracked,
|
|
.free = lzma_free_tracked,
|
|
.opaque = &tracker,
|
|
};
|
|
lzma_ret ret;
|
|
|
|
stream.allocator = &allocator;
|
|
ret = lzma_microlzma_decoder(&stream, input_size, output_size,
|
|
full != 0, dict_size);
|
|
if (ret != LZMA_OK) {
|
|
result.codec_error = 1;
|
|
} else {
|
|
stream.next_in = input;
|
|
stream.avail_in = input_size;
|
|
stream.next_out = output;
|
|
stream.avail_out = output_size;
|
|
while (stream.avail_out != 0) {
|
|
size_t in_before = stream.avail_in;
|
|
size_t out_before = stream.avail_out;
|
|
ret = lzma_code(&stream, LZMA_RUN);
|
|
if (ret == LZMA_STREAM_END) {
|
|
result.stream_end = 1;
|
|
break;
|
|
}
|
|
if (ret != LZMA_OK || (stream.avail_in == in_before &&
|
|
stream.avail_out == out_before)) {
|
|
result.codec_error = 1;
|
|
break;
|
|
}
|
|
}
|
|
result.consumed = input_size - stream.avail_in;
|
|
result.produced = output_size - stream.avail_out;
|
|
result.status = ret;
|
|
}
|
|
lzma_end(&stream);
|
|
result.cleanup = tracker.current == 0;
|
|
} else if (strcmp(codec, "zstd") == 0) {
|
|
ZSTD_customMem memory = {
|
|
.customAlloc = zstd_alloc_tracked,
|
|
.customFree = zstd_free_tracked,
|
|
.opaque = &tracker,
|
|
};
|
|
ZSTD_DCtx *context;
|
|
ZSTD_inBuffer in_buffer;
|
|
ZSTD_outBuffer out_buffer;
|
|
size_t ret = 1;
|
|
|
|
context = ZSTD_createDCtx_advanced(memory);
|
|
if (context == NULL) {
|
|
result.codec_error = 1;
|
|
} else {
|
|
in_buffer = (ZSTD_inBuffer){ input, input_size, 0 };
|
|
out_buffer = (ZSTD_outBuffer){ output, output_size, 0 };
|
|
while (out_buffer.pos != out_buffer.size) {
|
|
size_t in_before = in_buffer.pos;
|
|
size_t out_before = out_buffer.pos;
|
|
ret = ZSTD_decompressStream(context, &out_buffer, &in_buffer);
|
|
if (ZSTD_isError(ret)) {
|
|
result.codec_error = 1;
|
|
break;
|
|
}
|
|
if (ret == 0) {
|
|
result.stream_end = 1;
|
|
break;
|
|
}
|
|
if (in_buffer.pos == in_before && out_buffer.pos == out_before) {
|
|
result.codec_error = 1;
|
|
break;
|
|
}
|
|
}
|
|
result.consumed = in_buffer.pos;
|
|
result.produced = out_buffer.pos;
|
|
result.status = ret;
|
|
if (ZSTD_isError(ZSTD_freeDCtx(context)))
|
|
result.cleanup = 0;
|
|
else
|
|
result.cleanup = tracker.current == 0;
|
|
}
|
|
} else {
|
|
fail("unknown codec");
|
|
}
|
|
result.workspace_peak = tracker.peak;
|
|
return result;
|
|
}
|
|
|
|
static uint64_t
|
|
elapsed_ns(const struct timespec *start, const struct timespec *end)
|
|
{
|
|
return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL +
|
|
(uint64_t)(end->tv_nsec - start->tv_nsec);
|
|
}
|
|
|
|
int
|
|
main(int argc, char **argv)
|
|
{
|
|
unsigned char *input, *mapping, *output;
|
|
struct result result = { 0 }, current;
|
|
struct timespec start, end;
|
|
size_t input_size, output_size, usable, page_size;
|
|
uint32_t dict_size;
|
|
uint64_t cpu_ns;
|
|
int full, guards = 1, iterations;
|
|
|
|
if (argc != 8)
|
|
fail("usage: oracle CODEC INPUT OUTPUT OUTPUT_SIZE FULL DICT ITERATIONS");
|
|
output_size = (size_t)strtoull(argv[4], NULL, 10);
|
|
full = atoi(argv[5]);
|
|
dict_size = (uint32_t)strtoul(argv[6], NULL, 10);
|
|
iterations = atoi(argv[7]);
|
|
if (output_size == 0 || iterations <= 0)
|
|
fail("invalid output size or iteration count");
|
|
input = read_file(argv[2], &input_size);
|
|
page_size = (size_t)sysconf(_SC_PAGESIZE);
|
|
usable = (output_size + page_size - 1) & ~(page_size - 1);
|
|
mapping = mmap(NULL, usable + 2 * page_size, PROT_READ | PROT_WRITE,
|
|
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
|
if (mapping == MAP_FAILED)
|
|
fail("cannot allocate guarded output");
|
|
if (mprotect(mapping, page_size, PROT_NONE) != 0 ||
|
|
mprotect(mapping + page_size + usable, page_size, PROT_NONE) != 0)
|
|
fail("cannot protect output guards");
|
|
output = mapping + page_size;
|
|
if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start) != 0)
|
|
fail("cannot start CPU clock");
|
|
for (int iteration = 0; iteration < iterations; ++iteration) {
|
|
memset(output, 0xa5, usable);
|
|
current = decode_once(argv[1], input, input_size, output, output_size,
|
|
full, dict_size);
|
|
if (iteration == 0)
|
|
result = current;
|
|
else if (current.codec_error != result.codec_error ||
|
|
current.cleanup != result.cleanup ||
|
|
current.consumed != result.consumed ||
|
|
current.produced != result.produced ||
|
|
current.stream_end != result.stream_end)
|
|
fail("decoder result changed across iterations");
|
|
if (current.workspace_peak > result.workspace_peak)
|
|
result.workspace_peak = current.workspace_peak;
|
|
}
|
|
if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end) != 0)
|
|
fail("cannot stop CPU clock");
|
|
cpu_ns = elapsed_ns(&start, &end);
|
|
for (size_t index = output_size; index < usable; ++index) {
|
|
if (output[index] != 0xa5) {
|
|
guards = 0;
|
|
break;
|
|
}
|
|
}
|
|
write_file(argv[3], output, result.produced);
|
|
printf("cleanup=%d codec_error=%d consumed=%zu cpu_ns=%" PRIu64
|
|
" guards=%d iterations=%d produced=%zu status=%zu stream_end=%d "
|
|
"workspace_peak=%zu\n", result.cleanup, result.codec_error,
|
|
result.consumed, cpu_ns, guards, iterations, result.produced,
|
|
result.status, result.stream_end, result.workspace_peak);
|
|
munmap(mapping, usable + 2 * page_size);
|
|
free(input);
|
|
return 0;
|
|
}
|
|
'''
|
|
|
|
|
|
def compile_oracle(temp: Path) -> tuple[Path, dict[str, Any]]:
|
|
source = temp / "p15-086-oracle.c"
|
|
binary = temp / "p15-086-oracle"
|
|
source.write_text(ORACLE_SOURCE, encoding="ascii")
|
|
run_logged(
|
|
[
|
|
"cc", "-O2", "-std=c17", "-Wall", "-Wextra", "-Werror",
|
|
str(source), "-o", str(binary), "-llz4", "-llzma", "-lz", "-lzstd",
|
|
],
|
|
temp,
|
|
OUTPUT / "logs/oracle-build.log",
|
|
)
|
|
versions = {
|
|
name: subprocess.check_output(["pkg-config", "--modversion", package], text=True).strip()
|
|
for name, package in (
|
|
("liblz4", "liblz4"),
|
|
("liblzma", "liblzma"),
|
|
("zlib", "zlib"),
|
|
("libzstd", "libzstd"),
|
|
)
|
|
}
|
|
if versions != SPEC["libraries"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"independent library versions changed: {versions}")
|
|
return binary, {
|
|
"binary_sha256": sha256_path(binary),
|
|
"source_sha256": sha256_path(source),
|
|
**versions,
|
|
}
|
|
|
|
|
|
def decode(
|
|
oracle: Path,
|
|
codec: str,
|
|
data: bytes,
|
|
output_size: int,
|
|
full: bool,
|
|
dict_size: int,
|
|
iterations: int,
|
|
temp: Path,
|
|
label: str,
|
|
) -> tuple[dict[str, int], bytes]:
|
|
input_path = temp / f"{label}.input"
|
|
output_path = temp / f"{label}.output"
|
|
input_path.write_bytes(data)
|
|
completed = run_logged(
|
|
[
|
|
str(oracle), codec, str(input_path), str(output_path), str(output_size),
|
|
"1" if full else "0", str(dict_size), str(iterations),
|
|
],
|
|
temp,
|
|
OUTPUT / f"logs/oracle-{label}.log",
|
|
)
|
|
record: dict[str, int] = {}
|
|
for field in completed.stdout.strip().split():
|
|
key, value = field.split("=", 1)
|
|
record[key] = int(value)
|
|
required = {
|
|
"cleanup", "codec_error", "consumed", "cpu_ns", "guards", "iterations",
|
|
"produced", "status", "stream_end", "workspace_peak",
|
|
}
|
|
if set(record) != required:
|
|
raise GateFailure("INFRA_BLOCKED", f"oracle fields changed for {label}")
|
|
return record, output_path.read_bytes()
|
|
|
|
|
|
def full_errno(record: dict[str, int], data_size: int, expected: bytes, output: bytes) -> int:
|
|
if (
|
|
record["codec_error"] != 0
|
|
or record["cleanup"] != 1
|
|
or record["guards"] != 1
|
|
or record["consumed"] != data_size
|
|
or record["produced"] != len(expected)
|
|
or record["stream_end"] != 1
|
|
or output != expected
|
|
):
|
|
return EINTEGRITY
|
|
return 0
|
|
|
|
|
|
def partial_errno(record: dict[str, int], expected: bytes, output: bytes) -> int:
|
|
if (
|
|
record["codec_error"] != 0
|
|
or record["cleanup"] != 1
|
|
or record["guards"] != 1
|
|
or record["produced"] != len(expected)
|
|
or output != expected
|
|
):
|
|
return EINTEGRITY
|
|
return 0
|
|
|
|
|
|
def fsck_extract(fsck: Path, image: Path, destination: Path, label: str) -> dict[str, Any]:
|
|
destination.mkdir()
|
|
completed = run_logged(
|
|
[str(fsck), f"--extract={destination}", str(image)],
|
|
image.parent,
|
|
OUTPUT / f"logs/fsck-{label}.log",
|
|
expected=set(range(0, 256)),
|
|
)
|
|
return {"exit": completed.returncode, "success": completed.returncode == 0}
|
|
|
|
|
|
def select_extent(
|
|
dump: Path, image: Path, payload_path: str, minimum: int, label: str,
|
|
) -> dict[str, int]:
|
|
completed = run_logged(
|
|
[str(dump), f"--path={payload_path}", "-e", str(image)],
|
|
image.parent,
|
|
OUTPUT / f"logs/dump-{label}.log",
|
|
)
|
|
choices = [
|
|
record for record in parse_extents(completed.stdout)
|
|
if record["logical_length"] >= minimum and record["physical_length"] > 0
|
|
]
|
|
if not choices:
|
|
raise GateFailure("STOP", f"{label} has no representative compressed extent")
|
|
return max(choices, key=lambda record: record["logical_length"])
|
|
|
|
|
|
def dump_extents_compat(dump: Path, image: Path, label: str) -> tuple[list[dict[str, int]], dict[str, Any]]:
|
|
attempts = []
|
|
modern = subprocess.run(
|
|
[str(dump), "--path=/payload.bin", "-e", str(image)],
|
|
cwd=image.parent,
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
attempts.append(
|
|
"$ " + " ".join([str(dump), "--path=/payload.bin", "-e", str(image)])
|
|
+ "\n" + modern.stdout + f"\n[exit {modern.returncode}]\n"
|
|
)
|
|
if modern.returncode == 0:
|
|
records = parse_extents(modern.stdout)
|
|
(OUTPUT / f"logs/dump-{label}.log").write_text("\n".join(attempts), encoding="utf-8")
|
|
return records, {"mode": "path", "nid": None}
|
|
matches = []
|
|
for nid in range(1, 256):
|
|
completed = subprocess.run(
|
|
[str(dump), f"--nid={nid}", "-e", str(image)],
|
|
cwd=image.parent,
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
if completed.returncode == 0 and re.search(r"^(?:File|Path) : /payload\.bin$", completed.stdout, re.MULTILINE):
|
|
matches.append((nid, completed.stdout))
|
|
if len(matches) != 1:
|
|
(OUTPUT / f"logs/dump-{label}.log").write_text("\n".join(attempts), encoding="utf-8")
|
|
raise GateFailure("INFRA_BLOCKED", f"{label} bounded NID lookup found {len(matches)} payloads")
|
|
nid, stdout = matches[0]
|
|
attempts.append(
|
|
"$ " + " ".join([str(dump), f"--nid={nid}", "-e", str(image)])
|
|
+ "\n" + stdout + "\n[exit 0]\n"
|
|
)
|
|
(OUTPUT / f"logs/dump-{label}.log").write_text("\n".join(attempts), encoding="utf-8")
|
|
return parse_extents(stdout), {"mode": "bounded-nid", "nid": nid}
|
|
|
|
|
|
def benchmark(
|
|
oracle: Path, codec: str, stream: bytes, logical: bytes, dict_size: int,
|
|
temp: Path, label: str,
|
|
) -> dict[str, Any]:
|
|
samples = SPEC["cpu"]["samples"]
|
|
iterations = SPEC["cpu"]["iterations_per_sample"]
|
|
partial_end = SPEC["ranges"][0]["length"]
|
|
full_samples = []
|
|
partial_samples = []
|
|
full_peak = 0
|
|
partial_peak = 0
|
|
for sample in range(samples):
|
|
full_record, full_output = decode(
|
|
oracle, codec, stream, len(logical), True, dict_size, iterations,
|
|
temp, f"{label}-cpu-full-{sample}",
|
|
)
|
|
partial_record, partial_output = decode(
|
|
oracle, codec, stream, partial_end, False, dict_size, iterations,
|
|
temp, f"{label}-cpu-partial-{sample}",
|
|
)
|
|
if full_errno(full_record, len(stream), logical, full_output) != 0:
|
|
raise GateFailure("STOP", f"{label} full CPU sample lost exactness")
|
|
if partial_errno(partial_record, logical[:partial_end], partial_output) != 0:
|
|
raise GateFailure("STOP", f"{label} partial CPU sample lost exactness")
|
|
full_samples.append(full_record["cpu_ns"] / iterations)
|
|
partial_samples.append(partial_record["cpu_ns"] / iterations)
|
|
full_peak = max(full_peak, full_record["workspace_peak"])
|
|
partial_peak = max(partial_peak, partial_record["workspace_peak"])
|
|
full_median = statistics.median(full_samples)
|
|
partial_median = statistics.median(partial_samples)
|
|
return {
|
|
"full_ns_per_decode": full_samples,
|
|
"full_median_ns": full_median,
|
|
"iterations_per_sample": iterations,
|
|
"partial_ns_per_decode": partial_samples,
|
|
"partial_median_ns": partial_median,
|
|
"partial_to_full_percent": partial_median * 100.0 / full_median,
|
|
"samples": samples,
|
|
"workspace_full_peak": full_peak,
|
|
"workspace_partial_peak": partial_peak,
|
|
}
|
|
|
|
|
|
def evaluate_ranges(
|
|
oracle: Path,
|
|
codec: str,
|
|
stream: bytes,
|
|
logical: bytes,
|
|
physical_length: int,
|
|
dict_size: int,
|
|
temp: Path,
|
|
label: str,
|
|
) -> dict[str, Any]:
|
|
full_record, full_output = decode(
|
|
oracle, codec, stream, len(logical), True, dict_size, 1,
|
|
temp, f"{label}-full",
|
|
)
|
|
if full_errno(full_record, len(stream), logical, full_output) != 0:
|
|
raise GateFailure("STOP", f"{label} legal full decode is not exact")
|
|
ranges = []
|
|
prefix_record: dict[str, int] | None = None
|
|
for request in SPEC["ranges"]:
|
|
offset = request["offset"] if request["offset"] >= 0 else len(logical) + request["offset"]
|
|
if offset < 0 or request["length"] > len(logical) - offset:
|
|
raise GateFailure("STOP", f"{label} cannot cover range {request['name']}")
|
|
end = offset + request["length"]
|
|
use_partial = end < len(logical)
|
|
record, output = decode(
|
|
oracle, codec, stream, end, not use_partial, dict_size, 1,
|
|
temp, f"{label}-{request['name']}",
|
|
)
|
|
if codec == "lz4" and use_partial:
|
|
parsed_output, parsed_consumed = raw_lz4_decode(stream, end)
|
|
if parsed_output != logical[:end]:
|
|
raise GateFailure("STOP", f"{label} raw LZ4 range parser differs from full slice")
|
|
record["consumed"] = parsed_consumed
|
|
errno = (
|
|
partial_errno(record, logical[:end], output)
|
|
if use_partial
|
|
else full_errno(record, len(stream), logical, output)
|
|
)
|
|
if errno != 0 or output[offset:end] != logical[offset:end]:
|
|
raise GateFailure("STOP", f"{label} range {request['name']} differs from full slice")
|
|
if use_partial and record["consumed"] >= len(stream):
|
|
raise GateFailure("STOP", f"{label} range {request['name']} has no measurable input boundary")
|
|
if request["name"] == "prefix":
|
|
prefix_record = record
|
|
ranges.append(
|
|
{
|
|
"consumed": record["consumed"],
|
|
"decoded_prefix": end,
|
|
"fallback_full": not use_partial,
|
|
"guards": record["guards"],
|
|
"length": request["length"],
|
|
"name": request["name"],
|
|
"offset": offset,
|
|
"policy_errno": errno,
|
|
"slice_sha256": sha256_bytes(output[offset:end]),
|
|
"workspace_peak": record["workspace_peak"],
|
|
}
|
|
)
|
|
if prefix_record is None:
|
|
raise GateFailure("INFRA_BLOCKED", "prefix request disappeared")
|
|
corruption_start = max(prefix_record["consumed"] + 8, len(stream) - 64)
|
|
if corruption_start >= len(stream):
|
|
raise GateFailure("STOP", f"{label} cannot place corruption after partial consumption")
|
|
corrupted = bytearray(stream)
|
|
original_tail = bytes(corrupted[corruption_start:])
|
|
corrupted[corruption_start:] = b"\0" * (len(corrupted) - corruption_start)
|
|
if bytes(corrupted[corruption_start:]) == original_tail:
|
|
corrupted[-1] ^= 0x5A
|
|
partial_end = SPEC["ranges"][0]["length"]
|
|
corrupt_partial_record, corrupt_partial_output = decode(
|
|
oracle, codec, bytes(corrupted), partial_end, False, dict_size, 1,
|
|
temp, f"{label}-corrupt-partial",
|
|
)
|
|
if codec == "lz4":
|
|
parsed_output, parsed_consumed = raw_lz4_decode(bytes(corrupted), partial_end)
|
|
if parsed_output != logical[:partial_end]:
|
|
raise GateFailure("STOP", f"{label} corrupt raw LZ4 prefix differs from full slice")
|
|
corrupt_partial_record["consumed"] = parsed_consumed
|
|
corrupt_partial_errno = partial_errno(
|
|
corrupt_partial_record, logical[:partial_end], corrupt_partial_output
|
|
)
|
|
corrupt_full_record, corrupt_full_output = decode(
|
|
oracle, codec, bytes(corrupted), len(logical), True, dict_size, 1,
|
|
temp, f"{label}-corrupt-full",
|
|
)
|
|
corrupt_full_errno = full_errno(
|
|
corrupt_full_record, len(corrupted), logical, corrupt_full_output
|
|
)
|
|
truncated_record, truncated_output = decode(
|
|
oracle, codec, stream[:-1], len(logical), True, dict_size, 1,
|
|
temp, f"{label}-truncated",
|
|
)
|
|
truncated_errno = full_errno(
|
|
truncated_record, len(stream) - 1, logical, truncated_output
|
|
)
|
|
if corrupt_partial_errno != 0 or corrupt_full_errno != EINTEGRITY:
|
|
raise GateFailure("STOP", f"{label} range-after corruption contract is not exact")
|
|
if truncated_errno != EINTEGRITY:
|
|
raise GateFailure("STOP", f"{label} one-byte truncation is not positive EINTEGRITY")
|
|
cpu = benchmark(oracle, codec, stream, logical, dict_size, temp, label)
|
|
prefix_end = SPEC["ranges"][0]["length"]
|
|
caller = SPEC["ranges"][0]["length"]
|
|
baseline_peak = physical_length + len(logical) + caller + cpu["workspace_full_peak"]
|
|
candidate_peak = physical_length + prefix_end + caller + cpu["workspace_partial_peak"]
|
|
reduction = (baseline_peak - candidate_peak) * 100.0 / baseline_peak
|
|
workspace_budget = SPEC["hard_budget"]["workspace_bytes"][codec]
|
|
hard_cap = (
|
|
SPEC["hard_budget"]["input_bytes"]
|
|
+ SPEC["hard_budget"]["decoded_bytes"]
|
|
+ SPEC["hard_budget"]["caller_bytes"]
|
|
+ workspace_budget
|
|
)
|
|
if physical_length > SPEC["hard_budget"]["input_bytes"]:
|
|
raise GateFailure("STOP", f"{label} input exceeds the fixed pcluster budget")
|
|
if len(logical) > SPEC["hard_budget"]["decoded_bytes"]:
|
|
raise GateFailure("STOP", f"{label} output exceeds the fixed decoded budget")
|
|
if max(cpu["workspace_full_peak"], cpu["workspace_partial_peak"]) > workspace_budget:
|
|
raise GateFailure("STOP", f"{label} workspace exceeds its hard budget")
|
|
if max(baseline_peak, candidate_peak) > hard_cap:
|
|
raise GateFailure("STOP", f"{label} peak temporary bytes exceed the hard cap")
|
|
threshold = SPEC["thresholds"]["minimum_peak_temporary_reduction_percent"]
|
|
fallback_reasons = []
|
|
if reduction < threshold:
|
|
fallback_reasons.append(
|
|
f"peak temporary reduction {reduction:.3f}% is below {threshold}%"
|
|
)
|
|
if cpu["partial_median_ns"] > cpu["full_median_ns"] * 1.25:
|
|
fallback_reasons.append("partial CPU median exceeds the 125% hard regression budget")
|
|
return {
|
|
"budget": {
|
|
"baseline_peak_temporary_bytes": baseline_peak,
|
|
"candidate_peak_temporary_bytes": candidate_peak,
|
|
"hard_cap_bytes": hard_cap,
|
|
"peak_reduction_percent": reduction,
|
|
"threshold_percent": threshold,
|
|
"workspace_budget_bytes": workspace_budget,
|
|
},
|
|
"corruption": {
|
|
"full_errno": corrupt_full_errno,
|
|
"partial_consumed": corrupt_partial_record["consumed"],
|
|
"partial_errno": corrupt_partial_errno,
|
|
"starts_after_partial_consumed": corruption_start > prefix_record["consumed"],
|
|
"starts_at_stream_byte": corruption_start,
|
|
"truncated_errno": truncated_errno,
|
|
},
|
|
"cpu": cpu,
|
|
"full": {
|
|
**full_record,
|
|
"output_sha256": sha256_bytes(full_output),
|
|
"policy_errno": 0,
|
|
},
|
|
"ranges": ranges,
|
|
"partial_capability": "FULL_FALLBACK" if fallback_reasons else "GO",
|
|
"partial_capability_reasons": fallback_reasons,
|
|
}
|
|
|
|
|
|
def build_stream_codecs(
|
|
source_dir: Path, source: bytes, oracle: Path, temp: Path,
|
|
) -> list[dict[str, Any]]:
|
|
mkfs = Path(SPEC["tools"]["mkfs.erofs"]["path"])
|
|
fsck = Path(SPEC["tools"]["fsck.erofs"]["path"])
|
|
dump = Path(SPEC["tools"]["dump.erofs"]["path"])
|
|
records = []
|
|
for codec, codec_spec in sorted(SPEC["stream"]["codecs"].items()):
|
|
images = []
|
|
for repeat in ("a", "b"):
|
|
image = temp / f"{codec}-{repeat}.erofs"
|
|
run_logged(
|
|
[str(mkfs), *codec_spec["mkfs_args"], str(image), str(source_dir)],
|
|
temp,
|
|
OUTPUT / f"logs/mkfs-{codec}-{repeat}.log",
|
|
)
|
|
images.append(image)
|
|
hashes = [sha256_path(image) for image in images]
|
|
if hashes[0] != hashes[1]:
|
|
raise GateFailure("INFRA_BLOCKED", f"{codec} image is not byte reproducible")
|
|
valid_fsck = fsck_extract(fsck, images[0], temp / f"extract-{codec}", f"{codec}-valid")
|
|
if not valid_fsck["success"] or (temp / f"extract-{codec}/payload.bin").read_bytes() != source:
|
|
raise GateFailure("INFRA_BLOCKED", f"{codec} legal image extraction mismatch")
|
|
extent = select_extent(dump, images[0], "/payload.bin", 12288, codec)
|
|
image_bytes = images[0].read_bytes()
|
|
start = extent["physical_offset"]
|
|
end = start + extent["physical_length"]
|
|
block = image_bytes[start:end]
|
|
leading = next((index for index, value in enumerate(block) if value), len(block))
|
|
if leading == len(block):
|
|
raise GateFailure("STOP", f"{codec} selected pcluster is all padding")
|
|
stream = block[leading:]
|
|
logical = source[
|
|
extent["logical_offset"] : extent["logical_offset"] + extent["logical_length"]
|
|
]
|
|
evaluated = evaluate_ranges(
|
|
oracle, codec, stream, logical, extent["physical_length"],
|
|
codec_spec["dict_size"], temp, codec,
|
|
)
|
|
corrupt_start = evaluated["corruption"]["starts_at_stream_byte"]
|
|
corrupted_image = bytearray(image_bytes)
|
|
absolute = start + leading + corrupt_start
|
|
corrupted_image[absolute:end] = b"\0" * (end - absolute)
|
|
corrupt_path = temp / f"{codec}-corrupt.erofs"
|
|
corrupt_path.write_bytes(corrupted_image)
|
|
corrupt_fsck = fsck_extract(
|
|
fsck, corrupt_path, temp / f"extract-{codec}-corrupt", f"{codec}-corrupt"
|
|
)
|
|
if corrupt_fsck["success"]:
|
|
raise GateFailure("STOP", f"{codec} real EROFS tail corruption escaped full fsck")
|
|
records.append(
|
|
{
|
|
"codec": codec,
|
|
"extent": {**extent, "leading_zero_bytes": leading, "stream_bytes": len(stream)},
|
|
"image_repeated_sha256": hashes,
|
|
"real_corrupt_fsck": corrupt_fsck,
|
|
"real_valid_fsck": valid_fsck,
|
|
**evaluated,
|
|
}
|
|
)
|
|
return records
|
|
|
|
|
|
def build_lz4_generations(
|
|
source_dir: Path, source: bytes, oracle: Path, temp: Path,
|
|
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
clone = temp / "erofs-utils"
|
|
run_logged(
|
|
["git", "clone", "--no-checkout", SPEC["lz4"]["generator_url"], str(clone)],
|
|
temp,
|
|
OUTPUT / "logs/lz4-upstream-clone.log",
|
|
timeout=90,
|
|
)
|
|
generation_records = []
|
|
evaluated_records = []
|
|
for version in SPEC["lz4"]["generators"]:
|
|
name = version["name"]
|
|
worktree = temp / f"erofs-{name}"
|
|
run_logged(
|
|
["git", "-C", str(clone), "worktree", "add", "--detach", str(worktree), version["commit"]],
|
|
temp,
|
|
OUTPUT / f"logs/lz4-{name}-worktree.log",
|
|
)
|
|
resolved = subprocess.check_output(["git", "-C", str(worktree), "rev-parse", "HEAD"], text=True).strip()
|
|
if resolved != version["commit"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"LZ4 generator identity changed for {name}")
|
|
run_logged(["./autogen.sh"], worktree, OUTPUT / f"logs/lz4-{name}-autogen.log")
|
|
run_logged(["./configure", "--disable-fuse"], worktree, OUTPUT / f"logs/lz4-{name}-configure.log")
|
|
run_logged(["make", "-s", "-j2"], worktree, OUTPUT / f"logs/lz4-{name}-make.log")
|
|
mkfs = worktree / "mkfs/mkfs.erofs"
|
|
fsck = worktree / "fsck/fsck.erofs"
|
|
dump = worktree / "dump/dump.erofs"
|
|
images = []
|
|
for repeat in ("a", "b"):
|
|
image = temp / f"lz4-{name}-{repeat}.erofs"
|
|
run_logged(
|
|
[str(mkfs), *SPEC["lz4"]["mkfs_args"], str(image), str(source_dir)],
|
|
worktree,
|
|
OUTPUT / f"logs/lz4-{name}-mkfs-{repeat}.log",
|
|
)
|
|
images.append(image)
|
|
hashes = [sha256_path(image) for image in images]
|
|
if hashes[0] != hashes[1]:
|
|
raise GateFailure("INFRA_BLOCKED", f"LZ4 {name} image is not byte reproducible")
|
|
extract = temp / f"extract-lz4-{name}"
|
|
extract.mkdir()
|
|
run_logged(
|
|
[str(fsck), "--extract", str(images[0])],
|
|
extract,
|
|
OUTPUT / f"logs/lz4-{name}-fsck.log",
|
|
)
|
|
extents, dump_identity = dump_extents_compat(dump, images[0], f"lz4-{name}")
|
|
choices = [record for record in extents if record["logical_length"] >= 12288]
|
|
if not choices:
|
|
raise GateFailure("STOP", f"LZ4 {name} has no representative compressed extent")
|
|
extent = max(choices, key=lambda record: record["logical_length"])
|
|
image_bytes = images[0].read_bytes()
|
|
reconstructed = bytearray(len(source))
|
|
for current in extents:
|
|
current_block = image_bytes[
|
|
current["physical_offset"] :
|
|
current["physical_offset"] + current["physical_length"]
|
|
]
|
|
current_logical, _ = raw_lz4_decode(current_block, current["logical_length"])
|
|
logical_start = current["logical_offset"]
|
|
logical_end = logical_start + current["logical_length"]
|
|
if current_logical != source[logical_start:logical_end]:
|
|
raise GateFailure("STOP", f"LZ4 {name} extent differs from source")
|
|
reconstructed[logical_start:logical_end] = current_logical
|
|
if bytes(reconstructed) != source:
|
|
raise GateFailure("STOP", f"LZ4 {name} full reconstruction differs from source")
|
|
block = image_bytes[
|
|
extent["physical_offset"] : extent["physical_offset"] + extent["physical_length"]
|
|
]
|
|
logical = source[
|
|
extent["logical_offset"] : extent["logical_offset"] + extent["logical_length"]
|
|
]
|
|
parsed, consumed = raw_lz4_decode(block, len(logical))
|
|
if parsed != logical:
|
|
raise GateFailure("STOP", f"LZ4 {name} independent parser differs from source")
|
|
stream = block[:consumed]
|
|
evaluated = evaluate_ranges(
|
|
oracle, "lz4", stream, logical, extent["physical_length"], 0,
|
|
temp, f"lz4-{name}",
|
|
)
|
|
generation_records.append(
|
|
{
|
|
"commit": resolved,
|
|
"extent": {**extent, "stream_bytes": consumed},
|
|
"dump_identity": dump_identity,
|
|
"image_repeated_sha256": hashes,
|
|
"name": name,
|
|
}
|
|
)
|
|
evaluated_records.append({"generator": name, **evaluated})
|
|
representative = evaluated_records[-1]
|
|
capability = (
|
|
"GO"
|
|
if all(record["partial_capability"] == "GO" for record in evaluated_records)
|
|
else "FULL_FALLBACK"
|
|
)
|
|
lz4_record = {
|
|
"codec": "lz4",
|
|
"generations": generation_records,
|
|
"partial_capability": capability,
|
|
"representative": representative,
|
|
}
|
|
return lz4_record, evaluated_records
|
|
|
|
|
|
def verify_tools() -> dict[str, Any]:
|
|
records = {}
|
|
for name, expected in SPEC["tools"].items():
|
|
path = Path(expected["path"])
|
|
if not path.is_file() or sha256_path(path) != expected["sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"tool identity changed: {name}")
|
|
records[name] = expected
|
|
mkfs_version = subprocess.check_output(
|
|
[SPEC["tools"]["mkfs.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True
|
|
).strip()
|
|
fsck_version = subprocess.check_output(
|
|
[SPEC["tools"]["fsck.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True
|
|
).strip()
|
|
for codec in ("lz4", "lzma", "deflate", "zstd"):
|
|
if codec not in mkfs_version or codec not in fsck_version:
|
|
raise GateFailure("STOP", f"{codec} lacks a real mkfs/fsck path")
|
|
records["mkfs_version"] = mkfs_version
|
|
records["fsck_version"] = fsck_version
|
|
return records
|
|
|
|
|
|
def verify_source_contract(sources: dict[str, str]) -> dict[str, Any]:
|
|
req = sources["repo-pre-15/src/compress.h"]
|
|
if "size_t outputsize;" not in req or "bool partial_decoding;" not in req:
|
|
raise GateFailure("INFRA_BLOCKED", "FreeBSD partial request contract changed")
|
|
anchors = {
|
|
"lz4": "rq->partial_decoding",
|
|
"lzma": "!rq->partial_decoding",
|
|
"deflate": "if (rq->partial_decoding)",
|
|
"zstd": "if (rq->partial_decoding)",
|
|
}
|
|
for codec, anchor in anchors.items():
|
|
if anchor not in sources[f"repo-pre-15/src/decompressor_{codec}.c"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"FreeBSD {codec} partial primitive changed")
|
|
dispatch = sources["repo-pre-15/src/decompressor.c"]
|
|
if ".outputsize = dstlen" not in dispatch or ".partial_decoding = partial" not in dispatch:
|
|
raise GateFailure("INFRA_BLOCKED", "FreeBSD dispatch partial propagation changed")
|
|
zdata = sources["repo-pre-15/src/zdata.c"]
|
|
required = (
|
|
"partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0;",
|
|
"if (!partial && decoded_len != map->m_llen)",
|
|
"erofs_put_metabuf(&buf);",
|
|
"erofs_brelse(compressed);",
|
|
"free(decoded, M_EROFS);",
|
|
"*bufp = decoded;",
|
|
)
|
|
if not all(anchor in zdata for anchor in required):
|
|
raise GateFailure("INFRA_BLOCKED", "FreeBSD buffer/partial-ref contract changed")
|
|
decode_at = zdata.index("error = z_erofs_decompress")
|
|
release_at = min(
|
|
zdata.index("erofs_put_metabuf(&buf);", decode_at),
|
|
zdata.index("erofs_brelse(compressed);", decode_at),
|
|
)
|
|
error_at = zdata.index("if (error != 0) {", release_at)
|
|
free_at = zdata.index("free(decoded, M_EROFS);", error_at)
|
|
publish_at = zdata.index("*bufp = decoded;", free_at)
|
|
if not decode_at < release_at < error_at < free_at < publish_at:
|
|
raise GateFailure("INFRA_BLOCKED", "FreeBSD decode cleanup ordering changed")
|
|
linux_req = sources["src-linux/compress.h"]
|
|
if "partial_decoding" not in linux_req or "outputsize" not in linux_req:
|
|
raise GateFailure("INFRA_BLOCKED", "Linux partial request anchor changed")
|
|
linux_anchors = {
|
|
"lz4": "LZ4_decompress_safe_partial",
|
|
"lzma": "!rq->partial_decoding",
|
|
"deflate": "rq->partial_decoding",
|
|
"zstd": "rq->outputsize + dctx.avail_out",
|
|
}
|
|
for codec, anchor in linux_anchors.items():
|
|
path = "src-linux/decompressor.c" if codec == "lz4" else f"src-linux/decompressor_{codec}.c"
|
|
if anchor not in sources[path]:
|
|
raise GateFailure("INFRA_BLOCKED", f"Linux {codec} partial anchor changed")
|
|
return {
|
|
"backend_partial_primitives": sorted(anchors),
|
|
"buffer_release_after_decode": True,
|
|
"failed_output_freed": True,
|
|
"linux_prefix_contract": True,
|
|
"partial_ref_preserved": True,
|
|
"successful_output_published_only_after_decode": True,
|
|
}
|
|
|
|
|
|
def build_state_model() -> dict[str, Any]:
|
|
scenarios = [
|
|
{"name": "cache-hit", "decode": "none", "cache_after": "ready-full"},
|
|
{"name": "cache-miss-partial-success", "decode": "bounded-prefix", "cache_after": "absent"},
|
|
{"name": "cache-miss-partial-failure", "decode": "bounded-prefix", "cache_after": "absent", "errno": EINTEGRITY},
|
|
{"name": "concurrent-partial-miss", "decode": "per-request-local", "shared_owner_count": 0, "shared_waiter_count": 0},
|
|
{"name": "full-request", "decode": "full", "cache_after": "existing-policy"},
|
|
{"name": "unsupported-backend", "decode": "full-fallback", "cache_after": "existing-policy"},
|
|
{"name": "partial-reference", "decode": "bounded-prefix", "cache_after": "ineligible"},
|
|
{"name": "eviction", "partial_reference_retained": False, "cache_after": "existing-policy"},
|
|
{"name": "reclaim", "partial_reference_retained": False, "cache_after": "existing-policy"},
|
|
{"name": "unmount", "partial_reference_retained": False, "drain_required": False},
|
|
{"name": "key-reuse", "partial_reference_retained": False, "cache_after": "existing-policy"},
|
|
]
|
|
prototype = {
|
|
"probed_codecs": ["lz4", "lzma", "deflate", "zstd"],
|
|
"fallback_backends": ["shifted", "interlaced", "unknown"],
|
|
"new_persistent_state": False,
|
|
"overflow_errno": EOVERFLOW,
|
|
"pool_exhaustion": "not applicable; B28 adds no pool",
|
|
"rule": "cache hit first; strict subextent on a gate-authorized codec uses mapoff+want bytes and never publishes a partial cache entry; all other cases use exact full decode",
|
|
"scenarios": scenarios,
|
|
}
|
|
write_json(OUTPUT / "candidate-state-model.json", prototype)
|
|
prototype_source = """P15-086 test-only candidate model\n\ncapable = lz4|lzma|deflate|zstd\nrequest_end = checked_add(mapoff, want)\nif cache_hit: copy full cache\nelif capable and request_end < m_llen: decode(request_end, partial=true), no cache publish\nelse: decode(m_llen, partial=partial_ref), preserve existing full cache policy\nfailed local output is freed; input is released after callback; no persistent state is added\n"""
|
|
prototype_path = OUTPUT / "candidate-prototype.txt"
|
|
prototype_path.write_text(prototype_source, encoding="ascii")
|
|
return {
|
|
"prototype_sha256": sha256_path(prototype_path),
|
|
"scenario_count": len(scenarios),
|
|
"state_machine": "PASS",
|
|
}
|
|
|
|
|
|
def finalize() -> None:
|
|
entries = []
|
|
for path in sorted(OUTPUT.rglob("*")):
|
|
if path.is_file() and path.name != "SHA256SUMS":
|
|
entries.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
|
|
(OUTPUT / "SHA256SUMS").write_text("\n".join(entries) + "\n", encoding="ascii")
|
|
|
|
|
|
result: dict[str, Any] = {}
|
|
exit_code = 0
|
|
owned_temp: str | None = None
|
|
try:
|
|
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-086" or SPEC.get("gates") != ["G04", "G05"]:
|
|
raise GateFailure("INFRA_BLOCKED", "gate input identity changed")
|
|
resolved = git("rev-parse", REQUESTED_BASE)
|
|
if resolved != SPEC["required_base"]:
|
|
raise GateFailure("INFRA_BLOCKED", f"wrong DUT base: {resolved}")
|
|
sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]}
|
|
source_hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()}
|
|
if source_hashes != SPEC["source_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed")
|
|
write_json(OUTPUT / "source-sha256.json", source_hashes)
|
|
freebsd_head = subprocess.check_output(
|
|
["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True
|
|
).strip()
|
|
freebsd_hashes = {
|
|
path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"]
|
|
}
|
|
if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed")
|
|
errno_source = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8")
|
|
if "#define\tEINTEGRITY\t97" not in errno_source or "#define\tEOVERFLOW\t84" not in errno_source:
|
|
raise GateFailure("INFRA_BLOCKED", "positive FreeBSD errno identity changed")
|
|
write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes})
|
|
write_json(OUTPUT / "toolchain.json", verify_tools())
|
|
write_json(OUTPUT / "source-contract.json", verify_source_contract(sources))
|
|
write_json(OUTPUT / "state-model-result.json", build_state_model())
|
|
with tempfile.TemporaryDirectory(prefix="p15-086-g04-g05-") as temporary:
|
|
owned_temp = temporary
|
|
temp = Path(temporary)
|
|
oracle, oracle_identity = compile_oracle(temp)
|
|
write_json(OUTPUT / "independent-oracle.json", oracle_identity)
|
|
stream_dir = temp / "stream-source"
|
|
stream_source = make_stream_source(stream_dir)
|
|
stream_records = build_stream_codecs(stream_dir, stream_source, oracle, temp)
|
|
lz4_dir = temp / "lz4-source"
|
|
lz4_source = make_lz4_source(lz4_dir)
|
|
lz4_record, lz4_generation_results = build_lz4_generations(
|
|
lz4_dir, lz4_source, oracle, temp
|
|
)
|
|
codec_records = [lz4_record, *stream_records]
|
|
write_json(OUTPUT / "codec-results.json", codec_records)
|
|
write_json(OUTPUT / "lz4-generation-results.json", lz4_generation_results)
|
|
per_codec = {
|
|
record["codec"]: (
|
|
record["partial_capability"]
|
|
if record["codec"] == "lz4"
|
|
else record["partial_capability"]
|
|
)
|
|
for record in codec_records
|
|
}
|
|
if set(per_codec) != {"lz4", "lzma", "deflate", "zstd"}:
|
|
raise GateFailure("STOP", "the real supported codec set is incomplete")
|
|
enabled = sorted(codec for codec, decision in per_codec.items() if decision == "GO")
|
|
if not enabled:
|
|
raise GateFailure("STOP", "no codec meets the P15-086 quantified benefit gate")
|
|
write_json(
|
|
OUTPUT / "authorized-capabilities.json",
|
|
{
|
|
"full_fallback": sorted(
|
|
codec for codec, decision in per_codec.items() if decision == "FULL_FALLBACK"
|
|
),
|
|
"partial": enabled,
|
|
"persistent_state_added": False,
|
|
"schema": 1,
|
|
},
|
|
)
|
|
result = {
|
|
"b28": "AUTHORIZED",
|
|
"candidate": "P15-086",
|
|
"cleanup": "PASS",
|
|
"codecs": per_codec,
|
|
"cpu_samples_per_mode": SPEC["cpu"]["samples"],
|
|
"full_feature_suite": "NOT_RUN",
|
|
"g04": "GO",
|
|
"g05": "GO",
|
|
"gates": ["G04", "G05"],
|
|
"hard_budget": "PASS",
|
|
"partial_enabled_codecs": enabled,
|
|
"oracle": "real EROFS images plus independent liblz4/liblzma/zlib/libzstd consumed-byte oracle",
|
|
"qemu": "NOT_RUN",
|
|
"qemu_reason": "Stage0 host gate owns correctness and budget authorization; B28 acceptance owns strict TC176 QEMU",
|
|
"requested_base": REQUESTED_BASE,
|
|
"resolved_base": resolved,
|
|
"schema": 1,
|
|
"status": "GO",
|
|
"typed_errno": "PASS",
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
except GateFailure as failure:
|
|
result = {
|
|
"b28": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN",
|
|
"candidate": "P15-086",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gates": ["G04", "G05"],
|
|
"qemu": "NOT_RUN",
|
|
"reason": failure.reason,
|
|
"requested_base": REQUESTED_BASE,
|
|
"schema": 1,
|
|
"status": failure.status,
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
exit_code = 1 if failure.status == "STOP" else 21
|
|
except (OSError, subprocess.SubprocessError, ValueError) as failure:
|
|
result = {
|
|
"b28": "NOT_RUN",
|
|
"candidate": "P15-086",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gates": ["G04", "G05"],
|
|
"qemu": "NOT_RUN",
|
|
"reason": f"gate infrastructure failure: {failure}",
|
|
"requested_base": REQUESTED_BASE,
|
|
"schema": 1,
|
|
"status": "INFRA_BLOCKED",
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
exit_code = 21
|
|
finally:
|
|
cleanup_record = {
|
|
"owned_temp": owned_temp,
|
|
"owned_temp_removed": owned_temp is None or not Path(owned_temp).exists(),
|
|
"protected_pid_touched": False,
|
|
"protected_port_touched": False,
|
|
"qemu_started": False,
|
|
"shared_base_image_touched": False,
|
|
}
|
|
if not cleanup_record["owned_temp_removed"]:
|
|
result = {
|
|
"b28": "NOT_RUN",
|
|
"candidate": "P15-086",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"gates": ["G04", "G05"],
|
|
"qemu": "NOT_RUN",
|
|
"reason": "owned gate temporary directory survived cleanup",
|
|
"requested_base": REQUESTED_BASE,
|
|
"schema": 1,
|
|
"status": "INFRA_BLOCKED",
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
exit_code = 21
|
|
write_json(OUTPUT / "owned-cleanup.json", cleanup_record)
|
|
finalize()
|
|
|
|
print(json.dumps(result, sort_keys=True))
|
|
raise SystemExit(exit_code)
|
|
PY
|