#!/bin/sh set -eu gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) input=$gate_dir/P15-086-input.json freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} base= output= while test "$#" -gt 0; do case "$1" in --base) test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } base=$2 shift 2 ;; --output) test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } output=$2 shift 2 ;; *) printf 'unknown argument: %s\n' "$1" >&2 exit 2 ;; esac done test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; } for tool in autoreconf cc dump.erofs fsck.erofs git make mkfs.erofs pkg-config python3 sha256sum; do command -v "$tool" >/dev/null 2>&1 || { printf 'missing required host tool: %s\n' "$tool" >&2 exit 2 } done case "$output" in /*) ;; *) output=$PWD/$output ;; esac test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } mkdir -p "$output" python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY' from __future__ import annotations import hashlib import json import os from pathlib import Path import re import shutil import statistics import subprocess import sys import tempfile from typing import Any ROOT = Path(sys.argv[1]) INPUT = Path(sys.argv[2]) REQUESTED_BASE = sys.argv[3] OUTPUT = Path(sys.argv[4]) FREEBSD_SRC = Path(sys.argv[5]) SPEC = json.loads(INPUT.read_text(encoding="ascii")) EINTEGRITY = 97 EOVERFLOW = 84 class GateFailure(Exception): def __init__(self, status: str, reason: str): super().__init__(reason) self.status = status self.reason = reason def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def sha256_path(path: Path) -> str: return sha256_bytes(path.read_bytes()) def write_json(path: Path, value: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") def git(*args: str) -> str: return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip() def source_at(commit: str, path: str) -> str: completed = subprocess.run( ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) if completed.returncode != 0: raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}") return completed.stdout def run_logged( argv: list[str], cwd: Path, log: Path, timeout: int = 120, expected: set[int] | None = None, ) -> subprocess.CompletedProcess[str]: try: completed = subprocess.run( argv, cwd=cwd, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=timeout, ) except subprocess.TimeoutExpired as failure: log.parent.mkdir(parents=True, exist_ok=True) log.write_text( "$ " + " ".join(argv) + f"\n[TIMEOUT after {timeout}s]\n", encoding="utf-8", ) raise GateFailure("INFRA_BLOCKED", f"command timed out: {' '.join(argv)}") from failure log.parent.mkdir(parents=True, exist_ok=True) log.write_text( "$ " + " ".join(argv) + "\n" + completed.stdout + f"\n[exit {completed.returncode}]\n", encoding="utf-8", ) allowed = {0} if expected is None else expected if completed.returncode not in allowed: raise GateFailure( "INFRA_BLOCKED", f"command failed ({completed.returncode}): {' '.join(argv)}", ) return completed EXTENT_RE = re.compile( r"^\s*(\d+):\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:\s*" r"(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$", re.MULTILINE, ) def parse_extents(text: str) -> list[dict[str, int]]: records = [] for match in EXTENT_RE.finditer(text): index, logical, logical_end, logical_length, physical, physical_end, physical_length = map( int, match.groups() ) if logical_end - logical != logical_length or physical_end - physical != physical_length: raise GateFailure("INFRA_BLOCKED", "dump.erofs extent arithmetic changed") records.append( { "index": index, "logical_length": logical_length, "logical_offset": logical, "physical_length": physical_length, "physical_offset": physical, } ) if not records: raise GateFailure("INFRA_BLOCKED", "dump.erofs returned no extents") return records def make_stream_source(path: Path) -> bytes: path.mkdir(parents=True) content = b"".join( f"P15-086-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode("ascii") for index in range(SPEC["stream"]["source"]["line_count"]) ) expected = SPEC["stream"]["source"] if len(content) != expected["size"] or sha256_bytes(content) != expected["sha256"]: raise GateFailure("INFRA_BLOCKED", "stream source identity changed") payload = path / "payload.bin" payload.write_bytes(content) os.utime(payload, (0, 0)) os.utime(path, (0, 0)) return content def make_lz4_source(path: Path) -> bytes: path.mkdir(parents=True) content = b"".join( bytes([segment["byte"]]) * segment["length"] for segment in SPEC["lz4"]["source"]["segments"] ) expected = SPEC["lz4"]["source"] if len(content) != expected["size"] or sha256_bytes(content) != expected["sha256"]: raise GateFailure("INFRA_BLOCKED", "LZ4 source identity changed") payload = path / "payload.bin" payload.write_bytes(content) os.utime(payload, (0, 0)) os.utime(path, (0, 0)) return content def raw_lz4_decode(data: bytes, target: int) -> tuple[bytes, int]: ip = 0 output = bytearray() while ip < len(data): token = data[ip] ip += 1 literal_length = token >> 4 if literal_length == 15: while True: if ip >= len(data): raise GateFailure("STOP", "independent LZ4 parser saw truncated literal length") value = data[ip] ip += 1 literal_length += value if value != 255: break if ip + literal_length > len(data): raise GateFailure("STOP", "independent LZ4 parser saw truncated literals") output.extend(data[ip : ip + literal_length]) ip += literal_length if len(output) >= target: return bytes(output[:target]), ip if ip == len(data): break if ip + 2 > len(data): raise GateFailure("STOP", "independent LZ4 parser saw truncated match offset") offset = data[ip] | data[ip + 1] << 8 ip += 2 if offset == 0 or offset > len(output): raise GateFailure("STOP", "independent LZ4 parser saw invalid match offset") match_length = token & 15 if match_length == 15: while True: if ip >= len(data): raise GateFailure("STOP", "independent LZ4 parser saw truncated match length") value = data[ip] ip += 1 match_length += value if value != 255: break for _ in range(match_length + 4): output.append(output[-offset]) if len(output) >= target: return bytes(output[:target]), ip raise GateFailure("STOP", "independent LZ4 parser ended before requested output") ORACLE_SOURCE = r''' #define _POSIX_C_SOURCE 200809L #define ZSTD_STATIC_LINKING_ONLY #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct allocation { size_t size; }; struct tracker { size_t current; size_t peak; }; struct result { size_t consumed; size_t produced; size_t status; size_t workspace_peak; int cleanup; int codec_error; int stream_end; }; static void fail(const char *message) { fprintf(stderr, "%s\n", message); exit(2); } static void * tracked_alloc(struct tracker *tracker, size_t size) { struct allocation *allocation; if (size > SIZE_MAX - sizeof(*allocation)) return NULL; allocation = malloc(sizeof(*allocation) + size); if (allocation == NULL) return NULL; allocation->size = size; tracker->current += size; if (tracker->current > tracker->peak) tracker->peak = tracker->current; return allocation + 1; } static void tracked_free(struct tracker *tracker, void *address) { struct allocation *allocation; if (address == NULL) return; allocation = (struct allocation *)address - 1; if (allocation->size > tracker->current) fail("allocation tracker underflow"); tracker->current -= allocation->size; free(allocation); } static voidpf zalloc_tracked(voidpf opaque, uInt items, uInt size) { if (items != 0 && size > SIZE_MAX / items) return NULL; return tracked_alloc(opaque, (size_t)items * size); } static void zfree_tracked(voidpf opaque, voidpf address) { tracked_free(opaque, address); } static void * lzma_alloc_tracked(void *opaque, size_t items, size_t size) { if (items != 0 && size > SIZE_MAX / items) return NULL; return tracked_alloc(opaque, items * size); } static void lzma_free_tracked(void *opaque, void *address) { tracked_free(opaque, address); } static void * zstd_alloc_tracked(void *opaque, size_t size) { return tracked_alloc(opaque, size); } static void zstd_free_tracked(void *opaque, void *address) { tracked_free(opaque, address); } static unsigned char * read_file(const char *path, size_t *sizep) { struct stat st; unsigned char *data; ssize_t amount; size_t done; int fd; fd = open(path, O_RDONLY); if (fd < 0 || fstat(fd, &st) != 0 || st.st_size <= 0) fail("cannot open oracle input"); *sizep = (size_t)st.st_size; data = malloc(*sizep); if (data == NULL) fail("cannot allocate oracle input"); done = 0; while (done < *sizep) { amount = read(fd, data + done, *sizep - done); if (amount <= 0) fail("cannot read oracle input"); done += (size_t)amount; } close(fd); return data; } static void write_file(const char *path, const unsigned char *data, size_t size) { ssize_t amount; size_t done; int fd; fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd < 0) fail("cannot create oracle output"); done = 0; while (done < size) { amount = write(fd, data + done, size - done); if (amount <= 0) fail("cannot write oracle output"); done += (size_t)amount; } close(fd); } static struct result decode_once(const char *codec, const unsigned char *input, size_t input_size, unsigned char *output, size_t output_size, int full, uint32_t dict_size) { struct result result = { 0 }; struct tracker tracker = { 0 }; if (strcmp(codec, "lz4") == 0) { int ret; if (full) ret = LZ4_decompress_safe((const char *)input, (char *)output, (int)input_size, (int)output_size); else ret = LZ4_decompress_safe_partial((const char *)input, (char *)output, (int)input_size, (int)output_size, (int)output_size); if (ret < 0) { result.codec_error = 1; } else { result.produced = (size_t)ret; result.stream_end = full && result.produced == output_size; } result.consumed = full && !result.codec_error ? input_size : 0; result.status = ret < 0 ? (size_t)-ret : 0; result.cleanup = 1; } else if (strcmp(codec, "deflate") == 0) { z_stream stream; int ret = Z_OK, endret; memset(&stream, 0, sizeof(stream)); stream.zalloc = zalloc_tracked; stream.zfree = zfree_tracked; stream.opaque = &tracker; stream.next_in = (Bytef *)(uintptr_t)input; stream.avail_in = (uInt)input_size; stream.next_out = output; stream.avail_out = (uInt)output_size; ret = inflateInit2(&stream, -15); if (ret != Z_OK) { result.codec_error = 1; } else { while (stream.avail_out != 0) { uInt in_before = stream.avail_in; uInt out_before = stream.avail_out; ret = inflate(&stream, Z_SYNC_FLUSH); if (ret == Z_STREAM_END) { result.stream_end = 1; break; } if (ret != Z_OK || (stream.avail_in == in_before && stream.avail_out == out_before)) { result.codec_error = 1; break; } } result.consumed = input_size - stream.avail_in; result.produced = output_size - stream.avail_out; result.status = (size_t)(unsigned int)ret; endret = inflateEnd(&stream); result.cleanup = endret == Z_OK; } } else if (strcmp(codec, "lzma") == 0) { lzma_stream stream = LZMA_STREAM_INIT; lzma_allocator allocator = { .alloc = lzma_alloc_tracked, .free = lzma_free_tracked, .opaque = &tracker, }; lzma_ret ret; stream.allocator = &allocator; ret = lzma_microlzma_decoder(&stream, input_size, output_size, full != 0, dict_size); if (ret != LZMA_OK) { result.codec_error = 1; } else { stream.next_in = input; stream.avail_in = input_size; stream.next_out = output; stream.avail_out = output_size; while (stream.avail_out != 0) { size_t in_before = stream.avail_in; size_t out_before = stream.avail_out; ret = lzma_code(&stream, LZMA_RUN); if (ret == LZMA_STREAM_END) { result.stream_end = 1; break; } if (ret != LZMA_OK || (stream.avail_in == in_before && stream.avail_out == out_before)) { result.codec_error = 1; break; } } result.consumed = input_size - stream.avail_in; result.produced = output_size - stream.avail_out; result.status = ret; } lzma_end(&stream); result.cleanup = tracker.current == 0; } else if (strcmp(codec, "zstd") == 0) { ZSTD_customMem memory = { .customAlloc = zstd_alloc_tracked, .customFree = zstd_free_tracked, .opaque = &tracker, }; ZSTD_DCtx *context; ZSTD_inBuffer in_buffer; ZSTD_outBuffer out_buffer; size_t ret = 1; context = ZSTD_createDCtx_advanced(memory); if (context == NULL) { result.codec_error = 1; } else { in_buffer = (ZSTD_inBuffer){ input, input_size, 0 }; out_buffer = (ZSTD_outBuffer){ output, output_size, 0 }; while (out_buffer.pos != out_buffer.size) { size_t in_before = in_buffer.pos; size_t out_before = out_buffer.pos; ret = ZSTD_decompressStream(context, &out_buffer, &in_buffer); if (ZSTD_isError(ret)) { result.codec_error = 1; break; } if (ret == 0) { result.stream_end = 1; break; } if (in_buffer.pos == in_before && out_buffer.pos == out_before) { result.codec_error = 1; break; } } result.consumed = in_buffer.pos; result.produced = out_buffer.pos; result.status = ret; if (ZSTD_isError(ZSTD_freeDCtx(context))) result.cleanup = 0; else result.cleanup = tracker.current == 0; } } else { fail("unknown codec"); } result.workspace_peak = tracker.peak; return result; } static uint64_t elapsed_ns(const struct timespec *start, const struct timespec *end) { return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL + (uint64_t)(end->tv_nsec - start->tv_nsec); } int main(int argc, char **argv) { unsigned char *input, *mapping, *output; struct result result = { 0 }, current; struct timespec start, end; size_t input_size, output_size, usable, page_size; uint32_t dict_size; uint64_t cpu_ns; int full, guards = 1, iterations; if (argc != 8) fail("usage: oracle CODEC INPUT OUTPUT OUTPUT_SIZE FULL DICT ITERATIONS"); output_size = (size_t)strtoull(argv[4], NULL, 10); full = atoi(argv[5]); dict_size = (uint32_t)strtoul(argv[6], NULL, 10); iterations = atoi(argv[7]); if (output_size == 0 || iterations <= 0) fail("invalid output size or iteration count"); input = read_file(argv[2], &input_size); page_size = (size_t)sysconf(_SC_PAGESIZE); usable = (output_size + page_size - 1) & ~(page_size - 1); mapping = mmap(NULL, usable + 2 * page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (mapping == MAP_FAILED) fail("cannot allocate guarded output"); if (mprotect(mapping, page_size, PROT_NONE) != 0 || mprotect(mapping + page_size + usable, page_size, PROT_NONE) != 0) fail("cannot protect output guards"); output = mapping + page_size; if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start) != 0) fail("cannot start CPU clock"); for (int iteration = 0; iteration < iterations; ++iteration) { memset(output, 0xa5, usable); current = decode_once(argv[1], input, input_size, output, output_size, full, dict_size); if (iteration == 0) result = current; else if (current.codec_error != result.codec_error || current.cleanup != result.cleanup || current.consumed != result.consumed || current.produced != result.produced || current.stream_end != result.stream_end) fail("decoder result changed across iterations"); if (current.workspace_peak > result.workspace_peak) result.workspace_peak = current.workspace_peak; } if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end) != 0) fail("cannot stop CPU clock"); cpu_ns = elapsed_ns(&start, &end); for (size_t index = output_size; index < usable; ++index) { if (output[index] != 0xa5) { guards = 0; break; } } write_file(argv[3], output, result.produced); printf("cleanup=%d codec_error=%d consumed=%zu cpu_ns=%" PRIu64 " guards=%d iterations=%d produced=%zu status=%zu stream_end=%d " "workspace_peak=%zu\n", result.cleanup, result.codec_error, result.consumed, cpu_ns, guards, iterations, result.produced, result.status, result.stream_end, result.workspace_peak); munmap(mapping, usable + 2 * page_size); free(input); return 0; } ''' def compile_oracle(temp: Path) -> tuple[Path, dict[str, Any]]: source = temp / "p15-086-oracle.c" binary = temp / "p15-086-oracle" source.write_text(ORACLE_SOURCE, encoding="ascii") run_logged( [ "cc", "-O2", "-std=c17", "-Wall", "-Wextra", "-Werror", str(source), "-o", str(binary), "-llz4", "-llzma", "-lz", "-lzstd", ], temp, OUTPUT / "logs/oracle-build.log", ) versions = { name: subprocess.check_output(["pkg-config", "--modversion", package], text=True).strip() for name, package in ( ("liblz4", "liblz4"), ("liblzma", "liblzma"), ("zlib", "zlib"), ("libzstd", "libzstd"), ) } if versions != SPEC["libraries"]: raise GateFailure("INFRA_BLOCKED", f"independent library versions changed: {versions}") return binary, { "binary_sha256": sha256_path(binary), "source_sha256": sha256_path(source), **versions, } def decode( oracle: Path, codec: str, data: bytes, output_size: int, full: bool, dict_size: int, iterations: int, temp: Path, label: str, ) -> tuple[dict[str, int], bytes]: input_path = temp / f"{label}.input" output_path = temp / f"{label}.output" input_path.write_bytes(data) completed = run_logged( [ str(oracle), codec, str(input_path), str(output_path), str(output_size), "1" if full else "0", str(dict_size), str(iterations), ], temp, OUTPUT / f"logs/oracle-{label}.log", ) record: dict[str, int] = {} for field in completed.stdout.strip().split(): key, value = field.split("=", 1) record[key] = int(value) required = { "cleanup", "codec_error", "consumed", "cpu_ns", "guards", "iterations", "produced", "status", "stream_end", "workspace_peak", } if set(record) != required: raise GateFailure("INFRA_BLOCKED", f"oracle fields changed for {label}") return record, output_path.read_bytes() def full_errno(record: dict[str, int], data_size: int, expected: bytes, output: bytes) -> int: if ( record["codec_error"] != 0 or record["cleanup"] != 1 or record["guards"] != 1 or record["consumed"] != data_size or record["produced"] != len(expected) or record["stream_end"] != 1 or output != expected ): return EINTEGRITY return 0 def partial_errno(record: dict[str, int], expected: bytes, output: bytes) -> int: if ( record["codec_error"] != 0 or record["cleanup"] != 1 or record["guards"] != 1 or record["produced"] != len(expected) or output != expected ): return EINTEGRITY return 0 def fsck_extract(fsck: Path, image: Path, destination: Path, label: str) -> dict[str, Any]: destination.mkdir() completed = run_logged( [str(fsck), f"--extract={destination}", str(image)], image.parent, OUTPUT / f"logs/fsck-{label}.log", expected=set(range(0, 256)), ) return {"exit": completed.returncode, "success": completed.returncode == 0} def select_extent( dump: Path, image: Path, payload_path: str, minimum: int, label: str, ) -> dict[str, int]: completed = run_logged( [str(dump), f"--path={payload_path}", "-e", str(image)], image.parent, OUTPUT / f"logs/dump-{label}.log", ) choices = [ record for record in parse_extents(completed.stdout) if record["logical_length"] >= minimum and record["physical_length"] > 0 ] if not choices: raise GateFailure("STOP", f"{label} has no representative compressed extent") return max(choices, key=lambda record: record["logical_length"]) def dump_extents_compat(dump: Path, image: Path, label: str) -> tuple[list[dict[str, int]], dict[str, Any]]: attempts = [] modern = subprocess.run( [str(dump), "--path=/payload.bin", "-e", str(image)], cwd=image.parent, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=30, ) attempts.append( "$ " + " ".join([str(dump), "--path=/payload.bin", "-e", str(image)]) + "\n" + modern.stdout + f"\n[exit {modern.returncode}]\n" ) if modern.returncode == 0: records = parse_extents(modern.stdout) (OUTPUT / f"logs/dump-{label}.log").write_text("\n".join(attempts), encoding="utf-8") return records, {"mode": "path", "nid": None} matches = [] for nid in range(1, 256): completed = subprocess.run( [str(dump), f"--nid={nid}", "-e", str(image)], cwd=image.parent, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=5, ) if completed.returncode == 0 and re.search(r"^(?:File|Path) : /payload\.bin$", completed.stdout, re.MULTILINE): matches.append((nid, completed.stdout)) if len(matches) != 1: (OUTPUT / f"logs/dump-{label}.log").write_text("\n".join(attempts), encoding="utf-8") raise GateFailure("INFRA_BLOCKED", f"{label} bounded NID lookup found {len(matches)} payloads") nid, stdout = matches[0] attempts.append( "$ " + " ".join([str(dump), f"--nid={nid}", "-e", str(image)]) + "\n" + stdout + "\n[exit 0]\n" ) (OUTPUT / f"logs/dump-{label}.log").write_text("\n".join(attempts), encoding="utf-8") return parse_extents(stdout), {"mode": "bounded-nid", "nid": nid} def benchmark( oracle: Path, codec: str, stream: bytes, logical: bytes, dict_size: int, temp: Path, label: str, ) -> dict[str, Any]: samples = SPEC["cpu"]["samples"] iterations = SPEC["cpu"]["iterations_per_sample"] partial_end = SPEC["ranges"][0]["length"] full_samples = [] partial_samples = [] full_peak = 0 partial_peak = 0 for sample in range(samples): full_record, full_output = decode( oracle, codec, stream, len(logical), True, dict_size, iterations, temp, f"{label}-cpu-full-{sample}", ) partial_record, partial_output = decode( oracle, codec, stream, partial_end, False, dict_size, iterations, temp, f"{label}-cpu-partial-{sample}", ) if full_errno(full_record, len(stream), logical, full_output) != 0: raise GateFailure("STOP", f"{label} full CPU sample lost exactness") if partial_errno(partial_record, logical[:partial_end], partial_output) != 0: raise GateFailure("STOP", f"{label} partial CPU sample lost exactness") full_samples.append(full_record["cpu_ns"] / iterations) partial_samples.append(partial_record["cpu_ns"] / iterations) full_peak = max(full_peak, full_record["workspace_peak"]) partial_peak = max(partial_peak, partial_record["workspace_peak"]) full_median = statistics.median(full_samples) partial_median = statistics.median(partial_samples) return { "full_ns_per_decode": full_samples, "full_median_ns": full_median, "iterations_per_sample": iterations, "partial_ns_per_decode": partial_samples, "partial_median_ns": partial_median, "partial_to_full_percent": partial_median * 100.0 / full_median, "samples": samples, "workspace_full_peak": full_peak, "workspace_partial_peak": partial_peak, } def evaluate_ranges( oracle: Path, codec: str, stream: bytes, logical: bytes, physical_length: int, dict_size: int, temp: Path, label: str, ) -> dict[str, Any]: full_record, full_output = decode( oracle, codec, stream, len(logical), True, dict_size, 1, temp, f"{label}-full", ) if full_errno(full_record, len(stream), logical, full_output) != 0: raise GateFailure("STOP", f"{label} legal full decode is not exact") ranges = [] prefix_record: dict[str, int] | None = None for request in SPEC["ranges"]: offset = request["offset"] if request["offset"] >= 0 else len(logical) + request["offset"] if offset < 0 or request["length"] > len(logical) - offset: raise GateFailure("STOP", f"{label} cannot cover range {request['name']}") end = offset + request["length"] use_partial = end < len(logical) record, output = decode( oracle, codec, stream, end, not use_partial, dict_size, 1, temp, f"{label}-{request['name']}", ) if codec == "lz4" and use_partial: parsed_output, parsed_consumed = raw_lz4_decode(stream, end) if parsed_output != logical[:end]: raise GateFailure("STOP", f"{label} raw LZ4 range parser differs from full slice") record["consumed"] = parsed_consumed errno = ( partial_errno(record, logical[:end], output) if use_partial else full_errno(record, len(stream), logical, output) ) if errno != 0 or output[offset:end] != logical[offset:end]: raise GateFailure("STOP", f"{label} range {request['name']} differs from full slice") if use_partial and record["consumed"] >= len(stream): raise GateFailure("STOP", f"{label} range {request['name']} has no measurable input boundary") if request["name"] == "prefix": prefix_record = record ranges.append( { "consumed": record["consumed"], "decoded_prefix": end, "fallback_full": not use_partial, "guards": record["guards"], "length": request["length"], "name": request["name"], "offset": offset, "policy_errno": errno, "slice_sha256": sha256_bytes(output[offset:end]), "workspace_peak": record["workspace_peak"], } ) if prefix_record is None: raise GateFailure("INFRA_BLOCKED", "prefix request disappeared") corruption_start = max(prefix_record["consumed"] + 8, len(stream) - 64) if corruption_start >= len(stream): raise GateFailure("STOP", f"{label} cannot place corruption after partial consumption") corrupted = bytearray(stream) original_tail = bytes(corrupted[corruption_start:]) corrupted[corruption_start:] = b"\0" * (len(corrupted) - corruption_start) if bytes(corrupted[corruption_start:]) == original_tail: corrupted[-1] ^= 0x5A partial_end = SPEC["ranges"][0]["length"] corrupt_partial_record, corrupt_partial_output = decode( oracle, codec, bytes(corrupted), partial_end, False, dict_size, 1, temp, f"{label}-corrupt-partial", ) if codec == "lz4": parsed_output, parsed_consumed = raw_lz4_decode(bytes(corrupted), partial_end) if parsed_output != logical[:partial_end]: raise GateFailure("STOP", f"{label} corrupt raw LZ4 prefix differs from full slice") corrupt_partial_record["consumed"] = parsed_consumed corrupt_partial_errno = partial_errno( corrupt_partial_record, logical[:partial_end], corrupt_partial_output ) corrupt_full_record, corrupt_full_output = decode( oracle, codec, bytes(corrupted), len(logical), True, dict_size, 1, temp, f"{label}-corrupt-full", ) corrupt_full_errno = full_errno( corrupt_full_record, len(corrupted), logical, corrupt_full_output ) truncated_record, truncated_output = decode( oracle, codec, stream[:-1], len(logical), True, dict_size, 1, temp, f"{label}-truncated", ) truncated_errno = full_errno( truncated_record, len(stream) - 1, logical, truncated_output ) if corrupt_partial_errno != 0 or corrupt_full_errno != EINTEGRITY: raise GateFailure("STOP", f"{label} range-after corruption contract is not exact") if truncated_errno != EINTEGRITY: raise GateFailure("STOP", f"{label} one-byte truncation is not positive EINTEGRITY") cpu = benchmark(oracle, codec, stream, logical, dict_size, temp, label) prefix_end = SPEC["ranges"][0]["length"] caller = SPEC["ranges"][0]["length"] baseline_peak = physical_length + len(logical) + caller + cpu["workspace_full_peak"] candidate_peak = physical_length + prefix_end + caller + cpu["workspace_partial_peak"] reduction = (baseline_peak - candidate_peak) * 100.0 / baseline_peak workspace_budget = SPEC["hard_budget"]["workspace_bytes"][codec] hard_cap = ( SPEC["hard_budget"]["input_bytes"] + SPEC["hard_budget"]["decoded_bytes"] + SPEC["hard_budget"]["caller_bytes"] + workspace_budget ) if physical_length > SPEC["hard_budget"]["input_bytes"]: raise GateFailure("STOP", f"{label} input exceeds the fixed pcluster budget") if len(logical) > SPEC["hard_budget"]["decoded_bytes"]: raise GateFailure("STOP", f"{label} output exceeds the fixed decoded budget") if max(cpu["workspace_full_peak"], cpu["workspace_partial_peak"]) > workspace_budget: raise GateFailure("STOP", f"{label} workspace exceeds its hard budget") if max(baseline_peak, candidate_peak) > hard_cap: raise GateFailure("STOP", f"{label} peak temporary bytes exceed the hard cap") threshold = SPEC["thresholds"]["minimum_peak_temporary_reduction_percent"] fallback_reasons = [] if reduction < threshold: fallback_reasons.append( f"peak temporary reduction {reduction:.3f}% is below {threshold}%" ) if cpu["partial_median_ns"] > cpu["full_median_ns"] * 1.25: fallback_reasons.append("partial CPU median exceeds the 125% hard regression budget") return { "budget": { "baseline_peak_temporary_bytes": baseline_peak, "candidate_peak_temporary_bytes": candidate_peak, "hard_cap_bytes": hard_cap, "peak_reduction_percent": reduction, "threshold_percent": threshold, "workspace_budget_bytes": workspace_budget, }, "corruption": { "full_errno": corrupt_full_errno, "partial_consumed": corrupt_partial_record["consumed"], "partial_errno": corrupt_partial_errno, "starts_after_partial_consumed": corruption_start > prefix_record["consumed"], "starts_at_stream_byte": corruption_start, "truncated_errno": truncated_errno, }, "cpu": cpu, "full": { **full_record, "output_sha256": sha256_bytes(full_output), "policy_errno": 0, }, "ranges": ranges, "partial_capability": "FULL_FALLBACK" if fallback_reasons else "GO", "partial_capability_reasons": fallback_reasons, } def build_stream_codecs( source_dir: Path, source: bytes, oracle: Path, temp: Path, ) -> list[dict[str, Any]]: mkfs = Path(SPEC["tools"]["mkfs.erofs"]["path"]) fsck = Path(SPEC["tools"]["fsck.erofs"]["path"]) dump = Path(SPEC["tools"]["dump.erofs"]["path"]) records = [] for codec, codec_spec in sorted(SPEC["stream"]["codecs"].items()): images = [] for repeat in ("a", "b"): image = temp / f"{codec}-{repeat}.erofs" run_logged( [str(mkfs), *codec_spec["mkfs_args"], str(image), str(source_dir)], temp, OUTPUT / f"logs/mkfs-{codec}-{repeat}.log", ) images.append(image) hashes = [sha256_path(image) for image in images] if hashes[0] != hashes[1]: raise GateFailure("INFRA_BLOCKED", f"{codec} image is not byte reproducible") valid_fsck = fsck_extract(fsck, images[0], temp / f"extract-{codec}", f"{codec}-valid") if not valid_fsck["success"] or (temp / f"extract-{codec}/payload.bin").read_bytes() != source: raise GateFailure("INFRA_BLOCKED", f"{codec} legal image extraction mismatch") extent = select_extent(dump, images[0], "/payload.bin", 12288, codec) image_bytes = images[0].read_bytes() start = extent["physical_offset"] end = start + extent["physical_length"] block = image_bytes[start:end] leading = next((index for index, value in enumerate(block) if value), len(block)) if leading == len(block): raise GateFailure("STOP", f"{codec} selected pcluster is all padding") stream = block[leading:] logical = source[ extent["logical_offset"] : extent["logical_offset"] + extent["logical_length"] ] evaluated = evaluate_ranges( oracle, codec, stream, logical, extent["physical_length"], codec_spec["dict_size"], temp, codec, ) corrupt_start = evaluated["corruption"]["starts_at_stream_byte"] corrupted_image = bytearray(image_bytes) absolute = start + leading + corrupt_start corrupted_image[absolute:end] = b"\0" * (end - absolute) corrupt_path = temp / f"{codec}-corrupt.erofs" corrupt_path.write_bytes(corrupted_image) corrupt_fsck = fsck_extract( fsck, corrupt_path, temp / f"extract-{codec}-corrupt", f"{codec}-corrupt" ) if corrupt_fsck["success"]: raise GateFailure("STOP", f"{codec} real EROFS tail corruption escaped full fsck") records.append( { "codec": codec, "extent": {**extent, "leading_zero_bytes": leading, "stream_bytes": len(stream)}, "image_repeated_sha256": hashes, "real_corrupt_fsck": corrupt_fsck, "real_valid_fsck": valid_fsck, **evaluated, } ) return records def build_lz4_generations( source_dir: Path, source: bytes, oracle: Path, temp: Path, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: clone = temp / "erofs-utils" run_logged( ["git", "clone", "--no-checkout", SPEC["lz4"]["generator_url"], str(clone)], temp, OUTPUT / "logs/lz4-upstream-clone.log", timeout=90, ) generation_records = [] evaluated_records = [] for version in SPEC["lz4"]["generators"]: name = version["name"] worktree = temp / f"erofs-{name}" run_logged( ["git", "-C", str(clone), "worktree", "add", "--detach", str(worktree), version["commit"]], temp, OUTPUT / f"logs/lz4-{name}-worktree.log", ) resolved = subprocess.check_output(["git", "-C", str(worktree), "rev-parse", "HEAD"], text=True).strip() if resolved != version["commit"]: raise GateFailure("INFRA_BLOCKED", f"LZ4 generator identity changed for {name}") run_logged(["./autogen.sh"], worktree, OUTPUT / f"logs/lz4-{name}-autogen.log") run_logged(["./configure", "--disable-fuse"], worktree, OUTPUT / f"logs/lz4-{name}-configure.log") run_logged(["make", "-s", "-j2"], worktree, OUTPUT / f"logs/lz4-{name}-make.log") mkfs = worktree / "mkfs/mkfs.erofs" fsck = worktree / "fsck/fsck.erofs" dump = worktree / "dump/dump.erofs" images = [] for repeat in ("a", "b"): image = temp / f"lz4-{name}-{repeat}.erofs" run_logged( [str(mkfs), *SPEC["lz4"]["mkfs_args"], str(image), str(source_dir)], worktree, OUTPUT / f"logs/lz4-{name}-mkfs-{repeat}.log", ) images.append(image) hashes = [sha256_path(image) for image in images] if hashes[0] != hashes[1]: raise GateFailure("INFRA_BLOCKED", f"LZ4 {name} image is not byte reproducible") extract = temp / f"extract-lz4-{name}" extract.mkdir() run_logged( [str(fsck), "--extract", str(images[0])], extract, OUTPUT / f"logs/lz4-{name}-fsck.log", ) extents, dump_identity = dump_extents_compat(dump, images[0], f"lz4-{name}") choices = [record for record in extents if record["logical_length"] >= 12288] if not choices: raise GateFailure("STOP", f"LZ4 {name} has no representative compressed extent") extent = max(choices, key=lambda record: record["logical_length"]) image_bytes = images[0].read_bytes() reconstructed = bytearray(len(source)) for current in extents: current_block = image_bytes[ current["physical_offset"] : current["physical_offset"] + current["physical_length"] ] current_logical, _ = raw_lz4_decode(current_block, current["logical_length"]) logical_start = current["logical_offset"] logical_end = logical_start + current["logical_length"] if current_logical != source[logical_start:logical_end]: raise GateFailure("STOP", f"LZ4 {name} extent differs from source") reconstructed[logical_start:logical_end] = current_logical if bytes(reconstructed) != source: raise GateFailure("STOP", f"LZ4 {name} full reconstruction differs from source") block = image_bytes[ extent["physical_offset"] : extent["physical_offset"] + extent["physical_length"] ] logical = source[ extent["logical_offset"] : extent["logical_offset"] + extent["logical_length"] ] parsed, consumed = raw_lz4_decode(block, len(logical)) if parsed != logical: raise GateFailure("STOP", f"LZ4 {name} independent parser differs from source") stream = block[:consumed] evaluated = evaluate_ranges( oracle, "lz4", stream, logical, extent["physical_length"], 0, temp, f"lz4-{name}", ) generation_records.append( { "commit": resolved, "extent": {**extent, "stream_bytes": consumed}, "dump_identity": dump_identity, "image_repeated_sha256": hashes, "name": name, } ) evaluated_records.append({"generator": name, **evaluated}) representative = evaluated_records[-1] capability = ( "GO" if all(record["partial_capability"] == "GO" for record in evaluated_records) else "FULL_FALLBACK" ) lz4_record = { "codec": "lz4", "generations": generation_records, "partial_capability": capability, "representative": representative, } return lz4_record, evaluated_records def verify_tools() -> dict[str, Any]: records = {} for name, expected in SPEC["tools"].items(): path = Path(expected["path"]) if not path.is_file() or sha256_path(path) != expected["sha256"]: raise GateFailure("INFRA_BLOCKED", f"tool identity changed: {name}") records[name] = expected mkfs_version = subprocess.check_output( [SPEC["tools"]["mkfs.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True ).strip() fsck_version = subprocess.check_output( [SPEC["tools"]["fsck.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True ).strip() for codec in ("lz4", "lzma", "deflate", "zstd"): if codec not in mkfs_version or codec not in fsck_version: raise GateFailure("STOP", f"{codec} lacks a real mkfs/fsck path") records["mkfs_version"] = mkfs_version records["fsck_version"] = fsck_version return records def verify_source_contract(sources: dict[str, str]) -> dict[str, Any]: req = sources["repo-pre-15/src/compress.h"] if "size_t outputsize;" not in req or "bool partial_decoding;" not in req: raise GateFailure("INFRA_BLOCKED", "FreeBSD partial request contract changed") anchors = { "lz4": "rq->partial_decoding", "lzma": "!rq->partial_decoding", "deflate": "if (rq->partial_decoding)", "zstd": "if (rq->partial_decoding)", } for codec, anchor in anchors.items(): if anchor not in sources[f"repo-pre-15/src/decompressor_{codec}.c"]: raise GateFailure("INFRA_BLOCKED", f"FreeBSD {codec} partial primitive changed") dispatch = sources["repo-pre-15/src/decompressor.c"] if ".outputsize = dstlen" not in dispatch or ".partial_decoding = partial" not in dispatch: raise GateFailure("INFRA_BLOCKED", "FreeBSD dispatch partial propagation changed") zdata = sources["repo-pre-15/src/zdata.c"] required = ( "partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0;", "if (!partial && decoded_len != map->m_llen)", "erofs_put_metabuf(&buf);", "erofs_brelse(compressed);", "free(decoded, M_EROFS);", "*bufp = decoded;", ) if not all(anchor in zdata for anchor in required): raise GateFailure("INFRA_BLOCKED", "FreeBSD buffer/partial-ref contract changed") decode_at = zdata.index("error = z_erofs_decompress") release_at = min( zdata.index("erofs_put_metabuf(&buf);", decode_at), zdata.index("erofs_brelse(compressed);", decode_at), ) error_at = zdata.index("if (error != 0) {", release_at) free_at = zdata.index("free(decoded, M_EROFS);", error_at) publish_at = zdata.index("*bufp = decoded;", free_at) if not decode_at < release_at < error_at < free_at < publish_at: raise GateFailure("INFRA_BLOCKED", "FreeBSD decode cleanup ordering changed") linux_req = sources["src-linux/compress.h"] if "partial_decoding" not in linux_req or "outputsize" not in linux_req: raise GateFailure("INFRA_BLOCKED", "Linux partial request anchor changed") linux_anchors = { "lz4": "LZ4_decompress_safe_partial", "lzma": "!rq->partial_decoding", "deflate": "rq->partial_decoding", "zstd": "rq->outputsize + dctx.avail_out", } for codec, anchor in linux_anchors.items(): path = "src-linux/decompressor.c" if codec == "lz4" else f"src-linux/decompressor_{codec}.c" if anchor not in sources[path]: raise GateFailure("INFRA_BLOCKED", f"Linux {codec} partial anchor changed") return { "backend_partial_primitives": sorted(anchors), "buffer_release_after_decode": True, "failed_output_freed": True, "linux_prefix_contract": True, "partial_ref_preserved": True, "successful_output_published_only_after_decode": True, } def build_state_model() -> dict[str, Any]: scenarios = [ {"name": "cache-hit", "decode": "none", "cache_after": "ready-full"}, {"name": "cache-miss-partial-success", "decode": "bounded-prefix", "cache_after": "absent"}, {"name": "cache-miss-partial-failure", "decode": "bounded-prefix", "cache_after": "absent", "errno": EINTEGRITY}, {"name": "concurrent-partial-miss", "decode": "per-request-local", "shared_owner_count": 0, "shared_waiter_count": 0}, {"name": "full-request", "decode": "full", "cache_after": "existing-policy"}, {"name": "unsupported-backend", "decode": "full-fallback", "cache_after": "existing-policy"}, {"name": "partial-reference", "decode": "bounded-prefix", "cache_after": "ineligible"}, {"name": "eviction", "partial_reference_retained": False, "cache_after": "existing-policy"}, {"name": "reclaim", "partial_reference_retained": False, "cache_after": "existing-policy"}, {"name": "unmount", "partial_reference_retained": False, "drain_required": False}, {"name": "key-reuse", "partial_reference_retained": False, "cache_after": "existing-policy"}, ] prototype = { "probed_codecs": ["lz4", "lzma", "deflate", "zstd"], "fallback_backends": ["shifted", "interlaced", "unknown"], "new_persistent_state": False, "overflow_errno": EOVERFLOW, "pool_exhaustion": "not applicable; B28 adds no pool", "rule": "cache hit first; strict subextent on a gate-authorized codec uses mapoff+want bytes and never publishes a partial cache entry; all other cases use exact full decode", "scenarios": scenarios, } write_json(OUTPUT / "candidate-state-model.json", prototype) prototype_source = """P15-086 test-only candidate model\n\ncapable = lz4|lzma|deflate|zstd\nrequest_end = checked_add(mapoff, want)\nif cache_hit: copy full cache\nelif capable and request_end < m_llen: decode(request_end, partial=true), no cache publish\nelse: decode(m_llen, partial=partial_ref), preserve existing full cache policy\nfailed local output is freed; input is released after callback; no persistent state is added\n""" prototype_path = OUTPUT / "candidate-prototype.txt" prototype_path.write_text(prototype_source, encoding="ascii") return { "prototype_sha256": sha256_path(prototype_path), "scenario_count": len(scenarios), "state_machine": "PASS", } def finalize() -> None: entries = [] for path in sorted(OUTPUT.rglob("*")): if path.is_file() and path.name != "SHA256SUMS": entries.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}") (OUTPUT / "SHA256SUMS").write_text("\n".join(entries) + "\n", encoding="ascii") result: dict[str, Any] = {} exit_code = 0 owned_temp: str | None = None try: if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-086" or SPEC.get("gates") != ["G04", "G05"]: raise GateFailure("INFRA_BLOCKED", "gate input identity changed") resolved = git("rev-parse", REQUESTED_BASE) if resolved != SPEC["required_base"]: raise GateFailure("INFRA_BLOCKED", f"wrong DUT base: {resolved}") sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]} source_hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()} if source_hashes != SPEC["source_sha256"]: raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed") write_json(OUTPUT / "source-sha256.json", source_hashes) freebsd_head = subprocess.check_output( ["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True ).strip() freebsd_hashes = { path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"] } if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]: raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed") errno_source = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8") if "#define\tEINTEGRITY\t97" not in errno_source or "#define\tEOVERFLOW\t84" not in errno_source: raise GateFailure("INFRA_BLOCKED", "positive FreeBSD errno identity changed") write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes}) write_json(OUTPUT / "toolchain.json", verify_tools()) write_json(OUTPUT / "source-contract.json", verify_source_contract(sources)) write_json(OUTPUT / "state-model-result.json", build_state_model()) with tempfile.TemporaryDirectory(prefix="p15-086-g04-g05-") as temporary: owned_temp = temporary temp = Path(temporary) oracle, oracle_identity = compile_oracle(temp) write_json(OUTPUT / "independent-oracle.json", oracle_identity) stream_dir = temp / "stream-source" stream_source = make_stream_source(stream_dir) stream_records = build_stream_codecs(stream_dir, stream_source, oracle, temp) lz4_dir = temp / "lz4-source" lz4_source = make_lz4_source(lz4_dir) lz4_record, lz4_generation_results = build_lz4_generations( lz4_dir, lz4_source, oracle, temp ) codec_records = [lz4_record, *stream_records] write_json(OUTPUT / "codec-results.json", codec_records) write_json(OUTPUT / "lz4-generation-results.json", lz4_generation_results) per_codec = { record["codec"]: ( record["partial_capability"] if record["codec"] == "lz4" else record["partial_capability"] ) for record in codec_records } if set(per_codec) != {"lz4", "lzma", "deflate", "zstd"}: raise GateFailure("STOP", "the real supported codec set is incomplete") enabled = sorted(codec for codec, decision in per_codec.items() if decision == "GO") if not enabled: raise GateFailure("STOP", "no codec meets the P15-086 quantified benefit gate") write_json( OUTPUT / "authorized-capabilities.json", { "full_fallback": sorted( codec for codec, decision in per_codec.items() if decision == "FULL_FALLBACK" ), "partial": enabled, "persistent_state_added": False, "schema": 1, }, ) result = { "b28": "AUTHORIZED", "candidate": "P15-086", "cleanup": "PASS", "codecs": per_codec, "cpu_samples_per_mode": SPEC["cpu"]["samples"], "full_feature_suite": "NOT_RUN", "g04": "GO", "g05": "GO", "gates": ["G04", "G05"], "hard_budget": "PASS", "partial_enabled_codecs": enabled, "oracle": "real EROFS images plus independent liblz4/liblzma/zlib/libzstd consumed-byte oracle", "qemu": "NOT_RUN", "qemu_reason": "Stage0 host gate owns correctness and budget authorization; B28 acceptance owns strict TC176 QEMU", "requested_base": REQUESTED_BASE, "resolved_base": resolved, "schema": 1, "status": "GO", "typed_errno": "PASS", } write_json(OUTPUT / "result.json", result) except GateFailure as failure: result = { "b28": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN", "candidate": "P15-086", "full_feature_suite": "NOT_RUN", "gates": ["G04", "G05"], "qemu": "NOT_RUN", "reason": failure.reason, "requested_base": REQUESTED_BASE, "schema": 1, "status": failure.status, } write_json(OUTPUT / "result.json", result) exit_code = 1 if failure.status == "STOP" else 21 except (OSError, subprocess.SubprocessError, ValueError) as failure: result = { "b28": "NOT_RUN", "candidate": "P15-086", "full_feature_suite": "NOT_RUN", "gates": ["G04", "G05"], "qemu": "NOT_RUN", "reason": f"gate infrastructure failure: {failure}", "requested_base": REQUESTED_BASE, "schema": 1, "status": "INFRA_BLOCKED", } write_json(OUTPUT / "result.json", result) exit_code = 21 finally: cleanup_record = { "owned_temp": owned_temp, "owned_temp_removed": owned_temp is None or not Path(owned_temp).exists(), "protected_pid_touched": False, "protected_port_touched": False, "qemu_started": False, "shared_base_image_touched": False, } if not cleanup_record["owned_temp_removed"]: result = { "b28": "NOT_RUN", "candidate": "P15-086", "full_feature_suite": "NOT_RUN", "gates": ["G04", "G05"], "qemu": "NOT_RUN", "reason": "owned gate temporary directory survived cleanup", "requested_base": REQUESTED_BASE, "schema": 1, "status": "INFRA_BLOCKED", } write_json(OUTPUT / "result.json", result) exit_code = 21 write_json(OUTPUT / "owned-cleanup.json", cleanup_record) finalize() print(json.dumps(result, sort_keys=True)) raise SystemExit(exit_code) PY