565 lines
18 KiB
Bash
Executable File
565 lines
18 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
umask 022
|
|
|
|
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
|
input=$gate_dir/P15-045-input.json
|
|
base=
|
|
output=
|
|
deadline=90
|
|
|
|
while test "$#" -gt 0; do
|
|
case "$1" in
|
|
--base)
|
|
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 20; }
|
|
base=$2
|
|
shift 2
|
|
;;
|
|
--output)
|
|
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 20; }
|
|
output=$2
|
|
shift 2
|
|
;;
|
|
*)
|
|
printf 'unknown argument: %s\n' "$1" >&2
|
|
exit 20
|
|
;;
|
|
esac
|
|
done
|
|
|
|
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 20; }
|
|
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 20; }
|
|
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 20; }
|
|
for tool in git mktemp python3 sha256sum timeout; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'missing required host tool: %s\n' "$tool" >&2
|
|
exit 21
|
|
}
|
|
done
|
|
case "$output" in
|
|
/*) ;;
|
|
*) output=$PWD/$output ;;
|
|
esac
|
|
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 20; }
|
|
mkdir -p "$output"
|
|
work=$(mktemp -d "${TMPDIR:-/tmp}/P15-045.XXXXXX")
|
|
|
|
cleanup_trap()
|
|
{
|
|
rm -rf -- "$work"
|
|
}
|
|
trap cleanup_trap EXIT HUP INT TERM
|
|
|
|
python3 -B - "$output/command-argv.json" "$0" --base "$base" --output "$output" <<'PY'
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
Path(sys.argv[1]).write_text(
|
|
json.dumps(sys.argv[2:], ensure_ascii=True, separators=(",", ":")) + "\n",
|
|
encoding="ascii",
|
|
)
|
|
PY
|
|
python3 -B - "$output/ownership.json" "$work" "$output" <<'PY'
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
Path(sys.argv[1]).write_text(
|
|
json.dumps(
|
|
{
|
|
"owned_temporary_paths": [sys.argv[2]],
|
|
"persistent_output": sys.argv[3],
|
|
"owned_processes": [],
|
|
"owned_ports": [],
|
|
"qemu": "NOT_RUN",
|
|
},
|
|
ensure_ascii=True,
|
|
indent=2,
|
|
sort_keys=True,
|
|
) + "\n",
|
|
encoding="ascii",
|
|
)
|
|
PY
|
|
|
|
set +e
|
|
timeout -k 5 "$deadline" python3 -B - \
|
|
"$root" "$input" "$base" "$output" "$deadline" <<'PY' \
|
|
>"$output/stdout.log" 2>"$output/stderr.log"
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(sys.argv[1])
|
|
INPUT = Path(sys.argv[2])
|
|
REQUESTED_BASE = sys.argv[3]
|
|
OUTPUT = Path(sys.argv[4])
|
|
DEADLINE = int(sys.argv[5])
|
|
SPEC = json.loads(INPUT.read_text(encoding="utf-8"))
|
|
|
|
|
|
class GateFailure(RuntimeError):
|
|
def __init__(self, status: str, reason: str):
|
|
super().__init__(reason)
|
|
self.status = status
|
|
self.reason = reason
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.write_text(
|
|
json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
|
|
encoding="ascii",
|
|
)
|
|
|
|
|
|
def run_bytes(argv: list[str], allowed: set[int] | None = None) -> bytes:
|
|
try:
|
|
completed = subprocess.run(
|
|
argv,
|
|
cwd=ROOT,
|
|
check=False,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
timeout=30,
|
|
)
|
|
except subprocess.TimeoutExpired as error:
|
|
raise GateFailure(
|
|
"INFRA_BLOCKED", f"command timed out: {' '.join(argv)}"
|
|
) from error
|
|
accepted = {0} if allowed is None else allowed
|
|
if completed.returncode not in accepted:
|
|
detail = completed.stderr.decode("utf-8", errors="replace").strip()
|
|
raise GateFailure(
|
|
"RUNNER_FAIL",
|
|
f"command failed ({completed.returncode}): {' '.join(argv)}: {detail}",
|
|
)
|
|
return completed.stdout
|
|
|
|
|
|
def git_text(*args: str, allowed: set[int] | None = None) -> str:
|
|
return run_bytes(["git", "-C", str(ROOT), *args], allowed).decode(
|
|
"utf-8", errors="strict"
|
|
).strip()
|
|
|
|
|
|
def source_at(commit: str, relative: str) -> bytes:
|
|
return run_bytes(["git", "-C", str(ROOT), "show", f"{commit}:{relative}"])
|
|
|
|
|
|
def truthy(value: Any) -> bool:
|
|
if value is True:
|
|
return True
|
|
if isinstance(value, str):
|
|
return value.strip().lower() in {"true", "yes", "required", "enabled", "1"}
|
|
if isinstance(value, int):
|
|
return value == 1
|
|
return False
|
|
|
|
|
|
def flattened(value: Any, prefix: str = "") -> list[tuple[str, Any]]:
|
|
records: list[tuple[str, Any]] = []
|
|
if isinstance(value, dict):
|
|
for key, child in value.items():
|
|
path = f"{prefix}.{key}" if prefix else str(key)
|
|
records.append((path.lower().replace("-", "_"), child))
|
|
records.extend(flattened(child, path))
|
|
elif isinstance(value, list):
|
|
for index, child in enumerate(value):
|
|
records.extend(flattened(child, f"{prefix}[{index}]"))
|
|
return records
|
|
|
|
|
|
def json_demand_witness(value: Any) -> dict[str, Any] | None:
|
|
if not isinstance(value, dict):
|
|
return None
|
|
records = flattened(value)
|
|
kind = any(
|
|
any(token in key for token in ("manifest_type", "manifest_kind", "document_type"))
|
|
and isinstance(item, str)
|
|
and any(token in item.lower() for token in ("support", "deploy"))
|
|
for key, item in records
|
|
)
|
|
version = any(
|
|
"version" in key and isinstance(item, (str, int)) and str(item).strip()
|
|
for key, item in records
|
|
)
|
|
targets = any(
|
|
any(token in key for token in ("deployment_target", "support_target", "product_target"))
|
|
and ((isinstance(item, str) and item.strip()) or (isinstance(item, list) and item))
|
|
for key, item in records
|
|
)
|
|
required = any(
|
|
"zstd" in key
|
|
and any(token in key for token in ("default", "required", "out_of_box", "outofthebox"))
|
|
and truthy(item)
|
|
for key, item in records
|
|
)
|
|
if kind and version and targets and required:
|
|
return {
|
|
"format": "json",
|
|
"manifest_kind": True,
|
|
"version": True,
|
|
"deployment_targets": True,
|
|
"zstd_default_required": True,
|
|
}
|
|
return None
|
|
|
|
|
|
def text_demand_witness(text: str) -> dict[str, Any] | None:
|
|
fields: dict[str, str] = {}
|
|
for line in text.splitlines():
|
|
match = re.fullmatch(r"([A-Za-z][A-Za-z0-9_-]*):[ \t]*(.*)", line)
|
|
if match is not None:
|
|
fields[match.group(1).lower().replace("-", "_")] = match.group(2).strip()
|
|
kind = fields.get("manifest_type", "").lower()
|
|
version = fields.get("manifest_version", "")
|
|
targets = fields.get("deployment_targets", "") or fields.get("support_targets", "")
|
|
required = fields.get("zstd_default_required", "")
|
|
if any(token in kind for token in ("support", "deploy")) and version and targets and truthy(required):
|
|
return {
|
|
"format": "text-header",
|
|
"manifest_kind": True,
|
|
"version": True,
|
|
"deployment_targets": True,
|
|
"zstd_default_required": True,
|
|
}
|
|
return None
|
|
|
|
|
|
def parse_ls_tree(raw: bytes) -> dict[str, dict[str, Any]]:
|
|
entries: dict[str, dict[str, Any]] = {}
|
|
for record in raw.split(b"\0"):
|
|
if not record:
|
|
continue
|
|
metadata, path_bytes = record.split(b"\t", 1)
|
|
mode, kind, oid, size = metadata.decode("ascii").split()
|
|
path = path_bytes.decode("utf-8", errors="strict")
|
|
entries[path] = {
|
|
"mode": mode,
|
|
"kind": kind,
|
|
"oid": oid,
|
|
"size": int(size),
|
|
}
|
|
return entries
|
|
|
|
|
|
def grep_paths(commit: str) -> list[str]:
|
|
raw = run_bytes(
|
|
[
|
|
"git", "-C", str(ROOT), "grep", "-I", "-l", "-i",
|
|
"-e", "zstd", "-e", "zstandard", commit, "--",
|
|
"planning/pre15/evidence",
|
|
],
|
|
{0, 1},
|
|
)
|
|
prefix = f"{commit}:"
|
|
paths = []
|
|
for line in raw.decode("utf-8", errors="strict").splitlines():
|
|
if not line.startswith(prefix):
|
|
raise GateFailure("RUNNER_FAIL", f"unexpected git grep path: {line}")
|
|
paths.append(line[len(prefix):])
|
|
return sorted(paths)
|
|
|
|
|
|
def main() -> int:
|
|
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-045":
|
|
raise GateFailure("RUNNER_FAIL", "invalid P15-045 input identity")
|
|
if SPEC.get("gate") != "G07" or SPEC.get("batch") != "B35":
|
|
raise GateFailure("RUNNER_FAIL", "invalid G07/B35 input identity")
|
|
resolved = git_text("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
|
|
if resolved != SPEC["required_base"]:
|
|
raise GateFailure(
|
|
"INFRA_BLOCKED",
|
|
f"P15-045 must replay {SPEC['required_base']}, got {resolved}",
|
|
)
|
|
evidence_tree = git_text("rev-parse", f"{resolved}:planning/pre15/evidence")
|
|
if evidence_tree != SPEC["evidence_tree_oid"]:
|
|
raise GateFailure("INFRA_BLOCKED", "frozen project evidence tree changed")
|
|
|
|
source_hashes = {
|
|
path: sha256_bytes(source_at(resolved, path))
|
|
for path in SPEC["source_sha256"]
|
|
}
|
|
if source_hashes != SPEC["source_sha256"]:
|
|
raise GateFailure("INFRA_BLOCKED", "frozen G07 source identity changed")
|
|
write_json(OUTPUT / "source-sha256.json", source_hashes)
|
|
addendum_paths = (
|
|
"repo-pre-15/tests/pre15/gates/P15-045.sh",
|
|
"repo-pre-15/tests/pre15/gates/P15-045-input.json",
|
|
"repo-pre-15/docs/pre15-stage0/P15-045.md",
|
|
)
|
|
addendum_hashes = {
|
|
path: sha256_bytes((ROOT / path).read_bytes()) for path in addendum_paths
|
|
}
|
|
write_json(OUTPUT / "gate-addendum-sha256.json", addendum_hashes)
|
|
|
|
tree_entries = parse_ls_tree(
|
|
run_bytes(
|
|
[
|
|
"git", "-C", str(ROOT), "ls-tree", "-r", "--long", "-z",
|
|
resolved, "--", "planning/pre15/evidence",
|
|
]
|
|
)
|
|
)
|
|
if len(tree_entries) != SPEC["evidence_tracked_path_count"]:
|
|
raise GateFailure("INFRA_BLOCKED", "frozen evidence path count changed")
|
|
prohibited = set(SPEC["prohibited_paths"])
|
|
if prohibited.intersection(tree_entries):
|
|
raise GateFailure("RUNNER_FAIL", "prohibited path entered evidence scan")
|
|
|
|
zstd_paths = grep_paths(resolved)
|
|
maximum_bytes = SPEC["demand_manifest_contract"]["maximum_text_candidate_bytes"]
|
|
demand_terms = tuple(SPEC["demand_manifest_contract"]["demand_terms"])
|
|
candidates = []
|
|
qualified = []
|
|
oversized = []
|
|
for path in zstd_paths:
|
|
entry = tree_entries[path]
|
|
if entry["size"] > maximum_bytes:
|
|
oversized.append({"path": path, **entry})
|
|
continue
|
|
data = source_at(resolved, path)
|
|
text = data.decode("utf-8", errors="replace")
|
|
lowered = text.lower()
|
|
signals = sorted(term for term in demand_terms if term.lower() in lowered)
|
|
if not signals:
|
|
continue
|
|
witness = None
|
|
if path.endswith(".json"):
|
|
try:
|
|
witness = json_demand_witness(json.loads(text))
|
|
except json.JSONDecodeError:
|
|
witness = None
|
|
if witness is None:
|
|
witness = text_demand_witness(text)
|
|
record = {
|
|
"path": path,
|
|
"blob_oid": entry["oid"],
|
|
"bytes": entry["size"],
|
|
"sha256": sha256_bytes(data),
|
|
"demand_signals": signals,
|
|
"qualification": witness,
|
|
"disposition": (
|
|
"QUALIFIED versioned deployment-demand manifest"
|
|
if witness is not None
|
|
else "test/build/planning evidence; no explicit versioned deployment demand"
|
|
),
|
|
}
|
|
candidates.append(record)
|
|
if witness is not None:
|
|
qualified.append(record)
|
|
|
|
if oversized:
|
|
write_json(OUTPUT / "oversized-candidates.json", oversized)
|
|
raise GateFailure(
|
|
"INFRA_BLOCKED", "a Zstd evidence candidate exceeds the bounded review size"
|
|
)
|
|
|
|
approved = SPEC["approved_demand_manifest_sha256"]
|
|
actual_approved = {record["path"]: record["sha256"] for record in qualified}
|
|
if actual_approved != approved:
|
|
raise GateFailure(
|
|
"INFRA_BLOCKED", "qualified demand inventory differs from frozen gate input"
|
|
)
|
|
|
|
makefile = source_at(resolved, "repo-pre-15/src/Makefile").decode("utf-8")
|
|
zstd_source = source_at(
|
|
resolved, "repo-pre-15/src/decompressor_zstd.c"
|
|
).decode("utf-8")
|
|
readme = source_at(resolved, "repo-pre-15/README.md").decode("utf-8")
|
|
manual = source_at(resolved, "repo-pre-15/docs/erofs.5").decode("utf-8")
|
|
policy_checks = {
|
|
"makefile_default_is_opt_in_zero": makefile.count("WITH_ZSTDIO?= 0") == 1,
|
|
"enabled_build_defines_zstdio": "CFLAGS.decompressor_zstd.c+= -DZSTDIO" in makefile,
|
|
"disabled_stub_returns_eopnotsupp": (
|
|
"#else\nstatic int\nz_erofs_zstd_decompress" in zstd_source
|
|
and "return (EOPNOTSUPP);\n}\n#endif" in zstd_source
|
|
),
|
|
"readme_names_kernel_option": "kernel built with `options ZSTDIO`" in readme,
|
|
"manual_names_kernel_option": '.Cd "options ZSTDIO"' in manual,
|
|
}
|
|
if not all(policy_checks.values()):
|
|
raise GateFailure("INFRA_BLOCKED", "current opt-in ZSTDIO contract changed")
|
|
|
|
path_inventory = "\0".join(zstd_paths).encode("utf-8")
|
|
candidate_inventory = "\0".join(
|
|
record["path"] for record in candidates
|
|
).encode("utf-8")
|
|
demand_scan = {
|
|
"schema": 1,
|
|
"base": resolved,
|
|
"evidence_tree_oid": evidence_tree,
|
|
"tracked_path_count": len(tree_entries),
|
|
"zstd_text_path_count": len(zstd_paths),
|
|
"zstd_text_path_inventory_sha256": sha256_bytes(path_inventory),
|
|
"demand_candidate_count": len(candidates),
|
|
"demand_candidate_path_inventory_sha256": sha256_bytes(candidate_inventory),
|
|
"qualified_manifest_count": len(qualified),
|
|
"qualified_manifests": qualified,
|
|
"candidates": candidates,
|
|
"contract": SPEC["demand_manifest_contract"],
|
|
"prohibited_path_read": False,
|
|
}
|
|
write_json(OUTPUT / "demand-scan.json", demand_scan)
|
|
|
|
demand_present = bool(qualified)
|
|
conditions = [
|
|
{
|
|
"id": "versioned-deployment-demand",
|
|
"status": "PASS" if demand_present else "STOP",
|
|
"observation": (
|
|
f"{len(qualified)} qualified manifest(s)"
|
|
if demand_present
|
|
else "zero qualifying manifests in the frozen project evidence tree"
|
|
),
|
|
},
|
|
{
|
|
"id": "disabled-runtime-exact-EOPNOTSUPP",
|
|
"status": "NOT_RUN",
|
|
"reason": "deployment-demand prerequisite is absent",
|
|
},
|
|
{
|
|
"id": "enabled-real-reproducible-Zstd-EROFS-read",
|
|
"status": "NOT_RUN",
|
|
"reason": "deployment-demand prerequisite is absent",
|
|
},
|
|
{
|
|
"id": "FreeBSD-kernel-ZSTDIO-symbol-capability-model",
|
|
"status": "NOT_RUN",
|
|
"reason": "deployment-demand prerequisite is absent; no dependency KLD is claimed",
|
|
},
|
|
{
|
|
"id": "KLD-size-delta-at-most-64KiB-and-10-percent",
|
|
"status": "NOT_RUN",
|
|
"reason": "deployment-demand prerequisite is absent",
|
|
},
|
|
]
|
|
all_go = all(item["status"] == "PASS" for item in conditions)
|
|
if all_go:
|
|
raise GateFailure(
|
|
"RUNNER_FAIL", "false-GO guard: downstream runtime probes were not executed"
|
|
)
|
|
result = {
|
|
"schema": 1,
|
|
"gate": "G07",
|
|
"candidate": "P15-045",
|
|
"batch": "B35",
|
|
"status": "STOP",
|
|
"reason": "no concrete versioned support/deployment manifest requires Zstd enabled out of the box",
|
|
"requested_base": REQUESTED_BASE,
|
|
"resolved_base": resolved,
|
|
"deadline_seconds": DEADLINE,
|
|
"expected_exit_code": 22,
|
|
"conditions": conditions,
|
|
"false_go_guard": "GO requires every condition PASS; evaluated false",
|
|
"current_policy_static_checks": policy_checks,
|
|
"with_zstdio_default": 0,
|
|
"b35": "STOP-NO-SOURCE",
|
|
"builds": "NOT_RUN",
|
|
"qemu": "NOT_RUN",
|
|
"full_feature_suite": "NOT_RUN",
|
|
"smoke_suite": "NOT_RUN",
|
|
"generic_dependency_kld_claimed": False,
|
|
"production_source_changed": False,
|
|
}
|
|
write_json(OUTPUT / "result.json", result)
|
|
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
|
|
return 22
|
|
|
|
|
|
try:
|
|
raise SystemExit(main())
|
|
except GateFailure as error:
|
|
failure = {
|
|
"schema": 1,
|
|
"gate": "G07",
|
|
"candidate": "P15-045",
|
|
"status": error.status,
|
|
"reason": error.reason,
|
|
"requested_base": REQUESTED_BASE,
|
|
"deadline_seconds": DEADLINE,
|
|
"b35": "NOT_RUN",
|
|
"qemu": "NOT_RUN",
|
|
"production_source_changed": False,
|
|
}
|
|
write_json(OUTPUT / "result.json", failure)
|
|
print(json.dumps(failure, ensure_ascii=True, sort_keys=True))
|
|
raise SystemExit({"RUNNER_FAIL": 20, "INFRA_BLOCKED": 21}.get(error.status, 20))
|
|
PY
|
|
gate_rc=$?
|
|
set -e
|
|
|
|
cleanup_status=PASS
|
|
if ! rm -rf -- "$work"; then
|
|
cleanup_status=FAIL
|
|
gate_rc=20
|
|
fi
|
|
trap - EXIT HUP INT TERM
|
|
printf 'owned temporary path removed: %s\ncleanup=%s\n' \
|
|
"$work" "$cleanup_status" >"$output/cleanup.log"
|
|
|
|
python3 -B - "$output" "$gate_rc" "$cleanup_status" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
|
|
output = Path(sys.argv[1])
|
|
exit_code = int(sys.argv[2])
|
|
cleanup = sys.argv[3]
|
|
if cleanup != "PASS":
|
|
status = "RUNNER_FAIL"
|
|
origin = "runner"
|
|
elif exit_code == 22:
|
|
status = "STOP"
|
|
origin = "gate"
|
|
elif exit_code in (124, 137):
|
|
status = "INFRA_BLOCKED"
|
|
origin = "infrastructure"
|
|
else:
|
|
status = "RUNNER_FAIL"
|
|
origin = "runner"
|
|
(output / "attempt.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema": 1,
|
|
"exit_code": exit_code,
|
|
"status": status,
|
|
"failure_origin": origin,
|
|
"cleanup": cleanup,
|
|
"target_marker": "reached" if status == "STOP" else "not_reached",
|
|
},
|
|
ensure_ascii=True,
|
|
indent=2,
|
|
sort_keys=True,
|
|
) + "\n",
|
|
encoding="ascii",
|
|
)
|
|
lines = []
|
|
for path in sorted(output.iterdir(), key=lambda item: item.name):
|
|
if path.is_file() and path.name != "SHA256SUMS":
|
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
lines.append(f"{digest} {path.name}")
|
|
(output / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
|
|
PY
|
|
|
|
printf 'P15-045 gate exit=%s cleanup=%s output=%s\n' \
|
|
"$gate_rc" "$cleanup_status" "$output"
|
|
exit "$gate_rc"
|