update
This commit is contained in:
Executable
+464
@@ -0,0 +1,464 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
umask 022
|
||||
|
||||
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
||||
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
|
||||
dut=$root/repo-pre-15
|
||||
fixture_dir=$dut/tests/pre15/fixtures
|
||||
input=$gate_dir/P15-022-input.json
|
||||
freebsd_src=${FREEBSD_SRC:-/work/dev-freebsd-releng}
|
||||
base=
|
||||
output=
|
||||
|
||||
while test "$#" -gt 0; do
|
||||
case "$1" in
|
||||
--base)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
|
||||
base=$2
|
||||
shift 2
|
||||
;;
|
||||
--output)
|
||||
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
|
||||
output=$2
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
printf 'unknown argument: %s\n' "$1" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
|
||||
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
|
||||
case "$output" in
|
||||
/*) ;;
|
||||
*) output=$PWD/$output ;;
|
||||
esac
|
||||
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
|
||||
mkdir -p "$output"
|
||||
work=$output/.work
|
||||
mkdir "$work"
|
||||
commands=$output/COMMANDS.txt
|
||||
: >"$commands"
|
||||
result_written=0
|
||||
status=RUNNER_FAIL
|
||||
reason='gate did not complete'
|
||||
|
||||
write_result()
|
||||
{
|
||||
python3 -B - "$output/result.json" "$status" "$reason" "$base" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
status = sys.argv[2]
|
||||
result = {
|
||||
"b19a": "AUTHORIZED" if status == "GO" else "STOP-NO-SOURCE",
|
||||
"candidate": "P15-022",
|
||||
"decision": status,
|
||||
"full_feature_suite": "NOT_RUN",
|
||||
"gates": ["G03", "G05"],
|
||||
"qemu": "NOT_RUN",
|
||||
"reason": sys.argv[3],
|
||||
"requested_base": sys.argv[4],
|
||||
"schema": 1,
|
||||
"source_modified": False,
|
||||
"status": status,
|
||||
}
|
||||
path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
PY
|
||||
result_written=1
|
||||
}
|
||||
|
||||
finish()
|
||||
{
|
||||
rc=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
if test -d "$work"; then
|
||||
find "$work" -depth -delete
|
||||
fi
|
||||
if test "$result_written" -eq 0; then
|
||||
write_result
|
||||
fi
|
||||
python3 -B - "$output/cleanup.json" "$status" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
Path(sys.argv[1]).write_text(json.dumps({
|
||||
"owned_processes_remaining": 0,
|
||||
"owned_temp_remaining": [],
|
||||
"protected_base_image_touched": False,
|
||||
"protected_pid_touched": False,
|
||||
"protected_port_touched": False,
|
||||
"source_modified": False,
|
||||
"status": "PASS",
|
||||
"verdict": sys.argv[2],
|
||||
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
PY
|
||||
find "$output" -type f ! -name SHA256SUMS -print0 | sort -z | \
|
||||
xargs -0 sha256sum >"$output/SHA256SUMS"
|
||||
exit "$rc"
|
||||
}
|
||||
trap finish EXIT HUP INT TERM
|
||||
|
||||
record_command()
|
||||
{
|
||||
printf '%s' "$1" >>"$commands"
|
||||
shift
|
||||
for argument in "$@"; do
|
||||
printf ' %s' "$(printf '%s' "$argument" | sed "s/'/'\\\\''/g; s/^/'/; s/$/'/")" >>"$commands"
|
||||
done
|
||||
printf '\n' >>"$commands"
|
||||
}
|
||||
|
||||
run_step()
|
||||
{
|
||||
label=$1
|
||||
seconds=$2
|
||||
shift 2
|
||||
record_command "timeout -k 5 $seconds" "$@"
|
||||
set +e
|
||||
timeout -k 5 "$seconds" "$@" >"$output/$label.stdout" \
|
||||
2>"$output/$label.stderr"
|
||||
rc=$?
|
||||
set -e
|
||||
printf '%s\n' "$rc" >"$output/$label.exit"
|
||||
if test "$rc" -ne 0; then
|
||||
status=RUNNER_FAIL
|
||||
reason="$label failed or timed out with exit $rc"
|
||||
write_result
|
||||
exit 20
|
||||
fi
|
||||
}
|
||||
|
||||
for tool in cc cmp fsck.erofs git mkfs.erofs python3 sha256sum timeout; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
status=INFRA_BLOCKED
|
||||
reason="missing required host tool: $tool"
|
||||
write_result
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test -d "$freebsd_src/sys" || {
|
||||
status=INFRA_BLOCKED
|
||||
reason="missing exact-ABI FreeBSD source: $freebsd_src"
|
||||
write_result
|
||||
exit 21
|
||||
}
|
||||
|
||||
run_step identity 30 python3 -B - "$root" "$dut" "$input" "$base" \
|
||||
"$output/identity.json" "$freebsd_src" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
root = Path(sys.argv[1])
|
||||
dut = Path(sys.argv[2])
|
||||
spec = json.loads(Path(sys.argv[3]).read_text(encoding="ascii"))
|
||||
requested = sys.argv[4]
|
||||
output = Path(sys.argv[5])
|
||||
freebsd = Path(sys.argv[6])
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
value.update(block)
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def git(*args: str, cwd: Path = root) -> str:
|
||||
completed = subprocess.run(
|
||||
["git", "-C", str(cwd), *args], check=False,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(completed.stderr.strip())
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
resolved = git("rev-parse", f"{requested}^{{commit}}")
|
||||
if resolved != spec["required_base"]:
|
||||
raise SystemExit(f"required base {spec['required_base']}, got {resolved}")
|
||||
source_hashes = {}
|
||||
for relative, expected in {**spec["source_sha256"], **spec["linux_source_sha256"]}.items():
|
||||
data = subprocess.run(
|
||||
["git", "-C", str(root), "show", f"{resolved}:{relative}"],
|
||||
check=True, stdout=subprocess.PIPE,
|
||||
).stdout
|
||||
actual = hashlib.sha256(data).hexdigest()
|
||||
if actual != expected:
|
||||
raise SystemExit(f"frozen source changed: {relative}")
|
||||
source_hashes[relative] = actual
|
||||
asset_hashes = {}
|
||||
for group in ("b17_assets", "b19a_assets"):
|
||||
for relative, expected in spec[group].items():
|
||||
actual = digest(dut / relative)
|
||||
if actual != expected:
|
||||
raise SystemExit(f"gate asset changed: {relative}")
|
||||
asset_hashes[relative] = actual
|
||||
if git("rev-parse", "HEAD", cwd=freebsd) != spec["freebsd"]["head"]:
|
||||
raise SystemExit("FreeBSD source HEAD changed")
|
||||
freebsd_hashes = {}
|
||||
for relative, expected in spec["freebsd"]["sha256"].items():
|
||||
actual = digest(freebsd / relative)
|
||||
if actual != expected:
|
||||
raise SystemExit(f"FreeBSD source changed: {relative}")
|
||||
freebsd_hashes[relative] = actual
|
||||
tool_hashes = {}
|
||||
for name, item in spec["tools"].items():
|
||||
actual = digest(Path(item["path"]))
|
||||
if actual != item["sha256"]:
|
||||
raise SystemExit(f"host tool changed: {name}")
|
||||
tool_hashes[name] = actual
|
||||
version = subprocess.run(
|
||||
[spec["tools"]["mkfs.erofs"]["path"], "-V"], check=True,
|
||||
stdout=subprocess.PIPE, text=True,
|
||||
).stdout.splitlines()[0]
|
||||
if version != spec["tools"]["mkfs.erofs"]["version"]:
|
||||
raise SystemExit("mkfs.erofs version changed")
|
||||
output.write_text(json.dumps({
|
||||
"assets": asset_hashes,
|
||||
"base": resolved,
|
||||
"freebsd_head": spec["freebsd"]["head"],
|
||||
"freebsd_sha256": freebsd_hashes,
|
||||
"source_sha256": source_hashes,
|
||||
"status": "PASS",
|
||||
"tool_sha256": tool_hashes,
|
||||
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
PY
|
||||
|
||||
spec=$fixture_dir/B19a-xattr-spec.json
|
||||
generator=$fixture_dir/B19a-xattr-generate.py
|
||||
oracle=$fixture_dir/B19a-xattr-oracle.py
|
||||
model=$fixture_dir/B19a-cache-model.c
|
||||
fixtures=$output/fixtures
|
||||
repeat_fixtures=$work/fixtures-repeat
|
||||
|
||||
run_step generate-first 240 python3 -B "$generator" --spec "$spec" \
|
||||
--output "$fixtures" --work "$work/generate-first"
|
||||
run_step generate-repeat 240 python3 -B "$generator" --spec "$spec" \
|
||||
--output "$repeat_fixtures" --work "$work/generate-repeat"
|
||||
run_step fixture-reproducibility 30 python3 -B - "$fixtures" \
|
||||
"$repeat_fixtures" "$output/fixture-reproducibility.json" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
first = Path(sys.argv[1])
|
||||
second = Path(sys.argv[2])
|
||||
names = sorted(path.name for path in first.glob("*.erofs"))
|
||||
if names != sorted(path.name for path in second.glob("*.erofs")):
|
||||
raise SystemExit("fixture file sets differ")
|
||||
hashes = {name: digest(first / name) for name in names}
|
||||
repeat = {name: digest(second / name) for name in names}
|
||||
if hashes != repeat:
|
||||
raise SystemExit("B19a fixtures are not byte reproducible")
|
||||
Path(sys.argv[3]).write_text(json.dumps({
|
||||
"first": hashes,
|
||||
"repeat": repeat,
|
||||
"status": "PASS",
|
||||
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
PY
|
||||
|
||||
run_step oracle-selftest 30 python3 -B "$oracle" --spec "$spec" selftest \
|
||||
--output "$output/oracle-selftest.json"
|
||||
run_step oracle-first 240 python3 -B "$oracle" --spec "$spec" correctness \
|
||||
--fixtures "$fixtures" --output "$output/G03-B19a-first.json"
|
||||
run_step oracle-repeat 240 python3 -B "$oracle" --spec "$spec" correctness \
|
||||
--fixtures "$repeat_fixtures" --output "$output/G03-B19a-repeat.json"
|
||||
|
||||
mkdir "$output/G03-B17"
|
||||
b17_spec=$fixture_dir/B17-xattr-spec.json
|
||||
b17_seed=$fixture_dir/B17-xattr-seed.erofs
|
||||
b17_generator=$fixture_dir/B17-xattr-generate.py
|
||||
b17_oracle=$fixture_dir/B17-xattr-oracle.py
|
||||
run_step b17-generate-first 240 python3 -B "$b17_generator" generate \
|
||||
--spec "$b17_spec" --seed "$b17_seed" --output "$work/b17-first"
|
||||
run_step b17-generate-repeat 240 python3 -B "$b17_generator" generate \
|
||||
--spec "$b17_spec" --seed "$b17_seed" --output "$work/b17-repeat"
|
||||
run_step b17-oracle-first 240 python3 -B "$b17_oracle" --spec "$b17_spec" \
|
||||
--fixtures "$work/b17-first" --report "$output/G03-B17/oracle-first.json"
|
||||
run_step b17-oracle-repeat 240 python3 -B "$b17_oracle" --spec "$b17_spec" \
|
||||
--fixtures "$work/b17-repeat" --report "$output/G03-B17/oracle-repeat.json"
|
||||
cp "$work/b17-first/fixture-index.json" "$output/G03-B17/fixture-index.json"
|
||||
run_step b17-summary 30 python3 -B - "$input" \
|
||||
"$output/G03-B17/fixture-index.json" "$output/G03-B17/oracle-first.json" \
|
||||
"$output/G03-B17/oracle-repeat.json" "$output/G03-B17/summary.json" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
gate = json.loads(Path(sys.argv[1]).read_text(encoding="ascii"))
|
||||
index = json.loads(Path(sys.argv[2]).read_text(encoding="ascii"))
|
||||
reports = [json.loads(Path(path).read_text(encoding="ascii")) for path in sys.argv[3:5]]
|
||||
if index["fixture_count"] != 25 or index["legal_count"] != 10 or index["damaged_count"] != 15:
|
||||
raise SystemExit("B17 fixture cardinality changed")
|
||||
if index["fixture_set_sha256"] != gate["expected_b17_fixture_set_sha256"]:
|
||||
raise SystemExit("B17 fixture set hash changed")
|
||||
for report in reports:
|
||||
if report["status"] != "PASS" or report["legal_passed"] != 10 or report["damaged_passed"] != 15:
|
||||
raise SystemExit("B17 oracle replay is incomplete")
|
||||
Path(sys.argv[5]).write_text(json.dumps({
|
||||
"damaged_passed": 15,
|
||||
"fixture_count": 25,
|
||||
"fixture_set_sha256": index["fixture_set_sha256"],
|
||||
"legal_passed": 10,
|
||||
"replays": 2,
|
||||
"status": "PASS",
|
||||
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
PY
|
||||
|
||||
run_step cache-model-compile 30 cc -std=c11 -O2 -Wall -Wextra -Werror \
|
||||
-pthread "$model" -o "$work/B19a-cache-model"
|
||||
run_step cache-model 120 "$work/B19a-cache-model"
|
||||
cp "$output/cache-model.stdout" "$output/G05-cache-model.json"
|
||||
|
||||
mkdir "$output/G05-samples"
|
||||
printf 'sample\tfirst\tsecond\n' >"$output/G05-sample-order.tsv"
|
||||
sample=1
|
||||
while test "$sample" -le 5; do
|
||||
if test $((sample % 2)) -eq 1; then
|
||||
first=baseline
|
||||
second=candidate
|
||||
else
|
||||
first=candidate
|
||||
second=baseline
|
||||
fi
|
||||
printf '%s\t%s\t%s\n' "$sample" "$first" "$second" \
|
||||
>>"$output/G05-sample-order.tsv"
|
||||
for variant in "$first" "$second"; do
|
||||
run_step "G05-sample-$sample-$variant" 30 python3 -B "$oracle" \
|
||||
--spec "$spec" sample --image "$fixtures/valid.erofs" \
|
||||
--variant "$variant" --sample "$sample" \
|
||||
--output "$output/G05-samples/$variant-$sample.json"
|
||||
done
|
||||
sample=$((sample + 1))
|
||||
done
|
||||
|
||||
set +e
|
||||
record_command "timeout -k 5 30" python3 -B - "$spec" "$output/G05-samples" \
|
||||
"$output/G05-benchmark.json" "$output/G05-raw-samples.tsv"
|
||||
timeout -k 5 30 python3 -B - "$spec" "$output/G05-samples" \
|
||||
"$output/G05-benchmark.json" "$output/G05-raw-samples.tsv" \
|
||||
>"$output/G05-aggregate.stdout" 2>"$output/G05-aggregate.stderr" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import sys
|
||||
|
||||
spec = json.loads(Path(sys.argv[1]).read_text(encoding="ascii"))
|
||||
sample_dir = Path(sys.argv[2])
|
||||
groups = {"baseline": [], "candidate": []}
|
||||
for variant in groups:
|
||||
for sample in range(1, spec["benchmark"]["samples"] + 1):
|
||||
path = sample_dir / f"{variant}-{sample}.json"
|
||||
value = json.loads(path.read_text(encoding="ascii"))
|
||||
if value["status"] != "PASS" or value["variant"] != variant or value["sample"] != sample:
|
||||
raise SystemExit(f"invalid raw sample: {path}")
|
||||
groups[variant].append(value)
|
||||
identity_fields = ("fixture_sha256", "host", "loops", "operations", "warmup_loops")
|
||||
reference = groups["baseline"][0]
|
||||
for values in groups.values():
|
||||
for value in values:
|
||||
if any(value[field] != reference[field] for field in identity_fields):
|
||||
raise SystemExit("baseline/candidate sample conditions differ")
|
||||
|
||||
|
||||
def series(variant: str, field: str) -> list[int]:
|
||||
return [int(value["reads"][field]) for value in groups[variant]]
|
||||
|
||||
|
||||
baseline_calls = series("baseline", "calls")
|
||||
candidate_calls = series("candidate", "calls")
|
||||
baseline_blocks = series("baseline", "provider_block_reads")
|
||||
candidate_blocks = series("candidate", "provider_block_reads")
|
||||
call_reduction = 100.0 * (
|
||||
statistics.median(baseline_calls) - statistics.median(candidate_calls)
|
||||
) / statistics.median(baseline_calls)
|
||||
block_reduction = 100.0 * (
|
||||
statistics.median(baseline_blocks) - statistics.median(candidate_blocks)
|
||||
) / statistics.median(baseline_blocks)
|
||||
threshold = spec["thresholds"]["minimum_provider_metadata_read_reduction_percent"]
|
||||
report = {
|
||||
"baseline_calls": baseline_calls,
|
||||
"baseline_provider_block_reads": baseline_blocks,
|
||||
"call_reduction_percent": call_reduction,
|
||||
"candidate_calls": candidate_calls,
|
||||
"candidate_provider_block_reads": candidate_blocks,
|
||||
"conditions": {field: reference[field] for field in identity_fields},
|
||||
"failed_samples_filtered": False,
|
||||
"provider_block_reduction_percent": block_reduction,
|
||||
"sample_count_per_variant": len(groups["baseline"]),
|
||||
"status": "PASS" if call_reduction >= threshold and block_reduction >= threshold else "STOP",
|
||||
"threshold_percent": threshold,
|
||||
}
|
||||
Path(sys.argv[3]).write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
lines = ["variant\tsample\tcalls\tprovider_block_reads\tbytes\telapsed_ns\tstatus"]
|
||||
for variant in ("baseline", "candidate"):
|
||||
for value in groups[variant]:
|
||||
lines.append("\t".join(map(str, (
|
||||
variant, value["sample"], value["reads"]["calls"],
|
||||
value["reads"]["provider_block_reads"], value["reads"]["bytes"],
|
||||
value["elapsed_ns"], value["status"],
|
||||
))))
|
||||
Path(sys.argv[4]).write_text("\n".join(lines) + "\n", encoding="ascii")
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
raise SystemExit(0 if report["status"] == "PASS" else 22)
|
||||
PY
|
||||
aggregate_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "$aggregate_rc" >"$output/G05-aggregate.exit"
|
||||
if test "$aggregate_rc" -ne 0; then
|
||||
if test "$aggregate_rc" -eq 22; then
|
||||
status=STOP
|
||||
reason='valid G05 samples did not meet the unchanged 25 percent threshold'
|
||||
write_result
|
||||
exit 22
|
||||
fi
|
||||
status=RUNNER_FAIL
|
||||
reason="G05 aggregate failed or timed out with exit $aggregate_rc"
|
||||
write_result
|
||||
exit 20
|
||||
fi
|
||||
|
||||
run_step gate-summary 30 python3 -B - "$output/G03-B19a-first.json" \
|
||||
"$output/G03-B19a-repeat.json" "$output/G03-B17/summary.json" \
|
||||
"$output/G05-cache-model.json" "$output/G05-benchmark.json" \
|
||||
"$output/GATE-SUMMARY.json" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
values = [json.loads(Path(path).read_text(encoding="ascii")) for path in sys.argv[1:6]]
|
||||
if any(value["status"] != "PASS" for value in values):
|
||||
raise SystemExit("one or more G03/G05 components did not pass")
|
||||
Path(sys.argv[6]).write_text(json.dumps({
|
||||
"b19a": "AUTHORIZED",
|
||||
"g03": "GO",
|
||||
"g05": "GO",
|
||||
"source_modified": False,
|
||||
"status": "GO",
|
||||
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
PY
|
||||
|
||||
status=GO
|
||||
reason='G03 and G05 both reached GO with valid multi-block oracle and symmetric samples'
|
||||
write_result
|
||||
printf '%s\n' 'P15-022 G03/G05 GO; B19a source is authorized'
|
||||
exit 0
|
||||
Reference in New Issue
Block a user