Files
erofs-freebsd-out-tree/tests/pre15/cases/B22-zmap-arithmetic.sh
T
2026-08-18 09:20:44 +02:00

571 lines
21 KiB
Bash
Executable File

#!/bin/sh
set -eu
: "${PRE15_DUT:?PRE15_DUT is required}"
: "${PRE15_ROOT:?PRE15_ROOT is required}"
: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}"
: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}"
: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}"
. "$PRE15_LIB_DIR/runner.sh"
baseline=8a8761af91654a025fd991d14da906c0b612bd7d
fixture=$PRE15_DUT/tests/pre15/fixtures/B22-zmap-arithmetic.json
kld_builder=$PRE15_DUT/tests/pre15/fixtures/B22-build-kld.sh
zmap=$PRE15_DUT/src/zmap.c
linux_zmap=$PRE15_ROOT/src-linux/zmap.c
artifacts=$PRE15_RUN_DIR/artifacts
for tool in cc git python3 sha256sum; do
command -v "$tool" >/dev/null 2>&1 || \
pre15_infra_blocked "missing B22 host tool: $tool"
done
pre15_record_fixture b22-arithmetic "$fixture"
pre15_record_fixture b22-kld-builder "$kld_builder"
pre15_record_fixture b22-case "$PRE15_DUT/tests/pre15/cases/B22-zmap-arithmetic.sh"
pre15_record_fixture b22-zmap "$zmap"
pre15_record_fixture b22-linux-zmap "$linux_zmap"
mkdir -p "$artifacts"
pre15_target_reached
if python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" "$fixture" \
"$artifacts" <<'PY'
from __future__ import annotations
import json
from pathlib import Path
import re
import subprocess
import sys
root = Path(sys.argv[1])
dut = Path(sys.argv[2])
baseline = sys.argv[3]
fixture_path = Path(sys.argv[4])
artifacts = Path(sys.argv[5])
src = dut / "src"
u32_max = (1 << 32) - 1
u64_max = (1 << 64) - 1
eintegrity = 97
def committed(path: str) -> str:
completed = subprocess.run(
["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if completed.returncode != 0:
raise SystemExit(f"cannot read B22 baseline {path}: {completed.stderr}")
return completed.stdout
def function(source: str, name: str) -> str:
match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE)
if match is None:
raise SystemExit(f"missing function: {name}")
name_line = source.rfind("\n", 0, match.start()) + 1
start = source.rfind("\n", 0, name_line - 1) + 1
brace = source.find("{", match.end())
if brace < 0:
raise SystemExit(f"missing function body: {name}")
depth = 0
state = "code"
index = brace
while index < len(source):
char = source[index]
following = source[index + 1] if index + 1 < len(source) else ""
if state == "code":
if char == "/" and following == "*":
state = "block"
index += 2
continue
if char == "/" and following == "/":
state = "line"
index += 2
continue
if char == '"':
state = "string"
elif char == "'":
state = "character"
elif char == "{":
depth += 1
elif char == "}":
depth -= 1
if depth == 0:
return source[start : index + 1]
elif state == "block" and char == "*" and following == "/":
state = "code"
index += 2
continue
elif state == "line" and char == "\n":
state = "code"
elif state in {"string", "character"}:
if char == "\\":
index += 2
continue
if (state == "string" and char == '"') or (
state == "character" and char == "'"
):
state = "code"
index += 1
raise SystemExit(f"unterminated function: {name}")
def add(left: int, right: int) -> tuple[str, int | None]:
value = left + right
if value > u64_max:
return ("EINTEGRITY", None)
return ("PASS", value)
def multiply(left: int, right: int) -> tuple[str, int | None]:
value = left * right
if value > u64_max:
return ("EINTEGRITY", None)
return ("PASS", value)
def model(case: dict[str, object]) -> tuple[str, int | None]:
operation = case["operation"]
if operation == "compact_pblk":
return add(int(case["base"]), int(case["nblk"]))
if operation == "index_base":
status, end = add(int(case["inode_off"]), int(case["inode_isize"]))
if status != "PASS":
return (status, None)
status, end = add(int(end), int(case.get("xattr_isize", 0)))
if status != "PASS":
return (status, None)
status, end = add(int(end), 7)
if status != "PASS":
return (status, None)
end = int(end) & ~7
return add(end, 8)
if operation == "index_advance":
status, delta = multiply(int(case["count"]), int(case["unit"]))
if status != "PASS":
return (status, None)
return add(int(case["position"]), int(delta))
if operation == "lcluster_count":
bits = int(case["lclusterbits"])
if bits >= 64:
return ("EINTEGRITY", None)
size = int(case["size"])
count = size >> bits
if size & ((1 << bits) - 1):
return add(count, 1)
return ("PASS", count)
if operation == "lcluster_pos":
bits = int(case["lclusterbits"])
if bits >= 64:
return ("EINTEGRITY", None)
status, base = multiply(int(case["lcn"]), 1 << bits)
if status != "PASS":
return (status, None)
return add(int(base), int(case["clusterofs"]))
if operation == "lcn_advance":
return add(int(case["lcn"]), int(case["delta"]))
if operation == "physical_end":
status, pend = add(int(case["pa"]), int(case["plen"]))
if status != "PASS":
return (status, None)
limit = (1 << 48) * int(case["block_size"])
if limit <= u64_max and int(pend) > limit:
return ("EINTEGRITY", None)
return ("PASS", None)
if operation == "post_eof":
la = int(case["la"])
size = int(case["size"])
if la < size:
return ("EINTEGRITY", None)
length = la - size + 1
return ("PASS", min(length, u64_max))
if operation == "fragment_offset":
low = int(case["low"])
high = int(case["high"])
if low > u32_max or high > u32_max:
return ("EINTEGRITY", None)
return ("PASS", low | (high << 32))
raise SystemExit(f"unknown B22 operation: {operation}")
fixture = json.loads(fixture_path.read_text(encoding="ascii"))
if fixture.get("schema") != 1 or fixture.get("batch") != "B22":
raise SystemExit("invalid B22 fixture identity")
if fixture.get("test") != "TC170-zmap-arithmetic":
raise SystemExit("invalid B22 test identity")
if fixture.get("candidates") != [
"P15-047",
"P15-059",
"P15-071",
"P15-079",
"P15-084",
]:
raise SystemExit("B22 candidate set changed")
if fixture.get("errno") != {
"corruption": eintegrity,
"provider_io": "unchanged",
"sign": "positive",
}:
raise SystemExit("B22 errno contract changed")
cases = fixture.get("cases")
if not isinstance(cases, list) or not cases:
raise SystemExit("B22 fixture has no cases")
names: set[str] = set()
markers: set[str] = set()
operations: set[str] = set()
decoded: list[dict[str, object]] = []
for case in cases:
if not isinstance(case, dict):
raise SystemExit("B22 fixture case is not an object")
name = str(case.get("name", ""))
marker = str(case.get("target_marker", ""))
if not name or name in names or not marker.startswith("TC170:"):
raise SystemExit(f"invalid B22 case identity: {name!r}")
names.add(name)
markers.add(marker)
operations.add(str(case.get("operation", "")))
status, value = model(case)
if status != case.get("status"):
raise SystemExit(f"B22 independent status mismatch: {name}")
if status == "PASS" and value is not None and value != case.get("expected"):
raise SystemExit(f"B22 independent value mismatch: {name}")
mutated = case.get("mutated_fields")
if status == "EINTEGRITY" and (
not isinstance(mutated, list) or len(mutated) != 1
):
raise SystemExit(f"B22 negative is not single-field: {name}")
if status == "PASS" and mutated != []:
raise SystemExit(f"B22 positive declares a mutation: {name}")
decoded.append({"name": name, "status": status, "value": value})
required_markers = {
"TC170:compact-pblk-32bit-crossing",
"TC170:index-position",
"TC170:delta-lcn",
"TC170:physical-end-48bit",
"TC170:post-eof",
"TC170:fragment-high-bits",
}
if markers != required_markers:
raise SystemExit("B22 target-marker set is incomplete")
if operations != {
"compact_pblk",
"fragment_offset",
"index_advance",
"index_base",
"lcluster_count",
"lcluster_pos",
"lcn_advance",
"physical_end",
"post_eof",
}:
raise SystemExit("B22 operation set is incomplete")
current = (src / "zmap.c").read_text(encoding="utf-8")
base = committed("src/zmap.c")
linux = (root / "src-linux/zmap.c").read_text(encoding="utf-8")
helpers = [
"z_erofs_index_base",
"z_erofs_index_advance",
"z_erofs_lcluster_count",
"z_erofs_lcluster_pos",
"z_erofs_lcn_advance",
"z_erofs_compact_pblk",
"z_erofs_fragment_offset",
"z_erofs_post_eof_len",
"z_erofs_physical_end",
]
for helper in helpers:
if helper in base or current.count(helper) < 2:
raise SystemExit(f"B22 helper is missing, duplicated, or pre-existing: {helper}")
old_freebsd = (
"m->pblk = le32dec(in + packsize - sizeof(uint32_t)) + nblk;",
"map->m_llen = map->m_la + 1 - vi->size;",
"vi->z_fragmentoff |= map->m_pa << 32;",
"lcn += m->delta[1];",
"pos += lcn << amortizedshift;",
"(pend >> sbi->blkszbits) >= (1ULL << 48)",
)
if not all(anchor in base for anchor in old_freebsd):
raise SystemExit("B22 baseline arithmetic anchors changed")
if any(anchor in current for anchor in old_freebsd):
raise SystemExit("B22 left an unchecked baseline arithmetic anchor")
linux_anchors = (
"m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;",
"map->m_llen = map->m_la + 1 - inode->i_size;",
"vi->z_fragmentoff |= map->m_pa << 32;",
"lcn += m->delta[1];",
"(pend >> sbi->blkszbits) >= BIT_ULL(48)",
)
if not all(anchor in linux for anchor in linux_anchors):
raise SystemExit("Linux zmap semantic anchor changed")
if function(current, "z_erofs_read_index") != function(base, "z_erofs_read_index"):
raise SystemExit("B22 changed the FreeBSD metadata/provider reader")
if current.count("erofs_read_metadata") != base.count("erofs_read_metadata"):
raise SystemExit("B22 changed metadata I/O call count")
if current.count("erofs_put_metabuf") != base.count("erofs_put_metabuf"):
raise SystemExit("B22 changed metadata release call count")
for public in ("z_erofs_fill_inode", "z_erofs_map_blocks"):
old_signature = function(base, public).split("{", 1)[0]
new_signature = function(current, public).split("{", 1)[0]
if old_signature != new_signature:
raise SystemExit(f"B22 changed public zmap ABI: {public}")
if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", current):
raise SystemExit("B22 introduced Linux negative errno")
legacy_mismatches: set[str] = set()
for case in cases:
operation = case["operation"]
status = case["status"]
expected = case.get("expected")
if operation == "compact_pblk" and status == "PASS":
old = (int(case["base"]) + int(case["nblk"])) & u32_max
if old != expected:
legacy_mismatches.add("compact-pblk")
elif operation == "index_advance" and status == "EINTEGRITY":
old = (int(case["position"]) + int(case["count"]) * int(case["unit"])) & u64_max
if old <= u64_max:
legacy_mismatches.add("index-position")
elif operation == "lcluster_pos" and status == "EINTEGRITY":
old = ((int(case["lcn"]) << int(case["lclusterbits"])) + int(case["clusterofs"])) & u64_max
if old <= u64_max:
legacy_mismatches.add("delta-lcn")
elif operation == "physical_end" and case["name"] == "physical-last-48bit-block":
pend = int(case["pa"]) + int(case["plen"])
if (pend >> 12) >= (1 << 48):
legacy_mismatches.add("physical-end")
elif operation == "post_eof" and case["name"] == "post-eof-saturates-empty-inode":
old = ((int(case["la"]) + 1) & u64_max) - int(case["size"])
if old != expected:
legacy_mismatches.add("post-eof")
elif operation == "fragment_offset" and status == "EINTEGRITY":
old = int(case["low"]) | ((int(case["high"]) << 32) & u64_max)
if old <= u64_max:
legacy_mismatches.add("fragment-high")
if legacy_mismatches != {
"compact-pblk",
"delta-lcn",
"fragment-high",
"index-position",
"physical-end",
"post-eof",
}:
raise SystemExit("B22 fixtures do not distinguish every legacy bug")
helper_source = "\n\n".join(function(current, name) for name in helpers)
c_lines = [
"#include <stdint.h>",
"#include <stdio.h>",
"#define EINTEGRITY 97",
"#define rounddown2(x, y) ((x) & ~((y) - 1))",
"typedef uint64_t erofs_off_t;",
"typedef uint64_t erofs_blk_t;",
"struct z_erofs_map_header { uint8_t bytes[8]; };",
"struct erofs_inode { erofs_off_t inode_off; unsigned int inode_isize; unsigned int xattr_isize; };",
helper_source,
"int main(void)",
"{",
"\tuint64_t value;",
"\tint error, failures = 0;",
]
def u64(value: object) -> str:
return f"UINT64_C({int(value)})"
for case in cases:
name = str(case["name"])
operation = case["operation"]
expected_error = 0 if case["status"] == "PASS" else eintegrity
c_lines.append(f"\t/* {name} */")
if operation == "compact_pblk":
call = f"z_erofs_compact_pblk(UINT32_C({int(case['base'])}), {int(case['nblk'])}U, &value)"
elif operation == "index_base":
c_lines.extend([
"\t{",
"\t\tstruct erofs_inode vi = {",
f"\t\t\t.inode_off = {u64(case['inode_off'])},",
f"\t\t\t.inode_isize = {int(case['inode_isize'])}U,",
f"\t\t\t.xattr_isize = {int(case.get('xattr_isize', 0))}U,",
"\t\t};",
])
call = "z_erofs_index_base(&vi, &value)"
elif operation == "index_advance":
c_lines.append(f"\tvalue = {u64(case['position'])};")
call = f"z_erofs_index_advance(&value, {u64(case['count'])}, {u64(case['unit'])})"
elif operation == "lcluster_count":
call = f"z_erofs_lcluster_count({u64(case['size'])}, {int(case['lclusterbits'])}U, &value)"
elif operation == "lcluster_pos":
call = f"z_erofs_lcluster_pos({u64(case['lcn'])}, {int(case['lclusterbits'])}U, {u64(case['clusterofs'])}, &value)"
elif operation == "lcn_advance":
c_lines.append(f"\tvalue = {u64(case['lcn'])};")
call = f"z_erofs_lcn_advance(&value, {u64(case['delta'])})"
elif operation == "physical_end":
call = f"z_erofs_physical_end({u64(case['pa'])}, {u64(case['plen'])}, {u64(case['block_size'])})"
elif operation == "post_eof":
call = f"z_erofs_post_eof_len({u64(case['la'])}, {u64(case['size'])}, &value)"
elif operation == "fragment_offset":
call = f"z_erofs_fragment_offset({u64(case['low'])}, {u64(case['high'])}, &value)"
else:
raise SystemExit(f"cannot generate C for operation: {operation}")
c_lines.append(f"\terror = {call};")
c_lines.append(f"\tif (error != {expected_error}) {{")
c_lines.append(f"\t\tfprintf(stderr, \"{name}: error=%d\\n\", error);")
c_lines.append("\t\tfailures++;\n\t}")
if case["status"] == "PASS" and "expected" in case:
c_lines.append(f"\tif (value != {u64(case['expected'])}) {{")
c_lines.append(f"\t\tfprintf(stderr, \"{name}: value mismatch\\n\");")
c_lines.append("\t\tfailures++;\n\t}")
if operation == "index_base":
c_lines.append("\t}")
c_lines.extend([
"\tif (failures != 0)",
"\t\treturn (1);",
f"\tprintf(\"B22 extracted arithmetic PASS cases={len(cases)}\\n\");",
"\treturn (0);",
"}",
])
c_path = artifacts / "B22-zmap-arithmetic.c"
binary = artifacts / "B22-zmap-arithmetic"
c_path.write_text("\n".join(c_lines) + "\n", encoding="ascii")
compiled = subprocess.run(
["cc", "-std=gnu11", "-Wall", "-Wextra", "-Werror", str(c_path), "-o", str(binary)],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
(artifacts / "B22-compile.stdout").write_text(compiled.stdout, encoding="utf-8")
(artifacts / "B22-compile.stderr").write_text(compiled.stderr, encoding="utf-8")
if compiled.returncode != 0:
raise SystemExit("B22 extracted arithmetic compilation failed")
executed = subprocess.run(
[str(binary)],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
(artifacts / "B22-extracted.stdout").write_text(executed.stdout, encoding="utf-8")
(artifacts / "B22-extracted.stderr").write_text(executed.stderr, encoding="utf-8")
if executed.returncode != 0:
raise SystemExit("B22 extracted arithmetic execution failed")
result = {
"status": "PASS",
"batch": "B22",
"test": "TC170-zmap-arithmetic",
"baseline": baseline,
"case_count": len(cases),
"negative_count": sum(case["status"] == "EINTEGRITY" for case in cases),
"target_markers": sorted(markers),
"legacy_mismatches": sorted(legacy_mismatches),
"freebsd_positive_errno": True,
"linux_algorithm_locations_audited": True,
"provider_io_calls_unchanged": True,
"metadata_release_calls_unchanged": True,
"decompressor_abi_unchanged": True,
"ondisk_abi_unchanged": True,
"qemu_scope": "exact-source KLD load plus extracted helper replay when PRE15_MODE=qemu",
"full_feature_suite": "NOT_RUN",
}
(artifacts / "B22-decoded-cases.json").write_text(
json.dumps(decoded, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
encoding="ascii",
)
(artifacts / "B22-result.json").write_text(
json.dumps(result, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
encoding="ascii",
)
print(
f"B22 host PASS cases={len(cases)} negatives={result['negative_count']} "
f"markers={len(markers)}"
)
PY
then
:
else
pre15_dut_fail 'B22 source, fixture, or extracted arithmetic check failed'
fi
sha256sum "$artifacts/B22-zmap-arithmetic.c" \
"$artifacts/B22-zmap-arithmetic" \
"$artifacts/B22-decoded-cases.json" "$artifacts/B22-result.json" \
> "$artifacts/SHA256SUMS"
if test "${PRE15_MODE:-host}" != qemu; then
printf '%s\n' \
'TC170 host extracted zmap arithmetic and fixture oracle PASS' \
'QEMU exact-source KLD load and FreeBSD helper replay NOT_RUN in host mode'
exit 0
fi
for tool in awk clang file nm scp; do
command -v "$tool" >/dev/null 2>&1 || \
pre15_infra_blocked "missing B22 QEMU tool: $tool"
done
: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}"
: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}"
: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}"
: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}"
module=$PRE15_CASE_TMP/B22-erofs.ko
if ! /bin/sh "$kld_builder" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" "$module" \
"$PRE15_CASE_TMP/kld-work" >"$artifacts/B22-kld-build.stdout" \
2>"$artifacts/B22-kld-build.stderr"; then
pre15_dut_fail 'B22 cross-target KLD build failed'
fi
pre15_record_module "$module"
file "$module" > "$artifacts/B22-kld-file.txt"
sha256sum "$module" > "$artifacts/B22-kld-sha256.txt"
nm -u "$module" | LC_ALL=C sort > "$artifacts/B22-kld-nm-u.txt"
pre15_scp()
{
timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \
-o BatchMode=yes -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
-o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \
-i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \
"$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2"
}
if ! pre15_scp "$module" /root/B22-erofs.ko || \
! pre15_scp "$artifacts/B22-zmap-arithmetic.c" \
/root/B22-zmap-arithmetic.c; then
pre15_infra_blocked 'could not transfer B22 module or arithmetic source'
fi
if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then
pre15_infra_blocked 'FreeBSD guest already has an EROFS module loaded'
fi
if ! pre15_guest_ssh_bounded kldload /root/B22-erofs.ko \
>"$artifacts/B22-kldload.stdout" 2>"$artifacts/B22-kldload.stderr"; then
if grep -q 'module already loaded or in kernel' \
"$artifacts/B22-kldload.stderr"; then
pre15_infra_blocked 'guest kernel already owns the erofs.1 interface'
fi
pre15_dut_fail 'B22 exact-source KLD failed to load'
fi
pre15_own_guest_kld erofs 'B22 exact-source KLD'
if ! pre15_guest_ssh_bounded cc -std=gnu11 -Wall -Wextra -Werror \
/root/B22-zmap-arithmetic.c -o /root/B22-zmap-arithmetic; then
pre15_infra_blocked 'could not compile B22 arithmetic helper in the guest'
fi
if ! pre15_guest_ssh_bounded /root/B22-zmap-arithmetic \
>"$artifacts/B22-guest-arithmetic.stdout" \
2>"$artifacts/B22-guest-arithmetic.stderr"; then
pre15_dut_fail 'B22 FreeBSD arithmetic helper replay failed'
fi
pre15_guest_ssh_bounded kldstat -n erofs > "$artifacts/B22-kldstat.txt"
printf '%s\n' \
'TC170 QEMU PASS exact-source KLD load and FreeBSD arithmetic helper replay' \
'No full feature suite or shared GEOM provider was used'