307 lines
10 KiB
Bash
Executable File
307 lines
10 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
script=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/B32-qemu-run.sh
|
|
|
|
if test "${1:-}" != --worker; then
|
|
test "$#" -eq 1 || {
|
|
printf '%s\n' 'usage: B32-qemu-run.sh TC168-cache-inflight|TC184-vnode-races' >&2
|
|
exit 20
|
|
}
|
|
case "$1" in
|
|
TC168-cache-inflight|TC184-vnode-races) ;;
|
|
*) printf 'unknown B32 QEMU scenario: %s\n' "$1" >&2; exit 20 ;;
|
|
esac
|
|
total=${PRE15_QEMU_TIMEOUT:-1200}
|
|
case "$total" in
|
|
''|*[!0-9]*|0) printf '%s\n' 'invalid PRE15_QEMU_TIMEOUT' >&2; exit 20 ;;
|
|
esac
|
|
exec timeout -k 30 "$total" "$script" --worker "$1"
|
|
fi
|
|
|
|
shift
|
|
test "$#" -eq 1 || exit 20
|
|
scenario=$1
|
|
: "${PRE15_EVIDENCE_ROOT:?PRE15_EVIDENCE_ROOT is required}"
|
|
: "${PRE15_QEMU_BASE_IMAGE:?PRE15_QEMU_BASE_IMAGE is required}"
|
|
: "${PRE15_QEMU_SSH_KEY:?PRE15_QEMU_SSH_KEY is required}"
|
|
|
|
dut=$(CDPATH= cd -- "$(dirname -- "$script")/../../.." && pwd -P)
|
|
root=$(CDPATH= cd -- "$dut/.." && pwd -P)
|
|
lib=$dut/tests/pre15/lib
|
|
case_script=$dut/tests/pre15/cases/B32-cache-state.sh
|
|
freebsd_src=${PRE15_FREEBSD_SRC:-${FREEBSD_SRC:-/work/build/freebsd-src}}
|
|
user=${PRE15_QEMU_SSH_USER:-root}
|
|
boot_timeout=${PRE15_QEMU_BOOT_TIMEOUT:-180}
|
|
guest_timeout=${PRE15_GUEST_COMMAND_TIMEOUT:-60}
|
|
case_timeout=${PRE15_B32_QEMU_CASE_TIMEOUT:-900}
|
|
run_id=$(date -u '+%Y%m%dT%H%M%SZ')-qemu-B32-$scenario-$$
|
|
run=$PRE15_EVIDENCE_ROOT/$run_id
|
|
case_run=$run/case
|
|
case_tmp=$run/case-tmp
|
|
overlay=$run/guest-overlay.qcow2
|
|
control=$run/ssh-control
|
|
serial=$run/serial.log
|
|
qemu_log=$run/qemu.log
|
|
ownership=$case_run/ownership.tsv
|
|
cleanup_log=$case_run/cleanup.log
|
|
target_marker=$case_run/target.marker
|
|
reason_file=$case_run/reason.txt
|
|
fixture_hashes=$case_run/fixture-hashes.tsv
|
|
module_hash=$case_run/module-hash.tsv
|
|
qemu_pid=
|
|
port=
|
|
ssh_ready=0
|
|
cleanup_done=0
|
|
cleanup_status=PASS
|
|
|
|
mkdir -p "$case_run" "$case_tmp"
|
|
: >"$ownership"
|
|
: >"$fixture_hashes"
|
|
: >"$module_hash"
|
|
: >"$reason_file"
|
|
|
|
cleanup()
|
|
{
|
|
test "$cleanup_done" -eq 0 || return
|
|
cleanup_done=1
|
|
if test "$ssh_ready" -eq 1; then
|
|
PRE15_QEMU_SSH_PORT=$port
|
|
PRE15_QEMU_SSH_USER=$user
|
|
PRE15_QEMU_SSH_KEY=$PRE15_QEMU_SSH_KEY
|
|
PRE15_QEMU_CONTROL_PATH=$control
|
|
PRE15_GUEST_COMMAND_TIMEOUT=$guest_timeout
|
|
PRE15_OWNERSHIP_FILE=$ownership
|
|
PRE15_CLEANUP_LOG=$cleanup_log
|
|
PRE15_RUN_DIR=$case_run
|
|
export PRE15_QEMU_SSH_PORT PRE15_QEMU_SSH_USER PRE15_QEMU_SSH_KEY
|
|
export PRE15_QEMU_CONTROL_PATH PRE15_GUEST_COMMAND_TIMEOUT
|
|
export PRE15_OWNERSHIP_FILE PRE15_CLEANUP_LOG PRE15_RUN_DIR
|
|
. "$lib/runner.sh"
|
|
pre15_cleanup_owned || cleanup_status=FAIL
|
|
test "${PRE15_CLEANUP_STATUS_RESULT:-FAIL}" = PASS || cleanup_status=FAIL
|
|
else
|
|
printf '%s\n' 'PASS no guest resources reached before SSH readiness' \
|
|
>"$cleanup_log"
|
|
fi
|
|
if test -S "$control"; then
|
|
timeout -k 5 15 ssh -S "$control" -O exit -p "${port:-22}" \
|
|
"$user@127.0.0.1" >>"$run/host-cleanup.log" 2>&1 || true
|
|
fi
|
|
if test -n "$qemu_pid" && test "$qemu_pid" != 26318 && \
|
|
kill -0 "$qemu_pid" 2>/dev/null; then
|
|
kill -TERM "$qemu_pid" 2>/dev/null || cleanup_status=FAIL
|
|
wait_count=0
|
|
while kill -0 "$qemu_pid" 2>/dev/null && test "$wait_count" -lt 100; do
|
|
sleep 0.1
|
|
wait_count=$((wait_count + 1))
|
|
done
|
|
if kill -0 "$qemu_pid" 2>/dev/null; then
|
|
kill -KILL "$qemu_pid" 2>/dev/null || cleanup_status=FAIL
|
|
sleep 0.2
|
|
fi
|
|
fi
|
|
if test -n "$qemu_pid" && kill -0 "$qemu_pid" 2>/dev/null; then
|
|
cleanup_status=FAIL
|
|
printf 'FAIL owned QEMU PID survived: %s\n' "$qemu_pid" \
|
|
>>"$run/host-cleanup.log"
|
|
else
|
|
printf 'PASS owned QEMU PID stopped: %s\n' "${qemu_pid:-not-started}" \
|
|
>>"$run/host-cleanup.log"
|
|
fi
|
|
if test -f "$overlay"; then
|
|
rm -f -- "$overlay" || cleanup_status=FAIL
|
|
fi
|
|
test ! -e "$overlay" || cleanup_status=FAIL
|
|
printf 'overlay_removed=%s\n' "$(test ! -e "$overlay" && printf yes || printf no)" \
|
|
>>"$run/host-cleanup.log"
|
|
rm -rf -- "$case_tmp"
|
|
stat -c 'path=%n size=%s mode=%a inode=%i mtime=%Y ctime=%Z' \
|
|
"$PRE15_QEMU_BASE_IMAGE" >"$run/base-identity-after.txt" || \
|
|
cleanup_status=FAIL
|
|
if ! cmp -s "$run/base-identity-before.txt" "$run/base-identity-after.txt"; then
|
|
cleanup_status=FAIL
|
|
printf '%s\n' 'FAIL protected base metadata changed' \
|
|
>>"$run/host-cleanup.log"
|
|
else
|
|
printf '%s\n' 'PASS protected base metadata unchanged' \
|
|
>>"$run/host-cleanup.log"
|
|
fi
|
|
if test -n "$port"; then
|
|
if timeout 10s python3 -B - "$port" <<'PY'
|
|
import socket
|
|
import sys
|
|
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.settimeout(0.2)
|
|
raise SystemExit(0 if sock.connect_ex(("127.0.0.1", int(sys.argv[1]))) != 0 else 1)
|
|
PY
|
|
then
|
|
printf 'PASS dynamic port free: %s\n' "$port" \
|
|
>>"$run/host-cleanup.log"
|
|
else
|
|
cleanup_status=FAIL
|
|
printf 'FAIL dynamic port remains open: %s\n' "$port" \
|
|
>>"$run/host-cleanup.log"
|
|
fi
|
|
fi
|
|
printf '%s\n' "$cleanup_status" >"$run/cleanup.status"
|
|
}
|
|
|
|
finish()
|
|
{
|
|
rc=$?
|
|
trap - EXIT HUP INT TERM
|
|
cleanup
|
|
if test "$cleanup_status" != PASS; then
|
|
printf '%s\n' 'RUNNER_FAIL owned cleanup or base metadata check failed' \
|
|
>>"$run/RESULT.txt"
|
|
rc=20
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
|
|
trap finish EXIT HUP INT TERM
|
|
|
|
for tool in cmp qemu-img qemu-system-x86_64 ssh stat timeout; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'INFRA_BLOCKED missing QEMU tool: %s\n' "$tool" >"$run/RESULT.txt"
|
|
exit 21
|
|
}
|
|
done
|
|
test -f "$PRE15_QEMU_BASE_IMAGE" || {
|
|
printf '%s\n' 'INFRA_BLOCKED protected base image is absent' >"$run/RESULT.txt"
|
|
exit 21
|
|
}
|
|
test -f "$PRE15_QEMU_SSH_KEY" || {
|
|
printf '%s\n' 'INFRA_BLOCKED SSH key is absent' >"$run/RESULT.txt"
|
|
exit 21
|
|
}
|
|
|
|
stat -c 'path=%n size=%s mode=%a inode=%i mtime=%Y ctime=%Z' \
|
|
"$PRE15_QEMU_BASE_IMAGE" >"$run/base-identity-before.txt"
|
|
timeout -k 5 30 qemu-img info --output=json "$PRE15_QEMU_BASE_IMAGE" \
|
|
>"$run/base-qemu-img-info.json"
|
|
timeout -k 5 30 qemu-img create -f qcow2 -F qcow2 \
|
|
-b "$PRE15_QEMU_BASE_IMAGE" "$overlay" >"$run/qemu-img-create.stdout" \
|
|
2>"$run/qemu-img-create.stderr"
|
|
|
|
port=$(timeout 10s python3 -B - <<'PY'
|
|
import socket
|
|
|
|
while True:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
if port != 9222:
|
|
print(port)
|
|
break
|
|
PY
|
|
)
|
|
python3 -B - "$run/qemu-argv.json" "$overlay" "$port" "$serial" <<'PY'
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
argv = [
|
|
"qemu-system-x86_64", "-accel", "tcg,thread=multi", "-cpu", "qemu64",
|
|
"-m", "6144", "-smp", "4",
|
|
"-drive", f"file={sys.argv[2]},if=virtio,format=qcow2",
|
|
"-netdev", f"user,id=net0,hostfwd=tcp:127.0.0.1:{sys.argv[3]}-:22",
|
|
"-device", "virtio-net-pci,netdev=net0", "-display", "none",
|
|
"-serial", f"file:{sys.argv[4]}", "-monitor", "none",
|
|
]
|
|
Path(sys.argv[1]).write_text(json.dumps(argv, separators=(",", ":")) + "\n", encoding="ascii")
|
|
PY
|
|
|
|
qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 -m 6144 -smp 4 \
|
|
-drive "file=$overlay,if=virtio,format=qcow2" \
|
|
-netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$port-:22" \
|
|
-device virtio-net-pci,netdev=net0 -display none -serial "file:$serial" \
|
|
-monitor none >"$run/qemu.stdout" 2>"$qemu_log" &
|
|
qemu_pid=$!
|
|
printf '%s\n' "$qemu_pid" >"$run/qemu.pid"
|
|
test "$qemu_pid" != 26318 || {
|
|
printf '%s\n' 'RUNNER_FAIL QEMU PID collided with protected PID' >"$run/RESULT.txt"
|
|
exit 20
|
|
}
|
|
|
|
boot_marker=$run/ssh-ready
|
|
boot_start=$(date -u '+%s')
|
|
set +e
|
|
timeout -s KILL "$boot_timeout" env B32_QEMU_PID="$qemu_pid" \
|
|
B32_QEMU_PORT="$port" B32_QEMU_USER="$user" \
|
|
B32_QEMU_KEY="$PRE15_QEMU_SSH_KEY" B32_QEMU_MARKER="$boot_marker" \
|
|
/bin/sh -c '
|
|
while kill -0 "$B32_QEMU_PID" 2>/dev/null; do
|
|
if timeout -s KILL 8 ssh -o BatchMode=yes \
|
|
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
|
-o ConnectTimeout=5 -i "$B32_QEMU_KEY" \
|
|
-p "$B32_QEMU_PORT" "[email protected]" true \
|
|
>/dev/null 2>&1; then
|
|
: >"$B32_QEMU_MARKER"
|
|
exit 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
exit 21
|
|
'
|
|
boot_rc=$?
|
|
set -e
|
|
boot_end=$(date -u '+%s')
|
|
printf 'start_epoch=%s\nend_epoch=%s\nelapsed_seconds=%s\ndeadline_seconds=%s\nexit=%s\n' \
|
|
"$boot_start" "$boot_end" "$((boot_end - boot_start))" "$boot_timeout" \
|
|
"$boot_rc" >"$run/boot-timing.txt"
|
|
if test -f "$boot_marker"; then
|
|
ssh_ready=1
|
|
elif ! kill -0 "$qemu_pid" 2>/dev/null; then
|
|
printf '%s\n' 'INFRA_BLOCKED QEMU exited before guest SSH' >"$run/RESULT.txt"
|
|
exit 21
|
|
else
|
|
printf 'INFRA_BLOCKED guest SSH not ready within absolute %ss boot deadline\n' \
|
|
"$boot_timeout" >"$run/RESULT.txt"
|
|
exit 21
|
|
fi
|
|
|
|
if ! timeout -k 5 30 ssh -MNf -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
-o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o ControlMaster=yes \
|
|
-o "ControlPath=$control" -i "$PRE15_QEMU_SSH_KEY" -p "$port" \
|
|
"$user@127.0.0.1"; then
|
|
printf '%s\n' 'INFRA_BLOCKED SSH ControlMaster failed' >"$run/RESULT.txt"
|
|
exit 21
|
|
fi
|
|
|
|
export PRE15_MODE=qemu
|
|
export PRE15_DUT=$dut PRE15_ROOT=$root PRE15_LIB_DIR=$lib
|
|
export PRE15_CASE_TMP=$case_tmp PRE15_RUN_DIR=$case_run
|
|
export PRE15_FREEBSD_SRC=$freebsd_src PRE15_B32_SCENARIO=$scenario
|
|
export PRE15_QEMU_CONTROL_PATH=$control PRE15_QEMU_SSH_PORT=$port
|
|
export PRE15_QEMU_SSH_USER=$user PRE15_QEMU_SSH_KEY
|
|
export PRE15_GUEST_COMMAND_TIMEOUT=$guest_timeout
|
|
export PRE15_OWNERSHIP_FILE=$ownership PRE15_CLEANUP_LOG=$cleanup_log
|
|
export PRE15_TARGET_MARKER_FILE=$target_marker PRE15_REASON_FILE=$reason_file
|
|
export PRE15_FIXTURE_HASHES=$fixture_hashes PRE15_MODULE_HASH=$module_hash
|
|
|
|
set +e
|
|
timeout -k 10 "$case_timeout" /bin/sh "$case_script" \
|
|
>"$case_run/stdout.log" 2>"$case_run/stderr.log"
|
|
case_rc=$?
|
|
set -e
|
|
case "$case_rc" in
|
|
0)
|
|
if test -f "$target_marker"; then
|
|
printf '%s PASS target=reached\n' "$scenario" >"$run/RESULT.txt"
|
|
else
|
|
printf '%s\n' 'RUNNER_FAIL case exited zero without target marker' \
|
|
>"$run/RESULT.txt"
|
|
case_rc=20
|
|
fi
|
|
;;
|
|
10) printf '%s DUT_FAIL\n' "$scenario" >"$run/RESULT.txt" ;;
|
|
20) printf '%s RUNNER_FAIL\n' "$scenario" >"$run/RESULT.txt" ;;
|
|
21) printf '%s INFRA_BLOCKED\n' "$scenario" >"$run/RESULT.txt" ;;
|
|
124|137) printf '%s RUNNER_FAIL case-timeout=%ss\n' "$scenario" "$case_timeout" >"$run/RESULT.txt"; case_rc=20 ;;
|
|
*) printf '%s RUNNER_FAIL exit=%s\n' "$scenario" "$case_rc" >"$run/RESULT.txt"; case_rc=20 ;;
|
|
esac
|
|
exit "$case_rc"
|