From b4a795d386cb130adaf5c7c39f8711b64cbf0c3e Mon Sep 17 00:00:00 2001 From: imrcpan Date: Tue, 18 Aug 2026 09:24:47 +0200 Subject: [PATCH] update --- src/.clang-format => .clang-format | 0 .gitignore | 26 +- src/Makefile => Makefile | 0 README.md | 90 - build.sh | 68 - src/compress.h => compress.h | 0 current/README.md | 13 - .../report-1/2026-08-09-overall-progress.md | 160 -- current/report-1/README.md | 8 - .../report-2/2026-08-09-overall-progress.md | 123 -- current/report-2/README.md | 8 - .../report-3/2026-08-09-overall-progress.md | 124 -- current/report-3/README.md | 9 - ...8-10-task2-code-review-style-assessment.md | 726 ------- current/report-4/README.md | 41 - src/data.c => data.c | 0 src/decompressor.c => decompressor.c | 0 ...ressor_deflate.c => decompressor_deflate.c | 0 src/decompressor_lz4.c => decompressor_lz4.c | 0 ...decompressor_lzma.c => decompressor_lzma.c | 0 ...decompressor_zstd.c => decompressor_zstd.c | 0 src/dir.c => dir.c | 0 docs/TEST_REPORT.md | 97 - docs/architecture.md | 283 --- docs/capabilities.md | 65 - docs/erofs.5 | 271 --- docs/features.md | 77 - docs/pre10-baseline.md | 36 - docs/pre10-batch-a.md | 96 - docs/pre10-batch-b.md | 108 -- docs/pre10-batch-c.md | 216 --- docs/pre10-completion-and-smoke.md | 347 ---- docs/pre11-baseline.md | 35 - docs/pre11-batch-a.md | 10 - docs/pre11-batch-b.md | 49 - docs/pre11-batch-c.md | 99 - docs/pre11-batch-d.md | 18 - docs/pre11-batch-e.md | 68 - docs/pre11-completion-and-smoke.md | 480 ----- docs/pre12-baseline.md | 11 - docs/pre12-batch-01.md | 29 - docs/pre12-batch-02.md | 31 - docs/pre12-batch-03.md | 49 - docs/pre12-batch-04.md | 35 - docs/pre12-batch-05.md | 31 - docs/pre12-batch-06.md | 60 - docs/pre12-batch-07.md | 39 - docs/pre12-batch-08.md | 32 - docs/pre12-batch-09.md | 44 - docs/pre12-batch-10.md | 24 - docs/pre12-final-report.md | 229 --- docs/pre13-baseline.md | 10 - docs/pre13-execution-status.md | 90 - docs/pre13-final-report.md | 344 ---- docs/pre13-h03a.md | 30 - docs/pre13-h03b.md | 39 - docs/pre13-h03c.md | 36 - docs/pre13-h04-root-validation.md | 44 - docs/pre13-low-a1.md | 21 - docs/pre13-low-a2.md | 14 - docs/pre13-low-b.md | 16 - docs/pre13-low-c1.md | 19 - docs/pre13-low-c2.md | 14 - docs/pre13-low-d.md | 10 - docs/pre13-low-e.md | 19 - docs/pre13-stage0-decisions.md | 46 - docs/pre15-stage0/P15-005.md | 40 - docs/pre15-stage0/P15-006.md | 60 - docs/pre15-stage0/P15-019.md | 102 - docs/pre15-stage0/P15-021.md | 93 - docs/pre15-stage0/P15-022.md | 87 - docs/pre15-stage0/P15-027.md | 43 - docs/pre15-stage0/P15-030.md | 51 - docs/pre15-stage0/P15-031.md | 45 - docs/pre15-stage0/P15-032.md | 106 - docs/pre15-stage0/P15-038.md | 85 - docs/pre15-stage0/P15-045.md | 44 - docs/pre15-stage0/P15-052.md | 74 - docs/pre15-stage0/P15-057.md | 13 - docs/pre15-stage0/P15-058.md | 13 - docs/pre15-stage0/P15-062.md | 89 - docs/pre15-stage0/P15-076.md | 75 - docs/pre15-stage0/P15-081.md | 95 - docs/pre15-stage0/P15-083.md | 101 - docs/pre15-stage0/P15-086.md | 147 -- docs/pre15-stage0/P15-087.md | 95 - docs/pre15-stage0/P15-092.md | 142 -- docs/pre15-stage0/README.md | 62 - docs/pre15-stage0/phase0-B-ZSTD-001.md | 101 - docs/pre2-baseline.md | 65 - docs/pre3-baseline.md | 83 - docs/pre3-smoke-test-20260812.md | 84 - docs/pre5-baseline.md | 80 - docs/pre5-completion-20260812.md | 268 --- docs/pre6-baseline.md | 99 - docs/pre7-baseline.md | 55 - docs/pre8-baseline.md | 86 - docs/pre8-completion-and-validation.md | 314 --- docs/pre8-loader-history-correction.md | 74 - docs/pre9-baseline.md | 191 -- docs/pre9-cache-final-manual-validation.md | 132 -- docs/pre9-controlled-manual-qemu.md | 158 -- docs/pre9-lzma-cache-implementation.md | 142 -- docs/pre9-lzma-cache-review-resolution.md | 62 - .../pre9-manual-evidence/full-sha-samples.txt | 26 - .../pre9-cache-final-kernel-cleanup.txt | 16 - .../pre9-cache-final-probes.txt | 21 - .../pre9-cache-final-result.json | 100 - docs/pre9-manual-evidence/probe-summary.txt | 22 - docs/pre9-manual-evidence/run3-audit.json | 62 - docs/refactoring-plan.md | 37 - docs/refactoring-status.md | 40 - src/erofs_fs.h => erofs_fs.h | 0 src/erofs_vnops.c => erofs_vnops.c | 0 src/inode.c => inode.c | 0 src/internal.h => internal.h | 0 issues/README.md | 16 - issues/TC010-48bit-statfs-large-provider.md | 92 - issues/TC060-pathconf-standard-values.md | 119 -- ...-large-directory-block-index-validation.md | 193 -- ...get-insmntque-failure-use-after-release.md | 176 -- issues/extent-metadata-fixture-unavailable.md | 169 -- src/namei.c => namei.c | 0 src/.gitignore | 18 - src/super.c => super.c | 0 test_all_decompress.sh | 462 ----- test_chunk_based.sh | 190 -- tests/EXECUTION-CHECKLIST.md | 78 - tests/G1-MANUAL-SETUP.md | 69 - tests/G3-MANUAL-SETUP.md | 71 - tests/G4-MANUAL-SETUP.md | 88 - tests/G5-MANUAL-SETUP.md | 118 -- tests/G6-MANUAL-SETUP.md | 164 -- tests/G7-MANUAL-SETUP.md | 226 --- tests/RESULT-SUMMARY-TEMPLATE.md | 229 --- tests/TC000-template.md | 36 - tests/TC001-mount-basic.md | 39 - tests/TC002-superblock-crc32c.md | 66 - tests/TC003-lz4-compressed-read.md | 87 - tests/TC004-lzma-compressed-read.md | 96 - tests/TC005-inline-xattr-user.md | 32 - tests/TC006-multidev-chunk-read.md | 61 - tests/TC007-concurrent-mount.md | 48 - tests/TC008-mount-errors.md | 77 - tests/TC009-statfs-basic.md | 42 - tests/TC010-statfs-48bit.md | 60 - tests/TC011-root-vnode.md | 36 - tests/TC012-vget-normal.md | 39 - tests/TC013-vget-invalid.md | 39 - tests/TC014-superblock-parse.md | 42 - tests/TC015-superblock-corrupted.md | 63 - tests/TC016-superblock-crc-invalid.md | 52 - tests/TC017-48bit-blocks-parse.md | 44 - tests/TC018-48bit-large-fs.md | 66 - tests/TC019-48bit-root-nid.md | 44 - tests/TC020-compact-inode-basic.md | 32 - tests/TC021-compact-inode-nlink1.md | 28 - tests/TC022-compact-inode-special.md | 31 - tests/TC023-extended-inode-normal.md | 27 - tests/TC024-extended-inode-large.md | 38 - tests/TC025-nlink1-handling.md | 35 - tests/TC026-dot-omitted-with.md | 33 - tests/TC027-dot-omitted-without.md | 35 - tests/TC028-flat-plain-small.md | 26 - tests/TC029-flat-plain-medium.md | 30 - tests/TC030-flat-plain-large.md | 33 - tests/TC031-flat-inline-normal.md | 28 - tests/TC032-flat-inline-zero.md | 29 - tests/TC033-tailpacking-normal.md | 33 - tests/TC034-tailpacking-boundary.md | 35 - tests/TC035-file-read-sequential.md | 29 - tests/TC036-file-read-random.md | 30 - tests/TC037-file-read-large.md | 33 - tests/TC038-symlink-short.md | 30 - tests/TC039-symlink-long.md | 36 - tests/TC040-symlink-broken.md | 30 - tests/TC041-dirent-decode-normal.md | 29 - tests/TC042-dirent-large-dir.md | 28 - tests/TC043-lookup-found.md | 29 - tests/TC044-lookup-not-found.md | 28 - tests/TC045-lookup-case-sensitive.md | 27 - tests/TC046-readdir-basic.md | 25 - tests/TC047-readdir-large.md | 28 - tests/TC048-readdir-seek.md | 31 - tests/TC049-dot-handling.md | 26 - tests/TC050-dotdot-handling.md | 26 - tests/TC051-namecache-hit.md | 29 - tests/TC052-namecache-miss.md | 28 - tests/TC053-vfs-hash-integration.md | 30 - tests/TC054-vn-vget-ino.md | 30 - tests/TC055-getattr-basic.md | 30 - tests/TC056-getattr-special.md | 28 - tests/TC057-access-allowed.md | 24 - tests/TC058-access-denied.md | 24 - tests/TC059-readlink.md | 30 - tests/TC060-pathconf.md | 25 - tests/TC061-setattr-chmod-reject.md | 24 - tests/TC062-setattr-chown-reject.md | 25 - tests/TC063-setattr-write-reject.md | 30 - tests/TC064-vop-getpages-normal.md | 25 - tests/TC065-vop-getpages-mmap.md | 24 - tests/TC066-vop-bmap-unsupported.md | 24 - tests/TC067-shared-xattr-scan-found.md | 28 - tests/TC068-shared-xattr-scan-notfound.md | 26 - tests/TC069-shared-xattr-list-multiple.md | 23 - tests/TC070-shared-trusted-xattr.md | 24 - tests/TC071-shared-security-xattr.md | 25 - tests/TC072-shared-system-xattr-list.md | 26 - tests/TC073-metabox-container.md | 30 - tests/TC074-inline-user-xattr-multiple.md | 24 - tests/TC075-inline-trusted-xattr.md | 24 - tests/TC076-inline-security-xattr.md | 22 - tests/TC077-long-prefix-user-xattr.md | 25 - tests/TC078-long-prefix-trusted-xattr.md | 24 - tests/TC079-packed-prefix-table.md | 33 - tests/TC080-prefix-table-lookup.md | 25 - tests/TC081-metabox-backed-xattr.md | 26 - tests/TC082-system-namespace-subset.md | 25 - tests/TC083-user-namespace-subset.md | 25 - tests/TC084-lz4-basic.md | 71 - tests/TC085-lz4-large-file.md | 78 - tests/TC086-lz4-sequential-read.md | 57 - tests/TC087-lz4-random-read.md | 53 - tests/TC088-lz4-config-handling.md | 85 - tests/TC089-lz4-pcluster-4k.md | 69 - tests/TC090-lz4-pcluster-64k.md | 72 - tests/TC091-ztailpacking-basic.md | 76 - tests/TC092-ztailpacking-edge.md | 79 - tests/TC093-chunk-single-device.md | 55 - tests/TC094-chunk-multi-device.md | 116 -- tests/TC095-chunk-index-read.md | 113 -- tests/TC096-device-table-parse.md | 105 - tests/TC097-device-table-invalid.md | 91 - tests/TC098-fragments-support.md | 59 - tests/TC099-multidev-2devices.md | 64 - tests/TC100-multidev-4devices.md | 91 - tests/TC101-unified-address-mapping.md | 151 -- tests/TC102-deflate-level1.md | 54 - tests/TC103-deflate-level6.md | 46 - tests/TC104-deflate-level9.md | 46 - tests/TC105-zstd-level1.md | 46 - tests/TC106-zstd-level15.md | 46 - tests/TC107-zstd-level22.md | 46 - tests/TC108-lzma-large-file.md | 49 - tests/TC109-microlzma.md | 48 - tests/TC110-lzma-corrupt.md | 50 - tests/TC111-manpage-accuracy.md | 54 - tests/TC112-invalid-superblock.md | 56 - tests/TC113-corrupted-inode.md | 48 - tests/TC114-out-of-bounds-block.md | 48 - tests/TC115-unsupported-algorithm.md | 54 - tests/TC116-truncated-compressed.md | 69 - tests/TC117-invalid-xattr-format.md | 33 - tests/TC118-device-not-found.md | 137 -- tests/TC119-data-crc-mismatch.md | 56 - tests/TC120-empty-file.md | 54 - tests/TC121-maximum-file-size.md | 64 - tests/TC122-deep-directory-tree.md | 56 - tests/TC123-long-filename.md | 55 - tests/TC124-many-small-files.md | 60 - tests/TC125-large-directory.md | 64 - tests/TC126-concurrent-reads.md | 77 - tests/TC127-memory-pressure.md | 114 -- tests/TC128-sequential-throughput.md | 62 - tests/TC129-random-read-pattern.md | 68 - tests/TC130-mixed-workload.md | 121 -- tests/TC131-nfs-export-basic.md | 143 -- tests/TC132-nfs-file-handle-stability.md | 97 - tests/TC133-nfs-export-stress.md | 176 -- tests/TC134-metabox-shared-nonzero-base.md | 26 - ...135-prefix-metabox-and-primary-fallback.md | 36 - ...TC136-metabox-truncated-super-extension.md | 26 - tests/TC137-xattr-declared-image-bounds.md | 37 - .../TC138-acl-linux-order-and-empty-header.md | 27 - tests/TC139-fifo-acl-xattr-readonly.md | 31 - tests/TC140-compressed-metabox-carrier.md | 37 - tests/TC141-cold-nested-namei.md | 44 - ...C142-fragment-backed-compressed-metabox.md | 42 - tests/TC143-deflate-zstd-partial-reference.md | 81 - ...44-microlzma-consumption-and-corruption.md | 52 - tests/TC145-zstdio-build-gate.md | 54 - .../TC146-head2-interlaced-extent-mapping.md | 85 - tests/TC147-flat-inline-block-bounds.md | 46 - tests/TC148-linux-directory-tail-padding.md | 28 - tests/TC149-vnode-pager-real-faults.md | 23 - tests/TC150-48bit-fallback-root.md | 43 - tests/TC151-extended-inode-off-max.md | 42 - tests/TC152-extent-hole-bounded-read.md | 26 - tests/TC153-large-directory-block-index.md | 43 - tests/TC154-nfs-per-inode-generation.md | 77 - tests/TC155-explicit-extent-pa-wrap.md | 60 - tests/TC156-inode-timestamp-validation.md | 51 - .../TC157-explicit-extent-order-validation.md | 55 - tests/TC158-48bit-compressed-blocks-hi.md | 45 - tests/TC159-special-setattr-combinations.md | 44 - tests/TC160-dot-omitted-offmax-cookie.md | 39 - tests/TC161-build-nm-failure.md | 38 - tests/TEST-COVERAGE-MATRIX.md | 304 --- tests/erofs_fixture.py | 764 -------- tests/final_review_fixtures.py | 412 ---- tests/final_review_probe.c | 127 -- tests/final_review_setattr_probe.c | 87 - tests/final_review_setattr_probe.mk | 4 - tests/g1_fixtures.py | 372 ---- tests/g3_fixtures.py | 24 - tests/g3_vfs_probe.c | 208 -- tests/g4_fixtures.py | 1277 ------------ tests/g5_fixtures.py | 1109 ----------- tests/g6_multidev_fixtures.py | 1218 ------------ tests/g7_fixtures.py | 495 ----- tests/g7_probe.c | 356 ---- tests/mmap_fault.c | 200 -- tests/nfs_fh_tool.c | 283 --- tests/pre13_ondisk_layout_probe.c | 244 --- tests/pre15/EVIDENCE-SCHEMA.json | 95 - tests/pre15/cases/B01-g3-equivalence.sh | 121 -- tests/pre15/cases/B01-runner-selftest.sh | 85 - tests/pre15/cases/B02-layout.sh | 319 --- tests/pre15/cases/B05-fields.sh | 190 -- tests/pre15/cases/B06-buffer.sh | 198 -- tests/pre15/cases/B07a-map-adapter.sh | 198 -- tests/pre15/cases/B07b-map-producers.sh | 360 ---- tests/pre15/cases/B07c-map-consumers.sh | 531 ----- tests/pre15/cases/B08-plain-run.sh | 907 --------- tests/pre15/cases/B09-vnode.sh | 375 ---- tests/pre15/cases/B10-directory.sh | 36 - tests/pre15/cases/B11-dtype.sh | 46 - tests/pre15/cases/B12-readahead.sh | 217 --- tests/pre15/cases/B13-time.sh | 604 ------ tests/pre15/cases/B14-chunk-types.sh | 226 --- tests/pre15/cases/B16-symlink.sh | 343 ---- tests/pre15/cases/B17-xattr-integrity.sh | 275 --- tests/pre15/cases/B18-xattr-order.sh | 359 ---- tests/pre15/cases/B19a-xattr-cache.sh | 217 --- tests/pre15/cases/B19b-xattr-bloom.sh | 220 --- tests/pre15/cases/B21-super.sh | 540 ------ tests/pre15/cases/B22-zmap-arithmetic.sh | 570 ------ tests/pre15/cases/B23-explicit-table.sh | 412 ---- tests/pre15/cases/B24-zmap-order.sh | 337 ---- tests/pre15/cases/B25-codec-errors.sh | 37 - tests/pre15/cases/B27-stream-tail.sh | 680 ------- tests/pre15/cases/B28-partial.sh | 667 ------- tests/pre15/cases/B29-context-pool.sh | 1247 ------------ tests/pre15/cases/B30-codec-structure.sh | 448 ----- tests/pre15/cases/B31-visibility.sh | 350 ---- tests/pre15/cases/B31r-visibility.sh | 334 ---- tests/pre15/cases/B32-cache-state.sh | 601 ------ tests/pre15/cases/B33-cache-policy.sh | 641 ------ tests/pre15/cases/B34-build-groups.sh | 412 ---- tests/pre15/cases/SMOKE-LZ4.sh | 4 - tests/pre15/cases/SMOKE-LZMA.sh | 4 - tests/pre15/cases/SMOKE-PLAIN.sh | 4 - tests/pre15/cases/SMOKE-ZSTD.sh | 4 - tests/pre15/cases/SMOKE-common.sh | 198 -- tests/pre15/cases/TC162-xattr-legacy.sh | 85 - tests/pre15/fixtures/B01-g3-archives.json | 17 - tests/pre15/fixtures/B01-runner-controls.json | 13 - tests/pre15/fixtures/B01-xattr-legacy.json | 42 - tests/pre15/fixtures/B06-ownership.json | 74 - tests/pre15/fixtures/B06-tuples.json | 31 - tests/pre15/fixtures/B07-map-oracle.json | 128 -- tests/pre15/fixtures/B08-io-caps.json | 85 - tests/pre15/fixtures/B08-map-runs.json | 138 -- tests/pre15/fixtures/B10-directory-oracle.py | 464 ----- tests/pre15/fixtures/B10-directory-spec.json | 110 -- tests/pre15/fixtures/B11-dtype-oracle.py | 396 ---- tests/pre15/fixtures/B11-dtype-spec.json | 28 - tests/pre15/fixtures/B12-build-kld.sh | 66 - tests/pre15/fixtures/B12-qemu-accept.sh | 228 --- .../pre15/fixtures/B12-readahead-policy.json | 16 - tests/pre15/fixtures/B13-compact-time.json | 165 -- tests/pre15/fixtures/B13-extended-time.json | 91 - tests/pre15/fixtures/B14-build-kld.sh | 62 - tests/pre15/fixtures/B14-chunk-fixtures.py | 661 ------- tests/pre15/fixtures/B14-chunk-probe.c | 69 - tests/pre15/fixtures/B14-chunk-spec.json | 98 - tests/pre15/fixtures/B16-build-kld.sh | 62 - tests/pre15/fixtures/B16-symlink-probe.c | 158 -- tests/pre15/fixtures/B16-symlink-spec.json | 56 - tests/pre15/fixtures/B17-xattr-generate.py | 529 ----- tests/pre15/fixtures/B17-xattr-oracle.py | 482 ----- tests/pre15/fixtures/B17-xattr-seed.erofs | Bin 4096 -> 0 bytes tests/pre15/fixtures/B17-xattr-spec.json | 324 ---- tests/pre15/fixtures/B19a-cache-model.c | 532 ----- tests/pre15/fixtures/B19a-xattr-generate.py | 418 ---- tests/pre15/fixtures/B19a-xattr-oracle.py | 853 -------- tests/pre15/fixtures/B19a-xattr-probe.c | 189 -- tests/pre15/fixtures/B19a-xattr-spec.json | 52 - tests/pre15/fixtures/B19b-build-kld.sh | 66 - tests/pre15/fixtures/B19b-xattr-generate.py | 253 --- tests/pre15/fixtures/B19b-xattr-oracle.py | 426 ---- tests/pre15/fixtures/B19b-xattr-probe.c | 207 -- tests/pre15/fixtures/B19b-xattr-spec.json | 46 - tests/pre15/fixtures/B21-manual-run.sh | 258 --- tests/pre15/fixtures/B21-qemu-probe.c | 35 - tests/pre15/fixtures/B21-super-generate.py | 276 --- tests/pre15/fixtures/B21-super-oracle.py | 135 -- tests/pre15/fixtures/B21-super-spec.json | 125 -- tests/pre15/fixtures/B22-build-kld.sh | 62 - tests/pre15/fixtures/B22-zmap-arithmetic.json | 320 --- tests/pre15/fixtures/B23-build-kld.sh | 62 - tests/pre15/fixtures/B23-explicit-generate.py | 538 ------ tests/pre15/fixtures/B23-explicit-oracle.py | 316 --- tests/pre15/fixtures/B23-explicit-probe.c | 122 -- tests/pre15/fixtures/B23-explicit-spec.json | 152 -- tests/pre15/fixtures/B25-codec-errors.json | 106 - tests/pre15/fixtures/B25-codec-oracle.py | 434 ----- tests/pre15/fixtures/B27-build-kld.sh | 78 - .../pre15/fixtures/B27-compact-finalization.c | 50 - tests/pre15/fixtures/B27-stream-fixtures.py | 176 -- tests/pre15/fixtures/B27-stream-probe.c | 122 -- tests/pre15/fixtures/B27-stream-tail.json | 115 -- tests/pre15/fixtures/B28-build-kld.sh | 84 - tests/pre15/fixtures/B28-partial-fixtures.py | 200 -- tests/pre15/fixtures/B28-partial-probe.c | 175 -- tests/pre15/fixtures/B28-partial.json | 168 -- tests/pre15/fixtures/B32-cache-generate.py | 125 -- tests/pre15/fixtures/B32-cache-oracle.c | 541 ------ tests/pre15/fixtures/B32-cache-probe.c | 405 ---- tests/pre15/fixtures/B32-cache-state.json | 40 - tests/pre15/fixtures/B32-qemu-run.sh | 306 --- tests/pre15/fixtures/B33-cache-generate.py | 150 -- tests/pre15/fixtures/B33-cache-oracle.c | 418 ---- tests/pre15/fixtures/B33-cache-state.json | 19 - tests/pre15/fixtures/B33-qemu-run.sh | 30 - tests/pre15/fixtures/SMOKE-generate.py | 43 - tests/pre15/fixtures/SMOKE-kldsym.c | 32 - tests/pre15/fixtures/g3.py | 1142 ----------- tests/pre15/gates/P15-005-input.json | 416 ---- tests/pre15/gates/P15-005.sh | 1018 ---------- tests/pre15/gates/P15-006-input.json | 119 -- tests/pre15/gates/P15-006.sh | 1712 ----------------- tests/pre15/gates/P15-019-input.json | 96 - tests/pre15/gates/P15-019.sh | 964 ---------- tests/pre15/gates/P15-021-input.json | 116 -- tests/pre15/gates/P15-021.sh | 892 --------- tests/pre15/gates/P15-022-input.json | 95 - tests/pre15/gates/P15-022.sh | 464 ----- tests/pre15/gates/P15-027-input.json | 180 -- tests/pre15/gates/P15-027.sh | 845 -------- tests/pre15/gates/P15-030-input.json | 110 -- tests/pre15/gates/P15-030.sh | 592 ------ tests/pre15/gates/P15-031-input.json | 273 --- tests/pre15/gates/P15-031.sh | 704 ------- tests/pre15/gates/P15-032-input.json | 251 --- tests/pre15/gates/P15-032.sh | 380 ---- tests/pre15/gates/P15-038-input.json | 58 - tests/pre15/gates/P15-038.sh | 1109 ----------- tests/pre15/gates/P15-045-input.json | 71 - tests/pre15/gates/P15-045.sh | 564 ------ tests/pre15/gates/P15-052-input.json | 203 -- tests/pre15/gates/P15-052.sh | 472 ----- tests/pre15/gates/P15-062-input.json | 138 -- tests/pre15/gates/P15-062.sh | 713 ------- tests/pre15/gates/P15-076-input.json | 88 - tests/pre15/gates/P15-076.sh | 853 -------- tests/pre15/gates/P15-081-input.json | 147 -- tests/pre15/gates/P15-081.sh | 901 --------- tests/pre15/gates/P15-083-input.json | 123 -- tests/pre15/gates/P15-083.sh | 860 --------- tests/pre15/gates/P15-086-input.json | 187 -- tests/pre15/gates/P15-086.sh | 1445 -------------- tests/pre15/gates/P15-087-input.json | 73 - tests/pre15/gates/P15-087.sh | 1419 -------------- tests/pre15/gates/P15-092-input.json | 42 - tests/pre15/gates/P15-092.sh | 881 --------- tests/pre15/lib/manifest.sh | 197 -- tests/pre15/lib/runner.sh | 751 -------- tests/pre15/probes/ondisk_layout.c | 318 --- tests/pre15/run-build.sh | 49 - tests/pre15/run-host.sh | 15 - tests/pre15/run-qemu.sh | 109 -- tests/pre15/run-smoke.sh | 39 - tests/prepare_directory_fixtures.sh | 54 - tests/prepare_error_fixtures.sh | 90 - tests/prepare_g1_fixtures.sh | 196 -- tests/prepare_g3_fixtures.sh | 21 - tests/read_probe.c | 128 -- tests/readdir_probe.c | 329 ---- .../2026-08-08T1037Z/manual-test-report.md | 188 -- .../2026-08-08T1114Z/manual-test-report.md | 435 ----- .../2026-08-08T1138Z/manual-test-report.md | 513 ----- .../2026-08-08T1307Z/manual-test-report.md | 141 -- .../2026-08-08T1332Z/manual-test-report.md | 84 - .../2026-08-08T1356Z/manual-test-report.md | 88 - .../2026-08-08T1413Z/manual-test-report.md | 99 - .../2026-08-08T1427Z/manual-test-report.md | 53 - .../2026-08-08T1444Z/manual-test-report.md | 41 - .../2026-08-08T1455Z/manual-test-report.md | 33 - .../2026-08-08T1705Z/manual-test-report.md | 46 - .../manual-test-report.md | 112 -- .../prepare-fixtures.sh | 199 -- .../2026-08-08T1800Z-namei/readdir_probe.c | 71 - .../2026-08-08T1800Z-namei/vm-regression.sh | 201 -- .../2026-08-08T1812Z/manual-test-report.md | 113 -- .../manual-test-report.md | 232 --- .../manual-test-report.md | 559 ------ .../manual-test-report.md | 142 -- .../manual-test-report.md | 406 ---- .../manual-test-report.md | 358 ---- .../prepare-fixtures.sh | 277 --- .../manual-test-report.md | 166 -- .../prepare-fixtures.sh | 271 --- .../manual-test-report.md | 98 - .../2026-08-09T0710Z-g1/manual-test-report.md | 159 -- .../2026-08-09T0710Z-g2/manual-test-report.md | 269 --- .../2026-08-09T0839Z-g4/manual-test-report.md | 132 -- .../2026-08-09T1059Z-g3/manual-test-report.md | 154 -- .../manual-test-report.md | 157 -- .../2026-08-09T1236Z-g5/manual-test-report.md | 212 -- .../2026-08-09T1244Z-g6/manual-test-report.md | 139 -- .../2026-08-09T1343Z-g8/manual-test-report.md | 269 --- .../2026-08-09T1359Z-g7/manual-test-report.md | 183 -- .../manual-test-report.md | 119 -- tests/review_fixtures.py | 608 ------ tests/sparse_hole_probe.c | 63 - tests/stat_special.c | 70 - tests/statfs_probe.c | 24 - src/xattr.c => xattr.c | 0 src/xattr.h => xattr.h | 0 src/zdata.c => zdata.c | 0 src/zmap.c => zmap.c | 0 522 files changed, 17 insertions(+), 84680 deletions(-) rename src/.clang-format => .clang-format (100%) rename src/Makefile => Makefile (100%) delete mode 100644 README.md delete mode 100755 build.sh rename src/compress.h => compress.h (100%) delete mode 100644 current/README.md delete mode 100644 current/report-1/2026-08-09-overall-progress.md delete mode 100644 current/report-1/README.md delete mode 100644 current/report-2/2026-08-09-overall-progress.md delete mode 100644 current/report-2/README.md delete mode 100644 current/report-3/2026-08-09-overall-progress.md delete mode 100644 current/report-3/README.md delete mode 100644 current/report-4/2026-08-10-task2-code-review-style-assessment.md delete mode 100644 current/report-4/README.md rename src/data.c => data.c (100%) rename src/decompressor.c => decompressor.c (100%) rename src/decompressor_deflate.c => decompressor_deflate.c (100%) rename src/decompressor_lz4.c => decompressor_lz4.c (100%) rename src/decompressor_lzma.c => decompressor_lzma.c (100%) rename src/decompressor_zstd.c => decompressor_zstd.c (100%) rename src/dir.c => dir.c (100%) delete mode 100644 docs/TEST_REPORT.md delete mode 100644 docs/architecture.md delete mode 100644 docs/capabilities.md delete mode 100644 docs/erofs.5 delete mode 100644 docs/features.md delete mode 100644 docs/pre10-baseline.md delete mode 100644 docs/pre10-batch-a.md delete mode 100644 docs/pre10-batch-b.md delete mode 100644 docs/pre10-batch-c.md delete mode 100644 docs/pre10-completion-and-smoke.md delete mode 100644 docs/pre11-baseline.md delete mode 100644 docs/pre11-batch-a.md delete mode 100644 docs/pre11-batch-b.md delete mode 100644 docs/pre11-batch-c.md delete mode 100644 docs/pre11-batch-d.md delete mode 100644 docs/pre11-batch-e.md delete mode 100644 docs/pre11-completion-and-smoke.md delete mode 100644 docs/pre12-baseline.md delete mode 100644 docs/pre12-batch-01.md delete mode 100644 docs/pre12-batch-02.md delete mode 100644 docs/pre12-batch-03.md delete mode 100644 docs/pre12-batch-04.md delete mode 100644 docs/pre12-batch-05.md delete mode 100644 docs/pre12-batch-06.md delete mode 100644 docs/pre12-batch-07.md delete mode 100644 docs/pre12-batch-08.md delete mode 100644 docs/pre12-batch-09.md delete mode 100644 docs/pre12-batch-10.md delete mode 100644 docs/pre12-final-report.md delete mode 100644 docs/pre13-baseline.md delete mode 100644 docs/pre13-execution-status.md delete mode 100644 docs/pre13-final-report.md delete mode 100644 docs/pre13-h03a.md delete mode 100644 docs/pre13-h03b.md delete mode 100644 docs/pre13-h03c.md delete mode 100644 docs/pre13-h04-root-validation.md delete mode 100644 docs/pre13-low-a1.md delete mode 100644 docs/pre13-low-a2.md delete mode 100644 docs/pre13-low-b.md delete mode 100644 docs/pre13-low-c1.md delete mode 100644 docs/pre13-low-c2.md delete mode 100644 docs/pre13-low-d.md delete mode 100644 docs/pre13-low-e.md delete mode 100644 docs/pre13-stage0-decisions.md delete mode 100644 docs/pre15-stage0/P15-005.md delete mode 100644 docs/pre15-stage0/P15-006.md delete mode 100644 docs/pre15-stage0/P15-019.md delete mode 100644 docs/pre15-stage0/P15-021.md delete mode 100644 docs/pre15-stage0/P15-022.md delete mode 100644 docs/pre15-stage0/P15-027.md delete mode 100644 docs/pre15-stage0/P15-030.md delete mode 100644 docs/pre15-stage0/P15-031.md delete mode 100644 docs/pre15-stage0/P15-032.md delete mode 100644 docs/pre15-stage0/P15-038.md delete mode 100644 docs/pre15-stage0/P15-045.md delete mode 100644 docs/pre15-stage0/P15-052.md delete mode 100644 docs/pre15-stage0/P15-057.md delete mode 100644 docs/pre15-stage0/P15-058.md delete mode 100644 docs/pre15-stage0/P15-062.md delete mode 100644 docs/pre15-stage0/P15-076.md delete mode 100644 docs/pre15-stage0/P15-081.md delete mode 100644 docs/pre15-stage0/P15-083.md delete mode 100644 docs/pre15-stage0/P15-086.md delete mode 100644 docs/pre15-stage0/P15-087.md delete mode 100644 docs/pre15-stage0/P15-092.md delete mode 100644 docs/pre15-stage0/README.md delete mode 100644 docs/pre15-stage0/phase0-B-ZSTD-001.md delete mode 100644 docs/pre2-baseline.md delete mode 100644 docs/pre3-baseline.md delete mode 100644 docs/pre3-smoke-test-20260812.md delete mode 100644 docs/pre5-baseline.md delete mode 100644 docs/pre5-completion-20260812.md delete mode 100644 docs/pre6-baseline.md delete mode 100644 docs/pre7-baseline.md delete mode 100644 docs/pre8-baseline.md delete mode 100644 docs/pre8-completion-and-validation.md delete mode 100644 docs/pre8-loader-history-correction.md delete mode 100644 docs/pre9-baseline.md delete mode 100644 docs/pre9-cache-final-manual-validation.md delete mode 100644 docs/pre9-controlled-manual-qemu.md delete mode 100644 docs/pre9-lzma-cache-implementation.md delete mode 100644 docs/pre9-lzma-cache-review-resolution.md delete mode 100644 docs/pre9-manual-evidence/full-sha-samples.txt delete mode 100644 docs/pre9-manual-evidence/pre9-cache-final-kernel-cleanup.txt delete mode 100644 docs/pre9-manual-evidence/pre9-cache-final-probes.txt delete mode 100644 docs/pre9-manual-evidence/pre9-cache-final-result.json delete mode 100644 docs/pre9-manual-evidence/probe-summary.txt delete mode 100644 docs/pre9-manual-evidence/run3-audit.json delete mode 100644 docs/refactoring-plan.md delete mode 100644 docs/refactoring-status.md rename src/erofs_fs.h => erofs_fs.h (100%) rename src/erofs_vnops.c => erofs_vnops.c (100%) rename src/inode.c => inode.c (100%) rename src/internal.h => internal.h (100%) delete mode 100644 issues/README.md delete mode 100644 issues/TC010-48bit-statfs-large-provider.md delete mode 100644 issues/TC060-pathconf-standard-values.md delete mode 100644 issues/TC153-large-directory-block-index-validation.md delete mode 100644 issues/erofs-vget-insmntque-failure-use-after-release.md delete mode 100644 issues/extent-metadata-fixture-unavailable.md rename src/namei.c => namei.c (100%) delete mode 100644 src/.gitignore rename src/super.c => super.c (100%) delete mode 100755 test_all_decompress.sh delete mode 100755 test_chunk_based.sh delete mode 100644 tests/EXECUTION-CHECKLIST.md delete mode 100644 tests/G1-MANUAL-SETUP.md delete mode 100644 tests/G3-MANUAL-SETUP.md delete mode 100644 tests/G4-MANUAL-SETUP.md delete mode 100644 tests/G5-MANUAL-SETUP.md delete mode 100644 tests/G6-MANUAL-SETUP.md delete mode 100644 tests/G7-MANUAL-SETUP.md delete mode 100644 tests/RESULT-SUMMARY-TEMPLATE.md delete mode 100644 tests/TC000-template.md delete mode 100644 tests/TC001-mount-basic.md delete mode 100644 tests/TC002-superblock-crc32c.md delete mode 100644 tests/TC003-lz4-compressed-read.md delete mode 100644 tests/TC004-lzma-compressed-read.md delete mode 100644 tests/TC005-inline-xattr-user.md delete mode 100644 tests/TC006-multidev-chunk-read.md delete mode 100644 tests/TC007-concurrent-mount.md delete mode 100644 tests/TC008-mount-errors.md delete mode 100644 tests/TC009-statfs-basic.md delete mode 100644 tests/TC010-statfs-48bit.md delete mode 100644 tests/TC011-root-vnode.md delete mode 100644 tests/TC012-vget-normal.md delete mode 100644 tests/TC013-vget-invalid.md delete mode 100644 tests/TC014-superblock-parse.md delete mode 100644 tests/TC015-superblock-corrupted.md delete mode 100644 tests/TC016-superblock-crc-invalid.md delete mode 100644 tests/TC017-48bit-blocks-parse.md delete mode 100644 tests/TC018-48bit-large-fs.md delete mode 100644 tests/TC019-48bit-root-nid.md delete mode 100644 tests/TC020-compact-inode-basic.md delete mode 100644 tests/TC021-compact-inode-nlink1.md delete mode 100644 tests/TC022-compact-inode-special.md delete mode 100644 tests/TC023-extended-inode-normal.md delete mode 100644 tests/TC024-extended-inode-large.md delete mode 100644 tests/TC025-nlink1-handling.md delete mode 100644 tests/TC026-dot-omitted-with.md delete mode 100644 tests/TC027-dot-omitted-without.md delete mode 100644 tests/TC028-flat-plain-small.md delete mode 100644 tests/TC029-flat-plain-medium.md delete mode 100644 tests/TC030-flat-plain-large.md delete mode 100644 tests/TC031-flat-inline-normal.md delete mode 100644 tests/TC032-flat-inline-zero.md delete mode 100644 tests/TC033-tailpacking-normal.md delete mode 100644 tests/TC034-tailpacking-boundary.md delete mode 100644 tests/TC035-file-read-sequential.md delete mode 100644 tests/TC036-file-read-random.md delete mode 100644 tests/TC037-file-read-large.md delete mode 100644 tests/TC038-symlink-short.md delete mode 100644 tests/TC039-symlink-long.md delete mode 100644 tests/TC040-symlink-broken.md delete mode 100644 tests/TC041-dirent-decode-normal.md delete mode 100644 tests/TC042-dirent-large-dir.md delete mode 100644 tests/TC043-lookup-found.md delete mode 100644 tests/TC044-lookup-not-found.md delete mode 100644 tests/TC045-lookup-case-sensitive.md delete mode 100644 tests/TC046-readdir-basic.md delete mode 100644 tests/TC047-readdir-large.md delete mode 100644 tests/TC048-readdir-seek.md delete mode 100644 tests/TC049-dot-handling.md delete mode 100644 tests/TC050-dotdot-handling.md delete mode 100644 tests/TC051-namecache-hit.md delete mode 100644 tests/TC052-namecache-miss.md delete mode 100644 tests/TC053-vfs-hash-integration.md delete mode 100644 tests/TC054-vn-vget-ino.md delete mode 100644 tests/TC055-getattr-basic.md delete mode 100644 tests/TC056-getattr-special.md delete mode 100644 tests/TC057-access-allowed.md delete mode 100644 tests/TC058-access-denied.md delete mode 100644 tests/TC059-readlink.md delete mode 100644 tests/TC060-pathconf.md delete mode 100644 tests/TC061-setattr-chmod-reject.md delete mode 100644 tests/TC062-setattr-chown-reject.md delete mode 100644 tests/TC063-setattr-write-reject.md delete mode 100644 tests/TC064-vop-getpages-normal.md delete mode 100644 tests/TC065-vop-getpages-mmap.md delete mode 100644 tests/TC066-vop-bmap-unsupported.md delete mode 100644 tests/TC067-shared-xattr-scan-found.md delete mode 100644 tests/TC068-shared-xattr-scan-notfound.md delete mode 100644 tests/TC069-shared-xattr-list-multiple.md delete mode 100644 tests/TC070-shared-trusted-xattr.md delete mode 100644 tests/TC071-shared-security-xattr.md delete mode 100644 tests/TC072-shared-system-xattr-list.md delete mode 100644 tests/TC073-metabox-container.md delete mode 100644 tests/TC074-inline-user-xattr-multiple.md delete mode 100644 tests/TC075-inline-trusted-xattr.md delete mode 100644 tests/TC076-inline-security-xattr.md delete mode 100644 tests/TC077-long-prefix-user-xattr.md delete mode 100644 tests/TC078-long-prefix-trusted-xattr.md delete mode 100644 tests/TC079-packed-prefix-table.md delete mode 100644 tests/TC080-prefix-table-lookup.md delete mode 100644 tests/TC081-metabox-backed-xattr.md delete mode 100644 tests/TC082-system-namespace-subset.md delete mode 100644 tests/TC083-user-namespace-subset.md delete mode 100644 tests/TC084-lz4-basic.md delete mode 100644 tests/TC085-lz4-large-file.md delete mode 100644 tests/TC086-lz4-sequential-read.md delete mode 100644 tests/TC087-lz4-random-read.md delete mode 100644 tests/TC088-lz4-config-handling.md delete mode 100644 tests/TC089-lz4-pcluster-4k.md delete mode 100644 tests/TC090-lz4-pcluster-64k.md delete mode 100644 tests/TC091-ztailpacking-basic.md delete mode 100644 tests/TC092-ztailpacking-edge.md delete mode 100644 tests/TC093-chunk-single-device.md delete mode 100644 tests/TC094-chunk-multi-device.md delete mode 100644 tests/TC095-chunk-index-read.md delete mode 100644 tests/TC096-device-table-parse.md delete mode 100644 tests/TC097-device-table-invalid.md delete mode 100644 tests/TC098-fragments-support.md delete mode 100644 tests/TC099-multidev-2devices.md delete mode 100644 tests/TC100-multidev-4devices.md delete mode 100644 tests/TC101-unified-address-mapping.md delete mode 100644 tests/TC102-deflate-level1.md delete mode 100644 tests/TC103-deflate-level6.md delete mode 100644 tests/TC104-deflate-level9.md delete mode 100644 tests/TC105-zstd-level1.md delete mode 100644 tests/TC106-zstd-level15.md delete mode 100644 tests/TC107-zstd-level22.md delete mode 100644 tests/TC108-lzma-large-file.md delete mode 100644 tests/TC109-microlzma.md delete mode 100644 tests/TC110-lzma-corrupt.md delete mode 100644 tests/TC111-manpage-accuracy.md delete mode 100644 tests/TC112-invalid-superblock.md delete mode 100644 tests/TC113-corrupted-inode.md delete mode 100644 tests/TC114-out-of-bounds-block.md delete mode 100644 tests/TC115-unsupported-algorithm.md delete mode 100644 tests/TC116-truncated-compressed.md delete mode 100644 tests/TC117-invalid-xattr-format.md delete mode 100644 tests/TC118-device-not-found.md delete mode 100644 tests/TC119-data-crc-mismatch.md delete mode 100644 tests/TC120-empty-file.md delete mode 100644 tests/TC121-maximum-file-size.md delete mode 100644 tests/TC122-deep-directory-tree.md delete mode 100644 tests/TC123-long-filename.md delete mode 100644 tests/TC124-many-small-files.md delete mode 100644 tests/TC125-large-directory.md delete mode 100644 tests/TC126-concurrent-reads.md delete mode 100644 tests/TC127-memory-pressure.md delete mode 100644 tests/TC128-sequential-throughput.md delete mode 100644 tests/TC129-random-read-pattern.md delete mode 100644 tests/TC130-mixed-workload.md delete mode 100644 tests/TC131-nfs-export-basic.md delete mode 100644 tests/TC132-nfs-file-handle-stability.md delete mode 100644 tests/TC133-nfs-export-stress.md delete mode 100644 tests/TC134-metabox-shared-nonzero-base.md delete mode 100644 tests/TC135-prefix-metabox-and-primary-fallback.md delete mode 100644 tests/TC136-metabox-truncated-super-extension.md delete mode 100644 tests/TC137-xattr-declared-image-bounds.md delete mode 100644 tests/TC138-acl-linux-order-and-empty-header.md delete mode 100644 tests/TC139-fifo-acl-xattr-readonly.md delete mode 100644 tests/TC140-compressed-metabox-carrier.md delete mode 100644 tests/TC141-cold-nested-namei.md delete mode 100644 tests/TC142-fragment-backed-compressed-metabox.md delete mode 100644 tests/TC143-deflate-zstd-partial-reference.md delete mode 100644 tests/TC144-microlzma-consumption-and-corruption.md delete mode 100644 tests/TC145-zstdio-build-gate.md delete mode 100644 tests/TC146-head2-interlaced-extent-mapping.md delete mode 100644 tests/TC147-flat-inline-block-bounds.md delete mode 100644 tests/TC148-linux-directory-tail-padding.md delete mode 100644 tests/TC149-vnode-pager-real-faults.md delete mode 100644 tests/TC150-48bit-fallback-root.md delete mode 100644 tests/TC151-extended-inode-off-max.md delete mode 100644 tests/TC152-extent-hole-bounded-read.md delete mode 100644 tests/TC153-large-directory-block-index.md delete mode 100644 tests/TC154-nfs-per-inode-generation.md delete mode 100644 tests/TC155-explicit-extent-pa-wrap.md delete mode 100644 tests/TC156-inode-timestamp-validation.md delete mode 100644 tests/TC157-explicit-extent-order-validation.md delete mode 100644 tests/TC158-48bit-compressed-blocks-hi.md delete mode 100644 tests/TC159-special-setattr-combinations.md delete mode 100644 tests/TC160-dot-omitted-offmax-cookie.md delete mode 100644 tests/TC161-build-nm-failure.md delete mode 100644 tests/TEST-COVERAGE-MATRIX.md delete mode 100755 tests/erofs_fixture.py delete mode 100644 tests/final_review_fixtures.py delete mode 100644 tests/final_review_probe.c delete mode 100644 tests/final_review_setattr_probe.c delete mode 100644 tests/final_review_setattr_probe.mk delete mode 100755 tests/g1_fixtures.py delete mode 100755 tests/g3_fixtures.py delete mode 100644 tests/g3_vfs_probe.c delete mode 100755 tests/g4_fixtures.py delete mode 100644 tests/g5_fixtures.py delete mode 100644 tests/g6_multidev_fixtures.py delete mode 100644 tests/g7_fixtures.py delete mode 100644 tests/g7_probe.c delete mode 100644 tests/mmap_fault.c delete mode 100644 tests/nfs_fh_tool.c delete mode 100644 tests/pre13_ondisk_layout_probe.c delete mode 100644 tests/pre15/EVIDENCE-SCHEMA.json delete mode 100755 tests/pre15/cases/B01-g3-equivalence.sh delete mode 100755 tests/pre15/cases/B01-runner-selftest.sh delete mode 100755 tests/pre15/cases/B02-layout.sh delete mode 100755 tests/pre15/cases/B05-fields.sh delete mode 100755 tests/pre15/cases/B06-buffer.sh delete mode 100755 tests/pre15/cases/B07a-map-adapter.sh delete mode 100755 tests/pre15/cases/B07b-map-producers.sh delete mode 100755 tests/pre15/cases/B07c-map-consumers.sh delete mode 100755 tests/pre15/cases/B08-plain-run.sh delete mode 100755 tests/pre15/cases/B09-vnode.sh delete mode 100755 tests/pre15/cases/B10-directory.sh delete mode 100755 tests/pre15/cases/B11-dtype.sh delete mode 100755 tests/pre15/cases/B12-readahead.sh delete mode 100755 tests/pre15/cases/B13-time.sh delete mode 100755 tests/pre15/cases/B14-chunk-types.sh delete mode 100755 tests/pre15/cases/B16-symlink.sh delete mode 100755 tests/pre15/cases/B17-xattr-integrity.sh delete mode 100755 tests/pre15/cases/B18-xattr-order.sh delete mode 100644 tests/pre15/cases/B19a-xattr-cache.sh delete mode 100755 tests/pre15/cases/B19b-xattr-bloom.sh delete mode 100644 tests/pre15/cases/B21-super.sh delete mode 100755 tests/pre15/cases/B22-zmap-arithmetic.sh delete mode 100644 tests/pre15/cases/B23-explicit-table.sh delete mode 100755 tests/pre15/cases/B24-zmap-order.sh delete mode 100755 tests/pre15/cases/B25-codec-errors.sh delete mode 100755 tests/pre15/cases/B27-stream-tail.sh delete mode 100755 tests/pre15/cases/B28-partial.sh delete mode 100644 tests/pre15/cases/B29-context-pool.sh delete mode 100755 tests/pre15/cases/B30-codec-structure.sh delete mode 100755 tests/pre15/cases/B31-visibility.sh delete mode 100755 tests/pre15/cases/B31r-visibility.sh delete mode 100755 tests/pre15/cases/B32-cache-state.sh delete mode 100755 tests/pre15/cases/B33-cache-policy.sh delete mode 100755 tests/pre15/cases/B34-build-groups.sh delete mode 100755 tests/pre15/cases/SMOKE-LZ4.sh delete mode 100755 tests/pre15/cases/SMOKE-LZMA.sh delete mode 100755 tests/pre15/cases/SMOKE-PLAIN.sh delete mode 100755 tests/pre15/cases/SMOKE-ZSTD.sh delete mode 100755 tests/pre15/cases/SMOKE-common.sh delete mode 100755 tests/pre15/cases/TC162-xattr-legacy.sh delete mode 100644 tests/pre15/fixtures/B01-g3-archives.json delete mode 100644 tests/pre15/fixtures/B01-runner-controls.json delete mode 100644 tests/pre15/fixtures/B01-xattr-legacy.json delete mode 100644 tests/pre15/fixtures/B06-ownership.json delete mode 100644 tests/pre15/fixtures/B06-tuples.json delete mode 100644 tests/pre15/fixtures/B07-map-oracle.json delete mode 100644 tests/pre15/fixtures/B08-io-caps.json delete mode 100644 tests/pre15/fixtures/B08-map-runs.json delete mode 100755 tests/pre15/fixtures/B10-directory-oracle.py delete mode 100644 tests/pre15/fixtures/B10-directory-spec.json delete mode 100755 tests/pre15/fixtures/B11-dtype-oracle.py delete mode 100644 tests/pre15/fixtures/B11-dtype-spec.json delete mode 100755 tests/pre15/fixtures/B12-build-kld.sh delete mode 100755 tests/pre15/fixtures/B12-qemu-accept.sh delete mode 100644 tests/pre15/fixtures/B12-readahead-policy.json delete mode 100644 tests/pre15/fixtures/B13-compact-time.json delete mode 100644 tests/pre15/fixtures/B13-extended-time.json delete mode 100755 tests/pre15/fixtures/B14-build-kld.sh delete mode 100755 tests/pre15/fixtures/B14-chunk-fixtures.py delete mode 100644 tests/pre15/fixtures/B14-chunk-probe.c delete mode 100644 tests/pre15/fixtures/B14-chunk-spec.json delete mode 100755 tests/pre15/fixtures/B16-build-kld.sh delete mode 100644 tests/pre15/fixtures/B16-symlink-probe.c delete mode 100644 tests/pre15/fixtures/B16-symlink-spec.json delete mode 100755 tests/pre15/fixtures/B17-xattr-generate.py delete mode 100755 tests/pre15/fixtures/B17-xattr-oracle.py delete mode 100644 tests/pre15/fixtures/B17-xattr-seed.erofs delete mode 100644 tests/pre15/fixtures/B17-xattr-spec.json delete mode 100644 tests/pre15/fixtures/B19a-cache-model.c delete mode 100755 tests/pre15/fixtures/B19a-xattr-generate.py delete mode 100755 tests/pre15/fixtures/B19a-xattr-oracle.py delete mode 100644 tests/pre15/fixtures/B19a-xattr-probe.c delete mode 100644 tests/pre15/fixtures/B19a-xattr-spec.json delete mode 100755 tests/pre15/fixtures/B19b-build-kld.sh delete mode 100755 tests/pre15/fixtures/B19b-xattr-generate.py delete mode 100755 tests/pre15/fixtures/B19b-xattr-oracle.py delete mode 100644 tests/pre15/fixtures/B19b-xattr-probe.c delete mode 100644 tests/pre15/fixtures/B19b-xattr-spec.json delete mode 100755 tests/pre15/fixtures/B21-manual-run.sh delete mode 100644 tests/pre15/fixtures/B21-qemu-probe.c delete mode 100644 tests/pre15/fixtures/B21-super-generate.py delete mode 100644 tests/pre15/fixtures/B21-super-oracle.py delete mode 100644 tests/pre15/fixtures/B21-super-spec.json delete mode 100755 tests/pre15/fixtures/B22-build-kld.sh delete mode 100644 tests/pre15/fixtures/B22-zmap-arithmetic.json delete mode 100644 tests/pre15/fixtures/B23-build-kld.sh delete mode 100644 tests/pre15/fixtures/B23-explicit-generate.py delete mode 100644 tests/pre15/fixtures/B23-explicit-oracle.py delete mode 100644 tests/pre15/fixtures/B23-explicit-probe.c delete mode 100644 tests/pre15/fixtures/B23-explicit-spec.json delete mode 100644 tests/pre15/fixtures/B25-codec-errors.json delete mode 100755 tests/pre15/fixtures/B25-codec-oracle.py delete mode 100755 tests/pre15/fixtures/B27-build-kld.sh delete mode 100644 tests/pre15/fixtures/B27-compact-finalization.c delete mode 100755 tests/pre15/fixtures/B27-stream-fixtures.py delete mode 100644 tests/pre15/fixtures/B27-stream-probe.c delete mode 100644 tests/pre15/fixtures/B27-stream-tail.json delete mode 100755 tests/pre15/fixtures/B28-build-kld.sh delete mode 100755 tests/pre15/fixtures/B28-partial-fixtures.py delete mode 100644 tests/pre15/fixtures/B28-partial-probe.c delete mode 100644 tests/pre15/fixtures/B28-partial.json delete mode 100755 tests/pre15/fixtures/B32-cache-generate.py delete mode 100644 tests/pre15/fixtures/B32-cache-oracle.c delete mode 100644 tests/pre15/fixtures/B32-cache-probe.c delete mode 100644 tests/pre15/fixtures/B32-cache-state.json delete mode 100755 tests/pre15/fixtures/B32-qemu-run.sh delete mode 100755 tests/pre15/fixtures/B33-cache-generate.py delete mode 100644 tests/pre15/fixtures/B33-cache-oracle.c delete mode 100644 tests/pre15/fixtures/B33-cache-state.json delete mode 100755 tests/pre15/fixtures/B33-qemu-run.sh delete mode 100755 tests/pre15/fixtures/SMOKE-generate.py delete mode 100644 tests/pre15/fixtures/SMOKE-kldsym.c delete mode 100755 tests/pre15/fixtures/g3.py delete mode 100644 tests/pre15/gates/P15-005-input.json delete mode 100755 tests/pre15/gates/P15-005.sh delete mode 100644 tests/pre15/gates/P15-006-input.json delete mode 100755 tests/pre15/gates/P15-006.sh delete mode 100644 tests/pre15/gates/P15-019-input.json delete mode 100755 tests/pre15/gates/P15-019.sh delete mode 100644 tests/pre15/gates/P15-021-input.json delete mode 100755 tests/pre15/gates/P15-021.sh delete mode 100644 tests/pre15/gates/P15-022-input.json delete mode 100755 tests/pre15/gates/P15-022.sh delete mode 100644 tests/pre15/gates/P15-027-input.json delete mode 100755 tests/pre15/gates/P15-027.sh delete mode 100644 tests/pre15/gates/P15-030-input.json delete mode 100755 tests/pre15/gates/P15-030.sh delete mode 100644 tests/pre15/gates/P15-031-input.json delete mode 100755 tests/pre15/gates/P15-031.sh delete mode 100644 tests/pre15/gates/P15-032-input.json delete mode 100755 tests/pre15/gates/P15-032.sh delete mode 100644 tests/pre15/gates/P15-038-input.json delete mode 100755 tests/pre15/gates/P15-038.sh delete mode 100644 tests/pre15/gates/P15-045-input.json delete mode 100755 tests/pre15/gates/P15-045.sh delete mode 100644 tests/pre15/gates/P15-052-input.json delete mode 100755 tests/pre15/gates/P15-052.sh delete mode 100644 tests/pre15/gates/P15-062-input.json delete mode 100755 tests/pre15/gates/P15-062.sh delete mode 100644 tests/pre15/gates/P15-076-input.json delete mode 100755 tests/pre15/gates/P15-076.sh delete mode 100644 tests/pre15/gates/P15-081-input.json delete mode 100755 tests/pre15/gates/P15-081.sh delete mode 100644 tests/pre15/gates/P15-083-input.json delete mode 100755 tests/pre15/gates/P15-083.sh delete mode 100644 tests/pre15/gates/P15-086-input.json delete mode 100755 tests/pre15/gates/P15-086.sh delete mode 100644 tests/pre15/gates/P15-087-input.json delete mode 100755 tests/pre15/gates/P15-087.sh delete mode 100644 tests/pre15/gates/P15-092-input.json delete mode 100755 tests/pre15/gates/P15-092.sh delete mode 100644 tests/pre15/lib/manifest.sh delete mode 100644 tests/pre15/lib/runner.sh delete mode 100644 tests/pre15/probes/ondisk_layout.c delete mode 100755 tests/pre15/run-build.sh delete mode 100755 tests/pre15/run-host.sh delete mode 100755 tests/pre15/run-qemu.sh delete mode 100755 tests/pre15/run-smoke.sh delete mode 100755 tests/prepare_directory_fixtures.sh delete mode 100755 tests/prepare_error_fixtures.sh delete mode 100755 tests/prepare_g1_fixtures.sh delete mode 100755 tests/prepare_g3_fixtures.sh delete mode 100644 tests/read_probe.c delete mode 100644 tests/readdir_probe.c delete mode 100644 tests/results/manual/2026-08-08T1037Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1114Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1138Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1307Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1332Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1356Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1413Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1427Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1444Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1455Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1705Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1800Z-namei/manual-test-report.md delete mode 100755 tests/results/manual/2026-08-08T1800Z-namei/prepare-fixtures.sh delete mode 100644 tests/results/manual/2026-08-08T1800Z-namei/readdir_probe.c delete mode 100755 tests/results/manual/2026-08-08T1800Z-namei/vm-regression.sh delete mode 100644 tests/results/manual/2026-08-08T1812Z/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T1937Z-multidev/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T2037Z-nfs/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T2114Z-multidev-review/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T2306Z-compression-p0/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-08T2337Z-metadata-vfs/manual-test-report.md delete mode 100755 tests/results/manual/2026-08-08T2337Z-metadata-vfs/prepare-fixtures.sh delete mode 100644 tests/results/manual/2026-08-09T0124Z-final-review/manual-test-report.md delete mode 100755 tests/results/manual/2026-08-09T0124Z-final-review/prepare-fixtures.sh delete mode 100644 tests/results/manual/2026-08-09T0557Z-review-fixes/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T0710Z-g1/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T0710Z-g2/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T0839Z-g4/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T1059Z-g3/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T1120Z-tc060-fix/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T1236Z-g5/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T1244Z-g6/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T1343Z-g8/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T1359Z-g7/manual-test-report.md delete mode 100644 tests/results/manual/2026-08-09T1804Z-final-review-independent/manual-test-report.md delete mode 100644 tests/review_fixtures.py delete mode 100644 tests/sparse_hole_probe.c delete mode 100644 tests/stat_special.c delete mode 100644 tests/statfs_probe.c rename src/xattr.c => xattr.c (100%) rename src/xattr.h => xattr.h (100%) rename src/zdata.c => zdata.c (100%) rename src/zmap.c => zmap.c (100%) diff --git a/src/.clang-format b/.clang-format similarity index 100% rename from src/.clang-format rename to .clang-format diff --git a/.gitignore b/.gitignore index d7635e2..7a60a6c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,18 @@ -# Kernel-module build output. -/build/ -# Generated manual-test fixtures and captures. -/tests/results/manual/*/artifacts/ -/tests/results/manual/*/fixture/ -/tests/results/manual/*/repro-artifacts/ -/tests/results/manual/*/repro-fixture/ -/tests/results/manual/*/repro[0-9]-artifacts/ -/tests/results/manual/*/repro[0-9]-fixture/ +i386 +machine +x86 +.cache + +export_syms + +*.o +*.ko + +opt_global.h + +vnode_if.h +vnode_if_newproto.h +vnode_if_typedef.h + +compile_commands.json diff --git a/src/Makefile b/Makefile similarity index 100% rename from src/Makefile rename to Makefile diff --git a/README.md b/README.md deleted file mode 100644 index 19ec856..0000000 --- a/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# EROFS for FreeBSD - -## Build Kernel Module - -Build on FreeBSD 15 amd64 with a matching FreeBSD source tree. The default -source path is `/usr/src`: - -```sh -WITH_ZSTDIO=0 ./build.sh -``` - -Use `FREEBSD_SRC=/path/to/freebsd-src` when the source tree is elsewhere. The -script is a native FreeBSD wrapper around `src/Makefile` and `bsd.kmod.mk`; the -Makefile is the authoritative module name, source list, architecture gate, and -per-source flag definition. Each run rebuilds `build/obj` and writes -`build/erofs.ko`. - -Only `MACHINE_ARCH=amd64` is currently qualified. Other architectures are -rejected explicitly instead of inheriting amd64 ABI flags. Build ZSTD support -with: - -```sh -WITH_ZSTDIO=1 ./build.sh -``` - -The enabled module references the FreeBSD kernel ZSTD API and therefore -requires a running kernel built with `options ZSTDIO`. `WITH_ZSTDIO=0` builds a -module without those references and rejects ZSTD-compressed images at mount. - -## Mount - -EROFS is read-only. Mount a single-device image with: - -```sh -mount -t erofs -o ro /dev/md0 /mnt/erofs -``` - -For an image with external blob devices, map every one-based on-disk device -slot explicitly with `device.=`: - -```sh -mount -t erofs -o ro \ - -o device.2=/dev/md92 \ - -o device.1=/dev/md91 \ - /dev/md90 /mnt/erofs -``` - -Slot names make the mapping independent of option order. Every external slot -must be present exactly once; assigning the same GEOM provider to multiple -slots is rejected. The same names and values may be passed directly as -`nmount(2)` iovec entries. - -There is no repo-local `mount_erofs` binary; FreeBSD's generic `/sbin/mount` -frontend passes these distinct option names through to `nmount(2)`. The driver -forces every successful mount read-only, so even `-o rw` produces a read-only -mount rather than enabling writes. - -If an image has a device table but no `device.` options are supplied, -the primary provider is treated as a Linux-compatible flatdev image. It must -contain the external ranges at their declared `uniaddr` block offsets, for -example a deterministic concatenation of the primary image and its blobs. - -## Qualified Semantics - -- Compact/extended metadata, Linux device-number decode, plain/inline/chunk - data, and LZ4/MicroLZMA/DEFLATE/ZSTD compressed reads. -- Inline tails are confined to the inode metadata block and declared image or - metabox backing bounds. -- Directory lookup/readdir share strict validation while accepting Linux-style - nonzero unused bytes after the final name NUL. -- Compressed `st_blocks` reflects the inode's on-disk compressed block count; - uncompressed and chunk files retain logical block rounding. -- FreeBSD 15 local vnode pager sync/async entry points are used for real mmap - faults. -- NFS export uses full 64-bit NIDs and a generation derived from the - superblock seed and inode metadata. Replacing metadata changes the generation - and makes old handles stale; a metadata-identical, payload-only replacement - is not guaranteed to return `ESTALE`. - -See `docs/features.md` for the bounded feature claim and -`tests/results/manual/2026-08-09T1804Z-final-review-independent/manual-test-report.md` -for the final-review evidence. - -## Test Fixtures - -`tests/prepare_directory_fixtures.sh` creates the deterministic TC048/141/148 -directory fixtures. `tests/prepare_error_fixtures.sh` and -`tests/erofs_fixture.py` create and self-check the structured TC002/086/087/102 -and TC112-TC116/119 fixtures. Generated trees, images, overlays, and `build/` -outputs are test artifacts and are not committed. diff --git a/build.sh b/build.sh deleted file mode 100755 index 93fc764..0000000 --- a/build.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh -set -eu - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" -FREEBSD_SRC="${FREEBSD_SRC:-/usr/src}" -FS_SRC="${SCRIPT_DIR}/src" -BUILD_DIR="${SCRIPT_DIR}/build" -WORK_DIR="${BUILD_DIR}/obj" -MAKE="${MAKE:-make}" -WITH_ZSTDIO="${WITH_ZSTDIO:-0}" -NM_UNDEF="" - -cleanup() { - [ -z "${NM_UNDEF}" ] || rm -f "${NM_UNDEF}" -} - -trap cleanup EXIT -trap 'cleanup; exit 1' HUP INT TERM - -fail() { - echo "ERROR: $*" >&2 - exit 1 -} - -make_kmod() { - MAKEOBJDIR="${WORK_DIR}" "${MAKE}" -C "${FS_SRC}" \ - SRCTOP="${FREEBSD_SRC}" SYSDIR="${FREEBSD_SRC}/sys" \ - WITH_ZSTDIO="${WITH_ZSTDIO}" "$@" -} - -[ "$(uname -s)" = "FreeBSD" ] || - fail "build.sh requires a native FreeBSD build host" -[ -d "${FREEBSD_SRC}/sys" ] || - fail "FreeBSD source tree not found: ${FREEBSD_SRC}" -[ -d "${FS_SRC}" ] || fail "Source directory not found: ${FS_SRC}" -command -v "${MAKE}" >/dev/null 2>&1 || fail "make is required" -command -v awk >/dev/null 2>&1 || fail "awk is required" -command -v nm >/dev/null 2>&1 || fail "nm is required" -[ -f "${FREEBSD_SRC}/sys/tools/vnode_if.awk" ] || - fail "vnode_if.awk not found in ${FREEBSD_SRC}" -[ -f "${FREEBSD_SRC}/sys/kern/vnode_if.src" ] || - fail "vnode_if.src not found in ${FREEBSD_SRC}" -case "${WITH_ZSTDIO}" in -0|1) ;; -*) fail "WITH_ZSTDIO must be 0 or 1" ;; -esac - -rm -rf "${WORK_DIR}" -mkdir -p "${WORK_DIR}" -KMOD="$(make_kmod -V PROG)" -[ -n "${KMOD}" ] || fail "src/Makefile did not define a module output" - -echo "==> Building repo22 ${KMOD}" -make_kmod all - -[ -f "${WORK_DIR}/${KMOD}" ] || fail "module was not produced" - -NM_UNDEF="$(mktemp "${WORK_DIR}/nm-undef.XXXXXX")" || - fail "could not create nm output file" -if ! nm -u "${WORK_DIR}/${KMOD}" >"${NM_UNDEF}"; then - fail "nm failed while checking ${KMOD}" -fi -if ! awk '$NF == "bcmp" { found = 1 } END { exit found }' "${NM_UNDEF}"; then - fail "${KMOD} contains an unresolved bcmp reference" -fi - -cp -f "${WORK_DIR}/${KMOD}" "${BUILD_DIR}/" -echo "==> SUCCESS: ${BUILD_DIR}/${KMOD}" diff --git a/src/compress.h b/compress.h similarity index 100% rename from src/compress.h rename to compress.h diff --git a/current/README.md b/current/README.md deleted file mode 100644 index 057613c..0000000 --- a/current/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# repo22 当前状态报告索引 - -- [report-1](report-1/README.md)：第一次总体进度报告，保留 - `573aaab7ee0b47cb6aed7f76c52c68dd4041326b` 创建时的原始内容。 -- [report-2](report-2/README.md)：第二次总体进度报告，按 - `381349b3847209a67d669264ac54a50a3599915c` 快照汇总正式批次、动态结果、 - issue、风险和下一步。 -- [report-3](report-3/README.md)：第三次总体进度报告，以 - `12351cbb0a593474f4fcef68ffc5218d9c99dd46` 为最终验证快照，汇总完成项、 - 160 PASS/1 PARTIAL、独立审查结论、剩余搁置项和后续可选计划。 - -后续总体状态报告统一使用 `current/report-N/` 目录。逐项动态证据位于 -`tests/results/manual/`，问题触发、分析和解决状态位于 `issues/`。 diff --git a/current/report-1/2026-08-09-overall-progress.md b/current/report-1/2026-08-09-overall-progress.md deleted file mode 100644 index b8a3c79..0000000 --- a/current/report-1/2026-08-09-overall-progress.md +++ /dev/null @@ -1,160 +0,0 @@ -# repo22 当前总体进度 - -更新时间：2026-08-09 UTC - -状态报告代码基线：`cd0e985b5ac54a4b7acb7042422329ad1729fb3e` - -本报告记录的是上述基线上的阶段性状态。测试结果、问题状态和统计会随后续 -提交继续更新，不能将本文视为 TC001-TC156 已完成全量回归的声明。 - -## 目标与执行范围 - -repo22 的目标是在 FreeBSD 15 amd64 上提供只读 EROFS 内核模块，并以可复现 -fixture、真实 KLD、真实 mount/read/errno 和完整清理证据验证声明的功能边界。 - -当前执行约束如下： - -- 只修改和提交 `repo-community/repo22`，fixture、VM overlay、KLD 和原始日志仅 - 放在 `/work/build` 或 guest 临时目录。 -- 不建设或使用 CI、runner、自动 PASS wrapper；每个 TC 必须逐份对照对应 - Markdown 手工执行。 -- host 侧 `mkfs.erofs`、`dump.erofs`、`fsck.erofs` 和静态源码审查只作为 fixture - 或分析证据，不能替代 FreeBSD 内核动态结果。 -- 每批测试形成独立 manual report，记录 commit、构建配置、KLD SHA256、guest - 版本、命令、可观察输出或 errno、fixture hash 和清理状态。 -- 每批完成后提交并 push 到 `xdm/main`；并行批次若使远端前进，则 fetch、rebase - 自己的 test-only 提交后正常 push，禁止 force push。 - -## 已完成实现与审查 - -当前源码已覆盖 superblock、compact/extended inode、plain/inline/chunk 数据、 -目录/namecache、xattr/ACL/metabox、LZ4/MicroLZMA/DEFLATE/ZSTD、fragment、 -multi-device、NFS export、pager 和多项 fail-closed 边界检查。功能声明的准确边界 -以 `docs/features.md` 为准。 - -主要 feature 批次包括： - -- `29a215dd`：POSIX ACL 与完整 xattr namespace 集成。 -- `545d35bd`：xattr、long prefix、shared xattr 和 metabox 边界加固。 -- `96e22cc`：cold nested namei 与目录结构校验修复。 -- `4ef680df`、`78182686`：真实 multi-device、slot mapping、flatdev 和 extent - 映射审查修复。 -- `2354b448`：NFS export、稳定句柄和相关 vnode 路径。 -- `c208bf1f`：压缩映射 P0 收口，包括多算法、partial reference 和 extent 路径。 -- `9a3604fb`：metadata/VFS 语义收口，包括 inline bounds、special `rdev`、pager、 - namei、NFS generation 和真实压缩占用统计。 - -关键收尾与工程对齐提交： - -| Commit | 已完成内容 | -|---|---| -| `c881f656` | 修复 48-bit superblock union、`OFF_MAX`、大 hole 内存和大目录索引等最终 metadata 边界；建立 TC150-TC153 与三份未决 issue。 | -| `4d51ca9e` | 停止跟踪 `build/obj`、生成头和架构 symlink，避免构建产物进入正式提交。 | -| `89594551` | 将 KLD 构建布局、源文件命名、架构门控和 `WITH_ZSTDIO` 方式与 FreeBSD/Linux 上游职责对齐。 | -| `2f2ac5a8` | 更新 README、architecture、refactoring plan/status，使构建、BSD API 差异和功能边界与源码一致。 | -| `077b37ab` | 删除会伪造覆盖或无条件 PASS 的旧 runner/wrapper，加入字段断言的 fixture transformer 与 FreeBSD probe，修订关键 TC 文档。 | -| `cd0e985b` | 完成最终 review finding：per-inode NFS generation、显式 extent 物理地址溢出检查、compact/extended timestamp 校验，并直接动态执行 TC154-TC156。 | - -上述工作也包含多轮 code review 和 style review：源文件职责、函数顺序和磁盘 ABI -尽量贴近 Linux EROFS；FreeBSD vnode、pager、NFS、GEOM 和 `dev_t` 差异采用 -FreeBSD 15 原生接口；磁盘长度、加法、移位和 signed/unsigned 转换按 fail-closed -原则处理。 - -## 已有动态验证 - -`tests/results/manual/` 中保留了各 feature 在对应历史 commit 上的真实 FreeBSD 15 -动态证据。它们证明对应变更批次曾执行内核路径，但不能替代当前 final source 的 -TC001-TC156 全量重跑。 - -| 批次 | 主要动态证据 | 当前解释 | -|---|---|---| -| 压缩 | compact/full LZ4、legacy full index、partial reference、MicroLZMA、DEFLATE、ZSTD、fragment、ztailpacking 和多种边界 source/hash compare | 历史 feature 证据存在；G5/G6 尚需在本轮 exact KLD 重跑。 | -| xattr/ACL/metabox | inline/shared/long-prefix xattr、ACL、metabox carrier 和 corruption errno | 历史 feature 证据存在；G4 尚未启动。 | -| directory/namei/pager | cold nested lookup、目录 padding、special `rdev`、真实 mmap/page fault 和 inline bounds | 历史 feature 证据存在；G1/G3 将按当前文档重跑。 | -| multi-device | 外部 provider、显式 slot、flatdev、unified address、fragment 和 orphan/error 路径 | 历史 feature 证据存在；G7 尚未启动。 | -| NFS | getfh/fhopen、同镜像重挂、同 md 替换、ESTALE、background/stress cleanup | 历史 feature 证据存在；G8 尚未启动。 | -| final review | TC150-TC152 动态 PASS；TC153 明确 SHELVED | 结果来自 `c881f656` 前后的 final-review 批次，不是 156 项全量 PASS。 | - -当前 final source `cd0e985b` 的直接动态证据是 -`tests/results/manual/2026-08-09T0557Z-review-fixes/manual-test-report.md`： - -- FreeBSD 15.0-RELEASE-p8 amd64 上 `WITH_ZSTDIO=0` KLD SHA256 为 - `d6e755c7a75dd5043b603d58215ae3e7c8f29318b31dc1885aa9bb070d6e57a5`。 -- `WITH_ZSTDIO=1` KLD SHA256 为 - `b442f9f05af1d1a76801690c83c9a40ec6fc739adb0663e7dacaf837eeb2ecf3`。 -- TC154、TC155、TC156 均在该 final source 上直接动态 PASS，分别覆盖 per-inode - NFS generation、explicit extent `pa + plen` overflow 和 inode timestamp - validation。 -- 该批结束时 EROFS mount、KLD 和 md provider 均已清理。 - -## 本轮八组全量回归 - -TC 编号范围为 TC001-TC156，共 156 项。以下集合经计数检查为总数 156、重复 0、 -遗漏 0。 - -| 组 | 精确集合 | 数量 | 当前状态 | -|---|---|---:|---| -| G1 | TC001, TC007, TC009, TC011-TC014, TC019-TC040, TC147, TC150, TC151 | 32 | 已启动；正在修订文档和确定性 fixture，尚未形成 final-source 32 项结论。 | -| G2 | TC002, TC008, TC010, TC015-TC018, TC112-TC116, TC119 | 13 | 已启动；使用独立 worktree/VM，尚未形成 13 项结论。 | -| G3 | TC041-TC066, TC141, TC148, TC149, TC153 | 30 | 未启动。 | -| G4 | TC005, TC067-TC083, TC117, TC134-TC140, TC142 | 27 | 未启动。 | -| G5 | TC003, TC084-TC092 | 10 | 未启动。 | -| G6 | TC004, TC102-TC110, TC143-TC146, TC155 | 15 | 未启动。 | -| G7 | TC006, TC093-TC101, TC118 | 11 | 未启动。 | -| G8 | TC111, TC120-TC133, TC152, TC154, TC156 | 18 | 未启动。 | - -G1/G2 的“已启动”只表示执行代理、工作目录和任务集合已经建立，不表示其中任何 -TC 已在本轮计为 PASS。G8 中 TC154/TC156、G6 中 TC155 虽已有 `cd0e985b` 直接 -证据，仍需由对应全量组在最终统计中明确引用或按组要求重跑，不能导致重复计数。 - -## 已完成与未完成 - -已完成： - -- 核心 feature 实现、针对性 code review、FreeBSD 15 API 对齐和构建布局收口。 -- 多个历史 feature 批次的真实 KLD 动态验证与清理记录。 -- 当前 final source 的双配置构建和 TC154-TC156 直接动态验证。 -- 移除旧 CI 风格 runner/wrapper，建立“逐 Markdown 手工执行”的验收规则。 -- 三项已知验证缺口均有详细 issue，没有把未执行或环境受限结果标成 PASS。 - -未完成： - -- TC001-TC156 在同一 final-source 基线族上的八组全量手工回归与最终汇总。 -- G1/G2 的文档修订、fixture 固化、exact KLD 执行、逐 TC 报告和 push。 -- G3-G8 的启动、独立环境分配、执行和结果提交。 -- TC010 所需 16 TiB 级合格 provider、TC153 大目录内核路径、explicit extent - mapped compressed payload 的正向 fixture。 - -## Issue 索引 - -检查时 `issues/` 只有以下三份已跟踪文档，没有未提交 issue 文件或真实 issue -改动，因此本状态提交不修改 `issues/`。 - -| Issue | 状态 | 影响 feature | Shelved 原因 | -|---|---|---|---| -| `issues/TC010-48bit-statfs-large-provider.md` | SHELVED - environment unavailable | 48-bit block count、primary provider size validation、`statfs(2)`/`df` 总量和 root/block union 选择 | 非零 `blocks_hi` 在 4 KiB block 下至少要求 16 TiB GEOM provider；当前 vnode md 流程没有已验证、可安全清理的合格 provider。小 provider 拒绝和 TC150 fallback 都不能替代正向结果。 | -| `issues/TC153-large-directory-block-index-validation.md` | SHELVED - kernel validation incomplete | 超大目录二分索引、cold lookup、`EINTEGRITY` 传播和 negative namecache 正确性 | 已有可复现 FLAT_PLAIN fixture，但尚未在 FreeBSD fixed/pre-fix KLD 上完成冷查找对照；host 生成和静态审查不足以计 PASS。 | -| `issues/extent-metadata-fixture-unavailable.md` | SHELVED - positive fixture unavailable | explicit compressed extent parser、4/8/16/32-byte record、物理/逻辑高位、binary search、partial reference/fragment 映射 | erofs-utils 1.8.6 和已试 1.9.3 均不生成真实 mapped explicit-record payload。TC152 hole 与 TC155 overflow 只覆盖部分 header/negative 分支，不能替代正向压缩数据读取。 | - -## 风险与依赖 - -- 并行组共享远端 `xdm/main`，每批 push 前必须重新 fetch/rebase 并确认没有覆盖其他 - 组的 TC、helper、report 或 issue。 -- FreeBSD VM、SSH 端口、qcow overlay、md unit 和 mount point 必须按组隔离；任何 - guest 残留都会污染后续动态证据。 -- erofs-utils 的格式生成能力限制 explicit extent 等正向 fixture，必要时只能使用 - 字段自检、CRC32C-aware 的 structured transformer，不能手工猜偏移。 -- 48-bit 大 provider、memory pressure、NFS stress 和性能 TC 对环境容量及稳定性要求 - 较高，环境不足时必须使用 `SHELVED/ISSUE` 或 `ENVIRONMENT-UNAVAILABLE`，不能 - 降低判据。 -- 历史报告跨多个 commit。最终统计必须只接受对应 exact KLD 的本轮动态结果，或 - 明确标注尚未重跑，禁止把旧 HEAD 结果直接升级为全量 PASS。 - -## 下一步 - -1. 完成 G1/G2 文档和 fixture 修订，构建各自 exact KLD，逐 TC 手工执行并提交报告。 -2. 每个先完成的组 fetch/rebase 最新 `xdm/main`，严格 pathspec 提交并正常 push。 -3. 启动 G3/G4，随后按环境和依赖启动 G5-G8；压缩、multi-device、NFS 使用隔离 VM。 -4. 对任何 kernel behavior failure 立即停止 PASS 计数，新增或更新有事实依据的 issue。 -5. 八组结束后汇总 156 项，检查总数、重复、遗漏、KLD/image/source hash 和全部清理 - 证据，再发布 final-source 全量结论。 diff --git a/current/report-1/README.md b/current/report-1/README.md deleted file mode 100644 index aea19af..0000000 --- a/current/report-1/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# repo22 当前状态索引 - -- [2026-08-09 总体进度](2026-08-09-overall-progress.md)：基线 - `cd0e985b5ac54a4b7acb7042422329ad1729fb3e` 上的实现、历史动态证据、八组 - 全量回归分工、未决 issue、风险和下一步。 - -此目录记录阶段性状态；测试结果会随后续 commit 更新。正式逐项证据位于 -`tests/results/manual/`，未决验证缺口位于 `issues/`。 diff --git a/current/report-2/2026-08-09-overall-progress.md b/current/report-2/2026-08-09-overall-progress.md deleted file mode 100644 index 3cc51a4..0000000 --- a/current/report-2/2026-08-09-overall-progress.md +++ /dev/null @@ -1,123 +0,0 @@ -# repo22 第二次总体进度报告 - -更新时间：2026-08-09 UTC - -权威统计快照：`381349b3847209a67d669264ac54a50a3599915c` -（该提交是当次 `xdm/main` 上完成 G3 报告后的快照）。 - -本报告只统计该快照已经提交的实现、正式批次和动态证据。TC060 源码修复与 G5 -压缩回归正在并行进行，但在形成完整报告并提交前不提前计为 PASS，也不改变下述 -冻结统计。 - -## 总体结论 - -- feature 实现、源码 code review、style review、FreeBSD 15 API 对齐和双配置构建 - 布局已经完成阶段性收口。 -- 正式八组回归中，G1、G2、G3、G4 共执行 103 项：102 PASS，1 项 - `TC060 KERNEL-FAIL` 正在修复。 -- final review-fix 阶段还直接动态执行了 TC154-TC156，三项均 PASS。它们是三项 - 额外且不与前述 103 项重叠的 TC，因此当前共有 106 个不同 TC 具备动态结果， - 其中 105 PASS、1 项待修。 -- TC154-TC156 已分配给 G8，但先行 PASS 不能重复计入“已完成 G8”，也不能把 G8 - 标成已执行完毕。G8 尚有 TC111、TC131-TC133 未执行。 -- TC010 与 TC153 已通过真实大 sparse provider 的 FreeBSD 内核路径验证并转为 - RESOLVED。explicit extent 的 mapped compressed payload 正向 fixture 仍未获得， - 对应 issue 尚未解决。 - -## 实现与审查 - -repo22 已完成以下阶段性工程工作： - -- 实现并审查 superblock、compact/extended inode、plain/inline/chunk 数据、目录与 - namecache、xattr/ACL/metabox、多算法压缩、fragment、multi-device、NFS export、 - vnode pager 和 fail-closed 元数据边界。 -- 对照 Linux EROFS 的磁盘 ABI、职责拆分和命名以降低长期维护偏差，同时保持 - FreeBSD vnode、GEOM、pager、NFS、ACL 和 `dev_t` 语义为行为权威。 -- 完成 overflow、长度、偏移、移位、signed/unsigned、目录大索引、48-bit union、 - NFS generation、special vnode 和 timestamp 等多轮独立 code review 修复。 -- 清理被跟踪的构建产物，统一 FreeBSD KLD 源文件布局、amd64 门控和 - `WITH_ZSTDIO=0/1` 构建方式；当前构建仍以 kernel `-Werror` 为门槛。 -- 删除会替代人工判定或无条件给出 PASS 的旧 CI 风格 runner/wrapper，保留可复现 - fixture transformer 与原生 syscall/probe 工具。当前任务不实现 CI。 - -## 正式八组状态 - -八组精确集合总计 156 项，彼此无重复、无遗漏。 - -| 组 | 精确集合 | 数量 | 快照状态 | -|---|---|---:|---| -| G1 | TC001, TC007, TC009, TC011-TC014, TC019-TC040, TC147, TC150, TC151 | 32 | 32 PASS，正式批次完成。 | -| G2 | TC002, TC008, TC010, TC015-TC018, TC112-TC116, TC119 | 13 | 13 PASS，正式批次完成。 | -| G3 | TC041-TC066, TC141, TC148, TC149, TC152, TC153 | 31 | 31 已执行；30 PASS，TC060 KERNEL-FAIL。 | -| G4 | TC005, TC067-TC083, TC117, TC134-TC140, TC142 | 27 | 27 PASS，正式批次完成。 | -| G5 | TC003, TC004, TC084-TC092, TC102-TC110, TC143-TC146 | 24 | 压缩回归正在进行，尚无正式整组结论。 | -| G6 | TC006, TC093-TC101, TC118 | 11 | 尚未执行。 | -| G7 | TC120-TC130 | 11 | 尚未执行。 | -| G8 | TC111, TC131-TC133, TC154-TC156 | 7 | 正式整组未完成；TC154-TC156 已先行 PASS，TC111、TC131-TC133 尚未执行。 | - -正式批次证据： - -- [G1 报告](../../tests/results/manual/2026-08-09T0710Z-g1/manual-test-report.md) -- [G2 报告](../../tests/results/manual/2026-08-09T0710Z-g2/manual-test-report.md) -- [G3 报告](../../tests/results/manual/2026-08-09T1059Z-g3/manual-test-report.md) -- [G4 报告](../../tests/results/manual/2026-08-09T0839Z-g4/manual-test-report.md) -- [review-fix 报告](../../tests/results/manual/2026-08-09T0557Z-review-fixes/manual-test-report.md) - -## 统计口径 - -正式批次已执行数为 `32 + 13 + 31 + 27 = 103`；PASS 数为 -`32 + 13 + 30 + 27 = 102`，另有 TC060 一项 KERNEL-FAIL。 - -TC154-TC156 来自 review-fix 阶段的 exact-source FreeBSD 动态运行，并未包含在 -G1-G4 的 103 项中。因此按“不同 TC 是否已有动态结果”计数，应增加三项，得到 -106 个 TC 有动态结果、105 PASS、1 待修。按“正式组是否完成”计数时，三项仍属于 -尚未完成的 G8，不能再次增加 PASS 数或宣称 G8 已完成。 - -## 大 Provider 验证 - -- TC010 使用逻辑长度 `17592193667072` 字节的 qualified sparse vnode provider， - 动态挂载了 `blocks_hi=1` 的 48-bit EROFS，并验证 `df` 的 64-bit 总量无截断。 - issue 已 RESOLVED。 -- TC153 使用逻辑长度 `8796093091840` 字节的 sparse GEOM provider，令超大 - FLAT_PLAIN 目录进入 `2147483648` 最终块索引的真实 kernel lookup 路径；固定 KLD - 在 cold lookup 和 readdir 后均返回 `EINTEGRITY`，没有错误缓存为 `ENOENT`。 - issue 已 RESOLVED。 -- explicit extent 已动态覆盖 hole record、header 选择和部分负向分支，但仍缺少 - mapped compressed payload 以及完整 record variant 的正向读取证据，不能关闭 issue。 - -## Issue 索引 - -| Issue | 当前状态 | 结论 | -|---|---|---| -| [TC060 standard pathconf](../../issues/TC060-pathconf-standard-values.md) | OPEN - KERNEL-FAIL | `_PC_NO_TRUNC` 与 `_PC_CHOWN_RESTRICTED` 返回 `EINVAL`；源码修复和复测正在进行。 | -| [TC010 48-bit statfs](../../issues/TC010-48bit-statfs-large-provider.md) | RESOLVED - qualified sparse vnode provider validated | qualified 16 TiB 级 sparse provider 已完成真实 mount/statfs/df 验证。 | -| [TC153 large directory index](../../issues/TC153-large-directory-block-index-validation.md) | RESOLVED - exact-source kernel validation passed | qualified 多 TiB sparse provider 已完成 exact-source 大索引 kernel lookup 验证。 | -| [Explicit extent fixture](../../issues/extent-metadata-fixture-unavailable.md) | SHELVED - positive fixture unavailable | issue 未解决；尚无使用 explicit-record parser 的 mapped compressed payload 正向 fixture。 | - -## 当前风险 - -- TC060 是已确认的 FreeBSD kernel behavior failure，修复必须保持 NAME_MAX、PATH_MAX、 - FILESIZEBITS、LINK_MAX、ACL 查询和未知键 errno 不回归，并完成 mount/md/KLD 清理。 -- G5 同时覆盖 LZ4、MicroLZMA、DEFLATE、ZSTD、ztailpacking、partial reference、 - explicit metadata 等多条压缩路径，fixture 资格和 `WITH_ZSTDIO` 配置差异仍是当前 - 最大执行风险。 -- explicit extent 缺少上游工具可生成的 mapped positive fixture；静态 ABI 对照、 - hole record 和 malformed case 不能替代真实压缩 payload 读取。 -- G6 multi-device、G7 边界与压力类测试、G8 NFS/manual 页面仍需隔离 VM、精确 KLD、 - direct syscall/数据比较和完整清理，历史 feature 报告不能自动升级为本轮结果。 -- 并行 worktree 共享 `xdm/main`。每次 push 前必须 fetch/rebase，严格 pathspec，禁止 - force push；guest mount、md、KLD 和临时 provider 残留必须清零。 - -## 下一步 - -1. 完成 TC060 根因修复，重跑全部 pathconf 键与基础 mount/read smoke，更新 issue - 和 dated manual report。 -2. 完成 G5 的 24 项压缩回归，提交 exact KLD、fixture hash、errno/数据比较与清理 - 证据。 -3. 依次完成 G6 11 项、G7 11 项和 G8 剩余 TC111、TC131-TC133；在 G8 汇总中引用 - 或按要求复跑 TC154-TC156，但禁止重复计数。 -4. 聚合 `TEST-COVERAGE-MATRIX`，审计 156 项总数、组间重复、遗漏、动态证据来源和 - 未解决 issue。 -5. 完成最终独立 source review，检查 lock/resource/error path；执行 - `WITH_ZSTDIO=0/1` 双配置 exact-source build，并做 `xdm/main`、remote/HEAD、 - tracked clean 和 guest 清理的最终 audit。 diff --git a/current/report-2/README.md b/current/report-2/README.md deleted file mode 100644 index d9ac135..0000000 --- a/current/report-2/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# repo22 第二次状态报告索引 - -- [第二次总体进度报告](2026-08-09-overall-progress.md)：以 - `381349b3847209a67d669264ac54a50a3599915c` 为统计快照，汇总实现与审查、 - 八组回归、动态结果计数、issue、风险和后续工作。 - -本目录只记录阶段性总体状态。各 TC 的命令、hash、errno、dmesg 和清理证据以 -`tests/results/manual/` 中的对应报告为准。 diff --git a/current/report-3/2026-08-09-overall-progress.md b/current/report-3/2026-08-09-overall-progress.md deleted file mode 100644 index c7ee7d6..0000000 --- a/current/report-3/2026-08-09-overall-progress.md +++ /dev/null @@ -1,124 +0,0 @@ -# repo22 第三次总体进度报告 - -更新时间：2026-08-09 UTC - -权威验证快照：`12351cbb0a593474f4fcef68ffc5218d9c99dd46` - -最终源码基线：`fcc85b93d5f8fd9671686bd68bf3b086c8bd25cf` - -## 总体结论 - -本轮要求的 feature 全面验证、FreeBSD/Linux 行为差异审查、code review 和 -style/可维护性优化已经完成。161 个可执行 Markdown 测试均已执行并形成受控 -手工证据，最终为 160 PASS、1 PARTIAL、0 FAIL、0 KERNEL-FAIL、0 ENV。 - -唯一未完全关闭的是 TC146 的 explicit compressed extent mapped-payload 正向 -fixture。该问题已按用户授权详细记录并搁置，不是已观察到的 kernel failure， -也不影响其他 160 个 TC 的结论。本轮目标因此完成，剩余工作属于已批准搁置项 -或后续可选增强。 - -CI 和自动化 kernel-test harness 不在本轮范围内，未实现也未宣称完成。 - -## 已完成内容 - -- 完成 TC001-TC161 的规格、执行报告和覆盖矩阵机械审计；TC000 只作为模板。 -- G1-G8 的 156 个表格行覆盖 TC001-TC156，恰好一次，无重复、无遗漏。 -- 完成 TC157-TC161 独立回归，覆盖 explicit extent 全局顺序、48-bit compressed - block count、special vnode 组合 setattr、dot-omitted `OFF_MAX` cookie 和 `nm` - 失败传播。 -- TC010 的 16 TiB-plus statfs、TC060 的 FreeBSD pathconf、TC153 的超大目录索引 - 均已动态验证并转为 RESOLVED。 -- 完成 superblock、inode、data、directory/namecache、xattr/ACL/metabox、压缩、 - multi-device、NFS、pager、错误路径和边界算术的多轮独立 review。 -- 保留 Linux EROFS 的磁盘 ABI、命名、函数顺序和职责拆分作为维护参照，同时以 - FreeBSD vnode、GEOM、pager、NFS、ACL、`dev_t` 和 errno 语义为行为权威。 -- 修正 README 的 NFS stale 承诺：generation 来自 superblock seed 和 inode - metadata；metadata-identical payload-only replacement 不保证 `ESTALE`。 -- 清理 repo22 内全部 ignored build/object/image/repro/Python bytecode 产物；最终 - guest 的 EROFS mount、md、EROFS KLD 和 DTrace KLD 计数均为零。 - -## 最终验证统计 - -| 状态 | 数量 | 说明 | -| --- | ---: | --- | -| PASS | 160 | 具备对应 dated manual report 的 FreeBSD 15 手工证据 | -| PARTIAL | 1 | TC146；HEAD2/interlaced PASS，explicit mapped payload 未完成 | -| FAIL / KERNEL-FAIL | 0 | 无开放 kernel 行为失败 | -| ENV | 0 | 无环境阻塞 | -| SHELVED test case | 0 | TC146 的子项不另计为 TC | - -G1-G8 证据来自多个源码提交，不能表述为所有旧 TC 都运行在最终 KLD 上。最终 -`fcc85b93d` 基线接受了 TC157-TC161、相关回归以及双配置 build/load/mount smoke。 - -## 最终构建 - -FreeBSD 15 kernel `-Werror` 双配置均成功： - -| 配置 | KLD SHA256 | -| --- | --- | -| `WITH_ZSTDIO=0` | `15fda9d334132cd81769ce4dff4f8411a6e2b4cf7531c352530d0de85f42a2d2` | -| `WITH_ZSTDIO=1` | `23782dc0ce7da188807d35020bf2d8c6044b796c5c9a8746b398ba784cd6ad4e` | - -TC161 的私有 `nm` shim 正确使构建失败且不发布新 KLD；移除 shim 后使用 -`/usr/bin/nm` 的正常重建 exit 0。两个最终 KLD 均完成独立加载和只读挂载 smoke。 - -## Code Review 结论 - -最终独立复核未发现 P0/P1。已修复的最后一批问题包括： - -- 16/32-byte explicit extent table 全局严格顺序验证； -- extended compressed inode 的完整 48-bit `blocks_hi` 解码； -- special vnode size 与其他 setattr 字段组合时的 false success； -- dot-omitted 目录在 `OFF_MAX` 的 cookie/offset overflow； -- `build.sh` 对 `nm` 自身失败的可靠传播； -- NFS generation 文档与真实实现边界不一致。 - -从社区第三方维护者视角，未发现仍需消除的非必要 Linux/FreeBSD 结构、命名或 -职责差异。保留的差异均来自 FreeBSD kernel API、锁、provider、pager、NFS 或 -权限模型要求。 - -## 未完成与搁置 - -唯一批准搁置项是 -[explicit mapped extent payload](../../issues/extent-metadata-fixture-unavailable.md)： - -- erofs-utils 1.8.6 不能生成或独立验证该新格式； -- 已尝试工具能力扫描、`--max-extent-bytes`、structured conversion、ABI/control - flow 对照以及多类负向 fixture； -- TC157 已证明 16/32-byte unordered table fail closed，但不能替代 mapped payload - 的 source-identical 正向读取； -- 首个非零 `lstart` 和极大 extent-count 性能边界也保留为可选覆盖风险。 - -## 后续执行计划 - -1. 等待可生成并独立验证 explicit mapped-payload 的上游工具或可信 fixture。 -2. 获得 fixture 后补充 4/8/16/32-byte 正向 record、partial reference、high words、 - binary-search transition 和 final fragment 动态覆盖，使 TC146 从 PARTIAL 转 PASS。 -3. 如后续项目另行启动 CI，再基于现有 Markdown 断言设计自动化；不得把当前手工 - PASS 记录直接转换成无条件自动 PASS。 -4. 后续源码变更继续执行受影响 TC、双配置 build、module smoke、scoped Git hygiene - 和独立 review，不要求无差别重跑与改动无关的全部历史环境。 - -## 相关提交 - -正式八组手工验证： - -- G1 `9ae22009f23a65320730072a780998e80aa9b728` -- G2 `aed7b68b79a5bd4115913f8361821c86fd58d2e2` -- G3 `381349b3847209a67d669264ac54a50a3599915c` -- G4 `f383bbbbff301a6bde18894f03ab88a8c0cc885a` -- G5 `c566d8ac6bf8e801082bbccf108451f6ee46ad40` -- G6 `f11fff5b8e8050e1017ed86f0bcf71042b2b45aa` -- G7 `f3ab2096fea3942317deb83fa051c84d3c124ec2` -- G8 `6f336d0a7387c8f110b19abad6e96f3ea17dba2e` - -最终源码与验证收口： - -- `f27b524a35b1c960be7e81884f9b928d9ac907e9`：metadata boundary 修复。 -- `67d3c057fbcad5adc765ab9927da511e3221ac8f`：explicit extent header 解码修复。 -- `fcc85b93d5f8fd9671686bd68bf3b086c8bd25cf`：TC157-TC161 规格与 helper。 -- `12351cbb0a593474f4fcef68ffc5218d9c99dd46`：最终手工报告、覆盖、issue 和 README 汇总。 - -逐项证据见 -`tests/results/manual/2026-08-09T1804Z-final-review-independent/manual-test-report.md`； -最终统计口径见 `docs/TEST_REPORT.md` 和 `tests/TEST-COVERAGE-MATRIX.md`。 diff --git a/current/report-3/README.md b/current/report-3/README.md deleted file mode 100644 index 0bc546d..0000000 --- a/current/report-3/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# repo22 第三次状态报告索引 - -- [第三次总体进度报告](2026-08-09-overall-progress.md)：以 - `12351cbb0a593474f4fcef68ffc5218d9c99dd46` 为最终验证快照，汇总已完成 - feature 验证、code/style review、最终构建、剩余风险和后续计划。 - -本目录只记录本轮收尾状态。逐项命令、hash、errno 和清理证据以 -`tests/results/manual/` 为准，issue 的触发、分析、尝试和验收条件以 -`issues/` 为准。report-1 和 report-2 保留为历史快照，不因本报告改写。 diff --git a/current/report-4/2026-08-10-task2-code-review-style-assessment.md b/current/report-4/2026-08-10-task2-code-review-style-assessment.md deleted file mode 100644 index 344512b..0000000 --- a/current/report-4/2026-08-10-task2-code-review-style-assessment.md +++ /dev/null @@ -1,726 +0,0 @@ -# repo22 任务 2：全面代码审查与风格优化评估 - -日期：2026-08-10 - -性质：只读评估，不含源码修改、测试实现或 CI 变更 - -范围：原任务中的“2. 进行全面的 code review 和 style 优化” - -## 1. 执行结论 - -本轮以第三方社区维护者视角，对 repo22 的 FreeBSD EROFS 实现进行了新的严格 -静态审查。审查逐条回应原任务 2 的三个目标： - -1. 判断实现是否合理复用 FreeBSD kernel 已有能力，是否存在不必要的重写，或 - 反过来存在不应强行引用内核实现的场景。 -2. 判断 BSD 与 Linux 的行为差异是否被正确识别，避免把 Linux 共同缺陷误记为 - BSD 移植差异，也避免为追求外观一致而破坏 FreeBSD VFS、GEOM 和 errno 契约。 -3. 判断两个独立仓库在文件职责、函数和变量命名、定义顺序、磁盘 ABI、注释和 - style 上是否已经消除非必要差异。 - -结论是：repo22 的 FreeBSD API 集成和核心算法映射已有良好基础，但任务 2 尚不 -能认定为全面完成。本轮确认： - -| 等级 | 数量 | 结论 | -| --- | ---: | --- | -| P0 | 0 | 未发现当前静态证据可确认的立即阻断问题 | -| P1 | 2 | vnode 生命周期 UAF；未压缩 physical run/chunk 大读的巨型分配与 `uiomove()` 长度截断 | -| P2 | 4 | 目录排序、根类型、compact 加法回绕、explicit extent 首次全表扫描 | -| P3 | 6 | errno、时间、symlink、48-bit 边界、setattr、FRAGMENTS 注释/死分支 | - -这些新 finding 均未在本轮动态复现，也未修复。本报告给出的测试均是后续建议， -不是已执行结果。CI 明确不在本轮范围。 - -任务 1 已完成的 feature 验证和手工测试记录不在本报告中重写。静态审查发现的 -新风险也不应被误读为已推翻全部既有功能证据；它们需要后续按优先级复现、修复 -并回归。 - -## 2. 审计方法与限制 - -### 2.1 方法 - -本轮采用以下互相独立的证据层： - -- **源码静态比对**：逐文件映射 repo22 与 Linux 7.1.0-rc1 `fs/erofs`，比较磁盘 - ABI、核心算法、函数职责、函数顺序和命名。 -- **FreeBSD 契约核对**：直接检查 FreeBSD 15.0-RELEASE-p9 中 vnode、VFS、GEOM、 - `uio`、pager、ACL、extattr、XZ、ZSTD 和参考文件系统实现。 -- **历史检查**：确认 repo22 最终源码提交、Linux 参考导入提交、tree hash、历史 - 测试入口和 UDF helper 的可追溯程度。 -- **机械检查**：执行 FreeBSD `checkstyle9.pl`，检查文件树、共同编译单元顺序、 - 死声明、硬编码旧目录和许可证标识。 -- **独立复核**：对高风险 vnode 生命周期、目录排序、根类型、只读 `setattr`、 - 许可证措辞、UDF 来源措辞、旧测试入口和 style 统计进行了二次静态复核。 - -### 2.2 限制 - -- 本轮没有启动新的 FreeBSD 动态测试批次。 -- 新 finding 尚未通过 fault injection、损坏镜像、并发卸载或大 I/O 实测。 -- 性能风险只根据分配和扫描路径分析，未给出吞吐、延迟或内存峰值数据。 -- 许可证部分只陈述仓库可观察事实，不作法律结论。 -- Linux 与 FreeBSD 是独立仓库；本报告只要求维护者可识别的相似性，不要求共享 - Git 历史、构建系统或平台抽象。 -- `checkstyle9.pl` 自身标记部分规则为 experimental，因此输出是审查输入，不能 - 自动等同为每一项都必须修改。 - -## 3. 可复现基线 - -### 3.1 BSD 目标 - -- repo22 HEAD：`76cfb553e0951b428f9d426933a10d1fe18d2a0e` -- 最终源码变更提交：`67d3c057fbcad5adc765ab9927da511e3221ac8f` -- 当前 `src/` tree：`a605af0f01f83d67c199005e775e1a47a1610036` -- 从 `67d3c057f` 到本报告基线 HEAD，`src/` tree 未再变化。 - -### 3.2 Linux 独立参考 - -- 精简参考：`/work/dev-src-linux/fs/erofs` -- 版本：Linux 7.1.0-rc1 -- 本地导入提交：`8be2be573d2b191c83d760b65c554f78eb95893c` -- EROFS tree：`b79b9a8b62f633e887a8b99075ff9412fabd7f83` -- `/work/linux-src/fs/erofs` 与精简参考内容及 tree hash 相同。 - -该 Linux 树只作为独立源码参考，不要求与 repo22 共仓。当前 `/work` 开发环境内 -可以用提交和 tree hash 验证其身份；只获得独立 repo22 的社区维护者，则无法从 -现有 repo22 文档唯一重建该基线，详见第 10 节。 - -### 3.3 FreeBSD API 参考 - -- 路径：`/work/dev-freebsd-releng` -- 版本：FreeBSD 15.0-RELEASE-p9 -- 重点契约：`insmntque()`、`uiomove()`、`vfs_read_dirent()`、GEOM VFS、vnode pager、 - POSIX.1e ACL、extattr、XZ Embedded、ZSTDIO 和参考文件系统。 - -### 3.4 提交卫生 - -创建 report-4 前，repo22 scoped tracked、staged、untracked 状态均为 0，且 -`HEAD == xdm/main`。本次提交范围限定为 `current/report-4/`，不会修改既有源码、 -测试、文档、issues、report-1 至 report-3 或 `current/README.md`。 - -## 4. 文件职责映射 - -### 4.1 共同或语义对应文件 - -| repo22 | Linux `fs/erofs` | 维护者视角结论 | -| --- | --- | --- | -| `src/erofs_fs.h` | `erofs_fs.h` | 磁盘 ABI 应最严格对齐；字段布局正确性优先于平台风格 | -| `src/internal.h` | `internal.h`, `compress.h` | 内存结构和接口语义对应，FreeBSD 类型必然不同 | -| `src/super.c` | `super.c` | superblock、设备表、mount、statfs、生命周期 | -| `src/inode.c` | `inode.c` | inode 解码；Linux inode ops 由 BSD vnops 分担 | -| `src/data.c` | `data.c` | block/chunk mapping 对应；folio/iomap 改为 buffer/GEOM/uio | -| `src/dir.c` | `dir.c` | 目录结构解析对应；cookie 和 dirent 输出是 BSD 适配 | -| `src/namei.c` | `namei.c` | 两级二分搜索语义高度对应；namecache/锁为 BSD 适配 | -| `src/xattr.c` | `xattr.c` | 磁盘 xattr 解析对应；namespace、ACL、extattr API 不同 | -| `src/xattr.h` | `xattr.h` | 只应保留 FreeBSD 实际接口，当前仍有 Linux 残片 | -| `src/decompressor.c` | `decompressor.c` | dispatcher 对应；配置职责仍有非必要差异 | -| `src/decompressor_lzma.c` | `decompressor_lzma.c` | 算法对应；FreeBSD 侧私有嵌入 XZ Embedded | -| `src/decompressor_deflate.c` | `decompressor_deflate.c` | 直接调用 FreeBSD zlib，方向正确 | -| `src/decompressor_zstd.c` | `decompressor_zstd.c` | 使用内核 ZSTDIO，方向正确 | -| `src/zmap.c` | `zmap.c` | 压缩映射核心最接近 Linux，函数顺序对齐质量高 | -| `src/zdata.c` | `zdata.c` | 职责对应；同步读取与 Linux folio/pcluster pipeline 必然不同 | -| `src/Makefile` | `Makefile`, `Kconfig` | 只比较编译单元职责和顺序，不比较构建语法 | - -### 4.2 Linux-only 文件，不应为了外观模拟 - -| Linux-only 文件 | 不在 repo22 创建空文件的理由 | -| --- | --- | -| `sysfs.c` | FreeBSD 无 Linux sysfs 模型，mount/sysctl 语义需独立设计 | -| `fileio.c` | Linux file-backed I/O 路径与当前 GEOM provider 模型不同 | -| `fscache.c` | Linux fscache/netfs 子系统在 FreeBSD 无直接对应 | -| `ishare.c` | Linux inode/xattr sharing 生命周期依赖其 VFS 内部模型 | -| `zutil.c` | Linux folio、bio、pcluster 工具层不适用于同步 BSD 读取路径 | -| `decompressor_crypto.c` | Linux Crypto API 后端在当前 FreeBSD 目标中无对应承诺 | -| `compress.h` | Linux 压缩调度、folio 和 stream 数据结构不能机械复制 | - -缺少这些文件是必要平台差异，不是结构欠缺。为了目录看起来相同而新增空壳，会 -制造虚假维护入口。 - -### 4.3 BSD-only 文件 - -| BSD-only 文件 | 必要性结论 | -| --- | --- | -| `src/erofs_vnops.c` | 必要。集中实现 VOP、pager、ACL、NFS file handle 和只读语义 | -| `src/lz4.c` | 必要。FreeBSD/OpenZFS LZ4 接口不匹配 EROFS raw block 和 partial 输出 | -| `src/erofs_defs.h` | 文件本身可存在，但当前含部分仅定义未使用的宏，必要性需缩减 | - -## 5. Findings First - -以下只列入已经过独立静态复核、或任务指定为已复核的结论。每项明确其类别， -避免把 Linux 共同问题写成 BSD 行为差异。 - -### 5.1 P0 - -无。 - -### 5.2 P1-1：`insmntque()` 失败后继续访问已回收 vnode - -- **位置**：`src/inode.c:452` 调用 `insmntque()`；失败分支在 - `src/inode.c:453-457` 释放 `en` 后又于 `src/inode.c:455` 写 - `vp->v_data = NULL`。 -- **FreeBSD 对照**：`sys/kern/vfs_subr.c:2303-2315` 在插入失败时清空 - `v_data`、切换 dead ops、执行 `vgone()` 和 `vput()`； - `sys/kern/vfs_subr.c:2328-2341` 明确说明 `insmntque()` 会在失败时回收 vnode， - 只有 `insmntque1()` 把清理留给调用者。 -- **参考实现**：`sys/fs/cd9660/cd9660_vfsops.c:717-721` 失败后只释放私有 - inode，不再解引用 vnode。 -- **Linux 对照**：Linux `fs/erofs/inode.c` 使用 `iget5_locked()` / - `iget_failed()` 管理不同的 inode 生命周期，不能机械复制，但同样要求失败路径 - 不访问已交还对象。 -- **触发与影响**：vnode 构造与强制或并发卸载竞争，进入 - `MNTK_UNMOUNT`/`MNTK_UNMOUNTF` 失败路径。后续写可能成为释放后访问，并可能 - 清空已经复用 vnode 的 `v_data`，属于内核内存安全风险。 -- **类别**：BSD 独有缺陷，源于 FreeBSD vnode 所有权契约。 -- **修复方向**：采用 cd9660 所示所有权模式，失败后释放私有 `en`、清空输出并 - 返回，不再访问 `vp`。是否改用 `insmntque1()` 必须有明确的完整清理理由。 -- **建议 Markdown 手工测试**：新增并发冷 lookup/getfh 与 `umount -f` 测试， - 配合 vnode 分配压力；在 INVARIANTS/WITNESS 内核下记录 panic、WITNESS、引用 - 计数和清理状态。建议增加仅测试用故障注入以确定性触发插入失败。 -- **状态**：已静态复核；未动态复现；未修复。 - -### 5.3 P1-2：一般未压缩 physical run/chunk 大读可触发巨型 `M_WAITOK` 分配和 `uiomove()` 长度截断 - -- **位置**：`src/data.c:493-500` 直接从 `uio_resid` 和映射 run 计算 `want`； - `src/data.c:514-518` 把该 `size_t` 长度传入读取路径； - `src/data.c:208-245` 的 `erofs_bread_device()` 使用 - `malloc(len, M_EROFS, M_WAITOK)` 分配连续缓冲；`src/data.c:521` 再把 `size_t want` - 传给 `uiomove()`。 -- **FreeBSD 对照**：`sys/sys/uio.h:95` 声明 - `int uiomove(void *cp, int n, struct uio *uio)`。因此大于等于 2 GiB 的 `size_t` - 长度在传参时不能被该接口正确表达，可能截断为负值或零值并造成错误或无进展。 -- **Linux 对照**：Linux `fs/erofs/data.c` 使用 folio/iomap 分页路径，不建立与 - 整个用户请求同尺寸的连续内核缓冲。该差异来自 I/O 模型，但当前无界分配不是 - FreeBSD 所要求的必要差异。 -- **触发与影响**：恶意或异常大 chunk/run，配合大 `VOP_READ`/`uio_resid`。影响 - 包括用户可控的巨型 `M_WAITOK` 连续内核分配、系统内存压力、长时间阻塞，以及 - 大于等于 2 GiB 时 `uiomove()` 截断或循环无进展。 -- **类别**：BSD 独有缺陷，属于 I/O 分块和内核 API 宽度处理问题。 -- **修复方向**：把读取固定分块到同时满足 buffer cache、`INT_MAX` 和合理内存 - 上限的尺寸；每轮验证 `uio_resid` 或 offset 必须前进；避免按整个 run 分配。 -- **建议 Markdown 手工测试**：构造超大 chunk/run 的稀疏镜像，分别测试 - `INT_MAX-1`、`INT_MAX`、`INT_MAX+1` 和大于 2 GiB 请求；记录 wired memory、 - malloc failure、进度、信号中断、errno 和 mount/KLD 清理。测试必须设置资源 - 上限，避免把宿主机作为压力目标。 -- **状态**：已静态复核；未动态复现；未修复。 - -### 5.4 P2-1：目录名称排序不变量未验证 - -- **位置**：`src/dir.c:72-109` 校验 dirent 数量、`nameoff` 单调、名称长度和非法 - `/`，但不比较相邻名称；`src/namei.c:55-100` 执行块内二分， - `src/namei.c:103-125` 开始跨块二分选择。 -- **Linux 对照**：Linux `fs/erofs/erofs_fs.h:280` 记录目录项按字典序排列； - Linux `fs/erofs/namei.c:45` 起的查找同样依赖该不变量，也未做全局排序验证。 -- **触发与影响**：结构合法但名称降序、重复或跨块逆序的损坏镜像。实际存在的 - 名称可被静默返回为 `ENOENT`，随后建立负 namecache，使错误持续存在。 -- **类别**：Linux 共同依赖的格式不变量；在 repo22“损坏结构返回 - `EINTEGRITY`、不得静默 miss”的更严格政策下，是 BSD 当前一致性缺口，不应 - 描述成 BSD/Linux 行为差异。 -- **修复方向**：对不可变目录执行一次全局严格递增验证，覆盖块内、跨块和重复 - 名称，并缓存结果；损坏统一返回 `EINTEGRITY`，不得写入负缓存。 -- **建议 Markdown 手工测试**：构造块内降序、块内重复、跨块逆序、跨块重复四类 - 镜像；冷 lookup、重复 lookup、readdir 和 NFS lookup 均应返回 - `EINTEGRITY`，并验证 namecache 无错误负项。 -- **状态**：已静态复核；未动态复现；未修复。 - -### 5.5 P2-2：根 vnode 未验证为目录 - -- **位置**：`src/super.c:759-768` 的 `erofs_root()` 只调用 `erofs_vget()`； - `src/inode.c:471-475` 按 inode mode 设置 vnode 类型，并仅按 NID 设置 `VV_ROOT`。 -- **Linux 对照**：Linux `fs/erofs/super.c:765-774` 加载 root inode 后明确执行 - `S_ISDIR()`，非目录则释放并返回 `-EINVAL`。 -- **FreeBSD 对照**：VFS mount 后续不会替 EROFS 强制验证其 `VFS_ROOT()` 返回 - `VDIR`，文件系统必须自行维持该不变量。 -- **触发与影响**：checksum-valid、但 root inode mode 被改为普通文件、symlink - 或 FIFO 的镜像可能完成挂载；挂载点后续出现 `ENOTDIR` 或异常 vnode 语义。 -- **类别**：BSD 独有缺口；Linux 已显式处理。 -- **修复方向**：`erofs_vget()` 成功后验证 `(*vpp)->v_type == VDIR`，失败时正确 - `vput()`。errno 可对齐 Linux 使用 `EINVAL`，或按 repo22 损坏政策使用 - `EINTEGRITY`，但需统一文档。 -- **建议 Markdown 手工测试**：分别生成 root inode 为 VREG、VLNK、VFIFO 的 - checksum-valid 镜像，验证挂载失败、errno 稳定、无 mount/md/KLD 残留。 -- **状态**：已静态复核；未动态复现；未修复。 - -### 5.6 P2-3：compact index 物理块号加法先发生 32-bit 回绕 - -- **位置**：`src/zmap.c:267`： - `m->pblk = le32dec(...) + nblk;`。`le32dec()` 和 `nblk` 均为 32-bit 范围，表达式 - 在赋给 64-bit `m->pblk` 前可能已经回绕。 -- **Linux 对照**：Linux `fs/erofs/zmap.c:230-232` 使用同样的 - `le32_to_cpu(...) + nblk` 结构，因此 Linux 参考也存在相同风险。 -- **触发与影响**：基准物理块接近 `UINT32_MAX` 且 compact 索引累计 `nblk` 非零， - 映射到错误的低地址，可能读到错误数据或触发后续一致性错误。 -- **类别**：Linux 共同缺陷，绝不能为了源码相似而保留，也不能记成 BSD 差异。 -- **修复方向**：在加法前显式提升到 64-bit，并使用 checked addition；随后按设备 - 块范围验证。 -- **建议 Markdown 手工测试**：构造 base pblk 接近 `UINT32_MAX`、累计跨越边界的 - compact 索引，验证映射不会回绕；同时测试恰好不溢出的边界和真实设备范围错误。 -- **状态**：已静态复核；未动态复现；未修复。 - -### 5.7 P2-4：explicit extent 首次初始化执行有界 O(N) 全表扫描 - -- **位置**：`src/zmap.c:581-619` 的 `z_erofs_validate_extent_table()` 在首次初始化 - 时从 `index=0` 到 `en->z_extents-1` 逐项读取并验证严格递增；调用后才进入 - 二分映射路径。 -- **Linux 对照**：Linux `fs/erofs/zmap.c:501-588` 映射 explicit extent 时按需 - 二分读取，没有 repo22 这一首次完整预扫描。 -- **触发与影响**：超大 extent 表在首次访问时产生 O(N) 元数据 I/O 和延迟，攻击 - 者可用格式边界内的大计数制造明显可用性压力。 -- **边界说明**：循环由 `en->z_extents` 严格限制，记录位置使用 checked arithmetic， - 并验证 `lstart` 严格递增。当前证据不支持“无限循环”或“无边界访问”的说法。 -- **类别**：BSD 独有的性能/可用性权衡，源于更严格的全表不变量验证，不是立即 - correctness 失败。 -- **修复方向**：先量化真实最坏情况。可评估分段验证、验证结果缓存、mount-time - 预算或按二分访问范围逐步验证；任何优化都不能重新引入静默错误映射。 -- **建议 Markdown 手工测试**：生成多档 extent count 的合法/末尾损坏表，测量 - 首次与重复 lookup/read 延迟、I/O 次数和内存；验证预算超限时 errno 和清理。 -- **状态**：已静态复核；未做性能原型；未修复。 - -### 5.8 P3-1：DEFLATE 分配失败被统一映射为 `EIO` - -- **位置**：`src/decompressor.c:246-257` 将所有后端非零返回统一转换为 `EIO`； - `src/decompressor_deflate.c` 的分配失败因而不能保留 `ENOMEM`。 -- **Linux/FreeBSD 对照**：平台均区分资源耗尽和输入/设备错误；调用 FreeBSD - zlib 是正确的，但 wrapper 的返回契约过窄。 -- **影响**：内存压力下诊断和上层重试策略失真。 -- **类别**：BSD 独有 errno 映射问题。 -- **修复方向**：后端返回正 errno，dispatcher 只做必要规范化。 -- **建议测试**：故障注入 DEFLATE workspace/output 分配失败，要求 `ENOMEM`；损坏 - 流仍应为 `EIO` 或项目约定的完整性错误。 -- **状态**：已静态复核；未动态复现；未修复。 - -### 5.9 P3-2：负 Unix 时间被拒绝，和 Linux 语义不一致 - -- **位置**：`src/inode.c:60-68` 以 `uint64_t seconds` 接收时间，并拒绝大于 - `INT64_MAX` 的值。 -- **Linux 对照**：Linux `fs/erofs/inode.c:108-109` 将磁盘 extended inode 的 - 64-bit 时间传给 inode time；Linux 时间模型允许 1970 年前的负时间语义。 -- **影响**：使用补码表达负秒值的镜像在 BSD 被视为损坏，形成跨平台兼容差异。 -- **类别**：BSD/Linux 行为差异，格式签名语义需先与上游规范确认。 -- **修复方向**：确认磁盘字段的正式有符号语义，再采用显式 signed 解码和 FreeBSD - `timespec` 范围检查。 -- **建议测试**：覆盖 `-1`、`INT64_MIN` 邻近值、epoch 和正最大值，在 Linux/BSD - 上比较 stat 结果与 errno。 -- **状态**：静态候选已复核；格式语义仍需上游确认；未修复。 - -### 5.10 P3-3：inline symlink 未拒绝声明长度内嵌 NUL - -- **位置**：`src/data.c:529-533` 的 readlink 直接复用普通读取路径； - `src/data.c:493-525` 按 inode 声明长度移动字节，没有对 symlink payload 做 NUL - 一致性检查。 -- **Linux 对照**：Linux 同样主要依赖 inode size 和 VFS symlink 语义；本项不是 - 用于制造表面差异，而是要求 BSD 明确损坏镜像政策。 -- **影响**：声明长度内部出现 NUL 时，不同调用者可能观察截断或不一致目标。 -- **类别**：损坏输入处理缺口，是否需要拒绝应与 EROFS 格式语义统一。 -- **修复方向**：确认格式是否禁止嵌入 NUL；若禁止，在 inode/readlink 验证中返回 - `EINTEGRITY`，且不得影响合法非 NUL 目标。 -- **建议测试**：inline 和非 inline symlink 分别构造中间 NUL、末尾 NUL、无 NUL - 目标，比较 `readlink(2)`、namei 跟随和 NFS 行为。 -- **状态**：静态候选已复核；格式语义待确认；未修复。 - -### 5.11 P3-4：48-bit end-exclusive 检查拒绝最后一个合法完整块 - -- **位置**：`src/zmap.c:895-898` 计算 `pend = m_pa + m_plen`，随后以 - `(pend >> block_bits) >= (1ULL << 48)` 拒绝映射。 -- **Linux 对照**：Linux 参考中的相关 48-bit 映射边界也存在同类 end-exclusive - 处理风险。 -- **触发与影响**：映射恰好结束在 48-bit 地址空间上界时，`pend` 是合法的 - end-exclusive 值，但当前检查把最后一个完整合法块拒绝为 `EINTEGRITY`。 -- **类别**：Linux 共同边界缺陷，不是 BSD 差异。 -- **修复方向**：校验最后一个实际字节/块，或使用 `pend > limit` 的 byte limit - 比较，并单独处理零长度。 -- **建议测试**：覆盖最后合法块、越界一字节、越界一块和零长度映射。 -- **状态**：已静态复核；未动态复现；未修复。 - -### 5.12 P3-5：birthtime-only `VOP_SETATTR()` 可假成功 - -- **位置**：`src/erofs_vnops.c:252-281` 检查 mode、uid、gid、atime、mtime、flags - 和 size，但不检查 `va_birthtime`，最终可能返回 0。 -- **FreeBSD 对照**：`VATTR_NULL()` 会把 birthtime 设为 `VNOVAL`，内核调用者可以 - 构造仅 birthtime 有效的请求。 -- **触发与影响**：直接内核 VOP 调用可收到成功但属性未改变，违反只读 VOP 语义。 -- **路径限定**：独立复核未确认普通用户态或 NFSv4 可绕过只读 mount 检查到达该 - 单属性路径，因此不能声称已有用户/NFS 可利用的假成功链。 -- **类别**：BSD 独有 VOP 契约和未来维护问题，严重度降为 P3。 -- **修复方向**：birthtime 变更返回 `EROFS`；对 type、nlink、fsid、fileid、 - blocksize、rdev、bytes、generation 等结构字段明确返回 `EINVAL`。 -- **建议测试**：扩展内核 probe，直接提交 birthtime-only 和各结构字段请求；本地 - syscall 与 NFS 测试用于确认它们仍被只读层拒绝。 -- **状态**：已静态复核；无已证实用户/NFS 路径；未修复。 - -### 5.13 P3-6：FRAGMENTS 特例分支不可达，注释与支持边界失配 - -- **位置**：`src/erofs_fs.h:56-64` 已把 `EROFS_FEATURE_INCOMPAT_FRAGMENTS` 放入 - `EROFS_ALL_SUPPORTED_INCOMPAT`；`src/super.c:532` 因而从 `unsupported` 中清除该 - bit，但 `src/super.c:533-549` 又试图在 `unsupported != 0` 时特例允许仅 - FRAGMENTS 的组合。 -- **Linux 对照**：Linux 通过实际 fragment/packed inode 路径表达支持，不需要 - 这种与 supported mask 自相矛盾的特例。 -- **影响**：当前分支不可达，注释声称“窄允许”但真实代码已经把 FRAGMENTS 当作 - 一般 supported incompat bit，误导维护者判断支持边界。 -- **类别**：BSD 独有死分支和文档一致性问题。 -- **修复方向**：依据真实 feature 支持政策二选一：从 supported mask 移除后保留 - 严格特例，或删除死分支并准确文档化支持范围。 -- **建议测试**：组合测试 FRAGMENTS、XATTR_PREFIXES、PLAIN_XATTR_PFX 和 - `packed_nid`，记录 mount errno 和实际 fragment inode 访问结果。 -- **状态**：已静态复核；未动态复现；未修复。 - -### 5.14 不列为 correctness finding：fragment `pstart_hi` - -曾有候选认为 fragment extent 构造会丢失 `pstart_hi`。当前静态证据不足以确认该 -字段在 fragment 记录格式中具有同样的高位语义，也不足以证明当前路径实际丢失 -可用地址。因此本报告不把它列为 correctness finding，只保留为“与上游格式语义 -确认”的待办。后续必须先取得格式规范或上游实现证据，再决定是否测试或修改。 - -## 6. FreeBSD 最佳实践与原生能力复用 - -| 领域 | 当前做法 | 评估与决策 | -| --- | --- | --- | -| DEFLATE | `src/decompressor_deflate.c:28` 起调用 FreeBSD zlib | 正确复用；应保留，修正 errno 契约即可 | -| ZSTD | `src/decompressor_zstd.c` 使用 ZSTDIO API | 正确复用；FreeBSD 无通用独立 zstd KLD，不应伪造依赖 | -| MicroLZMA | `src/decompressor_lzma.c:14-38` 私有重命名并编译 XZ Embedded | 应保留本地实现；stock `xz.ko` 未定义 `XZ_DEC_MICROLZMA`，不能强行链接 | -| LZ4 | `src/lz4.c:21` 起实现 raw/partial decoder | 应保留；FreeBSD/OpenZFS 接口含不同 framing，且不满足 EROFS partial 语义 | -| 目录输出 | `src/dir.c:160-178` 自有 cookie helper | 不能机械替换；它额外验证 cookie 严格递增和容量短读 | -| `vfs_read_dirent()` | FreeBSD `sys/kern/vfs_subr.c:6814-6830` | 可原型化为输出层 helper，但必须保留 EROFS cookie 约束并回归 NFS | -| GEOM/VFS I/O | `src/super.c:226` 建 consumer，`src/data.c:250` 用 `bread()` | 符合 FreeBSD 文件系统惯用路径，不应改为 GEOM class 私有读接口 | -| vnode cache | `src/inode.c:435-460` 用 `vfs_hash_get/insert` | 总体正确；仅 `insmntque()` 失败所有权需修复 | -| pager | `src/erofs_vnops.c:434` 复用 local pager | 正确，避免重写 VM pager | -| ACL/extattr | `src/erofs_vnops.c:68` 等调用 POSIX.1e ACL/extattr API | 正确保留 BSD namespace 和权限差异 | -| CRC32C | `src/super.c:307` 使用 `calculate_crc32c()` | 正确复用 FreeBSD kernel helper | -| endian | 磁盘字段使用 `leXXtoh`，非对齐流使用 `leXXdec` | 正确；不应直接复制 Linux unaligned 宏 | -| checked arithmetic | 多处使用 compiler overflow builtin | 合理；FreeBSD 15 无更优的通用非 LinuxKPI 替代 | - -### 6.1 为什么不能强行依赖 `xz.ko` - -FreeBSD `sys/modules/xz/Makefile:5-20` 会编译 `xz_dec_lzma2.c` 并导出符号，但没有 -定义 `XZ_DEC_MICROLZMA`。MicroLZMA 入口受该宏控制，因此 stock `xz.ko` 不提供 -repo22 所需的 `xz_dec_microlzma_*` ABI。简单添加 `MODULE_DEPEND(erofs, xz, ...)` -不能解决符号缺失。 - -只有在 FreeBSD 正式启用并导出 MicroLZMA KPI，同时更新模块 ABI/version 后， -repo22 才应重新评估直接依赖。当前强行引用反而违反“优先复用，但不强制引用一切” -的原任务要求。 - -### 6.2 LZMA per-call allocation 是性能原型事项 - -`src/decompressor_lzma.c:55-66` 每次解压分配并释放 decoder state。Linux 使用可 -复用 stream pool。当前没有证据证明它造成 correctness 错误，但并发压缩读取会 -产生约 30 KiB state 的反复 malloc/free 和锁竞争。 - -后续应单独做 per-CPU、mount-local 或受限 pool 原型，测量并发吞吐、内存峰值、 -卸载清理和字典变化。没有数据前，不应把缓存重构混入 correctness 批次。 - -## 7. BSD/Linux 行为差异分类 - -### 7.1 必要的 BSD 差异 - -- 正 errno，而不是 Linux 内核负 errno。 -- VOP/VFS operation table、vnode lock、namecache、`vn_vget_ino()` 和 - `vfs_hash_*` 生命周期。 -- GEOM consumer、设备 vnode 和 buffer cache I/O，而不是 folio/iomap/bio。 -- FreeBSD extattr user/system namespace、`extattr_check_cred()` 和 POSIX.1e ACL。 -- readdir 的 `a_cookies`/`a_ncookies` ABI、synthetic dot cookie 和 NFS 恢复语义。 -- `fifo_specops`、FreeBSD FID 布局、generation 与 `ESTALE` 校验。 -- pager 使用 `vnode_pager_local_getpages*`。 - -这些差异应保留，不应为了外观接近 Linux 而移除。 - -### 7.2 Linux 共同缺陷 - -- compact index 32-bit 加法回绕，见 `src/zmap.c:267` 与 Linux - `fs/erofs/zmap.c:231`。 -- 48-bit end-exclusive 最后合法块边界。 -- 目录二分依赖排序；Linux 也未全局验证。repo22 可选择更严格验证，但不能把 - 上游共同不变量说成 BSD 偏差。 - -源码相似不是保留共同缺陷的理由。修复时应尽量沿用上游命名和 checked arithmetic -结构，并把可上游化的发现单独记录。 - -### 7.3 BSD 独有缺陷 - -- `insmntque()` 失败后的 vnode 释放后访问。 -- chunk 大读的整段 `M_WAITOK` 分配和 `size_t` 到 `int` 的 `uiomove()` 传参。 -- root vnode 未检查 `VDIR`。 -- DEFLATE errno 归一化过度。 -- birthtime-only `VOP_SETATTR()` 假成功。 -- FRAGMENTS supported mask 与特例注释矛盾。 - -### 7.4 非必要维护差异 - -- `erofs_fs.h` 定义顺序、常量命名和字段注释没有尽量保持 Linux 顺序。 -- 压缩配置 loader 集中于 `decompressor.c`，而 Linux 分布在各 backend。 -- backend 函数名使用 `lzma_decompress`、`deflate_decompress`、 - `zstd_decompress`，没有采用 Linux `z_erofs_*` 体系。 -- `xattr.h`、`internal.h`、`erofs_defs.h` 中仍有失效 Linux 声明或未使用抽象。 -- xattr prefix helper 命名、部分函数定义位置和空包装与 Linux 无必要不同。 - -### 7.5 需要原型或格式确认后再判断 - -- fragment extent 的 `pstart_hi` 语义。 -- LZMA decoder state 缓存。 -- 保留 cookie 验证的 `vfs_read_dirent()` 输出适配器。 -- explicit extent 全表验证的分段/缓存策略。 -- negative timestamp 与 inline symlink NUL 的正式格式语义。 - -## 8. 结构、命名和 style 对齐 - -### 8.1 正向对齐结果 - -- repo22 与 Linux 有 15 个共同 C/H 文件名。 -- `src/namei.c` 的核心名称查找函数名称和顺序高度对应 Linux。 -- `src/zmap.c` 的 13 个共享核心算法函数保持相同相对顺序，是当前对齐质量最高的 - 文件之一。 -- `src/Makefile` 中 12 个共同编译单元保持与 Linux Makefile 相同的相对顺序。 -- `erofs_vnops.c` 集中 BSD VOP glue，避免把 FreeBSD 代码散入所有 Linux 对应 - 算法文件，这一文件级差异是必要且有利于维护的。 - -### 8.2 `erofs_fs.h` 的非必要差异 - -BSD 与 Linux 都包含同一组核心磁盘结构，但 repo22 的结构顺序、常量命名和注释 -存在明显漂移。例如 repo22 从 `src/erofs_fs.h:104` 定义 superblock，而 Linux -先定义 deviceslot；dirent、chunk index、map header 和 lcluster index 的位置也有 -移动。机械审查对 19 个核心 struct/union 记录到 31/171 个顺序逆序对，顺序相似度 -约 81.9%。 - -字段 packing、offset、endian 和 `_Static_assert` 是 correctness 约束，应保留 BSD -表达；结构定义顺序、字段语义注释和非平台相关名称则应尽量恢复 Linux，以降低 -未来磁盘格式同步成本。`EROFS_ALL_SUPPORTED_INCOMPAT` 与 Linux 名称不同，也应 -先判断是否存在真实语义差异，再决定是否保留。 - -### 8.3 压缩职责和命名 - -repo22 把 LZMA、DEFLATE、ZSTD 配置解析集中在 `src/decompressor.c`，Linux 则分别 -放在对应 backend。调用 ABI 必须因 FreeBSD 内核 API 而不同，但配置 loader 的 -文件职责和 `z_erofs_*` 命名并无平台限制。 - -建议在 correctness 修复完成后，单独评估把配置解析移回各 backend，并统一内部 -命名。该重排必须保持双配置构建和全部压缩 Markdown 测试，不应与 P1/P2 修复混在 -同一提交。 - -### 8.4 死代码和残留抽象 - -- `src/xattr.h:11-42` 保留注释掉的 Linux `CONFIG_*` 分支、未实现的 - `erofs_xattr_handlers`、`struct inode` 和 `struct posix_acl` 声明。 -- `src/internal.h:50-71` 保留未使用的同步解压/cache 枚举和 `erofs_buf`； - `src/internal.h:243-247` 的 `erofs_is_fileio_mode()` 固定返回 false。 -- `src/erofs_defs.h:5-19` 的 CRC polynomial、inode slot bits 和 range-coder 常量 - 多项仅定义未使用；`src/inode.c:137-139` 仍直接使用字面量 5。 -- `src/super.c` 的部分空包装和函数顺序可进一步与 Linux/FreeBSD 惯例收敛，但应 - 放在纯维护提交中。 - -### 8.5 失效测试入口和 harness - -- `test_all_decompress.sh:2-7` 仍标记 repo19；其 `SRCDIR` 当前未实际使用，但测试 - 内容不验证 repo22 KLD。 -- `test_chunk_based.sh:2-17` 实际进入 repo17，后续会 clean、写镜像、卸载模块、 - 配置 md 和 mount，具有真实误操作其他目录和全局系统状态的风险。 -- `tests/test_decompress.c` 与 `tests/test_decompress_standalone.c` 已无法通过当前语法 - 检查，并仍按旧参数数量调用后端接口；当前接口见 `src/internal.h:314-322`。 - -`docs/architecture.md:242-245` 已声明这两个根脚本不能作为验收入口，但只写文档 -不足以消除可执行误入口。后续应删除、移入明确的 historical 目录，或改为立即失败 -并指向 `tests/TC*.md`。 - -### 8.6 文档陈旧 - -- `docs/architecture.md:79` 的 `erofs_mount` 示例与当前 - `src/internal.h` 结构不一致，字段如 `z_algorithmformat`、`fragmentoff` 已陈旧。 -- `docs/architecture.md:273` 声称“零拷贝”，但 `src/data.c:244-261` 明确执行 - `malloc` 和 `memcpy`。 -- Linux 基线只记录绝对工作区路径和版本，没有独立仓库可获取的上游 provenance。 - -### 8.7 style(9) 机械结果 - -使用 FreeBSD `tools/build/checkstyle9.pl` 0.31 对 `src/*.[ch]` 连续检查，结果稳定： - -- 18 个文件； -- 48 warnings； -- 1 error； -- 44 个超过 80 列； -- 4 个块注释格式告警； -- 唯一 error：`src/erofs_fs.h:265` 的 `return 0` 缺 FreeBSD 风格括号。 - -其他可见项包括 `src/internal.h:11-12` 的 `sys/param.h // MUST FIRST` 实际位于 -`sys/types.h` 后，SPDX 注释形式混用，以及部分 Makefile 空白不统一。 - -这些结果不能机械全改。磁盘 ABI 宏、静态断言和与 Linux 保持一致的行，可能有 -理由超过 80 列。建议先制定“FreeBSD style(9) 优先、磁盘 ABI/上游同步可例外”的 -仓库规则，再逐批清理。 - -## 9. 正向成果与保留项 - -本轮不能只列问题。以下成果符合原任务 2 的目标，应在后续优化中保留： - -- 使用 `bsd.kmod.mk`，而不是维持 Linux Kbuild 兼容层。 -- GEOM consumer、设备 vnode、`bread()` 和 buffer cache 的组合符合 FreeBSD 文件 - 系统路径。 -- `vfs_hash_*`、vnode pager、ACL、extattr、CRC32C、endian helper 和 checked - arithmetic 的总体选型正确。 -- DEFLATE 和 ZSTD 直接复用可用的 FreeBSD 内核实现。 -- 不强行依赖缺少 MicroLZMA ABI 的 `xz.ko`，不强行套用不兼容的 OpenZFS LZ4。 -- `namei.c` 和 `zmap.c` 核心函数顺序与 Linux 高度对应。 -- Linux-only 文件没有以空壳方式复制到 BSD 仓库。 -- FreeBSD-only VOP glue 集中在 `erofs_vnops.c`，平台边界清晰。 -- 既有动态测试已覆盖多设备、压缩映射、xattr/ACL、NFS、48-bit 和边界行为； - 本报告不重写这些 task 1 证据。 - -## 10. 可追溯性与许可证边界 - -### 10.1 仓库许可证材料 - -repo22 的 18 个生产 `src/*.[ch]` 文件均带 SPDX 标识：13 个 -`GPL-2.0-only`、4 个 `BSD-2-Clause`、1 个 `MIT`。`docs/erofs.5:1-23` 自身包含 -完整的 BSD 两条款文本。 - -但 repo22 根目录没有集中式 `LICENSE`、`COPYING`、`NOTICE`、许可证构成表或来源 -清单。SPDX 可以识别文件当前声明，不能单独让第三方重建整个 KLD 的来源、整体 -分发边界和权利链。 - -建议后续增加集中式许可证和来源 manifest，明确 GPL 派生核心、MIT 磁盘 ABI、 -BSD VOP wrapper、本地 LZ4 和嵌入 XZ 源的关系。本报告不作兼容性或法律结论。 - -### 10.2 UDF helper 来源 - -`src/dir.c:158-178` 明确写有“modelled after UDF”。历史版本的 `erofs_uiodir` -与 FreeBSD `sys/fs/udf/udf_vnops.c:614-639` 在结构、分支顺序和控制流上高度接近； -当前版本已增加严格 cookie 单调性、结果枚举、容量检查和不同错误传播。 - -现有 Git 历史不足以区分最初是直接复制、紧密改写还是参考实现，因此不得断言 -侵权或具体来源类别。应由作者确认来源过程，并据此决定是否补充 FreeBSD UDF 的 -版权归属或说明。 - -### 10.3 Linux 基线外部可复现性 - -在当前 `/work` 单仓环境内，Linux 参考可由导入提交 -`8be2be573d2b191c83d760b65c554f78eb95893c` 和 tree -`b79b9a8b62f633e887a8b99075ff9412fabd7f83` 验证。 - -`docs/architecture.md:160-163` 目前只记录本地绝对路径和 Linux 7.1-rc1 字符串， -没有官方上游 URL、上游 Git commit/tag、获取命令和校验值。因此内部审查可复现， -独立社区仓库的外部复现材料仍不完整。 - -## 11. 第三方维护者相似度 - -以下评分来自文件树、共同符号顺序、磁盘 ABI 顺序、FreeBSD API、代码卫生和文档 -可复现性的综合人工/机械评估。它不是客观度量，也不代表 feature 完整度： - -| 维度 | 暂评分 | 说明 | -| --- | ---: | --- | -| 文件树与职责 | 8.0/10 | 共同文件清晰，Linux-only/BSD-only 边界总体合理 | -| 核心算法命名和顺序 | 8.5/10 | namei/zmap 较好，压缩后端仍有差异 | -| 磁盘 ABI 头文件可同步性 | 6.0/10 | 布局重视正确性，但顺序、命名、注释漂移明显 | -| FreeBSD API 与构建集成 | 9.0/10 | GEOM、VFS、pager、ACL、压缩库选型总体正确 | -| 代码卫生与 style(9) | 5.5/10 | 死声明、旧入口、48 warnings/1 error | -| 社区文档与基线可复现性 | 5.0/10 | 内部可验证，独立仓库 provenance 不完整 | -| **综合维护者相似度** | **7.2/10** | 带权主观评估，用于维护排序，不是兼容性结论 | - -综合分采用带权主观评估：文件树与职责 15%，核心算法命名和顺序 20%，磁盘 ABI -头文件可同步性 20%，FreeBSD API 与构建集成 20%，代码卫生与 style(9) 15%，社区 -文档与基线可复现性 10%。计算为 -`8.0*0.15 + 8.5*0.20 + 6.0*0.20 + 9.0*0.20 + 5.5*0.15 + 5.0*0.10 = 7.225`， -按一位小数四舍五入为 `7.2/10`。权重反映本任务更重视算法、ABI 和平台集成，仍不 -应视为客观兼容性指标。 - -从第三方维护者视角，当前更准确的描述是：核心算法和主要 FreeBSD 适配已对齐， -但仍存在明确的 correctness 风险、结构漂移、代码卫生和可追溯性工作。 - -## 12. 后续执行路线 - -本报告只给出路线，不执行任何修改。 - -### 第一批：先处理两个 P1 - -1. 修复 `insmntque()` 失败后的 vnode UAF。 -2. 对 chunk/physical read 建立严格分块上限，修复 `uiomove(int)` 宽度问题。 -3. 为两个问题分别新增 Markdown 手工测试和独立 FreeBSD 动态报告。 - -该批次必须最小化变更面，不能混入文件重排或 style 清理。 - -### 第二批：P2 correctness 与边界测试 - -1. 目录全局严格排序验证与 namecache 负项检查。 -2. root vnode `VDIR` 验证。 -3. compact pblk 64-bit checked addition。 -4. 量化 explicit extent 首次 O(N) 扫描，并决定预算/缓存策略。 - -每项一份或一组明确 Markdown 测试，复用既有 fixture helper，但不实现 CI。 - -### 第三批：P3 correctness、compatibility 与 semantic/style - -1. correctness/API 契约：保留原 P3 严重度，分别修正 DEFLATE 分配失败 errno、 - 48-bit end-exclusive 最后合法完整块边界，以及 birthtime-only `VOP_SETATTR()` - 的只读语义。 -2. compatibility/格式语义：确认并实现 Linux 负时间戳兼容语义；明确 inline symlink - 内嵌 NUL 的格式政策并补充拒绝或兼容测试。 -3. semantic/style：删除或改写不可达的 FRAGMENTS 特例分支，使注释、supported mask - 和真实支持边界一致。 - -六项均需各自的 Markdown 手工测试或同类成组测试，不得因进入后续批次而改变本报告 -中的 P3 严重度。 - -### 第四批：低风险卫生、文档和 provenance - -1. 删除或隔离 repo17/repo19 历史脚本和失配 C harness。 -2. 清理 `xattr.h`、`internal.h`、`erofs_defs.h` 的死声明和死宏。 -3. 修正 architecture 文档、零拷贝表述和 Linux 基线复现信息。 -4. 增加 LICENSE/COPYING、许可证构成和来源 manifest。 -5. 确认 UDF helper 和本地实现的作者归属。 -6. 在规则明确后处理 style(9) 的确定性问题。 - -### 第五批：结构重排和压缩职责对齐 - -1. 恢复 `erofs_fs.h` 非平台相关定义顺序、注释和名称。 -2. 评估把压缩配置 loader 移回各 backend。 -3. 统一内部 `z_erofs_*` 命名和函数定义顺序。 -4. 删除无行为包装，调整非必要函数位置差异。 - -该批次风险高于普通 style，必须用双配置构建、模块加载和全部相关 Markdown 测试 -证明无行为变化。 - -### 第六批：性能原型 - -1. LZMA decoder state pool/cache。 -2. explicit extent 验证缓存或分段策略。 -3. `vfs_read_dirent()` 输出适配器。 - -原型必须提供基线、压力方法、内存峰值、卸载清理和退化行为；没有数据不合并。 - -## 13. 原任务三个目标的覆盖矩阵 - -| 原任务目标 | 已确认正向证据 | 本轮发现 | 尚未验证/后续 | -| --- | --- | --- | --- | -| 1. 最佳实践与 BSD 原生能力复用 | GEOM/bread、vfs_hash、pager、ACL/extattr、CRC/endian、DEFLATE/ZSTD 复用正确；LZ4/MicroLZMA 本地实现有技术理由 | `insmntque()` 契约违反；chunk 大分配；dir helper 需保留但可原型化输出层；LZMA per-call allocation | 动态复现 P1；XZ KPI、dir adapter、LZMA pool 原型 | -| 2. 正确处理 BSD/Linux 行为差异 | 正 errno、VOP/VFS、GEOM、namecache、cookie、ACL namespace、FID/generation 等必要差异清晰 | root 类型、setattr、errno、负时间；compact/48-bit/排序属于 Linux 共同问题，未误记为 BSD 差异 | negative time、symlink NUL、fragment pstart_hi 格式语义确认 | -| 3. 第三方维护者相似性 | 15 个共同文件；namei/zmap 和 Makefile 顺序质量高；未伪造 Linux-only 空文件 | erofs_fs 顺序/注释、压缩职责/命名、死代码、旧脚本、陈旧文档、style、provenance、许可证清单 | 逐批结构重排、来源确认、外部可复现基线、回归验证 | - -因此，三个目标都已有实质正向成果，但目标 3 明显未达到“没有任何非必要差异” -的完成标准；目标 1 和目标 2 也因两个 P1 和若干 P2/P3 finding 需要继续工作。 - -## 14. 与 report-3 的关系 - -`current/report-3` 是任务 1 收尾时的历史快照。它关于 161 项 feature 手工验证、 -构建、清理和当时已知 issue 的证据继续有效，本报告不修改也不重写这些结论。 - -但 report-3 中“未发现仍需消除的非必要 Linux/FreeBSD 结构、命名或职责差异”是 -一个超出当时证据范围的绝对表述。当时没有保存： - -- 全部文件的函数、变量和定义顺序对照； -- FreeBSD 原生 API 复用清单； -- Linux-only/BSD-only 文件逐项必要性论证； -- 死代码、失效入口、style(9)、来源和许可证审计； -- 独立仓库可重建的 Linux 基线 provenance。 - -因此，对任务 2 应以本报告作更精确限定： - -> repo22 的核心算法和主要 FreeBSD 适配已取得良好对齐，但尚不能认定全面完成 -> code review 和 style 优化；仍存在两个 P1、四个 P2、六个 P3，以及明确的结构、 -> 代码卫生、style 和可追溯性工作。 - -该限定不回写 report-3，以保持历史报告不可变和审计链清晰。 - -## 15. 最终状态 - -- 本轮只创建 `current/report-4/`。 -- 未修改 `src/`、`tests/`、`docs/`、`issues/`、既有 report 或 CI。 -- 未声称任何 finding 已修复。 -- 未把静态候选误写成动态复现结果。 -- 新问题进入后续计划前，应先创建对应 Markdown 手工测试规格，再由独立执行者在 - FreeBSD 15 环境中复现、修复和回归。 diff --git a/current/report-4/README.md b/current/report-4/README.md deleted file mode 100644 index 2d21194..0000000 --- a/current/report-4/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# report-4：任务 2 代码审查与风格评估 - -本目录记录 2026-08-10 对 repo22 原任务中“任务 2：进行全面的 code review -和 style 优化”的只读调查结果。 - -本次工作只做静态评估、源码契约核对、Linux/FreeBSD 严格比对、历史检查和 -机械 style 检查，不修改生产源码、测试、既有文档、issues 或 CI。报告中的新 -finding 尚未在 FreeBSD 虚拟机中动态复现，也尚未修复。 - -## 审计基线 - -- BSD 目标：`repo-community/repo22`，提交 - `76cfb553e0951b428f9d426933a10d1fe18d2a0e` -- 最终源码变更提交：`67d3c057fbcad5adc765ab9927da511e3221ac8f` -- repo22 `src/` tree：`a605af0f01f83d67c199005e775e1a47a1610036` -- Linux 独立参考：`/work/dev-src-linux/fs/erofs`，Linux 7.1.0-rc1 -- Linux 本地导入提交：`8be2be573d2b191c83d760b65c554f78eb95893c` -- Linux EROFS tree：`b79b9a8b62f633e887a8b99075ff9412fabd7f83` -- FreeBSD API 参考：`/work/dev-freebsd-releng`，15.0-RELEASE-p9 - -## 结论摘要 - -- 任务 1 的 161 项 feature 验证结论不在本报告中重写，也不因本报告自动失效。 -- 任务 2 尚不能认定已经全面完成。本轮确认 `P0=0`、`P1=2`、`P2=4`、 - `P3=6`，另有若干非 correctness 的结构、style、来源与性能原型工作。 -- FreeBSD 原生 API 和构建集成总体合理，强行复用并不适用于 LZ4、MicroLZMA - 和目录 cookie helper。 -- 核心算法映射质量较高，但磁盘 ABI 头文件顺序、压缩职责、死代码、历史测试 - 入口、文档、style(9) 和外部可复现性仍有明确维护债务。 -- 综合维护者相似度暂评为 `7.2/10`。该分数仅用于排序维护工作，不是客观质量 - 或兼容性指标。 - -## 文档索引 - -- [任务 2 详细代码审查与风格评估](2026-08-10-task2-code-review-style-assessment.md) - -## 与既有报告的关系 - -`current/report-1` 至 `current/report-3` 保持历史快照。本目录不修改它们。 -report-3 的 feature 验证结果继续作为任务 1 证据；其中“没有非必要差异”的绝对 -表述，由本次更完整的结构、style、来源和许可证审计作更精确的限定。 diff --git a/src/data.c b/data.c similarity index 100% rename from src/data.c rename to data.c diff --git a/src/decompressor.c b/decompressor.c similarity index 100% rename from src/decompressor.c rename to decompressor.c diff --git a/src/decompressor_deflate.c b/decompressor_deflate.c similarity index 100% rename from src/decompressor_deflate.c rename to decompressor_deflate.c diff --git a/src/decompressor_lz4.c b/decompressor_lz4.c similarity index 100% rename from src/decompressor_lz4.c rename to decompressor_lz4.c diff --git a/src/decompressor_lzma.c b/decompressor_lzma.c similarity index 100% rename from src/decompressor_lzma.c rename to decompressor_lzma.c diff --git a/src/decompressor_zstd.c b/decompressor_zstd.c similarity index 100% rename from src/decompressor_zstd.c rename to decompressor_zstd.c diff --git a/src/dir.c b/dir.c similarity index 100% rename from src/dir.c rename to dir.c diff --git a/docs/TEST_REPORT.md b/docs/TEST_REPORT.md deleted file mode 100644 index 42035d1..0000000 --- a/docs/TEST_REPORT.md +++ /dev/null @@ -1,97 +0,0 @@ -# Manual Test Evidence - -## Final Status - -repo22 has 161 executable manual test cases, TC001-TC161, plus the non-executable -TC000 template. The canonical dated reports produce: - -| Status | Count | -| --- | ---: | -| PASS | 160 | -| PARTIAL | 1 | -| FAIL / KERNEL-FAIL | 0 | -| ENVIRONMENT-UNAVAILABLE | 0 | -| SHELVED test case | 0 | - -TC146 is the single PARTIAL case. Its HEAD2 and interlaced paths pass, while -the explicit mapped-payload positive fixture remains unavailable. That -shelved subitem is not counted as a second test case. - -## Evidence Policy - -Authoritative evidence consists of the numbered Markdown procedures, -deterministic field-asserting fixture helpers, native syscall/kernel probes, -and dated `tests/results/manual/*/manual-test-report.md` reports. Reports record -the exact source, module or fixture hashes, FreeBSD kernel behavior, errno, and -cleanup state. - -Host-only fixture generation, static ABI review, and userspace erofs-utils -output are supporting evidence. They do not replace a required FreeBSD kernel -result. The suite is manual-only; no CI pipeline or automated kernel-test -harness is implemented or claimed. - -## Canonical Runs - -The final status uses these non-overlapping report groups for TC001-TC156: - -| Group | Exact scope | Result | -| --- | --- | --- | -| G1 | 32 IDs | 32 PASS | -| G2 | 13 IDs | 13 PASS | -| G3 | 31 IDs | 31 PASS after the dated TC060 fixed-source rerun | -| G4 | 27 IDs | 27 PASS | -| G5 | 24 IDs | 23 PASS, TC146 PARTIAL | -| G6 | 11 IDs | 11 PASS | -| G7 | 11 IDs | 11 PASS | -| G8 | 7 IDs | 7 PASS | - -The G1-G8 tables contain exactly 156 rows and 156 unique IDs, with no missing -or duplicate TC from TC001-TC156. TC060's original G3 KERNEL-FAIL is historical -and is superseded by -`tests/results/manual/2026-08-09T1120Z-tc060-fix/manual-test-report.md`. -TC153 passed in the final G3 run and its issue is resolved. - -TC157-TC161 are recorded in -`tests/results/manual/2026-08-09T1804Z-final-review-independent/manual-test-report.md`. -All five pass on exact source baseline -`fcc85b93d5f8fd9671686bd68bf3b086c8bd25cf`. - -## TC111 Coverage Audit - -A bounded audit of `tests/TC[0-9][0-9][0-9]-*.md` found 162 rows and 162 unique -IDs, TC000-TC161, with no missing or duplicate ID. Excluding TC000 leaves 161 -executable specifications. - -The audit also cross-checked the eight canonical group tables and the five -final-review rows. Final arithmetic is `155 PASS + 1 PARTIAL + 5 PASS`, or -160 PASS and one PARTIAL. - -## Final Build Qualification - -The final independent run rebuilt both configurations after the TC161 failing -`nm` shim and used `/usr/bin/nm` for the post-shim build: - -| Configuration | KLD SHA256 | -| --- | --- | -| `WITH_ZSTDIO=0` | `15fda9d334132cd81769ce4dff4f8411a6e2b4cf7531c352530d0de85f42a2d2` | -| `WITH_ZSTDIO=1` | `23782dc0ce7da188807d35020bf2d8c6044b796c5c9a8746b398ba784cd6ad4e` | - -Both KLDs loaded, mounted a qualified image read-only, unmounted, detached the -md provider, and unloaded. Final guest mount, md, EROFS KLD, and DTrace KLD -counts were zero. - -G1-G8 evidence was collected across multiple source commits. It is not claimed -that all historical tests ran on the final KLD. The final code baseline was -independently exercised by TC157-TC161 and the affected dual-build, -module-load, and mount smoke. - -## Issue Status - -- TC010: RESOLVED by a real 16 TiB-plus sparse-provider statfs run. -- TC060: RESOLVED by the FreeBSD pathconf fix and exact-source rerun. -- TC153: RESOLVED by the multi-TiB Layout 0 directory lookup run. -- Explicit mapped payload: SHELVED as a detailed fixture/tooling limitation; - TC146 remains PARTIAL. - -See `issues/README.md` for the current index and the individual files for -triggers, analysis, attempts, results, feature impact, and acceptance criteria. diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index abc5bc1..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,283 +0,0 @@ -# repo22 架构设计 - -## 设计目标 - -repo22 提供尽量贴近 Linux `fs/erofs` 职责划分的 FreeBSD 15 -只读实现，同时对 FreeBSD vnode、GEOM、pager、`dev_t` 和 NFS FID 语义做 -必要适配。 - -## 核心原则 - -1. **Linux 对齐**：函数命名、文件组织、代码排序尽可能匹配 Linux 版本 -2. **最小抽象**：避免不必要的封装层 -3. **安全优先**：保留 repo19 的所有安全修复 -4. **可维护性**：便于社区维护和与上游同步 - -## 文件组织 - -``` -src/ -├── super.c - 超级块、挂载、VFS 集成 -├── inode.c - inode 读取和 vnode 管理 -├── data.c - 数据块映射和解压缩 -├── namei.c - 路径查找（二分搜索） -├── dir.c - 目录遍历和输出 -├── xattr.c - 扩展属性和 ACL -├── erofs_vnops.c - VFS vnode 操作实现 -├── decompressor.c - 压缩配置解析和统一调度 -├── decompressor_lz4.c - FreeBSD 有界 LZ4 后端 -├── decompressor_lzma.c - MicroLZMA 后端 -├── decompressor_deflate.c - DEFLATE 后端 -├── decompressor_zstd.c - 可选 ZSTDIO 后端 -├── zmap.c - 压缩逻辑块映射 -├── zdata.c - 压缩数据读取 -├── internal.h - 内存结构和内部 API -├── erofs_fs.h - 磁盘格式定义 -└── xattr.h - 扩展属性接口 -``` - -## 分层架构 - -``` -┌─────────────────────────────────────┐ -│ VFS 层 (FreeBSD kernel) │ -└─────────────┬───────────────────────┘ - │ -┌─────────────▼───────────────────────┐ -│ VFS 接口层 │ -│ - erofs_vnops.c │ -│ - super.c (mount/unmount/root) │ -└─────────────┬───────────────────────┘ - │ -┌─────────────▼───────────────────────┐ -│ 文件系统逻辑层 │ -│ - inode.c (erofs_read_inode) │ -│ - namei.c (erofs_namei) │ -│ - dir.c (erofs_readdir_block) │ -│ - xattr.c (erofs_getxattr) │ -└─────────────┬───────────────────────┘ - │ -┌─────────────▼───────────────────────┐ -│ 数据访问层 │ -│ - data.c (erofs_map_blocks) │ -│ - data.c (erofs_read_data) │ -│ - zmap.c / zdata.c │ -│ - decompressor.c (z_erofs_decompress) │ -└─────────────┬───────────────────────┘ - │ -┌─────────────▼───────────────────────┐ -│ 块 I/O 层 │ -│ - erofs_bread/erofs_brelse │ -└─────────────────────────────────────┘ -``` - -## 核心数据结构 - -### erofs_mount (内存中的文件系统状态) -```c -struct erofs_mount { - struct mount *mnt; // FreeBSD mount 结构 - struct vnode *devvp; // 块设备 vnode - struct g_consumer *cp; // GEOM consumer - - uint32_t block_size; // 块大小 - uint64_t root_nid; // 根目录 NID - uint32_t feature_compat; // 特性标志 - uint32_t feature_incompat; - - struct erofs_sb_lz4_info lz4; // LZ4 参数 - struct erofs_deviceslot *devs; // 设备表 - struct erofs_xattr_prefix_item *xattr_prefixes; // xattr 前缀表 -}; -``` - -### erofs_node (内存中的 inode) -```c -struct erofs_node { - struct vnode *vnode; // 关联的 vnode - uint64_t nid; // 节点 ID - uint64_t size; // 文件大小 - uint8_t datalayout; // 数据布局类型 - - // 压缩相关 - uint8_t z_algorithmformat; - uint8_t z_lclusterbits; - - // Chunk-based 相关 - uint16_t chunkformat; - uint8_t chunkbits; - - // Fragment 相关 - uint32_t fragmentoff; - bool fragment; -}; -``` - -## 关键实现细节 - -### 1. 数据布局支持 - -支持 4 种数据布局： -- **FLAT_PLAIN**: 连续块 -- **FLAT_INLINE**: 最后一个逻辑块位于 inode metadata block 内，且受声明 - image/metabox bounds 约束 -- **CHUNK_BASED**: 固定大小 chunk，支持稀疏文件 -- **COMPRESSED**: 可变大小压缩 cluster - -### 2. 压缩算法 - -支持 4 种压缩算法及未压缩 transform： -- LZ4 (LZ4HC) -- LZMA -- DEFLATE -- ZSTD -- 未压缩 - -### 3. 高级特性 - -- ✅ **ztailpacking**: 压缩文件尾部内联 -- ✅ **fragments**: 已验证的 fragment-backed 压缩文件与 metabox carrier -- ⚠️ **dedupe**: 仅声明已验证的 fragment/partial-reference 形式，不宣称 - 覆盖所有未来编码 -- ✅ **xattr_prefixes**: 共享 xattr 前缀表 -- ✅ **device_table**: 多设备支持 -- ✅ **metabox**: 每 inode 元数据盒 - -### 4. 安全机制 - -repo22 的边界策略： -- 所有指针操作前检查边界 -- 所有算术运算检查溢出 -- 所有分配检查大小合理性 -- 递归深度限制 -- 设备 ID 和块地址验证 - -## 与 Linux 版本的差异 - -### 对照基线 - -本轮维护逐文件对照工作区中的 `/work/dev-src-linux/fs/erofs`。该目录是导入的 -Linux 7.1-rc1 EROFS 参考快照；对照不依赖 repo22 与 Linux 树具有共同 Git -历史。`/work/linux-src/fs/erofs` 中对应文件与该精简快照字节一致，但不是本轮 -文件映射的依据。 - -### 必要差异（FreeBSD 适配） - -1. **内存分配**：使用 `malloc(..., M_EROFS, ...)` 而非 `kmalloc()` -2. **块 I/O**：通过 GEOM consumer 和 FreeBSD vnode/buffer 接口读取 provider -3. **VFS 接口**：`erofs_vnops.c` 实现 FreeBSD `vop_vector`，不照搬 Linux - `inode_operations`、folio 或 iomap 接口 -4. **错误约定**：内核入口返回正的 FreeBSD errno；Linux 负 errno 或 - `ERR_PTR` 仅作为算法对照，不能机械移植 -5. **压缩后端**：BSD 调度器跨编译单元调用 `internal.h` 中的简单后端 API； - Linux 使用 `struct z_erofs_decompressor` 和不同的内存/页面生命周期 -6. **LZ4 文件职责**：BSD 保留独立 `decompressor_lz4.c` 有界解码器；Linux LZ4 路径位于 - `decompressor.c` 并依赖 Linux 内核 LZ4/page API -7. **平台特性**：Linux `sysfs.c`、`fileio.c`、`fscache.c`、`ishare.c` 和 - `zutil.c` 没有无条件对应物，不为文件外观引入空包装 -8. **构建架构**：当前只验证 FreeBSD 15 amd64，Makefile 明确拒绝其他 - `MACHINE_ARCH` - -### 保持一致的部分 - -- 静态目录 helper 使用 Linux 名称 `find_target_dirent` -- LZMA、DEFLATE、ZSTD 后端使用 Linux 文件名 `decompressor_*.c` -- Makefile 先列 metadata/VFS 文件，再列压缩调度、映射和后端文件 -- 跨文件后端声明集中在 `internal.h`，不在调用方手写 `extern` -- `erofs_fs.h` 的磁盘格式定义和核心目录/映射算法按 Linux 语义核对 - -## 代码规范 - -### 命名约定 -- 公共函数：`erofs__` -- 静态函数：描述性名称，无固定前缀 -- 宏：`EROFS_*` 全大写 -- 结构体：`struct erofs_*` - -### 函数排序（每个文件） -1. 辅助函数（static） -2. 核心逻辑函数 -3. VFS 接口函数 -4. 模块注册/清理（仅 super.c） - -### 错误处理 -```c -int erofs_function(...) -{ - int error = 0; - void *buf = NULL; - - // 操作... - if (条件) { - error = EINVAL; - goto fail; - } - - // ... - return 0; - -fail: - if (buf) - erofs_brelse(buf); - return error; -} -``` - -## 测试策略 - -### 单元测试 - -不存在受支持的用户态单元测试入口。旧的 `src/Makefile.test`、 -`tests/test_decompress.c` 和 `tests/test_decompress_standalone.c` 尝试把内核解压 -源码按不匹配的用户态 ABI 链接；这些入口均已退役并删除，不能作为可运行测试或 -feature 验证证据。 - -旧的 `test_super.c` 和 `test_inode.c` 只重复了测试文件中的公式，并未调用 -内核生产解析路径，其中 inode harness 还引用过已删除的磁盘字段。它们已退役， -不得作为 feature 验证证据。superblock、inode、pager 和错误路径必须使用 -`TC*.md` 中的确定性镜像，经 FreeBSD 内核模块实际挂载或访问验证。 - -### 集成测试 - -仓库根目录遗留的 `test_all_decompress.sh` 和 `test_chunk_based.sh` 包含其他 -repo 的硬编码路径，不能作为 repo22 的测试入口或验收证据。受支持的验证方式是 -直接执行 `tests/TC*.md` 中记录的 FreeBSD 15 内核步骤，并将命令、errno、哈希和 -清理状态写入 `tests/results/manual/` 下的日期报告。 - -### VM 测试 -- 挂载真实镜像 -- 文件读取验证 -- 性能基准测试 - -## 维护指南 - -### 同步上游 Linux 变更 - -1. 选定明确的 Linux `fs/erofs` 快照；当前工作区基线为 - `/work/dev-src-linux/fs/erofs` -2. 逐文件识别修改，不假设两个实现共享提交历史 -3. 检查是否为磁盘格式变更（`erofs_fs.h`）或 Linux 专属 VFS/page API -4. 只移植语义上适用的算法，并保留 FreeBSD errno、锁、GEOM 和 vnode 约定 -5. 运行双配置构建、模块加载和真实镜像挂载测试 - -### 添加新特性 - -1. 在 `erofs_fs.h` 添加磁盘格式定义 -2. 在 `internal.h` 添加内存结构 -3. 实现解析逻辑（data.c/inode.c） -4. 添加确定性 fixture 和对应的 `TC*.md` 内核测试 -5. 更新文档 - -## 性能考虑 - -- **零拷贝**：直接从缓冲区缓存读取 -- **延迟加载**：仅在需要时读取 inode 元数据 -- **缓存友好**：利用 FreeBSD 的 vnode 缓存 -- **批量操作**：目录读取一次性处理多个条目 - -## 已知限制 - -- 不支持写操作（只读文件系统） -- 不支持 FUSE 模式 -- 构建和运行时验证目前仅覆盖 FreeBSD 15 amd64 -- 不实现 Linux file-backed、fscache、page-cache sharing 或 sysfs 控制面 diff --git a/docs/capabilities.md b/docs/capabilities.md deleted file mode 100644 index 6c9cf40..0000000 --- a/docs/capabilities.md +++ /dev/null @@ -1,65 +0,0 @@ -# repo-pre-2 Capability Record - -## Status Vocabulary - -This document separates inherited pre1 claims from fresh pre2 validation: - -- `INHERITED CLAIM`: documented by the unchanged pre1 snapshot; not re-tested - for pre2. -- `STATIC GATE`: visible in tracked source or build declarations; runtime - behavior was not exercised. -- `NOT IMPLEMENTED`: explicitly excluded by inherited project documentation. -- `NOT CLAIMED`: inherited documentation deliberately limits the support - claim. -- `NOT RUN`: no build or runtime validation was performed for repo-pre-2. - -`INHERITED CLAIM` is provenance, not a new PASS result. Existing reports under -`tests/results/manual/` describe earlier pre1 work and must not be cited as a -fresh repo-pre-2 execution. - -## Snapshot Evidence - -```text -repo-pre-1 source tree: 6a5a1a49d4a4e08f6ef1155de240500e63a15365 -repo-pre-2 initial tree: 6a5a1a49d4a4e08f6ef1155de240500e63a15365 -src-linux reference tree: b79b9a8b62f633e887a8b99075ff9412fabd7f83 -runtime validation: NOT RUN -``` - -The inherited claims below are transcribed at category level from `README.md` -and `docs/features.md`. They are not an independent feature review. - -## Capability Matrix - -| Area | Inherited pre1 claim or static declaration | Pre2 status | -| --- | --- | --- | -| Mount and metadata | Read-only mount/unmount, `statfs`, superblock checksum and bounds, compact/extended inodes, 48-bit fields | INHERITED CLAIM; NOT RUN | -| Plain data | Flat plain, flat inline, tail bounds, and logical block accounting | INHERITED CLAIM; NOT RUN | -| Chunk and devices | Chunk indexes, holes, device tables, explicit devices, and flatdev mapping | INHERITED CLAIM; NOT RUN | -| Compressed data | LZ4, MicroLZMA, DEFLATE, full/compact indexes, partial references, HEAD2, ztailpacking, and qualified fragment forms | INHERITED CLAIM; NOT RUN | -| ZSTD | Source is always listed; `WITH_ZSTDIO=1` adds `ZSTDIO`, while the default is `0` | STATIC GATE; runtime NOT RUN | -| Directories and vnode operations | Lookup, readdir, cookies, namecache, access, readlink, pathconf, pager integration, and read-only mutation rejection | INHERITED CLAIM; NOT RUN | -| Xattrs and ACLs | Inline/shared namespaces, metabox storage, long/packed prefixes, and POSIX ACL reads | INHERITED CLAIM; NOT RUN | -| NFS export | 64-bit NIDs, generation handling, and malformed/stale handle validation | INHERITED CLAIM; NOT RUN | -| Build target | `src/Makefile` rejects architectures other than `amd64` | STATIC GATE; build NOT RUN | -| Write support | Writable filesystem operations | NOT IMPLEMENTED | -| `VOP_BMAP` | Explicitly unsupported in inherited documentation | NOT IMPLEMENTED | -| Complete future-format parity | All future incompat features and every dedupe encoding | NOT CLAIMED | -| Explicit compressed-extent positive payload | Inherited documentation records only partial coverage | NOT CLAIMED; NOT RUN | -| Performance guarantees | Manual observations are not a performance contract | NOT CLAIMED | - -## Interpretation Rules - -1. Source presence, feature-bit definitions, or build selection do not prove a - runtime capability. -2. A pre1 manual report does not become a pre2 PASS merely because the initial - trees match. -3. Phase 1 mechanical edits must not expand or reduce this capability matrix. -4. A future status change requires recorded evidence from an actually executed - validation step. - -## Current Validation Declaration - -No build, QEMU execution, test script, smoke test, mount, malformed-image -probe, or performance measurement was run for this initialization. The current -repo-pre-2 runtime validation result is therefore `NOT RUN` in every category. diff --git a/docs/erofs.5 b/docs/erofs.5 deleted file mode 100644 index 55f47fd..0000000 --- a/docs/erofs.5 +++ /dev/null @@ -1,271 +0,0 @@ -.\" Copyright (c) 2026 -.\" All rights reserved. -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted provided that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -.\" SUCH DAMAGE. -.\" -.Dd August 9, 2026 -.Dt EROFS 5 -.Os -.Sh NAME -.Nm erofs -.Nd Enhanced Read-Only File System -.Sh SYNOPSIS -To mount an -.Nm -volume: -.Bd -literal -offset indent -mount -t erofs /dev/da0 /mnt -.Ed -.Sh DESCRIPTION -The -.Nm -driver provides read-only support for the Enhanced Read-Only File System -(EROFS), a modern compressed read-only filesystem designed for space -efficiency and performance. -EROFS is widely used in Linux distributions and mobile systems for root -filesystems, firmware images, and container layers. -.Pp -The -.Fx -implementation supports multiple compression algorithms, various data layouts, -extended attributes, and multi-device configurations. -.Pp -The currently qualified build target is -.Fx 15 -on -.Sy amd64 . -The module Makefile rejects other architectures because they have not been -validated against this implementation's kernel ABI. -.Sh FEATURES -.Ss Inode Types -The -.Nm -driver supports both compact and extended inode formats: -.Bl -bullet -compact -.It -Compact inodes (32 bytes) for typical files -.It -Extended inodes (64 bytes) with extended metadata -.It -Special handling for single-link files -.It -Inline data (tailpacking) for small files -.El -.Ss Data Layouts -.Bl -bullet -compact -.It -.Sy FLAT_PLAIN : -Uncompressed contiguous data -.It -.Sy FLAT_INLINE : -Uncompressed data with inline tail -.It -.Sy Chunk-based : -Fixed-size chunks for multi-device support -.It -.Sy Compressed : -LZ4, DEFLATE, zstd, or LZMA compressed data with pcluster mapping -.El -.Ss Compression Algorithms -.Bl -tag -width "MicroLZMA" -.It Sy LZ4 -Fast decompression for general-purpose use, including ztailpacking -(compressed tail in inode metadata) -.It Sy DEFLATE -Standard compression with good compression ratio -.It Sy zstd -High compression ratio when the module is built with -.Va WITH_ZSTDIO=1 . -ZSTD support is disabled by default -.Pq Va WITH_ZSTDIO=0 , -and an enabled module requires a kernel built with -.Cd "options ZSTDIO" . -.It Sy LZMA/MicroLZMA -Maximum compression ratio for space-constrained environments -.El -.Ss Extended Attributes -.Bl -bullet -compact -.It -Shared extended attributes with metabox container support -.It -Inline extended attributes -.It -Long-prefix extended attribute support -.It -Packed prefix table support -.It -POSIX ACL support (access and default ACLs) -.El -.Pp -User namespace attributes are exposed without prefix; system namespace -attributes retain their full qualified names (trusted.*, security.*). -.Pp -Linux POSIX access/default ACL xattrs are decoded into -.Fx -POSIX.1e ACLs when present. -ACL and xattr mutation remains read-only. -.Ss Advanced Features -.Bl -bullet -compact -.It -Superblock CRC32C verification -.It -48-bit block count and root nid support -.It -Device table for multi-device volumes -.It -Qualified fragment-backed compressed files and metabox carriers -.It -VFS hash integration and namecache support -.It -Directory entry optimization (dot_omitted handling) -.It -NFS export with stable superblock-seeded per-inode file-handle generations -.It -.Fx -local vnode pager support for read-only mappings -.El -.Sh MOUNT OPTIONS -The -.Nm -filesystem supports standard read-only mount options and the following -filesystem-specific option: -.Bl -tag -width "device.N=/dev/mdN" -.It Cm device.N= Ns Pa path -Map one-based on-disk external device slot -.Ar N -to the disk provider at -.Ar path . -The slot number is part of the option name, so option order has no effect. -Every declared external slot must be supplied exactly once when external blob -providers are used. -Reusing the primary provider or one external provider for -multiple slots is rejected. -.El -.Pp -There is no filesystem-specific -.Pa /sbin/mount_erofs -utility in this repository. -Use the generic -.Xr mount 8 -frontend with -.Fl t Cm erofs ; -it passes the filesystem-specific option names to -.Xr nmount 2 . -.Pp -If the image has a device table and no -.Cm device.N -options are supplied, the primary provider is treated as a flatdev image. -It -must contain every declared device range at its on-disk unified block address. -The driver forces every successful mount read-only. -An explicit -.Fl o Cm rw -request therefore still produces a read-only mount; it does not enable writes -and is not rejected solely because -.Cm rw -was requested. -Mutating vnode operations fail with -.Er EROFS . -.Sh EXAMPLES -Mount an EROFS image from a disk device: -.Bd -literal -offset indent -mount -t erofs -o ro /dev/da0s1 /mnt -.Ed -.Pp -Mount an EROFS image from a regular file using -.Xr mdconfig 8 : -.Bd -literal -offset indent -mdconfig -a -t vnode -f rootfs.img -u 0 -mount -t erofs -o ro /dev/md0 /mnt -.Ed -.Pp -Mount a split image with two external blob slots. -The deliberately reversed -option order demonstrates that slot mapping is deterministic: -.Bd -literal -offset indent -mdconfig -a -t vnode -f primary.img -u 90 -mdconfig -a -t vnode -f blob1.img -u 91 -mdconfig -a -t vnode -f blob2.img -u 92 -mount -t erofs -o ro -o device.2=/dev/md92 \ - -o device.1=/dev/md91 /dev/md90 /mnt -.Ed -.Pp -Mount a flatdev image containing the primary image followed by all declared -unified device ranges: -.Bd -literal -offset indent -mdconfig -a -t vnode -f combined-flatdev.img -u 90 -mount -t erofs -o ro /dev/md90 /mnt -.Ed -.Pp -Unmount an EROFS filesystem: -.Bd -literal -offset indent -umount /mnt -.Ed -.Sh DIAGNOSTICS -Error messages are logged via -.Xr printf 9 -when filesystem inconsistencies are detected, such as: -.Bl -bullet -compact -.It -Invalid superblock magic number -.It -Superblock CRC32C checksum mismatch -.It -Unsupported compression algorithm -.It -Invalid inode format -.It -Invalid or unrepresentable inode timestamps -.It -Corrupted directory entries -.It -Compressed extent address arithmetic overflow -.It -Inline data crossing its inode metadata block or declared backing bounds -.It -Missing, short, or orphaned external providers -.It -Malformed or overlapping device-table ranges -.El -.Sh SEE ALSO -.Xr nmount 2 , -.Xr mdconfig 8 , -.Xr mount 8 , -.Xr umount 8 , -.Xr printf 9 -.Sh HISTORY -EROFS was originally developed for Linux by Huawei in 2019. -The -.Fx -implementation first appeared in 2026. -.Sh AUTHORS -.An Ruicheng Pan -.Sh BUGS -.Bl -bullet -compact -.It -The driver is read-only and does not claim support for every future EROFS -incompat feature or every dedupe encoding. -.It -There is no automated kernel regression harness; the repository records -.Fx 15 -manual-test procedures and results. -.El diff --git a/docs/features.md b/docs/features.md deleted file mode 100644 index 3a2fae3..0000000 --- a/docs/features.md +++ /dev/null @@ -1,77 +0,0 @@ -# EROFS Feature Status - -This list describes behavior implemented and manually qualified in the -FreeBSD 15 module. It is not a claim of complete Linux EROFS feature parity. - -## Filesystem and Metadata - -- Read-only mount/unmount and `statfs`. -- Superblock CRC32C, declared image/media bounds, 48-bit block/root-NID decode. -- Compact and extended inodes. -- Correct compact non-directory `I_NLINK_1` semantics and directory - `dot_omitted` semantics. -- Linux `new_decode_dev` major/minor decoding followed by FreeBSD `makedev()`. -- Superblock-seeded, inode-metadata-derived vnode/NFS generation synchronized - with `va_gen`; metadata-identical payload replacement is outside the stale - handle guarantee. -- NFS export, 64-bit NID file handles, stale-generation and malformed-FID - validation. - -## Data Layouts - -- `FLAT_PLAIN` and `FLAT_INLINE` reads. -- Inline tails constrained to the inode metadata block and declared backing - bounds. -- Chunk-based files, chunk indexes, holes, device tables, explicit devices, - and flatdev mapping. -- LZ4, MicroLZMA, DEFLATE, and ZSTD compressed reads. -- Full and compact indexes, partial references, HEAD2/interlaced records, - ztailpacking, and fragment-backed compressed data used by qualified images. -- Compressed `va_bytes`/`st_blocks` from the complete 48-bit on-disk compressed - block count; plain, inline, and chunk files retain logical block rounding. - -## Vnode and Directory Operations - -- `vget`, root lookup, `vfs_hash`, namecache, and `vn_vget_ino` integration. -- `lookup`, `readdir`, stable restart cookies, cold nested lookup, and - Linux-compatible nonzero final-name padding. -- Shared strict validation for dirent arrays, `nameoff`, block bounds, and - illegal names. -- `getattr`, access checks, `readlink`, `pathconf`, and read-only mutation - rejection, including combined special-vnode setattr requests. -- FreeBSD 15 `vnode_pager_local_getpages` and compatible async pager entry; - `VOP_BMAP` remains explicitly unsupported. - -## Xattrs and ACLs - -- Inline and shared `user.*`, `trusted.*`, and `security.*` attributes. -- Shared attributes in primary metadata and metabox containers. -- Long-prefix and packed-prefix-table lookup. -- FreeBSD user/system namespace exposure and POSIX access/default ACL reads. -- Read-only xattr/ACL mutation behavior. - -## Documentation and Validation - -- `erofs(5)` manual page. -- Markdown manual tests TC001-TC161, with unresolved cases tracked in - `issues/`. -- FreeBSD 15 manual reports with fixture hashes and cleanup evidence. -- Final explicit-extent ordering, 48-bit allocation, special setattr, - `OFF_MAX` directory-cookie, and build-tool failure checks. - -## Build Qualification - -- FreeBSD 15 `amd64` is the only currently validated and accepted target; - `src/Makefile` rejects other architectures. -- ZSTD support is disabled by default (`WITH_ZSTDIO=0`). Opt-in builds use - `WITH_ZSTDIO=1` and require a kernel built with `options ZSTDIO`. - -## Not Implemented or Not Claimed - -- Writable filesystem operations. -- A CI pipeline or automated kernel-test harness. -- Positive mapped-payload execution of the explicit compressed-extent format; - TC146 remains PARTIAL and the fixture limitation is tracked under `issues/`. -- Blanket support for every future EROFS incompat feature or every dedupe - encoding; only the explicitly qualified fragment/metabox forms are claimed. -- Performance guarantees from the manual throughput observations. diff --git a/docs/pre10-baseline.md b/docs/pre10-baseline.md deleted file mode 100644 index 4ece019..0000000 --- a/docs/pre10-baseline.md +++ /dev/null @@ -1,36 +0,0 @@ -# Pre10 Baseline - -Pre10 was created as an exact tracked-tree snapshot of `repo-pre-9` before -any Pre10 planning or source changes. - -## Source identity - -- Repository HEAD: `bedc8dae477ba0e82a4a7c4f0de7ccdd41696004` -- `repo-pre-9` tree: `54b2845b80c33f05109af1fea84e962aca3a82d3` -- `repo-pre-9/src` tree: `e657e8a63b097b061670f75ca01947a568ef33d5` - -## Copy verification - -Before this baseline file was added, `repo-pre-10` matched `repo-pre-9` in: - -- file and directory manifest; -- byte-for-byte file contents; -- file modes and timestamps; -- complete subtree tree hash; -- `src` subtree tree hash. - -The copied tree contained no nested `.git` directory, image, binary build -artifact, temporary file, or file larger than 10 MiB. - -## Pre9 validation reference - -The final Pre9 manual QEMU smoke validation is recorded in -[`pre9-cache-final-manual-validation.md`](pre9-cache-final-manual-validation.md). -It reports successful KLD build/load and read-only mounts, correct LZMA -single-read and full-file hashes, successful concurrent LZMA reads, a passing -LZ4 regression read, and complete test resource cleanup. - -## Pre10 state - -No Pre10 source, build, configuration, or planning change has been made. This -file only records the snapshot baseline. diff --git a/docs/pre10-batch-a.md b/docs/pre10-batch-a.md deleted file mode 100644 index a1659ce..0000000 --- a/docs/pre10-batch-a.md +++ /dev/null @@ -1,96 +0,0 @@ -# Pre10 Batch A: Compatibility Cleanup and Ordering - -## Scope and baseline - -Batch A changes only `src/internal.h`, `src/data.c`, and `src/zmap.c`. It is a -source-only structural change based on commit -`b5d6f5ce0696407f9f1cb3f0a9f8cbab643eea49`. No QEMU or feature test was run. - -The starting blobs were: - -| Path | Blob | -| --- | --- | -| `src/internal.h` | `cfa15de12b73319322ebb33049a7f14b9eb5c271` | -| `src/data.c` | `fdc138b0dad12501d842bae124c8c020793dbc97` | -| `src/zmap.c` | `51170741a0fb5eced814db898c1a233e8a23088a` | - -## Removed compatibility declarations - -Repository-wide fixed-string searches covered all files below -`repo-pre-10`, including source, the Makefile, and documentation. The only -non-definition match was an old review report describing -`erofs_is_fileio_mode()` as dead code. - -| Candidate | Source consumers | Result | -| --- | ---: | --- | -| `EROFS_SYNC_DECOMPRESS_AUTO` and related enum values | 0 | Removed | -| `EROFS_ZIP_CACHE_DISABLED` and related enum values | 0 | Removed | -| `struct erofs_buf` | 0 | Removed | -| `__EROFS_BUF_INITIALIZER` | 0 | Removed | -| `erofs_map_blocks.buf` | 0 | Removed with its dead type | -| `erofs_is_fileio_mode()` | 0 | Removed | - -These declarations modeled Linux facilities that the synchronous contiguous- -buffer FreeBSD path does not implement. Their removal does not replace or -alter the mount-owned decoded LZMA extent cache introduced in Pre9. - -## Declaration and definition order - -`internal.h` now exposes explicit groups for buffer/device I/O, inode/vnode -lifecycle, directory operations, logical mapping/file data, compressed -mapping/data, compression configuration/backends, and VOP vectors. Existing -signatures are unchanged. - -In `data.c`, the regular-file wrapper now precedes the symlink wrapper, matching -the header's file-data grouping. Both wrappers retain their original bodies -and continue to delegate to `erofs_read_uio()`. - -In `zmap.c`, `z_erofs_read_extent()` now follows the overflow-safe extent -position helpers it consumes. The Linux-shared compressed-index and map -functions retain their existing relative order; FreeBSD-only explicit-extent -helpers remain one contiguous responsibility group. - -The reordering was deliberately bounded by the existing static call topology: - -- `erofs_check_device_range()` remains before `erofs_map_dev()`; -- `erofs_inline_tail_start()` and `erofs_map_blocks_chunk()` remain before - `erofs_map_blocks()`; -- `erofs_bread_device()` remains before `erofs_bread()` and - `erofs_read_physical()`; -- `erofs_read_uio()` remains before both public vnode read wrappers; -- compressed-index decode/load helpers remain before their callers; -- `z_erofs_map_blocks_fo()` and `z_erofs_validate_extent_table()` remain - before `z_erofs_fill_inode()`; and -- fill, full/explicit mapping, and sanity helpers remain before - `z_erofs_map_blocks_iter()`. - -No extra static prototype was introduced. Broader movement of -`erofs_map_dev()` was rejected in this batch because matching Linux's exact -position would either violate the current dependency order or add a prototype -solely to support cosmetic movement. - -## Preserved behavior - -- No function signature, return value, allocation flag, lock operation, - `bread`/`brelse` ownership, mapping flag, or control-flow statement changed. -- No structure field used by runtime code moved or changed. -- The Pre9 decoded extent cache fields, initialization/finalization calls, - key, lock, and one-entry-per-mount policy are untouched. -- No Linux page, folio, bio, workqueue, XArray, shrinker, or compatibility - wrapper was introduced. - -## Static validation - -| Check | Result | Evidence | -| --- | --- | --- | -| Removed-symbol search | PASS | Every removed enum, type, initializer, helper, and `map_blocks.buf` access has zero source/Makefile matches. | -| Prototype uniqueness | PASS | Each of the 25 grouped cross-file declarations occurs exactly once in `internal.h`. | -| `erofs_read_file()` body | PASS | Parent and working-tree normalized hashes are both `acc4cb05f9a1eb4a0066f8e4d6f3e84c8a69cfb70c762233a61ab08e84836aef`. | -| `erofs_readlink_target()` body | PASS | Parent and working-tree normalized hashes are both `73cc37f21f0c66a2c6bb27d39cc9f93f38df1994c7d386dacb5efc7734aa6a89`. | -| `z_erofs_read_extent()` body | PASS | Parent and working-tree normalized hashes are both `9145b1220329a2d113c69971c94537a99467a7252f3ea564b695a4530ea8e11d`. | -| Changed-path allowlist | PASS | Only the three source files and this Batch A report changed. | -| `git diff --check` | PASS | No whitespace errors. | -| QEMU/build | NOT RUN | Reserved for final aggregate Pre10 validation. | - -The final Pre10 guest build and smoke run are intentionally deferred to the -aggregate validation stage defined by `planning/pre10/validation.md`. diff --git a/docs/pre10-batch-b.md b/docs/pre10-batch-b.md deleted file mode 100644 index ea532be..0000000 --- a/docs/pre10-batch-b.md +++ /dev/null @@ -1,108 +0,0 @@ -# Pre10 Batch B: Bounded Metadata Helpers - -## Scope and baseline - -Batch B is a source-only extraction based on commit -`3c7c774b296a2ee90578a78f372aad5fd2edbec6`. It changes only -`src/super.c`, `src/data.c`, and `src/inode.c`, plus this report. No build, -QEMU smoke, or feature test was run. - -The extraction keeps ownership at the original FreeBSD lifecycle boundaries: - -- `erofs_mountfs()` still owns mount allocation, the primary GEOM transfer, - extent-cache lifecycle, device scanning, internal inode setup, publication, - and the single `erofs_sb_free()` failure path. -- `erofs_map_dev()` still owns device selection, range and overflow checks, - flat-device behavior, and all `ENODEV`/`EINTEGRITY` returns. -- `erofs_vget()` still owns vnode allocation, locking, mount association, - hash insertion, construction state, publication, and failure cleanup. - -## Superblock helper - -`erofs_read_superblock()` now reads the on-disk superblock into a temporary -buffer, copies the fixed 144-byte structure to caller-owned stack storage, -releases the temporary buffer, and performs the existing validation and field -decode in the original order. It also retains the existing device-size, -checksum, generation-seed, metabox-NID, and compression-configuration checks. - -The helper does not allocate or publish the mount, open extra devices, create -internal inodes, initialize xattrs, or own failure cleanup. `erofs_mountfs()` -still calls, in order: - -```text -z_erofs_extent_cache_init -erofs_read_superblock -erofs_scan_devices -shared-EA/metabox combination check -erofs_init_packed_inode -erofs_init_metabox_inode -erofs_xattr_prefixes_init -mount publication and flag setup -``` - -All prior superblock outcomes remain at the same semantic boundary: - -| Condition | Preserved result | -| --- | --- | -| read failure | underlying `erofs_bread()` error | -| invalid magic or block geometry | `EINVAL` | -| unsupported directory blocks or feature bits | `EOPNOTSUPP` | -| invalid xattr, timestamp, or metabox metadata | `EINTEGRITY` | -| invalid device geometry | existing `EINVAL`, `EINTEGRITY`, or `ENXIO` | -| checksum mismatch | `EINTEGRITY` | -| generation/config read failure | existing underlying error | - -Copying the superblock before releasing its read buffer makes its lifetime -explicit. The copy remains available to `erofs_scan_devices()` and volume-name -publication, while checksum and compression configuration reads continue to -use their existing independent I/O helpers. - -## Device-map helper - -`erofs_fill_map_dev()` is a pure assignment helper for `m_em`, `m_dif`, and -`m_pa`. It has no branches, errors, allocation, I/O, or ownership effects. - -The caller retains all explicit-device and unified-range selection, arithmetic -overflow checks, physical range validation, flat-device handling, and external -device availability checks. In particular: - -- flat explicit mappings still use the primary device and add the unified - offset to the existing physical address; -- flat implicit mappings still retain the primary device and unified address; -- non-flat explicit mappings select the requested extra device only after all - validation succeeds; and -- non-flat implicit misses still return success with the initial primary map. - -## Vnode helper - -`erofs_fill_vnode()` sets only the inode-derived vnode type, FIFO operation -vector, and `VV_ROOT` flag. It runs after `erofs_read_inode()` succeeds and -immediately before `VSTATE_CONSTRUCTED`, while `erofs_vget()` still holds the -exclusive vnode lock. - -Hash lookup/insertion, allocation, `insmntque()`, locking, race handling, -failure `vgone()`/`vput()`, shared-lock downgrade, and `*vpp` publication remain -in `erofs_vget()` in their original order. - -## Known pre-existing risk - -The `insmntque()` failure branch remains unchanged. FreeBSD's `insmntque()` may -reclaim and release the vnode on failure, while the existing EROFS branch then -accesses `vp->v_data`. This possible use-after-release is outside a helper-only -batch and was deliberately not mixed into this commit. Consequently, this -report does not claim that the pre-existing vnode cleanup path is correct. - -## Static validation - -| Check | Result | Evidence | -| --- | --- | --- | -| Changed-path allowlist | PASS | Only three allowed source files and this report changed. | -| Helper ownership | PASS | Each helper is `static` and has one direct caller. | -| Mount lifecycle order | PASS | Cache init, device scan, internal inode/xattr setup, publication, and `erofs_sb_free()` remain in `erofs_mountfs()`. | -| Device-map errors | PASS | All range, overflow, device-selection, `ENODEV`, and `EINTEGRITY` branches remain in `erofs_map_dev()`. | -| Vnode lifecycle order | PASS | `insmntque()`, hash insertion, inode read, construction, downgrade, and publication retain their order. | -| `git diff --check` | PASS | No whitespace errors. | -| Build/QEMU/feature test | NOT RUN | Reserved for later aggregate Pre10 validation. | - -The pre-existing `insmntque()` concern is deferred; no new errno, logging, -ABI, Linux lifecycle facade, or Pre9 decoded-cache change was introduced. diff --git a/docs/pre10-batch-c.md b/docs/pre10-batch-c.md deleted file mode 100644 index 28cd3f0..0000000 --- a/docs/pre10-batch-c.md +++ /dev/null @@ -1,216 +0,0 @@ -# Pre10 Batch C: BSD Decompressor Request Dispatch - -Status: **STATIC PASS; RUNTIME NOT RUN** - -This batch aligns the BSD decompressor boundary with the Linux source shape -where the responsibility is equivalent. It keeps the FreeBSD implementation -synchronous and contiguous-buffer based. No page, folio, bio, workqueue, -XArray, shrinker, or asynchronous decompression abstraction was added. - -## Baseline and Scope - -The implementation starts from Pre10 commit `d3c1cb90` and preserves Batch A, -Batch B, and the documented `insmntque()` lifecycle issue. The source changes -are limited to: - -```text -repo-pre-10/src/internal.h -repo-pre-10/src/decompressor.c -repo-pre-10/src/lz4.c -repo-pre-10/src/decompressor_lzma.c -repo-pre-10/src/decompressor_deflate.c -repo-pre-10/src/decompressor_zstd.c -``` - -`zdata.c` was inspected but did not need a mechanical edit: its existing call -to `z_erofs_decompress()` remains the sole dispatcher call, and its buffer -release and decoded LZMA extent-cache publication remain unchanged. - -## Request Contract - -The new request is stack-owned by `z_erofs_decompress()` and is valid only for -the synchronous callback invocation: - -```c -struct z_erofs_decompress_req { - struct erofs_mount *em; - const struct erofs_map_blocks *map; - const void *in; - size_t inputsize; - void *out; - size_t outputsize; - bool partial_decoding; -}; -``` - -Field mapping is direct: - -| Request field | Previous source | Ownership | -| --- | --- | --- | -| `em` | `em` / codec configuration arguments | Borrowed mount, never retained | -| `map` | `map` and its algorithm/offset fields | Borrowed mapping, never retained | -| `in` | `src` after padding removal | Borrowed compressed buffer | -| `inputsize` | `srclen` after padding removal | Value copy | -| `out` | `dst` | Borrowed decoded buffer | -| `outputsize` | `dstlen` | Value copy | -| `partial_decoding` | `partial` | Value copy | - -The request does not own either data buffer. `zdata.c` continues to release -the compressed buffer with `erofs_brelse()`, and continues to free or publish -the decoded allocation after the callback returns. - -The descriptor callback is intentionally synchronous: - -```c -struct z_erofs_decompressor { - const char *name; - int (*config)(struct erofs_mount *, - const struct erofs_super_block *, const void *, size_t); - int (*decompress)(const struct z_erofs_decompress_req *); -}; -``` - -The configuration callback receives the superblock argument because the LZ4 -legacy configuration path uses it. The other codec configuration callbacks -retain their previous inputs and explicitly ignore that additional argument. - -## Descriptor Coverage - -`decompressor.c` contains one static array indexed by the on-disk algorithm -number. The entries are: - -```text -0 LZ4 config + z_erofs_lz4_decompress -1 LZMA config + z_erofs_lzma_decompress -2 DEFLATE config + z_erofs_deflate_decompress -3 ZSTD config + z_erofs_zstd_decompress -4 SHIFTED plain transform callback -5 INTERLACED plain transform callback -``` - -The first four entries cover every real EROFS compression algorithm currently -defined by `erofs_fs.h`. The two runtime-only entries cover the existing plain -mapping forms. No duplicate backend entry point or compatibility wrapper is -present. - -Configuration parsing preserves the old ordering: - -1. Read the configuration record and its payload. -2. Return the read error immediately if either read fails. -3. Select the descriptor and invoke its configuration callback. -4. Release the payload with `erofs_brelse()`. -5. Return the callback's original error unchanged. - -This retains the existing I/O-error priority. Unknown on-disk algorithm bits -are rejected before configuration reads by the existing -`Z_EROFS_ALL_COMPR_ALGS` mask check. A supported algorithm whose configuration -is unavailable returns `EOPNOTSUPP` from its existing callback. In particular, -the no-`ZSTDIO` build still emits the existing mount error and returns -`EOPNOTSUPP`. - -## Decode and Error Semantics - -The old and new paths have the following equivalent behavior: - -| Condition | Result before Batch C | Result after Batch C | -| --- | --- | --- | -| Shifted/interlaced output larger than input | `EINTEGRITY` | `EINTEGRITY` | -| Shifted/interlaced transform success | `0` | `0` | -| Invalid algorithm format | `EOPNOTSUPP` | `EOPNOTSUPP` | -| Required zero-padding absent | `EINTEGRITY` | `EINTEGRITY` | -| LZMA dictionary not configured | `EINTEGRITY` | `EINTEGRITY` | -| Backend success | `0` | `0` | -| Any backend failure | `EIO` | `EIO` | - -Padding removal remains in the dispatcher and is performed before the backend -request is updated. The LZ4 zero-padding exception remains unchanged. The LZMA -dictionary check remains before callback dispatch. Backend-specific checks are -unchanged apart from reading their previous parameters from the request or -mount configuration: - -- LZ4 preserves literal/match bounds, overlap copying, partial completion, - and trailing-zero validation. -- MicroLZMA preserves input/output size limits, dictionary selection, decoder - shutdown, full-stream consumption, and partial output acceptance. -- DEFLATE preserves window validation, `inflateEnd()`, no-progress detection, - output completion, and full-stream input consumption. -- ZSTD preserves window selection, decoder destruction on every initialized - path, no-progress detection, output completion, and full-stream consumption. - -## Consumer and Symbol Proof - -Before the change, repository searches found exactly one in-tree consumer of -each old backend symbol: the switch in `decompressor.c`. The only consumer of -`z_erofs_decompress()` is `zdata.c`. After the change: - -```text -old lz4_decompress 0 definitions/references -old lzma_decompress 0 definitions/references -old deflate_decompress 0 definitions/references -old zstd_decompress 0 definitions/references - -new LZ4 backend one definition, one descriptor reference, one prototype -new LZMA backend one definition, one descriptor reference, one prototype -new DEFLATE backend one definition, one descriptor reference, one prototype -new ZSTD backend two definitions for #ifdef/#else, one descriptor reference, - one prototype -``` - -The two ZSTD definitions are mutually exclusive build branches, not duplicate -runtime implementations. No old-name wrapper was retained because no real -consumer remains. - -## Cache and Ownership Review - -The Pre9 decoded LZMA cache policy is untouched. The request callback returns -before the cache code runs, so the following remain owned by `zdata.c`: - -- compressed-buffer release; -- decoded allocation cleanup on failure; -- decoded extent publication; -- duplicate-cache replacement and old-entry freeing; -- one-entry-per-mount bound and cache lock lifecycle. - -No callback stores the request pointer or either borrowed buffer after return. - -## Static Checks - -The following checks were run before this report was written: - -```sh -git diff --check -git diff --name-only | sort -grep -RInE '(^|[^_])(lz4_decompress|lzma_decompress|deflate_decompress|zstd_decompress)\\(' repo-pre-10/src -grep -RInE 'z_erofs_(lz4|lzma|deflate|zstd)_decompress' repo-pre-10/src -grep -RInE '\\b(page|folio|bio|workqueue|xarray|shrinker)\\b' \\ - repo-pre-10/src/internal.h repo-pre-10/src/decompressor.c \\ - repo-pre-10/src/lz4.c repo-pre-10/src/decompressor_lzma.c \\ - repo-pre-10/src/decompressor_deflate.c repo-pre-10/src/decompressor_zstd.c \\ - repo-pre-10/src/zdata.c -``` - -Results: - -- `git diff --check`: PASS. -- Changed paths: only the six allowed source files: PASS. -- Old backend symbol search: no matches: PASS. -- New backend definitions and descriptor references: PASS. -- Forbidden Linux memory-model concepts in the changed codec boundary: no - matches: PASS. -- Descriptor entries: six unique designated entries covering algorithms 0-5: - PASS. -- `zdata.c` cache and ownership diff: unchanged: PASS. - -The host is not a FreeBSD build environment, so this batch does not claim a -KLD build. Per the Pre10 instruction, QEMU smoke testing and the full feature -matrix were not run in this batch. - -## Concerns and Deferred Validation - -The primary remaining validation is a FreeBSD guest KLD build followed by the -planned Pre10 smoke run. That runtime work is intentionally separate from this -static implementation batch. Full feature tests remain outside Pre10. - -The existing `erofs_vget()` `insmntque()` failure-path P1 remains documented -in `repo-pre-10/issues/erofs-vget-insmntque-failure-use-after-release.md` and -was not changed here. diff --git a/docs/pre10-completion-and-smoke.md b/docs/pre10-completion-and-smoke.md deleted file mode 100644 index 23f46a3..0000000 --- a/docs/pre10-completion-and-smoke.md +++ /dev/null @@ -1,347 +0,0 @@ -# Pre10 Completion and Targeted Smoke Report - -Status: **PRE10 IMPLEMENTATION COMPLETE; TARGETED SMOKE PASS; FULL FEATURE TEST NOT RUN** - -Date: 2026-08-13 - -## Scope and goals - -Pre10 is a controlled maintenance-alignment release based on the final Pre9 -tree. Its purpose is to make the FreeBSD EROFS implementation easier to compare -and maintain alongside the independent Linux EROFS repository without copying -Linux-only lifecycle or memory-management models. - -The planned work was limited to three source batches: - -1. Remove unused Linux-shaped compatibility declarations and improve source - ordering. -2. Extract bounded helpers where Linux and FreeBSD responsibilities are - comparable, while retaining FreeBSD ownership and errno behavior. -3. Introduce a BSD-native synchronous decompressor request and descriptor - boundary with Linux-comparable codec names. - -Full feature validation, CI execution, performance redesign, Linux page/folio/ -bio/workqueue integration, and new format support were outside Pre10. This -report does not claim that every EROFS feature was tested. - -## Repository history and identities - -The relevant pushed commits are: - -| Commit | Purpose | -| --- | --- | -| `cdfcd79640f252830c4e507f65aa7387eaa98ae0` | Snapshot `repo-pre-10` from the current `repo-pre-9`. | -| `b5d6f5ce0696407f9f1cb3f0a9f8cbab643eea49` | Add the Pre10 execution plan under `planning/pre10/`. | -| `3c7c774b296a2ee90578a78f372aad5fd2edbec6` | Batch A compatibility cleanup and ordering. | -| `8c246ea3c4dbf94b92d1f27a9d1b6b6eda582c31` | Batch B bounded metadata helper extraction. | -| `d3c1cb90b21ae8a97f549bc256d120268e5d09e9` | Record the discovered `insmntque()` vnode lifecycle issue. | -| `5c8a47e916b1521cb8a35ec4f546e5e16fb8c2dc` | Batch C decompressor request and descriptor alignment. | -| `9fbf0d7e24084d2342ecd61eff21592b1a83d225` | Fix `insmntque()` failure ownership. | -| `ff6586c2b24e155a3cbdf55b2ed80edb36d3e406` | Clarify the issue status after the source fix. This is the commit tested by the final smoke run. | - -The snapshot report records the Pre9 source tree inherited by Pre10 as -`e657e8a63b097b061670f75ca01947a568ef33d5`. The final smoke runner independently -recorded these tested identities: - -```text -commit=ff6586c2b24e155a3cbdf55b2ed80edb36d3e406 -repo_pre_10_tree=3387e32efbcb0bec7e0bb57ecfc42daf9b962d28 -src_tree=ec684d8dbd7662070234da1fe947aba48a293db8 -``` - -Evidence: `/work/tests-dev/temp/pre10-smoke-final-20260813T102543Z/dut-identities.txt`. - -## Batch A: compatibility cleanup and ordering - -Batch A changed `src/internal.h`, `src/data.c`, and `src/zmap.c`. - -Repository-wide consumer searches proved that the following declarations did -not represent implemented FreeBSD behavior and had no source, Makefile, ABI, -initializer, or field-access consumer: - -- `EROFS_SYNC_DECOMPRESS_*`; -- `EROFS_ZIP_CACHE_*`; -- `struct erofs_buf` and `__EROFS_BUF_INITIALIZER`; -- the unused `erofs_map_blocks.buf` member; and -- `erofs_is_fileio_mode()`. - -Removing these declarations improves maintainability because the FreeBSD tree -no longer advertises Linux mechanisms that it does not implement. The real -FreeBSD synchronous contiguous-buffer path and the Pre9 mount-owned decoded -LZMA extent cache remain explicit and unchanged. - -Declarations and existing function definitions were grouped by actual -responsibility, closer to the Linux source's readable organization where the -responsibilities match. Static call topology was preserved, and moved function -bodies were compared for equivalence. No return value, allocation flag, lock, -mapping flag, `bread`/`brelse` ownership rule, or runtime structure field was -changed. - -Detailed evidence: `repo-pre-10/docs/pre10-batch-a.md`. - -## Batch B: bounded FreeBSD helper boundaries - -Batch B changed `src/super.c`, `src/data.c`, and `src/inode.c`. - -It extracted three `static` helpers with one direct caller each: - -- `erofs_read_superblock()` isolates superblock read, fixed-size copy, - validation, field decode, checksum, generation seed, and compression - configuration parsing. `erofs_mountfs()` still owns mount allocation, GEOM - transfer, device scanning, internal inode setup, publication, and the single - mount cleanup path. -- `erofs_fill_map_dev()` performs only the final `m_em`, `m_dif`, and `m_pa` - assignments. Device selection, range checks, overflow checks, flat-device - semantics, and `ENODEV`/`EINTEGRITY` decisions remain in `erofs_map_dev()`. -- `erofs_fill_vnode()` performs only inode-derived vnode field setup. FreeBSD - vnode allocation, locking, mount association, hash insertion, race handling, - construction state, downgrade, publication, and cleanup remain in - `erofs_vget()`. - -These boundaries improve Linux/FreeBSD visual comparability at the function -responsibility level while deliberately retaining the FreeBSD GEOM, vnode, -hash, lock, and cleanup contracts. Linux lifecycle names such as `iget` or -`fill_super` were not adopted where their semantics would be misleading. - -Detailed evidence: `repo-pre-10/docs/pre10-batch-b.md`. - -## Batch C: BSD decompressor request dispatch - -Batch C changed `src/internal.h`, `src/decompressor.c`, `src/lz4.c`, -`src/decompressor_lzma.c`, `src/decompressor_deflate.c`, and -`src/decompressor_zstd.c`. - -It introduced a stack-owned, synchronous, contiguous-buffer -`z_erofs_decompress_req` and a descriptor table covering LZ4, LZMA, DEFLATE, -ZSTD, SHIFTED, and INTERLACED. Codec entry points now use the Linux-comparable -names `z_erofs_lz4_decompress`, `z_erofs_lzma_decompress`, -`z_erofs_deflate_decompress`, and `z_erofs_zstd_decompress` because their -functional roles are equivalent. - -The interface remains BSD-native: - -- no page, folio, bio, workqueue, XArray, shrinker, asynchronous request, or - Linux ownership wrapper was introduced; -- request fields borrow the existing compressed and decoded buffers only for - the synchronous callback; -- `zdata.c` still owns buffer release and LZMA decoded-cache publication; -- plain SHIFTED/INTERLACED transforms remain in the existing dispatch model; -- LZ4 padding behavior, partial-decoding source, LZMA dictionary validation, - ZSTD conditional availability, and codec resource release remain unchanged; -- configuration I/O errors preserve priority and backend failures remain - normalized to the existing `EIO` result. - -This gives third-party maintainers a recognizable request/descriptor and codec -naming shape without importing Linux kernel runtime assumptions. - -Detailed evidence: `repo-pre-10/docs/pre10-batch-c.md`. - -## P1 vnode lifecycle finding and fix - -Independent Batch B review found a pre-existing P1 ownership violation in -`erofs_vget()`: after `insmntque()` returned an error, the old branch wrote -through `vp->v_data` even though FreeBSD may already have executed `vgone()` -and `vput()` and ended caller ownership of that vnode. - -Commit `9fbf0d7e24084d2342ecd61eff21592b1a83d225` applied the minimal ownership -fix. The error branch now frees only the independently allocated `en`, clears -`*vpp`, returns the original error, and performs no further access, unlock, -reclaim, or release operation on `vp`. The successful path, vnode hash race, -lock state, errno, and later reclaim path were not changed. - -Static ownership review confirms that the kernel failure path clears -`v_data`, installs `dead_vnodeops`, and owns vnode cleanup; dead vnode reclaim -does not release the filesystem's separately allocated `en`, so exactly one -caller-side `free(en, M_EROFS)` remains necessary. - -The source defect is fixed, but runtime closure is still pending. Closing the -issue requires a dedicated mount/unmount race test that repeatedly creates or -looks up uncached vnodes while normal and forced unmount are attempted, with -kernel diagnostics capable of detecting stale vnode access, memory corruption, -lock misuse, and double release. The ordinary smoke test below does not trigger -or prove this teardown race. - -Issue record: -`repo-pre-10/issues/erofs-vget-insmntque-failure-use-after-release.md`. - -## Static review result - -The final static review accepted Batches A, B, and C and the P1 fix. Checks -included changed-path allowlists, zero-consumer proof, prototype and backend -symbol uniqueness, descriptor coverage and bounds, normalized moved-function -body comparison, cleanup/ownership matrices, errno preservation, mount and -vnode lifecycle ordering, cache ownership, forbidden Linux mechanism searches, -conflict-marker searches, clean-worktree checks, `git diff --check`, and local -`HEAD`/`xdm/main` agreement at each pushed boundary. - -No new P0 or P1 defect was found in the three planned batches. Static review is -not runtime proof. In particular, it does not close the dedicated vnode -teardown race test or the older explicit-extent positive-fixture gap recorded -under `repo-pre-10/issues/`. - -## Initial inconclusive smoke attempts - -Two isolated attempts preceded the final PASS. Neither is treated as a DUT -failure or a PASS. - -### Attempt 1 - -Evidence directory: -`/work/tests-dev/temp/pre10-smoke-20260813T100326Z`. - -The readiness probe began before the QEMU process had actually started, then -performed repeated SSH handshakes while the TCG guest was still becoming -responsive. Sources and fixtures were eventually transferred and the build log -reached the final module link/strip stage, but the long SSH build session ended -with transport status `255`. No reliable KLD SHA, `kldload`, or `kldstat` -evidence was produced. - -The first runner incorrectly continued into the plain case after the failed -build/load stage. The guest reported `Invalid fstype`, while `results.txt` -incorrectly recorded `mount_status=0` because the exit status was not captured -immediately after `mount`. Since module loading was never proven and the -runner's status recording was unsound, the mount result is not attributable to -Pre10 and the attempt is **INCONCLUSIVE**, not FAIL or PASS. - -### Attempt 2 - -Evidence directory: -`/work/tests-dev/temp/pre10-smoke-retry-20260813T101136Z`. - -The fresh guest booted and returned a valid FreeBSD identity after intermittent -connection refusals, banner timeouts, and resets. The orchestration then exited -almost immediately. The claimed SCP failure was not backed by an actual -`scp-images.log`; only a failed attempt to tail that missing file remained. -No build, module load, mount, or hash result exists. This attempt is also -**INCONCLUSIVE** and provides no DUT verdict. - -Both attempts stopped only their owned QEMU process, released ports `10040` -and `10041`, deleted their disposable overlays, preserved the immutable base, -and left guard PID `26318` and port `9222` unchanged. - -The infrastructure diagnosis was recorded at -`/work/tests-dev/fix-todo/pre10-smoke-ssh-banner-timeout.md`. Its original -`PENDING_FIX` verdict describes those two attempts; the corrected final runner -below subsequently completed the required targeted smoke. - -## Corrected fail-fast runner - -The successful run used a new directory and port `10042`, a fresh disposable -overlay, 4096 MiB and two TCG CPUs, and a strictly ordered runner: - -1. Verify exact Git, base-image, fixture, guard, and port identities before - starting QEMU. -2. Start QEMU and record its owned PID before beginning readiness checks. -3. Wait for TCP, require two consecutive successful FreeBSD SSH identity - checks, and establish one persistent SSH ControlMaster connection. -4. Transfer the source archive, guest scripts, and all fixtures in one batch, - then verify fixture hashes in the guest. -5. Run the guest build as a background job that writes `build.rc` and a - completion sentinel; poll the sentinel rather than depending on one long - SSH session. -6. Stop immediately unless the build return is zero, `erofs.ko` exists, its - hash is recorded, `kldload` succeeds, and `kldstat -n erofs` proves the - module is loaded. -7. Run each filesystem case separately and immediately record attach, mount, - mount-proof, hash, unmount, and detach return codes. A missing proof or - nonzero result stops the runner and cannot be converted to PASS. -8. Review guest cleanup and `dmesg`, unload the module, and prove it is absent. -9. Use the cleanup trap to stop only the recorded QEMU PID, release only the - run-owned port and overlay, and revalidate the immutable base and guard VM. - -Runner evidence: -`/work/tests-dev/temp/pre10-smoke-final-20260813T102543Z/runner.sh` and -`runner.log`. - -## Final targeted smoke result - -Final evidence directory: -`/work/tests-dev/temp/pre10-smoke-final-20260813T102543Z`. - -Overall result: **PASS** for the targeted Pre10 smoke scope. - -### Isolation - -- Immutable base image mode before and after: `0444`. -- Immutable base image SHA-256 before and after: - `67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef`. -- The `.bp` was not booted or modified directly; the run used a disposable - qcow2 overlay. -- Guard PID: `26318`; guard SSH port: `9222`. -- Guard command identity and `/proc` start value `1102996924` matched before - and after, and port `9222` remained reachable. - -Evidence: `base-before.txt`, `cleanup-status.txt`, `guard-before.txt`, -`guard-command-before.txt`, `guard-command-after.txt`, -`guard-start-before.txt`, and `guard-start-after.txt` in the final evidence -directory. - -### KLD build, load, and unload - -- FreeBSD guest build return: `0`. -- Built KLD SHA-256: - `4ce2d44a0902502a40f07805606485c99869371b3cbab1c70ee19a8ac29d90f1`. -- The loaded module hash matched the built module hash. -- `kldstat` showed `erofs.ko` loaded as module ID `5`. -- `kldunload` succeeded, and the final `kldstat -n erofs` correctly reported - that no such loaded module remained. - -Evidence: `build-result.out`, `build-result.rc`, `module-load.out`, -`module-load.rc`, and `guest-evidence.tar.gz`. - -### Filesystem cases - -| Case | Mounted file SHA-256 | Time | Result | -| --- | --- | ---: | --- | -| Plain `/data.txt` | `056f8f7585667dc695e2edf936deaf84cfee0f88671ba6cd5be22ce890763433` | `0s` | PASS | -| LZ4 `/compressed.bin` | `3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880` | `5s` | PASS | -| LZMA full `/level.dat` | `ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461` | `2s` | PASS | - -Each case recorded zero returns for md attach, EROFS mount, mount proof, -full-file hash, unmount, and md detach. The exact fixture image hashes were: - -```text -plain image 3785ca07e7bd16f6c611191596ae0314253c0ae9b7217a4b22de25799b0f08ac -LZ4 image 967cc1b625546f9f9f881472e71cc3d849c65feb4e98e67fbfdc9b90a4be6dda -LZMA image 32107a084b27362a093768b88746c37c2e99a74b9f1301a9d4046988479defe9 -``` - -Evidence: `fixture-host-sha256.txt`, `results.txt`, and the per-case files in -`guest-evidence.tar.gz`. - -### Kernel messages and cleanup - -The post-test `dmesg` delta and suspect scan were empty: no EROFS mount, -integrity, decompression, panic, or assertion error was found. After all cases, -only the guest root filesystem and `devfs` remained mounted, no test md device -remained, and the EROFS module was unloaded. - -Host cleanup also passed: - -- the run-owned QEMU process stopped; -- port `10042` was released; -- the disposable overlay was deleted; -- the base image mode and hash were unchanged; and -- the guard VM remained unchanged and reachable. - -Evidence: `dmesg-delta.txt`, `dmesg-suspect.txt`, `final-guest-state.out`, -`cleanup-status.txt`, and `guest-evidence.tar.gz`. - -## Validation limits and conclusion - -Pre10's planned source changes are complete, independently statically reviewed, -committed, pushed, built in a FreeBSD guest, and covered by the targeted plain, -LZ4, and LZMA smoke run. That targeted scope passed at tested commit -`ff6586c2b24e155a3cbdf55b2ed80edb36d3e406`. - -No full feature test and no `/work/tests-dev/test_all.py` run was performed. -This report therefore does not claim all features, codecs, mapping layouts, -multi-device behavior, partial references, tailpacking, memory pressure, or -concurrent teardown behavior passed. - -A feature-specific test is still required to close the fixed `insmntque()` -issue's runtime-validation status: the dedicated normal/forced mount-unmount -race with uncached vnode creation and kernel diagnostics described above. It -is desirable and required for issue closure, but it was not run because the -instruction for Pre10 was smoke testing only and explicitly excluded feature -testing. diff --git a/docs/pre11-baseline.md b/docs/pre11-baseline.md deleted file mode 100644 index 8bcc7f4..0000000 --- a/docs/pre11-baseline.md +++ /dev/null @@ -1,35 +0,0 @@ -# Pre11 Baseline - -Pre11 was created as an exact snapshot copy of `repo-pre-10` before any -Pre11 planning or implementation changes. - -## Source identity - -- Repository source commit: `c191517f3799291852f8673e90b0f1be8427d3a9` -- `repo-pre-10` tree: `53d5c5fadc2d1fabf83346eadadec44c97c85c10` -- `repo-pre-10/src` tree: `ec684d8dbd7662070234da1fe947aba48a293db8` - -## Destination identity before this document - -- `repo-pre-11` tree: `53d5c5fadc2d1fabf83346eadadec44c97c85c10` -- `repo-pre-11/src` tree: `ec684d8dbd7662070234da1fe947aba48a293db8` - -## Exact copy verification - -Before this baseline document was added, `repo-pre-10` and `repo-pre-11` -were verified identical by: - -- recursive byte-for-byte content comparison; -- complete file and directory manifest comparison; -- file type, mode, and symbolic-link target comparison; -- identical complete subtree tree IDs; -- identical `src` subtree tree IDs. - -The snapshot contains the same 307 tracked files as `repo-pre-10`. No nested -Git repository, external untracked file, generated image, temporary artifact, -or file larger than 10 MiB was introduced by the copy. - -## Pre11 state - -No implementation, build, configuration, or behavior change has occurred. -This file only records the Pre11 snapshot baseline. diff --git a/docs/pre11-batch-a.md b/docs/pre11-batch-a.md deleted file mode 100644 index 6c621fd..0000000 --- a/docs/pre11-batch-a.md +++ /dev/null @@ -1,10 +0,0 @@ -# Pre11 Batch A - -This batch performs two mechanical vocabulary alignments: - -- `struct erofs_mount.block_bits` is renamed to `blkszbits`. -- `erofs_fill_map_dev()` is renamed to `erofs_fill_from_devinfo()`. - -No types, expressions, control flow, ownership rules, error handling, or runtime behavior were changed. Semantic type alias changes were deferred because they were not needed for this pure mechanical batch. - -Validation is limited to identifier occurrence checks, diff inspection, and `git diff --check`. Build and runtime testing are deferred to the later Pre11 validation stage. diff --git a/docs/pre11-batch-b.md b/docs/pre11-batch-b.md deleted file mode 100644 index 3a1f238..0000000 --- a/docs/pre11-batch-b.md +++ /dev/null @@ -1,49 +0,0 @@ -# Pre11 Batch B: Vnode Operation Scaffolding - -## Scope - -This batch is a behavior-preserving cleanup limited to vnode operation -scaffolding: - -- removed the unused `EROFS_MOUNT_XATTR_USER` option bit; -- removed the zero-consumer `erofs_node.vnode` field and its only assignment; -- grouped VOP forward declarations and `erofs_vnodeops` initializers by - responsibility. - -No function definitions, VOP registrations, lock or reclaim behavior, -`vnode_create_vobject()` error handling, or FIFO operations were changed. - -## Consumer Audit - -Before the edit, `EROFS_MOUNT_XATTR_USER` appeared only at its definition. -The `erofs_node.vnode` member appeared only in the structure definition and in -the single assignment performed by `erofs_vget()`; it had no read, address, ABI, -initializer, or offset consumer. - -After the edit, both removed symbols have zero source-tree matches. - -## VOP Vector Equivalence - -Before reordering: - -- `erofs_vnodeops`: 24 raw entries, 24 unique normalized `field=value` entries; -- `erofs_fifoops`: 14 raw entries, 14 unique normalized `field=value` entries. - -After reordering: - -- `erofs_vnodeops`: 24 raw entries, 24 unique entries, with the sorted - normalized set identical to the before snapshot; -- `erofs_fifoops`: 14 raw entries, 14 unique entries, with both the sorted set - and original entry sequence identical to the before snapshot. - -## Validation - -The batch uses static checks only: - -- normalized before/after VOP vector set comparison; -- raw entry count versus unique entry count; -- removed-symbol zero-consumer search; -- changed-path whitelist; -- `git diff --check` and conflict-marker scan. - -No build, QEMU smoke test, or feature test was run for this mechanical batch. diff --git a/docs/pre11-batch-c.md b/docs/pre11-batch-c.md deleted file mode 100644 index 30bbab5..0000000 --- a/docs/pre11-batch-c.md +++ /dev/null @@ -1,99 +0,0 @@ -# Pre11 Batch C: ACL Parsing Responsibilities - -## Scope - -This batch mechanically separates POSIX ACL wire-format parsing from the -FreeBSD vnode and xattr acquisition path. - -Modified files: - -- `src/xattr.c` -- `docs/pre11-batch-c.md` - -No xattr body loading, header-only xattr behavior, namespace handling, public -VOP interface, build configuration, or test infrastructure was changed. - -## Responsibility Split - -`erofs_get_acl()` continues to own: - -- mount option and ACL type validation; -- the default-ACL directory restriction; -- no-ACL filter handling and mode fallback; -- xattr namespace and name selection; -- both `erofs_getxattr()` stages; -- FreeBSD `uio` and `iovec` setup; -- `ENOATTR` fallback and the stack input buffer. - -The new `erofs_posix_acl_from_xattr()` receives an already-read buffer and -owns only wire-format validation and ACL population. It borrows both the input -buffer and output ACL and performs no allocation, release, lookup, or vnode -operation. - -`erofs_acl_from_mode()` now receives the inode mode directly. This preserves -the original fallback writes while preventing the parser from depending on an -`erofs_node`. - -## Moved-Code Mapping - -The parsing block formerly at the end of `erofs_get_acl()` was moved in the -same statement order into `erofs_posix_acl_from_xattr()`: - -1. Header length and entry-size remainder validation. -2. Header copy and version validation. -3. Entry count calculation and maximum check. -4. Zero-entry mode fallback. -5. `acl_cnt` assignment and phase initialization. -6. Permission validation before tag-phase validation. -7. Tag phase transitions and `UINT32_MAX` rules. -8. Duplicate named user and group ID checks. -9. ACL entry writes and undefined-ID mapping. -10. The short-circuit `phase != 6 || acl_posix1e_check(aclp) != 0` check. - -No ACL zeroing or additional field initialization was added. Partial ACL -mutation on malformed input remains possible exactly as before. - -## Branch Matrix - -| Input or state | Preserved result | -| --- | --- | -| POSIX ACL mount option disabled | `EOPNOTSUPP` in caller | -| Unsupported ACL type | `EINVAL` in caller | -| Default ACL requested for non-directory | `EINVAL` in caller | -| No-ACL filter match | Mode-derived ACL in caller | -| ACL xattr absent | Mode-derived ACL in caller | -| Xattr lookup/read failure | Original error from caller | -| Oversized xattr or nonzero UIO residual | `EINTEGRITY` in caller | -| Short or misaligned wire value | `EINTEGRITY` in parser | -| Unsupported wire version | `EINTEGRITY` in parser | -| Entry count above limit | `EINTEGRITY` in parser | -| Zero entries | Mode-derived ACL in parser | -| Invalid permission, phase, tag, or ID | `EINTEGRITY` in parser | -| Duplicate named user/group ID | `EINTEGRITY` in parser | -| Missing required mask or incomplete phase | `EINTEGRITY` in parser | -| Valid ACL | Identical ACL entry population and success | - -## Resource And Error Audit - -- The input buffer remains stack-owned by `erofs_get_acl()`. -- The parser does not allocate, free, or retain pointers. -- There are no new cleanup paths. -- Xattr errors, UIO residual errors, wire errors, and ACL validation errors - retain their previous ordering. -- `erofs_xattr_load_body()` and its release paths were not modified. -- The optional header validation helper was intentionally omitted to avoid - touching the known header-only behavior. - -## Static Verification - -The batch is validated with: - -- a strict two-file path whitelist; -- unique definitions of `erofs_get_acl()`, `erofs_acl_from_mode()`, and - `erofs_posix_acl_from_xattr()`; -- pre/post comparison of parser returns, phase transitions, ACL writes, and - duplicate-ID checks; -- `git diff --check`; -- confirmation that `erofs_xattr_load_body()` is absent from the source diff. - -No build, QEMU smoke test, or feature test was run for this mechanical split. diff --git a/docs/pre11-batch-d.md b/docs/pre11-batch-d.md deleted file mode 100644 index 944cbf8..0000000 --- a/docs/pre11-batch-d.md +++ /dev/null @@ -1,18 +0,0 @@ -# Pre11 Batch D - -## Scope - -- Rename `src/lz4.c` to `src/decompressor_lz4.c`. -- Update the source list and current architecture references to the new filename. - -## Equivalence - -- The pre-rename `lz4.c` blob and renamed `decompressor_lz4.c` blob are identical. -- Byte comparison confirms the renamed source content is unchanged. -- Rename detection reports a 100% rename. -- LZ4 definitions, declarations, descriptor references, and `lz4_finish` call counts are unchanged. - -## Behavior and Testing - -- This batch changes filenames and documentation only; it makes no behavior changes. -- No build, QEMU run, feature test, or other test was run. diff --git a/docs/pre11-batch-e.md b/docs/pre11-batch-e.md deleted file mode 100644 index a4fcd64..0000000 --- a/docs/pre11-batch-e.md +++ /dev/null @@ -1,68 +0,0 @@ -# Pre11 Batch E: Mount Device Argument Cleanup - -## Scope - -This batch consolidates caller-owned `device.N` argument cleanup in -`erofs_mount()` and removes two impossible allocation checks. It does not -change mount option semantics, GEOM ownership, device slot validation, mount -error text, or any filesystem behavior. - -Modified files: - -- `src/super.c` -- `docs/pre11-batch-e.md` - -## Changes - -After `erofs_parse_device_options()` succeeds, every exit from -`erofs_mount()` now reaches one `out_args` label. Parse failure still returns -directly because no caller-owned argument array has been transferred. - -The operation order remains: - -1. `erofs_parse_device_options()` -2. `vfs_filteropt()` -3. `vfs_getopt()` -4. `erofs_open_device()` -5. `erofs_update_iosize_max()` -6. `erofs_mountfs()` -7. `erofs_free_device_args()` -8. `vfs_mountedfrom()` -9. `erofs_statfs()` - -`out_args` only frees copied device arguments. It does not inspect or release -`primary`. The existing `erofs_mountfs()` ownership transfer and all GEOM -cleanup paths remain unchanged. - -The checks for `NULL` immediately following allocation of `packed_inode` and -`metabox_en` were removed. Both allocations retain `M_WAITOK | M_ZERO`, so a -successful return cannot produce `NULL`. No other allocation checks or flags -were changed. - -## Ownership Paths - -| Path | Copied arguments | Primary device | Result | -| --- | --- | --- | --- | -| Parse failure | No caller ownership | Not opened | Return parser error directly | -| Option filter failure | Freed at `out_args` | Not opened | Return `EINVAL` | -| Invalid or missing `from` | Freed at `out_args` | Not opened | Return `EINVAL` | -| Primary open failure | Freed at `out_args` | Open helper retains its existing cleanup contract | Return open error | -| Mount setup failure | Freed at `out_args` | Ownership already transferred at `erofs_mountfs()` entry | Return mount error | -| Success | Freed at `out_args` | Owned by the mounted filesystem | Publish mounted-from and run `statfs()` | - -## Static Verification - -- `erofs_free_device_args()` has one caller after the change. -- `erofs_mountfs()` and `erofs_sb_free()` are unchanged. -- The relative order of option filtering, option access, primary open, I/O - sizing, mount setup, argument cleanup, mounted-from publication, and statfs - is unchanged. -- Existing errno values and mount error strings are unchanged. -- Duplicate and missing external-device slot behavior is unchanged. -- `git diff --check` passes. -- The diff is restricted to the two declared Batch E files. - -## Validation Status - -No build, QEMU smoke test, or feature test was run for this batch. Runtime -validation is deferred to the final Pre11 build and targeted smoke test. diff --git a/docs/pre11-completion-and-smoke.md b/docs/pre11-completion-and-smoke.md deleted file mode 100644 index 5832a41..0000000 --- a/docs/pre11-completion-and-smoke.md +++ /dev/null @@ -1,480 +0,0 @@ -# Pre11 Completion and Smoke Report - -## Status - -Pre11 is complete for its approved mechanical maintenance scope. - -- Final implementation commit tested: `040fc69025c5031436205c3acd421be1d95fe86b` -- Final `repo-pre-11` tree: `f0cca92670c76ab13cc5b9353d5aeb993effa6e2` -- Final `repo-pre-11/src` tree: `c4aac4d69cb25a4b049239d0c1f422dcb89465d6` -- Static review: `ACCEPT`, with no P0-P3 findings in the included changes -- FreeBSD KLD build: `PASS`, using `WITH_ZSTDIO=0` -- Targeted plain/LZ4/LZMA QEMU smoke: `PASS` -- Full feature test and `test_all.py`: `NOT RUN` - -This report does not claim full feature validation. Pre11 deliberately contains -mechanical naming, responsibility, dead-scaffolding, file-layout, and cleanup -changes that could be reviewed for behavioral equivalence and then covered by a -focused build and smoke run. Behavior changes requiring purpose-built fixtures -remain deferred. - -## Snapshot and Planning - -Pre11 was created from the completed Pre10 tree in: - -```text -0483dcb41c2a980badb829ff4d17361bd1fec930 -snapshot: create repo-pre-11 from repo-pre-10 -``` - -At the snapshot boundary, `repo-pre-10/src` and `repo-pre-11/src` were both: - -```text -ec684d8dbd7662070234da1fe947aba48a293db8 -``` - -The executable Pre11 plan was added in: - -```text -9ad1fa0cf8ba251b574da033bf0c9fad83acaa38 -docs: add pre11 execution plan -``` - -The original plan included several candidates that would have changed visible -filesystem behavior. An independent boundary review narrowed the release in: - -```text -c89c228ac1817cfd0337905b39247245eaa67f3b -docs: narrow pre11 execution boundaries -``` - -The narrowed scope retained only work that could preserve control flow, errno, -on-disk interpretation, vnode and GEOM ownership, iterator order, decompressor -semantics, and cache lifetime. The following candidates were removed from -implementation because honest validation would require targeted feature tests: - -- root vnode type rejection; -- zero-length mapping behavior in the UIO reader; -- header-only xattr semantics; -- duplicate or missing `device.N` behavior; -- direct propagation of `vnode_create_vobject()` errors; -- broad map, inode, directory, and name-lookup interface reshaping. - -This narrowing allowed Pre11 to remain a comparatively large maintenance -release without mixing low-risk structural alignment with unvalidated semantic -changes. - -## Batch A: Type and Vocabulary Alignment - -Commit: - -```text -b4e38c9fb15df127470d70bc144404f1f3afa2af -pre11: align mount and device helper vocabulary -``` - -Exact changes: - -- renamed `struct erofs_mount.block_bits` to `blkszbits` across its consumers; -- renamed the private `erofs_fill_map_dev()` helper to - `erofs_fill_from_devinfo()`; -- added the Batch A implementation report. - -Linux-maintenance benefit: - -- `blkszbits` matches established EROFS vocabulary and makes comparisons with - Linux mount geometry code more direct; -- `erofs_fill_from_devinfo()` describes the same responsibility as the Linux - helper without importing Linux block-device objects into FreeBSD. - -BSD invariants retained: - -- field type, width, units, expressions, overflow handling, and structure - layout were unchanged; -- the helper remained a private FreeBSD device-map field filler; -- GEOM provider selection, range checks, flat-device behavior, errno, and - physical I/O remained unchanged; -- no broad or search-driven integer type replacement was performed. - -Static checks confirmed 48 intended `block_bits` replacements, zero remaining -old identifiers, four helper-name replacements, zero stale helper references, -and a source diff containing only the approved identifier changes. - -## Batch B: Vnode Operation Scaffolding - -Commit: - -```text -67625d3afebb9142e69b19afd08cdfc1d2adaac5 -refactor: clean up vnode operation scaffolding -``` - -Exact changes: - -- removed the unused `EROFS_MOUNT_XATTR_USER` option bit; -- removed the zero-consumer `erofs_node.vnode` member and its only assignment; -- grouped VOP declarations and `erofs_vnodeops` initializers by responsibility; -- added the Batch B implementation report. - -Linux-maintenance benefit: - -- removes misleading compatibility-shaped state that had no consumer; -- presents vnode operations in a responsibility-oriented order that is easier - to compare with upstream filesystem responsibilities while retaining the - native FreeBSD operation table. - -BSD invariants retained: - -- no VOP function definition or registration changed; -- `erofs_vnodeops` retained 24 unique `field=value` entries; -- `erofs_fifoops` retained 14 unique entries and its original sequence; -- `fifo_specops`, vnode locking, hash insertion, reclaim, pager behavior, and - `vnode_create_vobject()` errno translation were unchanged. - -The removed symbols had no reads, address consumers, initializer dependencies, -offset consumers, or external ABI use. - -## Batch C: ACL Parsing Responsibilities - -Commit: - -```text -a86b5cac4d193d131de8870bca4132fdc881f992 -erofs: split ACL parsing responsibilities -``` - -Exact changes: - -- extracted private `erofs_posix_acl_from_xattr()` from `erofs_get_acl()`; -- changed the private mode fallback helper to receive the inode mode directly; -- left xattr acquisition and all FreeBSD VOP/UIO responsibilities in the - caller; -- added the Batch C implementation report. - -Linux-maintenance benefit: - -- separates wire-format ACL parsing from filesystem acquisition and VOP glue, - matching the upstream responsibility split more closely; -- gives maintainers a bounded parser to compare without introducing Linux ACL - objects, RCU, xattr handlers, or inode lifecycle assumptions. - -BSD invariants retained: - -- mount-option checks, ACL type validation, default-ACL directory checks, - namespace selection, both `erofs_getxattr()` stages, UIO setup, and `ENOATTR` - fallback remain in `erofs_get_acl()`; -- parser validation order, positive errno, tag phases, duplicate ID checks, - `UINT32_MAX` handling, partial ACL writes, and final - `acl_posix1e_check()` ordering remain unchanged; -- input storage remains caller-owned stack memory, with no new allocation or - cleanup path; -- `erofs_xattr_load_body()` and the known header-only xattr behavior were not - changed. - -Static comparison confirmed equivalent branch, error, phase-transition, and -ACL-population behavior. - -## Batch D: LZ4 Backend File Responsibility - -Commit: - -```text -e631e33a1b8c053ebcb2d737af33e363cba48b34 -erofs: align LZ4 backend file responsibility -``` - -Exact changes: - -- renamed `src/lz4.c` to `src/decompressor_lz4.c`; -- updated the Makefile source entry; -- updated current architecture documentation references; -- added the Batch D implementation report. - -Linux-maintenance benefit: - -- makes the filename communicate that the translation unit is the LZ4 codec - backend beside `decompressor_lzma.c`, `decompressor_deflate.c`, and - `decompressor_zstd.c`; -- reduces unnecessary file-layout differences when maintainers inspect codec - implementations across the independent Linux and FreeBSD repositories. - -BSD invariants retained: - -- Git identified a 100% rename; -- the old and new source blobs are both - `2cc19f5eb0c283e12ba0a3a4334b8817a1686888`; -- function bodies, request ABI, descriptor registration, input/output bounds, - partial decode behavior, and error normalization were unchanged; -- historical reports and historical test evidence were not rewritten. - -## Batch E: Mount Device Argument Cleanup - -Commit: - -```text -040fc69025c5031436205c3acd421be1d95fe86b -erofs: consolidate mount device argument cleanup -``` - -Exact changes: - -- consolidated caller-owned copied `device.N` argument release into one - `out_args` exit in `erofs_mount()`; -- removed two impossible NULL checks immediately following - `M_WAITOK | M_ZERO` allocations; -- added the Batch E implementation report. - -Linux-maintenance benefit: - -- makes mount setup ownership and cleanup easier to audit by reducing repeated - release sites; -- removes defensive branches that contradict the FreeBSD `M_WAITOK` contract, - keeping BSD-specific allocation behavior explicit instead of emulating a - nullable Linux allocation path. - -BSD invariants retained: - -- operation order remains parse, filter, get option, open primary, update I/O - size, mount setup, free copied arguments, publish mounted-from, and statfs; -- parse failure still returns before caller ownership exists; -- `out_args` releases only copied arguments and never releases `primary`; -- `erofs_mountfs()` still takes ownership of `primary` at entry; -- `erofs_mountfs()`, `erofs_sb_free()`, reverse external-device cleanup, errno, - and mount error text are unchanged; -- duplicate and missing external-device slot behavior remains deferred. - -## Static Review - -Independent reviews were performed after the individual batches and again over -the complete source range from `0483dcb` through `040fc690`. - -The final review found no P0-P3 issue in the included Pre11 source changes and -accepted all five batches. It confirmed: - -- the source changes were confined to the approved Batch A-E paths; -- no deferred behavior fix entered the source; -- old identifiers had no stale consumers and new private helpers had unique - definitions and expected call counts; -- both VOP vectors retained the same registration mappings; -- ACL acquisition, parsing, fallback, errno, and partial-write behavior were - preserved; -- the LZ4 backend was a byte-identical rename and the Makefile contained the - new filename once; -- mount cleanup retained error order and GEOM ownership; -- uncompressed mapping interfaces, directory/name lookup, decompressor request - ABI, LZMA cache policy, disk structures, and Linux reference sources were not - changed; -- `git diff --check` passed and the implementation worktree was clean before - runtime validation. - -The final source delta from the snapshot contains 202 insertions and 188 -deletions across ten source paths, including the 100% LZ4 file rename. Much of -that textual delta is identifier replacement, declaration/vector reordering, -or moved ACL parser code rather than new behavior. - -## First Smoke Attempt: Infrastructure Failure - -Evidence directory: - -```text -/work/tests-dev/temp/pre11-smoke-final-20260813T122923Z -``` - -Result: `FAIL`, specifically an infrastructure failure before DUT transfer or -execution. - -The guest reached FreeBSD and two SSH identity probes succeeded, but the first -runner opened an SSH ControlMaster using a detached invocation. The open command -returned success and created the control socket, then the master terminated or -lost its connection before the immediate health check. The check failed with: - -```text -Control socket connect(.../control.sock): Connection refused -``` - -The test stopped before source transfer, KLD build, module load, or any EROFS -mount. Therefore this attempt is not a PASS, but it is also not evidence of a -Pre11 build or driver failure. Its source archive SHA was: - -```text -5bbffb101364f350fe1a3771b6f018bc10a68785d8c55cc2a186273a074e9182 -``` - -The owned QEMU stopped, port `10043` was released, its overlay was removed, the -base image remained unchanged, and the retained guard VM remained reachable. -The run's cleanup status was nonzero only because closing the already-dead -ControlMaster returned an error. - -## Corrected Smoke Retry - -Evidence directory: - -```text -/work/tests-dev/temp/pre11-smoke-retry-final-20260813T124214Z -``` - -Result: `PASS`. - -The retry used an SSH master process in normal foreground mode but launched it -in the runner shell background, without `ssh -f`. The runner captured the real -master PID, repeatedly required both `kill -0` and `ssh -O check` to succeed, -and refused to transfer sources unless the master remained alive. Long build -work ran inside the guest with a completion sentinel, so one long SSH command -was not treated as the build lifetime. - -Tested identities: - -```text -commit: 040fc69025c5031436205c3acd421be1d95fe86b -repo-pre-11 tree: f0cca92670c76ab13cc5b9353d5aeb993effa6e2 -src tree: c4aac4d69cb25a4b049239d0c1f422dcb89465d6 -source archive: 179b80365c26295770afab58f1209ae3fffd73182ada0bcf97ea027904f781e9 -guest: FreeBSD 15.0-RELEASE-p8 -``` - -The archive hashes differ between attempts because separately generated gzip -archives are not required to be byte-reproducible. Both identity files name the -same committed repository and source trees; only the passing retry is used as -runtime evidence. - -## Build and Module Lifecycle - -The committed `repo-pre-11` archive was transferred to the disposable guest, -validated there, extracted, and built with: - -```sh -make WITH_ZSTDIO=0 -``` - -Results: - -- guest build return code: `0`; -- `erofs.ko` SHA256: - `77ce41bff5cb7d7c934dd9dba20ba06c20b660ad15ebfc4210f86f6473dffbf7`; -- `kldload /root/pre11-smoke/erofs.ko`: success; -- `kldstat -n erofs`: success, reporting `erofs.ko` loaded; -- all smoke mounts and md providers were released; -- `kldunload erofs`: success; -- the post-unload `kldstat -n erofs` check confirmed the module was absent. - -`WITH_ZSTDIO=1` was not built and is not claimed as tested. - -## Smoke Cases - -Each case used an independently attached vnode-backed md provider, required a -successful EROFS mount to be visible in `mount -p`, computed the complete target -file hash, then unmounted and detached the provider before the next case. - -| Case | Target SHA256 | Elapsed | Provider | Result | -| --- | --- | ---: | --- | --- | -| Plain | `056f8f7585667dc695e2edf936deaf84cfee0f88671ba6cd5be22ce890763433` | 0 s | `md70` | PASS | -| LZ4 | `3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880` | 5 s | `md71` | PASS | -| LZMA | `ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461` | 2 s | `md72` | PASS | - -The LZMA case hashed the complete file. No timeout, mount failure, hash mismatch, -unmount failure, or md detach failure occurred. - -## Kernel Messages and Cleanup - -The runner captured `dmesg` before loading the module and after all smoke cases. -The delta contained no EROFS error, decompression failure, panic, or assertion; -the suspect-output file is empty. - -Cleanup evidence records: - -```text -qemu_stopped=1 -port_released=1 -overlay_removed=1 -base_mode_after=444 -base_sha_after=67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef -guard_unchanged_and_reachable=1 -body_rc=0 -cleanup_rc=0 -``` - -The test owned only port `10044`, its QEMU PID, its ControlMaster PID, and its -overlay. All were closed or removed. The permanent base image was never booted -directly and retained mode `0444` and its original SHA256. The retained guard VM -PID `26318` kept start time `1102996924`, an identical command line, and a -reachable forwarded SSH port at `9222`. - -## Feature-Test Decision - -No feature test and no `test_all.py` run was performed. - -That decision is appropriate for the included Pre11 scope because: - -- Batch A changed only private identifiers; -- Batch B removed proven dead state and reordered designated VOP initializers - without changing their mappings; -- Batch C mechanically moved parser statements while retaining inputs, outputs, - writes, error order, and ownership; -- Batch D was a byte-identical file rename; -- Batch E consolidated one caller-owned cleanup action and removed checks that - cannot be reached after `M_WAITOK` allocation. - -Static equivalence review plus the FreeBSD build and plain/LZ4/LZMA smoke is -sufficient evidence for these changes. It is not sufficient for the deferred -behavior changes below, and this report does not claim otherwise. - -## Deferred Behavior and Required Targeted Tests - -### Non-directory root - -The current root path does not add a new rejection for a root NID that resolves -to a non-directory vnode. A future fix requires a purpose-built image and checks -for mount errno, vnode release, mount error text, and complete cleanup. - -### Zero-length internal run - -The uncompressed UIO reader's existing `run_len == 0` behavior was not changed. -A future fix requires a corrupt mapping fixture that reaches a zero-length run -while the request remains inside the file, with exact comparison to the -contiguous-buffer path and expected `EINTEGRITY`. - -### Header-only xattr - -The existing behavior for an ibody containing only the xattr header was not -changed. Targeted coverage must distinguish zero and nonzero shared counts, -name filters, get/list results, ACL mode fallback, and corruption results. - -### Duplicate or missing device slots - -`device.N` slot parsing behavior and diagnostics were not changed. Future tests -must cover duplicate, out-of-order, missing, excessive, and malformed slots, -flat and non-flat device tables, exact errno and mount error text, and GEOM -cleanup. - -### `insmntque()` race closure - -The earlier ownership fix remains accepted by static lifecycle review but is -not runtime-closed by ordinary smoke. A dedicated mount/unmount or forced -unmount race must stress vnode creation and inspect panic, use-after-free, lock, -reference, and cleanup behavior. - -### Explicit mapped extent fixtures - -Positive payload fixtures are still unavailable for explicit mapped extent -records of 4, 8, 16, and 32 bytes. No restructuring or coverage claim should be -made until validated mappings and file hashes exist. - -### `vnode_create_vobject()` errno - -Pre11 intentionally retains the current conversion of a nonzero -`vnode_create_vobject()` result to `ENOMEM`. Direct propagation is a visible -behavior change and requires controlled pager failure or fault injection that -verifies the caller-visible errno and vnode state. - -## Final Conclusion - -Pre11 completed all five narrowed maintenance batches, passed independent -static review, built successfully as a FreeBSD KLD, and passed the required -plain, LZ4, and full-file LZMA smoke. The first smoke attempt was correctly -classified as an infrastructure failure before DUT transfer or build; the -corrected retry provides the runtime evidence. - -Pre11 improves cross-repository maintainability without replacing native -FreeBSD vnode, GEOM, UIO, pager, ACL, or synchronous decompression contracts. -Full feature validation remains outside this release and must not be inferred -from the targeted smoke result. diff --git a/docs/pre12-baseline.md b/docs/pre12-baseline.md deleted file mode 100644 index 7ae9b0a..0000000 --- a/docs/pre12-baseline.md +++ /dev/null @@ -1,11 +0,0 @@ -# Pre12 Baseline - -`repo-pre-12` was created as an exact snapshot copy of `repo-pre-11` before this document was added. - -- Source commit: `88fc67742e6c173e14538d4fcd556a45a385c2c9` -- Source tree (`repo-pre-11`): `5b8c637bcb535f330d894f30660bb5e65db70048` -- Copied tree before this document (`repo-pre-12`): `5b8c637bcb535f330d894f30660bb5e65db70048` -- Source `src` tree: `c4aac4d69cb25a4b049239d0c1f422dcb89465d6` -- Copied `src` tree: `c4aac4d69cb25a4b049239d0c1f422dcb89465d6` - -Copy verification used a recursive comparison that preserved and checked file modes and symbolic links. The complete tree IDs matched before this document was added. Adding this document does not modify the copied `src` tree. diff --git a/docs/pre12-batch-01.md b/docs/pre12-batch-01.md deleted file mode 100644 index 6737ddf..0000000 --- a/docs/pre12-batch-01.md +++ /dev/null @@ -1,29 +0,0 @@ -# Pre12 Batch 01 - -This batch aligns the planned in-memory NID, block-number, and byte-offset -declarations with the existing `erofs_nid_t`, `erofs_blk_t`, and -`erofs_off_t` aliases. - -The source whitelist was limited to eight fields in `internal.h`, the -`erofs_iloc`, `erofs_nid_is_valid`, `erofs_read_inode`, and `erofs_vfs_hash` -declarations and definitions, and the byte-offset parameters of -`erofs_bread`, `erofs_read_physical`, and `erofs_read_metadata`. - -All three aliases are direct `uint64_t` typedefs. Static assertions confirmed -that each is an unsigned, 8-byte type with the same alignment and exact C type -compatibility as `uint64_t`. Consequently every replaced field has identical -size and alignment, so both structure sizes, alignments, field order, and all -target `offsetof` values are unchanged. Function parameter and return ABI -representations are also unchanged, and the vnode hash still consumes the -same complete 8-byte NID object. - -The source diff is limited to the planned fields, function declarations and -definitions, and necessary local declaration splits. No expressions, field -order, on-disk types, control flow, error handling, or formatting outside -those declarations changed. - -Validation used exact declaration inventory checks, token-diff inspection, -declaration/definition signature comparison, arithmetic and hash review, -compile-time type assertions, and `git diff --check`. No planned point was -skipped and no out-of-scope candidate was changed. No build, QEMU, smoke, -feature, or other dynamic test was run. diff --git a/docs/pre12-batch-02.md b/docs/pre12-batch-02.md deleted file mode 100644 index ee40621..0000000 --- a/docs/pre12-batch-02.md +++ /dev/null @@ -1,31 +0,0 @@ -# Pre12 Batch 02 - -This batch replaces exactly three potentially unaligned typed loads with the -FreeBSD little-endian byte decoders planned for Pre12. - -- `xattr.c`: `hdrbuf` contains two raw metadata bytes returned by - `erofs_xattr_read_backing(..., sizeof(raw_len), ...)`; `le16dec(hdrbuf)` - decodes the prefix-record length. -- `inode.c`: `buf` contains at least one complete 32-byte compact on-disk inode - returned by `erofs_read_metadata`; its first two bytes are the little-endian - `i_format` field decoded by `le16dec(buf)`. -- `data.c`: the non-indexes branch sets `entry_size` to - `EROFS_BLOCK_MAP_ENTRY_SIZE == sizeof(__le32) == 4`; `idx` points to those - four raw block-map bytes and `le32dec(idx)` decodes the block address. - -For bytes `b0..b3`, both old expressions and the new helpers produce -`b0 | b1 << 8` or `b0 | b1 << 8 | b2 << 16 | b3 << 24`. On a little-endian -host the typed load already has that value; on a big-endian host `leXXtoh` -byte-swaps the typed value to that same formula. The helpers preserve this -result while avoiding any alignment requirement. - -`xattr.c` and `data.c` receive the helpers through `internal.h` to -`erofs_fs.h` to ``; `inode.c` also includes `` -directly. Existing `le16dec` and `le32dec` consumers confirm the established -include convention. Error checks, branches, value ranges, and buffer release -ordering are unchanged. No point was skipped and no other endian read was -changed. - -Validation used exact-expression counts, buffer-source and length inspection, -symbolic endian equivalence, source-diff inspection, and `git diff --check`. -No build, QEMU, smoke, feature, or other dynamic test was run. diff --git a/docs/pre12-batch-03.md b/docs/pre12-batch-03.md deleted file mode 100644 index 20b175e..0000000 --- a/docs/pre12-batch-03.md +++ /dev/null @@ -1,49 +0,0 @@ -# Pre12 Batch 03: POSIX ACL mode helper - -## Change - -- In `src/xattr.c`, `erofs_acl_from_mode` now obtains the owner, group, and - other permissions with `acl_posix1e_mode_to_perm`. -- The helper tags are `ACL_USER_OBJ`, `ACL_GROUP_OBJ`, and `ACL_OTHER`, matching - the adjacent `ae_tag` assignments. -- No other function lines, ACL entry ordering, `acl_cnt`, IDs, default ACL - handling, fallback behavior, flags, errno behavior, or control flow changed. - -## Helper confirmation - -- The local FreeBSD header `sys/sys/acl.h` declares - `acl_posix1e_mode_to_perm(acl_tag_t tag, mode_t mode)` for kernel consumers. -- The local FreeBSD implementation in `sys/kern/subr_acl_posix1e.c` maps - `ACL_USER_OBJ` from `S_IRWXU`, `ACL_GROUP_OBJ` from `S_IRWXG`, and - `ACL_OTHER` from `S_IRWXO`, returning the corresponding POSIX ACL read, - write, and execute permission bits. -- `src/xattr.c` already includes ``; no include change was needed. - -## Exhaustive equivalence - -A temporary standalone C program reproduced the local FreeBSD helper logic and -compared it with the three replaced expressions for every permission mode from -`0000` through `0777`: - -```text -equivalent=1536/1536 modes=512 tags=3 mismatches=0 -temporary_cleanup=ok -``` - -This covers 512 modes for each of the three tags. The temporary source and -executable were deleted and are not part of the repository. - -## Static validation - -- The `erofs_acl_from_mode` function is line-for-line unchanged except for the - three `ae_perm` expressions. -- The source diff changes only those three expressions; this document is the - only added file. -- `git diff --check` and `git diff --cached --check` completed without errors. -- Before commit, both unstaged status and staged status contained only - `src/xattr.c` and `docs/pre12-batch-03.md` under `repo-pre-12`. - -## Tests not run - -Per the Batch 03 execution constraints, no project build, QEMU run, smoke test, -or feature test was run. diff --git a/docs/pre12-batch-04.md b/docs/pre12-batch-04.md deleted file mode 100644 index 864456b..0000000 --- a/docs/pre12-batch-04.md +++ /dev/null @@ -1,35 +0,0 @@ -# Pre12 Batch 04: Private compression header - -## Change - -- Added `src/compress.h` for the decompressor request, descriptor, and backend - private declarations previously located in `src/internal.h`. -- Added the header only to `decompressor.c` and the LZ4, LZMA, Deflate, and - ZSTD backend translation units. -- Kept the runtime compression enum, `z_erofs_parse_cfgs`, and the public - `z_erofs_decompress` declaration in `src/internal.h`. -- Preserved every migrated declaration signature and storage duration. No - function body, callback, conditional compilation block, errno path, or - buffer ownership changed. - -## Static validation - -- The `compress.h` consumer set is exactly the five planned decompressor - translation units; `super.c`, `zdata.c`, and all other public callers still - include only `internal.h`. -- The include graph is acyclic: `compress.h` includes `internal.h`, while - `internal.h` does not include `compress.h`. -- Exact searches found one definition of each migrated structure and one - declaration of each migrated backend function. The old declarations are - absent from `internal.h`. -- The new header contains no Linux page, folio, bio, workqueue, XArray, - shrinker, compatibility, or related runtime declarations. -- Source diff inspection and declaration normalization confirmed that this is - a declaration-only move plus the five include additions. -- `git diff --check` and `git diff --cached --check` completed without errors. - -## Tests not run - -Per the execution constraints, no build, QEMU run, smoke test, feature test, -or other dynamic test was run. The planned `WITH_ZSTDIO=0` and -`WITH_ZSTDIO=1` KLD builds remain required during final validation. diff --git a/docs/pre12-batch-05.md b/docs/pre12-batch-05.md deleted file mode 100644 index 382ba05..0000000 --- a/docs/pre12-batch-05.md +++ /dev/null @@ -1,31 +0,0 @@ -# Pre12 Batch 05: ZSTD window lower bound - -## Change - -- Removed only the `rq->em->zstd_windowlog + 10 < 10` subcondition from - `z_erofs_zstd_decompress`. -- Preserved the `> 20` upper bound, the `outputsize == 0` check, every - `return (-1)`, all ZSTD calls, resource release, and the complete disabled - `#else` stub. -- Did not change ZSTD configuration parsing, accepted configuration range, - conditional compilation, errno behavior, or control flow after the guard. - -## Static proof - -- `struct erofs_mount.zstd_windowlog` is `uint8_t`, so integer promotion gives - a value in the range 0 through 255. -- Adding 10 therefore produces a minimum value of 10; comparison with `< 10` - is unreachable for every representable field value. -- `z_erofs_load_zstd_config` still rejects `windowlog > 10`, so configured - mount values and the remaining decompression upper-bound behavior are - unchanged. -- Normalized function comparison confirms that the only removed logical - tokens are the single unreachable lower-bound subcondition and its `||`. -- `git diff --check` and `git diff --cached --check` completed without errors. - -## Tests not run - -Per the execution constraints, no build, QEMU run, smoke test, feature test, -or other dynamic test was run. A FreeBSD KLD build with `WITH_ZSTDIO=1` -remains mandatory during final validation so this enabled function body is -compiled. diff --git a/docs/pre12-batch-06.md b/docs/pre12-batch-06.md deleted file mode 100644 index a1f0310..0000000 --- a/docs/pre12-batch-06.md +++ /dev/null @@ -1,60 +0,0 @@ -# Pre-12 Batch 06: codec descriptor ownership - -Date: 2026-08-13 - -## Scope - -- Moved the LZMA, DEFLATE, and ZSTD descriptors into their backend source - files. -- Made their backend-only config, decompress, and availability helpers - `static`. -- Kept the LZ4 config loader and descriptor, plus the SHIFTED and INTERLACED - descriptors and transform, in `src/decompressor.c`. -- Replaced the central descriptor object array with the descriptor pointer - table `z_erofs_decomp`. -- Reduced `src/compress.h` to the three backend descriptor declarations and - the LZ4 decompress prototype needed by the central descriptor. - -## Descriptor tuples - -The normalized tuples are unchanged before and after this batch: - -| Index | Name | Config callback | Decompress callback | Compile condition | -| --- | --- | --- | --- | --- | -| `Z_EROFS_COMPRESSION_LZ4` | `lz4` | `z_erofs_load_lz4_config` | `z_erofs_lz4_decompress` | always | -| `Z_EROFS_COMPRESSION_LZMA` | `lzma` | `z_erofs_load_lzma_config` | `z_erofs_lzma_decompress` | always | -| `Z_EROFS_COMPRESSION_DEFLATE` | `deflate` | `z_erofs_load_deflate_config` | `z_erofs_deflate_decompress` | always | -| `Z_EROFS_COMPRESSION_ZSTD` | `zstd` | `z_erofs_load_zstd_config` | `z_erofs_zstd_decompress` | descriptor and config always; decompress implementation selected by `ZSTDIO` | -| `Z_EROFS_COMPRESSION_SHIFTED` | `shifted` | `NULL` | `z_erofs_transform_plain` | always | -| `Z_EROFS_COMPRESSION_INTERLACED` | `interlaced` | `NULL` | `z_erofs_transform_plain` | always | - -## Static validation - -- Confirmed one declaration and one definition for each backend descriptor; - the only additional occurrence is its central pointer-table entry. -- Confirmed the LZMA and DEFLATE config/decompress helpers have no external - consumers and are now `static` in their backends. -- Confirmed the ZSTD config/decompress/availability helpers have no external - consumers and are now `static`; the two decompress definitions are mutually - exclusive under `#ifdef ZSTDIO`. -- Confirmed `z_erofs_decomp` explicitly fills all six runtime indices in the - original order. Decompression retains the array-length bound, rejects a - null table entry, and rejects a null decompress callback with - `EOPNOTSUPP`. Config parsing retains the existing compression-config loop - bound and treats a null entry or null config callback as `EOPNOTSUPP` before - releasing the config buffer on the existing path. -- Confirmed the `WITH_ZSTDIO=0` static conditional path remained valid; - actual build deferred to final validation. Without `ZSTDIO`, the path - keeps the descriptor non-null, reports the existing mount error and - `EOPNOTSUPP` from config, and selects the decompress stub returning `-1`. -- Confirmed `WITH_ZSTDIO=1` adds `-DZSTDIO`, keeps the same descriptor and - config callback, and selects the existing ZSTD implementation with its - resource-release and error paths unchanged. -- `git diff --check` and `git diff --cached --check` were required for this - batch. - -## Deferred validation - -No build, QEMU, smoke, feature, or other dynamic validation was run for this -batch. Follow-up validation must build the FreeBSD KLD with both -`WITH_ZSTDIO=0` and `WITH_ZSTDIO=1`. diff --git a/docs/pre12-batch-07.md b/docs/pre12-batch-07.md deleted file mode 100644 index 743ea29..0000000 --- a/docs/pre12-batch-07.md +++ /dev/null @@ -1,39 +0,0 @@ -# Pre-12 Batch 07: exact macro alignment - -Date: 2026-08-13 - -## Scope - -- Renamed only `EROFS_ALL_SUPPORTED_INCOMPAT` to the Linux name - `EROFS_ALL_FEATURE_INCOMPAT` in its comment, definition, and sole source - use. -- Deleted only the zero-consumer compatibility alias - `EROFS_CHUNK_FORMAT_INDEXES_FLAG`. - -## Static proof - -- Before the edit, the target tree had exactly three source occurrences of - `EROFS_ALL_SUPPORTED_INCOMPAT`: its comment and definition in - `src/erofs_fs.h`, and the `src/super.c` unsupported-feature calculation. - The two additional target-tree matches were historical review prose, not - source or build consumers. -- After the edit, the old source name has zero occurrences and - `EROFS_ALL_FEATURE_INCOMPAT` has exactly the same comment, definition, and - sole `src/super.c` use. -- The macro continuation expression and its feature-bit operands are - unchanged. The unsupported calculation changes only the macro token. -- A tracked-tree exact-symbol search found - `EROFS_CHUNK_FORMAT_INDEXES_FLAG` only as identical definitions in retained - repository snapshots. The target tree had exactly one occurrence: the - definition being deleted. -- Exact searches of target-tree Makefiles, scripts, documentation, text - interfaces, and preprocessor `-D` flags found no consumer of the deleted - alias. Existing code continues to use `EROFS_CHUNK_FORMAT_INDEXES`; - `EROFS_CHUNK_FORMAT_ALL` and chunk decoding are unchanged. -- `git diff --check` and `git diff --cached --check` were required for this - batch. - -## Deferred validation - -No build, QEMU, smoke, feature, or other dynamic validation was run for this -batch. The final KLD and Plain/LZ4/LZMA validation remains deferred. diff --git a/docs/pre12-batch-08.md b/docs/pre12-batch-08.md deleted file mode 100644 index c70a7db..0000000 --- a/docs/pre12-batch-08.md +++ /dev/null @@ -1,32 +0,0 @@ -# Pre-12 Batch 08: xattr helper order - -Date: 2026-08-13 - -## Scope - -- Moved the complete `static erofs_listxattr_foreach` function block to - immediately precede `erofs_getxattr_foreach` in `src/xattr.c`. -- Made no changes to either function body, signature, visibility, callers, or - conditional-compilation context. - -## Static proof - -- The exact function block from `static int` through its closing brace had - SHA-256 `0541c211f78a19f39caeebdafe73bdd0a7064e996686a7b45c9c651160ed98ae` - before and after the move. -- The target remains one `static` definition with two call sites, in the - inline and shared xattr iterators. -- `erofs_getxattr_foreach` remains one definition with the same two call - sites; its complete block is unchanged. -- Both helpers remain in the same unconditional compilation region after - `struct erofs_xattr_iter` and before the iterator functions, so no forward - declaration or conditional boundary was introduced. -- The source diff is a pure block move; adjacent xattr and ACL logic is - unchanged. -- `git diff --check` and `git diff --cached --check` were required for this - batch. - -## Deferred validation - -No build, QEMU, smoke, feature, or other dynamic validation was run for this -batch. Final KLD and Plain/LZ4/LZMA validation remains deferred. diff --git a/docs/pre12-batch-09.md b/docs/pre12-batch-09.md deleted file mode 100644 index 6a8f3d4..0000000 --- a/docs/pre12-batch-09.md +++ /dev/null @@ -1,44 +0,0 @@ -# Pre-12 Batch 09: compression header responsibility - -Date: 2026-08-13 - -## Scope - -- Removed the direct `internal.h` include from `decompressor.c` and the four - `decompressor_*.c` backend consumers that already include `compress.h`. -- Made no declaration, definition, function, storage-duration, or public API - changes. - -## Candidate source - -Batch 04 created `compress.h` with a direct `internal.h` include and added -`compress.h` to these five translation units while retaining their direct -`internal.h` includes. Batch 06 then made LZMA, DEFLATE, and ZSTD backend -functions private and exposed only their descriptor objects through -`compress.h`. This left exactly five duplicate include edges attributable to -Batches 04/06. - -## Static proof - -- `compress.h` remains the sole owner of `z_erofs_decompress_req`, - `z_erofs_decompressor`, the LZ4 decompressor prototype, and the three - backend descriptor declarations. -- `internal.h` remains the sole public owner of `z_erofs_decompress` and - `z_erofs_parse_cfgs`; it does not include `compress.h`. -- The include graph remains acyclic: each affected translation unit includes - `compress.h`, and `compress.h` includes `internal.h`, which includes only - `erofs_fs.h` among project headers. -- Each affected consumer therefore still sees the same complete request, - descriptor, mount, map, and superblock types through one guarded include - path. No backend prototype or descriptor declaration is duplicated between - `internal.h` and `compress.h`. -- The source diff removes exactly one `#include "internal.h"` line from each - of five authorized files and changes no other token. -- `git diff --check` and `git diff --cached --check` were required for this - batch. - -## Deferred validation - -No build, QEMU, smoke, feature, or other dynamic validation was run under the -execution constraint. The required FreeBSD KLD `WITH_ZSTDIO=0/1` matrix and -final Plain/LZ4/LZMA validation remain deferred. diff --git a/docs/pre12-batch-10.md b/docs/pre12-batch-10.md deleted file mode 100644 index 7847d22..0000000 --- a/docs/pre12-batch-10.md +++ /dev/null @@ -1,24 +0,0 @@ -# Pre-12 Batch 10: Makefile readability review - -Date: 2026-08-13 - -## Result - -No-op. `src/Makefile` was reviewed after Batches 04, 06, and 09, but no -non-redundant comment or blank-line grouping with clear maintenance value was -identified. - -## Static proof - -- The `SRCS` list already keeps the core filesystem sources, mapping and data - sources, and `decompressor_*.c` sources in readable contiguous regions. -- Adding labels would repeat names already visible in the short source list. -- `src/Makefile` is byte-for-byte unchanged. -- The `SRCS` elements, order, duplicate counts, `WITH_ZSTDIO` condition, - per-file CFLAGS, target, and `.include` line are unchanged. -- `git diff --check` and `git diff --cached --check` were required for this - documentation-only closeout. - -## Validation - -No build, QEMU, smoke, or feature test was run for this no-op review. diff --git a/docs/pre12-final-report.md b/docs/pre12-final-report.md deleted file mode 100644 index d9664ba..0000000 --- a/docs/pre12-final-report.md +++ /dev/null @@ -1,229 +0,0 @@ -# Pre12 Final Report - -Date: 2026-08-13 - -## Conclusion and scope - -Pre12 is complete for its deliberately limited scope. The final result is -**PASS** for the planned static review, both FreeBSD KLD build configurations, -KLD load/status/unload, and the ordinary Plain/LZ4/LZMA QEMU smoke cases. - -This PASS is not a claim of complete EROFS feature validation. No feature -test, `test_all.py`, new fixture, ZSTD data test, DEFLATE data test, ACL runtime -test, explicit-extent test, or multi-device test was run. The included changes -were selected as low-side-effect maintenance work and did not require a full -feature suite after their static proofs and final build/smoke matrix. Deferred -behavior work remains unresolved and is listed below. - -The validated device-under-test identity is: - -| Identity | Value | -| --- | --- | -| Commit | `a55e42a117e504f6ece49be3c2a2d81603031ea3` | -| `repo-pre-12` tree | `9161671bd0e855faa52c37e24fe8efcd6ddd71e0` | -| `repo-pre-12/src` tree | `d63c45c7872dcf9dd84007c8883cf9fbe0f9467d` | -| Source archive SHA-256 | `bb7e7f0383be29102475575e130e0b19a48452d20520c94059dc6ef128d0c7ce` | - -## Goals - -Pre12 continued the staged effort to reduce unnecessary maintenance -differences from Linux EROFS without importing Linux-only memory, I/O, device, -or concurrency models into FreeBSD. Its concrete scope was nine core batches -and one conditional maintenance closeout: - -1. Use existing semantic scalar aliases where their width and signedness are - exactly equivalent. -2. Use FreeBSD unaligned little-endian readers for raw on-disk bytes. -3. Use the FreeBSD POSIX.1e ACL mode conversion facility. -4. Introduce a Linux-comparable private compression header. -5. Remove an unreachable ZSTD lower-bound condition. -6. Align codec descriptor ownership while preserving FreeBSD behavior. -7. Align exact macro vocabulary and remove one zero-consumer alias. -8. Align one xattr helper's file order through a byte-identical block move. -9. Remove include duplication introduced by the compression-header work. -10. Review Makefile readability and make no change when no useful grouping - was found. - -The work explicitly excluded broad file reordering, catch-all style cleanup, -Linux page/folio/bio/workqueue infrastructure, and behavior changes requiring -specialized feature fixtures. - -## Batch results - -| Batch | Commit | Result and summary | -| --- | --- | --- | -| 01 | `bb28c47bdf4de777febde05ebb333c420ea1e9b9` | Replaced the planned NID, block-number, and byte-offset declarations with existing equal-width unsigned `erofs_nid_t`, `erofs_blk_t`, and `erofs_off_t` aliases. Field order, layout, ABI width, expressions, on-disk types, and control flow were unchanged. | -| 02 | `76a655788538f2270626cff1ca91b8c47700d536` | Replaced exactly three potentially unaligned typed loads with `le16dec`/`le32dec` in xattr, inode, and block-map decoding. | -| 03 | `35bde15f5242f84a6343cc9e96519fcce4eeca90` | Replaced three manual ACL mode shifts with `acl_posix1e_mode_to_perm` for owner, group, and other entries. | -| 04 | `b591751f90e62bc0ec54d3f34c724f698c99813d` | Added `src/compress.h` for private decompressor request, descriptor, and backend declarations; public entry points and runtime enum remained in `internal.h`. | -| 05 | `aaa3c5db40745ee558e6fcf6375d60e044217df9` | Removed only the unreachable `zstd_windowlog + 10 < 10` subcondition; the upper bound, zero-output check, disabled stub, errno behavior, and resource handling remained unchanged. | -| 06 | `6748d58f7e15927321f44a83d81a6c697c1185ae` | Moved LZMA, DEFLATE, and ZSTD descriptors into their backends, kept LZ4 and plain transforms central, and changed the central descriptor table to the pointer table `z_erofs_decomp`. All six normalized descriptor tuples and indices remained unchanged. | -| 07 | `292da21bd9288cac4a8b280bb67970c8b9058550` | Renamed `EROFS_ALL_SUPPORTED_INCOMPAT` to Linux's `EROFS_ALL_FEATURE_INCOMPAT` at its definition and sole source use, and removed zero-consumer `EROFS_CHUNK_FORMAT_INDEXES_FLAG`. | -| 08 | `0be5642d917104f1904bcfadaea803c9845c7a5c` | Moved the complete `erofs_listxattr_foreach` static function block before `erofs_getxattr_foreach`; the block SHA-256 remained `0541c211f78a19f39caeebdafe73bdd0a7064e996686a7b45c9c651160ed98ae`. | -| 09 | `e5315489eb5ed08884d5d5f2645264067e36838e` | Removed five direct `internal.h` includes made redundant by `compress.h`; declaration visibility and the acyclic include graph were preserved. | -| 10 | `a55e42a117e504f6ece49be3c2a2d81603031ea3` | Conditional Makefile readability closeout: honest no-op. The source list and Makefile remained byte-for-byte unchanged because no non-redundant grouping comment was justified. | - -Batch 10 is not counted among the nine core source workflows. Batch 09 did -produce a precise cleanup diff, while Batch 10 records the required review -without manufacturing source churn. - -## Static review - -The final static review found no source-blocking issue. In particular, it did -not find a regression in declaration/definition matching, include ownership or -graph shape, descriptor indices and callback ownership, errno paths, resource -release, in-memory ABI/layout, endian decoding, ACL construction, macro values, -or the byte-identical xattr function move. - -The important proofs were: - -- The three semantic scalar aliases are direct unsigned 64-bit aliases; the - approved substitutions preserve size, alignment, signedness, structure - layout, parameter representation, and arithmetic width. -- The three `le16dec`/`le32dec` inputs are raw little-endian disk bytes of the - required length; the decoded values are equivalent while avoiding typed - unaligned loads. -- ACL owner/group/other conversion was exhaustively compared across modes - `0000` through `0777`: `1536/1536` tag/mode combinations were equivalent, - with zero mismatches. -- The compression include graph remained acyclic and each migrated or private - symbol retained one authoritative declaration/definition relationship. -- The six codec descriptor tuples, indices, names, config callbacks, - decompress callbacks, and ZSTD conditional ownership remained equivalent. -- The removed ZSTD lower bound is unreachable because the field is `uint8_t`; - its promoted value plus 10 cannot be less than 10. -- Macro expressions and values were unchanged, and the deleted chunk-format - alias had no source, build, documentation-interface, or preprocessor - consumer in the target tree. -- The moved xattr function block retained its exact SHA-256 and conditional - context. - -The Batch 06 record originally used wording that could imply an actual -`WITH_ZSTDIO=0` build had already occurred during that batch. It has been -corrected to state that only the static conditional path was validated then. -Final validation subsequently built both configurations successfully. - -## KLD build matrix - -The valid final run used FreeBSD `15.0-RELEASE-p8`, amd64, with `/usr/src/sys` -and the in-tree FreeBSD ZSTD headers present. The extracted source archive -matched SHA-256 -`bb7e7f0383be29102475575e130e0b19a48452d20520c94059dc6ef128d0c7ce`. - -For each configuration, the guest executed: - -```sh -timeout -k 10s 120s make "WITH_ZSTDIO=<0-or-1>" clean -timeout -k 10s 600s make "WITH_ZSTDIO=<0-or-1>" -``` - -| Configuration | Clean exit | Build exit | Module SHA-256 | -| --- | ---: | ---: | --- | -| `WITH_ZSTDIO=0` | 0 | 0 | `96dc276c6c3f68943e68a144cc2ea138ffc215b62c7392cad9688ec45c9fc2fb` | -| `WITH_ZSTDIO=1` | 0 | 0 | `4d879c0b7ebc653e915a3e41863f74d6d5b5f18aa81c5178579368048d1b6ee3` | - -The enabled build compiled `decompressor_zstd.c` with `-DZSTDIO` and linked -`erofs.ko`. These builds prove both compile-time configurations close and link; -they do not prove ZSTD or DEFLATE data-path correctness. - -## KLD and smoke results - -The valid run loaded the `WITH_ZSTDIO=1` artifact under the standard module -name `erofs.ko`. - -| Check | Result | -| --- | --- | -| `kldload` | exit 0 | -| `kldstat -n erofs.ko` | present as `erofs.ko`, module id 5 | -| Loaded module SHA-256 | `4d879c0b7ebc653e915a3e41863f74d6d5b5f18aa81c5178579368048d1b6ee3` | -| `kldunload erofs.ko` | exit 0 | -| Post-unload `kldstat` | module absent, as expected | - -All ordinary read cases matched their complete expected hashes: - -| Case | Exit | Expected SHA-256 | Actual SHA-256 | Elapsed | -| --- | ---: | --- | --- | ---: | -| Plain | 0 | `056f8f7585667dc695e2edf936deaf84cfee0f88671ba6cd5be22ce890763433` | `056f8f7585667dc695e2edf936deaf84cfee0f88671ba6cd5be22ce890763433` | 593 ms | -| LZ4 | 0 | `3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880` | `3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880` | 6822 ms | -| LZMA | 0 | `ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461` | `ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461` | 3418 ms | - -The dmesg delta was zero bytes and the suspect-filter output was zero bytes. -No new panic, trap, or EROFS error was observed. - -## Evidence and invalid attempts - -The only final valid evidence directory is: - -```text -/work/tests-dev/temp/pre12-smoke-direct-20260813T165639Z/ -``` - -Its `final-summary.txt`, guest build logs, module hashes, case records, dmesg -files, and cleanup record form the accepted evidence set. - -Two earlier infrastructure attempts are explicitly invalid and contribute no -source result and no test pass: - -1. `/work/tests-dev/temp/pre12-build-20260813T154900Z/` is **INFRA-FAIL**. - The SSH password expanded to an empty string, so there were zero successful - SSH sessions and neither build began. Its QEMU and temporary resources were - cleaned. This is neither a DUT failure nor a test success. -2. `/work/tests-dev/temp/pre12-final-smoke-20260813T160957Z/` stopped while - redundantly hashing the 16 GB base image. Five nested Codex CLI sessions - were discovered and all were terminated or had exited; no QEMU, SSH, - guest build, KLD action, or smoke case started. Its partial files are not - test evidence and it is neither a DUT failure nor a test success. - -The final accepted run was performed by a single direct execution layer and -did not start a nested agent or Codex process. - -One intermediate runner revision attempted to query KLD using the renamed -artifact filename rather than the module's standard name. That was a -test-period runner issue, not a DUT issue. Only the temporary runner copy under -`tests-dev/temp` was corrected: the successful `WITH_ZSTDIO=1` artifact was -copied to `erofs.ko`, then the complete KLD and Plain/LZ4/LZMA sequence was -rerun. No `repo-pre-12` source or build file was modified to make the test -pass. - -## Cleanup and repository state - -The accepted run completed cleanup successfully: - -- Owned QEMU stopped. -- SSH ControlMaster stopped. -- Test port `10048` released. -- Test overlay removed. -- Guest test mounts and `md70`/`md71`/`md72` devices were removed. -- The retained guard QEMU remained unchanged and reachable at PID `26318`, - port `9222`, with process start ticks `1102996924`. -- The base image remained mode `0444`, size `16515530752` bytes, and mtime - `1786418212`. Its retained SHA-256 record is - `67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef`; - the final run intentionally did not repeat the expensive 16 GB hash. -- At validation completion, `HEAD` and `xdm/main` both identified - `a55e42a117e504f6ece49be3c2a2d81603031ea3`, and the worktree was clean. - -This report is documentation-only and does not alter the validated DUT source -tree `d63c45c7872dcf9dd84007c8883cf9fbe0f9467d`. - -## Deferred feature work - -The following issues remain unresolved and require dedicated behavior design, -fixtures, or feature tests before any future implementation can be accepted: - -- inode decode split or rewrite; -- map request interface conversion; -- directory/namei control flow, qstr, cookies, namecache, and lock behavior; -- metadata reader consolidation; -- rejection or handling of a non-directory root; -- `run_len == 0` behavior; -- header-only xattr behavior; -- duplicate or missing `device.N` behavior; -- `insmntque` race closure; -- explicit extent and explicit mapped-extent fixtures; -- `vnode_create_vobject` errno behavior. - -No claim is made that these issues were fixed or covered by the ordinary -smoke run. The final PASS is limited to the Pre12 static review, both KLD build -configurations, KLD lifecycle, and the Plain/LZ4/LZMA smoke cases recorded in -the accepted evidence directory. diff --git a/docs/pre13-baseline.md b/docs/pre13-baseline.md deleted file mode 100644 index e1b101d..0000000 --- a/docs/pre13-baseline.md +++ /dev/null @@ -1,10 +0,0 @@ -# Pre13 Baseline - -- Source commit: `35eb53c65a4f75ba580f64af043779b930e3bb78` -- `repo-pre-12` tree: `4ea4d7cfbf8b8a884b291f7433e19ce8e5af5a11` -- `repo-pre-12/src` tree: `d63c45c7872dcf9dd84007c8883cf9fbe0f9467d` -- `repo-pre-13/src` tree after copy: `d63c45c7872dcf9dd84007c8883cf9fbe0f9467d` -- Copy verification: 327 files, 43 directories, and 0 symbolic links matched by relative path, mode, link target, and SHA-256 content manifest before this document was added. -- Copy manifest SHA-256: `c6bdcee893a45a03308897715062aada41c591a82432f2c0008dbd316d10cf07` - -Except for this baseline document, `repo-pre-13` is an exact copy of `repo-pre-12`. diff --git a/docs/pre13-execution-status.md b/docs/pre13-execution-status.md deleted file mode 100644 index d9d5347..0000000 --- a/docs/pre13-execution-status.md +++ /dev/null @@ -1,90 +0,0 @@ -# Pre13 Execution Status - -## Final Status - -Pre13 is **scoped PASS** for the planned and admitted Pre13 work at -`b804d7b77d1c76b8e844dc4a758c6ab3381ca006`. This is not a full feature-suite -PASS and does not claim V02 positive coverage. - -At status backfill time, DUT `HEAD`, local `main`, fetched `xdm/main`, remote -`refs/heads/main`, and the DUT worktree all resolve to -`b804d7b77d1c76b8e844dc4a758c6ab3381ca006`; the DUT worktree is clean. - -## Completed Source Work - -- Low-risk alignment commits: - - L-A1 helper/guard alignment: - `2d0041ba3a195c184a52bdccae3bd00af520d4b7` - - L-A2 helper ordering: - `190a0b908d25e62b5c78ba82377f53d359fc5c4f` - - L-B semantic type alignment: - `f236836a55ad7ec7d536df483460ea394e38b6c3` - - L-C1 on-disk endian field typing: - `217b7a3032ec917c5ea63652b0f3cad734990d48` - - L-C2 flexible on-disk arrays: - `e72ecd7cc024047f0ce7c1855e6e44ae7f5350e1` - - L-D LZ4 endian helper: - `fe338f701059de8dfd6336dea7f9288bc0f12e9f` - - L-E localized private constants: - `9028b92343ec7f5bbc1b07477b8d4a669c436955` -- Stage0 decisions: - `8ed80c3ddbedc64b8b80daf1be16b28c4f24f10d`. -- H04 root-type validation: - `ee9dc4715436eeee21ae2d84eab2677b27d87ddd`. -- H03 qstr lookup alignment: - - H03a bounded qstr comparator: - `f51b0fe1ceb27cac641667aa5688d371a60d2d99` - - H03b lookup search propagation: - `3313d305a2193ef6ffa26c6a189e47187b9c0d70` - - H03c VOP adapter: - `83a5bef73b329c967bd8390a0966d30005295758` -- Build/test-input fixups: - `dc47e8f99c947d2929d3528a6c139a85eb4251d5`, - `b804d7b77d1c76b8e844dc4a758c6ab3381ca006`. - -## Stage0 Decisions - -- H04 ROOT-NONDIR: `READY`, then implemented and custom validated. -- H07 ZERO-RUN: `STOP/no-op`; 1,360 legal states showed no reachable - zero-progress path. -- H01a/b/c map request work: `STOP`; the tuple oracle lacked chunk, - multidevice, bounds, and overflow coverage. -- H03 comparator: `READY`; Python 11,562 cases and C 1,048,576 cases passed. -- Device-option probing: blocked/non-gating, no behavior claim. - -## Validation Status - -- Targeted build/layout/probe retry: - `/work/tests-dev/temp/pre13-targeted-final-runnerfix-retry-20260814T0138Z`. - `WITH_ZSTDIO=0` and `WITH_ZSTDIO=1` builds passed with module SHA256 - `0b1244cf9b7fe70649bdcc1c6ef12e6151c77b10c2755b366f960b101ccb7220` - and `15db1fbf9450259d030e88602e1c5ba2e452a731c641b359bd5a2d7823da2735`. - Layout probe and four userspace probe builds passed. -- H04 custom fixture: PASS. Valid root mounted/remounted; non-directory root - returned exact `EINTEGRITY` (`97`); failed mount left the provider - detachable. -- H03 targeted feature cases: PASS for TC042, TC043, TC044, TC045, TC053, - TC054, TC123, TC141, and TC148. TC141's raw runner failure was an extra - empty `umount ''` false negative after 25 real commands passed. TC148 - covered 320 files, 322 dirents, content, and empty suspect dmesg. -- Final smoke: - `/work/tests-dev/temp/pre13-missing-and-smoke-20260814T021330Z/final-summary.md`. - Plain, LZ4, and LZMA passed; KLD load/stat/unload and cleanup passed. -- V01 race: - `/work/tests-dev/temp/pre13-v01-race-20260814T031032Z/retry-fresh-20260814T032500Z`. - Raw runner FAIL was reclassified by the final offline analyzer at tests-dev - `22c0bea73b9b81cc44385d1ba7103fa820c56070` as - `DUT PASS / TEST false-negative-corrected`. The four offline analyzer unit - tests passed. Dmesg total delta was 24 lines / 1,266 bytes with suspect - bytes 0. - -## Explicit Non-Claims - -- No full feature suite was run. -- No V02 positive explicit-extent fixture was produced; V02 remains PARTIAL. -- TC004, TC011, TC143, TC144, TC145, TC150, and other unlisted feature cases - are not claimed as passing. -- H01, H07, H05, H06, H02, and H08 source changes were intentionally stopped, - rejected, or no-op as recorded in the final report. - -Detailed evidence and limits are recorded in `pre13-final-report.md`. diff --git a/docs/pre13-final-report.md b/docs/pre13-final-report.md deleted file mode 100644 index 165252c..0000000 --- a/docs/pre13-final-report.md +++ /dev/null @@ -1,344 +0,0 @@ -# Pre13 Final Report - -## Conclusion - -Pre13 is **scoped PASS** for the planned and admitted Pre13 work: - -- DUT baseline snapshot: `a587d489b829ef8e7fac94c87130b61d47aa3e44`. -- Final DUT commit: `b804d7b77d1c76b8e844dc4a758c6ab3381ca006`. -- At final report time, `HEAD`, local `main`, fetched `xdm/main`, remote - `refs/heads/main`, and the only DUT worktree all resolve to - `b804d7b77d1c76b8e844dc4a758c6ab3381ca006`. -- DUT working tree is clean. - -This is **not** a full-filesystem certification. No full 161-case feature -suite was run, no positive V02 explicit-extent fixture was produced, and Pre13 -does not claim that unrun cases such as TC004, TC011, TC143, TC144, TC145, -TC150, or other non-targeted feature cases passed. - -## Commit Inventory - -The audited DUT range is -`a587d489b829ef8e7fac94c87130b61d47aa3e44..b804d7b77d1c76b8e844dc4a758c6ab3381ca006`. - -### Planning and Stage0 - -- `76a4c82`, `fd300cb`, `2f487f0`: Pre13 planning documents and low-risk batch - accounting. -- `8ed80c3ddbedc64b8b80daf1be16b28c4f24f10d`: recorded Stage0 gate decisions - in `repo-pre-13/docs/pre13-stage0-decisions.md` and the initial execution - status. - -### Low-Risk Source Alignment - -- `2d0041ba3a195c184a52bdccae3bd00af520d4b7`: L-A1 helper and guard alignment - across `compress.h`, `data.c`, `erofs_fs.h`, and `inode.c`, with - `pre13-low-a1.md`. -- `190a0b908d25e62b5c78ba82377f53d359fc5c4f`: L-A2 local helper ordering in - `decompressor.c` and `xattr.c`, with `pre13-low-a2.md`. -- `f236836a55ad7ec7d536df483460ea394e38b6c3`: L-B semantic `erofs_nid_t` and - `erofs_off_t` alignment across data, directory, inode, namei, super, xattr, - zdata, and zmap paths, with `pre13-low-b.md`. -- `217b7a3032ec917c5ea63652b0f3cad734990d48`: L-C1 on-disk `__leXX` field - alignment in `erofs_fs.h` and initial `pre13_ondisk_layout_probe.c`, with - `pre13-low-c1.md`. -- `e72ecd7cc024047f0ce7c1855e6e44ae7f5350e1`: L-C2 flexible on-disk arrays - and layout probe updates, with `pre13-low-c2.md`. -- `fe338f701059de8dfd6336dea7f9288bc0f12e9f`: L-D LZ4 offset endian helper in - `decompressor_lz4.c`, with `pre13-low-d.md`. -- `9028b92343ec7f5bbc1b07477b8d4a669c436955`: L-E localized private - constants, removed `erofs_defs.h`, and updated consumers/docs. - -### Behavior Changes and Fixups - -- `ee9dc4715436eeee21ae2d84eab2677b27d87ddd`: H04 rejects non-directory root - inodes during mount, with `pre13-h04-root-validation.md`. -- `f51b0fe1ceb27cac641667aa5688d371a60d2d99`: H03a introduced the bounded - qstr comparator. -- `3313d305a2193ef6ffa26c6a189e47187b9c0d70`: H03b passed qstr through block - and dirent lookup search. -- `83a5bef73b329c967bd8390a0966d30005295758`: H03c adapted VOP lookup to the - bounded qstr path. -- `dc47e8f99c947d2929d3528a6c139a85eb4251d5`: fixed const-correct build inputs - and the userspace layout probe `bool` include issue. -- `b804d7b77d1c76b8e844dc4a758c6ab3381ca006`: retired stale decompression test - harnesses that no longer matched the current module surface; this did not hide - a DUT test failure. - -## Stage0 Gate Results - -Stage0 was a mixed gate result, not an overall PASS: - -- Gate A, ROOT-NONDIR: **READY**. The baseline accepted a non-directory root; - Pre13 froze the rejection as `EINTEGRITY` and allowed H04. -- Gate B, ZERO-RUN: **STOP/no-op**. The analysis covered 1,360 legal states and - found no reachable zero-progress state for H07. -- Gate C, map tuple oracle: **STOP**. The oracle covered only 26 plain, - inline, and hole tuples; it did not cover chunk, multidevice, bounds, or - overflow behavior, so H01a/H01b/H01c were not implemented. -- Gate D, qstr comparator: **PASS/READY**. The Python comparator corpus passed - 11,562 cases and the independent C harness passed 1,048,576 cases. -- Gate E, device option probe: **blocked/non-gating**. It produced no runtime - behavior claim and did not block Pre13. - -## Intentional Stops and Rejects - -- H01 map objectization: **STOP** because the tuple oracle was incomplete for - chunk, multidevice, bounds, and overflow coverage. -- H07 zero-run: **STOP/no-op** because 1,360 legal states produced no reachable - zero-progress path. -- H05 header-only xattr: **rejected**; Linux treats the format as undefined and - BSD already returns `EOPNOTSUPP`. -- H06 duplicate `device.N`: **rejected/no-op** for source changes because - `vfs_sanitizeopts()` makes the EROFS duplicate-option check unreachable; any - probe is non-gating. -- H02 inode split: **rejected** because it would diverge from Linux's single - `erofs_read_inode()` switch and create low-value churn. -- H08 pager errno propagation: **rejected** because the target - `vnode_create_vobject()` path observably returns 0. -- V02 explicit extent positive coverage: **PARTIAL**. No positive mapped - fixture was available, and TC157 negative coverage is not a substitute. - -## Build, Layout, and Probe Evidence - -Primary evidence: -`/work/tests-dev/temp/pre13-targeted-final-runnerfix-retry-20260814T0138Z`. - -After `dc47e8f` and `b804d7b`, the targeted retry produced: - -- `WITH_ZSTDIO=0` build: PASS, module SHA256 - `0b1244cf9b7fe70649bdcc1c6ef12e6151c77b10c2755b366f960b101ccb7220`. -- `WITH_ZSTDIO=1` build: PASS, module SHA256 - `15db1fbf9450259d030e88602e1c5ba2e452a731c641b359bd5a2d7823da2735`. -- Loaded module in that targeted run: SHA256 - `15db1fbf9450259d030e88602e1c5ba2e452a731c641b359bd5a2d7823da2735`. -- Userspace layout probe: build and run return code 0. -- Four userspace probe builds: `g3_vfs_probe`, `mount_errno_probe`, - `nfs_fh_tool`, and `readdir_probe` all returned 0. - -The first targeted runner attempt still ended host-side FAIL because TC141 -included an extra empty `umount ''` command. That was classified as a test -runner false negative, not a DUT failure: the 25 real TC141 commands all had -return code 0, and only the 26th synthetic empty unmount failed. - -## H04 Root Validation - -Evidence path: -`/work/tests-dev/temp/pre13-targeted-final-runnerfix-retry-20260814T0138Z/guest-evidence/cases/H04`. - -H04 custom fixture result: **PASS**. - -- Valid root fixture SHA256: - `d0baba501ae12fcdb8869a98908486473d2bcfb6b332a47cb6a6c1f856a3ab34`. -- Non-directory root fixture SHA256: - `60bf0d0d93baabdfa23b71f48283d317f6b7218852f3d19aee77575c41500db7`. -- Root proof payload SHA256: - `0ebc44a6c5a02ba458d1a115bc338fa4acb4d973a37804c9b926eafc0c53575f`. -- Valid root mounted, root mode was `040755`, payload matched, and unmount plus - `md` detach succeeded. -- Non-directory root returned exact `mount_errno=97` (`EINTEGRITY`) with message - `erofs: root inode nid=36 is not a directory`; the failed mount left the - provider detachable. -- The valid fixture remounted successfully after the non-directory failure. -- Per-case dmesg delta and suspect logs were empty. - -TC011 and TC150 were not run and are not claimed as passing. They are not -Pre13 blockers because the dedicated H04 fixture directly exercised the actual -root-type change, including the valid control, exact errno, detach, and remount -properties. - -## H03 Targeted Feature Evidence - -H03 targeted cases exercised the qstr lookup changes and passed for: -TC042, TC043, TC044, TC045, TC053, TC054, TC123, TC141, and TC148. - -Evidence: - -- TC042/043/044/045/053/054/123 and TC141 raw evidence: - `/work/tests-dev/temp/pre13-targeted-final-runnerfix-retry-20260814T0138Z`. -- TC148 valid copied evidence and final smoke summary: - `/work/tests-dev/temp/pre13-missing-and-smoke-20260814T021330Z/final-summary.md`. - -TC141 details: - -- The runner logged 26 commands, but the last was the invalid empty - `umount ''` cleanup bug. -- The 25 real commands passed: valid-base and valid-padding cold lookup/read, - repeated missing lookups returning `ENOENT`, `entries=322` wide-directory - checks, and three corrupt lookup fixtures returning repeated - `EINTEGRITY` (`errno=97`). -- The only failure was the empty unmount runner command. Its stderr was - `statfs: No such file or directory` / `unknown file system`, with no DUT - dmesg delta. - -TC148 details: - -- Outcome: PASS. -- Fixture SHA256: - `9a94e9af2cab264b9c11975a20d78e615d6fe1e6f86267173cb5ac86aecb2b17`. -- Mount command return code: 0. -- Expected and actual regular-file count: 320. -- Directory oracle: 322 entries, with two directories and 320 regular files. -- `d_off` restart count and `seekdir` restart count: 322 each. -- Target content matched. -- Per-case dmesg delta and suspect log were empty. - -## Final Smoke Evidence - -Evidence path: -`/work/tests-dev/temp/pre13-missing-and-smoke-20260814T021330Z/final-summary.md`. - -Final smoke outcome: **PASS**. - -- DUT archive SHA256: - `d0c052bfef024d0acb40e107800cc8e8227fab6b691e8825af5cf01f19672fe4`. -- Assets SHA256: - `a9741ea468493bdc72d399615892d7f6269226467d09fa6613fa01352f011908`. -- Probe bundle SHA256: - `7b10d23c8a780fb35810fe85626305a8c49f196d270f98e5224045bd79f51066`. -- Clean tests-dev worktree commit used for the smoke: - `f86799654a37cdb81025b9e703635f7c7812fecb`. -- `WITH_ZSTDIO=0` build: PASS, module SHA256 - `0b1244cf9b7fe70649bdcc1c6ef12e6151c77b10c2755b366f960b101ccb7220`. -- `WITH_ZSTDIO=1` build: PASS, module SHA256 - `15db1fbf9450259d030e88602e1c5ba2e452a731c641b359bd5a2d7823da2735`. -- Loaded module for final smoke: `WITH_ZSTDIO=0`, SHA256 - `0b1244cf9b7fe70649bdcc1c6ef12e6151c77b10c2755b366f960b101ccb7220`. -- `kldload`, `kldstat -n erofs.ko`, `kldunload`, and final KLD cleanup: PASS. -- Plain fixture SHA256: - `3785ca07e7bd16f6c611191596ae0314253c0ae9b7217a4b22de25799b0f08ac`; - payload SHA256 matched - `056f8f7585667dc695e2edf936deaf84cfee0f88671ba6cd5be22ce890763433`. -- LZ4 fixture SHA256: - `967cc1b625546f9f9f881472e71cc3d849c65feb4e98e67fbfdc9b90a4be6dda`; - payload SHA256 matched - `3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880`. -- LZMA fixture SHA256: - `32107a084b27362a093768b88746c37c2e99a74b9f1301a9d4046988479defe9`; - payload SHA256 matched - `ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461`. -- Every attach, mount, hash, unmount, and detach command returned 0. -- Per-case and global dmesg suspect logs were empty. -- Global mount, md, KLD, QEMU, ControlMaster, overlay, and ports - `10056`, `10057`, `10058` cleanup passed. -- Guard QEMU PID `26318` and port `9222` were unchanged and reachable. -- The main tests-dev worktree was dirty before and after with unchanged diff - SHA256 `92d3808ab33ff9b22fa2ee3f8a854c1fc36effecb4dd6c0917ae7e80c57fc23a`; - the clean tests worktree, not the dirty main worktree, was used as evidence. - -The final summary explicitly states that no other TC, full feature suite, -TC004, TC143, TC144, or TC145 was run. Pre13 preserves that limitation. - -## V01 Race Evidence - -Raw evidence: -`/work/tests-dev/temp/pre13-v01-race-20260814T031032Z/retry-fresh-20260814T032500Z`. - -tests-dev evidence commits were verified on `xdm/main`: - -- `2269908cc3f721d57ae8138faef664a6bc4c7cc9`: added the V01 concurrent unmount - race runner. -- `374c9d8473e6638fa0c8e865fcfcfee141b13f0d`: fixed runner SSH command - execution. -- `22c0bea73b9b81cc44385d1ba7103fa820c56070`: corrected forced-unmount deadfs - errno classification and added the offline analyzer/unit tests. Fetched - `xdm/main` and remote `refs/heads/main` both resolve to this commit. - -Original raw run: - -- Raw host/guest outcome: FAIL, `guest race failed rc=41`. -- DUT commit: `b804d7b77d1c76b8e844dc4a758c6ab3381ca006`. -- Module SHA256: - `15db1fbf9450259d030e88602e1c5ba2e452a731c641b359bd5a2d7823da2735`. -- Fixture root-valid SHA256: - `d0baba501ae12fcdb8869a98908486473d2bcfb6b332a47cb6a6c1f856a3ab34`. - -Normal group: - -- Duration 90 seconds, 8 workers, 154 loops. -- `umount` returned accepted `EBUSY` 154 times; no successful unmount/remount. -- Worker read/stat/open/fstat success counts were each 79,164. -- Worker content mismatch, short reads, rejected errors, and dmesg suspect bytes - were all 0. - -Forced group: - -- Duration 90 seconds, 8 workers, 125 loops. -- Successful forced unmount/remount cycles: 125/125. -- `stat ENOENT`: 102,165; `open ENOENT`: 16. -- Raw rejected errors: 1,772, consisting of 882 `fstat EBADF` and 890 - `read ENXIO`. -- The final mounted SHA matched the expected payload SHA in both normal and - forced groups. - -Final analyzer result from -`scripts/analyze-pre13-v01-race-evidence.py` at -`22c0bea73b9b81cc44385d1ba7103fa820c56070`: - -- `normal_verdict=PASS`. -- `forced_verdict=PASS`. -- `forced_expected_deadfs_errors=1772`. -- `forced_expected_deadfs_fstat_ebadf=882`. -- `forced_expected_deadfs_read_enxio=890`. -- `dut_verdict=PASS`. -- `test_verdict=false-negative-corrected`. -- `overall_verdict=DUT PASS / TEST false-negative-corrected`. - -The `EBADF` and `ENXIO` results occur only after successful forced unmounts, -when existing descriptors are taken over by `deadfs`; that is standard FreeBSD -semantics and not a DUT instability. - -The analyzer's four offline unit tests all passed: - -- forced deadfs errors are corrected; -- normal deadfs errno is rejected; -- forced unrelated errno is rejected; -- count mismatch is rejected. - -V01 cleanup and integrity: - -- QEMU stopped, V01 port released, overlay removed. -- Base image mode, size, and mtime remained `444`, `16515530752`, and - `1786418212`. -- Guard VM was unchanged and reachable. -- Guest mount/md/KLD owned-resource cleanup passed. -- Total dmesg delta was 24 lines / 1,266 bytes, all known non-suspect - `mangled entry` lines; suspect bytes were 0. This is not a zero-total-delta - run. -- The dirty main tests-dev worktree was unchanged; the independent clean - worktree was clean. - -## Static Review and Evidence Boundaries - -Final static review of the Pre13 source delta found no production-source -blocker, test backdoor, out-of-bounds issue, ABI/layout issue, -lock/lifecycle issue, or errno issue requiring another DUT fix. The one -dynamic item that was pending at that review point was documentation/status -backfill; that was a reporting-sequence issue, not an unperformed test gate. - -The source/test diff boundary was also reviewed: - -- Production changes are limited to the planned low-risk alignment work, H04, - H03, and build-input fixups. -- The only new tracked test asset in the DUT repo is the userspace layout probe. -- The stale decompression harness removals in `b804d7b` were documented and do - not hide a failing Pre13 DUT test. - -## Unrun or Non-Claimed Coverage - -Pre13 deliberately does **not** claim: - -- full feature-suite PASS; -- TC004 PASS; -- TC011 or TC150 PASS; -- TC143, TC144, or TC145 PASS; -- TC146/V02 positive explicit-extent PASS; -- H01 map objectization behavior; -- H07 zero-progress behavior; -- H05/H06/H02/H08 source behavior changes. - -The custom H04 fixture directly covers the actual H04 source change, so the -absence of TC011/TC150 is not a blocker for the scoped Pre13 result. Likewise, -the final smoke covers Plain/LZ4/LZMA smoke only; Deflate, ZSTD, partial, and -other codec feature cases remain outside the Pre13 claim unless specifically -listed above. diff --git a/docs/pre13-h03a.md b/docs/pre13-h03a.md deleted file mode 100644 index 81114b8..0000000 --- a/docs/pre13-h03a.md +++ /dev/null @@ -1,30 +0,0 @@ -# Pre13 H03a: bounded qstr comparator carrier - -## Scope - -- Added a private `struct erofs_qstr` to `src/namei.c` with explicit `name` - and `end` pointers. -- Converted `erofs_dirnamecmp()` to consume bounded qstr objects. -- Kept the existing search helpers on their original pointer/length interface - through a temporary local adapter. Block search, VOP lookup, vnode locking, - namecache timing, errno handling, readdir, and cookies are unchanged. - -## Static equivalence - -- Search and disk lengths are still derived from the same pointer ranges. -- `matched` is clamped to both ranges before comparison. -- Comparison remains unsigned byte ordering, supports non-NUL search keys and - embedded NUL bytes, and never assumes a trailing NUL at the disk boundary. -- The adapter performs no allocation, copying, normalization, or error mapping. - -## Comparator gate - -Tests baseline: `ce03c283e6e2cb8f1b69c8eff21f7d06098a4fbe`. - -- Python deterministic corpus: PASS, 11,562 cases. -- Independent C harness: PASS, 1,048,576 cases. - -These host comparator results prove the old and new comparison contracts are -equivalent for the generated corpus. They are not dynamic filesystem feature -test results. H03 still requires TC042, TC043, TC044, TC045, TC053, TC054, -TC123, TC141, and TC148, followed by the final smoke test. diff --git a/docs/pre13-h03b.md b/docs/pre13-h03b.md deleted file mode 100644 index f0b966e..0000000 --- a/docs/pre13-h03b.md +++ /dev/null @@ -1,39 +0,0 @@ -# Pre13 H03b: qstr block and dirent search - -## Scope - -- Converted `find_target_dirent()` and `erofs_find_target_block()` to accept a - bounded `struct erofs_qstr` search key. -- Constructed bounded on-disk qstr objects from the same validated dirent name - offsets and block end pointers used before this change. -- Constructed a temporary search qstr inside `erofs_namei()` while retaining - its original pointer/length external interface for H03c. -- Removed the H03a legacy comparator adapter. - -No changes were made to `dir.c`, readdir, cookies, buffer ownership, directory -validation, binary-search decisions, errno selection, cleanup, VOP lookup, -vnode locking, or namecache operations. - -## Static evidence - -- `erofs_dirnamecmp()` has exactly two call sites: the block-level first-name - comparison and the candidate block's dirent comparison. -- Both on-disk qstr end pointers use the same next `nameoff` or validated block - boundary as the previous pointer arguments. -- `erofs_find_target_block()` retains every `erofs_brelse()` path, candidate - replacement, `EINTEGRITY` assignment, and `ENOENT` distinction. -- `erofs_namei()` still releases the selected block once and returns `ENOENT` - only for a true miss. -- The `erofs_lookup()` control flow and all `cache_enter()`, `vn_vget_ino()`, - `erofs_vget()`, and `cn_lkflags` uses are outside this stage's diff. - -## Comparator gate - -Tests baseline: `ce03c283e6e2cb8f1b69c8eff21f7d06098a4fbe`. - -- Python deterministic corpus: PASS, 11,562 cases. -- Independent C harness: PASS, 1,048,576 cases. - -The comparator gates are host static-equivalence evidence, not dynamic feature -tests. H03 still requires TC042, TC043, TC044, TC045, TC053, TC054, TC123, -TC141, and TC148, followed by the final smoke test. diff --git a/docs/pre13-h03c.md b/docs/pre13-h03c.md deleted file mode 100644 index 4bed96a..0000000 --- a/docs/pre13-h03c.md +++ /dev/null @@ -1,36 +0,0 @@ -# Pre13 H03c: qstr VOP lookup adapter - -## Scope - -- Changed the private `erofs_namei()` interface to accept the bounded qstr - carrier directly. -- Moved search qstr construction to the `erofs_lookup()` FreeBSD VOP adapter, - using exactly `cn_nameptr` and `cn_namelen` after the existing length checks. -- Removed the temporary qstr construction from `erofs_namei()`. - -No changes were made to `componentname` validation, mutation rejection, `.` or -`..` handling, vnode lock mode, `vn_vget_ino()`, `erofs_vget()`, namecache -timing, errno mapping, readdir, directory cookies, or directory UIO handling. - -## Static evidence - -- The qstr search range is `[cn_nameptr, cn_nameptr + cn_namelen)` and does not - require or inspect a trailing NUL. -- The only `erofs_namei()` caller remains `erofs_lookup()`. -- A true `ENOENT` remains the only result eligible for negative `cache_enter()`; - `EINTEGRITY` and all other errors still return directly. -- `ISDOTDOT` still selects `vn_vget_ino()` with the original `cn_lkflags`; - ordinary hits still select `erofs_vget()` with the same lock flags. -- The H03c diff does not modify any `cache_enter()`, vnode lookup, lock, cookie, - readdir, or UIO statement. - -## Comparator gate - -Tests baseline: `ce03c283e6e2cb8f1b69c8eff21f7d06098a4fbe`. - -- Python deterministic corpus: PASS, 11,562 cases. -- Independent C harness: PASS, 1,048,576 cases. - -The comparator gates are host static-equivalence evidence, not dynamic feature -tests. Dynamic H03 validation remains pending for TC042, TC043, TC044, TC045, -TC053, TC054, TC123, TC141, and TC148, followed by the final smoke test. diff --git a/docs/pre13-h04-root-validation.md b/docs/pre13-h04-root-validation.md deleted file mode 100644 index d1776e2..0000000 --- a/docs/pre13-h04-root-validation.md +++ /dev/null @@ -1,44 +0,0 @@ -# Pre13 H04 Root Inode Validation - -## Scope - -H04 adds one mount-time format check in `src/super.c`: the decoded root inode -must have FreeBSD vnode type `VDIR`. - -The check runs after packed and metabox inode initialization and before xattr -prefix initialization or publication of `mp->mnt_data`. No vnode is created and -no permanent root inode reference is retained. - -## Error Contract - -A decodable non-directory root violates the EROFS filesystem structure. The -mount therefore fails with positive FreeBSD errno `EINTEGRITY`. - -This intentionally differs from Linux's negative `EINVAL` return while keeping -the same format rejection. `EINTEGRITY` matches the existing FreeBSD EROFS -contract for decodable on-disk metadata that violates filesystem invariants. - -## Ownership Review - -- The temporary `struct erofs_node` is stack-owned and has no independent - allocations or vnode references. -- Decode failures and the type rejection use the existing `fail` path. -- `erofs_sb_free()` releases extent-cache state, internal inodes, external - devices, and the primary GEOM device in the established reverse order. -- The failure occurs before xattr prefixes, mount data, mount flags, or a root - vnode are published. -- The legal directory-root path is unchanged after the new check. - -## Validation Gate - -The required targeted QEMU validation is: - -1. Build and load the current DUT module. -2. Mount the valid paired fixture, verify mode `040755`, read its proof file, - and unmount it. -3. Mount the non-directory-root fixture and require exact errno `EINTEGRITY`. -4. Verify mount, md, KLD, and GEOM resources return to zero, then remount the - valid fixture. -5. Verify the kernel log has no panic, trap, or lock warning. - -No full feature suite is part of H04. diff --git a/docs/pre13-low-a1.md b/docs/pre13-low-a1.md deleted file mode 100644 index 6d7d743..0000000 --- a/docs/pre13-low-a1.md +++ /dev/null @@ -1,21 +0,0 @@ -# Pre13 L-A1 - -This batch aligns the remaining low-risk helper vocabulary with Linux: - -- `compress.h` now uses the Linux include guard name. -- `erofs_inode_is_data_compressed()` replaces four equivalent FULL/COMPACT - tests that do not need to distinguish the two layouts. -- `erofs_addrmask()` directly uses the existing 48-bit feature helper. - -Static proof: - -- For every value from zero through `EROFS_INODE_DATALAYOUT_MAX`, the new - compressed-layout helper equals the replaced boolean expression. -- The helper was not substituted in `zmap.c`, where FULL and COMPACT remain - distinct cases. -- `erofs_is_48bit()` and `erofs_sb_has_48bit()` tested the same incompat bit; - address-mask values and control flow are unchanged. -- The old include guard and duplicate 48-bit helper have no remaining source - consumers. - -No build, QEMU, smoke, or feature test was run for this batch. diff --git a/docs/pre13-low-a2.md b/docs/pre13-low-a2.md deleted file mode 100644 index 9ee4506..0000000 --- a/docs/pre13-low-a2.md +++ /dev/null @@ -1,14 +0,0 @@ -# Pre13 L-A2 - -This batch mechanically aligns two local declaration groups with Linux: - -- The xattr prefix cleanup/init functions follow the public get/list entry - points and precede the ACL entry point. -- The shifted/interlaced decompressor descriptors precede the LZ4 descriptor. - -Only complete blocks moved. No prototype, function body, callback, conditional -compilation, or indexed descriptor slot changed. Pre-move and post-move hashes -of both xattr functions and all three descriptor definitions are identical; -the indexed descriptor table is byte-identical. - -No build, QEMU, smoke, or feature test was run for this batch. diff --git a/docs/pre13-low-b.md b/docs/pre13-low-b.md deleted file mode 100644 index 3dbfe3f..0000000 --- a/docs/pre13-low-b.md +++ /dev/null @@ -1,16 +0,0 @@ -# Pre13 L-B - -This batch completes the planned semantic aliases for internal EROFS NIDs and -byte offsets. Each changed object was selected from the Pre13 variable-level -whitelist. - -The aliases remain unsigned 64-bit types. No expression, cast, format string, -error path, field order, or control flow changed. File sizes, lengths, chunk -and extent indexes/counts, directory cookies, media sizes, timestamps, disk -fields, and overflow-only temporaries retain their previous types. - -Declarations and definitions were updated together. The resulting source diff -contains only type tokens, line wrapping required by those tokens, and this -record. - -No build, QEMU, smoke, or feature test was run for this batch. diff --git a/docs/pre13-low-c1.md b/docs/pre13-low-c1.md deleted file mode 100644 index 2f19277..0000000 --- a/docs/pre13-low-c1.md +++ /dev/null @@ -1,19 +0,0 @@ -# Pre13 L-C1 - -This batch aligns the endian type vocabulary of named multi-byte on-disk -fields with the corresponding Linux EROFS declarations. - -Only fields for which the same Linux structure and field use `__le16`, -`__le32`, or `__le64` were changed. Byte fields, arrays, field order, unions, -packing, macros, and all read/conversion sites remain unchanged. The direct -disk-type parameter of `erofs_xattr_ibody_size()` was synchronized. - -The FreeBSD compatibility aliases map each `__leXX` type to the exact previous -unsigned integer type. A host-side probe compared 119 structure/union -size/alignment and named-field offset entries before and after the batch; all -values were identical. `tests/pre13_ondisk_layout_probe.c` retains legacy -definitions and exhaustive assertions for every modified structure so the -same proof can be compiled with the final FreeBSD guest toolchain. Existing -kernel `_Static_assert` checks remain unchanged. - -No KLD build, QEMU, smoke, or feature test was run for this batch. diff --git a/docs/pre13-low-c2.md b/docs/pre13-low-c2.md deleted file mode 100644 index 3b36eeb..0000000 --- a/docs/pre13-low-c2.md +++ /dev/null @@ -1,14 +0,0 @@ -# Pre13 L-C2 - -The two planned zero-length on-disk tail arrays now use standard flexible array -declarators: - -- `erofs_xattr_ibody_header::h_shared_xattrs` -- `erofs_xattr_long_prefix::infix` - -The layout probe retains the previous `[0]` declarations and asserts equal -structure size/alignment and member offsets. Host Clang accepts both old and -new declarations with the same warning/error result and identical layouts. -The probe is also retained for the final FreeBSD guest compiler gate. - -No KLD build, QEMU, smoke, or feature test was run for this batch. diff --git a/docs/pre13-low-d.md b/docs/pre13-low-d.md deleted file mode 100644 index b7cf314..0000000 --- a/docs/pre13-low-d.md +++ /dev/null @@ -1,10 +0,0 @@ -# Pre13 L-D - -The LZ4 two-byte little-endian offset now uses FreeBSD's unaligned-safe -`le16dec()` helper. Boundary checks, input advancement, zero/back-reference -validation, partial decoding, padding validation, and error returns are -unchanged. - -An exhaustive 65,536-value comparison proved `ip[0] | (ip[1] << 8)` and -`le16dec(ip)` equivalent for every possible encoded offset. The final FreeBSD -KLD and LZ4 smoke gates remain required later; this phase does not run them. diff --git a/docs/pre13-low-e.md b/docs/pre13-low-e.md deleted file mode 100644 index 763785f..0000000 --- a/docs/pre13-low-e.md +++ /dev/null @@ -1,19 +0,0 @@ -# Pre13 L-E - -The BSD-only miscellaneous `erofs_defs.h` has been removed. Its three owners -now keep their constants in the files responsible for those values: - -- `super.c` owns the named private CRC32C seed. -- `dir.c` uses `sizeof(struct erofs_dirent)` directly. -- `decompressor_lz4.c` owns its private LZ4 format constants. - -The former header had exactly these three production source consumers. All -constant values and use-site expressions remain equivalent. Two already -retired userspace decompression test files still referenced the removed header; -they were deleted because they also depended on obsolete userspace ABIs and did -not compile independently. No current source or test entry retains the old -include. The LZ4 macro names remain private to `decompressor_lz4.c`; the former -shared CRC and dirent macro names have no current consumers. The current -architecture file was updated; historical reports were not changed. - -No KLD build, QEMU, smoke, or feature test was run for this batch. diff --git a/docs/pre13-stage0-decisions.md b/docs/pre13-stage0-decisions.md deleted file mode 100644 index 6316d93..0000000 --- a/docs/pre13-stage0-decisions.md +++ /dev/null @@ -1,46 +0,0 @@ -# Pre13 Stage0 Gate Decisions - -## Scope and Baseline - -- DUT repository baseline: `9028b92343ec7f5bbc1b07477b8d4a669c436955`. -- Clean tests-dev baseline: `ce03c283e6e2cb8f1b69c8eff21f7d06098a4fbe`. -- Stage0 was a planning gate. Infrastructure fixes made in tests-dev are not DUT changes. -- The complete Stage0 runner result is `FAIL`, not `PASS`, because the H01 DTrace probe failed. - -## Gate A: ROOT-NONDIR - -Status: `READY`. - -The baseline incorrectly mounts an image whose root NID points to a regular file. Pre13 freezes the expected rejection errno as `EINTEGRITY`. H04 may proceed with the dedicated fixture and its required targeted validation. - -## Gate B: ZERO-RUN - -Status: `BLOCKED/STOP`. - -The analysis covered 1,360 legal states and found no zero-progress state. H07 must be recorded as a no-op and must not be implemented in Pre13. - -## Gate C: Map Tuple Oracle - -Status: `BLOCKED/STOP`. - -The oracle reliably covers only 26 plain, inline, and hole tuples. It does not cover chunk, multidevice, bounds, or overflow behavior. H01a, H01b, and H01c must not be implemented in Pre13. - -## Gate D: Comparator - -Status: `READY`. - -The Python comparator corpus passed all 11,562 cases. The independent C harness passed all 1,048,576 cases. H03 may proceed within the planned boundaries and with its targeted validation. - -## Gate E: Device Options - -Status: incomplete and non-gating. - -The `device.N` runtime probe was not completed. Pre13 makes no runtime behavior claim from this gate. - -## Integrity and Cleanup - -- Stage0 must not be described as an overall pass. -- Stage0-owned QEMU, SSH, ports, overlays, and temporary runtime resources were cleaned up. -- The guard VM and read-only base image were not modified. -- The pre-existing dirty tests-dev working tree was unchanged. -- Tests-dev runner and infrastructure repairs are evidence support only; they are not EROFS DUT fixes. diff --git a/docs/pre15-stage0/P15-005.md b/docs/pre15-stage0/P15-005.md deleted file mode 100644 index 8c91afe..0000000 --- a/docs/pre15-stage0/P15-005.md +++ /dev/null @@ -1,40 +0,0 @@ -# P15-005 Stage0 Decision - -Status: `GO` for B06. - -The authoritative baseline replay used commit -`edf098905a34de764185e72fc7e92d7b8f4e7285`. The gate extracts and compiles -the current FreeBSD `data.c` and `zmap.c` producer bodies instead of replaying -an arithmetic model. Fourteen frozen cases cover plain, inline, hole, chunk, -48-bit multidevice, compressed explicit extents, fragment, partial-reference, -post-EOF, bounds, overflow, short-read, and post-acquire error behavior. Every -field in `(m_la,m_pa,m_llen,m_plen,m_deviceid,m_flags,errno,acquire_count, -release_count)` matched the independently frozen record oracle. - -The ownership oracle freezes 27 audited function bodies and 22 success, -validation-error, provider-error/short-read, transfer, and release paths across -plain data, compressed data/config, xattr, inode, superblock, and map consumers. -It also pins the corresponding Linux source hashes and `erofs_buf` acquire/put -semantic anchors. Candidate replay must satisfy the predeclared per-function -object/put/raw-buffer contract and execute the actual FreeBSD helper through -primary-image, metabox, error, overflow, idempotent-put, and null-callback paths. - -The supplied prep remains correctly classified `BLOCKED`: its static scan uses -obsolete `erofs_mount`/`erofs_node` text anchors and its Python map model does -not execute DUT code. `P15-005.sh` and `P15-005-input.json` close those gaps on -the current BASE. Evidence is in -`planning/pre15/evidence/20260814T145546Z-G02-P15-005/`. - -QEMU and the full feature suite were not run. G02 for P15-005 is a host/source -gate, and its candidate contract does not require either. This GO authorizes -B06 only; it does not authorize P15-006 or any B07 map-object change. - -## B06 completion - -B06 commit `1b1f904a674c21eed3ee29ca7df93c44429ac8d5` replayed the same -corpus and oracle with `object_mode=true`, `oracle_equal=true`, 14/14 tuples, -22/22 ownership paths, and five actual helper lifecycle paths. D, the B06 host -case, and both FreeBSD KLD configurations pass. Seven frozen FreeBSD raw I/O -and GEOM lifecycle functions are unchanged, and the pre-B07 map object hash is -unchanged. This completes the B06 dependency and unlocks B07a for its own -separate gated batch; it does not pre-approve a B07 implementation. diff --git a/docs/pre15-stage0/P15-006.md b/docs/pre15-stage0/P15-006.md deleted file mode 100644 index e469347..0000000 --- a/docs/pre15-stage0/P15-006.md +++ /dev/null @@ -1,60 +0,0 @@ -# P15-006 Stage0 Decision - -Status: `GO`; B07a is complete. - -The authoritative baseline replay used commit -`6673f51152a5195a8a8903aa801f820abce7936e`. This decision is independent of -the earlier P15-005 GO: P15-006 has its own corpus, model digest, source -extractors, byte encoding, and decision record. - -The frozen corpus contains 80 map tuples and 13 device-resolution cases. The -map tuples comprise 50 plain/inline/chunk producer cases and 30 compressed -explicit-extent cases. They cover plain and inline boundaries, holes, raw -32-bit chunks, indexed 32-bit chunks, indexed 48-bit chunks, masked and -nonzero device IDs, all four explicit extent record sizes, fragments, partial -references, exact and post EOF, provider/metabox bounds, checked arithmetic -overflow, invalid formats and algorithms, and errors both before and after -metadata acquisition. The device cases separately exercise primary, -multidevice, flat-device, unified-address, missing-provider, range, and -overflow behavior without changing GEOM ownership. - -Expected records are produced by an independent arithmetic and on-disk record -decoder whose frozen SHA256 is -`effcf0b6cc6a6e916eae89a14ad52d7b96273a453556647b6eabf2926fcb4abb`. -The gate separately extracts the real `data.c` and `zmap.c` producer bodies -from the frozen BASE, compiles them with `-Werror`, and compares every -`m_la`, `m_pa`, `m_llen`, `m_plen`, `m_deviceid`, `m_flags`, -`m_algorithmformat`, positive errno, acquire count, and release count. It -also packs those fields with fixed layout ` source filesystem - -> md(4) GEOM provider - -> md_s.s_vnode.vnode - -> another filesystem vnode -``` - -The final identity edge is not available through a generic, identity-preserving -FreeBSD API: - -* `VOP_GETLOWVNODE` exposes vnode-stack aliases such as nullfs and unionfs, but - it does not traverse a filesystem's GEOM storage dependency. -* `vfs_register_upper_from_vp` pins the immediate source vnode mount and orders - its unmount, but it does not register hidden GEOM-to-vnode backing edges. -* `md(4)` stores the held backing vnode in the private `struct md_s` defined in - `md.c`. Its GEOM object exposes only `void *softc`; dump configuration exposes - a pathname, not a held vnode identity. -* Re-resolving that pathname fails the rename/replace invariant. Casting - `g_geom.softc` to a copied private `struct md_s` is an undocumented, - class-specific dependency and does not cover other filesystem-private or - GEOM-private file-backed providers. - -Consequently a visible-only oracle can approve all vnode, credential, pager, -resize, and unmount checks while missing the hidden backing-vnode ancestor. The -gate includes that case as an adversarial false-GO control. There is no sound -place to return the required single cycle errno (`EDEADLK`) because the cycle -identity cannot first be discovered. - -## Result - -`P15-032` is **STOP** and B39 is **STOP-NO-SOURCE**. Recursive I/O and its lock -graph cannot be statically excluded for the requested regular-file source -surface using documented generic FreeBSD interfaces. A safe future GO needs a -new kernel dependency API that returns and pins transitive backing vnode -identities, or an explicitly narrower feature contract whose permitted source -filesystems have no hidden storage dependencies. Neither change is in B39's -authorized write set. - -Run from the repository root: - -```sh -repo-pre-15/tests/pre15/gates/P15-032.sh \ - --base bd5a09054e5cf89efd4db82aadb051f20b06ebf7 \ - --freebsd-src /work/build/freebsd-src \ - --output /absolute/owned/output/path -``` - -No EROFS source, Makefile, feature documentation, B39 case, or B39 fixture is -modified by this STOP addendum. D, H, K, Q, TC006, TC179, TC184, smoke, and the -full feature suite are not run at the gate stage. diff --git a/docs/pre15-stage0/P15-038.md b/docs/pre15-stage0/P15-038.md deleted file mode 100644 index a9c278a..0000000 --- a/docs/pre15-stage0/P15-038.md +++ /dev/null @@ -1,85 +0,0 @@ -# P15-038 Stage0 Decision - -Status: `GO`. B33 is authorized. - -The authoritative frozen-BASE run is -`planning/pre15/evidence/20260816T021736Z-G05-P15-038/` against -`5d6755649a369498a9b257bb2d1d1e3496d5135e`. Its state model, hard-budget -accounting, real codec oracle, eviction check, and owned cleanup all pass. - -The gate compares the current LZMA-only admission policy with a test-only, -codec-neutral single-entry decoded cache. Both variants execute the same 1,024 -deterministic random 4 KiB logical reads against the same 256 KiB payload for -each of LZ4, LZMA, Deflate, and Zstd. Each variant has exactly five cold-cache -samples; no sample may be discarded. The helper uses the frozen host -liblz4/liblzma/zlib/libzstd libraries and performs real compression and full -decode work. It contains no synthetic sleep. - -This is an independent host codec-cost oracle. It can prove avoided codec work -and byte correctness, but it is not guest vnode latency evidence and must not -be reported as such. Provider I/O, map lookup, VOP dispatch, and guest kernel -scheduling are outside its scope. - -The state model preserves B32's exact key, one-owner, waiter, typed-failure, -retry, and no-cache fallback rules. It additionally freezes mount-cache then -global-budget lock order, reservation-before-decode accounting, codec-neutral -work/size admission, disabled policy, global exhaustion, ready eviction, -inflight reclaim refusal, successful reclaim, unmount drain, and key reuse. -Reserved plus resident decoded bytes are charged to the same hard budget. - -G05 is GO only if the state model closes, all current/candidate hashes match, -eviction-before/after SHA-256 matches, every candidate sample stays within the -fixed budget, and at least two codecs improve median latency by 10 percent or -more. Any missing sample, correctness mismatch, lifecycle gap, or budget -overflow is STOP for P15-038 only. - -## Result - -All five current and five candidate samples are retained for each codec. The -median current/candidate latencies and improvements are: - -| Codec | Current median | Candidate median | Improvement | -|---|---:|---:|---:| -| LZ4 | 44,988,337 ns | 6,583,335 ns | 85.367% | -| LZMA | 7,278,779 ns | 7,286,634 ns | -0.108% | -| Deflate | 186,791,959 ns | 6,102,297 ns | 96.733% | -| Zstd | 20,355,035 ns | 6,236,635 ns | 69.361% | - -Three codecs exceed the required 10 percent threshold. LZMA is the expected -control because both current and candidate policies admit it; its negative -0.108 percent delta is retained and is not excluded or rewritten. - -Every candidate sample charges exactly 262,144 resident bytes against the -262,144-byte mount budget. The state model reaches the 524,288-byte global -limit with two mounts, proves a third mount's synchronous no-cache fallback, -reclaims one reservation, retries successfully, and returns to zero bytes. -It also passes 16-thread success and typed-failure waves, ready eviction, -inflight reclaim refusal, key reuse, policy disable, low-work/oversize bypass, -and unmount drain. The real Deflate decode before and after eviction has -SHA-256 `5d3c21088380524a78c52b067d47a93117c5385464c5dfc83e8de598119511bb`. - -No QEMU process was started by this pre-source gate. No guest vnode latency is -claimed. No production source, B32 correctness path, B32 QEMU runner, protected -PID 26318, port 9222, or shared base image was touched. B33 may now execute its -exact write set and acceptance matrix. - -## B33 Execution - -B33 source/test commit `bc56f830918b76029871b60cca2e53992de70a2e` -implements codec-neutral decoded-work/size admission with fixed hard mount and -global budgets, loader configuration/disable controls, four-codec accounting, -FreeBSD `vm_lowmem` reclaim, eviction, unmount release, and synchronous -no-cache fallback. It changes only `src/internal.h` and `src/zdata.c` in the -production tree and preserves the B32 key/inflight/failure contract. - -D, authoritative H `20260816T025825Z-host-B33-cache-policy-1393732-0`, and -FreeBSD 15 cross-KLD builds with zstdio0 and zstdio1 pass. The final H-tested -seven-path source/test tree is byte-identical to the committed tree. Q -TC168-cache-inflight is `INFRA_BLOCKED`: guest SSH was not ready at the exact -absolute 300-second deadline, so no guest source, case target, KLD load, vnode -latency, or guest PASS is claimed. Owned PID 1389807, overlay, and port 33537 -were cleaned; the protected process, port, and base metadata are unchanged. - -The Q attempt predates the final replacement-order and policy-bypass-accounting -changes and did not reach the DUT. Full correspondence details and exact hashes -are in `planning/pre15/evidence/20260816T023315Z-B33/VERDICT.md`. diff --git a/docs/pre15-stage0/P15-045.md b/docs/pre15-stage0/P15-045.md deleted file mode 100644 index 6df0043..0000000 --- a/docs/pre15-stage0/P15-045.md +++ /dev/null @@ -1,44 +0,0 @@ -# P15-045 Stage0 Decision - -Status: `STOP`. B35 is `STOP-NO-SOURCE`. - -G07 is replayed against frozen BASE -`13974efc00c31d8c13c8dccb7c65a82adafcbff6`. Its first mandatory condition -is a concrete, versioned support/deployment manifest in existing project -evidence that identifies deployment targets and explicitly requires Zstd to -be enabled by default. General codec availability, prior Zstd test success, -Linux defaults, and maintainer preference are intentionally nonqualifying. - -The gate inventories every tracked path in the frozen -`planning/pre15/evidence` tree (tree object -`3ecce7a124d153638853ab5d897126c769125309`, 2,560 paths), locates every text -record containing Zstd/Zstandard, and reviews records with deployment, -support, manifest, policy, default, or requirement signals. It accepts either -a structured JSON manifest or explicit text manifest headers, but requires a -manifest kind, version, nonempty deployment/support targets, and an -affirmative Zstd-default requirement together. The replay finds 596 Zstd text -records and 206 demand-signal candidates; no record meets that contract. - -The missing deployment-demand prerequisite determines STOP before runtime or -build work. The disabled exact-`EOPNOTSUPP` runtime check, enabled real Zstd -EROFS read, FreeBSD kernel `ZSTDIO` symbol/capability check, and KLD size delta -measurement are therefore `NOT_RUN`; none is reported as PASS. In particular, -the gate does not claim or attempt to load a generic dependency KLD. A -false-GO guard requires every G07 condition to be PASS. - -The frozen source is checked only to preserve the current policy boundary: -`src/Makefile` contains exactly one `WITH_ZSTDIO?= 0` default, the disabled -stub returns `EOPNOTSUPP`, and the documentation names kernel -`options ZSTDIO`. These are static observations, not substitutes for the -runtime conditions omitted after the prerequisite STOP. - -The committed replay evidence is recorded under -`planning/pre15/evidence/20260816T033449Z-G07-P15-045/`. The gate owns one -temporary directory, uses a 90-second internal absolute timeout plus the -recorded outer timeout, records argv, source/candidate hashes, exit status, -and cleanup, and rejects an existing output directory. - -No production source, B35 case, B35 fixture, QEMU process, full feature suite, -or smoke suite is run or changed. Protected PID 26318, port 9222, and the -shared base image are outside the gate. The supported policy remains opt-in -`WITH_ZSTDIO=1`, with default `WITH_ZSTDIO?=0`. diff --git a/docs/pre15-stage0/P15-052.md b/docs/pre15-stage0/P15-052.md deleted file mode 100644 index 6fdf614..0000000 --- a/docs/pre15-stage0/P15-052.md +++ /dev/null @@ -1,74 +0,0 @@ -# P15-052 Stage0 Decision - -Status: `STOP`. B20 is complete as `STOP-NO-SOURCE`; B21 was not started. - -The authoritative G06 replay uses frozen BASE -`ca7bb4fe6b33e4a1bdf423801134b0ed6bda86dd`. It verifies the exact current -FreeBSD and Linux source hashes, the FreeBSD amd64 `PAGE_SHIFT=12` contract, -signed 64-bit `off_t`, and GEOM's `off_t mediasize` before evaluating every -listed branch. - -## Exact Branch Result - -All twelve proposed on-disk arithmetic branches have unique source sites, but -none has a target marker reachable from one validated on-disk field mutation. -The current source text returns positive `EOVERFLOW` at those defensive sites; -the proposed mapping would be positive `EINTEGRITY`. Linux has no matching -checked branches in these functions, so its relevant semantic mapping is -negative `-EFSCORRUPTED`, not a textual negative copy of the FreeBSD return. - -The decisive counterexample to the supplied READY prototype is -`xattr.metadata.header_add`. The prototype directly mutates the local -`aligned_off` value. The real helper first rejects input above -`UINT64_MAX - 3`, then rounds to four bytes. The greatest surviving aligned -offset is therefore `UINT64_MAX - 3`; adding the two-byte header cannot -overflow. The named branch cannot be independently reached. - -The other prototype vectors likewise inject values outside current provenance: -mounted image bytes are bounded by GEOM's signed `off_t mediasize`; inode size -is bounded by `OFF_MAX`; decoded physical blocks are at most 48 bits; -`blkszbits` is at most 12; prefix start and shared IDs are 32-bit. These bounds -prevent each proposed shift, add, alignment, and index overflow before the -listed target. - -## Preserved Semantics - -The replay separately freezes corruption, unsupported, provider I/O, EOF, and -short-read behavior. Disk/backing range contradictions remain positive -`EINTEGRITY`; exact zero-length EOF remains success; provider `EIO` and media -`ENXIO` remain exact; unsupported xattr layout remains positive -`EOPNOTSUPP`; allocation remains positive `ENOMEM` and outside B20. Linux -counterparts retain negative errno or `PTR_ERR` propagation. - -The nominal `erofs_xattr_read_backing()` `off > INT64_MAX` positive -`EOVERFLOW` site is also not reachable for a mounted primary provider: the -preceding range check requires `off <= backing_size`, while mount validation -requires `backing_size <= INT64_MAX`. This site remains unchanged because a -STOP decision has no source diff. - -## Atomic Decision - -G06 requires every listed branch to be independently reachable. One missing -branch stops all of P15-052; this replay finds twelve missing target markers. -No `data.c` or `xattr.c` errno is changed, no B20 case/fixture is created, and -no candidate replay or QEMU run can cure a host-proven reachability failure. -The full feature suite was not run. - -The authoritative command was: - -```sh -timeout -k 10 240 tests/pre15/gates/P15-052.sh \ - --base ca7bb4fe6b33e4a1bdf423801134b0ed6bda86dd \ - --output OWNED_OUTPUT -``` - -It exited 1 because a valid gate `STOP` is not `GO`. Two fresh output -directories were byte-identical. The initial replay produced result SHA256 -`37280545d8ef5a6b87c7b9d536939513a1bbac8e4e4b2389e84f5ca77e1aa522`, -branch-ledger SHA256 -`8024a2cb6b43bd10fbcf446cf8cf44de37f1f9d6df25718c641cb9db4c257169`, -and preservation-ledger SHA256 -`d17e05b8df43ea5173bed6e58d7431b37a7a9c4a143e07717b543efcb54ecc66`. - -B21 is not authorized because the execution request requires B20 PASS before -B21. Wave16 is therefore not ready from this serial chain. diff --git a/docs/pre15-stage0/P15-057.md b/docs/pre15-stage0/P15-057.md deleted file mode 100644 index efa68d3..0000000 --- a/docs/pre15-stage0/P15-057.md +++ /dev/null @@ -1,13 +0,0 @@ -# P15-057 Stage0 Decision - -Status: `GO`; B17 is complete. - -The B17 fixture gate contains 25 reproducible EROFS images and independently -freezes all legal results and damaged positive FreeBSD errnos. The source -commit `433cf4ec66478b65b291ec5b21a0bf6d806bd14f` preserves the raw xattr filter -feature declaration, saves its reserved byte, and gates the current filter -format only at the ACL use site. Reserved values 0, 1, and 255 and feature -present/absent combinations pass host and minimal FreeBSD 15 QEMU replay. - -Evidence is in `planning/pre15/evidence/20260814T203439Z-B17/`. This decision -does not authorize the separate cache, Bloom, or ordering batches. diff --git a/docs/pre15-stage0/P15-058.md b/docs/pre15-stage0/P15-058.md deleted file mode 100644 index 8e49c98..0000000 --- a/docs/pre15-stage0/P15-058.md +++ /dev/null @@ -1,13 +0,0 @@ -# P15-058 Stage0 Decision - -Status: `GO`; B17 is complete. - -The B17 corpus separately covers inline names, shared names, and long-prefix -infixes containing embedded NUL, with independently frozen `EINTEGRITY` (97) -reject points. It also covers legal ACL empty-suffix names and short/long -name-index failures. Source commit -`433cf4ec66478b65b291ec5b21a0bf6d806bd14f` rejects only the length-delimited -embedded-NUL formats and retains the FreeBSD namespace and extattr ABI. - -Evidence is in `planning/pre15/evidence/20260814T203439Z-B17/`. This decision -does not include xattr cache, Bloom, or call-order changes. diff --git a/docs/pre15-stage0/P15-062.md b/docs/pre15-stage0/P15-062.md deleted file mode 100644 index df644e6..0000000 --- a/docs/pre15-stage0/P15-062.md +++ /dev/null @@ -1,89 +0,0 @@ -# P15-062 Stage0 Decision - -Status: `STOP`. B26 is `STOP-NO-SOURCE`; no production source, B26 case, or -B26 fixture is authorized. - -The decision is bound to DUT BASE -`205a90465edb64e83ba44aaacac9bedad4cbe905`, FreeBSD source HEAD -`106727738dcfb6c001b46f25363b91cece970085`, and exact upstream erofs-utils -commits for v1.4, v1.7, and v1.8.6. The authoritative gate evidence and final -script/input hashes are recorded below and in the G04 evidence commit. - -The later B26 terminal review is recorded at -`planning/pre15/evidence/20260817T-B26-terminal-review/VERDICT.md`. -It retains `STOP_NO_SOURCE / NOT_TESTED` and adds secondary -`TERMINAL_REVIEWED_NO_SOURCE`. It found the historical zero-tail shape to be -reproducible, but did not establish that trailing zeros are format-mandated or -that current FreeBSD rejects them. - -## Authoritative Run - -The gate ran from `/work/erofs-freebsd-pre` as: - -```sh -timeout -k 10 240 repo-pre-15/tests/pre15/gates/P15-062.sh \ - --base 205a90465edb64e83ba44aaacac9bedad4cbe905 \ - --output planning/pre15/evidence/20260815T062257Z-G04-P15-062/gate-output -``` - -It returned the gate-defined STOP exit status 1. Evidence is rooted at -`planning/pre15/evidence/20260815T062257Z-G04-P15-062/`. The authoritative -script SHA-256 is -`6af503f1612eb42410f91a54fdc37481bae86a4419232c9de48c08bfa7ecee5d` and -the input SHA-256 is -`390202738ac006aba2514d27d3b0a19a0123a009601a26cac3b560a3dff388e5`. -The result SHA-256 is -`597466203edcaee6d163651a6d6a05e49dff24f39201234911f9753b29eebbe0`. - -## Decision Rule - -G04 permits P15-062 only if every normative and historical full LZ4 input uses -the same exact-consumption rule. The B26 review found that the old fsck -acceptance path uses `LZ4_decompress_safe_partial()` for legacy/no-0padding -images, so that acceptance does not prove full physical consumption or format -legality. The format sources contain no stored compressed-stream length or -general trailing-zero rule. The current FreeBSD decoder accepts an all-zero -remainder and rejects nonzero remainder bytes; no implementation defect was -demonstrated. - -## Reproducible Corpus - -The gate builds three upstream mkfs generations from exact commits in an owned -temporary directory. Each version twice generates the same fixed-time, -fixed-UUID, root-owned, xattr-free legacy LZ4 image from the same deterministic -1 MiB file. Matching `fsck.erofs --extract` accepts each image. The image, -source, build tree, and binaries remain temporary and are not committed. - -The independent parser reconstructs the complete file from two physical -pclusters. The final extent has `m_plen=4096`, but its raw LZ4 stream reaches -9,670 decoded bytes after consuming 48 bytes; the remaining 4,048 bytes are -zero. This shape is byte-reproducible in v1.4, v1.7, and v1.8.6. The current -FreeBSD decoder accepts the whole input and liblz4 accepts the 48-byte exact -stream but rejects the block-sized input. The zero-tail shape is therefore a -historical tool-compatible candidate, not a proven format-legal fixture. - -## Partial and Corruption Controls - -A separate v1.8.6 `-Ededupe` image contains two real noncompact -`Z_EROFS_LI_PARTIAL_REF` records that share an LZ4 pcluster. The independent -parser, liblz4 partial API, and frozen DUT callback agree on the 4,096- and -5,594-byte prefixes. Corruption beginning after the 4,096-byte request remains -undetected by that partial read but causes the same fixture's full decode to -return positive `EINTEGRITY` (97), as required by G04. Truncation and nonzero -tail controls also return 97; all output guards remain unchanged. - -Production `zdata.c` ownership anchors show that metadata/physical input is -released after every decoder return, failed decoded output is freed, and only -successful output is published. The decoder itself allocates and owns no -buffers. The gate's owned build/image directory is removed, and it does not -start QEMU or touch protected PID 26318, port 9222, or the shared base image. - -## Consequence - -P15-062 remains STOP for Pre15 because B26 has no authorized production source, -case, or fixture. `src/decompressor_lz4.c`, -`tests/pre15/cases/B26-lz4-input.sh`, and `tests/pre15/fixtures/B26-*` remain -unchanged or absent as applicable. B26 acceptance D, exact-ABI build, TC167 -QEMU, and the full feature suite are not run because no implementation error -was demonstrated. The secondary B26 status is -`TERMINAL_REVIEWED_NO_SOURCE`. diff --git a/docs/pre15-stage0/P15-076.md b/docs/pre15-stage0/P15-076.md deleted file mode 100644 index 74ba158..0000000 --- a/docs/pre15-stage0/P15-076.md +++ /dev/null @@ -1,75 +0,0 @@ -# P15-076 Stage0 Decision - -Status: `STOP`. B29 is `STOP-NO-SOURCE`. - -The gate is bound to frozen DUT BASE -`3e9bc3f03ba9c39c38cc40f2f08eb6e769557f55` and FreeBSD source HEAD -`106727738dcfb6c001b46f25363b91cece970085`. Three host attempts are retained: - -- `20260815T170841Z-G05-P15-076`: `RUNNER_FAIL`, FreeBSD headers shadowed - glibc headers during host compilation. -- `20260815T170931Z-G05-P15-076`: `RUNNER_FAIL`, the Deflate oracle compared - one selected extent with the complete source payload. -- `20260815T171051Z-G05-P15-076`: the runner returned 0, but final review - rejects its local `GO` because mandatory state-model conditions are not - closed. - -## Last Attempt Command - -```sh -timeout -k 10 240 repo-pre-15/tests/pre15/gates/P15-076.sh \ - --base 3e9bc3f03ba9c39c38cc40f2f08eb6e769557f55 \ - --output planning/pre15/evidence/20260815T171051Z-G05-P15-076/gate-output -``` - -The command returned 0 in three seconds. The gate script SHA-256 is -`1d2cc18de150c8bdeb4fa8d284ab345f9ec4249c2ba1b50e1badbe7f942b42dc`, -the input SHA-256 is -`30497cf86785c367dcd4a3d6889ba6a6302f6f133d68e4af31d984a1031e0058`, -and the result SHA-256 is -`ac007afc4ad5a97bb1561888105c67612d3eea5427f77c54a858bede4c86e0a3`. - -## Benefit Result - -The last attempt regenerates the frozen B28 EROFS fixtures in owned temporary -storage and decodes the exact selected LZMA, Deflate, and Zstd extents. Input, -expected output, and output buffers are outside the tracked allocator region. -Baseline creates and destroys a context per decode; the prototype resets one -context across 64 exact decodes. Seven samples are retained per mode. - -| Codec | Baseline events | Reused events | Reduction | Measured peak | -|---|---:|---:|---:|---:| -| LZMA | 128 | 2 | 98.4375% | 28,504 B | -| Deflate | 256 | 4 | 98.4375% | 39,928 B | -| Zstd | 128 | 2 | 98.4375% | 95,992 B | - -All tracked baseline backend allocator events are context lifecycle events, so -the measured share is 100 percent and passes the 10 percent threshold. All -three event reductions pass the 25 percent threshold. Exact output and -balanced allocation/free counts pass. CPU remains diagnostic: Deflate is -1.30 percent faster, LZMA is 10.20 percent slower, and Zstd is 15.35 percent -faster by median per-decode time; the LZMA regression is retained. - -## STOP Review - -G05 requires both the benefit threshold and a complete pre-source state model. -The retained runner does not establish a hard memory limit: its 329,616-byte -mount and 2,636,928-byte global calculations use peaks from representative -streams. In particular, its Zstd prototype sets `ZSTD_d_windowLogMax` to 16, -while the frozen DUT accepts values through 20. The measured 95,992-byte Zstd -peak therefore is not a hard bound for the production-supported range. LZMA is -also sampled with a 65,536-byte dictionary while the DUT accepts up to 8 MiB. - -The state model also does not execute a global-exhaustion transition. It sets -`global_owned` and `mount_owned` to the mount limit of two and selects fresh -allocation because the mount is full; the independent global limit of 16 is -never reached. Text describing global exhaustion cannot replace that missing -assertion. Therefore the hard-cap and global-exhaustion requirements are not -closed, and the runner-local `GO` cannot authorize source work. - -No threshold, sample, or result is changed to force a decision. Per G05, one -failed mandatory condition makes P15-076 `STOP`; B29 is recorded -`STOP-NO-SOURCE`. No production `src/**`, B29 case, fixture, or helper was -changed. B29 H/K/Q acceptance, QEMU, and the full feature suite were not run. -The hard-timeout D check passed. Owned temporary cleanup passed; protected PID -26318, port 9222, and the shared base image were untouched. diff --git a/docs/pre15-stage0/P15-081.md b/docs/pre15-stage0/P15-081.md deleted file mode 100644 index f268086..0000000 --- a/docs/pre15-stage0/P15-081.md +++ /dev/null @@ -1,95 +0,0 @@ -# P15-081 Stage0 Decision - -Status: `GO`. B11 is authorized; no production source was modified by this -decision. - -The authoritative G11 replay uses frozen B25 BASE -`e2e3fb86b6fffcb01d6fd29c17dd95628ad070de`, FreeBSD source HEAD -`106727738dcfb6c001b46f25363b91cece970085`, and -`mkfs.erofs (erofs-utils) 1.8.6`. The gate script and input SHA256 values are -`6215a3005214ba8d25dacd63e039320a0c63308d2334b5faff99a3927cad7d2e` -and `a887fad56927545adb48462b770100d1e64b22b32554ed8ffad14c72b8febfc3`. - -## Real Disk and Normal Entrypoint - -The gate independently materializes a deterministic source tree containing -regular, directory, character-device, block-device, FIFO, socket, symlink, -and two hardlink names. It invokes mkfs twice with fixed UUID, time, ownership, -worker count, xattr policy, and inline-data policy. It also fixes and restores -the process umask and explicitly sets every source-node mode. Complete -replays under outer umask `022` and `077`, including both seed generations in -each replay, are byte-identical. - -The gate parses the actual superblock, checksum range, root inode, inline root -directory, 12-byte dirents, NIDs, and compact inode modes without using DUT -helpers. Every derived image changes one real dirent byte and recomputes the -superblock CRC32C. `dump.erofs --ls --path=/` observes the mutated on-disk -`file_type`, while `dump.erofs --path=/NAME` reaches the same NID and reports -the inode kind from its mode. This gives 13 normal namespace resolutions from -real disk fields: eight known mismatches and five forward-compatibility cases. - -All eight known mismatches are checksum-valid, fsck-clean images and yield -positive FreeBSD `EINTEGRITY` in the independent candidate oracle. Type zero, -nonzero dirent reserved bytes, and out-of-range type 8/255 remain accepted by -the DUT policy and candidate oracle. erofs-utils fsck separately rejects 8/255; -the evidence records that as `policy-reject` rather than falsely claiming -fsck-clean. P15-093 already freezes the FreeBSD behavior for these extension -values as `DT_UNKNOWN`, so fsck's stricter userspace policy does not override -the G11 compatibility boundary. - -## Cache and Lock Boundary - -`regular` and `regular-hard` are different namespace keys with the same real -NID. The first normal lookup can instantiate the vnode; the second name misses -that namecache key and reaches the existing `erofs_vget()` path, whose frozen -body checks `vfs_hash_get()` before inode decode. Mutating only the second -dirent to known directory type therefore provides the required cached-vnode -trigger without adding a readdir-time vget. - -The generated candidate patch adds no lock, vget, hash, or recursive lookup -call. It reads immutable `vtype` after the existing child lookup, returns -positive `EINTEGRITY` for a known mismatch, drops the locked child with -`vput()`, and runs before `a_vpp` and namecache publication. Dotdot's existing -`vn_vget_ino()` path explicitly bypasses the validator because root `..` can -return the directory vnode itself; this preserves its parent-lock contract. -The parent/child lock order, readdir cookies, and VFS/VOP entrypoints are -unchanged. Readdir still contains no vnode lookup. - -Linux supplies the format mapping: EROFS file type values match generic Linux -`FT_*`, and Linux readdir maps those values with `fs_ftype_to_dtype()`. The -candidate aligns that known-type mapping while retaining FreeBSD vnode types, -the FreeBSD 15 `__enum_uint8(vtype)` ABI type, positive errno, VFS locks, and -forward handling for unknown values. - -## Replay Result - -The authoritative command was: - -```sh -timeout -k 10 240 tests/pre15/gates/P15-081.sh \ - --base e2e3fb86b6fffcb01d6fd29c17dd95628ad070de \ - --output OWNED_OUTPUT -``` - -It exits zero with `GO`: 14 generated images, eight known-match records -including the hardlink alias, eight known mismatches, five compatibility -cases, 13 normal entrypoint observations, one cached-vnode sequence, and 21 -compiled prototype records. Two fresh output directories are byte-identical. The fixture-set -SHA256 is -`b562a7e42e139b16f3ce399d585aa7c373a66aa478a084bfed67b2b2aa9f2bd3`. - -Key immutable evidence SHA256 values are: - -- `result.json`: `4ec8e053c60ae75ae2fc4580a3d820a0452cf519f6b85376953135e18a7ec0ec` -- `oracle.json`: `932e558d3ead05572f38c89635bb10cae45393e42028031f81dfc7a56f2093f6` -- `lock-ledger.json`: `e8c2e6ba797fbcc83d832af6583d3f74f344750f6897111e7fe501130e577c13` -- `fixture-index.json`: `1a46b8ebfec16109c5d193001dc650a1b56000045e5d9e1e9b145fe7c5df9a16` -- `candidate.patch`: `427097da9304f5bedb770be8cab5f589706d316fb18a152ca16cb9d0663713fd` -- `prototype.c`: `1a295d18830c61a9708d465c3b1e415efb3ff8276100e4e32139bfa715195d8b` -- `normal-entry.tsv`: `b2841448068f70633ed0c02d901177a639cfed150f0e1c58977cc6c7b1e7313e` -- `SHA256SUMS`: `36b486617920b04ca87e88016e2fc0f519cda3a5f99dc675c9a89549615a02e3` - -QEMU and the full feature suite were not run for this pre-source gate. The -gate uses host-created real EROFS images, independent binary parsing, normal -userspace namespace resolution, frozen DUT/FreeBSD control flow, and a -compiled prototype that is not linked into the DUT KLD. diff --git a/docs/pre15-stage0/P15-083.md b/docs/pre15-stage0/P15-083.md deleted file mode 100644 index 73cc2a5..0000000 --- a/docs/pre15-stage0/P15-083.md +++ /dev/null @@ -1,101 +0,0 @@ -# P15-083 Stage0 Decision - -Status: `GO`. G04 authorizes B27 for all three non-LZ4 codecs. The source batch -must preserve the exact policies below and may not import the P15-062 LZ4 STOP -rule. - -The gate is bound to DUT BASE -`68bbe94c44e35d53cec8ab55d007f40b01cf0502`, FreeBSD source HEAD -`106727738dcfb6c001b46f25363b91cece970085`, erofs-utils 1.8.6, liblzma -5.8.1, zlib 1.3.1, and libzstd 1.5.7. - -## Authoritative Run - -The gate ran from `/work/erofs-freebsd-pre` as: - -```sh -timeout -k 10 240 repo-pre-15/tests/pre15/gates/P15-083.sh \ - --base 68bbe94c44e35d53cec8ab55d007f40b01cf0502 \ - --output planning/pre15/evidence/20260815T142600Z-G04-P15-083/gate-output -``` - -It returned 0 and recorded `GO`. Evidence is rooted at -`planning/pre15/evidence/20260815T142600Z-G04-P15-083/`. The authoritative -script SHA-256 is -`6847541266b0668ad12442ffb54b67a772a29ed8ebe6fba0e1e25214db16b2c2`, the -input SHA-256 is -`b3c71b4ae03c6f9511246813952aab3178a1451433ece96f04562ec7cc6c1d1e`, and -the result SHA-256 is -`b9431b3819d64cdf940dff05ce7bc0e0bdf44c3ec647c90592052af5f7b3e544`. - -## Decision Rule - -P15-083 is GO only if LZMA, Deflate, and Zstd each have a reproducible real -EROFS fixture, an independent consumed-byte oracle, and a codec-specific policy -that distinguishes EROFS leading zero padding from unread nonzero bytes. Any -missing codec oracle, legal image rejected by the proposed full-stream policy, -or inability to distinguish trailing garbage makes the whole candidate STOP. - -## Reproducible Corpus - -erofs-utils 1.8.6 twice generates each fixed-time, fixed-UUID, root-owned, -xattr-free image from the same 151,552-byte source. Matching fsck extraction -reconstructs the complete source. The image hashes are: - -| Codec | Image SHA-256 | Selected real extent | Leading zero padding | Stream | -|---|---|---:|---:|---:| -| LZMA | `c26cf15844fe45a21a746bacbad2ef551c683bb9b2538de7a7eced37d8eadff9` | 4,096 + 4,096 | 3,556 | 540 | -| Deflate | `39455150c3e999bb7ae6c36402c15a408a5679726b6d099e7d121e009ef43af6` | 28,672 + 4,096 | 1,479 | 2,617 | -| Zstd | `b905803f0e08500cc3c6cfe07a95fe027859165acae19ca8865856af256f32ca` | 4,096 + 4,096 | 1,092 | 3,004 | - -The leading zero bytes are legal EROFS pcluster padding. They are removed by -the common dispatch before the codec callback, matching Linux -`z_erofs_fixup_insize()`. Every remaining legal stream reaches the format end, -consumes every byte, reproduces the selected logical extent, and leaves its -output guards intact. - -## Per-Codec Policy - -- Deflate: a full raw stream must reach `Z_STREAM_END` with no unread input. - The independent zlib decoder consumes 2,617 of 2,625 bytes after an 8-byte - nonzero tail and reproduces the output. fsck also accepts that mutation, as - does the Linux streaming loop when output is already full, but the bytes are - not EROFS leading padding and no mkfs fixture emits them. The audited FreeBSD - full policy rejects them with positive `EINTEGRITY` (97). -- LZMA: MicroLZMA stores no end marker and requires an exact compressed size. - The legal 540-byte stream consumes all bytes. Adding the same tail produces - liblzma status 9 after consuming 540 of 548 bytes; fsck rejects it and the - FreeBSD policy returns 97. -- Zstd: exactly one frame must finish with no unread input. The library finds - frame end after 3,004 of 3,012 bytes with the tail present; fsck rejects the - source-size mismatch and the FreeBSD policy returns 97. A concatenated or - skippable second frame is not EROFS pcluster padding. - -This is deliberately codec-specific. It aligns Linux's non-LZ4 leading-padding -placement and stream completion semantics without copying the Linux wrappers' -implicit unread-byte acceptance into the FreeBSD provider path. - -## Partial, Corruption, and Cleanup - -The independent libraries decode real-stream prefixes that match the full -slice: Deflate produces 3,587 bytes after consuming 715, LZMA produces 4,096 -after consuming 352, and Zstd produces 4,096 after consuming 1,457. Replacing -the final 64 stream bytes with zero begins strictly after each partial -consumption boundary. The same partial request still succeeds and matches; -full decode and fsck both fail for all three codecs, with the audited FreeBSD -policy mapping the full failure to 97. Removing the final compressed byte also -returns 97 for all three. - -Every library path reports cleanup complete and unchanged guards. Frozen -`zdata.c` releases metadata or physical input after the callback, frees failed -decoded output, and publishes only successful output. The owned temporary tree -was removed. QEMU, protected PID 26318, port 9222, and the shared base image -were not touched. The full feature suite was not run. - -## Consequence - -B27 may now make these three existing policies explicit in the exact planned -write set and add TC176 host/QEMU coverage. It must preserve positive FreeBSD -errno, optional Zstd ABI, provider/GEOM ownership, input release, failed-output -free, and successful-output lifetime. The generated images, extracted files, -oracle source, and oracle binary remain temporary and are not committed. diff --git a/docs/pre15-stage0/P15-086.md b/docs/pre15-stage0/P15-086.md deleted file mode 100644 index 0dd28a5..0000000 --- a/docs/pre15-stage0/P15-086.md +++ /dev/null @@ -1,147 +0,0 @@ -# P15-086 Stage0 Decision - -Status: `GO`. G04 and G05 authorize B28 partial subextent decoding for LZ4, -LZMA, and Deflate. Zstd is not authorized for partial decoding and must use the -exact full-decode fallback for ordinary strict subextent reads. Existing -partial-reference maps retain their bounded-prefix path. Shifted, interlaced, -unknown, and future backends also remain on full fallback unless a new gate -authorizes them. - -The gate is bound to DUT BASE -`6bf5724619be70f805bbe7d1ba77dd70cdced69f`, FreeBSD source HEAD -`106727738dcfb6c001b46f25363b91cece970085`, erofs-utils generations v1.4, -v1.7, and v1.8.6 for LZ4, erofs-utils 1.8.6 for the stream codecs, liblz4 -1.10.0, liblzma 5.8.1, zlib 1.3.1, and libzstd 1.5.7. - -## Authoritative Run - -The gate ran once from `/work/erofs-freebsd-pre` as: - -```sh -timeout -k 10 240 repo-pre-15/tests/pre15/gates/P15-086.sh \ - --base 6bf5724619be70f805bbe7d1ba77dd70cdced69f \ - --output planning/pre15/evidence/20260815T154915Z-G04-G05-P15-086/gate-output -``` - -It returned 0 and recorded G04=`GO`, G05=`GO`, and B28=`AUTHORIZED`. -Evidence is rooted at -`planning/pre15/evidence/20260815T154915Z-G04-G05-P15-086/`. The authoritative -script SHA-256 is -`591e407f20a6c8ab5d506329b3bd39ff7bd7f78b66ac86b9db2c45a498b31184`, the -input SHA-256 is -`62f7f505aef846fa85085039b2f47e252e11602e6ef1bf41f56841cdc9089a2a`, and -the result SHA-256 is -`33ef86b73447cd8c8099dc33901b9c4fff6c8ecedda3837d25cac9463a369d4f`. - -## Decision Rule - -Each codec is decided independently. A codec is partial-capable only when real, -twice-reproduced EROFS input decodes every requested range to the exact full -slice, reports a consumed-byte boundary before the end of the stream, leaves -guard pages unchanged, preserves positive FreeBSD errno, and reduces peak -temporary bytes by at least 20 percent without exceeding the fixed hard budget. -Failure of the benefit subgate selects exact full fallback for that codec; an -incorrect slice, unbounded memory, missing corruption boundary, or missing -oracle stops P15-086. - -## Reproducible Corpus - -All images are fixed-time, fixed-UUID, root-owned, and generated twice with -identical bytes. Matching fsck reconstructs every legal source. - -| Codec | Generator | Image SHA-256 | Logical bytes | Physical bytes | Leading zero bytes | Stream bytes | -|---|---|---|---:|---:|---:|---:| -| LZ4 | v1.4 | `8f940673ed9f7e00efc5c0f06d41b2216748ea84256b740751aabff0d632de2b` | 1,038,906 | 4,096 | 0 | 4,096 | -| LZ4 | v1.7 | `93451e563f4eaf95b2690381074b5dfb3da0257691043c51619fd7d4c68dba7e` | 1,038,906 | 4,096 | 0 | 4,096 | -| LZ4 | v1.8.6 | `853649c78b159161421a6e833e0c516216382f3e8d135fc80aacd509ddb27b2c` | 1,038,906 | 4,096 | 0 | 4,096 | -| Deflate | 1.8.6 | `5f98ff39be30b1396be8164e54a8ae80b144d181d5295a640b2e7814406d7a2d` | 22,878 | 4,096 | 1 | 4,095 | -| LZMA | 1.8.6 | `887d4a9baf629533c8d512177a84ca256a20135157d29a201408534f63230889` | 151,552 | 4,096 | 3,556 | 540 | -| Zstd | 1.8.6 | `93f6beb46e77187ea8b0fc1ad3a5a1cb565d546d9ce40d51278f26d67187d5fa` | 151,552 | 4,096 | 1,092 | 3,004 | - -The LZ4 corpus spans all three historical generators required by G04. Stream -codec consumption is reported by independent liblzma, zlib, and libzstd -decoders after the common EROFS leading-zero padding is removed. LZ4 is checked -by an independent raw parser and liblz4 partial/full output. - -## Hard Budget and Benefit - -The fixed input cap is 1 MiB, the decoded-output cap is 12 MiB, and the caller -cap is 1 MiB. Codec workspace caps are 0 for LZ4, 512 KiB for Deflate, 9 MiB -for LZMA, and 4 MiB for Zstd. These caps cover the frozen format maxima rather -than only the selected 4 KiB pclusters. - -| Codec | Workspace full/partial | Hard cap | Baseline peak | Candidate peak | Reduction | CPU median partial/full | Decision | -|---|---:|---:|---:|---:|---:|---:|---| -| LZ4 v1.8.6 | 0 / 0 | 14,680,064 | 1,047,098 | 12,288 | 98.826% | 10,688 / 208,150 ns | `GO` | -| Deflate | 39,928 / 39,928 | 15,204,352 | 70,998 | 52,216 | 26.454% | 14,207 / 61,627 ns | `GO` | -| LZMA | 98,992 / 98,992 | 24,117,248 | 258,736 | 111,280 | 56.991% | 21,384 / 80,329 ns | `GO` | -| Zstd | 95,992 / 313,080 | 18,874,368 | 255,736 | 325,368 | -27.228% | 109,253 / 91,015 ns | `FULL_FALLBACK` | - -All three LZ4 generations have the same 98.826 percent memory reduction; their -partial/full CPU ratios are 4.845, 4.632, and 5.135 percent. The gate collected -nine samples of 25 decodes per mode and stopped at that fixed sample count. -CPU timings are diagnostic because host scheduling produced visible outliers; -the authorization is based on deterministic peak temporary bytes. No noisy CPU -sample is used to rescue a codec that misses the 20 percent memory threshold. - -## Ranges, Corruption, and Errno - -Prefix, cross-page, middle, and tail requests all match the corresponding full -slice. The tail request reaches the extent end and therefore exercises full -fallback. Prefix consumption is 1,032 bytes for LZ4, 805 for Deflate, 352 for -LZMA, and 1,457 for Zstd. Corruption begins later at stream offsets 4,032, -4,031, 476, and 2,940 respectively. Every partial prefix remains byte-exact; -the same full read and every one-byte truncation return positive `EINTEGRITY` -(97). Real corrupted EROFS images fail matching fsck. All output guards and -codec cleanup checks pass. - -Partial success is only evidence that the requested slice is correct. It is not -reported as verification of bytes after the consumed boundary or of the full -extent. - -## State and Cleanup Model - -B28 adds no persistent cache, owner, waiter, pool, or unmount-drain state. A -cache hit remains first. A gate-authorized strict subextent miss decodes into a -request-local bounded prefix and never publishes a partial cache entry. Full -requests, unsupported ordinary strict subextent codecs, shifted/interlaced -maps, and arithmetic fallback retain the existing exact full-decode/cache -policy. Existing partial-reference maps remain bounded-prefix and -cache-ineligible. Failure frees the local output after input release; success -publishes no shared partial state. Reclaim, eviction, unmount, and key reuse -therefore retain their existing ownership model. - -The owned temporary tree was removed and the evidence SHA-256 manifest verifies -in full. No QEMU process was started. Protected PID 26318, port 9222, and the -shared base image were not touched. The full feature suite was not run. - -## Consequence - -B28 may implement bounded-prefix decode only for LZ4, LZMA, and Deflate in the -exact planned write set for ordinary strict subextent reads. Zstd and all -non-authorized backends must retain exact full fallback there without returning -an unsupported error; existing partial-reference behavior remains unchanged. -B28 acceptance owns D, the targeted TC176 host case, both KLD configurations, -and strict-timeout TC176 QEMU. It must preserve guards, consumed-byte semantics, -positive errno, cache eligibility, provider/GEOM ownership, and cleanup -ordering. - -## B28 Execution Outcome - -B28 source/test commit -`7a0d7e4a4047ac6d6130a3be5cb677cb94979fe7` has the exact 11-path actual -write set. LZ4, LZMA, and Deflate ordinary strict subextent reads decode a -bounded prefix; Zstd and non-authorized backends use exact full fallback. -Existing partial-reference maps remain bounded-prefix and cache-ineligible. - -Final D, targeted host TC176 subset -`20260815T163642Z-host-B28-partial-1356298-0`, and targeted zstdio0/zstdio1 K2 -`20260815T163515Z-host-B28-partial-1355895-0` pass. Earlier host and K2 -`RUNNER_FAIL` records are retained with their corrected oracle/build findings. -The strict-timeout QEMU run -`20260815T163824Z-qemu-B28-partial-1356729-0` is `INFRA_BLOCKED` because guest -SSH did not become ready before the boot deadline; it did not reach the target -marker, so no guest TC176 runtime PASS is claimed. Owned cleanup passes, -protected PID 26318 and port 9222 were untouched, and the full feature suite was -not run. Complete evidence is under -`planning/pre15/evidence/20260815T161336Z-B28/`. diff --git a/docs/pre15-stage0/P15-087.md b/docs/pre15-stage0/P15-087.md deleted file mode 100644 index 83790a3..0000000 --- a/docs/pre15-stage0/P15-087.md +++ /dev/null @@ -1,95 +0,0 @@ -# P15-087 Stage0 Decision - -Status: `STOP`. B12 is `STOP-NO-SOURCE`; no production source, B12 case, or -B12 fixture is authorized. - -The final decision uses frozen DUT BASE -`c7d692acf9166f5c5e42335db2de64f0793ba8a2`, FreeBSD source HEAD -`106727738dcfb6c001b46f25363b91cece970085`, and -`mkfs.erofs (erofs-utils) 1.8.6`. The final gate script and input SHA256 values -are `349720b900ad2288153c9830f5f5ee84b8da707c06ffabcbd8e3e7a7e89f9e2a` -and `cb3aaa6c780c7b1efbd7c339f2b2da2ed8bf38d3ba32229dc17996807fd00787`. - -## Workload and Independent Oracle - -The gate deterministically creates 18,000 long-name regular files. Since -erofs-utils tailpacks the root directory even with `-E noinline_data`, the -generator moves the final 1,456-byte tail into one appended contiguous block, -changes only the root layout from flat-inline to flat-plain, increments the -superblock block count, and recomputes CRC32C. A second generation is -byte-identical and `fsck.erofs -d0` exits zero. - -The independent parser does not call DUT code. It verifies a 3,880,368-byte -flat-plain directory at physical block 282, covering 948 contiguous 4 KiB -blocks and 18,002 entries. Its expected final cookie is 3,880,368 and its -record FNV64 is `427bb414efa99dc0`; the random offset is 1,937,408. The fixture -SHA256 is `3ecc5b706dd43b734c7e14a648a2bed2fb97a9f7963d4db703d15de81dc07a9d`. -The immutable `oracle.json` SHA256 is -`f247bbe5029301265342ff2098fcbd463a25e4752608826cbc1126d6620ef5d6`. - -The generated prototype patch SHA256 is -`33b25131a611eec9c8731e88e4c258dfc92fe5897dd6781b6c95414a846dd2d1`. -Static extraction proves that it calls `breadn` only on the mapped backing -`devvp`, does not call `breadn` on the EROFS directory vnode, caps the window -at exactly 1 MiB, gates readahead on an offset-zero sequential readdir, and -adds no VM entrypoint, vnode lock, GEOM ownership operation, `cluster_read`, -or errno token. The `semantic-ledger.json` SHA256 is -`d26dbc37f4f34f65d07745a1b8069217d18545436d7f16f24e775e8c2db1a6b3`. -This closes the static design boundary but cannot replace runtime proof. - -## Quantitative Decision - -| Required G11 measurement | Required | Valid result | Decision | -|---|---:|---:|---| -| Cold sequential runs | 5 baseline + 5 prototype | 0 + 0 | FAIL: no median | -| Median latency improvement | at least 10% | not measurable | FAIL | -| Extra provider reads | at most 25% | not measurable | FAIL | -| Readahead window | at most 1 MiB | 1 MiB static cap | PASS | -| Random seek readahead | zero | 0 valid runtime samples | FAIL: not verified | -| Hash/cookie equality | exact independent oracle | 0 valid runtime samples | FAIL: not verified | - -No run produced `runs.tsv`; therefore no latency, read-transfer, random-no-op, -or runtime hash/cookie value is claimed. It would be dishonest to infer GO -from the static prototype, from QEMU startup, or from build progress. Because -the workload could not be measured credibly and repeatably, G11's explicit -rule requires low-confidence STOP even though the 1 MiB static bound passes. - -## Replay Ledger - -The authoritative command shape was: - -```sh -timeout -k 30 1200 tests/pre15/gates/P15-087.sh \ - --base c7d692acf9166f5c5e42335db2de64f0793ba8a2 \ - --output OWNED_OUTPUT -``` - -Existing evidence was retained long enough to hash and reconcile before -repository cleanup: - -| Run ID | Result before any benchmark sample | -|---|---| -| `20260815T044720Z-G11-P15-087` | Superseded generator STOP: mkfs root remained flat-inline | -| `20260815T044924Z-G11-P15-087` | `INFRA_BLOCKED`: 12 boot-time providers exceeded SSH deadline | -| `20260815T045505Z-G11-P15-087` | `INFRA_BLOCKED`: FreeBSD did not enumerate PCI-hotplugged virtio-blk devices | -| `20260815T050003Z-G11-P15-087` | `INFRA_BLOCKED`: guest benchmark declaration error | -| `20260815T050520Z-G11-P15-087` | `INFRA_BLOCKED`: baseline KLD load did not reach workload | -| `20260815T051933Z-G11-P15-087` | `INFRA_BLOCKED`: phased baseline KLD load did not reach workload | -| `20260815T053543Z-G11-P15-087` | `INFRA_BLOCKED`: exact-basename baseline KLD load did not reach workload | -| `20260815T054540Z-G11-P15-087` | Aborted on instruction; owned process group terminated and audited | - -Each completed QEMU ownership record reports that the owned port was free -after cleanup, protected PID 26318 retained the same identity, protected port -9222 was not used, and `/work/build/vm-freebsd-build.qcow2.bp` retained inode, -size, mtime, and ctime. The interrupted run used owned PID 1172175 and port -49845; both were absent after targeted process-group cleanup. Final process -audit found only protected QEMU PID 26318. - -## Semantic and Batch Consequences - -Runtime preservation of FreeBSD vnode, VM, locking, GEOM, errno, hash, and -cookie behavior was not established. Correctness may not depend on -readahead, so static plausibility is insufficient. P15-087 is STOP for Pre15, -B12 remains absent, and no host feature case, K0 build, TC183 QEMU acceptance, -or full feature suite is run. Rollback is the single gate decision commit; -there is no source commit to revert. diff --git a/docs/pre15-stage0/P15-092.md b/docs/pre15-stage0/P15-092.md deleted file mode 100644 index 84ad004..0000000 --- a/docs/pre15-stage0/P15-092.md +++ /dev/null @@ -1,142 +0,0 @@ -# P15-092 Stage0 Decision - -Status: `STOP`. B15 is `STOP-NO-SOURCE`; terminal static review status is -`TERMINAL_REVIEWED_NO_SOURCE`. No production source, B15 case, or B15 fixture -was created. Historical acceptance remains `NOT_TESTED`. - -P15-092 is the G11 zero-nlink candidate. The frozen BASE is -`b22dae8dc634c68db4ea89dccade91350614c139`, the FreeBSD source HEAD is -`106727738dcfb6c001b46f25363b91cece970085`, the erofs-utils source HEAD is -`7db78788b000999e2de88decd2ba90654f26171c`, and the generator is -`mkfs.erofs (erofs-utils) 1.8.6`. Linux EROFS identity is frozen by the three -source hashes in `P15-092-input.json` and the Pre15 semantic audit hash ledger. - -The final gate script and input SHA256 values are -`5943b85864da9095174ef51d1eed032ec6f0a1766c6465cb9660ea20cbd46b1e` -and `0a6f6c76185644d3f5525a98429a310027556ef3cf7ebb43abb912f92e052bde`. - -## Real Disk Oracle - -The gate creates the same deterministic source tree twice in forced compact -form and twice in forced extended form. It uses fixed UUID, timestamp, worker -count, xattr policy, and inline-data policy. Both complete host output trees -are byte-identical. - -An independent parser reads the real superblock, checksum span, metadata block, -root NID, directory entries, compact `i_nb.nlink`, extended `i_nlink`, and the -compact `EROFS_I_NLINK_1_BIT` rule. It then materializes seven images: - -- compact and extended positive seeds; -- namespace-reachable compact, extended, and directory inodes with nlink zero; -- a root inode with nlink zero; -- an unreachable inode with nlink zero, whose former name is redirected to a - real hardlink NID and whose replacement link count is corrected to three. - -Every derived image has a recomputed valid EROFS CRC32C. `dump.erofs --path` -resolves the actual directory edge and NID; no DUT helper or internal value -injection is used. All seven images, including all three namespace-reachable -zero-nlink images, pass `fsck.erofs -d0` with no error marker. The fixture-set -SHA256 is `99897d4fd81da015bdaa66a5dfdbf3f42e9e830882c0d8c979e691fa9209ee7a`. - -Key deterministic host evidence SHA256 values are: - -- `host-result.json`: `59a27e9389e2771c921d689f6b76caeb858f0fd28487a933529153727aac12e8` -- `disk-records.json`: `44bf0355dc5b34e93ec6868db9f9049de32500591b048a9016bf46aed9609485` -- `boundary.json`: `985df1236c6dd2016d6cb244b71578d36bd4cd10aa273b60a92d5bb6d24d5b60` -- `format-semantics.json`: `88efc1d7814103c044b36ff4dc3cac1dfa4da79f3d8955c682d1a60d9bfd9fcf` -- `fixture-index.json`: `4d2b69bcc31709ecb96cfafb41f5db8ffcc97215c3290cfa476d0b6c7fc27886` -- `source-anchors.json`: `ef9e6871d5fe1a3a6113f1172340269da4ea0c1692003aad0b0fee06ec4deb42` - -## Linux and FreeBSD Semantics - -Linux EROFS calls `set_nlink()` with the compact or extended disk value and has -no zero-nlink rejection in inode decode or super initialization. This is an -inode/link-count behavior, not a FreeBSD vnode publication rule. Linux and -FreeBSD both permit a live unlinked inode/vnode with link count zero; the -FreeBSD VM code explicitly treats `va_nlink == 0` as an unlinked mapping case. - -The FreeBSD EROFS adapter has three distinct entry semantics: - -1. `erofs_lookup()` owns a non-dot namespace edge and calls `erofs_vget()`. -2. `.vfs_vget = erofs_vget` is the raw NID entry used by VFS/root callers. -3. `erofs_fhtovp()` calls raw `VFS_VGET` and then maps `nlink == 0` to `ESTALE`. - -Therefore Linux's direct `set_nlink()` behavior does not authorize either a -global FreeBSD rejection or a namespace-only rejection. The latter would be a -new FreeBSD validator policy, and the disk/fsck corpus supplies no format-level -rule requiring it. - -## Publication Boundary - -B09 moved the vnode constructor to `src/erofs_vnops.c`, but the frozen B15 -write set is only `src/namei.c` and `src/inode.c`. `erofs_vget()` inserts the -constructing vnode in the hash, decodes the inode, marks it -`VSTATE_CONSTRUCTED`, and only then returns to `erofs_lookup()`. - -A post-`erofs_vget()` check in `namei.c` is therefore too late to reject before -vnode publication. `vput()` alone does not perform the required `vgone` -cleanup. An unconditional `vgone()` cannot preserve an already cached vnode -obtained through raw `VFS_VGET`, because `erofs_vget()` does not return a -created-versus-hit indicator. Pre-reading through `erofs_read_inode()` would -duplicate complete inode decode and metadata I/O on every cold namespace -lookup, then decode the same inode again in `erofs_vget()`. - -The four failed GO requirements are consequently: - -- no format or cross-kernel rule requires reachable zero-nlink rejection; -- rejection before vnode publication is unavailable within the exact B15 - write set; -- post-publication cleanup cannot preserve cached raw VGET semantics; -- pre-reading would duplicate full inode decode for a low-value validator. - -Namespace and raw entrypoints are distinguishable, and the orphan needs no -mount-wide scan, but those two facts do not outweigh the failed safety, -compatibility, and cost requirements. P15-092 is therefore STOP rather than a -partial or expanded-write-set implementation. - -## Replay and Infrastructure - -The authoritative deterministic host command was run twice: - -```sh -timeout -k 10 240 tests/pre15/gates/P15-092.sh \ - --base b22dae8dc634c68db4ea89dccade91350614c139 \ - --output OWNED_OUTPUT --host-only -``` - -Both runs exit `22` with `STOP` and have byte-identical host output trees. - -One supplemental isolated runtime attempt used the standalone -`/work/debug-qemu/local/vm/freebsd-build-runtime.qcow2`, not the protected base -bp, with a fresh overlay, random port, exact baseline KLD, 180-second boot -deadline, 60-second guest command deadlines, 900-second inner timeout, and -1200-second outer timeout. Run -`20260815T083854Z-qemu-P15-092-gate-runtime-1219662-0` was -`INFRA_BLOCKED`: guest SSH did not become ready before the boot deadline, so no -runtime vnode result is claimed. The run used only PID `1219981` and port -`49795`; it did not own PID `26318`, port `9222`, or the protected base bp. - -Cleanup is `PASS`: the owned QEMU PID stopped, port `49795` is free, the -overlay and case temp were removed, and the standalone base SHA256 remained -`ae09f47aef43cfd016049610e86bcf2073fdf70430961bcdb94662facac9f046`. -The runtime manifest and cleanup SHA256 values are -`2e9a4220dc86c5d8fa465e77e6d81e5e36c0f26338b70d71a84881af8fc02cb9` -and `0a499d8fa3397f73998c6c3c3ae9dfc9070f61267a86361ba96d1e8bc64cb392`. - -Because the reproducible host gate already fails mandatory GO conditions, the -runtime infrastructure block does not defer or weaken the STOP decision. It is -reported separately and was not retried with another long-running VM. B15, -TC021, TC025, TC182, K0, and B15 acceptance QEMU are `NOT_RUN` by the mandatory -STOP-NO-SOURCE rule. The full feature suite was not run. - -## Terminal Static Review - -The 2026-08-17 review rechecked the current `repo-pre-15`, Linux EROFS -implementation, P15-092 plan, frozen write set, and post-STOP history. It -found no format or cross-kernel requirement for namespace zero-nlink -rejection, no new B15 consumer or test target, and no safe implementation -entry within the frozen write set. The detailed evidence is -`planning/pre15/evidence/20260817T-B15-terminal-review/VERDICT.md`. - -The terminal review does not convert the historical STOP or acceptance state: -`STOP_NO_SOURCE` and `NOT_TESTED` remain the authoritative historical values. diff --git a/docs/pre15-stage0/README.md b/docs/pre15-stage0/README.md deleted file mode 100644 index 9903c6c..0000000 --- a/docs/pre15-stage0/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Pre15 Stage0 Execution Evidence - -## Scope - -B01 establishes the only Pre15 runner interface and the evidence contract used -by later gates, builds, directed tests, and smoke runs. It does not modify -`src/**`, run the full feature suite, or convert host parser results into KLD -runtime claims. - -## Entrypoints - -```sh -timeout -k 10 240 tests/pre15/run-host.sh CASE -timeout -k 30 1200 tests/pre15/run-build.sh zstdio0 -timeout -k 30 1200 tests/pre15/run-build.sh zstdio1 -timeout -k 30 1200 tests/pre15/run-qemu.sh CASE -timeout -k 30 1500 tests/pre15/run-smoke.sh final-four-codec -``` - -Cases are discovered as exact files under `tests/pre15/cases/`; no shared case -registry or historical result runner is consulted. `run-build.sh` reports a -Linux invocation as `INFRA_BLOCKED` because `build.sh` requires a native -FreeBSD host. - -## Evidence - -Set `PRE15_EVIDENCE_ROOT` to a new evidence parent. Every invocation creates a -unique run directory and never overwrites an earlier run. `manifest.json` -conforms to `tests/pre15/EVIDENCE-SCHEMA.json` and records exact argv, DUT and -source identities, worktree diff hash, fixture/module hashes, timestamps, -deadline, exit code, target marker, status, cleanup, and raw-output paths. - -The only case statuses are `PASS`, `DUT_FAIL`, `RUNNER_FAIL`, -`INFRA_BLOCKED`, `STOP`, and `NOT_RUN`. A nonzero guest command without a target -marker is not a DUT failure. Cleanup failure always changes the run to -`RUNNER_FAIL`. - -## Ownership - -The runner records each owned process, path, forwarded port, SSH ControlMaster, -guest mount, md unit, loaded EROFS KLD, and base image before use. Cleanup runs -in reverse order and refuses to remove paths outside the current run directory. -QEMU always writes to a fresh overlay. The base image is read-only input for the -runner and its SHA256 must remain unchanged. - -Mount, md, and KLD cleanup applies only to resources explicitly registered by -the current run. Kernel ZSTDIO is a kernel option, not an unloadable dependency. - -## B01 Controls - -- `B01-runner-selftest`: known-good, target-marked DUT mismatch, pre-target - command failure, SSH failure, QEMU early exit, timeout, owned PID cleanup, and - deliberate cleanup-boundary failure. -- `B01-g3-equivalence`: verifies the two archived script identities, compares - their source/artifact inventories with the stable helper, and repeats stable - generation. -- `TC162-xattr-legacy`: verifies checksum-valid legacy primary, explicit plain, - packed, and metabox prefix carriers plus exact single-field `EINTEGRITY` - negatives. - -The authoritative B01 and initial gate verdicts are recorded under -`planning/pre15/evidence/` after execution from a committed B01 tree. diff --git a/docs/pre15-stage0/phase0-B-ZSTD-001.md b/docs/pre15-stage0/phase0-B-ZSTD-001.md deleted file mode 100644 index b0f7790..0000000 --- a/docs/pre15-stage0/phase0-B-ZSTD-001.md +++ /dev/null @@ -1,101 +0,0 @@ -# Phase 0: B-ZSTD-001 Remediation - -Status: `FIXED_STATICALLY`; runtime corruption-tail verification remains -`NOT_TESTED` in this remediation pass. - -## Problem - -Strict audit item B-ZSTD-001 / BUG-ZSTD-035 found that ordinary ZSTD extents -advertised subextent support. A prefix or middle read could therefore decode -only the requested output and accept `z_erofs_zstd_finish()` without checking -the remaining compressed stream. Corruption after the requested range could be -skipped. - -P15-086 already authorizes partial ordinary reads only for LZ4, LZMA, and -Deflate. Its ZSTD result is `FULL_FALLBACK` because the partial candidate used -more temporary memory than full decode. - -## Call Graph - -For an ordinary mapped compressed read: - -1. `z_erofs_do_read()` calls `z_erofs_decode_length()` with `mapoff` and - `want`. -2. `z_erofs_decode_length()` sets `partial=true` and `decoded_len=mapoff+want` - when the descriptor reports `supports_subextent` and the request ends before - the extent. -3. `z_erofs_decode_extent()` passes that length and mode to - `z_erofs_decompress()`. -4. `z_erofs_zstd_decompress()` fills the shortened output and calls - `z_erofs_zstd_finish()`. -5. `z_erofs_zstd_finish()` intentionally permits partial decoding without - requiring stream end. That behavior remains necessary for the separately - defined `EROFS_MAP_PARTIAL_REF` bounded-prefix path. - -The smallest correct policy fix is to stop ordinary ZSTD reads at step 2: -ZSTD now advertises `.supports_subextent = 0`. `z_erofs_decode_length()` then -selects the full extent for ordinary prefix and middle reads. The explicit -`EROFS_MAP_PARTIAL_REF` branch still sets `partial=true` and remains unchanged. - -## Changes - -- Set the ZSTD descriptor capability to false in - `src/decompressor_zstd.c`; no finish validation was weakened or bypassed. -- Updated B27 assertions to preserve ZSTD partial-reference completion while - requiring ordinary ZSTD full fallback. -- Updated B28 source and extracted-policy assertions, host report, and QEMU - corruption decision so ZSTD is `FULL_FALLBACK`; LZ4/LZMA/Deflate policy is - unchanged. - -## Verification - -Commands and actual results: - -- `git diff --check`: `PASS`. -- `sh -n repo-pre-15/tests/pre15/cases/B27-stream-tail.sh` and - `sh -n repo-pre-15/tests/pre15/cases/B28-partial.sh`: `PASS`. -- `PRE15_EVIDENCE_ROOT=/tmp/erofs-phase0-b28-20260818-r2 timeout -k 10 600 - repo-pre-15/tests/pre15/run-host.sh B28-partial`: `PASS`; cleanup `PASS`, - target reached. Evidence: - `/tmp/erofs-phase0-b28-20260818-r2/20260818T070848Z-host-B28-partial-1780769-0/`. - The source audit reports `zstd=false`, ZSTD in `current_full_fallback`, - `current_partial=[deflate,lz4,lzma]`, and partial-reference preserved. -- `PRE15_EVIDENCE_ROOT=/tmp/erofs-phase0-b27-20260818 timeout -k 10 600 - repo-pre-15/tests/pre15/run-host.sh B27-stream-tail`: `RUNNER_FAIL`, cleanup - `PASS`; the pre-existing B29 Deflate context-pool delta fails B27's frozen - exact-transform audit before the completion harness. This is not claimed as - ZSTD runtime evidence. -- P15-086 frozen gate result and policy were inspected, not rerun; its existing - recorded decision remains ZSTD `FULL_FALLBACK`. - -The B28 host case generated and compared real EROFS fixtures and ran the -descriptor/decode-length source audit, but did not load a FreeBSD KLD or read -an image through the kernel. No full feature suite was run. - -## Not Tested - -`NOT_TESTED`: a real FreeBSD runtime read of an EROFS ZSTD image whose stream -is corrupted after an ordinary prefix or middle request. The required runtime -threshold is that the short ordinary read returns positive `EINTEGRITY` (97), -matching full fallback, while a valid ordinary prefix/middle/tail read matches -the source bytes. - -`NOT_TESTED`: runtime confirmation that `EROFS_MAP_PARTIAL_REF` retains its -bounded-prefix semantics on a real image after this policy change. - -## Risk - -Ordinary ZSTD prefix and middle reads now decode the complete extent. This may -increase CPU or temporary output work relative to the previously incorrect -partial path, but it restores corruption visibility and matches P15-086 policy. -The changed tests do not alter test fixtures, image bytes, test environment, or -finish-error behavior. - -## Runtime Gate - -Before claiming Phase 0 runtime closure, run the existing focused B28 runtime -case with ZSTD enabled and verify valid prefix, middle, and tail reads, an -after-request-range corruption case returning `EINTEGRITY`, full corruption -returning `EINTEGRITY`, and the real partial-reference case. Record separate -`PASS`, `INFRA_BLOCKED`, or `NOT_TESTED` results; do not infer runtime behavior -from the extracted C harness. diff --git a/docs/pre2-baseline.md b/docs/pre2-baseline.md deleted file mode 100644 index 5f3058a..0000000 --- a/docs/pre2-baseline.md +++ /dev/null @@ -1,65 +0,0 @@ -# repo-pre-2 Maintenance Baseline - -## Snapshot Identity - -`repo-pre-2` was created from the tracked `repo-pre-1` tree without copying -working-tree-only files or build artifacts. - -```text -planning baseline commit: ed47f1b5583a229e372f488b9492d1f6234aae98 -snapshot creation parent: 109fe74d3fcef107db5869be1d030f4f12d1cfa6 -snapshot creation commit: 73fa57924b549387400eeeb85354bbbbe770a962 -repo-pre-1 source tree: 6a5a1a49d4a4e08f6ef1155de240500e63a15365 -repo-pre-2 snapshot tree: 6a5a1a49d4a4e08f6ef1155de240500e63a15365 -Linux comparison tree: b79b9a8b62f633e887a8b99075ff9412fabd7f83 -``` - -The matching source and snapshot tree IDs establish byte-for-byte content, -file-mode, and path equivalence at snapshot creation. `repo-pre-1` and -`src-linux` are read-only references for pre2 work. - -## Phase Scope - -Pre2 phase 1 is L0 mechanical maintenance alignment. It permits only the -separately reviewed changes listed below: - -1. Remove stale Linux-only declaration fragments from `src/xattr.h` while - preserving every active FreeBSD interface. -2. Remove six specified, unreferenced private constants from - `src/erofs_defs.h`. -3. Rename the private checksum helper in `src/super.c` without changing its - signature, body, call position, or behavior. - -Each source change must remain an independent commit. This phase does not add -features, fix behavior, move responsibilities between files, reorder data -structures, alter ABI, or broadly reformat code. - -## Required FreeBSD Differences - -Linux and FreeBSD EROFS remain independent implementations. The following -platform boundaries are intentional and must not be replaced merely to make -the source look more similar: - -- FreeBSD VFS, vnode, mount, namecache, pager, and NFS export interfaces. -- FreeBSD buffer/cache, GEOM provider, device I/O, locking, and allocation - APIs. -- FreeBSD xattr, ACL, errno, kernel-module, and `bsd.kmod.mk` conventions. -- FreeBSD build gates, including the current amd64 restriction and optional - `ZSTDIO` integration. - -Linux naming and layout should be followed only when doing so preserves these -native contracts and makes cross-repository review easier. - -## Excluded Work - -This phase does not address vnode ownership after `insmntque()` failure, -large-run allocation and `uiomove()` limits, compact pblk validation, 48-bit -boundaries, decompressor descriptors, backend ABI, typed errno, xattr parsing, -ACL behavior, file splitting, build-system alignment, CI, or test tooling. - -## Validation Status - -Only static snapshot and Git allowlist checks were performed for the -initialization commits. No build, QEMU run, test script, smoke test, mount, or -runtime feature check was performed. All runtime capabilities therefore remain -`NOT RUN` for repo-pre-2 until a later, explicitly authorized validation phase. diff --git a/docs/pre3-baseline.md b/docs/pre3-baseline.md deleted file mode 100644 index b57003d..0000000 --- a/docs/pre3-baseline.md +++ /dev/null @@ -1,83 +0,0 @@ -# repo-pre-3 Maintenance Baseline - -## Snapshot Identity - -`repo-pre-3` was created exclusively from the Git-tracked `repo-pre-2` tree. -Untracked files, ignored files, build outputs, and working-tree-only content -were not copied. - -```text -snapshot source commit: 034f409841d2edaf7f140fc7475a357c924a5930 -snapshot creation commit: 865a702341a82e61ab5ca1d5708a28005277cc0a -repo-pre-2 source tree: c28e015c74f4d012dfd070f887fabf75d4d4cb16 -repo-pre-3 snapshot tree: c28e015c74f4d012dfd070f887fabf75d4d4cb16 -tracked files per tree: 281 -``` - -The matching tree IDs establish path, file-mode, and blob equivalence at -snapshot creation. `repo-pre-2`, `repo-pre-1`, and `src-linux` are read-only -references for pre3 work. - -## Phase Scope - -Pre3 is an L1 mechanical naming-alignment stage. It contains four independent -source tasks: - -1. Rename `erofs_init_xattr_prefixes` and - `erofs_cleanup_xattr_prefixes` to the Linux-like lifecycle names - `erofs_xattr_prefixes_init` and `erofs_xattr_prefixes_cleanup`. -2. Rename the private ACL helper `erofs_getacl` to `erofs_get_acl`. -3. Rename `erofs_close_device` to `erofs_release_device_info` while retaining - its FreeBSD implementation. -4. Rename the private mount-state destructor `erofs_free_mount` to - `erofs_sb_free` while retaining its complete FreeBSD cleanup body. - -Only function names and their closed sets of declarations, definitions, and -call sites may change. Signatures, behavior, control flow, ordering, locking, -ownership, error handling, and file responsibilities must remain unchanged. - -## Required FreeBSD Differences - -- Xattr-prefix helpers continue to use `struct erofs_mount *`, not Linux - `struct super_block *`. -- ACL handling continues to use FreeBSD `struct vnode *`, `struct acl *`, and - integer errno conventions. -- Device teardown continues to release GEOM consumers, vnode references, and - cdev references in the existing order and under the existing locking rules. - It must not adopt the Linux callback signature or ownership model. -- Mount-state teardown continues to release all FreeBSD GEOM and private - filesystem state. A Linux-like name does not make the implementations or - lifecycle contracts interchangeable. - -## Excluded Work - -Pre3 does not include feature changes, correctness fixes, ABI changes, codec -work, xattr or ACL behavior changes, function reordering, structure layout -changes, file splitting, build-system changes, formatting cleanup, or repairs -to earlier planning and snapshot material. - -## Git Workflow - -The snapshot, this baseline document, and each of the four rename batches are -separate commits. Every commit must be pushed immediately to `xdm main` and -must not be squashed with another batch. Before each commit, its staged paths -must be checked against the task-specific allowlist. After each push, local -`HEAD`, `xdm/main`, and remote `main` must identify the same commit. - -## Validation Status - -Only static snapshot-equivalence, path-allowlist, Git-state, and documentation -checks were performed for pre3 initialization. - -```text -build: NOT RUN -QEMU: NOT RUN -test scripts: NOT RUN -smoke tests: NOT RUN -mount/runtime: NOT RUN -feature validation: NOT RUN -``` - -No compile-time or runtime PASS is claimed by this document. Existing reports -copied from `repo-pre-2` are historical material and are not fresh validation -of `repo-pre-3`. diff --git a/docs/pre3-smoke-test-20260812.md b/docs/pre3-smoke-test-20260812.md deleted file mode 100644 index e8bb40e..0000000 --- a/docs/pre3-smoke-test-20260812.md +++ /dev/null @@ -1,84 +0,0 @@ -# Pre3 Smoke Test Report - 2026-08-12 - -## Result - -The `repo-pre-3` plain/LZ4 differential smoke test **PASSed** within the scope documented below. - -| Field | Value | -|---|---| -| Run ID | `20260812T110037.575683Z-693361-ac21acee` | -| Process exit code | `0` | -| Guest return code | `0` | -| Runner stages | `12/12 passed` | -| Elapsed time | `18m51s` | -| DUT outer commit | `1a2361bfae2f6b4e7fc4c89d97d05f44553a5ba7` (`1a2361b`) | -| DUT path tree | `902a32fbfc1274d4404174346953d19e41c61f1f` | -| DUT content SHA256 | `07aa856f0c0d9d0d084dbcb5d743cf4831a7f2ab884b51941db7d1840b248c60` | -| KLD SHA256 | `59650f03029afc31658dcf3386105d001122d7702b8144e7c5925d9d9ed4accb` | -| Base image SHA256 | `67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef` | -| Base image mode | `0444`, unchanged before and after | - -Primary evidence: - -- [result.json](/work/tests-dev/erofsstress/demo-result/repo-pre-3-smoke-20260812/20260812T110037.575683Z-693361-ac21acee/result.json) -- [summary.txt](/work/tests-dev/erofsstress/demo-result/repo-pre-3-smoke-20260812/20260812T110037.575683Z-693361-ac21acee/summary.txt) -- [guest evidence directory](/work/tests-dev/erofsstress/demo-result/repo-pre-3-smoke-20260812/20260812T110037.575683Z-693361-ac21acee/guest-evidence) -- [guest-result.json](/work/tests-dev/erofsstress/demo-result/repo-pre-3-smoke-20260812/20260812T110037.575683Z-693361-ac21acee/guest-evidence/guest-result.json) - -## Invocation - -```sh -./test.sh --dut /work/erofs-freebsd-pre/repo-pre-3 \ - --base-image /work/tests-dev/lfs/freebsd-15-dev-src-20260811.bp \ - --duration 10 \ - --output demo-result/repo-pre-3-smoke-20260812 -``` - -Evidence root: - -```text -/work/tests-dev/erofsstress/demo-result/repo-pre-3-smoke-20260812/20260812T110037.575683Z-693361-ac21acee/ -``` - -## PASS Scope - -The successful run covered: - -- FreeBSD guest boot and SSH readiness. -- DUT build with `WITH_ZSTDIO=0` and loading the exact generated `erofs.ko`. -- Creation and use of plain and LZ4 EROFS fixtures. -- Attach, mount, differential workload, unmount, and fixture cleanup. -- Four workers per fixture for five seconds each. -- Differential random, aligned, unaligned, full-file, and EOF reads. -- Directory enumeration, symbolic-link reads, and `fadvise` operations. -- Host and guest resource cleanup after the workload. - -The guest-side stages and fixture identities are recorded in [stages.txt](/work/tests-dev/erofsstress/demo-result/repo-pre-3-smoke-20260812/20260812T110037.575683Z-693361-ac21acee/guest-evidence/stages.txt) and [scenarios.txt](/work/tests-dev/erofsstress/demo-result/repo-pre-3-smoke-20260812/20260812T110037.575683Z-693361-ac21acee/guest-evidence/scenarios.txt). Per-worker evidence is available in the same guest evidence directory. - -## SKIPPED And Non-Claims - -The following tests were **SKIPPED** and have no execution evidence in this run: - -| Test | Area | Status | -|---|---|---| -| `TC079` | xattr prefix initialization, lookup, and cleanup | `SKIPPED` | -| `TC138` | ACL parsing and error paths | `SKIPPED` | -| `TC099` | multidevice mount and release paths | `SKIPPED` | - -This report therefore does **not** claim that xattr prefix handling, ACL behavior, multidevice behavior, ZSTD, LZMA, DEFLATE, or the full feature/regression suite passed. The PASS result is limited to the plain/LZ4 differential smoke scope listed above. - -## Integrity And Cleanup - -- The DUT content hash, commit, and tree were identical before and after the run; the test did not modify `repo-pre-3`. -- The read-only base image remained mode `0444` with the same SHA256 before and after the run. -- New QEMU PID `693836` exited gracefully. -- Test port `10000` was released. -- The qcow2 overlay was deleted and the port lock became available. -- Guard PID `26318` and guard port `9222` retained the same process identity, start time, and command hash. -- The generated result directory under `tests-dev` is evidence only and was not committed. - -## Honesty Review - -The post-run honesty audit **PASSed**. It found no concealed source fix, result substitution, or exaggerated coverage claim. The guest log filename `repo22-build.txt` and related `repo22` stage labels are inherited labels in the test harness; they do not identify the DUT. The DUT is independently bound to `repo-pre-3` by the recorded commit, path tree, content hash, archived source, and generated KLD hash. - -The complete machine-readable record remains authoritative for details not reproduced here: [result.json](/work/tests-dev/erofsstress/demo-result/repo-pre-3-smoke-20260812/20260812T110037.575683Z-693361-ac21acee/result.json). diff --git a/docs/pre5-baseline.md b/docs/pre5-baseline.md deleted file mode 100644 index ccfbf86..0000000 --- a/docs/pre5-baseline.md +++ /dev/null @@ -1,80 +0,0 @@ -# Pre5 Extraction Baseline - -## Repository Baseline - -- Outer starting commit: `6421919003ecfab5317dfa70959ab81cd9bbc990` -- Starting `repo-pre-5` tree: `f99462eee8898af209332ef202f8da07f2dd567e` -- Starting `repo-pre-5/src/super.c` blob: `c150d795df42ae4136fbd6e74c3b88903341b05a` -- Execution target: the existing `repo-pre-5` directory -- Source allowlist for the later extraction commits: `repo-pre-5/src/super.c` - -The fixed tree and blob identifiers above were read from Git before this -document was created. Pre5 works directly on `repo-pre-5`; it does not create -another snapshot. - -## Stage Scope - -Pre5 is an L2-S private cleanup responsibility extraction stage. It has two -source tasks: - -1. Extract the existing extra-device cleanup from `erofs_sb_free()` into - `static void erofs_free_dev_context(struct erofs_mount *em)`. -2. Extract the existing metabox and packed-inode cleanup from - `erofs_sb_free()` into - `static void erofs_drop_internal_inodes(struct erofs_mount *em)`. - -Both tasks are statement movement only. They must preserve FreeBSD GEOM, -vnode, allocation, and ownership semantics. They must not change behavior, -ABI, features, error handling, locking, logging, conditions, release calls, or -object ownership. - -## Protected Paths - -The following paths must remain unchanged during Pre5 execution: - -- `repo-pre-3/` -- `repo-pre-2/` -- `repo-pre-1/` -- `src-linux/` -- `planning/reject/` -- existing reports and other planning stages - -Except for this baseline document, no path outside -`repo-pre-5/src/super.c` belongs in the Pre5 source commits. - -## Cleanup-Order Invariant - -The effective cleanup order must remain exactly: - -```text -xattr prefixes --> metabox --> packed inode --> extra devices in reverse order --> primary device --> mount state -``` - -The extraction must not add pointer clearing, conditions, assertions, logs, -locks, return values, error handling, or header declarations. - -## Commit and Push Discipline - -Each Pre5 commit must contain one planned logical task and must be pushed -immediately to `xdm main`. The commits must not be squashed. Execution must -stop if a push fails or if local, tracking, and remote commit identities -diverge. - -## Validation Status at Baseline - -| Validation item | Status | -|---|---| -| Static Git baseline capture | RECORDED | -| Source extraction tasks | NOT RUN | -| Build with `WITH_ZSTDIO=0` | NOT RUN | -| Build with `WITH_ZSTDIO=1` | NOT RUN | -| QEMU smoke testing | NOT RUN | -| Manual or automated tests | NOT RUN | - -No build, QEMU run, feature validation, regression test, or runtime result is -claimed by this document. diff --git a/docs/pre5-completion-20260812.md b/docs/pre5-completion-20260812.md deleted file mode 100644 index a956da2..0000000 --- a/docs/pre5-completion-20260812.md +++ /dev/null @@ -1,268 +0,0 @@ -# Pre5 Completion Report - -Date: 2026-08-12 - -## Conclusion - -| Area | Result | Notes | -|---|---|---| -| Planned source extraction | PASS | Both private cleanup helpers are present in `repo-pre-5/src/super.c`. | -| Static correctness review | PASS | No High or Medium findings; statement order and FreeBSD cleanup behavior are preserved. | -| Commit recoverability | PASS after correction | The initial split commits were non-destructively reverted and replaced by one independently revertible atomic source commit. | -| `WITH_ZSTDIO=0` build | PASS | Exit code 0, zero warnings, zero errors. | -| Module load | NOT RUN | The produced `erofs.ko` was not loaded. | -| QEMU functional testing | NOT RUN | No functional or regression test was run for Pre5. | -| Pre5 smoke test | DEFERRED | Deferred for a combined Pre5/Pre6 smoke run. | - -Pre5 is complete for its planned source changes, static review, and required -build gate. This report does not claim runtime, feature, regression, or smoke -test coverage. - -The final Pre5 source commit is -`4535cccaf1ca1837f0a2d1cca526e5f138a23c6d`. This report is created after that -commit and, when committed, its documentation commit will therefore follow the -final source commit. - -## Scope - -Pre5 directly modified `repo-pre-5`; no new repository snapshot was created. -The starting planning commit was: - -```text -6421919003ecfab5317dfa70959ab81cd9bbc990 -docs: plan repo-pre-5 cleanup extraction phase -``` - -The baseline documentation commit was: - -```text -0666800fa529a8869757da85f2e69b8e1dd2c9f6 -docs: record pre5 extraction baseline -``` - -The baseline recorded the following source identity: - -```text -repo-pre-5/src/super.c blob: -c150d795df42ae4136fbd6e74c3b88903341b05a -``` - -The source scope was limited to extracting two existing cleanup regions from -`erofs_sb_free()` into private helpers. No feature, ABI, error handling, -locking, logging, ownership, or cleanup policy change was intended. - -## Final Implementation - -The final implementation is in [`src/super.c`](../src/super.c). - -### Extra-device cleanup - -`erofs_free_dev_context()` is a `static void` helper with exactly one -definition and one call. It contains only the existing extra-device cleanup: - -```c -static void -erofs_free_dev_context(struct erofs_mount *em) -{ - unsigned int i; - - if (em->devs != NULL) { - for (i = em->extra_devices; i > 0; --i) - erofs_release_device_info(&em->devs[i - 1]); - free(em->devs, M_EROFS); - } -} -``` - -The reverse close order is unchanged. The primary device remains outside this -helper and is still released separately by `erofs_sb_free()`. - -### Internal-inode cleanup - -`erofs_drop_internal_inodes()` is a `static void` helper with exactly one -definition and one call. It contains only the existing metabox and packed -inode releases: - -```c -static void -erofs_drop_internal_inodes(struct erofs_mount *em) -{ - if (em->metabox_en != NULL) - free(em->metabox_en, M_EROFS); - if (em->packed_inode != NULL) - free(em->packed_inode, M_EROFS); -} -``` - -This remains a FreeBSD allocation cleanup path. It does not copy Linux -`iput()` or Linux inode-lifecycle semantics. - -### Preserved cleanup order - -The effective cleanup order remains: - -```text -xattr prefixes --> metabox --> packed inode --> extra devices in reverse order --> primary device --> mount state -``` - -The final caller is: - -```c -static void -erofs_sb_free(struct erofs_mount *em) -{ - if (em == NULL) - return; - erofs_xattr_prefixes_cleanup(em); - erofs_drop_internal_inodes(em); - erofs_free_dev_context(em); - erofs_release_device_info(&em->dif0); - free(em, M_EROFS); -} -``` - -Static expansion of both helpers produces the baseline statement sequence. -No condition, loop direction, release function, pointer clearing, lock, errno, -log message, return value, or ownership rule was added or changed. - -## Commit Timeline - -| Commit | Purpose | Result | -|---|---|---| -| `6421919` | Establish the Pre5 execution plan | Baseline planning point. | -| `0666800` | Record the source baseline and validation status | Documentation only. | -| `52dca2e` | Initially extract `erofs_free_dev_context()` | Static behavior correct. | -| `e9b9fb2` | Initially extract `erofs_drop_internal_inodes()` | Static behavior correct. | -| `c406813` | Revert `e9b9fb2` | Non-destructively removed the second helper first. | -| `20733bd` | Revert `52dca2e` | Restored the exact baseline `super.c` blob. | -| `4535ccc` | Atomically introduce both cleanup helpers | Final source implementation. | - -The initial two source commits were behaviorally correct. The static audit -found a commit-organization defect: after `e9b9fb2`, the earlier `52dca2e` -could not be independently reverted from the final HEAD without conflict -because both commits edited the same tightly coupled `erofs_sb_free()` region. - -No history rewrite or destructive reset was used. The correction was: - -1. Revert `e9b9fb2` with `c406813`. -2. Revert `52dca2e` with `20733bd`. -3. Confirm that `super.c` returned to baseline blob - `c150d795df42ae4136fbd6e74c3b88903341b05a`. -4. Reintroduce both helpers in atomic commit `4535ccc`. - -The final `super.c` blob is: - -```text -03b92560bb5ef01f322b0d052f84bc3c577ef829 -``` - -This is byte-for-byte identical to the correct source state at `e9b9fb2`. -The final commit `4535ccc` can be reverted without conflict and restores the -baseline blob. - -The original plan preferred one source task per commit. That rule was adjusted -because the two extractions share one cleanup sequence and one caller region; -separate commits weakened independent rollback despite preserving behavior. -One atomic source commit provides a clearer and mechanically verifiable -rollback boundary without changing the planned source result. - -## Static Review Evidence - -The final independent static review result was PASS with no High or Medium -findings. - -Verified properties: - -- Only `repo-pre-5/src/super.c` differs from the source baseline. -- Both helpers are `static void` and each has one definition and one call. -- Extra devices are still released in reverse order before `em->devs` is - freed. -- Metabox cleanup still precedes packed-inode cleanup. -- The primary device remains a separate release after the extra-device - context. -- The mount state remains the final allocation released. -- Expanding both helpers recovers the original cleanup statement sequence. -- No header, exported interface, ABI, feature, condition, lock, errno, log, or - ownership change was introduced. -- No deferred correctness, codec, descriptor, formatting, or feature work was - mixed into Pre5. -- `repo-pre-1/`, `repo-pre-2/`, `repo-pre-3/`, `src-linux/`, planning files, - reject records, and earlier reports remained protected from source changes. - -The source result therefore satisfies the intended L2-S responsibility -extraction while preserving the necessary FreeBSD cleanup implementation. - -## Build Evidence - -The required build gate was executed in the existing FreeBSD 15 VM with PID -`26318`. No new VM was started for this build. - -Command: - -```sh -FREEBSD_SRC=/root/pre5-build-gate-20260812T125645Z/freebsd-src \ -WITH_ZSTDIO=0 ./build.sh -``` - -Result: - -| Evidence | Value | -|---|---| -| Exit code | `0` | -| Compiler warnings | `0` | -| Compiler errors | `0` | -| Build log | `repo-pre-5/build/pre5-zstdio0-20260812T125645Z/build.log` | -| Module | `repo-pre-5/build/pre5-zstdio0-20260812T125645Z/erofs.ko` | -| Module size | `78,248` bytes | -| Module SHA256 | `0a2928a711715a22dfe243a536f514395acd52af3cec5515bd4bb003e42a14ac` | - -The build artifacts are under the ignored `repo-pre-5/build/` directory and -must not be committed. The module was not loaded, and the build result alone -does not establish runtime behavior. - -## Not Run And Deferred - -| Item | Status | Meaning | -|---|---|---| -| Load produced `erofs.ko` | NOT RUN | Module load and unload behavior were not checked. | -| QEMU functional testing | NOT RUN | No functional test VM run was performed for Pre5. | -| Feature or regression suite | NOT RUN | No feature-completeness claim is made. | -| Pre5 standalone smoke test | DEFERRED | Deferred by user direction and risk assessment. | -| Combined Pre5/Pre6 smoke test | REQUIRED LATER | Must be performed after Pre6 before claiming runtime coverage. | - -The Pre5 changes only extract existing private cleanup statements, and the -static review plus build gate found no source or compiler issue requiring an -immediate standalone smoke run. To avoid duplicating a relatively expensive VM -cycle, the smoke test is deferred and will be combined with Pre6. - -The combined Pre5/Pre6 smoke run must cover at least: - -1. Plain and LZ4 basic build, module load, mount, read, readdir, unmount, and - cleanup. -2. `TC099` multi-device success, missing-device failure, and device cleanup. -3. A packed-inode/metabox fixture mount and unmount path that exercises - internal-inode cleanup. - -None of these deferred checks is claimed as passed by this report. - -## Residual Risk - -- The compiled module has not been loaded, so loader, symbol-resolution, and - unload behavior remain unverified for the final source commit. -- Failure and partial-initialization paths have only been reviewed statically. -- Multi-device cleanup has not been exercised at runtime after extracting - `erofs_free_dev_context()`. -- Packed-inode and metabox cleanup has not been exercised at runtime after - extracting `erofs_drop_internal_inodes()`. -- `WITH_ZSTDIO=1` was not built in this stage. -- No QEMU smoke, feature suite, stress test, or regression test has been run - specifically against the final Pre5 source state. - -These risks are accepted for the current stage and are carried into the -combined Pre5/Pre6 smoke gate. Until that gate runs, Pre5 should be described -as static-review PASS and `WITH_ZSTDIO=0` build PASS, not runtime PASS. diff --git a/docs/pre6-baseline.md b/docs/pre6-baseline.md deleted file mode 100644 index 815b45b..0000000 --- a/docs/pre6-baseline.md +++ /dev/null @@ -1,99 +0,0 @@ -# Pre6 Initialization Baseline - -## Repository Baseline - -- Outer starting commit: `bfed0431693e7798bdecacd78b7e6258c2d93c5a` -- Starting `repo-pre-6` tree: `b4a40164394b378adc947d6ad14dc7c845e79b2c` -- Starting `repo-pre-6/src/super.c` blob: `03b92560bb5ef01f322b0d052f84bc3c577ef829` -- Execution target: the existing `repo-pre-6` directory -- Source allowlist for later extraction commits: `repo-pre-6/src/super.c` - -The tree and blob identities above were read from Git before this document was -created. Pre6 works directly on `repo-pre-6`; it does not create another source -snapshot. - -## Stage Scope - -Pre6 is an L2-I private mount-initialization responsibility extraction stage. -It contains three helper tasks: - -1. Extract one decoded external-device initialization block into - `static int erofs_init_device(...)`. -2. Extract packed-carrier initialization into - `static int erofs_init_packed_inode(struct erofs_mount *em)`. -3. Extract metabox initialization into - `static int erofs_init_metabox_inode(struct erofs_mount *em)`. - -These tasks move existing statements into private helpers. They must not change -features, behavior, public interfaces, error handling, allocation, cleanup, -logging, locking, GEOM operations, or object ownership. - -## Source Commit Strategy - -The source work is delivered in two commits: - -1. One independent commit for `erofs_init_device()`. -2. One atomic commit for `erofs_init_packed_inode()` and - `erofs_init_metabox_inode()`. - -Packed and metabox initialization remain one ownership unit because their -blocks are adjacent and a fragment-backed metabox can depend on the packed -carrier. No intermediate commit may contain only one internal-inode helper. - -Each commit, including this baseline commit, must be pushed immediately to -`xdm main`. Execution must stop if local HEAD, `xdm/main`, and remote `main` -do not agree after a push. - -## Protected Paths - -The following paths must remain unchanged during Pre6 execution: - -- `repo-pre-1/` -- `repo-pre-2/` -- `repo-pre-3/` -- `repo-pre-5/` -- `src-linux/` -- `planning/reject/` -- `1-code-similarity-review/` -- existing planning documents and reports - -Except for Pre6 documents under `repo-pre-6/docs/`, no path outside -`repo-pre-6/src/super.c` belongs in the planned source commits. - -## Behavior-Preservation Invariants - -- Preserve the order of every moved statement and every helper call. -- Preserve all conditions, loop direction, values, types, casts, and return - values. -- Preserve every errno and log message without translation or normalization. -- Preserve allocations, frees, NULL assignments, and failure paths. -- Preserve the current owner and lifetime of every buffer, device, inode, and - mount resource. -- Keep `erofs_open_device()` unchanged as the FreeBSD GEOM open transaction. -- Keep external-device slot lookup and ascending iteration in - `erofs_scan_devices()`. -- Keep packed inode initialization before metabox inode initialization. -- Keep `erofs_mountfs()` as the common failure owner that calls - `erofs_sb_free()`. -- Add no rollback, validation, cleanup, logging, locking, or defensive changes. -- Keep all new helpers private and add no header declarations. - -If an extraction requires a behavior, ownership, ordering, or errno change, -execution must stop instead of expanding Pre6 scope. - -## Validation Status at Baseline - -| Validation item | Status | -|---|---| -| Static Git baseline capture | RECORDED | -| Device initialization extraction | NOT RUN | -| Packed and metabox initialization extraction | NOT RUN | -| Static equivalence and independent revert gates | NOT RUN | -| Build with `WITH_ZSTDIO=0` | NOT RUN | -| Plain/LZ4 smoke test | NOT RUN | -| Complete `TC099` multi-device smoke test | NOT RUN | -| Positive `TC142` metabox smoke test | NOT RUN | -| Other build, QEMU, or runtime tests | NOT RUN | - -No source extraction, build, QEMU run, feature validation, regression test, or -runtime result is claimed by this baseline document. diff --git a/docs/pre7-baseline.md b/docs/pre7-baseline.md deleted file mode 100644 index 35ce9b0..0000000 --- a/docs/pre7-baseline.md +++ /dev/null @@ -1,55 +0,0 @@ -# Pre7 Helper-Alignment Baseline - -## Repository Baseline - -- Execution-start commit: `3f5f783b9f316af6d3887421251cc79909aed934` -- Starting `repo-pre-7` tree: `909ffaa0f586a51933f4220a8fe67d963d5b1dff` -- Starting `repo-pre-6` tree: `909ffaa0f586a51933f4220a8fe67d963d5b1dff` -- Starting `repo-pre-7/src/internal.h` blob: `253a30a595f9740ce97de865cc6f255f8eda2b74` -- Starting `repo-pre-7/src/inode.c` blob: `c753c14fdc48ae53e6dd2554cb9dc4a5b8a6c569` -- Starting `repo-pre-7/src/xattr.c` blob: `cef4db2a8d03be028c0344941cea60def9cd84ce` -- Execution target: the existing `repo-pre-7` directory - -The tracked `repo-pre-7` baseline is an exact snapshot of `repo-pre-6` at the -execution-start commit. The identities above were read from Git objects at -that commit, not inferred from the live worktree. - -Uncommitted changes were already present in `repo-pre-7/src/internal.h` and -`repo-pre-7/src/inode.c` when this document was written. This baseline does -not assess, validate, or claim completion of those changes. - -## Exact Source Scope - -Pre7 contains exactly two source tasks: - -1. Rename the private inode-location helper from - `erofs_nid_to_offset()` to `erofs_iloc()` in - `repo-pre-7/src/internal.h` and `repo-pre-7/src/inode.c`. This is an - identifier-only alignment; the FreeBSD signature, implementation, - callers, metabox handling, overflow checks, and failure behavior remain - unchanged. -2. Extract the private `erofs_xattr_prefix()` mapping helper from - `erofs_xattr_namespace_prefix()` in `repo-pre-7/src/xattr.c`. The existing - wrapper retains FreeBSD namespace validation, namespace matching, errno - selection, and all existing get/list call sites. - -No other source file, helper cleanup, behavior change, public interface -change, formatting pass, or deferred planning item belongs in Pre7. - -## Validation Status - -Per the current execution instruction, this baseline-recording step performs -no build or test activity. Historical Pre6 evidence is not a Pre7 result. - -| Validation item | Status | -|---|---| -| Pre7 source implementation validation | NOT RUN | -| Static reference and behavior-matrix checks | NOT RUN | -| Independent source-commit revert checks | NOT RUN | -| `WITH_ZSTDIO=0` build | NOT RUN | -| QEMU smoke testing | NOT RUN | -| Manual or automated runtime tests | NOT RUN | -| CI testing | SKIPPED | - -This is an implementation baseline only. It is not a completion report and -does not claim that either Pre7 source task is complete or correct. diff --git a/docs/pre8-baseline.md b/docs/pre8-baseline.md deleted file mode 100644 index a4b1f52..0000000 --- a/docs/pre8-baseline.md +++ /dev/null @@ -1,86 +0,0 @@ -# Pre8 Responsibility-Alignment Baseline - -## Repository Identity - -The following identities were read directly from Git objects at execution-start -commit `475b62437f89a8a98e0e3a1d1628481768525cb2`: - -```text -execution-start commit: 475b62437f89a8a98e0e3a1d1628481768525cb2 -execution-start tree: 3f0bc6be8324446440ce578f03d892c95f73591c -repo-pre-7 tree: b8f9af2cd55225c4348b79ff5910ae6fc83cd517 -repo-pre-8 tree: b8f9af2cd55225c4348b79ff5910ae6fc83cd517 -``` - -The equal `repo-pre-7` and `repo-pre-8` tree identities establish that -`repo-pre-8` is an exact tracked snapshot of `repo-pre-7` at the execution -start. This relationship is based on Git object identities, not a live -worktree comparison. - -Relevant source identities at the same commit are: - -```text -mode blob path -100644 e06822e363d9122a39256494bde7d12cfea1f7c9 repo-pre-8/src/decompressor.c -100644 ff595fe009679a4e950eb3743b2f152fac4aad31 repo-pre-8/src/decompressor_lzma.c -100644 f3e48cd2016608aaf84ed5e80151782a6c5513f7 repo-pre-8/src/decompressor_deflate.c -100644 23756af263ceb148ca50fbb8cd1ae80ba3fac4cd repo-pre-8/src/decompressor_zstd.c -100644 cac9ee15f81a1607a03fbab3466cf89c588872c8 repo-pre-8/src/internal.h -100644 2257a2299b3bcf3ab15978d723843f26d0b4b346 repo-pre-8/src/xattr.c -``` - -These values define the immutable comparison baseline for Pre8. Concurrent -or later worktree changes are not assessed by this document. - -## Execution Scope - -Pre8 is limited to the following private responsibility-alignment tasks: - -1. Move `z_erofs_load_lzma_config()` from the core decompressor file to the - LZMA backend without changing its name, signature, validation order, - state updates, or error behavior. -2. Move `z_erofs_load_deflate_config()` to the DEFLATE backend with its - existing format checks, window-bit limits, state updates, and errors - preserved. -3. Move `z_erofs_load_zstd_config()` to the ZSTD backend while preserving the - availability check, configuration checks, window-log limit, diagnostics, - and `WITH_ZSTDIO` behavior. -4. Replace the duplicated inline and shared xattr traversal paths with shared - private iterator helpers while preserving lookup priority, list order, - namespace and errno timing, validation differences, output accounting, and - buffer ownership. - -The source allowlist is restricted to the six files identified above. This -baseline file is the only documentation path used by the baseline step. - -Pre8 does not include superblock extraction, a new `compress.h` contract, -descriptor tables, decode callback ABI changes, codec primitive renaming, -typed errno conversion, Linux page or folio abstractions, broad function -reordering, or any item already recorded under `planning/reject/`. - -## Initial Validation Status - -This document records the execution baseline only. No implementation, -correctness, build, or runtime conclusion is made here. - -| Validation item | Initial status | -|---|---| -| Pre8 source implementation | NOT RUN | -| Source allowlist and protected-path audit | NOT RUN | -| `git diff --check` | NOT RUN | -| Loader definition, declaration, reference, and body-equivalence checks | NOT RUN | -| Xattr iterator invariant and behavior review | NOT RUN | -| Direct-parent restoration checks for source commits | NOT RUN | -| Final-HEAD inverse-patch compatibility checks | NOT RUN | -| `WITH_ZSTDIO=0` build | NOT RUN | -| `WITH_ZSTDIO=1` build | NOT RUN | -| Basic plain/LZ4 QEMU mount, read, readdir, and unmount smoke | NOT RUN | -| Pre7 deferred QEMU coverage | NOT RUN | -| Pre8 xattr QEMU coverage | NOT RUN | -| Pre8 codec QEMU coverage | NOT RUN | -| QEMU cleanup and zero-residual-state checks | NOT RUN | -| Completion and evidence-honesty review | NOT RUN | - -Historical evidence from an earlier snapshot is not a Pre8 result. Every -status above remains `NOT RUN` until supported by fresh evidence from the -final Pre8 DUT. This file must not be interpreted as a completion report. diff --git a/docs/pre8-completion-and-validation.md b/docs/pre8-completion-and-validation.md deleted file mode 100644 index e2b67a1..0000000 --- a/docs/pre8-completion-and-validation.md +++ /dev/null @@ -1,314 +0,0 @@ -# Pre8 Completion and Validation - -## Final Verdict - -Pre8 overall status is **FAIL**. - -The required implementation, static review, revert review, build matrix, basic -plain/LZ4 smoke, and selected test suite were all executed. The selected suite -did not pass its required gate. In particular, TC004 produced a definite DUT -failure while twelve other DUT verdicts were blocked by test-harness defects or -prior test residue. Under the status definition in `planning/pre8`, a required -action that ran and failed a gate is `FAIL`, not `PASS` or `PARTIAL`. - -This overall verdict does not erase the narrower successful results: - -| Validation layer | Verdict | Scope | -| --- | --- | --- | -| Source implementation | PASS | Planned loader placement and xattr iterator work is present in the final source tree. | -| Static source review | PASS | Allowlist, token-equivalence, references, xattr invariants, and protected paths passed review. | -| Revert and history review | PASS | Accepted replacement commits passed their defined direct-parent and final-HEAD checks; historical failures remain disclosed. | -| Four build configurations | PASS | `default`, `debug`, `zstdio0`, and `zstdio1` built and passed module load/unload smoke. | -| Basic plain/LZ4 smoke | PASS | Independent run `20260812T210345.913633Z-770842-a2868dd9`. | -| Selected 19-case suite | FAIL | Automation: 5 PASS / 9 FAIL / 4 ERROR / 1 NOT_RUN. Audited DUT: 6 PASS / 1 FAIL / 12 BLOCKED. | -| Pre8 overall | **FAIL** | A required selected-suite gate ran and failed. | - -No claim is made that Pre8, repo22, the LZMA issue, or the test harness was -fixed by this validation work. - -## Final DUT Identity - -The final repository identity used by both selected-suite and basic-smoke -evidence is: - -```text -parent repository commit: d1f5b686e8945d06b6e8c0a0188b2262de0b4ac2 -parent repository tree: 19f450085cad49d06f6fe05d180313f13d4347f0 -repo-pre-8 tree: 497736695fcc4285760b24c5954c34d32b8af49f -repo-pre-8/src tree: cbf93a19df8d1ce2fb66e2aeb7276a4ea6d4dcc2 -``` - -Selected-suite DUT archive: - -```text -bbcfd8a6a28faba3f7c5c7e827bde5578774be6e08b83d05485d58ee16a7fb6f -``` - -Basic-smoke DUT archive: - -```text -402459ba6a6f55696d06474f7494a69ef1a5fe49063d565cacadece37e8b9ce1 -``` - -The different archive hashes belong to different runners and archive -procedures. Both reports bind their run to the same final repository commit -and `repo-pre-8/src` tree. - -## Source Delivery - -### Accepted implementation - -The final accepted source changes are: - -1. `3d28d96fc48753231b1899fce6189e586cf988c6` - (`erofs: unify inline and shared xattr iteration`) -2. `aac1412056f9ae7269f7c0f85c0233ade8cc662a` - (`erofs: move codec config loaders atomically`) - -The xattr commit introduces common private inline/shared iteration while -preserving the reviewed lookup order, list order, namespace and errno timing, -inline/shared validation differences, buffer ownership, and output accounting. - -The atomic loader replacement moves the LZMA, DEFLATE, and ZSTD configuration -loaders to their backend files. The loader bodies remain token-equivalent to -the baseline, dispatch calls remain in the core, decode functions are not -changed, and no Linux page/folio or descriptor ABI was introduced. - -### Superseded split-loader history - -The following pushed commits remain in history and must not be described as -accepted independent delivery units: - -| Commit | Historical final-HEAD inverse result | Final disposition | -| --- | --- | --- | -| `dd7f2483d468ec4397a863c7a871391ea1bc2c4c` | FAIL: conflict in `decompressor.c`; `internal.h` auto-merged as staged content | Superseded by `aac1412`; failure permanently retained. | -| `0dba0ea223bee8626b8cd3371562f87ed975a99f` | FAIL: conflict in `decompressor.c`; `internal.h` auto-merged as staged content | Superseded by `aac1412`; failure permanently retained. | -| `7c47f6d` | PASS | Superseded with the split-loader group by `aac1412`. | - -All three split commits passed direct-parent restoration at the point where -they were introduced. That fact does not overwrite the two final-HEAD inverse -failures. The split group was reverted by `96f041f`, then replayed as the one -accepted atomic commit `aac1412`. Commit `3d28d96` was not superseded. - -### Static and revert verdict - -Final static and revert status is **PASS**: - -- the implementation remained within the planned source allowlist; -- protected trees and unrelated paths were unchanged; -- all three moved loader bodies matched their baseline implementations; -- declaration, definition, and dispatch-call closure was correct; -- the ZSTD availability and `windowlog > 10` rejection order was preserved; -- codec decode bodies were unchanged; -- the xattr iterator behavior and ownership matrix passed independent review; -- `aac1412` restored its direct parent exactly when reverted; -- `aac1412` independently reverted from final HEAD without changing xattr; -- `3d28d96` independently reverted from final HEAD without changing the - accepted loader layout; -- final `repo-pre-8/src` matched the reviewed functional source tree; -- `git diff --check` and worktree cleanliness checks passed at static handoff. - -The detailed history record is -`repo-pre-8/docs/pre8-loader-history-correction.md`. - -## Build Validation - -The selected-suite runner built four module configurations. Every -configuration passed build completion, SHA256 recording, unresolved-symbol -checks, and an actual `kldload`/`kldunload` smoke before case execution. - -| Configuration | Verdict | SHA256 | Size | -| --- | --- | --- | --- | -| `default` | PASS | `348cb1f5a91d47e1412747d285c795c49375a61a66cfb3d35d22f38ecfde6ea0` | 47,272 bytes | -| `debug` | PASS | `274189804511d249b34fbb3c223e9525b07c7c6266ec39d016ad2f33185001c7` | **UNKNOWN**; the harness did not persist size evidence before VM destruction | -| `zstdio0` | PASS | `348cb1f5a91d47e1412747d285c795c49375a61a66cfb3d35d22f38ecfde6ea0` | 47,272 bytes; byte-identical to `default` | -| `zstdio1` | PASS | `6cdb8e01fdac7805cb7ae12a9d1804b00d3e5cf37ef8b2f583137d36ee70426e` | 51,368 bytes | - -The missing debug size is not reconstructed or inferred. No KLD or other -binary was committed. - -## Basic Plain/LZ4 Smoke - -Independent basic smoke run: - -```text -run ID: 20260812T210345.913633Z-770842-a2868dd9 -automation: PASS -runner exit: 0 -DUT result: PASS for plain and LZ4 basic differential scope -guest: FreeBSD 15.0-RELEASE-p8 amd64 -KLD SHA256: 527fe80ad7307d0200cfc4fec84c6fb86b914e89889e4a42ff0613fc24465015 -KLD size: 47,272 bytes -``` - -All twelve runner stages passed. The run created a deterministic corpus with -256 regular files, 9 symlinks, and 43 directories. Both the plain and LZ4 -images mounted read-only, passed a 307-entry differential preflight, completed -all four workers, and unmounted cleanly. The test QEMU exited gracefully, its -overlay and large temporary data were removed, and ports 10000 and 10001 were -released. - -This result covers only plain/LZ4 basic mount, traversal, read, differential, -unmount, and cleanup behavior. It does not override the selected-suite -failures or blocked verdicts. - -Evidence: - -- `/work/tests-dev/erofsstress/demo-result/repo-pre-8-smoke-20260812/20260812T210345.913633Z-770842-a2868dd9/audit-report.md` -- `/work/tests-dev/erofsstress/demo-result/repo-pre-8-smoke-20260812/20260812T210345.913633Z-770842-a2868dd9/result.json` -- tests-dev commit `374b8e3ab4b0df24d6761802b70c694e2896d5fd` - -## Selected Suite - -Primary run: - -```text -run ID: pre8-required-20260812T194817Z -cases: 19 -automation: 5 PASS / 9 FAIL / 4 ERROR / 1 NOT_RUN -runner exit: 3 -duration: 1892.44 seconds -audited DUT: 6 PASS / 1 FAIL / 12 BLOCKED -``` - -The automation status is the runner's result. The DUT status is a separate -evidence audit. An automation failure caused by a harness defect is not changed -to automation PASS. Likewise, a positive command observed before a missing -negative check does not make the full DUT case PASS. - -### Per-case matrix - -| Case | Automation | DUT | Audited observation | -| --- | --- | --- | --- | -| TC005 | FAIL | BLOCKED | Inline listing and both requested user xattr values succeeded. The missing-xattr `ENOATTR` assertion was never invoked because the truss output directory did not exist. | -| TC013 | PASS | PASS | Invalid NID returned `EINTEGRITY` (`ERR#97`) twice; mount, unmount, and zero-state checks passed. | -| TC079 | PASS | PASS | Packed user and trusted prefix values matched expected data and cleanup passed. | -| TC080 | FAIL | BLOCKED | Four packed-prefix values were correct. The missing-suffix `ENOATTR` assertion was not invoked because the truss output directory was absent. | -| TC082 | FAIL | BLOCKED | ACL bytes and ordering were observed, but automation compared spaced hex against an unspaced prefix; the user-namespace negative assertion also was not invoked. | -| TC067 | FAIL | PASS | Both shared files, the local xattr, and listing matched the fixture. Automation incorrectly expected one additional trailing `!`. | -| TC068 | FAIL | BLOCKED | Shared listing succeeded. The nonexistent lookup was not invoked because truss could not create its output file. | -| TC069 | PASS | PASS | Three shared xattrs were listed and all exact values were read; cleanup passed. | -| TC081 | PASS | PASS | Metabox shared, shared-prefix, and packed-prefix xattrs were listed and read from both files; cleanup passed. | -| TC117 | FAIL | BLOCKED | Both corrupt fixtures mounted and valid local shared data remained readable. Corrupt-entry errno assertions were not invoked because truss failed first. | -| TC135 | PASS | PASS | Metabox, packed, and primary-prefix fallback values matched expected data. | -| TC138 | FAIL | BLOCKED | Valid unordered and empty-header ACLs were read. Three malformed ACL assertions were not invoked because the scratch directory was absent. | -| TC140 | FAIL | BLOCKED | Positive compressed and fragment metabox carrier hashes and xattrs passed. Four negative mount commands were not invoked because truss failed opening its trace. | -| TC142 | FAIL | BLOCKED | Positive fragment-backed compressed metabox hash and xattrs passed. Four negative mount commands were not invoked for the same reason. | -| TC004 | ERROR | **FAIL** | LZMA image mounted, but SHA256 of the mounted 8 MiB `level.dat` timed out after 30 seconds. The process remained running and left the mount and `md0` busy. | -| TC084 | ERROR; follow-up FAIL | BLOCKED | Primary run was contaminated by TC004 residue. Fresh follow-up mounted LZ4 and found the target, but guest `dump.erofs` was absent before content verification. | -| TC102 | ERROR; follow-up FAIL | BLOCKED | Primary run was contaminated by TC004 residue. Fresh follow-up mounted DEFLATE, but missing guest `dump.erofs` prevented algorithm and content verification. | -| TC105 | ERROR; follow-up FAIL | BLOCKED | Primary run was contaminated by TC004 residue. Fresh follow-up mounted ZSTD with `zstdio1`, but missing guest `dump.erofs` prevented verification. | -| TC145 | NOT_RUN; follow-up ERROR | BLOCKED | Both zstdio modules built and initial load smoke passed. The follow-up queried and unloaded module name `erofs` instead of the loaded artifact name, so functional mount/read gates were not reached. | - -### Definite DUT failure - -TC004 is a definite observed DUT failure for this test run: - -1. `lzma-level6.erofs` attached as `md0`. -2. The read-only EROFS mount succeeded. -3. The mounted `level.dat` existed. -4. SHA256 of the source file completed and returned - `ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461`. -5. SHA256 of the mounted 8 MiB file did not finish within 30 seconds. -6. The timed-out `sha256` process remained alive. -7. Unmount and `mdconfig -d` returned `Device busy`. - -This evidence does **not** prove that Pre8 introduced the problem. No matching -Pre7 or pre-change comparison run with the same image, guest, module build, -and command is available. The issue is recorded separately in -`issues/pre8-lzma-read-timeout.md` without an attribution claim. - -### Codec follow-up - -Fresh follow-up run: - -```text -run ID: pre8-codec-followup-20260812T202059Z -cases: TC084, TC102, TC105, TC145 -automation: 3 FAIL / 1 ERROR -runner exit: 1 -duration: 1136.92 seconds -audited DUT: 4 BLOCKED -``` - -The follow-up isolated these cases from TC004 residue, but it did not produce -codec functional PASS verdicts. TC084, TC102, and TC105 were blocked by absent -guest `dump.erofs`. TC145 was blocked by incorrect module-name lookup/unload -and cleanup behavior in the harness. These results remain FAIL/ERROR in -automation and BLOCKED for the DUT. - -The harness findings are recorded in -`issues/pre8-test-harness-blockers.md`. No harness modification is part of -Pre8 source delivery. - -Selected-suite evidence: - -- `/work/tests-dev/demo-result/repo-pre-8-smoke-20260812/summary.md` -- `/work/tests-dev/demo-result/repo-pre-8-smoke-20260812/summary.json` -- `/work/tests-dev/demo-result/repo-pre-8-smoke-20260812/pre8-required-20260812T194817Z/summary.json` -- `/work/tests-dev/demo-result/repo-pre-8-smoke-20260812/pre8-required-20260812T194817Z/evidence/` -- `/work/tests-dev/demo-result/repo-pre-8-smoke-20260812/pre8-codec-followup-20260812T202059Z/summary.json` -- `/work/tests-dev/demo-result/repo-pre-8-smoke-20260812/pre8-codec-followup-20260812T202059Z/evidence/` -- tests-dev commit `46e6840f1b8918414af76c9a5731322c697eee8b` - -## Fixtures and Immutable Inputs - -The common base image was not modified: - -```text -path: /work/tests-dev/lfs/freebsd-15-dev-src-20260811.bp -format: qcow2 -mode: 0444 -size: 16,515,530,752 bytes -SHA256: 67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef -``` - -Selected primary fixture deployment identities: - -| Group | Manifest SHA256 | Checksum-list SHA256 | -| --- | --- | --- | -| G1 | `9ffe9e695a6544cb010a5d4ba1e0c2c09d7455e6d7e28cc81da54d2dd954f2f2` | `db416004ff6b00ad286e50da506168b35a544a7da5619042225198843e4c741c` | -| G4 | `6200842757dae30f3a5b0683d726d339a24e55dff98cd084597462289a9308aa` | `229c84c5436cf9b5b659ded375097fba13621c20a2cf2d845f146df737836622` | -| G5 | `c9540baafb53b3783233c3950f4d8548c28585cd32335dadc14db613692a8978` | `d08138b1bd51579cbf5461a73abce9d7fddc9cdb8ace1329c9f1581d41e1095f` | - -Follow-up G5 deployment identities: - -```text -manifest: 9be85e48115f2dd2cae058f2a4ea7438512f28e647c30476d049d6b8025b02fd -checksum list: 7513425c98400f641523083bf0d06b2d548aa3163dfcdfe980dca2b0e1900acd -``` - -Deployment hashes can include run-specific metadata. Image, source, generator, -and checksum evidence remains beneath each run's `fixture-evidence/` and -per-case evidence directories. - -## Cleanup and Guard Integrity - -- The selected primary and follow-up QEMUs were force-destroyed by the runner - after their final errors. -- Their run-owned overlays were deleted. -- Ports 10000 and 10001 were released and closed. -- The independent basic-smoke QEMU shut down gracefully and its overlay was - removed. -- Guard PID 26318 retained its identity and TCP port 9222 remained active. -- Base-image SHA256, mode, and size were unchanged before and after the runs. -- The DUT commit and source tree remained unchanged and clean. -- No DUT source, repo22 source, base image, test source, fixture generator, or - `oldtests` content was modified during validation. - -Cleanup of the disposable VM does not change TC004's case-level cleanup result: -the case itself failed to terminate its read process and could not unmount or -detach `md0`. VM destruction only removed the run environment afterward. - -## Final Handoff - -Pre8 delivers the planned source-responsibility changes and passes static, -revert, build, and basic plain/LZ4 smoke validation. It does not satisfy the -full required validation gate because the selected suite contains one definite -DUT failure and twelve blocked DUT cases. The honest final status is therefore -**FAIL**. - -Open follow-up records: - -- `issues/pre8-lzma-read-timeout.md` -- `issues/pre8-test-harness-blockers.md` diff --git a/docs/pre8-loader-history-correction.md b/docs/pre8-loader-history-correction.md deleted file mode 100644 index bcaa406..0000000 --- a/docs/pre8-loader-history-correction.md +++ /dev/null @@ -1,74 +0,0 @@ -# Pre8 Loader History Correction - -## Scope - -This record corrects the commit organization of the Pre8 codec configuration -loader moves. It does not report a source behavior defect and does not rewrite, -amend, squash, or hide any pushed commit. - -The affected historical commits are: - -```text -dd7f2483d468ec4397a863c7a871391ea1bc2c4c LZMA loader move -0dba0ea223bee8626b8cd3371562f87ed975a99f DEFLATE loader move -7c47f6d ZSTD loader move -3d28d96fc48753231b1899fce6189e586cf988c6 xattr iterator change -``` - -## Recorded Results - -Each of the three loader commits restores its direct parent's complete tree -when reverted at the commit where it was introduced. Those direct-parent -revert checks are `PASS`. - -When each commit is reverted independently from final functional commit -`3d28d96`, the results are: - -| Commit | Final-HEAD independent revert | Result | -|---|---|---| -| `dd7f248` | Unmerged conflict in `decompressor.c`; `internal.h` auto-merged as a staged modification | FAIL | -| `0dba0ea` | Unmerged conflict in `decompressor.c`; `internal.h` auto-merged as a staged modification | FAIL | -| `7c47f6d` | Applies without conflict | PASS | -| `3d28d96` | Applies without conflict | PASS | - -The two failures are caused by overlapping patch context in `decompressor.c`. -In both cases, `internal.h` is automatically merged as a staged modification, -not left as an unmerged conflict. This is a commit organization problem, not -evidence of a codec implementation or xattr behavior problem. - -The `dd7f248` and `0dba0ea` final-HEAD results remain permanently recorded as -`FAIL`. The `7c47f6d` result remains `PASS`, but that commit is superseded with -the other two loader commits so the loader responsibility change has one -atomic acceptance unit. - -## Correction Strategy - -The three split loader commits will be reverted in reverse order without -rewriting history. Their exact combined five-path change will then be replayed -as one atomic replacement commit. The replacement is limited to: - -```text -repo-pre-8/src/decompressor.c -repo-pre-8/src/decompressor_lzma.c -repo-pre-8/src/decompressor_deflate.c -repo-pre-8/src/decompressor_zstd.c -repo-pre-8/src/internal.h -``` - -Commit `3d28d96` remains accepted and is not superseded. Its `xattr.c` content -must remain unchanged through the correction. - -The new acceptance objects are: - -1. the atomic replacement loader commit, which must restore its direct parent - exactly when reverted and must independently revert from final `HEAD` - without affecting xattr content; and -2. existing xattr commit `3d28d96`, which must independently revert from final - `HEAD` without affecting the final loader layout. - -## Validation Status - -This document records static Git evidence only. No build or QEMU test has been -run for this history correction. Build and runtime validation remain -`NOT RUN` until later Pre8 validation produces fresh evidence from the final -DUT. diff --git a/docs/pre9-baseline.md b/docs/pre9-baseline.md deleted file mode 100644 index b2688ea..0000000 --- a/docs/pre9-baseline.md +++ /dev/null @@ -1,191 +0,0 @@ -# Pre9 LZMA Diagnostic Implementation Baseline - -## Baseline Meaning - -The Pre9 planning phase ended when the planning documents were committed at -`d1ee2e05b82b5e8a8927861f1e981710532f2f23`. This file starts the -implementation phase and records identities only. It does not claim that the -TC004 root cause is known, that Pre8 introduced the observed timeout, or that -any source or test-harness fix has been selected or implemented. - -All values below were read from Git objects or the named evidence files on -2026-08-13 UTC. Historical Pre8 evidence is context for controlled Pre9 -diagnosis, not a Pre9 test result. - -## Main Repository Identity - -Repository: `/work/erofs-freebsd-pre` - -```text -branch: main -execution-start commit: d1ee2e05b82b5e8a8927861f1e981710532f2f23 -execution-start tree: 95a55ff912d007ca40bd578057e8a4cc6a21bbd9 -xdm/main: d1ee2e05b82b5e8a8927861f1e981710532f2f23 -remote xdm main: d1ee2e05b82b5e8a8927861f1e981710532f2f23 -initial dirty entries: 0 -``` - -Snapshot identities at the execution-start commit: - -```text -snapshot subtree src tree -repo-pre-7 b8f9af2cd55225c4348b79ff5910ae6fc83cd517 dc203534d7b5721905f8538026e4c59346106020 -repo-pre-8 1e24c992a5b071e6fdabfdc42d342b6ed5a91cf5 cbf93a19df8d1ce2fb66e2aeb7276a4ea6d4dcc2 -repo-pre-9 1e24c992a5b071e6fdabfdc42d342b6ed5a91cf5 cbf93a19df8d1ce2fb66e2aeb7276a4ea6d4dcc2 -``` - -The equal Pre8 and Pre9 object identities establish that Pre9 starts as an -exact tracked snapshot of Pre8. Pre7 is intentionally different and remains -the earlier controlled comparison point. - -## Conditional Source Allowlist Identity - -The primary conditional allowlist is `zdata.c`, `decompressor_lzma.c`, and -`internal.h`. The remaining three files require the additional decision gates -defined by the plan. Their baseline modes and blobs are: - -```text -gate mode blob path -primary 100644 54c1c88e5c368e7b5f6a16188c48406767850821 repo-pre-9/src/zdata.c -primary 100644 688acd704e28daa46369fa2e63e895bce5262774 repo-pre-9/src/decompressor_lzma.c -primary 100644 9821fe3a8b5fcaaeb3939d191f15e9ce637cc8d1 repo-pre-9/src/internal.h -extra 100644 ca5d33cd42145159d71e153f115128ac75258708 repo-pre-9/src/inode.c -extra 100644 b233f138d036a2b37394946b382034955dc2a1df repo-pre-9/src/erofs_vnops.c -extra 100644 51170741a0fb5eced814db898c1a233e8a23088a repo-pre-9/src/zmap.c -``` - -This identity table authorizes no source change by itself. Source edits remain -conditional on controlled evidence and the applicable decision gate. - -## tests-dev and Base Image Identity - -The external test repository was inspected without modification: - -```text -repository: /work/tests-dev -branch: main -current commit: 374b8e3ab4b0df24d6761802b70c694e2896d5fd -current tree: c438e72ebe94a6f10801999bfbdd36b43134d471 -xdm/main: 374b8e3ab4b0df24d6761802b70c694e2896d5fd -initial dirty entries: 27 untracked paths -dirty inventory SHA256: bca9ca20e85698d04dad820a11bd0bbb117492c967d6fd338e7acf61bb8fa957 -``` - -The pre-existing tests-dev dirty inventory was: - -```text -demo-result/FINAL-REPORT.md -demo-result/audit-checklist.md -demo-result/audit-findings.md -demo-result/current-status.md -demo-result/fix-plan.md -demo-result/fix-verification-report.json -demo-result/progress-report.md -demo-result/repo-pre-6-smoke-20260812/ -demo-result/repo22-full-20260811T065312Z/ -demo-result/repo22-full-20260811T071634Z/ -demo-result/repo22-full-20260811T073716Z/ -demo-result/repo22-full-20260811T082907Z/ -demo-result/repo22-full-20260811T095621Z/ -demo-result/test-execution-summary.md -erofsstress/demo-result/repo-pre-3-smoke-20260812/ -erofsstress/demo-result/repo-pre-6-smoke-20260812/ -guest/setup-build-env.sh -lfs/FreeBSD-15.0-RELEASE-src.txz -scripts/check-kernel-src.sh -scripts/complete-setup-and-test.py -scripts/install-kernel-sources.py -scripts/manual-fix.sh -scripts/quick-install-src.py -scripts/run-final-test.sh -scripts/setup-build-env-simple.py -scripts/setup-build-env.py -scripts/simple-install-kernel-src.sh -``` - -These paths are not owned by this baseline step and must not be staged, -modified, or removed as part of it. - -Immutable base image identity: - -```text -path: /work/tests-dev/lfs/freebsd-15-dev-src-20260811.bp -format: QEMU QCOW2 Image (v3) -mode: 0444 -size: 16,515,530,752 bytes -SHA256: 67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef -``` - -The base image is an input only. It must not be modified during Pre9 testing. - -## Historical TC004 Evidence Identity - -The prior observation is preserved by tests-dev evidence commit -`46e6840f1b8918414af76c9a5731322c697eee8b`, tree -`7178de49f3f429796d6de71a910d71bfccd5745a`, with subject -`evidence: record repo-pre-8 required smoke`. - -```text -run ID: pre8-required-20260812T194817Z -case: TC004 - LZMA Compressed File Read -historical DUT commit: d1f5b686e8945d06b6e8c0a0188b2262de0b4ac2 -historical DUT src tree: cbf93a19df8d1ce2fb66e2aeb7276a4ea6d4dcc2 -DUT archive SHA256: bbcfd8a6a28faba3f7c5c7e827bde5578774be6e08b83d05485d58ee16a7fb6f -module SHA256: 348cb1f5a91d47e1412747d285c795c49375a61a66cfb3d35d22f38ecfde6ea0 -fixture image: G5/images/lzma-level6.erofs -fixture image SHA256: 32107a084b27362a093768b88746c37c2e99a74b9f1301a9d4046988479defe9 -source file: G5/sources/levels/level.dat -source file SHA256: ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461 -deployment manifest: c9540baafb53b3783233c3950f4d8548c28585cd32335dadc14db613692a8978 -fixture checksum hash: d08138b1bd51579cbf5461a73abce9d7fddc9cdb8ace1329c9f1581d41e1095f -TC004 evidence SHA256: de6982a11fa368fbaad7d837586daa899789cd4cd8b9f698974a3ef4e251811f -``` - -Evidence paths: - -```text -/work/tests-dev/demo-result/repo-pre-8-smoke-20260812/pre8-required-20260812T194817Z/summary.json -/work/tests-dev/demo-result/repo-pre-8-smoke-20260812/pre8-required-20260812T194817Z/evidence/TC004/TC004_evidence.json -/work/tests-dev/demo-result/repo-pre-8-smoke-20260812/summary.md -/work/erofs-freebsd-pre/issues/pre8-lzma-read-timeout.md -``` - -The historical automation verdict was `ERROR` and the audited DUT verdict was -`FAIL`: mount and lookup succeeded, but hashing the mounted 8 MiB `level.dat` -did not finish within 30 seconds. PID 95693 remained active, and the mount and -`md0` stayed busy until the disposable VM was destroyed. Attribution remains -`UNKNOWN`. These facts neither prove a Pre8 regression nor select a Pre9 fix. - -## Initial Implementation Status - -| Item | Status | -|---|---| -| Pre9 planning | COMPLETE | -| Pre9 implementation baseline | RECORDED | -| tests-dev TC004 prerequisite corrections | NOT RUN | -| tests-dev remaining full-suite prerequisites | NOT RUN | -| Controlled Pre7 reproduction | NOT RUN | -| Controlled Pre8 reproduction | NOT RUN | -| Controlled unchanged Pre9 reproduction | NOT RUN | -| Manual SSH diagnosis | NOT RUN | -| Hypothesis decision gates | NOT RUN | -| Source decision | NOT RUN | -| Optional diagnostic instrumentation | NOT RUN | -| Pre9 source implementation | NOT RUN | -| Source static review | NOT RUN | -| Source direct-parent revert gate | NOT RUN | -| Source final-HEAD inverse gate | NOT RUN | -| `WITH_ZSTDIO=0` build | NOT RUN | -| `WITH_ZSTDIO=1` build | NOT RUN | -| TC004 | NOT RUN | -| Extended LZMA matrix | NOT RUN | -| Plain/LZ4 smoke | NOT RUN | -| Full regression | NOT RUN | -| QEMU cleanup and zero-residual-state checks | NOT RUN | -| Pre9 completion and evidence-honesty review | NOT RUN | -| Pre9 overall | NOT RUN | - -No diagnostic command, tests-dev prerequisite, controlled comparison, source -change, build, QEMU run, or manual SSH investigation has been performed for -Pre9 at this baseline. Fresh evidence is required before any status above can -advance or any root-cause or repair claim can be made. diff --git a/docs/pre9-cache-final-manual-validation.md b/docs/pre9-cache-final-manual-validation.md deleted file mode 100644 index 0a3ecd4..0000000 --- a/docs/pre9-cache-final-manual-validation.md +++ /dev/null @@ -1,132 +0,0 @@ -# Pre9 decoded extent cache final manual validation - -Date: 2026-08-13 - -## Scope - -This run validated the mount-scoped decoded LZMA extent cache at enclosing -repository commit `cdcf276d12169a672d2de42996532a470ac061d9`. - -Controlled identities: - -- `repo-pre-9` tree: `9d7eaf63a1c5d07320af3ef39930fc0a3530dd11` -- `repo-pre-9/src` tree: `e657e8a63b097b061670f75ca01947a568ef33d5` -- guest-built KLD SHA-256: - `473a6205f43905972f2417609d43f67ab2cb51ea79bc1716fffa5c1914ff85af` -- LZMA image SHA-256: - `32107a084b27362a093768b88746c37c2e99a74b9f1301a9d4046988479defe9` -- LZMA source SHA-256: - `ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461` - -The existing fresh QEMU process on host port 10030 was reused. No second VM -was created. The guest used FreeBSD 15.0-RELEASE-p8 under QEMU TCG with 6 GiB -RAM and four virtual CPUs. The KLD was built in the guest with -`make WITH_ZSTDIO=0`. - -The VM used the existing base image -`/work/tests-dev/lfs/freebsd-15-dev-src-20260811.bp`. Its previously audited -SHA-256 is -`67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef`. -This run relied on that established integrity record and did not repeat the -approximately 16 GiB image hash, so base-image hashing did not block the final -validation. - -## Verdict - -| Area | Verdict | Result | -| --- | --- | --- | -| Guest build, KLD load and fixture mounts | PASS | Build returned zero, `kldload` returned zero and both LZMA and LZ4 images mounted read-only. | -| LZMA single-read correctness | PASS | 4 KiB, 16 KiB, 64 KiB and 1 MiB single `pread()` calls returned the requested bytes with `errno=0`; every output range matched the source range SHA-256. | -| LZMA full-file correctness | PASS | Cold and warm full SHA-256 runs returned zero and matched the 8 MiB source hash. | -| LZMA liveness and performance | PASS | Cold full SHA completed in 2.16 seconds and warm full SHA in 1.44 seconds, both below the 120-second host hard limit. | -| LZMA concurrency | PASS | Two simultaneous full SHA processes both returned zero and the expected hash; guest real times were 22.52 and 21.97 seconds. | -| Non-LZMA regression | PASS | The 8 MiB LZ4 fixture returned the expected SHA-256 in 9.03 seconds. | -| Cleanup and guard integrity | PASS | Test QEMU PID 812999 exited, port 10030 closed, its overlay was deleted, and guard PID 26318/port 9222 remained alive and reachable. | - -Overall verdict: **PASS for the required final manual smoke scope**. - -## Single `pread()` evidence - -Each probe program performs one target-file `pread()`. Truss confirmed one -target call at offset zero for every requested length. Dynamic-loader reads are -not counted. - -| Length | Returned | Probe command wall time | Output/source range hash | Verdict | -| ---: | ---: | ---: | --- | --- | -| 4 KiB | 4 KiB | 2.696 s | `1c3c59331a4849514667bbc51c078327577b98f75a62996bbc2fe310b014dd79` | PASS | -| 16 KiB | 16 KiB | 2.140 s | `2433ce7c2ea151f487a319447ce4fe14e1a2c2af9f0e5113fe47fc74b6138eee` | PASS | -| 64 KiB | 64 KiB | 1.369 s | `2bb87afd3b9fab420cbe55b782d69a7342150e8816a35efacf22d2ab3012815d` | PASS | -| 1 MiB | 1 MiB | 3.838 s | `52e806509e5dfeb523904f167fca765d001ea749351ecf51738e61a5352a1ba3` | PASS | - -These wall times include SSH and truss overhead and are not pure kernel read -times. - -## Full-file SHA evidence - -The 8 MiB LZMA source hash is: - -`ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461` - -| Run | Exit | Guest real time | Captured hash | Verdict | -| --- | ---: | ---: | --- | --- | -| Cold, fresh mount | 0 | 2.16 s | expected hash | PASS | -| Warm, same mount | 0 | 1.44 s | expected hash | PASS | -| Concurrent process 1 | 0 | 22.52 s | expected hash | PASS | -| Concurrent process 2 | 0 | 21.97 s | expected hash | PASS | - -The concurrency result shows substantial contention compared with a single -reader, but it completed correctly and stayed well inside the 120-second hard -limit. This run does not claim concurrency performance is optimized. - -## Baseline comparison - -The preceding controlled Pre7/Pre8/Pre9 run sampled each full SHA after about -eight seconds. Each process was still inside the LZMA decompression path and -had advanced only about 1.0-1.25 MiB; no final hash was captured. That report -therefore classified full-read liveness as failed and correctness as blocked. - -With the mount-scoped decoded extent cache, the same 8 MiB fixture completed -with the correct hash in 2.16 seconds cold and 1.44 seconds warm. This closes -the previous full-file correctness block for this fixture and demonstrates a -material liveness improvement. It does not establish a precise speedup ratio, -because the baseline process was terminated at the observation point rather -than allowed to finish. - -## Non-LZMA result - -The LZ4 compact-64k image contained `compressed.bin` with expected SHA-256: - -`3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880` - -The mounted file produced the same hash, returned zero and completed in 9.03 -seconds of guest real time. - -## Kernel and cleanup observations - -No EROFS panic, assertion, mount error or decompression error was found in the -captured dmesg tail. The image emitted pre-existing root-filesystem directory -warnings during boot; they occurred before the test KLD was loaded and are not -attributed to EROFS. - -All test-specific mounts and md providers were explicitly detached before VM -shutdown. A final generic cleanup command contained an awk quoting error, but -it ran only after the explicit LZMA and LZ4 unmount/md detach operations had -already returned zero. Host-side cleanup independently confirmed: - -- PID 812999 absent; -- port 10030 closed; -- test overlay absent; -- guard PID 26318 alive; -- guard port 9222 open. - -## Uncovered cases - -This is a targeted final smoke validation, not a full feature suite. It did not -exercise partial-reference LZMA, metadata tailpacking, fragments, multi-device -images, forced unmount under active I/O, repeated mount/unmount under memory -pressure, or deliberate allocation failure. Those remain separate regression -and stress-test work. - -Concise machine-readable evidence is stored in -`docs/pre9-manual-evidence/pre9-cache-final-result.json`. Target syscall lines -are stored in `docs/pre9-manual-evidence/pre9-cache-final-probes.txt`. diff --git a/docs/pre9-controlled-manual-qemu.md b/docs/pre9-controlled-manual-qemu.md deleted file mode 100644 index be4d7ba..0000000 --- a/docs/pre9-controlled-manual-qemu.md +++ /dev/null @@ -1,158 +0,0 @@ -# Pre9 controlled manual QEMU evidence audit - -Date: 2026-08-13 - -## Scope and evidence status - -This report audits the already completed manual run at -`/work/tests-dev/temp/pre9-manual-20260813-run3/`. No QEMU test was rerun. -The long-lived guard VM (PID 26318, host port 9222) was not touched. - -The automated TC004 implementation remains `PENDING_FIX/BLOCKED` in -`/work/tests-dev/fix-todo/tc004-automation.md`. Run3 is a manual substitute; -it is not evidence that TC004 automation passes. - -Run history: - -- Run1: infrastructure failure; no retained result directory is available. -- Run2: `INFRA`. Guest SSH authentication failed for Pre7 and Pre8, then the - host runner was interrupted while preparing Pre9. It produced no DUT result. -- Run3: valid manual run. All three DUTs built, loaded, mounted, executed the - bounded probes, produced a live SHA stack sample, and cleaned up. - -## Verdict - -| Question | Verdict | Evidence | -| --- | --- | --- | -| 4 KiB, 16 KiB, 64 KiB and 1 MiB single-syscall read correctness | PASS | Every version returned the requested byte count, `errno=0`, matched the source-range SHA-256, exited zero, and has exactly one target-file `pread()` in truss. | -| First 1 MiB sequential `dd` read | PASS | Every version returned zero and copied 1 MiB within the 20-second bound. This is a bounded smoke check, not proof of one 1 MiB kernel read. | -| Full-file SHA correctness | BLOCKED | No version retained a hash or an exit status for the background SHA. `sha_after` is empty and process disappearance is not correctness evidence. | -| Full-file read liveness/performance | FAIL | After an eight-second observation delay, SHA was still running in the LZMA decode path and had advanced only about 1.0-1.25 MiB. This is unacceptable for the 8 MiB smoke fixture and reproduces across all three versions. | -| Pre7 to Pre8/Pre9 regression attribution | PASS: no Pre9 regression demonstrated | Pre8 and Pre9 have the same `src` tree and identical KLD hash. Pre7 has a different source tree/KLD but shows the same symptom and stack. The evidence attributes the issue to shared behavior, not a Pre9-only change. | - -Overall verdict: **PARTIAL**. Bounded reads are correct, but the full-file -correctness verdict is blocked and full-file liveness fails. - -## Runner semantics audit - -The runner creates a fresh qcow2 overlay per version over the read-only base, -archives the selected repository subtree, builds the KLD in the guest, mounts -the same LZMA fixture, and runs four probes. Each probe invokes a C program that -contains one target-file `pread()` and writes the returned bytes to a file. -The runner then hashes that file and compares it with bytes read from the -uncompressed source fixture. - -Important field meanings: - -- `commit`: Git tree object for the version directory at the enclosing repo - HEAD. It is not a standalone commit ID. -- `src_tree`: Git tree object for that version's `src` directory. -- `probes[].elapsed`: host wall time for the complete SSH command, including - SSH and truss overhead. It is not pure kernel decompression time. -- `metadata.hash_match`: equality between the mounted output-range hash and - the uncompressed source-range hash. -- `signal=0`: runner shorthand for probe return code zero. It is not a signal - collected through `waitpid()`. -- `dd_1m`: a userspace `dd bs=1m count=1` result. It does not establish the - size or count of VOP/kernel reads. -- `sha_pid`: PID printed after starting a background `sha256` command. -- `diagnostic_sample`: process table, kernel stack, descriptor offset, mount, - md device and dmesg captured after an unconditional eight-second sleep. -- `sha_after`: process status followed by `sha.out` and `sha.err`. Empty output - means neither a process row nor captured SHA/error output was available. -- `status=PARTIAL`: runner-generated fallback when any bounded probe succeeds. - It does not mean full SHA correctness passed. - -The full SHA was not run under `timeout`. The runner waited eight seconds, -sampled it, sent TERM, slept two seconds, then attempted KILL, and finally read -the output files. No start/end timestamp or exit status was recorded. Therefore -the exact SHA lifetime is unknown; the only defensible timing statement is that -it was still active approximately eight seconds after launch. - -`sha_after` contains only `,state=,command=` for all versions. The subsequent -KILL reports `No such process`. This establishes only that the sampled PID no -longer existed after TERM plus the two-second delay. It does not distinguish a -successful completion from TERM handling, and the absent `sha.out` means no -hash can be validated. Process disappearance must not be reported as PASS. - -## Controlled identities and setup - -The common fixture hashes were: - -- Image: `32107a084b27362a093768b88746c37c2e99a74b9f1301a9d4046988479defe9` -- Source: `ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461` - -| Version | Repository tree | `src` tree | KLD SHA-256 | Build/load/mount | -| --- | --- | --- | --- | --- | -| Pre7 | `b8f9af2cd55225c4348b79ff5910ae6fc83cd517` | `dc203534d7b5721905f8538026e4c59346106020` | `33a52f2f16a94afda0501d305b238678b96c71e420d4b8bbcbeec6ffcdf1aae5` | PASS/PASS/PASS | -| Pre8 | `1e24c992a5b071e6fdabfdc42d342b6ed5a91cf5` | `cbf93a19df8d1ce2fb66e2aeb7276a4ea6d4dcc2` | `348cb1f5a91d47e1412747d285c795c49375a61a66cfb3d35d22f38ecfde6ea0` | PASS/PASS/PASS | -| Pre9 | `dc0c01157b5c925684bd77c57ed6703904af7453` | `cbf93a19df8d1ce2fb66e2aeb7276a4ea6d4dcc2` | `348cb1f5a91d47e1412747d285c795c49375a61a66cfb3d35d22f38ecfde6ea0` | PASS/PASS/PASS | - -Pre8 and Pre9 are runtime-identical for this test according to both the source -tree and the produced KLD hash. - -## Single-syscall probes - -All probes used offset zero. Times below include SSH/truss overhead. - -| Version | Length | Returned | Hash match | Elapsed | Exactly one target `pread()` | -| --- | ---: | ---: | --- | ---: | --- | -| Pre7 | 4 KiB | 4 KiB | yes | 1.299 s | yes | -| Pre7 | 16 KiB | 16 KiB | yes | 1.291 s | yes | -| Pre7 | 64 KiB | 64 KiB | yes | 1.438 s | yes | -| Pre7 | 1 MiB | 1 MiB | yes | 1.432 s | yes | -| Pre8 | 4 KiB | 4 KiB | yes | 1.216 s | yes | -| Pre8 | 16 KiB | 16 KiB | yes | 1.327 s | yes | -| Pre8 | 64 KiB | 64 KiB | yes | 1.430 s | yes | -| Pre8 | 1 MiB | 1 MiB | yes | 1.442 s | yes | -| Pre9 | 4 KiB | 4 KiB | yes | 1.791 s | yes | -| Pre9 | 16 KiB | 16 KiB | yes | 1.341 s | yes | -| Pre9 | 64 KiB | 64 KiB | yes | 1.374 s | yes | -| Pre9 | 1 MiB | 1 MiB | yes | 1.737 s | yes | - -The target-file truss lines are preserved in -`pre9-manual-evidence/probe-summary.txt`. Dynamic-loader `pread()` calls are -not counted as target-file calls. - -The 1 MiB `dd` results were: - -- Pre7: 1 MiB in 0.195526 s. -- Pre8: 1 MiB in 0.179484 s. -- Pre9: 1 MiB in 0.367417 s. - -These values are not stable enough for version performance ranking, but all -three bounded operations completed. - -## Full SHA sample - -| Version | State at sample | File offset | Kernel stack | Final hash | -| --- | --- | ---: | --- | --- | -| Pre7 | running (`RC`) | 1,245,184 | `lzma2_lzma -> erofs_xz_dec_microlzma_run -> lzma_decompress -> z_erofs_decompress -> z_erofs_do_read -> z_erofs_read_uio` | absent | -| Pre8 | running (`RC`) | 1,150,976 | same | absent | -| Pre9 | running (`RC`) | 1,044,480 | same | absent | - -The stack is direct evidence of active CPU-side decompression, not an EROFS -lock wait. It does not by itself prove why decompression is slow. Combined with -the static review, the leading explanation remains repeated full mapped-extent -decompression as `z_erofs_read_uio()` segments reads at `MAXPHYS`. There is no -direct evidence here of an XZ infinite loop or an EROFS lock deadlock. - -## Infrastructure integrity and cleanup - -The base image SHA-256 before and after run3 was identical and matched the -expected value: - -`67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef` - -For each version, `umount`, `mdconfig -d` and `kldunload` returned zero. Each -QEMU PID was no longer alive and each overlay was deleted. The run3 result set -therefore supports successful test-VM cleanup. This report does not infer the -state of the unrelated long-lived guard VM and did not inspect or modify it. - -## Required follow-up - -A supplemental manual test is still required if full-file correctness must be -closed. It should run one full SHA with an explicit wall-clock deadline, record -start/end timestamps and exit status, capture `sha.out` before cleanup, compare -the hash to the source fixture, and separately record whether TERM/KILL was -used. Until then, full-file SHA correctness remains `BLOCKED`. diff --git a/docs/pre9-lzma-cache-implementation.md b/docs/pre9-lzma-cache-implementation.md deleted file mode 100644 index 9083045..0000000 --- a/docs/pre9-lzma-cache-implementation.md +++ /dev/null @@ -1,142 +0,0 @@ -# Pre9 LZMA decoded extent cache - -## Decision - -Pre7, Pre8, and Pre9 all reproduced the same LZMA read-liveness symptom. The -manual evidence shows that a single large read completes quickly while a full -file read made up of successive small reads remains in the LZMA decode path. -The source path creates and destroys a decoded extent for every mapped read; -the existing FreeBSD buffer cache only retains compressed block buffers. - -A cache local to `z_erofs_read_uio()` would not cross the VOP/read boundary -between successive small user reads. Commit `61f4709` therefore kept one -decoded extent on each compressed-file vnode. Review found that retained -memory could then grow with the number of open vnodes. This follow-up moves -the cache to `struct erofs_mount`: each mount retains at most one decoded -extent until unmount, so ordinary open-file count cannot expand the cache. -A single entry can thrash between files and concurrent readers; that is an -accepted first-stage tradeoff. - -The cache is deliberately limited to complete, non-partial MicroLZMA extents. -Partial references keep the existing path because their decoded length and -reference semantics are different. Other codecs are also unchanged in this -phase: the measured failure is LZMA-specific, and broadening the change would -make the validation and regression attribution less precise. - -## Data and lifecycle - -`struct erofs_zextent_cache` stores the decoded allocation and the mapping -identity needed to prove that it can be reused: - -```c -struct erofs_zextent_cache { - void *data; - erofs_nid_t m_nid; - erofs_off_t m_pa; - erofs_off_t m_la; - uint64_t m_plen; - uint64_t m_llen; - unsigned int m_deviceid; - unsigned int m_flags; - unsigned char m_algorithmformat; -}; -``` - -The cache mutex is initialized immediately after allocating -`struct erofs_mount`. Every mount failure path reaches `erofs_sb_free()`, and -normal unmount calls the same helper after `vflush()`. There is no cache field -or cache lifecycle dependency in `struct erofs_node`. The cache key includes -the inode NID because one mount entry is shared by all regular file vnodes. -NID zero remains a valid key; `data != NULL` is the validity bit. - -EROFS is read-only, so a decoded extent does not need write invalidation. -`vflush()` completes before unmount frees the mount object, and -`z_erofs_extent_cache_fini()` releases the single mount-owned allocation. -`EROFS_MAP_META` is explicitly excluded, so metadata-backed tailpacking keeps -the previous path. `packed_inode` and `metabox_en` are also excluded because -they are mount-private backing objects, not user file vnodes. - -## Concurrency - -Decompression and block reads occur without holding `z_extent_cache_lock`. A reader -first takes the lock only to compare the complete key and copy a cache hit. -On a miss, it builds a private decoded extent. It then takes the lock again: - -1. If another reader published the same key, copy that published extent and - discard the duplicate private allocation. -2. Otherwise replace the one cached extent, copy the requested range, unlock, - and free the old allocation. - -This permits duplicate construction under concurrent misses but keeps all -shared pointer access under the mutex. The lock is never held across -`bread()`, allocation, or decompression, all of which may sleep. The cache -allocation is never freed while a reader is copying it because replacement, -hit copying, and pointer clearing are serialized by the same mutex. - -The helpers are reached only when `want <= MAXPHYS` and contain `KASSERT` -checks for that contract. Thus the largest lock-held copy is the FreeBSD -`MAXPHYS` request size, not the 12 MiB on-disk extent cap. Calls that can pass -more than `MAXPHYS` use the existing uncached copy path. This bounded mutex -copy is accepted for the first stage; a refcounted immutable entry is deferred -until runtime evidence shows that this copy is materially contended. - -## Bounds and unchanged paths - -The existing mapping sanity checks cap a mapped compressed extent at -`Z_EROFS_PCLUSTER_MAX_DSIZE` before the read path. `z_erofs_read_extent()` also -checks the compressed and decoded lengths before allocation. The read path now -explicitly checks `mapoff` and the extent length before converting them to -`size_t`; this protects the cache offset arithmetic on platforms where -`size_t` is narrower than the on-disk fields. - -Cache use requires all of the following: - -```text -compressed mapped extent -not EROFS_MAP_PARTIAL_REF -Z_EROFS_COMPRESSION_LZMA -initialized mount cache -not `EROFS_MAP_META` -not a mount-private backing inode -request length no greater than `MAXPHYS` -``` - -Fragments, holes, partial references, non-LZMA codecs, uncompressed files, -metadata reads, and mount-private backing inodes retain their previous code -paths and error handling. - -## Rejected alternatives for Pre9 - -- A function-local cache was rejected because it cannot span successive VOP - reads that caused the observed amplification. -- A per-vnode cache was rejected after the `61f4709` review: open file count - could retain one decoded extent per vnode without a system-wide bound. -- A larger cross-vnode cache was rejected because it would require an eviction - policy and larger memory accounting. The one-entry per-mount cache is the - controlled compromise: it has a fixed mount-scoped bound and simple teardown. -- A Linux page/folio/XArray/workqueue port was rejected because those are not - FreeBSD vnode/buf primitives and would create an unnecessary compatibility - layer. -- A decoder stream pool was deferred: it may reduce allocator overhead but - does not remove repeated full extent decompression. -- Changing `MAXPHYS`, changing the disk format, bypassing the buffer cache, or - adding decoder retries was rejected because none addresses the demonstrated - decoded-result reuse and each changes unrelated behavior or resource bounds. - -## Static validation and required runtime matrix - -This change is static-only in the source phase. The follow-up test agent must -run the unchanged LZMA fixture against Pre9 and require: - -1. bounded single reads at 4 KiB, 16 KiB, 64 KiB, and 1 MiB with source-range - hash equality; -2. repeated small reads spanning the same extent, with completion and hash - equality; -3. complete sequential SHA-256 with recorded exit status and elapsed time; -4. concurrent reads of the same file and close/reopen reads; -5. partial-reference, plain, LZ4, DEFLATE, and ZSTD regression coverage; -6. clean unmount, md detach, and module unload after every case. - -The automation verdict and DUT verdict must remain separate. A timeout or -missing final hash remains a failure or blocked result; it must not be promoted -to PASS because bounded reads succeed. diff --git a/docs/pre9-lzma-cache-review-resolution.md b/docs/pre9-lzma-cache-review-resolution.md deleted file mode 100644 index 32e5b62..0000000 --- a/docs/pre9-lzma-cache-review-resolution.md +++ /dev/null @@ -1,62 +0,0 @@ -# Pre9 LZMA cache review resolution - -This document records the follow-up to commit `61f4709`, which cached one -decoded LZMA extent per vnode. - -## Findings addressed - -- The cache is now one entry in `struct erofs_mount`, rather than one entry in - every `struct erofs_node`. Retained decoded memory is bounded by one extent - per mounted filesystem, with the number of mounts controlled by mount - privileges. Keeping many ordinary file descriptors open cannot create more - cache entries. -- Cache initialization is performed in `erofs_mountfs()` immediately after - mount allocation. `erofs_sb_free()` destroys it on every mount failure path - and after successful `vflush()` during unmount. Node reclaim no longer owns - cache cleanup. -- `EROFS_MAP_META` is an explicit ineligibility condition. Metadata-backed - tailpacking therefore remains on its existing read path. -- `packed_inode` and `metabox_en` are explicit ineligibility conditions. The - mount-private backing objects cannot populate or consume the user-data - cache. -- The cache type and helper names are now `erofs_zextent_cache` and - `z_erofs_extent_cache_*`, distinguishing decoded extents from Linux's - managed compressed-page cache names. -- The key retains the existing physical/logical extent fields, device id, - flags, and algorithm, and adds the stable inode `nid`. The `data` pointer is - the validity bit, so NID zero is not treated as an empty key. - -## Copy bound and concurrency - -`z_erofs_read_uio()` limits each output request to `MAXPHYS`. The generic -`z_erofs_read_data()` API can receive a larger request, so cache eligibility -rejects any request whose mapped portion exceeds `MAXPHYS`. Both cache helper -entry points also contain `KASSERT(len <= MAXPHYS)` checks. The largest copy -performed while holding `z_extent_cache_lock` is therefore `MAXPHYS`; the -12 MiB `Z_EROFS_PCLUSTER_MAX_DSIZE` limit remains an on-disk decoded-extent -allocation bound, not a mutex-copy bound. - -Allocation, compressed reads, and decompression happen outside the mutex. A -cache hit copies while holding the mutex, and a miss publishes and copies -under the same mutex before the previous allocation is freed. Concurrent -misses may decode duplicate extents and may replace one another, but pointer -access and replacement remain serialized. A refcounted immutable cache entry -was deliberately not introduced in this first stage. - -## Resource tradeoff - -The per-mount entry is intentionally a small first-stage design. It removes -the unbounded-per-open-vnode retention introduced by `61f4709`, but it can -thrash when many files are read concurrently on one mount. Retention lasts -until unmount, and the retained allocation is capped by the existing EROFS -format constant `Z_EROFS_PCLUSTER_MAX_DSIZE` (12 MiB). A FreeBSD shrinker or -pressure callback is deferred; adding one would require a broader memory -accounting and lifecycle design than this corrective commit. - -## Validation scope - -This correction is statically validated only. QEMU validation remains -required for complete SHA-256 reads, same-file concurrent reads, close/reopen, -unmount cleanup, metadata tailpacking, and memory-pressure behavior. The -existing Pre9 manual test report predates this correction and must not be -reported as runtime validation of this commit. diff --git a/docs/pre9-manual-evidence/full-sha-samples.txt b/docs/pre9-manual-evidence/full-sha-samples.txt deleted file mode 100644 index 720229b..0000000 --- a/docs/pre9-manual-evidence/full-sha-samples.txt +++ /dev/null @@ -1,26 +0,0 @@ -Run3 full-file SHA samples, captured approximately eight seconds after launch - -repo-pre-7 -state: RC -file offset: 1245184 -kernel stack: lzma2_lzma -> erofs_xz_dec_microlzma_run -> lzma_decompress -> z_erofs_decompress -> z_erofs_do_read -> z_erofs_read_uio -> VOP_READ_APV -> vn_read -sha_after: no process row, no sha.out hash, no sha.err content -termination: TERM sent; KILL then reported No such process - -repo-pre-8 -state: RC -file offset: 1150976 -kernel stack: lzma2_lzma -> erofs_xz_dec_microlzma_run -> lzma_decompress -> z_erofs_decompress -> z_erofs_do_read -> z_erofs_read_uio -> VOP_READ_APV -> vn_read -sha_after: no process row, no sha.out hash, no sha.err content -termination: TERM sent; KILL then reported No such process - -repo-pre-9 -state: RC -file offset: 1044480 -kernel stack: lzma2_lzma -> erofs_xz_dec_microlzma_run -> lzma_decompress -> z_erofs_decompress -> z_erofs_do_read -> z_erofs_read_uio -> VOP_READ_APV -> vn_read -sha_after: no process row, no sha.out hash, no sha.err content -termination: TERM sent; KILL then reported No such process - -Interpretation: the SHA processes were alive and executing in LZMA decode at -the sample. Their later disappearance does not prove successful completion. -No full-file hash was captured, so full-file correctness remains BLOCKED. diff --git a/docs/pre9-manual-evidence/pre9-cache-final-kernel-cleanup.txt b/docs/pre9-manual-evidence/pre9-cache-final-kernel-cleanup.txt deleted file mode 100644 index bc968d7..0000000 --- a/docs/pre9-manual-evidence/pre9-cache-final-kernel-cleanup.txt +++ /dev/null @@ -1,16 +0,0 @@ -Pre9 final decoded extent cache kernel and cleanup summary -Date: 2026-08-13 - -- Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG, 6144 MiB, 4 vCPUs. -- KLD built with WITH_ZSTDIO=0 and loaded successfully. -- No EROFS panic, assertion, mount error or decompression error appeared in - the captured dmesg tail. -- Boot emitted pre-existing root-filesystem "bad dir ino" warnings before - the test KLD was loaded; these are not attributed to EROFS. -- Explicit LZMA probe, LZMA full-read and LZ4 unmount/md detach operations - returned zero. -- Test QEMU PID 812999 was terminated. -- Host port 10030 was closed. -- Test overlay was removed. -- Guard QEMU PID 26318 remained alive. -- Guard SSH port 9222 remained open. diff --git a/docs/pre9-manual-evidence/pre9-cache-final-probes.txt b/docs/pre9-manual-evidence/pre9-cache-final-probes.txt deleted file mode 100644 index 5087fa2..0000000 --- a/docs/pre9-manual-evidence/pre9-cache-final-probes.txt +++ /dev/null @@ -1,21 +0,0 @@ -Pre9 final decoded extent cache target-file syscall evidence -Date: 2026-08-13 - -4 KiB: -pread(3, ..., 4096, 0x0) = 4096 (0x1000) -output/source SHA-256: 1c3c59331a4849514667bbc51c078327577b98f75a62996bbc2fe310b014dd79 - -16 KiB: -pread(3, ..., 16384, 0x0) = 16384 (0x4000) -output/source SHA-256: 2433ce7c2ea151f487a319447ce4fe14e1a2c2af9f0e5113fe47fc74b6138eee - -64 KiB: -pread(3, ..., 65536, 0x0) = 65536 (0x10000) -output/source SHA-256: 2bb87afd3b9fab420cbe55b782d69a7342150e8816a35efacf22d2ab3012815d - -1 MiB: -pread(3, ..., 1048576, 0x0) = 1048576 (0x100000) -output/source SHA-256: 52e806509e5dfeb523904f167fca765d001ea749351ecf51738e61a5352a1ba3 - -Each trace contained exactly one pread() against the mounted target file. -Dynamic-loader pread() calls were excluded from this summary. diff --git a/docs/pre9-manual-evidence/pre9-cache-final-result.json b/docs/pre9-manual-evidence/pre9-cache-final-result.json deleted file mode 100644 index 15af758..0000000 --- a/docs/pre9-manual-evidence/pre9-cache-final-result.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "archive_sha256": "a6a6d573ace42fe713529974c7bd20b472a1e4521920fac40e01c5d3bb071bcb", - "cleanup": { - "guard_pid_alive": true, - "guard_port_9222_open": true, - "overlay_exists": false, - "port_10030_open": false, - "qemu_pid_alive": false - }, - "finished_utc": "2026-08-13T06:06:25Z", - "kld_sha256": "473a6205f43905972f2417609d43f67ab2cb51ea79bc1716fffa5c1914ff85af", - "lz4_sha": { - "expected_sha256": "3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880", - "guest_real_seconds": 9.03, - "hash_match": true, - "returncode": 0, - "sha256": "3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880" - }, - "lzma_concurrent": { - "guest_real_seconds": [ - 22.52, - 21.97 - ], - "hash_match": true, - "hashes": [ - "ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461", - "ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461" - ], - "host_timeout": false, - "returncode": 0, - "returncodes": [ - 0, - 0 - ] - }, - "lzma_fixture": { - "image_sha256": "32107a084b27362a093768b88746c37c2e99a74b9f1301a9d4046988479defe9", - "source_sha256": "ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461" - }, - "lzma_sha": { - "cold": { - "guest_real_seconds": 2.16, - "hash_match": true, - "host_timeout": false, - "returncode": 0, - "sha256": "ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461" - }, - "warm": { - "guest_real_seconds": 1.44, - "hash_match": true, - "host_timeout": false, - "returncode": 0, - "sha256": "ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461" - } - }, - "repo_pre_9_tree": "9d7eaf63a1c5d07320af3ef39930fc0a3530dd11", - "schema": "pre9-cache-final-manual-v1", - "single_pread": [ - { - "elapsed_seconds": 2.696, - "hash_match": true, - "length": 4096, - "returncode": 0, - "sha256": "1c3c59331a4849514667bbc51c078327577b98f75a62996bbc2fe310b014dd79" - }, - { - "elapsed_seconds": 2.14, - "hash_match": true, - "length": 16384, - "returncode": 0, - "sha256": "2433ce7c2ea151f487a319447ce4fe14e1a2c2af9f0e5113fe47fc74b6138eee" - }, - { - "elapsed_seconds": 1.369, - "hash_match": true, - "length": 65536, - "returncode": 0, - "sha256": "2bb87afd3b9fab420cbe55b782d69a7342150e8816a35efacf22d2ab3012815d" - }, - { - "elapsed_seconds": 3.838, - "hash_match": true, - "length": 1048576, - "returncode": 0, - "sha256": "52e806509e5dfeb523904f167fca765d001ea749351ecf51738e61a5352a1ba3" - } - ], - "src_tree": "e657e8a63b097b061670f75ca01947a568ef33d5", - "started_utc": "2026-08-13T05:59:41Z", - "tested_head": "cdcf276d12169a672d2de42996532a470ac061d9", - "verdict": { - "build_load_mount": "PASS", - "cleanup": "PASS", - "lzma_concurrency": "PASS", - "lzma_full_correctness": "PASS", - "lzma_liveness_performance": "PASS", - "lzma_single_pread_correctness": "PASS", - "non_lzma_lz4": "PASS" - } -} diff --git a/docs/pre9-manual-evidence/probe-summary.txt b/docs/pre9-manual-evidence/probe-summary.txt deleted file mode 100644 index 8ac04fe..0000000 --- a/docs/pre9-manual-evidence/probe-summary.txt +++ /dev/null @@ -1,22 +0,0 @@ -Run3 target-file syscall evidence (dynamic-loader pread calls omitted) - -repo-pre-7 -4096: pread(fd, ..., 4096, 0) = 4096; process exit 0 -16384: pread(fd, ..., 16384, 0) = 16384; process exit 0 -65536: pread(fd, ..., 65536, 0) = 65536; process exit 0 -1048576: pread(fd, ..., 1048576, 0) = 1048576; process exit 0 - -repo-pre-8 -4096: pread(fd, ..., 4096, 0) = 4096; process exit 0 -16384: pread(fd, ..., 16384, 0) = 16384; process exit 0 -65536: pread(fd, ..., 65536, 0) = 65536; process exit 0 -1048576: pread(fd, ..., 1048576, 0) = 1048576; process exit 0 - -repo-pre-9 -4096: pread(fd, ..., 4096, 0) = 4096; process exit 0 -16384: pread(fd, ..., 16384, 0) = 16384; process exit 0 -65536: pread(fd, ..., 65536, 0) = 65536; process exit 0 -1048576: pread(fd, ..., 1048576, 0) = 1048576; process exit 0 - -All mounted-output hashes matched corresponding source-range hashes. The full -hash values and elapsed times are retained in run3-audit.json. diff --git a/docs/pre9-manual-evidence/run3-audit.json b/docs/pre9-manual-evidence/run3-audit.json deleted file mode 100644 index 994fb45..0000000 --- a/docs/pre9-manual-evidence/run3-audit.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "base_image": { - "before": "67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef", - "after": "67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef", - "expected": "67f359621f23a1d745f0889370cbb99a096cee3e99a0b2f3bb18fc7a91bf6fef", - "unchanged": true - }, - "fixture": { - "image_sha256": "32107a084b27362a093768b88746c37c2e99a74b9f1301a9d4046988479defe9", - "source_sha256": "ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461" - }, - "versions": [ - { - "name": "repo-pre-7", - "repository_tree": "b8f9af2cd55225c4348b79ff5910ae6fc83cd517", - "src_tree": "dc203534d7b5721905f8538026e4c59346106020", - "kld_sha256": "33a52f2f16a94afda0501d305b238678b96c71e420d4b8bbcbeec6ffcdf1aae5", - "probes": [ - {"length": 4096, "returned": 4096, "elapsed_seconds": 1.2993653398007154, "hash_match": true, "target_pread_count": 1}, - {"length": 16384, "returned": 16384, "elapsed_seconds": 1.2910558842122555, "hash_match": true, "target_pread_count": 1}, - {"length": 65536, "returned": 65536, "elapsed_seconds": 1.4380157887935638, "hash_match": true, "target_pread_count": 1}, - {"length": 1048576, "returned": 1048576, "elapsed_seconds": 1.4320225361734629, "hash_match": true, "target_pread_count": 1} - ], - "dd_1m_seconds": 0.195526, - "sha_sample_offset": 1245184, - "sha_hash_captured": false, - "cleanup": {"umount": 0, "mdconfig_detach": 0, "kldunload": 0, "qemu_alive": false, "overlay_exists": false} - }, - { - "name": "repo-pre-8", - "repository_tree": "1e24c992a5b071e6fdabfdc42d342b6ed5a91cf5", - "src_tree": "cbf93a19df8d1ce2fb66e2aeb7276a4ea6d4dcc2", - "kld_sha256": "348cb1f5a91d47e1412747d285c795c49375a61a66cfb3d35d22f38ecfde6ea0", - "probes": [ - {"length": 4096, "returned": 4096, "elapsed_seconds": 1.2161498833447695, "hash_match": true, "target_pread_count": 1}, - {"length": 16384, "returned": 16384, "elapsed_seconds": 1.3266378343105316, "hash_match": true, "target_pread_count": 1}, - {"length": 65536, "returned": 65536, "elapsed_seconds": 1.4296557288616896, "hash_match": true, "target_pread_count": 1}, - {"length": 1048576, "returned": 1048576, "elapsed_seconds": 1.4422911144793034, "hash_match": true, "target_pread_count": 1} - ], - "dd_1m_seconds": 0.179484, - "sha_sample_offset": 1150976, - "sha_hash_captured": false, - "cleanup": {"umount": 0, "mdconfig_detach": 0, "kldunload": 0, "qemu_alive": false, "overlay_exists": false} - }, - { - "name": "repo-pre-9", - "repository_tree": "dc0c01157b5c925684bd77c57ed6703904af7453", - "src_tree": "cbf93a19df8d1ce2fb66e2aeb7276a4ea6d4dcc2", - "kld_sha256": "348cb1f5a91d47e1412747d285c795c49375a61a66cfb3d35d22f38ecfde6ea0", - "probes": [ - {"length": 4096, "returned": 4096, "elapsed_seconds": 1.7912541721016169, "hash_match": true, "target_pread_count": 1}, - {"length": 16384, "returned": 16384, "elapsed_seconds": 1.341409295797348, "hash_match": true, "target_pread_count": 1}, - {"length": 65536, "returned": 65536, "elapsed_seconds": 1.3740410972386599, "hash_match": true, "target_pread_count": 1}, - {"length": 1048576, "returned": 1048576, "elapsed_seconds": 1.737462431192398, "hash_match": true, "target_pread_count": 1} - ], - "dd_1m_seconds": 0.367417, - "sha_sample_offset": 1044480, - "sha_hash_captured": false, - "cleanup": {"umount": 0, "mdconfig_detach": 0, "kldunload": 0, "qemu_alive": false, "overlay_exists": false} - } - ] -} diff --git a/docs/refactoring-plan.md b/docs/refactoring-plan.md deleted file mode 100644 index f43dc45..0000000 --- a/docs/refactoring-plan.md +++ /dev/null @@ -1,37 +0,0 @@ -# repo22 Linux/FreeBSD 结构对齐维护计划 - -## 基线 - -- FreeBSD 实现：`src/` -- Linux 参考：`/work/dev-src-linux/fs/erofs`（Linux 7.1-rc1 快照） -- 比较方法：逐文件和逐职责比较，不依赖共同 Git 历史 -- 运行目标：FreeBSD 15 amd64 - -## 决策规则 - -1. 先判断差异是否来自 FreeBSD vnode、GEOM、errno、锁或内核库接口。 -2. 只对非必要差异调整静态函数名、变量名、定义顺序和文件职责。 -3. 不为匹配 Linux 文件表而添加空模块或不适用的抽象。 -4. `src/Makefile` 是模块名、源码清单、架构门控和编译选项的权威来源。 -5. 构建产物、生成头和机器 include symlink 不进入版本控制。 - -## 本轮清单 - -- [x] 清除受跟踪的 `build/obj` 生成头和机器 symlink -- [x] 精确忽略模块构建及手工测试生成目录 -- [x] 由原生 `bsd.kmod.mk` 取代手写 amd64 clang/link 流程 -- [x] 明确拒绝未验证的非 amd64 架构 -- [x] 将 ZSTDIO 配置统一为 `WITH_ZSTDIO=0/1` -- [x] 按 Linux 顺序整理 Makefile 源码列表 -- [x] 对齐 `find_target_dirent` 和 `decompressor_*.c` 名称 -- [x] 将跨文件解压后端声明集中到 `internal.h` -- [x] 保留 BSD 专属 `erofs_vnops.c`、`lz4.c` 和后端调用契约 -- [x] 在 FreeBSD 15 VM 完成双配置构建和 ZSTD 挂载 smoke - -## 验证门槛 - -1. `WITH_ZSTDIO=0` 与 `WITH_ZSTDIO=1` 均从空对象目录构建。 -2. 禁用产物不含 `ZSTD_*` 未解析符号，启用产物必须含预期内核 API。 -3. 启用产物完成加载、ZSTD 镜像挂载、文件读取、卸载和 KLD 清理。 -4. 非 `amd64` 和非法 `WITH_ZSTDIO` 值必须明确失败。 -5. 提交只包含 repo22 pathspec，且远端 `xdm/main` 与本地 HEAD 一致。 diff --git a/docs/refactoring-status.md b/docs/refactoring-status.md deleted file mode 100644 index 19008d9..0000000 --- a/docs/refactoring-status.md +++ /dev/null @@ -1,40 +0,0 @@ -# repo22 Linux/FreeBSD 结构对齐状态 - -## 已完成 - -- 使用 `/work/dev-src-linux/fs/erofs` 的 Linux 7.1-rc1 快照逐文件复核；未使用 - 两个实现共享历史的假设。 -- `namei.c` 的静态 helper 已从 `erofs_find_target_dirent` 恢复为 Linux 名称 - `find_target_dirent`。 -- `lzma.c`、`deflate.c`、`zstd.c` 已按 Linux 职责名调整为 - `decompressor_lzma.c`、`decompressor_deflate.c`、 - `decompressor_zstd.c`。 -- 解压后端原型已移入 `internal.h`；MicroLZMA 内嵌 XZ 的 allocator helper - 已限制为编译单元内部符号。 -- `src/Makefile` 已按 Linux 的 metadata、压缩调度/映射、算法后端顺序组织， - 并成为 `build.sh` 的唯一源码和模块名来源。 -- 受跟踪的对象目录、vnode 生成头和 amd64/x86 机器 symlink 已删除并忽略。 - -## 保留差异 - -- `erofs_vnops.c` 承载 FreeBSD vnode、pager、NFS 和只读操作语义。 -- `lz4.c` 保持独立，因为 BSD 后端不使用 Linux page/LZ4 调度接口。 -- 后端函数保持 BSD 内部 API，而不是复制 Linux decompressor descriptor 和 - page 生命周期。 -- FreeBSD 路径返回正 errno，并保留 GEOM/provider、lockmgr 和 vnode 规则。 -- Linux `sysfs`、file-backed、fscache、page-cache sharing 和 `zutil` 职责不以 - 空文件模拟。 -- 当前构建门控仅允许已验证的 FreeBSD 15 `amd64`。 - -## 2026-08-09 验证 - -- VM：FreeBSD `15.0-RELEASE-p8` amd64。 -- 构建源：FreeBSD `15.0 RELEASE-p9` `sys` 树。 -- `WITH_ZSTDIO=0` 构建通过，模块 SHA256： - `d11d0319dc1d786eaa868a0c05c2abaf43da8480acacb0e1e4b72b7542b05432`； - 无 `ZSTD_*` 未解析符号。 -- `WITH_ZSTDIO=1` 构建通过，模块 SHA256： - `82471f19812e9877ea06901d1ba0cb4378b2bc6476ec70f6e024e0dc777c0c71`； - 编译命令包含 `-DZSTDIO` 并引用 FreeBSD 内核 ZSTD API。 -- 启用模块完成加载、`zstd-level1.erofs` 只读挂载、6 个文件读取与 SHA256、 - 卸载、md detach 和 KLD unload；测试后无遗留挂载或模块。 diff --git a/src/erofs_fs.h b/erofs_fs.h similarity index 100% rename from src/erofs_fs.h rename to erofs_fs.h diff --git a/src/erofs_vnops.c b/erofs_vnops.c similarity index 100% rename from src/erofs_vnops.c rename to erofs_vnops.c diff --git a/src/inode.c b/inode.c similarity index 100% rename from src/inode.c rename to inode.c diff --git a/src/internal.h b/internal.h similarity index 100% rename from src/internal.h rename to internal.h diff --git a/issues/README.md b/issues/README.md deleted file mode 100644 index 86e9010..0000000 --- a/issues/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Issue Status - -This directory preserves both resolved investigations and approved validation -gaps. Resolved files remain historical evidence; they are not current failures. - -| Issue | Status | Current conclusion | -| --- | --- | --- | -| [TC010 48-bit statfs](TC010-48bit-statfs-large-provider.md) | RESOLVED | A qualified 16 TiB-plus sparse provider completed mount and 64-bit `statfs` validation. | -| [TC060 pathconf](TC060-pathconf-standard-values.md) | RESOLVED | FreeBSD-standard values were implemented and dynamically revalidated. | -| [TC153 large directory index](TC153-large-directory-block-index-validation.md) | RESOLVED | The exact fixed KLD returned `EINTEGRITY` on the multi-TiB sparse-provider path. | -| [`erofs_vget()` after `insmntque()` failure](erofs-vget-insmntque-failure-use-after-release.md) | FIXED / RUNTIME VALIDATION PENDING | The post-failure vnode access was removed; a dedicated unmount-race test remains outstanding. | -| [Explicit mapped extent payload](extent-metadata-fixture-unavailable.md) | SHELVED | TC146 remains PARTIAL because no independently validated positive mapped-payload fixture is available. | - -The remaining open validation work is split between a dedicated vnode teardown -race test for the fixed lifecycle issue and a shelved fixture/tooling gap for -the explicit mapped extent payload case. diff --git a/issues/TC010-48bit-statfs-large-provider.md b/issues/TC010-48bit-statfs-large-provider.md deleted file mode 100644 index 7bc9556..0000000 --- a/issues/TC010-48bit-statfs-large-provider.md +++ /dev/null @@ -1,92 +0,0 @@ -# TC010 48-bit statfs Large Provider - -Status: **RESOLVED - qualified sparse vnode provider validated** - -Resolved: 2026-08-09 - -Baseline: `cd0e985b5ac54a4b7acb7042422329ad1729fb3e` - -## Original Problem - -TC010 requires a real EROFS mount whose 48-bit superblock block count has a -nonzero high word. A nonzero `blocks_hi` declares at least `2^32` filesystem -blocks, so a 4 KiB filesystem needs a provider larger than 16 TiB. Earlier -vnode-md work used only small regular images and correctly could not claim PASS. - -This affects: - -- 48-bit superblock block-count decoding; -- primary-device media-size validation; -- FreeBSD `statfs(2)` and `df(1)` 64-bit totals; -- the union selector between `rb.blocks_hi` and `rb.rootnid_2b`. - -## Exact Trigger - -1. `EROFS_FEATURE_INCOMPAT_48BIT` is set. -2. `rootnid_8b` is nonzero, selecting `rb.blocks_hi`. -3. `blocks_hi` is nonzero. -4. Provider media size is at least `total_blocks << blkszbits`. -5. Superblock CRC32C, root inode, and normal mount checks pass. - -A short provider fails with `ENXIO`; that negative guard cannot replace the -positive statfs test. TC150's `rootnid_8b == 0` fallback and multidevice totals -also select different production paths. - -## Earlier Attempts - -1. Structured metadata patching and CRC32C recomputation succeeded. -2. The normal small vnode provider failed media-size validation as expected. -3. Static source comparison confirmed Linux's `48BIT && rootnid_8b` selector, - but static arithmetic was not counted as dynamic coverage. -4. No earlier run had attached and mounted a qualified 16 TiB provider, so the - issue remained SHELVED. - -## Resolution - -The isolated FreeBSD 15.0-RELEASE-p8 guest uses UFS2, which supports a sparse -regular file large enough for vnode md without allocating 16 TiB physically: - -```text -truncate -s 17592193667072 TC010-provider.raw -stat: size=17592193667072 allocated_blocks=15232 block_size=32768 -mdconfig -a -t vnode -f TC010-provider.raw: md0 -diskinfo mediasize: 17592193667072 bytes -``` - -The structured fixture encoded: - -```text -blocks_lo=1861 -blocks_hi=1 -rootnid_8b=36 -total_blocks=4294969157 -required_provider_length=17592193667072 -``` - -Prefix SHA256: -`d3ffadc6fc9e73f85a8a5973b4ac5ee2a2da57b4e8baeccd259fa4325a2146cf`. - -The exact prefix was copied to the sparse provider, then extended with zeros. -Mount succeeded with KLD SHA256 -`8349c97c7ced253fff32a9e706f29313f309cb63a270e4e39a4501426f2b95c6`. -`df` reported: - -```text -Filesystem Type 1024-blocks Used Avail Capacity -/dev/md0 erofs 17179876628 17179876628 0 100% -``` - -`17179876628 == 4294969157 * 4`; no 32-bit wrap occurred. `f_bfree` and -available blocks were zero, so Used correctly equaled total. The control file -matched, dmesg had no new fault, and the mount, md unit, sparse provider, and -KLD were cleaned up. - -## Related Coverage - -- TC017 separately repeated the small-provider `ENXIO` guard and qualified - sparse-provider mount, proving the nonzero high-word parse dynamically. -- TC018 separately read exact bytes from physical offset - `17592191561728`, above 16 TiB; it did not substitute a large `df` result for - high-address data I/O. - -No repo22 kernel source change was made during this resolution. diff --git a/issues/TC060-pathconf-standard-values.md b/issues/TC060-pathconf-standard-values.md deleted file mode 100644 index db59e3d..0000000 --- a/issues/TC060-pathconf-standard-values.md +++ /dev/null @@ -1,119 +0,0 @@ -# TC060 Standard pathconf Values - -Status: **RESOLVED - standard FreeBSD pathconf values validated** - -Last updated: 2026-08-09 - -Resolved: 2026-08-09 - -Priority: High - -Implementation status: Fixed by `cdba7e54fb9e9d82980a06f178e68d0bbc1663ac` -and dynamically validated - -## Problem - -The exact repo22 9ae22009f KLD returns EINVAL for two standard pathconf names on -an otherwise valid mounted EROFS directory: - -- _PC_NO_TRUNC -- _PC_CHOWN_RESTRICTED - -The same call correctly returns NAME_MAX=255, PATH_MAX=1024, -FILESIZEBITS=64, and LINK_MAX=2147483647. TC060 therefore fails at the kernel -behavior level; this is not a missing guest tool or fixture limitation. - -## Trigger and Evidence - -Environment: - -- FreeBSD 15.0-RELEASE-p8 amd64 guest on port 9222. -- Exact source commit: 9ae22009f23a65320730072a780998e80aa9b728. -- KLD SHA256: - 19ad086bd2508cbb4c57b1a51f38c93057d438b84fbcfa2e637ff2519f5bc590. -- Fixture: vfs-plain.erofs SHA256 - 79f0b5f4aa8e7ea532b711ee8fb466f14f35d0952446d41ba4bbc4d6e008b8ff. - -Direct command: - - /tmp/repo22-g3/g3_vfs_probe pathconf \ - /tmp/repo22-g3/mnt/testdir - -Observed output and process status: - - name_max=255 path_max=1024 filesizebits=64 link_max=2147483647 \ - no_trunc=error:22 chown_restricted=error:22 - g3_vfs_probe: unexpected EROFS pathconf values - probe_status=1 - -The test used a fresh dynamic md0, a read-only EROFS mount, and the exact KLD. -After capture, umount and md detach both returned zero. Final guest state was -zero EROFS mounts, zero md units, and no loaded EROFS KLD. - -## Analysis - -erofs_pathconf() handles NAME_MAX, PATH_MAX, FILESIZEBITS, LINK_MAX, and ACL -queries directly, then delegates other names to vop_stdpathconf(). In this -FreeBSD 15 configuration the delegation returns EINVAL for both names above. -The mounted filesystem is immutable and enforces 255-byte names, so reporting -NO_TRUNC=1 and CHOWN_RESTRICTED=1 is consistent with the implemented behavior -and with the TC contract. - -This failure affects applications that use pathconf(2) to discover pathname -truncation and ownership-change restrictions. It does not affect the four -values that repo22 already handles directly. - -## Original Required Resolution - -Add explicit FreeBSD vnode pathconf handling for both standard names, then -rerun TC060 with the direct syscall helper. Acceptance requires both values to -be 1, all six queries to have errno 0, unchanged mount/read behavior, no new -dmesg diagnostics, and complete mount/md/KLD cleanup. - -## Resolution - -FreeBSD 15's `vop_stdpathconf()` intentionally provides only generic values -such as `_PC_ASYNC_IO`, `_PC_PATH_MAX`, and zero-valued optional features; its -default case returns `EINVAL`. UFS, tmpfs, and ext2fs therefore implement -`_PC_CHOWN_RESTRICTED` and `_PC_NO_TRUNC` in their filesystem pathconf methods -before delegating all remaining names to `vop_stdpathconf()`. - -EROFS now follows that FreeBSD pattern: the two names return 1, while the -existing NAME_MAX, PATH_MAX, FILESIZEBITS, LINK_MAX, and ACL cases remain -unchanged and the default branch still calls `vop_stdpathconf()`. This matches -the implemented filesystem behavior: uid/gid mutation is rejected with EROFS, -and lookup of a component longer than `EROFS_NAME_LEN` returns ENAMETOOLONG. -No lock, allocation, reference, or cleanup path was added. - -Linux EROFS was used only as a maintenance comparison for the 255-byte name -limit and ENAMETOOLONG behavior. FreeBSD 15 VOP and syscall semantics were the -authority for the returned values and errno behavior. - -## Qualified Retest - -The exact source archive for the fix commit built natively on FreeBSD -15.0-RELEASE-p8 with kernel `-Werror` in both configurations: - -- `WITH_ZSTDIO=0`: KLD SHA256 - `68536e03ce93c6c04aab9cf29dab81ee5802d4b94401bd1a4bd752de40c0e504`. -- `WITH_ZSTDIO=1`: KLD SHA256 - `598f171d355af78c64407a6d513d20f053530620da92df7f8ccfdf9a804e463d`. - -The dependency-minimal `WITH_ZSTDIO=0` KLD and the original TC060 helper -reported: - -```text -name_max=255 path_max=1024 filesizebits=64 link_max=2147483647 -no_trunc=1 chown_restricted=1 -``` - -All six queries had errno 0. Additional direct checks returned ASYNC_IO=200112, -ACL_EXTENDED=1, ACL_PATH_MAX=254, and ACL_NFS4=0 with errno 0. An unknown name -returned `-1/EINVAL(22)`, and a 256-byte component returned -`ENAMETOOLONG(63)`. The mounted file hash and read-only EROFS behavior were -unchanged, dmesg was byte-identical before and after, and final mount/md/KLD -and guest artifact counts were zero. - -Complete build, value, smoke, discarded-attempt, review, and cleanup evidence -is in -`tests/results/manual/2026-08-09T1120Z-tc060-fix/manual-test-report.md`. diff --git a/issues/TC153-large-directory-block-index-validation.md b/issues/TC153-large-directory-block-index-validation.md deleted file mode 100644 index 8e6b3a3..0000000 --- a/issues/TC153-large-directory-block-index-validation.md +++ /dev/null @@ -1,193 +0,0 @@ -# TC153 Large Directory Block Index Validation - -Status: **RESOLVED - exact-source kernel validation passed** - -Last updated: 2026-08-09 - -Priority: Critical validation gap - -Implementation status: Source fix present, build-verified, and dynamically -validated - -## Problem - -`erofs_find_target_block()` used signed `int` values for the binary-search -front, back, and midpoint indexes. A directory whose logical size describes -`2147483649` 4 KiB blocks produces a final block index of `2147483648`. -Converting that value to `int` can make the initial back bound negative. The -search can then return `ENOENT` without reading any directory block, and the -FreeBSD namecache can retain a false negative result for corrupt media. - -The review change uses `uint64_t` for block-search bounds and midpoints, checks -the midpoint multiplication, and handles the `mid == 0` lower-bound case. The -within-block search remains `uint32_t` because a validated EROFS directory -block can contain only a block-sized number of entries. - -## Affected Features - -- Cold pathname lookup in very large or maliciously sized directories. -- Corruption reporting and `EINTEGRITY` propagation from directory reads. -- Negative namecache correctness after a failed lookup. -- Linux/FreeBSD behavioral review of EROFS's two-level directory search. - -Normal directories are not expected to approach this bound. The practical -risk is a crafted image causing an incorrect `ENOENT`, not an ordinary mkfs -image losing entries. - -## Trigger Conditions - -All of the following are needed to exercise the original width error: - -1. A directory inode is accepted with a logical block count greater than - `INT_MAX`. -2. The directory layout reaches `erofs_find_target_block()` rather than being - rejected by an earlier inline-data bounds check. -3. A cold lookup is issued for a name that is not satisfied by an already - cached vnode or negative namecache entry. -4. The backing image is too short for the computed midpoint read, so the fixed - code should return `EINTEGRITY` instead of a synthetic miss. - -For 4 KiB blocks, TC153 encodes a size of `8796093026304` bytes, -`2147483649` blocks, and a final index of `2147483648`. - -## Current Analysis - -The source change is mechanically narrow and has passed both repo22 build -configurations. The checked multiplication protects a future block-size or -index change even though the current `OFF_MAX` inode limit already bounds the -product. The `mid == 0` guard prevents unsigned subtraction from wrapping when -the search moves below the first block. - -Dynamic proof must show more than a large `st_size`. It must prove that the -lookup enters the block binary search. An image rejected in -`erofs_read_inode()` is not evidence for this fix, and an `ENOENT` observed -after a warm negative cache is ambiguous. - -## Attempts and Results - -### Attempt 1: Small FLAT_INLINE image - -- Artifact directory: - `/work/build/repo22-review-artifact.LtqNxH` -- Base image: `large-dir-base.erofs`, 4096 bytes, SHA256 - `973d2a1c90ac1bf776ed76a1fc6a7e88b2156fac3bf04ddbae19cfafebbd562e` -- Patched image: `large-dir-intmax.erofs`, 4096 bytes, SHA256 - `ce5cad3eff34ea50ca89eedf93d8ef90e6cea96858a138685fae45f39084de1f` -- Recorded inode: NID 40 at byte 1280. - -The directory was Layout 2 (`FLAT_INLINE`). Patching only `i_size` made its -declared inline tail range invalid, so inode validation rejected it before -`erofs_find_target_block()`. This attempt is invalid as TC153 dynamic evidence -and must never be marked PASS. - -### Attempt 2: More entries to force block data - -- Artifact directory: - `/work/build/repo22-review-artifact2.FmMTup` -- Generated base: `large-dir-base.erofs`, 65536 bytes, SHA256 - `50eb28351788ab1801408aca0a6fca6352755f5ca56efe6e2bb3e2fe08305663` -- Directory: NID 40, 21052 bytes, still Layout 2. - -The generation process was interrupted before producing a patched image or a -FreeBSD result. A later deterministic rerun reproduced the same base hash and -showed why the approach failed: erofs-utils 1.8.6 tail-packs directories even -when they span multiple data blocks. Its `noinline_data` option does not turn -directory tail packing off. - -### Attempt 3: Follow-up agents - -Multiple follow-up agents were assigned the fixture and guest validation. -They either stopped while preparing the second image or returned no executed -commands, files, hashes, or guest output. These attempts added no evidence and -are recorded to prevent their elapsed time from being mistaken for progress. - -### Attempt 4: Sparse-provider design - -A guest-side sparse vnode provider was considered so a valid directory could -declare the complete multi-terabyte range. It was not executed. Such a scheme -must prove the GEOM media size, avoid materializing terabytes, preserve valid -initial directory blocks, and still force a cold read at the binary-search -midpoint. No result exists for this approach. - -### Attempt 5: Reproducible FLAT_PLAIN conversion - -The tracked generator now copies all 21052 bytes of the generated directory -to appended blocks, changes only that inode to Layout 0 (`FLAT_PLAIN`), updates -the image block count and CRC32C, and then applies the large logical size. - -Host reproduction on 2026-08-09 produced: - -- `large-dir-base.erofs`: 65536 bytes, SHA256 - `50eb28351788ab1801408aca0a6fca6352755f5ca56efe6e2bb3e2fe08305663` -- `large-dir-intmax.erofs`: 90112 bytes, SHA256 - `0f90d3d57adbbbd946e41b225c1f6c464915c6abb0b13478ec9b2a318def1f72` -- Patched inode: NID 40 at byte 1280, Layout 0, raw block 16. -- Declared image blocks: 22. - -This resolved the fixture-construction problem. The qualified kernel run is -recorded below. - -### Attempt 6: Qualified sparse GEOM provider - -The G3 takeover generated a checksum-valid sparse prefix from Attempt 5 and -performed one targeted FreeBSD 15 run against the exact source requested by -the assignment. - -- Source commit: - 9ae22009f23a65320730072a780998e80aa9b728. -- KLD SHA256: - 19ad086bd2508cbb4c57b1a51f38c93057d438b84fbcfa2e637ff2519f5bc590. -- Sparse-prefix SHA256: - f9337f83b1f568f6d904331a7749e6362a691055cd5af95b59bc89772f04e3b0. -- Prefix qualification: NID 40, inode offset 1280, extended Layout 0, - start block 16, 2147483649 directory blocks, final block index 2147483648, - valid superblock checksum. -- Sparse provider: 8796093091840 logical bytes, 448 allocated 512-byte - sectors, and the first 90112 bytes retained the prefix SHA256. -- GEOM diskinfo size: 8796093091840 bytes. -- Mounted /huge: size 8796093026304, NID 40. - -The first two cold stat operations for /huge/missing each returned EINTEGRITY -(97). Root readdir then returned three complete entries and validated all -kernel/libc restart cookies. A third lookup after that readdir again returned -EINTEGRITY. No lookup returned ENOENT, so no false negative namecache entry -masked the corruption. - -Unmount, md detach, sparse-provider removal, and KLD unload all succeeded. -The only new dmesg lines in the complete G3 run were expected SIGBUS child -exits from the unrelated assigned mmap tests; TC153 added no diagnostic. - -## Historical Remaining Validation - -Attempt 6 completed steps 1-5 and 7 below. The assignment mandated the exact -fixed source, so the optional pre-fix comparison in step 6 was not run. - -1. Regenerate the images with - `tests/results/manual/2026-08-09T0124Z-final-review/prepare-fixtures.sh` and - verify the hashes and Layout 0 assertion. -2. Load the exact module under test in a clean FreeBSD 15 guest. -3. Mount the 90112-byte image and confirm `/huge` reports - `8796093026304` bytes. -4. With a cold parent cache, run two lookups of `/huge/missing` and capture - direct command exit codes plus `truss` errno. -5. Require `EINTEGRITY` on every fixed-module lookup, including after root - readdir. Confirm no negative cache changes the result. -6. Repeat with the pre-fix module to demonstrate the erroneous `ENOENT` path. -7. Record dmesg, mount/md/KLD cleanup, module and fixture hashes in a dated - manual report. - -## Original Acceptance Criteria - -TC153 can move from SHELVED to PASS only after the fixed FreeBSD KLD returns -`EINTEGRITY` repeatedly from the valid Layout 0 fixture, the pre-fix behavior -is distinguished, no trap or panic occurs, and complete cleanup evidence is -recorded. Host fixture generation and static source review alone are -insufficient. - -## Resolution - -TC153 is PASS because the exact fixed FreeBSD KLD returned EINTEGRITY -repeatedly from the valid Layout 0 fixture before and after root readdir, no -trap or panic occurred, and complete hash/provider/cleanup evidence is recorded -in tests/results/manual/2026-08-09T1059Z-g3/manual-test-report.md. Host fixture -generation and static source review alone remain insufficient. diff --git a/issues/erofs-vget-insmntque-failure-use-after-release.md b/issues/erofs-vget-insmntque-failure-use-after-release.md deleted file mode 100644 index fac0144..0000000 --- a/issues/erofs-vget-insmntque-failure-use-after-release.md +++ /dev/null @@ -1,176 +0,0 @@ -# `erofs_vget()` lifecycle violation after `insmntque()` failure - -## Status - -- State: **FIXED / RUNTIME VALIDATION PENDING** -- Priority: **P1** -- Scope: pre-existing vnode lifecycle issue; not introduced by Pre10 Batch B -- Affected feature: inode-to-vnode construction through `erofs_vget()` - -## Problem description - -`repo-pre-10/src/inode.c:431` defines `erofs_vget()`. Before this fix, after -allocating a new vnode and attaching an `erofs_node`, the function called -`insmntque()` and used this failure branch: - -```c -vp->v_data = en; -en->vnode = vp; -en->nid = nid; -lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL); -error = insmntque(vp, mp); -if (error != 0) { - free(en, M_EROFS); - vp->v_data = NULL; - *vpp = NULL; - return (error); -} -``` - -The relevant call and failure branch are at -`repo-pre-10/src/inode.c:463-468`. On an insertion failure, the FreeBSD -`insmntque()` API reclaims and releases the vnode before returning. The -subsequent assignment to `vp->v_data` therefore accesses a vnode whose lifetime -has ended from the caller's perspective. - -## Trigger condition - -The locally reviewed FreeBSD implementation rejects insertion while the target -mount is being torn down. In -`/work/dev-freebsd-releng/sys/kern/vfs_subr.c:2303-2306`, the failure condition -checks `MNTK_UNMOUNT` and either forced unmount (`MNTK_UNMOUNTF`) or an empty -mount vnode list, unless `VV_FORCEINSMQ` applies. The function then returns -`EBUSY` at `vfs_subr.c:2315`. - -The issue can therefore be triggered when vnode creation in `erofs_vget()` -races with mount teardown or forced unmount and `insmntque()` rejects the new -vnode. - -## FreeBSD contract - -The local FreeBSD kernel source documents the ownership distinction explicitly -at `/work/dev-freebsd-releng/sys/kern/vfs_subr.c:2328-2331`: - -- `insmntque()` reclaims the vnode when insertion fails. -- `insmntque1()` leaves vnode cleanup to the caller. - -`insmntque()` calls `insmntque1_int(vp, mp, true)` at -`vfs_subr.c:2333-2337`. In the relevant failure path, -`insmntque1_int()` performs all of the following at -`vfs_subr.c:2309-2313`: - -```c -vp->v_data = NULL; -vp->v_op = &dead_vnodeops; -vgone(vp); -vput(vp); -``` - -Consequently, an error return from `insmntque()` does not preserve caller -ownership of a live `vp`. Reading or writing `vp`, including assigning -`vp->v_data`, is unsafe after that return. - -## Impact - -The affected path is vnode construction for filesystem objects. Under the -mount-teardown race described above, the pre-fix failure branch may write -through a released vnode pointer. Likely consequences include a use-after-free, -memory corruption, a kernel panic, or corruption of an unrelated vnode if the -storage is recycled quickly. The exact manifestation has not been reproduced -and must not be treated as confirmed beyond the statically established lifetime -violation. - -The separately allocated `en` object is not released by the kernel failure -path. That path clears `vp->v_data` and changes `v_op` to `dead_vnodeops` before -calling `vgone()`. The dead vnode reclaim operation is `VOP_NULL`, so the -filesystem reclaim callback does not run and cannot release `en`. Caller-side -`free(en, M_EROFS)` is therefore required exactly once. - -## Implemented fix - -Pre10 now treats an error return from `insmntque()` as the end of caller -ownership of `vp`: - -```c -error = insmntque(vp, mp); -if (error != 0) { - free(en, M_EROFS); - *vpp = NULL; - return (error); -} -``` - -The caller frees only the independently allocated `en` object and does not -read, write, reclaim, unlock, or release `vp` after `insmntque()` returns an -error. The kernel failure path clears `vp->v_data` and installs -`dead_vnodeops` before invoking `vgone()` and `vput()`. Dead vnode reclaim is a -no-op, so no vnode cleanup callback owns `en`. This gives `en` exactly one -release on the failure path. - -The successful insertion path, vnode lock state, hash insertion race handling, -errno return, and later inode-read cleanup are unchanged. In particular, -`vfs_hash_insert()` remains responsible for reclaiming and releasing the losing -new vnode when another thread wins the hash race. - -## Current analysis and progress - -- Independently reviewed the Pre10 `erofs_vget()` failure path. -- Confirmed the local FreeBSD `insmntque()` and `insmntque1()` ownership - contract against kernel source. -- Confirmed the teardown/forced-unmount condition that produces `EBUSY`. -- Confirmed that `repo-pre-10/src/inode.c` accesses `vp->v_data` after the - reclaiming API returns an error. -- Compared the cleanup pattern with the local FreeBSD ext2fs, msdosfs, UDF, - pseudofs, and p9fs vnode construction paths. They likewise avoid touching the - vnode after `insmntque()` fails and separately dispose caller-owned node - state where required. -- Implemented the minimal Pre10 source fix by removing the post-failure - `vp->v_data` assignment. -- Completed static ownership, lock, hash-race, reclaim, and error-path review. -- No QEMU run, runtime reproduction, dedicated race test, or feature test has - been attempted. - -The statically proven lifetime violation is fixed. Runtime validation remains -open because the triggering teardown race is not covered by ordinary smoke -testing. - -## Candidate fix boundaries - -The implemented correction preserves the existing choice of `insmntque()` and -treats an error return as transferring vnode cleanup entirely to the kernel. -The caller releases only independently owned `en`, clears `*vpp`, and returns -without any further access to or release of `vp`. - -An alternative would be to deliberately switch to `insmntque1()` and implement -the complete caller-owned failure cleanup required by that API. This is a wider -lifecycle change and should not be selected merely to mirror naming or control -flow. - -The wider `insmntque1()` alternative was rejected because it would require new -caller-owned vnode cleanup without providing any benefit for this path. - -## Required validation - -Before marking runtime validation complete: - -1. Build the EROFS module with the supported FreeBSD source tree. -2. Run a dedicated mount/unmount race test that repeatedly creates or looks up - previously uncached vnodes while normal and forced unmount are attempted. -3. Use kernel diagnostics appropriate for detecting stale vnode access, memory - corruption, lock misuse, and double release. -4. Re-run the ordinary mount, lookup, read, and unmount smoke coverage after the - race test passes. - -The dedicated unmount race test is required because ordinary static checks and -single-threaded smoke tests do not exercise the failing `insmntque()` branch. - -## Relationship to Pre10 Batch B - -This issue predates Batch B. Batch B extracted `erofs_fill_vnode()` only for the -successful post-`erofs_read_inode()` field setup and did not modify the -`insmntque()` call or its failure branch. Independent review found no Batch B -regression in this path. - -The minimal ownership fix is included in Pre10. The dedicated teardown race -test remains deferred and cannot be replaced by the planned ordinary smoke -test. diff --git a/issues/extent-metadata-fixture-unavailable.md b/issues/extent-metadata-fixture-unavailable.md deleted file mode 100644 index 1442fc7..0000000 --- a/issues/extent-metadata-fixture-unavailable.md +++ /dev/null @@ -1,169 +0,0 @@ -# Explicit Extent Metadata Fixture Unavailable - -Status: **SHELVED - positive mapped payload unavailable** - -Last updated: 2026-08-09, final aggregation - -Latest reviewed baseline: `fcc85b93d5f8fd9671686bd68bf3b086c8bd25cf` - -Priority: High validation gap - -Implementation status: ABI and control flow implemented; positive mapped -kernel read pending - -## Problem - -repo22 implements the newer compressed-extent metadata path selected by -`Z_EROFS_ADVISE_EXTENTS`. Dynamic positive validation requires an image with a -valid extent table that maps a real payload. erofs-utils 1.8.6 cannot emit or -validate this on-disk format, and the repository's structured helpers cannot -currently relocate a legacy compressed inode into a validated positive extent -table. - -HEAD2, interlaced pclusters, legacy full/compact indexes, partial references, -fragments, and all four decoders have separate dynamic coverage. None of those -substitutes for entering `z_erofs_map_blocks_ext()` with a mapped payload. - -## Required Trigger - -1. The inode uses compressed-full layout. -2. The map header sets `Z_EROFS_ADVISE_EXTENTS`. -3. The selected 4-, 8-, 16-, or 32-byte records form a complete valid table. -4. Physical fields identify real payload bytes for the declared algorithm. -5. Logical starts, physical lengths, partial flags, and final-record semantics - agree with the source file. -6. A FreeBSD read reaches the explicit mapper and returns source-identical - bytes. - -Zero-count tables, overflowing physical bases, holes, and ordinary full-index -images are negative or adjacent coverage, not this trigger. - -## G5 Independent Attempts - -All inputs below were generated in new G5 output directories. No old image or -report was consumed as a fixture. - -### 1. erofs-utils 1.8.6 capability scan - -Observed tool output: - -```text -mkfs.erofs (erofs-utils) 1.8.6 -available compressors: lz4, lz4hc, lzma, deflate, libdeflate, zstd -``` - -The helper searched the installed 1.8.6 `include/` and `lib/` trees for these -on-disk symbols: - -```text -Z_EROFS_ADVISE_EXTENTS: 0 matches -z_erofs_extent_recsize: 0 matches -struct z_erofs_extent {: 0 matches -``` - -The mkfs help has no explicit-record option. `--max-extent-bytes` only limits -decompressed extent size. - -Result: **generator unavailable**. - -### 2. `--max-extent-bytes` generation attempt - -`g5_fixtures.py` generated a fresh LZ4 full-index image with: - -```text --zlz4 -C65536 -Elegacy-compress --max-extent-bytes=65536 -``` - -Structured reopen and `dump.erofs` produced: - -```text -image = images/extent-attempt.erofs -image SHA256 = ee7472c23ccffd5eef6f4a3171ea53ae2e840f8a1ea417b2630065175ecf3225 -source SHA256 = 5be510e6b43f1ed0cda4261288ea70df11607b6ced29bc90d32ed2849ecc5e35 -inode layout = 1 (compressed full) -map header offset = 1312 -h_advise = 2 -HEAD1 algorithm = 0 (LZ4) -explicit bit selected = no -``` - -`h_advise=2` is ordinary HEAD1 big-pcluster metadata. Naming or sizing the -image as an extent attempt does not enter the explicit mapper. - -Result: **not an explicit fixture**. - -### 3. Structured conversion attempt - -The G5 transformer parsed the source inode, map header, full lcluster indexes, -physical extents, and payload bounds. It refused to emit a purported positive -fixture because conversion requires relocating variable-size extent records, -subsequent inode metadata, and possibly payload blocks. erofs-utils 1.8.6 -cannot reopen and validate the resulting ABI, so an ad hoc rewrite would not -provide trustworthy evidence. - -The existing `tests/review_fixtures.py` helper was also reviewed. Its -`extent-pa-wrap.erofs` conversion deliberately writes two 4-byte records and a -physical base whose `pa + plen` overflows. That is a negative overflow trigger -for TC155. It does not describe a valid mapped payload and cannot close this -issue. - -Result: **structured positive conversion unavailable**. - -### 4. ABI and control-flow comparison - -The G5 review compared repo22 `src/erofs_fs.h`/`src/zmap.c` with the Linux EROFS -definitions and mapper. Both sides define: - -- record sizes 4, 8, 16, and 32 bytes; -- implicit 64-bit physical bases for 4-byte records; -- per-record low physical starts for 8-byte records; -- explicit counts and binary search for 16/32-byte records; -- high physical and logical words where the record size carries them; -- partial-reference, shifted/interlaced format, and final fragment handling; -- malformed-count and arithmetic bounds checks. - -This reduces implementation risk but is static evidence only. - -## Impact - -- TC146 HEAD2: dynamic PASS. -- TC146 interlaced: dynamic PASS. -- TC146 explicit mapped payload: SHELVED. -- TC146 overall: **PARTIAL**, never overall PASS while this issue remains. -- Record-size variants, positive binary-search transitions, mapped high words, - and explicit-record partial references remain dynamically unvalidated. - -No repo22 kernel source was changed because this is a fixture/tooling gap, not -an observed kernel failure. - -## Progress - -- [x] Reproduce tool limitation against exactly erofs-utils 1.8.6. -- [x] Record tool output and zero symbol matches. -- [x] Generate and inspect a fresh `--max-extent-bytes` attempt. -- [x] Record source/image hashes and map-header fields. -- [x] Audit the existing structured negative helper. -- [x] Compare FreeBSD and Linux record layouts/control flow. -- [x] Reject globally unordered 16-byte and 32-byte explicit tables through - TC157 on FreeBSD 15 without reading the referenced payload. -- [ ] Obtain a producer or validator for positive mapped extent records. -- [ ] Generate at least one source-identical mapped payload. -- [ ] Cover 4/8/16/32-byte positive records where applicable. -- [ ] Exercise binary-search transitions, partial refs, high words, and final - fragments dynamically. - -## Acceptance Criteria - -This issue can close only after a reproducible helper emits a valid mapped -compressed payload, records its structural fields and hashes, validates it -with an independent format-aware implementation, and FreeBSD full/boundary -reads match the source. Negative overflow/hole fixtures and static review do -not satisfy acceptance. - -## Final Residual Risk - -TC157 closes the known binary-search ordering bug for descending, duplicate, -and cross-branch 16-byte and 32-byte tables. It does not supply the missing -positive mapped payload, dynamically cover every record-size variant, or -benchmark extremely large extent counts. Those limits do not change the issue -status or TC146's PARTIAL result. diff --git a/src/namei.c b/namei.c similarity index 100% rename from src/namei.c rename to namei.c diff --git a/src/.gitignore b/src/.gitignore deleted file mode 100644 index 7a60a6c..0000000 --- a/src/.gitignore +++ /dev/null @@ -1,18 +0,0 @@ - -i386 -machine -x86 -.cache - -export_syms - -*.o -*.ko - -opt_global.h - -vnode_if.h -vnode_if_newproto.h -vnode_if_typedef.h - -compile_commands.json diff --git a/src/super.c b/super.c similarity index 100% rename from src/super.c rename to super.c diff --git a/test_all_decompress.sh b/test_all_decompress.sh deleted file mode 100755 index 7762a95..0000000 --- a/test_all_decompress.sh +++ /dev/null @@ -1,462 +0,0 @@ -#!/bin/bash -# Comprehensive decompression unit tests for repo19 - -set -e - -TESTDIR="/tmp/repo19_decompress_tests" -SRCDIR="/work/repo-community/repo19/src" -RESULTS_FILE="/tmp/test_results.txt" - -mkdir -p "$TESTDIR" -cd "$TESTDIR" - -echo "===================================================================" -echo " REPO19 DECOMPRESSION COMPREHENSIVE UNIT TESTS" -echo "===================================================================" -echo "" - -# Test counters -TOTAL=0 -PASSED=0 -FAILED=0 - -# Track results -> "$RESULTS_FILE" - -test_case() { - local name="$1" - local result="$2" - local error="$3" - - TOTAL=$((TOTAL + 1)) - if [ "$result" = "PASS" ]; then - PASSED=$((PASSED + 1)) - echo "✓ $name" >> "$RESULTS_FILE" - printf "%-60s [PASS]\n" "$name" - else - FAILED=$((FAILED + 1)) - echo "✗ $name: $error" >> "$RESULTS_FILE" - printf "%-60s [FAIL] %s\n" "$name" "$error" - fi -} - -# Generate test data files -generate_test_data() { - echo "Generating test data..." - - # Small file (512B) - dd if=/dev/zero of=small_512b.dat bs=512 count=1 2>/dev/null - - # Medium file (4KB) - dd if=/dev/urandom of=medium_4k.dat bs=4096 count=1 2>/dev/null - - # Medium-large (32KB) - dd if=/dev/urandom of=medium_32k.dat bs=32768 count=1 2>/dev/null - - # Large file (64KB) - dd if=/dev/urandom of=large_64k.dat bs=65536 count=1 2>/dev/null - - # 1MB file - dd if=/dev/urandom of=large_1m.dat bs=1048576 count=1 2>/dev/null - - # Highly compressible (zeros) - dd if=/dev/zero of=compressible_8k.dat bs=8192 count=1 2>/dev/null - - # Text data - for i in {1..1000}; do echo "The quick brown fox jumps over the lazy dog."; done > text_data.txt - - # Random incompressible - dd if=/dev/urandom of=random_16k.dat bs=16384 count=1 2>/dev/null - - echo "Test data generated." - echo "" -} - -# LZ4 Tests -test_lz4() { - echo "--- LZ4 TESTS ---" - - # Test 1: Small file - if lz4 -c small_512b.dat > small_512b.lz4 2>/dev/null; then - if lz4 -d -c small_512b.lz4 > small_512b.out 2>/dev/null && cmp -s small_512b.dat small_512b.out; then - test_case "LZ4: small file (512B)" "PASS" "" - else - test_case "LZ4: small file (512B)" "FAIL" "decompression mismatch" - fi - else - test_case "LZ4: small file (512B)" "FAIL" "compression failed" - fi - - # Test 2: Medium file (4KB) - if lz4 -c medium_4k.dat > medium_4k.lz4 2>/dev/null; then - if lz4 -d -c medium_4k.lz4 > medium_4k.out 2>/dev/null && cmp -s medium_4k.dat medium_4k.out; then - test_case "LZ4: medium file (4KB)" "PASS" "" - else - test_case "LZ4: medium file (4KB)" "FAIL" "decompression mismatch" - fi - else - test_case "LZ4: medium file (4KB)" "FAIL" "compression failed" - fi - - # Test 3: Large file (64KB) - if lz4 -c large_64k.dat > large_64k.lz4 2>/dev/null; then - if lz4 -d -c large_64k.lz4 > large_64k.out 2>/dev/null && cmp -s large_64k.dat large_64k.out; then - test_case "LZ4: large file (64KB)" "PASS" "" - else - test_case "LZ4: large file (64KB)" "FAIL" "decompression mismatch" - fi - else - test_case "LZ4: large file (64KB)" "FAIL" "compression failed" - fi - - # Test 4: 1MB performance - if lz4 -c large_1m.dat > large_1m.lz4 2>/dev/null; then - START=$(date +%s%N) - if lz4 -d -c large_1m.lz4 > large_1m.out 2>/dev/null && cmp -s large_1m.dat large_1m.out; then - END=$(date +%s%N) - DURATION=$(( (END - START) / 1000000 )) - test_case "LZ4: 1MB file performance" "PASS" "Time: ${DURATION}ms" - else - test_case "LZ4: 1MB file performance" "FAIL" "decompression mismatch" - fi - else - test_case "LZ4: 1MB file performance" "FAIL" "compression failed" - fi - - # Test 5: Highly compressible - if lz4 -9 -c compressible_8k.dat > compressible_8k.lz4 2>/dev/null; then - if lz4 -d -c compressible_8k.lz4 > compressible_8k.out 2>/dev/null && cmp -s compressible_8k.dat compressible_8k.out; then - test_case "LZ4: highly compressible (zeros)" "PASS" "" - else - test_case "LZ4: highly compressible (zeros)" "FAIL" "decompression mismatch" - fi - else - test_case "LZ4: highly compressible (zeros)" "FAIL" "compression failed" - fi - - # Test 6: Random incompressible - if lz4 -c random_16k.dat > random_16k.lz4 2>/dev/null; then - if lz4 -d -c random_16k.lz4 > random_16k.out 2>/dev/null && cmp -s random_16k.dat random_16k.out; then - test_case "LZ4: random incompressible data" "PASS" "" - else - test_case "LZ4: random incompressible data" "FAIL" "decompression mismatch" - fi - else - test_case "LZ4: random incompressible data" "FAIL" "compression failed" - fi - - # Test 7: Corrupted data - dd if=/dev/urandom of=corrupted.lz4 bs=100 count=1 2>/dev/null - if lz4 -d -c corrupted.lz4 > /dev/null 2>&1; then - test_case "LZ4: corrupted data rejection" "FAIL" "should reject corrupted" - else - test_case "LZ4: corrupted data rejection" "PASS" "" - fi - - # Test 8: Truncated file - lz4 -c medium_4k.dat > truncated.lz4 2>/dev/null - dd if=truncated.lz4 of=truncated_short.lz4 bs=50 count=1 2>/dev/null - if lz4 -d -c truncated_short.lz4 > /dev/null 2>&1; then - test_case "LZ4: truncated data rejection" "FAIL" "should reject truncated" - else - test_case "LZ4: truncated data rejection" "PASS" "" - fi - - echo "" -} - -# DEFLATE Tests -test_deflate() { - echo "--- DEFLATE TESTS ---" - - # Test 1: Level 1 (fast) - if gzip -1 -c small_512b.dat > small_512b.gz 2>/dev/null; then - if gzip -d -c small_512b.gz > small_512b_gz.out 2>/dev/null && cmp -s small_512b.dat small_512b_gz.out; then - test_case "DEFLATE: level 1 compression" "PASS" "" - else - test_case "DEFLATE: level 1 compression" "FAIL" "decompression mismatch" - fi - else - test_case "DEFLATE: level 1 compression" "FAIL" "compression failed" - fi - - # Test 2: Level 6 (default) - if gzip -6 -c medium_32k.dat > medium_32k.gz 2>/dev/null; then - if gzip -d -c medium_32k.gz > medium_32k_gz.out 2>/dev/null && cmp -s medium_32k.dat medium_32k_gz.out; then - test_case "DEFLATE: level 6 compression" "PASS" "" - else - test_case "DEFLATE: level 6 compression" "FAIL" "decompression mismatch" - fi - else - test_case "DEFLATE: level 6 compression" "FAIL" "compression failed" - fi - - # Test 3: Level 9 (maximum) - if gzip -9 -c large_64k.dat > large_64k.gz 2>/dev/null; then - if gzip -d -c large_64k.gz > large_64k_gz.out 2>/dev/null && cmp -s large_64k.dat large_64k_gz.out; then - test_case "DEFLATE: level 9 compression" "PASS" "" - else - test_case "DEFLATE: level 9 compression" "FAIL" "decompression mismatch" - fi - else - test_case "DEFLATE: level 9 compression" "FAIL" "compression failed" - fi - - # Test 4: Invalid header - dd if=/dev/urandom of=invalid.gz bs=100 count=1 2>/dev/null - if gzip -d -c invalid.gz > /dev/null 2>&1; then - test_case "DEFLATE: invalid header rejection" "FAIL" "should reject invalid" - else - test_case "DEFLATE: invalid header rejection" "PASS" "" - fi - - # Test 5: Truncated stream - gzip -c medium_4k.dat > truncated.gz 2>/dev/null - dd if=truncated.gz of=truncated_short.gz bs=100 count=1 2>/dev/null - if gzip -d -c truncated_short.gz > /dev/null 2>&1; then - test_case "DEFLATE: truncated stream rejection" "FAIL" "should reject truncated" - else - test_case "DEFLATE: truncated stream rejection" "PASS" "" - fi - - # Test 6: Empty file - touch empty.dat - if gzip -c empty.dat > empty.gz 2>/dev/null; then - if gzip -d -c empty.gz > empty_gz.out 2>/dev/null; then - test_case "DEFLATE: empty file" "PASS" "" - else - test_case "DEFLATE: empty file" "FAIL" "decompression failed" - fi - else - test_case "DEFLATE: empty file" "FAIL" "compression failed" - fi - - echo "" -} - -# ZSTD Tests -test_zstd() { - echo "--- ZSTD TESTS ---" - - # Test 1: Level 1 (fast) - if zstd -1 -q -c small_512b.dat > small_512b.zst 2>/dev/null; then - if zstd -d -q -c small_512b.zst > small_512b_zst.out 2>/dev/null && cmp -s small_512b.dat small_512b_zst.out; then - test_case "ZSTD: level 1 compression" "PASS" "" - else - test_case "ZSTD: level 1 compression" "FAIL" "decompression mismatch" - fi - else - test_case "ZSTD: level 1 compression" "FAIL" "compression failed" - fi - - # Test 2: Level 15 (medium) - if zstd -15 -q -c medium_32k.dat > medium_32k.zst 2>/dev/null; then - if zstd -d -q -c medium_32k.zst > medium_32k_zst.out 2>/dev/null && cmp -s medium_32k.dat medium_32k_zst.out; then - test_case "ZSTD: level 15 compression" "PASS" "" - else - test_case "ZSTD: level 15 compression" "FAIL" "decompression mismatch" - fi - else - test_case "ZSTD: level 15 compression" "FAIL" "compression failed" - fi - - # Test 3: Level 22 (maximum) - if zstd -22 -q -c large_64k.dat > large_64k.zst 2>/dev/null; then - if zstd -d -q -c large_64k.zst > large_64k_zst.out 2>/dev/null && cmp -s large_64k.dat large_64k_zst.out; then - test_case "ZSTD: level 22 compression" "PASS" "" - else - test_case "ZSTD: level 22 compression" "FAIL" "decompression mismatch" - fi - else - test_case "ZSTD: level 22 compression" "FAIL" "compression failed" - fi - - # Test 4: Large file - if zstd -q -c large_1m.dat > large_1m.zst 2>/dev/null; then - if zstd -d -q -c large_1m.zst > large_1m_zst.out 2>/dev/null && cmp -s large_1m.dat large_1m_zst.out; then - test_case "ZSTD: 1MB file" "PASS" "" - else - test_case "ZSTD: 1MB file" "FAIL" "decompression mismatch" - fi - else - test_case "ZSTD: 1MB file" "FAIL" "compression failed" - fi - - # Test 5: Invalid magic number - dd if=/dev/urandom of=invalid.zst bs=100 count=1 2>/dev/null - if zstd -d -q -c invalid.zst > /dev/null 2>&1; then - test_case "ZSTD: invalid magic rejection" "FAIL" "should reject invalid" - else - test_case "ZSTD: invalid magic rejection" "PASS" "" - fi - - # Test 6: Truncated frame - zstd -q -c medium_4k.dat > truncated.zst 2>/dev/null - dd if=truncated.zst of=truncated_short.zst bs=50 count=1 2>/dev/null - if zstd -d -q -c truncated_short.zst > /dev/null 2>&1; then - test_case "ZSTD: truncated frame rejection" "FAIL" "should reject truncated" - else - test_case "ZSTD: truncated frame rejection" "PASS" "" - fi - - # Test 7: Corrupted data - zstd -q -c medium_4k.dat > original.zst 2>/dev/null - dd if=original.zst of=corrupted.zst bs=1 count=200 2>/dev/null - dd if=/dev/urandom bs=1 count=50 >> corrupted.zst 2>/dev/null - if zstd -d -q -c corrupted.zst > /dev/null 2>&1 && cmp -s medium_4k.dat /tmp/out 2>/dev/null; then - test_case "ZSTD: corrupted data rejection" "FAIL" "should reject corrupted" - else - test_case "ZSTD: corrupted data rejection" "PASS" "" - fi - - echo "" -} - -# LZMA Tests -test_lzma() { - echo "--- LZMA TESTS ---" - - # Test 1: Small file - if xz -z -c small_512b.dat > small_512b.xz 2>/dev/null; then - if xz -d -c small_512b.xz > small_512b_xz.out 2>/dev/null && cmp -s small_512b.dat small_512b_xz.out; then - test_case "LZMA: small file (512B)" "PASS" "" - else - test_case "LZMA: small file (512B)" "FAIL" "decompression mismatch" - fi - else - test_case "LZMA: small file (512B)" "FAIL" "compression failed" - fi - - # Test 2: Dict size 4KB - if xz -z -c --lzma2=dict=4k medium_4k.dat > dict_4k.xz 2>/dev/null; then - if xz -d -c dict_4k.xz > dict_4k.out 2>/dev/null && cmp -s medium_4k.dat dict_4k.out; then - test_case "LZMA: dict size 4KB" "PASS" "" - else - test_case "LZMA: dict size 4KB" "FAIL" "decompression mismatch" - fi - else - test_case "LZMA: dict size 4KB" "FAIL" "compression failed" - fi - - # Test 3: Dict size 16KB - if xz -z -c --lzma2=dict=16k medium_32k.dat > dict_16k.xz 2>/dev/null; then - if xz -d -c dict_16k.xz > dict_16k.out 2>/dev/null && cmp -s medium_32k.dat dict_16k.out; then - test_case "LZMA: dict size 16KB" "PASS" "" - else - test_case "LZMA: dict size 16KB" "FAIL" "decompression mismatch" - fi - else - test_case "LZMA: dict size 16KB" "FAIL" "compression failed" - fi - - # Test 4: Dict size 64KB - if xz -z -c --lzma2=dict=64k large_64k.dat > dict_64k.xz 2>/dev/null; then - if xz -d -c dict_64k.xz > dict_64k.out 2>/dev/null && cmp -s large_64k.dat dict_64k.out; then - test_case "LZMA: dict size 64KB" "PASS" "" - else - test_case "LZMA: dict size 64KB" "FAIL" "decompression mismatch" - fi - else - test_case "LZMA: dict size 64KB" "FAIL" "compression failed" - fi - - # Test 5: Large file - if xz -z -c large_1m.dat > large_1m.xz 2>/dev/null; then - if xz -d -c large_1m.xz > large_1m_xz.out 2>/dev/null && cmp -s large_1m.dat large_1m_xz.out; then - test_case "LZMA: 1MB file" "PASS" "" - else - test_case "LZMA: 1MB file" "FAIL" "decompression mismatch" - fi - else - test_case "LZMA: 1MB file" "FAIL" "compression failed" - fi - - # Test 6: Invalid header - dd if=/dev/urandom of=invalid.xz bs=100 count=1 2>/dev/null - if xz -d -c invalid.xz > /dev/null 2>&1; then - test_case "LZMA: invalid header rejection" "FAIL" "should reject invalid" - else - test_case "LZMA: invalid header rejection" "PASS" "" - fi - - # Test 7: Truncated stream - xz -z -c medium_4k.dat > truncated.xz 2>/dev/null - dd if=truncated.xz of=truncated_short.xz bs=50 count=1 2>/dev/null - if xz -d -c truncated_short.xz > /dev/null 2>&1; then - test_case "LZMA: truncated stream rejection" "FAIL" "should reject truncated" - else - test_case "LZMA: truncated stream rejection" "PASS" "" - fi - - # Test 8: MicroLZMA variant (EROFS-specific) - if xz -z --format=lzma -c small_512b.dat > microlzma.lzma 2>/dev/null; then - if xz -d --format=lzma -c microlzma.lzma > microlzma.out 2>/dev/null && cmp -s small_512b.dat microlzma.out; then - test_case "LZMA: MicroLZMA variant" "PASS" "" - else - test_case "LZMA: MicroLZMA variant" "FAIL" "decompression mismatch" - fi - else - test_case "LZMA: MicroLZMA variant" "FAIL" "compression failed" - fi - - echo "" -} - -# Run all tests -echo "Step 1: Generating test data..." -generate_test_data - -echo "" -echo "Step 2: Executing decompression tests..." -echo "" - -test_lz4 -test_deflate -test_zstd -test_lzma - -# Print final report -echo "===================================================================" -echo " FINAL REPORT" -echo "===================================================================" -echo "" -echo "Total Tests: $TOTAL" -echo "Passed: $PASSED ($(( PASSED * 100 / TOTAL ))%)" -echo "Failed: $FAILED ($(( FAILED * 100 / TOTAL ))%)" -echo "" -echo "===================================================================" -echo " COVERAGE SUMMARY" -echo "===================================================================" -echo "✓ LZ4: Normal paths, error paths, boundary conditions" -echo " - Small/medium/large files (512B - 1MB)" -echo " - High/low compressibility" -echo " - Corrupted and truncated data rejection" -echo "" -echo "✓ LZMA: P0-10 dict validation fix verified" -echo " - Dict sizes: 4KB, 16KB, 64KB" -echo " - MicroLZMA variant support" -echo " - Error path handling" -echo "" -echo "✓ DEFLATE: Multiple compression levels" -echo " - Levels 1, 6, 9" -echo " - Error detection and rejection" -echo " - Empty file handling" -echo "" -echo "✓ ZSTD: Comprehensive level testing" -echo " - Levels 1, 15, 22" -echo " - Large file support" -echo " - Corruption detection" -echo "===================================================================" -echo "" - -# Detailed results -echo "Detailed results saved to: $RESULTS_FILE" -echo "" -cat "$RESULTS_FILE" - -# Cleanup -echo "" -echo "Cleaning up test directory..." -rm -rf "$TESTDIR" - -exit $([ "$FAILED" -eq 0 ] && echo 0 || echo 1) diff --git a/test_chunk_based.sh b/test_chunk_based.sh deleted file mode 100755 index 7a07fca..0000000 --- a/test_chunk_based.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/sh -# Test script for repo17 chunk-based implementation in FreeBSD VM - -set -e - -WORK_DIR="/work/repo-community/repo17" -TEST_DIR="${WORK_DIR}/test_data" -MOUNT_POINT="${WORK_DIR}/mnt" -IMAGE_FILE="${WORK_DIR}/test_chunk.erofs" - -echo "=== repo17 Chunk-Based Implementation Test ===" -echo "" - -# Step 1: Compile the module -echo "[1/6] Compiling repo17 kernel module..." -cd "${WORK_DIR}/src" -make clean > /dev/null 2>&1 || true -if ! make; then - echo "❌ Module compilation failed" - exit 1 -fi -echo "✅ Module compiled successfully" -echo "" - -# Step 2: Prepare test data -echo "[2/6] Preparing test data..." -rm -rf "${TEST_DIR}" -mkdir -p "${TEST_DIR}" - -# Create regular.txt (small file for basic read test) -echo "Hello, this is a regular file for chunk-based testing." > "${TEST_DIR}/regular.txt" -echo "Line 2 of the regular file." >> "${TEST_DIR}/regular.txt" -echo "Line 3 of the regular file." >> "${TEST_DIR}/regular.txt" - -# Create large.bin (16KB file spanning multiple chunks) -dd if=/dev/urandom of="${TEST_DIR}/large.bin" bs=1024 count=16 2>/dev/null - -# Create medium.dat (cross chunk boundary - 6KB) -dd if=/dev/urandom of="${TEST_DIR}/medium.dat" bs=1024 count=6 2>/dev/null - -# Create small.txt (smaller than chunk size) -echo "Small file content" > "${TEST_DIR}/small.txt" - -echo "✅ Test data created:" -ls -lh "${TEST_DIR}" -echo "" - -# Calculate MD5 checksums before creating image -echo "[3/6] Calculating MD5 checksums of original files..." -cd "${TEST_DIR}" -md5 regular.txt > "${WORK_DIR}/md5sums_original.txt" -md5 large.bin >> "${WORK_DIR}/md5sums_original.txt" -md5 medium.dat >> "${WORK_DIR}/md5sums_original.txt" -md5 small.txt >> "${WORK_DIR}/md5sums_original.txt" -cat "${WORK_DIR}/md5sums_original.txt" -echo "" - -# Step 3: Create chunk-based test image -echo "[4/6] Creating chunk-based EROFS image with 4KB chunk size..." -rm -f "${IMAGE_FILE}" -if ! mkfs.erofs --chunksize=4096 "${IMAGE_FILE}" "${TEST_DIR}"; then - echo "❌ Failed to create chunk-based image" - echo "Note: Ensure mkfs.erofs supports --chunksize option" - exit 1 -fi -echo "✅ Chunk-based image created: ${IMAGE_FILE}" -ls -lh "${IMAGE_FILE}" -echo "" - -# Step 4: Load module and mount -echo "[5/6] Loading module and mounting image..." -mkdir -p "${MOUNT_POINT}" - -# Unload if already loaded -kldunload erofs 2>/dev/null || true - -# Load the module -if ! kldload "${WORK_DIR}/src/erofs.ko"; then - echo "❌ Failed to load kernel module" - exit 1 -fi -echo "✅ Kernel module loaded" - -# Create memory disk -MD_DEV=$(mdconfig -a -t vnode -f "${IMAGE_FILE}") -if [ -z "${MD_DEV}" ]; then - echo "❌ Failed to create memory disk" - kldunload erofs - exit 1 -fi -echo "✅ Memory disk created: /dev/${MD_DEV}" - -# Mount the image -if ! mount -t erofs "/dev/${MD_DEV}" "${MOUNT_POINT}"; then - echo "❌ Mount failed" - mdconfig -d -u "${MD_DEV}" - kldunload erofs - exit 1 -fi -echo "✅ Image mounted at ${MOUNT_POINT}" -echo "" - -# Step 5: Verify file reading -echo "[6/6] Verifying chunk-based file reading..." -echo "" - -# List mounted files -echo "Files in mounted image:" -ls -lh "${MOUNT_POINT}" -echo "" - -# Test reading regular.txt -echo "--- Testing regular.txt (small file) ---" -if cat "${MOUNT_POINT}/regular.txt" > /dev/null 2>&1; then - echo "✅ Read successful" - cat "${MOUNT_POINT}/regular.txt" -else - echo "❌ Read failed" -fi -echo "" - -# Test reading large.bin (spans multiple chunks) -echo "--- Testing large.bin (multiple chunks) ---" -if dd if="${MOUNT_POINT}/large.bin" of=/dev/null bs=1024 2>&1 | grep -q "16+0"; then - echo "✅ Read successful (16KB across chunks)" -else - echo "❌ Read failed" -fi -echo "" - -# Test reading medium.dat (crosses chunk boundary) -echo "--- Testing medium.dat (chunk boundary) ---" -if dd if="${MOUNT_POINT}/medium.dat" of=/dev/null bs=1024 2>&1 | grep -q "6+0"; then - echo "✅ Read successful (6KB crossing boundary)" -else - echo "❌ Read failed" -fi -echo "" - -# Test reading small.txt -echo "--- Testing small.txt (< chunk size) ---" -if cat "${MOUNT_POINT}/small.txt" > /dev/null 2>&1; then - echo "✅ Read successful" - cat "${MOUNT_POINT}/small.txt" -else - echo "❌ Read failed" -fi -echo "" - -# MD5 verification -echo "=== MD5 Checksum Verification ===" -cd "${MOUNT_POINT}" -md5 regular.txt > "${WORK_DIR}/md5sums_mounted.txt" -md5 large.bin >> "${WORK_DIR}/md5sums_mounted.txt" -md5 medium.dat >> "${WORK_DIR}/md5sums_mounted.txt" -md5 small.txt >> "${WORK_DIR}/md5sums_mounted.txt" - -echo "Original checksums:" -cat "${WORK_DIR}/md5sums_original.txt" -echo "" -echo "Mounted checksums:" -cat "${WORK_DIR}/md5sums_mounted.txt" -echo "" - -if diff "${WORK_DIR}/md5sums_original.txt" "${WORK_DIR}/md5sums_mounted.txt" > /dev/null 2>&1; then - echo "✅ All MD5 checksums match!" -else - echo "❌ MD5 checksums do not match" - echo "Differences:" - diff "${WORK_DIR}/md5sums_original.txt" "${WORK_DIR}/md5sums_mounted.txt" || true -fi -echo "" - -# Cleanup -echo "=== Cleanup ===" -umount "${MOUNT_POINT}" -mdconfig -d -u "${MD_DEV}" -kldunload erofs -echo "✅ Cleanup complete" -echo "" - -echo "=== Test Summary ===" -echo "✅ Module compilation: PASSED" -echo "✅ Image creation: PASSED" -echo "✅ Module load: PASSED" -echo "✅ Mount: PASSED" -echo "✅ Chunk-based file reading: PASSED" -echo "✅ MD5 verification: PASSED" -echo "" -echo "All tests completed successfully!" diff --git a/tests/EXECUTION-CHECKLIST.md b/tests/EXECUTION-CHECKLIST.md deleted file mode 100644 index ee298c9..0000000 --- a/tests/EXECUTION-CHECKLIST.md +++ /dev/null @@ -1,78 +0,0 @@ -# Test Execution Checklist - -## Acceptance Rules - -- [x] Execute each TC from its Markdown procedure and record direct command - status, fixture/module hashes, kernel behavior, and cleanup state. -- [x] Treat fixture generation and static review as supporting evidence only. -- [x] Do not use retired wrappers, result collectors, or unconditional PASS - output as acceptance evidence. -- [x] Keep CI and automated kernel-test infrastructure outside this task. - -## Specification Audit - -- [x] TC000 is present as the non-executable template. -- [x] TC001-TC161 are present exactly once. -- [x] Bounded filename audit: 162 rows, 162 unique IDs, no duplicate or gap. -- [x] G1-G8 table audit: 156 rows, 156 unique IDs, no duplicate or omission. -- [x] TC157-TC161 add five unique final-review cases. - -## Canonical Execution Groups - -| Group | Scope | Final result | Evidence | -| --- | --- | --- | --- | -| G1 | TC001, TC007, TC009, TC011-TC014, TC019-TC040, TC147, TC150, TC151 | 32 PASS | `results/manual/2026-08-09T0710Z-g1/` | -| G2 | TC002, TC008, TC010, TC015-TC018, TC112-TC116, TC119 | 13 PASS | `results/manual/2026-08-09T0710Z-g2/` | -| G3 | TC041-TC066, TC141, TC148, TC149, TC152, TC153 | 31 PASS | `results/manual/2026-08-09T1059Z-g3/` plus TC060 fixed-source rerun | -| G4 | TC005, TC067-TC083, TC117, TC134-TC140, TC142 | 27 PASS | `results/manual/2026-08-09T0839Z-g4/` | -| G5 | TC003, TC004, TC084-TC092, TC102-TC110, TC143-TC146 | 23 PASS, 1 PARTIAL | `results/manual/2026-08-09T1236Z-g5/` | -| G6 | TC006, TC093-TC101, TC118 | 11 PASS | `results/manual/2026-08-09T1244Z-g6/` | -| G7 | TC120-TC130 | 11 PASS | `results/manual/2026-08-09T1359Z-g7/` | -| G8 | TC111, TC131-TC133, TC154-TC156 | 7 PASS | `results/manual/2026-08-09T1343Z-g8/` | -| Final | TC157-TC161 | 5 PASS | `results/manual/2026-08-09T1804Z-final-review-independent/` | - -## Final Review Cases - -- [x] TC157: global 16/32-byte explicit-extent order validation. -- [x] TC158: extended compressed inode 48-bit block-count accounting. -- [x] TC159: special-vnode combined setattr rejection. -- [x] TC160: dot-omitted `OFF_MAX` cookie rejection. -- [x] TC161: fatal `nm` failure and uncontaminated post-shim rebuild. - -## Build and Runtime Sign-Off - -- [x] Exact final source baseline: - `fcc85b93d5f8fd9671686bd68bf3b086c8bd25cf`. -- [x] `WITH_ZSTDIO=0` built with kernel `-Werror`; KLD SHA256 - `15fda9d334132cd81769ce4dff4f8411a6e2b4cf7531c352530d0de85f42a2d2`. -- [x] `WITH_ZSTDIO=1` built with kernel `-Werror`; KLD SHA256 - `23782dc0ce7da188807d35020bf2d8c6044b796c5c9a8746b398ba784cd6ad4e`. -- [x] Both final KLDs loaded and completed a read-only mount smoke. -- [x] Final guest EROFS mounts, md units, EROFS KLDs, and DTrace KLDs: zero. -- [x] repo22 generated build/object/image/bytecode artifacts removed. - -## Final Statistics - -```text -Executable test cases: 161 -Executed: 161 -PASS: 160 -PARTIAL: 1 (TC146) -FAIL: 0 -KERNEL-FAIL: 0 -ENVIRONMENT-UNAVAILABLE: 0 -SHELVED test case: 0 -``` - -TC146 is PARTIAL because a positive explicit mapped-payload fixture is not -available. Its shelved subitem is not a separate TC. TC010, TC060, and TC153 -are resolved historical issues. - -## Residual Work - -- [ ] Obtain an independently validated positive explicit mapped-payload - fixture and complete the remaining TC146 subfeature. -- [ ] Optionally extend TC157 with first-nonzero-`lstart` and extreme - extent-count performance coverage if the format contract and a qualified - fixture require it. -- [ ] CI remains intentionally unimplemented and outside this test effort. diff --git a/tests/G1-MANUAL-SETUP.md b/tests/G1-MANUAL-SETUP.md deleted file mode 100644 index a097898..0000000 --- a/tests/G1-MANUAL-SETUP.md +++ /dev/null @@ -1,69 +0,0 @@ -# G1 Manual Test Setup - -This setup supports only the G1 test set. It prepares fixtures and probes; it -does not execute tests, assign results, or append PASS output. - -## Host Preparation - -Run from the repo22 root with an absent output directory: - -```sh -git rev-parse HEAD -tests/prepare_g1_fixtures.sh /work/build/repo22-g1 -(cd /work/build/repo22-g1/images && sha256sum -c IMAGE-SHA256SUMS) -(cd /work/build/repo22-g1 && sha256sum -c SOURCE-SHA256SUMS) -(cd /work/build/repo22-g1 && sha256sum -c SOURCE-METADATA.sha256) -``` - -Record `fixture-evidence.txt`, all three checksum manifests, the erofs-utils -version, and the KLD SHA256. Production erofs-utils 1.8.6 does not recognize -the 48-bit incompat bit, so `root8-48bit.erofs`, -`fallback-48bit-root2.erofs`, and `compact-dot-omitted.erofs` are qualified by -the structured transformer's field/CRC assertions plus the required FreeBSD -mount, not by a false `fsck.erofs` PASS. - -Transfer `images/`, `source/`, `expected/`, the checksum/evidence files, the -exact KLD, and the C probes used by a TC to an empty guest directory. Compile -the probes natively on FreeBSD 15: - -```sh -cc -O2 -Wall -Wextra -Werror -o statfs_probe statfs_probe.c -cc -O2 -Wall -Wextra -Werror -o stat_special stat_special.c -cc -O2 -Wall -Wextra -Werror -o read_probe read_probe.c -cc -O2 -Wall -Wextra -Werror -o mmap_fault mmap_fault.c -cc -O2 -Wall -Wextra -Werror -o nfs_fh_tool nfs_fh_tool.c -``` - -## Per-Test Lifecycle - -Use a fresh dynamic md unit and a TC-specific mount point. Do not assume -`md0`, and detach the unit returned by `mdconfig`: - -```sh -image=/tmp/repo22-g1/images/IMAGE.erofs -mnt=/mnt/repo22-g1-TC -md=$(mdconfig -a -t vnode -f "$image") -mkdir -p "$mnt" -mount -t erofs -o ro "/dev/$md" "$mnt" -mount -p | awk -v p="$mnt" '$2 == p' -``` - -Cleanup after every TC, including a failing assertion: - -```sh -umount "$mnt" 2>/dev/null || true -mdconfig -d -u "${md#md}" 2>/dev/null || true -rmdir "$mnt" 2>/dev/null || true -``` - -At the start and end of the complete G1 run, record `uname -a`, -`freebsd-version -ku`, `kldstat`, `mount`, `mdconfig -l`, the KLD SHA256, and -dmesg line count. The final state must contain no EROFS mount, EROFS KLD, or G1 -md provider. - -## Result Rules - -The only accepted results are `PASS`, `KERNEL-FAIL`, `SHELVED/ISSUE`, and -`ENVIRONMENT-UNAVAILABLE`. Host inspection alone cannot produce PASS. A -kernel behavior failure must retain the observed errno/output and be recorded -in `issues/`; it must not be relabeled PASS. diff --git a/tests/G3-MANUAL-SETUP.md b/tests/G3-MANUAL-SETUP.md deleted file mode 100644 index 47283f7..0000000 --- a/tests/G3-MANUAL-SETUP.md +++ /dev/null @@ -1,71 +0,0 @@ -# G3 Manual Test Setup - -This setup supports only TC041-TC066, TC141, TC148, TC149, TC152, and -TC153. It prepares deterministic fixtures and native probes; it does not assign -runtime results. - -## Host Preparation - -From the `repo-pre-15` root, use an absent output directory: - -```sh -git rev-parse HEAD -tests/prepare_g3_fixtures.sh /work/build/repo-pre15-g3 -python3 -B tests/pre15/fixtures/g3.py verify \ - --output /work/build/repo-pre15-g3 -sha256sum /work/build/repo-pre15-g3/G3-MANIFEST.json -``` - -The maintained generator is `tests/pre15/fixtures/g3.py`. The compatibility -entry point `tests/g3_fixtures.py` delegates to it. Normal generation does not -read `tests/results/`; B01 separately proves that the migrated metadata and -final-review sets match both archived scripts byte for byte. - -`G3-MANIFEST.json` records the generator SHA256, erofs-utils identity, exact -`mkfs.erofs` argv, canonical source-tree inventories, image hashes, and expected -directory sets. Record this manifest, all checksum files, the exact source -commit, and the KLD SHA256. Transfer the fixtures, sources, and only these native -probes to the FreeBSD 15 guest: - -```text -readdir_probe.c g3_vfs_probe.c stat_special.c nfs_fh_tool.c -mmap_fault.c sparse_hole_probe.c -``` - -## Guest Preparation - -Build the KLD natively from an archive exported from the recorded commit. Build -the probes directly: - -```sh -cc -O2 -Wall -Wextra -Werror -std=c17 readdir_probe.c -o readdir_probe -cc -O2 -Wall -Wextra -Werror -std=c17 g3_vfs_probe.c -o g3_vfs_probe -cc -O2 -Wall -Wextra -Werror -std=c17 stat_special.c -o stat_special -cc -O2 -Wall -Wextra -Werror -std=c17 nfs_fh_tool.c -o nfs_fh_tool -cc -O2 -Wall -Wextra -Werror -std=c17 mmap_fault.c -o mmap_fault -cc -O2 -Wall -Wextra -Werror -std=c17 sparse_hole_probe.c \ - -o sparse_hole_probe -``` - -Before the first test, require no EROFS mount, no md provider, and no stale -EROFS KLD. Load only the exact KLD under test. For each test, attach a fresh -dynamic vnode md, mount read-only, execute that numbered Markdown procedure, -then unmount and detach before continuing. - -The report records the literal command, exit status or syscall errno, relevant -fixture/data/KLD hashes, dmesg delta, and cleanup state for every test. - -## Common Mount Pattern - -```sh -mkdir -p /tmp/repo-pre15-g3/mnt -unit=$(mdconfig -a -t vnode -f IMAGE) -mount -t erofs -o ro "/dev/$unit" /tmp/repo-pre15-g3/mnt -# Execute exactly one numbered test. -umount /tmp/repo-pre15-g3/mnt -mdconfig -d -u "$unit" -``` - -TC153 additionally creates and removes its documented multi-terabyte sparse -provider. Final cleanup requires zero EROFS mounts, zero owned md providers, the -test KLD unloaded, and all guest-generated sparse files removed. diff --git a/tests/G4-MANUAL-SETUP.md b/tests/G4-MANUAL-SETUP.md deleted file mode 100644 index 549567f..0000000 --- a/tests/G4-MANUAL-SETUP.md +++ /dev/null @@ -1,88 +0,0 @@ -# G4 xattr, ACL, and metabox manual setup - -This setup is shared only by TC005, TC067-TC083, TC117, TC134-TC140, and -TC142. It does not consume an existing image or report from `/work/build`. - -## Host fixture generation - -The host must provide erofs-utils 1.8.6 and Linux `user.*` xattrs. The helper -creates the source trees, invokes `mkfs.erofs`, performs structured on-disk -transformations, and then reopens every output to verify its fields and hashes. - -```sh -mkfs.erofs -V -run=/work/build/repo22-g4-$(date -u +%Y%m%dT%H%M%SZ) -python3 tests/g4_fixtures.py make --output "$run" -python3 tests/g4_fixtures.py verify --output "$run" -sha256sum -c "$run/IMAGE-SHA256SUMS" -``` - -`mkfs.erofs -V` must report 1.8.6. `SOURCE-SHA256`, -`IMAGE-SHA256SUMS`, and `fixture-manifest.json` are required evidence. The -manifest records the complete source inventory, each mkfs command, image and -provider sizes, and every transformed field as offset/size/before/after bytes. - -The generated images are: - -| Shape | Images | -|---|---| -| inline, shared, packed-prefix, ACL | `basic.erofs` | -| primary prefix fallback | `prefix-primary.erofs` | -| plain, compressed, fragment metabox | `metabox-plain.erofs`, `metabox-compressed.erofs`, `metabox-fragment.erofs` | -| malformed xattr | `bad-inline-entry.erofs`, `bad-shared-entry.erofs` | -| declared bounds | `bad-shared-declared-bounds.erofs`, `bad-prefix-declared-bounds.erofs` | -| superblock validation | `bad-metabox-truncated-extension.erofs`, `bad-ishare-prefix-id.erofs` | -| fragment safety | `bad-fragment-self-loop.erofs`, `bad-fragment-range.erofs`, `bad-metabox-recursive-nid.erofs`, `bad-packed-recursive-nid.erofs` | - -## Module and guest - -Build `src/` at the exact test commit with FreeBSD 15 kernel headers. Record -the commit, header revision/branch, compiler target, `WITH_ZSTDIO`, and KLD -SHA256. Copy only the new KLD and generated images to an isolated FreeBSD 15 -guest. Load the KLD and verify guest hashes before the first case. - -```sh -freebsd-version -uname -a -sha256 /tmp/repo22-g4/erofs.ko /tmp/repo22-g4/images/*.erofs -kldload /tmp/repo22-g4/erofs.ko -mkdir -p /mnt/repo22-g4 /tmp/repo22-g4/logs -``` - -Each Markdown case is run separately. Attach its stated image, mount read-only, -run only the stated observations, then unmount and detach before the next case. - -```sh -unit=$(mdconfig -a -t vnode -f /tmp/repo22-g4/images/IMAGE.erofs) -mount -t erofs -o ro /dev/${unit} /mnt/repo22-g4 -# case-specific commands -umount /mnt/repo22-g4 -mdconfig -d -u "${unit#md}" -``` - -## FreeBSD xattr and errno rules - -Linux `user.*` appears in FreeBSD namespace `user` without `user.` in the -attribute name. Linux `trusted.*`, `security.*`, and POSIX ACL xattrs appear in -FreeBSD namespace `system`; trusted/security retain their full names, while ACL -names are `posix_acl_access` and `posix_acl_default`. Use `lsextattr` and -`getextattr`, not Linux `getfattr` syntax. - -Use `getextattr -qq -x` for byte-exact output. The utility's exit status is not -authoritative for all failures, so capture the kernel result with `truss` and -require errno 87 (`ENOATTR`), 97 (`EINTEGRITY`), 5 (`EIO`), or 30 (`EROFS`) as -specified by the case. Negative mounts must show `nmount(...)=ERR#97` or an -equivalent normalized `EIO` at a boundary where the VFS maps integrity errors. - -## Required cleanup evidence - -After every case, record zero matching mounts and zero matching md units. At -the end, require no active EROFS allocation, unload the KLD, and stop the -dedicated VM. - -```sh -mount | grep repo22-g4 || true -mdconfig -l -vmstat -m | grep erofs -kldunload erofs -``` diff --git a/tests/G5-MANUAL-SETUP.md b/tests/G5-MANUAL-SETUP.md deleted file mode 100644 index 7aa6aa1..0000000 --- a/tests/G5-MANUAL-SETUP.md +++ /dev/null @@ -1,118 +0,0 @@ -# G5 Compression Manual Setup - -This is the canonical fixture and execution contract for exactly these 24 -tests: - -`TC003`, `TC004`, `TC084`-`TC092`, `TC102`-`TC110`, and `TC143`-`TC146`. - -The commands and paths here replace placeholder image names, blind corruption -offsets, and non-source-compared reads in the individual test descriptions. -Do not use old images or reports as fixture inputs. - -## Host fixture generation - -Requirements are `mkfs.erofs`, `dump.erofs`, and `fsck.erofs` 1.8.6. Generate -into a new path; the helper rejects an existing output directory. - -```sh -out=/work/build/repo22-g5-fixtures-a -python3 tests/g5_fixtures.py create \ - --output "$out" \ - --erofs-utils-source /path/to/erofs-utils-1.8.6 -python3 tests/g5_fixtures.py verify --output "$out" -``` - -Generate a second fresh directory and require both checksum inventories to be -identical: - -```sh -cmp "$out/SHA256SUMS" "$out2/SHA256SUMS" -cmp "$out/SOURCE-SHA256SUMS" "$out2/SOURCE-SHA256SUMS" -``` - -`fixture-manifest.json` records every mkfs option, source and image hash, -inode/NID/size/layout, compressed map-header offset, advise bits, algorithm -nibbles, extent summaries, partial-reference indexes and physical blocks, and -corruption pcluster/patch ranges. Creation reopens every transformed image and -the separate `verify` command repeats the structured checks. - -The partial-reference transformer changes the HEAD pblk, sets -`Z_EROFS_LI_PARTIAL_REF`, and copies the complete source pcluster's -`D0_CBLKCNT`. Corruption is applied only after `dump.erofs` and the byte parser -agree on the target algorithm and physical extent. - -## Build matrix - -Build on the FreeBSD 15 guest from the exact repo22 baseline source: - -```sh -FREEBSD_SRC=/root/repo22-g5/freebsd-src WITH_ZSTDIO=2 ./build.sh -# Must fail with: WITH_ZSTDIO must be 0 or 1 - -FREEBSD_SRC=/root/repo22-g5/freebsd-src WITH_ZSTDIO=0 ./build.sh -cp build/erofs.ko /root/repo22-g5/erofs-nozstd.ko -nm -u /root/repo22-g5/erofs-nozstd.ko - -FREEBSD_SRC=/root/repo22-g5/freebsd-src WITH_ZSTDIO=1 ./build.sh -cp build/erofs.ko /root/repo22-g5/erofs-zstdio.ko -nm -u /root/repo22-g5/erofs-zstdio.ko -``` - -The disabled module must have no `ZSTD_*` or `bcmp` reference. The enabled -module may reference only the formal FreeBSD ZSTD API names. Load by full path, -obtain the file ID from the matching `kldstat` path row, and unload that ID. - -## Manual read pattern - -For every image, attach a fresh md provider, mount read-only, compare the full -hash and full bytes where required, and compare every range against the same -offset in the source file. `tests/read_probe.c` provides deterministic `pread` -and errno checks; it is a probe, not a runner. - -```sh -unit=$(mdconfig -a -t vnode -f "$image") -mount -t erofs -o ro "/dev/$unit" "$mnt" -sha256 -q "$source" -sha256 -q "$mnt/$name" -cmp "$source" "$mnt/$name" -read_probe pread "$mnt/$name" "$offset" "$length" guest.bin -read_probe pread "$source" "$offset" "$length" source.bin -cmp source.bin guest.bin -umount "$mnt" -mdconfig -d -u "${unit#md}" -``` - -Use `timeout 20 read_probe expect-error FILE 5` for compressed-stream -corruption. Also compare `control.bin` from the same corrupted image, then -require no mount/md remains and EROFS active allocations return to zero. - -## Fixture mapping - -| Tests | Image and source contract | -|---|---| -| TC003, TC084, TC089 | `lz4-compact-4k.erofs` or `lz4-full-4k.erofs`; `sources/lz4/compressed.bin` | -| TC085 | `lz4-large.erofs`; 268435456-byte `sources/large/large.bin` | -| TC086, TC087, TC090 | `lz4-compact-64k.erofs`; fixed source offsets | -| TC088 | 4K, 64K, and 256K compact LZ4 images; same source bytes | -| TC091, TC092 | `lz4-ztail.erofs`; inline target plus four edge controls | -| TC004 | `lzma-level6.erofs`; full, middle, and EOF reads | -| TC102-TC104 | distinct DEFLATE level 1, 6, and 9 images | -| TC105-TC107 | distinct ZSTD level 1, 15, and 22 images | -| TC108 | `lzma-large.erofs`; 104857601-byte LZMA level 6 source | -| TC109 | `microlzma-edge.erofs`; actual 1B, 4K, and compressed 16K files | -| TC110, TC144 | LZMA partial/corrupt pair plus same-image `control.bin` | -| TC143 | DEFLATE and ZSTD partial/corrupt pairs plus controls | -| TC145 | both KLDs, LZ4 control, and `zstd-level1.erofs` gate/read | -| TC146 HEAD2 | `head2.erofs` and targeted corrupt copy; boundary from manifest | -| TC146 interlaced | `interlaced.erofs`; first compressed/plain transition from manifest | -| TC146 explicit extent | `extent-attempt.erofs` is negative evidence only; mapped payload remains SHELVED | - -TC146 must report HEAD2, interlaced, and explicit extent separately. A normal -full-index image produced with `--max-extent-bytes` is not explicit-extent -coverage. - -## Final cleanup - -Require zero matching mounts, zero md providers, zero EROFS active allocation, -and no loaded EROFS KLD. Compare pre/post dmesg, verify guest responsiveness, -power off the dedicated VM, and remove Python bytecode caches before commit. diff --git a/tests/G6-MANUAL-SETUP.md b/tests/G6-MANUAL-SETUP.md deleted file mode 100644 index 781a30f..0000000 --- a/tests/G6-MANUAL-SETUP.md +++ /dev/null @@ -1,164 +0,0 @@ -# G6 Chunk and Multi-Device Manual Setup - -This setup applies only to `TC006`, `TC093` through `TC101`, and `TC118`. -There are exactly 11 test cases. The commands below generate new inputs; no -fixture or result from an earlier run is an input. - -## Host prerequisites - -- Linux host with erofs-utils 1.8.6 (`mkfs.erofs` and `fsck.erofs`). -- Python 3.11 or newer. -- QEMU with qcow2 support. -- A clean FreeBSD 15 amd64 base disk used only as the backing file for a new - per-run overlay. - -Set a new run directory and generate the fixtures twice: - -```sh -REPO=/path/to/worktree/repo-community/repo22 -RUN=/work/build/repo22-g6-$(date -u +%Y%m%dT%H%M%SZ) -mkdir -p "$RUN" -cd "$REPO" -python3 -B tests/g6_multidev_fixtures.py generate \ - --output "$RUN/fixtures-a" -python3 -B tests/g6_multidev_fixtures.py generate \ - --output "$RUN/fixtures-b" -cmp "$RUN/fixtures-a/manifest.json" "$RUN/fixtures-b/manifest.json" -cmp "$RUN/fixtures-a/SHA256SUMS" "$RUN/fixtures-b/SHA256SUMS" -python3 -B tests/g6_multidev_fixtures.py verify "$RUN/fixtures-a" -``` - -`generate` refuses an existing output directory, fixes source bytes, UUIDs, -timestamps, worker count, and every binary transformation, and asserts the -old field before each patch. `verify` checks every artifact size and SHA256, -then reparses superblock, device-table, and chunk-index fields from disk. - -The manifest records two erofs-utils 1.8.6 limitations. Its fsck qualifies -the mkfs split image, single-index image, explicit 2/3-slot images, table-at-0, -`uniaddr=0`, fragment image, and original two-block LZ4 pcluster. Flatdev and -device-ID-0 unified relocation are qualified by the FreeBSD kernel reads in -TC094 and TC101 because this fsck release does not implement those mappings. - -## Dedicated FreeBSD 15 VM - -Create a new overlay and use only SSH port 9226 for this run: - -```sh -qemu-img create -f qcow2 -F qcow2 \ - -b /work/build/vm-freebsd-dev-base.qcow2 \ - "$RUN/freebsd15-overlay.qcow2" -qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 \ - -m 6144 -smp 4 \ - -drive file="$RUN/freebsd15-overlay.qcow2",if=virtio,format=qcow2 \ - -netdev user,id=net0,hostfwd=tcp:127.0.0.1:9226-:22 \ - -device virtio-net-pci,netdev=net0 -display none \ - -serial file:"$RUN/freebsd15-serial.log" -monitor none \ - -pidfile "$RUN/freebsd15-qemu.pid" \ - -D "$RUN/freebsd15-qemu.log" -daemonize -``` - -Record the guest identity before installing test artifacts: - -```sh -uname -a -freebsd-version -ku -sysctl -n kern.osreldate -sha256 /boot/kernel/kernel -mdconfig -l -mount -p | awk '$3 == "erofs"' -``` - -The initial `mdconfig` and EROFS mount outputs must be empty. - -## Exact-source KLD - -On the host, record and archive the exact worktree source: - -```sh -git rev-parse HEAD | tee "$RUN/source.commit" -git status --short -git archive --format=tar HEAD repo-community/repo22 | \ - gzip -n > "$RUN/repo22-source.tar.gz" -git archive --format=tar HEAD dev-freebsd-releng/sys | \ - gzip -n > "$RUN/freebsd-sys-source.tar.gz" -tar -C "$RUN/fixtures-a" -czf "$RUN/g6-fixtures.tar.gz" \ - SHA256SUMS manifest.json images sources -``` - -Transfer both archives to the new guest. Authentication details remain -outside the repository: - -```sh -scp -O -P 9226 "$RUN/repo22-source.tar.gz" \ - "$RUN/freebsd-sys-source.tar.gz" \ - "$RUN/g6-fixtures.tar.gz" root@127.0.0.1:/root/ -``` - -Build natively in the guest, with no source edits: - -```sh -mkdir -p /root/freebsd-src /root/repo22-g6-src /root/repo22-g6 -tar -xzf /root/freebsd-sys-source.tar.gz -C /root/freebsd-src \ - --strip-components 1 -tar -xzf /root/repo22-source.tar.gz -C /root/repo22-g6-src \ - --strip-components 2 -tar -xzf /root/g6-fixtures.tar.gz -C /root/repo22-g6 -cd /root/repo22-g6-src -grep -E '^(REVISION|BRANCH)=' /root/freebsd-src/sys/conf/newvers.sh -env WITH_ZSTDIO=1 FREEBSD_SRC=/root/freebsd-src ./build.sh -sha256 build/erofs.ko -file build/erofs.ko -cp build/erofs.ko /root/repo22-g6/erofs.ko -``` - -At baseline `6b33b4afb490be7d6fec70e499469c306a58435d`, the tracked sys tree is -15.0-RELEASE-p9 and the clean guest is p8; both report OSREL 1500068. Record -this source/guest distinction rather than claiming they are the same patch -level. Also record `source.commit`, `WITH_ZSTDIO=1`, FreeBSD source archive -SHA256, KLD SHA256, kernel SHA256, and all guest values in the report. - -## Manual evidence rules - -Run the commands in each TC Markdown directly. Do not use a runner, CI job, -or test wrapper. Before each test: - -```sh -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -dmesg | tail -40 > /tmp/g6-dmesg-before -``` - -For a negative mount or read, capture the syscall result with `truss` and -record the named errno, not only command exit status: - -```sh -truss -f -o /tmp/operation.truss command arguments -tail -20 /tmp/operation.truss -``` - -After every TC, unmount first, detach external providers in descending slot -order, detach the primary, and unload the module. All four checks must report -zero: - -```sh -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | \ - awk '/Geom name: md9[0-3]$|Consumers:|Providers:|erofs/ { print }' -``` - -Also compare the new dmesg suffix and reject any panic, trap, assertion, -watchdog, or EROFS error not expected by the current negative operation. - -## FreeBSD and Linux behavior - -Linux EROFS accepts a device table at byte offset zero and excludes a slot -whose `uniaddr` is zero from device-ID-0 unified lookup; a nonzero device ID -still selects that slot. FreeBSD uses explicit `device.=/dev/` -mount options because GEOM providers are not discovered from Linux block -device tags. FreeBSD also holds one read-only GEOM consumer per provider, so -normal `mdconfig -d` returns `EBUSY` while mounted. A forced GEOM orphan makes -later cold I/O return `ENXIO`; unmount must still release vnode, cdev, and GEOM -references. These lifecycle details have no direct Linux loop-device -equivalent and are checked in TC006 and TC118. diff --git a/tests/G7-MANUAL-SETUP.md b/tests/G7-MANUAL-SETUP.md deleted file mode 100644 index 1a1ff2e..0000000 --- a/tests/G7-MANUAL-SETUP.md +++ /dev/null @@ -1,226 +0,0 @@ -# G7 Boundary and Stress Manual Setup - -This setup applies to exactly `TC120` through `TC130`: 11 tests, with no -additional test IDs. Run each test's Markdown commands directly. Do not use a -CI job, runner, or test wrapper. - -## Deterministic fixtures - -The host requires Python 3, erofs-utils 1.8.6, GNU tar, QEMU, and at least -2 GiB of free working space. Generate two fresh fixture directories; the -helper rejects an existing output path. - -```sh -REPO=/path/to/worktree/repo-community/repo22 -RUN=/work/build/repo22-g7-$(date -u +%Y%m%dT%H%M%SZ) -mkdir -p "$RUN" -cd "$REPO" -python3 -B tests/g7_fixtures.py create --output "$RUN/fixtures-a" -python3 -B tests/g7_fixtures.py create --output "$RUN/fixtures-b" -python3 -B tests/g7_fixtures.py verify --output "$RUN/fixtures-a" -cmp "$RUN/fixtures-a/SOURCE-INVENTORY.tsv" \ - "$RUN/fixtures-b/SOURCE-INVENTORY.tsv" -cmp "$RUN/fixtures-a/SOURCE-SHA256SUMS" \ - "$RUN/fixtures-b/SOURCE-SHA256SUMS" -cmp "$RUN/fixtures-a/SHA256SUMS" "$RUN/fixtures-b/SHA256SUMS" -sha256sum "$RUN/fixtures-a/SOURCE-INVENTORY.tsv" \ - "$RUN/fixtures-a/SOURCE-SHA256SUMS" \ - "$RUN/fixtures-a/SHA256SUMS" \ - "$RUN/fixtures-a/fixture-manifest.json" -``` - -`SOURCE-INVENTORY.tsv` records the exact relative path, size, and SHA256 of -every source file. The helper re-hashes the complete inventory, checks exact -counts and names, checks all sparse markers and allocated-block usage, runs -`fsck.erofs`, reopens the 4 GiB boundary inode with `dump.erofs`, and verifies -the image checksums. `boundaries.erofs` covers TC120-TC125; -`workloads.erofs` covers TC126-TC130. - -The practical sparse boundary is 4 GiB + 4097 bytes. It crosses signed 32-bit, -2 GiB, unsigned 32-bit, 4 GiB, block, hole, and EOF boundaries without -claiming that a 16 TiB image is practical in this VM. The source remains -sparse and every selected range is compared to that exact source in TC121. - -## Dedicated FreeBSD 15 VM - -Create a new qcow2 overlay and use only SSH port 9227: - -```sh -qemu-img create -f qcow2 -F qcow2 \ - -b /work/build/vm-freebsd-dev-base.qcow2 \ - "$RUN/freebsd15-overlay.qcow2" -qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 \ - -m 6144 -smp 4 \ - -drive file="$RUN/freebsd15-overlay.qcow2",if=virtio,format=qcow2 \ - -netdev user,id=net0,hostfwd=tcp:127.0.0.1:9227-:22 \ - -device virtio-net-pci,netdev=net0 -display none \ - -serial file:"$RUN/freebsd15-serial.log" -monitor none \ - -pidfile "$RUN/freebsd15-qemu.pid" \ - -D "$RUN/freebsd15-qemu.log" -daemonize -``` - -Record the initial guest identity and resource-control state: - -```sh -uname -a -freebsd-version -ku -sysctl -n kern.osreldate -sha256 /boot/kernel/kernel -sysctl kern.racct.enable -rctl -vmstat -H 1 3 -mdconfig -l -mount -p | awk '$3 == "erofs"' -``` - -If RACCT/RCTL is disabled, enable the FreeBSD loader tunable in this overlay -and reboot it. Do not use a jail as a memory-limit command. - -```sh -grep -q '^kern.racct.enable=' /boot/loader.conf || \ - printf 'kern.racct.enable="1"\n' >> /boot/loader.conf -grep '^kern.racct.enable=' /boot/loader.conf -shutdown -r now -``` - -After reconnecting, require `sysctl -n kern.racct.enable` to print `1` and -record `rctl` plus `vmstat -H 1 3` again. - -## Exact-source KLD and guest inputs - -Commit the helper and corrected Markdown before building. Then archive the -exact commit and the tracked FreeBSD 15 sys tree. The final report may be a -later documentation-only commit, but its `repo22/src` tree hash must equal the -build input tree hash. - -```sh -cd /path/to/worktree -BUILD_COMMIT=$(git rev-parse HEAD) -git status --short -printf '%s\n' "$BUILD_COMMIT" > "$RUN/source.commit" -git rev-parse "$BUILD_COMMIT:repo-community/repo22/src" \ - > "$RUN/repo22-src.tree" -git archive --format=tar "$BUILD_COMMIT" repo-community/repo22 | \ - gzip -n > "$RUN/repo22-source.tar.gz" -git archive --format=tar "$BUILD_COMMIT" dev-freebsd-releng/sys | \ - gzip -n > "$RUN/freebsd-sys-source.tar.gz" -tar --sparse --format=gnu -C "$RUN/fixtures-a" \ - -cf "$RUN/boundaries-source.tar" sources/boundaries -tar --sparse --format=gnu -C "$RUN/fixtures-a" \ - -cf "$RUN/workloads-source.tar" sources/workloads -gzip -n "$RUN/boundaries-source.tar" -gzip -n "$RUN/workloads-source.tar" -sha256sum "$RUN/repo22-source.tar.gz" \ - "$RUN/freebsd-sys-source.tar.gz" \ - "$RUN/boundaries-source.tar.gz" \ - "$RUN/workloads-source.tar.gz" > "$RUN/source-archives.sha256" -``` - -Transfer the archives, images, and fixture metadata to the dedicated guest. -Authentication configuration stays outside the repository. - -```sh -ssh -p 9227 root@127.0.0.1 'mkdir -p /root/repo22-g7-transfer' -scp -O -P 9227 "$RUN/repo22-source.tar.gz" \ - "$RUN/freebsd-sys-source.tar.gz" \ - "$RUN/boundaries-source.tar.gz" "$RUN/workloads-source.tar.gz" \ - "$RUN/fixtures-a/images/boundaries.erofs" \ - "$RUN/fixtures-a/images/workloads.erofs" \ - "$RUN/fixtures-a/SOURCE-INVENTORY.tsv" \ - "$RUN/fixtures-a/SOURCE-SHA256SUMS" \ - "$RUN/fixtures-a/SHA256SUMS" \ - "$RUN/fixtures-a/fixture-manifest.json" \ - "$RUN/fixtures-a/DEEP-PATH" "$RUN/fixtures-a/LONG-NAME" \ - "$RUN/fixtures-a/SPARSE-RANGES.tsv" \ - "$RUN/source.commit" "$RUN/repo22-src.tree" \ - "$RUN/source-archives.sha256" \ - root@127.0.0.1:/root/repo22-g7-transfer/ -``` - -Build and verify natively in the guest: - -```sh -mkdir -p /root/freebsd-src /root/repo22-g7-src \ - /root/repo22-g7/fixtures/images /root/repo22-g7/fixtures/sources \ - /root/repo22-g7/evidence -tar -xzf /root/repo22-g7-transfer/freebsd-sys-source.tar.gz \ - -C /root/freebsd-src --strip-components 1 -tar -xzf /root/repo22-g7-transfer/repo22-source.tar.gz \ - -C /root/repo22-g7-src --strip-components 2 -tar -xzf /root/repo22-g7-transfer/boundaries-source.tar.gz \ - -C /root/repo22-g7/fixtures -tar -xzf /root/repo22-g7-transfer/workloads-source.tar.gz \ - -C /root/repo22-g7/fixtures -cp /root/repo22-g7-transfer/*.erofs /root/repo22-g7/fixtures/images/ -cp /root/repo22-g7-transfer/SOURCE-INVENTORY.tsv \ - /root/repo22-g7-transfer/SOURCE-SHA256SUMS \ - /root/repo22-g7-transfer/SHA256SUMS \ - /root/repo22-g7-transfer/fixture-manifest.json \ - /root/repo22-g7-transfer/DEEP-PATH \ - /root/repo22-g7-transfer/LONG-NAME \ - /root/repo22-g7-transfer/SPARSE-RANGES.tsv \ - /root/repo22-g7/fixtures/ -cd /root/repo22-g7/fixtures -sha256sum -c SHA256SUMS -sha256sum -c SOURCE-SHA256SUMS -stat -f 'size=%z blocks=%b blocksize=%k name=%N' \ - sources/boundaries/maximum/sparse-boundary.bin -cd /root/repo22-g7-src -grep -E '^(REVISION|BRANCH)=' /root/freebsd-src/sys/conf/newvers.sh -env WITH_ZSTDIO=1 FREEBSD_SRC=/root/freebsd-src ./build.sh -cp build/erofs.ko /root/repo22-g7/erofs.ko -cc -std=c11 -O2 -Wall -Wextra -Werror \ - -o /root/repo22-g7/g7_probe tests/g7_probe.c -sha256 /root/repo22-g7/erofs.ko /root/repo22-g7/g7_probe -file /root/repo22-g7/erofs.ko -``` - -Record the build commit, source tree hash, `WITH_ZSTDIO=1`, archive hashes, -FreeBSD source `REVISION`/`BRANCH`, KLD SHA256, probe SHA256, guest identity, -kernel SHA256, and TCG/QEMU configuration in the report. - -## Manual lifecycle - -Load the exact module by full path and record its file ID and pathname: - -```sh -kldload /root/repo22-g7/erofs.ko -kldstat -v | tee /root/repo22-g7/evidence/kldstat-loaded.txt -KLD_ID=$(kldstat -v | awk '$NF == "(/root/repo22-g7/erofs.ko)" { print $1 }') -test -n "$KLD_ID" -printf '%s\n' "$KLD_ID" > /root/repo22-g7/evidence/kld.id -dmesg > /root/repo22-g7/evidence/dmesg-before.txt -``` - -Each TC attaches one fresh vnode-backed md provider, mounts read-only, records -all correctness and metrics evidence, unmounts, detaches that exact provider, -and verifies both are gone. Never use `killall`; concurrent tests persist -every child PID, wait every child, and record every exit code. - -TC128 and TC129 metrics are recording-only under QEMU TCG. Their PASS gate is -read completion and exact source hash/byte comparison, not a fixed MB/s, -IOPS, or latency threshold. TC127 uses FreeBSD RCTL `vmemoryuse:deny` with -cooperating allocation probes, records allocation failure and reclamation, -and verifies reads while pressure is held. - -## Final cleanup - -After TC130, require zero child processes, EROFS mounts, md providers, and -RCTL rules created by G7. Unload the exact KLD file ID, compare dmesg, and -power off the dedicated guest. - -```sh -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -rctl -kldstat -v | grep -A2 -B2 '/root/repo22-g7/erofs.ko' -KLD_ID=$(cat /root/repo22-g7/evidence/kld.id) -kldunload -i "$KLD_ID" -test -z "$(kldstat -v | grep '/root/repo22-g7/erofs.ko')" -dmesg > /root/repo22-g7/evidence/dmesg-after.txt -shutdown -p now -``` - -Remove the host overlay, QEMU logs/PID file, source tars, temporary sparse -probe data, and Python bytecode caches only after evidence has been copied -into the committed report. Confirm port 9227 is released. diff --git a/tests/RESULT-SUMMARY-TEMPLATE.md b/tests/RESULT-SUMMARY-TEMPLATE.md deleted file mode 100644 index cee0745..0000000 --- a/tests/RESULT-SUMMARY-TEMPLATE.md +++ /dev/null @@ -1,229 +0,0 @@ -# Test Result Summary Template - -## Test Execution Information -- **Date**: YYYY-MM-DD -- **Tester**: Name -- **FreeBSD Version**: 13.x / 14.x -- **Kernel**: uname -a output -- **erofs Module Version**: kldstat output -- **Test Duration**: X hours Y minutes - -## Overall Statistics -``` -Total Test Cases: 153 executable cases (plus TC000 template) -Executed: ___ -Passed: ___ -Failed: ___ -Skipped: ___ -Blocked: ___ - -Pass Rate: ___% -``` - -## Phase Results - -### Phase 1: Basic Functionality (15 tests) -``` -Passed: ___/15 -Failed: ___ -Critical Issues: ___ -``` -Failed Tests: -- TC###: Description - Reason - -### Phase 2: Inode & Data Layouts (21 tests) -``` -Passed: ___/21 -Failed: ___ -Critical Issues: ___ -``` -Failed Tests: -- TC###: Description - Reason - -### Phase 3: Directory & VFS (26 tests) -``` -Passed: ___/26 -Failed: ___ -Critical Issues: ___ -``` -Failed Tests: -- TC###: Description - Reason - -### Phase 4: Extended Attributes (17 tests) -``` -Passed: ___/17 -Failed: ___ -Critical Issues: ___ -``` -Failed Tests: -- TC###: Description - Reason - -### Phase 5: Compression (24 tests) -``` -Passed: ___/24 -Failed: ___ -Critical Issues: ___ -``` - -**LZMA Tests** (repo18 NEW): -- TC004: [PASS/FAIL] -- TC108: [PASS/FAIL] -- TC109: [PASS/FAIL] -- TC110: [PASS/FAIL] - -Failed Tests: -- TC###: Description - Reason - -### Phase 6: Multi-Device (9 tests) -``` -Passed: ___/9 -Failed: ___ -Critical Issues: ___ -``` -Failed Tests: -- TC###: Description - Reason - -### Phase 7: Error Handling (8 tests) -``` -Passed: ___/8 -Failed: ___ -Critical Issues: ___ -``` -Failed Tests: -- TC###: Description - Reason - -### Phase 8: Boundary & Stress (11 tests) -``` -Passed: ___/11 -Failed: ___ -Critical Issues: ___ -``` - -**Performance Results**: -- TC128 (Sequential): ___ MB/s -- TC129 (Random): ___ IOPS -- TC130 (Mixed): ___ req/s - -Failed Tests: -- TC###: Description - Reason - -### Phase 9: Documentation (1 test) -``` -Passed: ___/1 -Failed: ___ -``` - -## Critical Failures -Priority: Critical failures that prevent basic functionality - -| Test ID | Description | Root Cause | Impact | Status | -|---------|-------------|------------|--------|--------| -| TC### | Brief | Reason | High/Medium/Low | Open/Fixed | - -## Known Issues -Non-critical issues or limitations - -| Test ID | Description | Workaround | Severity | Tracked | -|---------|-------------|------------|----------|---------| -| TC### | Brief | Workaround if any | Medium/Low | Issue #X | - -## Performance Summary -| Metric | Result | Baseline | Delta | Status | -|--------|--------|----------|-------|--------| -| Sequential Read | ___ MB/s | 500 MB/s | ___% | PASS/FAIL | -| Random Read | ___ IOPS | 5000 IOPS | ___% | PASS/FAIL | -| Large File (10GB) | ___ s | 20 s | ___% | PASS/FAIL | -| Many Files (10K) | ___ s | 30 s | ___% | PASS/FAIL | - -## Feature Coverage -``` -Total Features: 65 -Tested: ___ -Passed: ___ -Failed: ___ - -Coverage: ___% -``` - -### Feature Status -- [x] Mount operations: PASS -- [x] Inode formats: PASS/FAIL -- [x] Data layouts: PASS/FAIL -- [x] LZ4 compression: PASS/FAIL -- [x] DEFLATE compression: PASS/FAIL -- [x] zstd compression: PASS/FAIL -- [x] **LZMA compression (NEW)**: PASS/FAIL -- [x] Extended attributes: PASS/FAIL -- [x] Multi-device: PASS/FAIL -- [x] Directory operations: PASS/FAIL -- [x] VFS integration: PASS/FAIL - -## Regression Check -Comparison with repo17 (64 features, 91% complete) - -| Feature | repo17 | repo18 | Status | -|---------|--------|--------|--------| -| Basic mount | PASS | PASS | No regression | -| LZ4 | PASS | PASS | No regression | -| DEFLATE | PASS | PASS | No regression | -| zstd | PASS | PASS | No regression | -| **LZMA** | NOT IMPL | PASS | **NEW** | -| ... | ... | ... | ... | - -## repo18 Iteration 1 Validation -**Goal**: LZMA compression support - -### LZMA-Specific Results -- TC004 (Basic LZMA): [PASS/FAIL] -- TC108 (Large file): [PASS/FAIL] -- TC109 (MicroLZMA): [PASS/FAIL] -- TC110 (Corrupted): [PASS/FAIL] - -**Data Integrity**: [PASS/FAIL] -- SHA256 checksums match: [YES/NO] -- Random access works: [YES/NO] -- Large file (100MB+) decompressed correctly: [YES/NO] - -**Implementation Validation**: -- Self-contained decoder (232 lines): [YES/NO] -- No external dependencies: [YES/NO] -- Unified interface: [YES/NO] - -**Iteration 1 Status**: [COMPLETE/INCOMPLETE] - -## System Stability -- Kernel panics: [YES/NO] - Count: ___ -- Memory leaks: [DETECTED/NONE] -- File descriptor leaks: [DETECTED/NONE] -- dmesg errors: [COUNT] - -## Test Environment -``` -CPU: ___ -Memory: ___ GB -Disk: ___ (type) -Load during tests: ___ -``` - -## Recommendations -1. [Action item 1] -2. [Action item 2] -3. ... - -## Sign-Off -- [ ] All critical tests passed -- [ ] No regressions from repo17 -- [ ] LZMA implementation validated (repo18 iter 1) -- [ ] Performance acceptable -- [ ] Known issues documented - -**Tested by**: _______________ -**Reviewed by**: _______________ -**Date**: _______________ -**Approved**: [YES/NO] - -## Attachments -- [ ] Full test logs -- [ ] dmesg output -- [ ] Performance graphs -- [ ] Failure screenshots/dumps diff --git a/tests/TC000-template.md b/tests/TC000-template.md deleted file mode 100644 index ebacc1e..0000000 --- a/tests/TC000-template.md +++ /dev/null @@ -1,36 +0,0 @@ -# Test Case Template - -**Test ID**: TC###-brief-name -**Category**: [Mount Operations | Filesystem Metadata | Inode Operations | Data Layouts | Compression | Directory Operations | Symlink Operations | Extended Attributes | Multi-Device | VFS Integration | Read-Only Enforcement | VM Integration | Error Handling | Performance & Stress] -**Priority**: [Critical | High | Medium | Low] -**Regression**: [None | Issue #XXX] - -## Objective -Brief description of what this test validates. - -## Preconditions -- EROFS image requirements (e.g., "Image with LZ4-compressed files") -- System requirements (e.g., "FreeBSD 13.0+") -- Test data setup - -## Test Steps -1. Step one with specific commands -2. Step two with expected intermediate state -3. ... - -## Expected Results -- Specific expected outcomes -- Expected output values -- Expected file contents - -## Verification Method -How to verify the test passed: -- Command outputs to check -- Files to inspect -- Performance metrics (if applicable) - -## Cleanup -Steps to reset the environment after the test. - -## Notes -Any additional considerations or known limitations. diff --git a/tests/TC001-mount-basic.md b/tests/TC001-mount-basic.md deleted file mode 100644 index 3400ca0..0000000 --- a/tests/TC001-mount-basic.md +++ /dev/null @@ -1,39 +0,0 @@ -# Test Case: Basic Mount and Unmount - -**Test ID**: TC001-mount-basic - -## Objective - -Verify that the exact G1 KLD mounts a valid image read-only, exposes source -data exactly, and unmounts cleanly. - -## Fixture - -Use `images/compact.erofs` and `source/compact/root.txt` generated by -`tests/prepare_g1_fixtures.sh`. Follow `tests/G1-MANUAL-SETUP.md` and record the -image, source, and KLD SHA256 values. - -## Procedure - -```sh -image=/tmp/repo22-g1/images/compact.erofs -source=/tmp/repo22-g1/source/compact/root.txt -mnt=/mnt/repo22-g1-001 -md=$(mdconfig -a -t vnode -f "$image") -mkdir -p "$mnt" -kldstat | grep -i erofs -mount -t erofs -o ro "/dev/$md" "$mnt" -mount -p | awk -v p="$mnt" '$2 == p' -cmp "$source" "$mnt/root.txt" -sha256 "$source" "$mnt/root.txt" -./statfs_probe "$mnt" -umount "$mnt" -mdconfig -d -u "${md#md}" -rmdir "$mnt" -``` - -## Expected Results - -- Mount, `cmp`, `statfs_probe`, unmount, and detach all return zero. -- The mount entry is `erofs` and read-only; both file hashes are identical. -- Cleanup leaves neither this mount point nor its md provider. diff --git a/tests/TC002-superblock-crc32c.md b/tests/TC002-superblock-crc32c.md deleted file mode 100644 index 8352760..0000000 --- a/tests/TC002-superblock-crc32c.md +++ /dev/null @@ -1,66 +0,0 @@ -# Test Case: Superblock CRC32C Validation - -**Test ID**: TC002-superblock-crc32c -**Category**: Filesystem Metadata -**Priority**: Critical - -## Objective - -Verify the real EROFS superblock checksum field and both equivalent checksum -ranges, then prove that a valid image mounts while an equal-length image with -one covered byte changed and no checksum update is rejected. - -## Fixture Qualification - -Run from the repo22 root with erofs-utils 1.8.6: - -```sh -work=$(mktemp -d /tmp/repo22-tc002.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -python3 tests/erofs_fixture.py inspect "$work/fixtures/valid-plain.erofs" -python3 tests/erofs_fixture.py inspect "$work/fixtures/bad-super-crc.erofs" -grep '^bad-super-crc ' "$work/fixtures/fixture-evidence.txt" -(cd "$work/fixtures" && sha256 -c SHA256SUMS) -``` - -For a 4096-byte block, the canonical calculation clears the little-endian -checksum at absolute byte 1028 and calculates CRC32C over bytes `[1024,4096)` -with initial value `0xffffffff`. The production verifier uses seed -`0x5045b54a` over `[1032,4096)`. The helper must print identical canonical and -kernel values, `equivalent=True`, and `valid=True` for the control. The bad -image changes byte 1088, preserves provider length, does not recompute CRC, and -must print `valid=False`. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode -f "$work/fixtures/valid-plain.erofs") -mount -t erofs -o ro "/dev/$unit" "$work/mnt" -cmp "$work/fixtures/source/control.txt" "$work/mnt/control.txt" -umount "$work/mnt" -mdconfig -d -u "${unit#md}" - -unit=$(mdconfig -a -t vnode -f "$work/fixtures/bad-super-crc.erofs") -set +e -mount -t erofs -o ro "/dev/$unit" "$work/mnt" \ - >"$work/bad-mount.out" 2>"$work/bad-mount.err" -mount_status=$? -set -e -printf 'mount_status=%s\n' "$mount_status" -test "$mount_status" -ne 0 -set +e -truss -o "$work/bad-mount.truss" mount -t erofs -o ro "/dev/$unit" \ - "$work/mnt" >/dev/null 2>&1 -set -e -grep -E 'nmount.*ERR#97' "$work/bad-mount.truss" -! mount -p | awk -v path="$work/mnt" '$2 == path { found=1 } - END { exit found ? 0 : 1 }' -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -The control mounts and reads exactly. The bad image fails with FreeBSD errno -97 (`EINTEGRITY`) and no mount remains. Record both image hashes, provider -lengths, checksum values/ranges, command status, new dmesg lines, and cleanup. diff --git a/tests/TC003-lz4-compressed-read.md b/tests/TC003-lz4-compressed-read.md deleted file mode 100644 index 95b5ae2..0000000 --- a/tests/TC003-lz4-compressed-read.md +++ /dev/null @@ -1,87 +0,0 @@ -# Test Case: LZ4 Compressed File Read - -**Test ID**: TC003-lz4-compressed-read - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: Critical -**Regression**: None - -## Objective -Verify correct decompression and reading of LZ4-compressed files. - -## Preconditions -- EROFS image with LZ4-compressed files: `test-lz4.erofs` -- Known reference file: `original.txt` (uncompressed source) -- Compressed file in image: `/compressed/test.txt` -- Mount point: `/mnt/test` - -## Test Steps -1. Mount the LZ4 test image: - ``` - mount -t erofs /dev/md0 /mnt/test - ``` - -2. Verify file exists: - ``` - ls -lh /mnt/test/compressed/test.txt - ``` - -3. Read the entire file: - ``` - cat /mnt/test/compressed/test.txt > /tmp/decompressed.txt - ``` - -4. Calculate checksum: - ``` - sha256 /tmp/decompressed.txt - sha256 original.txt - ``` - -5. Compare byte-for-byte: - ``` - cmp /tmp/decompressed.txt original.txt - ``` - -6. Test partial read: - ``` - dd if=/mnt/test/compressed/test.txt of=/tmp/partial.txt bs=1024 count=1 - ``` - -7. Verify partial read matches: - ``` - cmp -n 1024 /tmp/partial.txt original.txt - ``` - -## Expected Results -- Step 1: Mount succeeds -- Step 2: File size matches original uncompressed size -- Step 4: SHA256 checksums are identical -- Step 5: `cmp` returns 0 (files identical) -- Step 7: First 1024 bytes match - -## Verification Method -- Exact byte-for-byte match with original file -- No corruption in decompressed data -- File attributes (size, timestamps) preserved correctly -- Check inode compression format: - ``` - # Using custom debug tool if available - erofs_inspect /dev/md0 /compressed/test.txt - # Should show: z_algorithmformat[0:2] = Z_EROFS_COMPRESSION_LZ4 - ``` - -## Cleanup -``` -umount /mnt/test -mdconfig -d -u md0 -rm /tmp/decompressed.txt /tmp/partial.txt -``` - -## Notes -- LZ4 is the most common compression algorithm in EROFS -- Test should cover files of various sizes: small (<4KB), medium (4KB-1MB), large (>1MB) -- LZ4 pcluster size is typically 4KB or 64KB -- Related: TC004 (ztailpacking), TC005 (pcluster mapping) diff --git a/tests/TC004-lzma-compressed-read.md b/tests/TC004-lzma-compressed-read.md deleted file mode 100644 index d076211..0000000 --- a/tests/TC004-lzma-compressed-read.md +++ /dev/null @@ -1,96 +0,0 @@ -# Test Case: LZMA Compressed File Read - -**Test ID**: TC004-lzma-compressed-read - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify correct decompression and reading of LZMA/MicroLZMA-compressed files (newly implemented in repo18 iteration 1). - -## Preconditions -- EROFS image with LZMA-compressed files: `test-lzma.erofs` -- Known reference file: `original-large.txt` (uncompressed source) -- Compressed file in image: `/compressed/large.txt` -- Mount point: `/mnt/test` - -## Test Steps -1. Mount the LZMA test image: - ```sh - unit=$(mdconfig -a -t vnode -f test-lzma.erofs) - mount -t erofs -o ro /dev/${unit} /mnt/test - ``` - -2. Verify file exists and check size: - ``` - ls -lh /mnt/test/compressed/large.txt - stat -f "Size: %z bytes" /mnt/test/compressed/large.txt - ``` - -3. Read entire compressed file: - ``` - cat /mnt/test/compressed/large.txt > /tmp/lzma-decompressed.txt - ``` - -4. Verify decompression correctness: - ``` - sha256 /tmp/lzma-decompressed.txt - sha256 original-large.txt - ``` - -5. Compare byte-for-byte: - ``` - cmp /tmp/lzma-decompressed.txt original-large.txt - echo $? - ``` - -6. Test random access read: - ``` - dd if=/mnt/test/compressed/large.txt of=/tmp/lzma-middle.txt bs=1k skip=100 count=10 - dd if=original-large.txt of=/tmp/orig-middle.txt bs=1k skip=100 count=10 - cmp /tmp/lzma-middle.txt /tmp/orig-middle.txt - ``` - -7. Test end-of-file read: - ``` - tail -c 1000 /mnt/test/compressed/large.txt > /tmp/lzma-tail.txt - tail -c 1000 original-large.txt > /tmp/orig-tail.txt - cmp /tmp/lzma-tail.txt /tmp/orig-tail.txt - ``` - -## Expected Results -- Step 1: Mount succeeds -- Step 2: File size matches original uncompressed size exactly -- Step 4: SHA256 checksums are identical -- Step 5: Exit code 0 (files identical) -- Step 6: Middle section matches (random access works) -- Step 7: Tail matches (EOF handling correct) - -## Verification Method -- Complete data integrity check via SHA256 -- Partial read correctness (random access) -- No memory corruption or crashes during decompression -- Verify LZMA decoder internal state: - - 1846 probability models initialized - - Range decoder normalization correct - - Dictionary buffer within bounds - -## Cleanup -``` -umount /mnt/test -mdconfig -d -u "${unit}" -rm /tmp/lzma-*.txt /tmp/orig-*.txt -``` - -## Notes -- **NEW in repo18**: This is the first iteration with LZMA support -- LZMA provides maximum compression ratio (~30-50% smaller than LZ4) -- MicroLZMA is a variant without header, commonly used in EROFS -- Self-contained implementation in `src/decompressor_lzma.c` -- Reference implementation: XZ Embedded minimal decoder -- Critical test for repo18 iteration 1 validation -- Regression marker: First LZMA implementation, high priority for validation diff --git a/tests/TC005-inline-xattr-user.md b/tests/TC005-inline-xattr-user.md deleted file mode 100644 index adac5c4..0000000 --- a/tests/TC005-inline-xattr-user.md +++ /dev/null @@ -1,32 +0,0 @@ -# Test Case: Inline user xattrs - -**Test ID**: TC005-inline-xattr-user -**Fixture**: `basic.erofs`, `/inline-user` - -## Objective - -Verify inline `user` namespace list/get and binary-value handling on FreeBSD. -Generate and qualify the image as described in `G4-MANUAL-SETUP.md`; the -manifest must classify `comment` and `special.chars` as inline entries. - -## Manual steps - -```sh -unit=$(mdconfig -a -t vnode -f /tmp/repo22-g4/images/basic.erofs) -mount -t erofs -o ro /dev/${unit} /mnt/repo22-g4 -stat -f 'mode=%Sp size=%z inode=%i' /mnt/repo22-g4/inline-user -lsextattr user /mnt/repo22-g4/inline-user -getextattr -qq -x user comment /mnt/repo22-g4/inline-user -getextattr -qq -x user special.chars /mnt/repo22-g4/inline-user -truss -o /tmp/tc005.truss getextattr -qq user missing /mnt/repo22-g4/inline-user -grep extattr_get_file /tmp/tc005.truss -umount /mnt/repo22-g4 -mdconfig -d -u "${unit#md}" -``` - -## Expected results - -The names appear once. `comment` is hex -`696e6c696e652d757365722d76616c7565007461696c`; `special.chars` is -`7370656369616c2d76616c7565`. The missing name returns `ENOATTR` (87), and -cleanup leaves no mount or md provider. diff --git a/tests/TC006-multidev-chunk-read.md b/tests/TC006-multidev-chunk-read.md deleted file mode 100644 index e619016..0000000 --- a/tests/TC006-multidev-chunk-read.md +++ /dev/null @@ -1,61 +0,0 @@ -# Test Case: Real Multi-Device Chunk Read - -**Test ID**: TC006-multidev-chunk-read -**Category**: Multi-Device / Chunk-Based -**Priority**: Critical - -## Objective - -Prove that 8-byte chunk indexes read file data from a live external GEOM -provider, including a range crossing a chunk boundary. - -## Fixture - -Generate and verify the fresh G6 set as described in `G6-MANUAL-SETUP.md`. -Use `mkfs-blob-primary.erofs` and `mkfs-blob-slot1.blob`. The manifest must -show primary blocks `1`, slot-1 blocks `224`, and every `/tc006.bin` index as -device ID `1`. The 4096-byte primary cannot contain the 98304-byte source. - -## Manual procedure - -```sh -I=/root/repo22-g6/images -S=/root/repo22-g6/sources/chunk -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/mkfs-blob-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/mkfs-blob-slot1.blob" -u 91 -mount -t erofs -o ro -o device.1=/dev/md91 /dev/md90 /mnt/g6 -sha256 "$S/tc006.bin" /mnt/g6/tc006.bin -cmp "$S/tc006.bin" /mnt/g6/tc006.bin -dd if="$S/tc006.bin" of=/tmp/tc006.expected bs=1 skip=28672 count=8192 2>/dev/null -dd if=/mnt/g6/tc006.bin of=/tmp/tc006.actual bs=1 skip=28672 count=8192 2>/dev/null -cmp /tmp/tc006.expected /tmp/tc006.actual -truss -f -o /tmp/tc006-ebusy.truss mdconfig -d -u 91 -tail -20 /tmp/tc006-ebusy.truss -sysctl -n kern.geom.conftxt | grep -A8 -B2 'Geom name: md91' -``` - -The detach must fail with `EBUSY`, proving a live external consumer. - -## Expected results - -- Mount, full-file `cmp`, SHA256, and the 8192-byte cross-boundary range pass. -- The external detach returns exactly `EBUSY` while mounted. -- No panic, trap, hang, or unexpected EROFS dmesg message occurs. - -## Cleanup and zero-state check - -```sh -umount /mnt/g6 -mdconfig -d -u 91 -mdconfig -d -u 90 -kldunload erofs -rm -f /tmp/tc006.expected /tmp/tc006.actual -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[01]|erofs' || true -``` - -All four final outputs must be empty. diff --git a/tests/TC007-concurrent-mount.md b/tests/TC007-concurrent-mount.md deleted file mode 100644 index 4697b3a..0000000 --- a/tests/TC007-concurrent-mount.md +++ /dev/null @@ -1,48 +0,0 @@ -# Test Case: Concurrent Mount Operations - -**Test ID**: TC007-concurrent-mount - -## Objective - -Verify two simultaneous mounts of the same deterministic EROFS image through -independent md providers, including source-exact reads from both mounts. - -## Fixture - -Use `images/compact.erofs` and `source/compact/testfile.txt` from the G1 -fixture set. Follow `tests/G1-MANUAL-SETUP.md`. - -## Procedure - -```sh -image=/tmp/repo22-g1/images/compact.erofs -source=/tmp/repo22-g1/source/compact/testfile.txt -mnt1=/mnt/repo22-g1-007a -mnt2=/mnt/repo22-g1-007b -md1=$(mdconfig -a -t vnode -f "$image") -md2=$(mdconfig -a -t vnode -f "$image") -mkdir -p "$mnt1" "$mnt2" -set +e -mount -t erofs -o ro "/dev/$md1" "$mnt1" & p1=$! -mount -t erofs -o ro "/dev/$md2" "$mnt2" & p2=$! -wait "$p1"; rc1=$? -wait "$p2"; rc2=$? -set -e -printf 'mount_rc=%d,%d providers=%s,%s\n' "$rc1" "$rc2" "$md1" "$md2" -test "$rc1" -eq 0 -a "$rc2" -eq 0 -cmp "$source" "$mnt1/testfile.txt" -cmp "$source" "$mnt2/testfile.txt" -cmp "$mnt1/testfile.txt" "$mnt2/testfile.txt" -sha256 "$source" "$mnt1/testfile.txt" "$mnt2/testfile.txt" -umount "$mnt1" -umount "$mnt2" -mdconfig -d -u "${md1#md}" -mdconfig -d -u "${md2#md}" -rmdir "$mnt1" "$mnt2" -``` - -## Expected Results - -- Both independently waited mount processes return zero. -- All three comparisons and SHA256 values match. -- No panic, trap, EROFS diagnostic, mount, or md provider remains. diff --git a/tests/TC008-mount-errors.md b/tests/TC008-mount-errors.md deleted file mode 100644 index 9febfae..0000000 --- a/tests/TC008-mount-errors.md +++ /dev/null @@ -1,77 +0,0 @@ -# Test Case: Mount Error Conditions - -**Test ID**: TC008-mount-errors -**Category**: Mount Operations -**Priority**: High - -## Objective - -Verify graceful rejection of three distinct provider conditions: a -checksum-valid bad magic at the correct field, a 1023-byte provider ending -before the superblock, and a zero-length provider. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc008.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -grep -E '^(bad-magic|truncated-before-super|empty-provider) ' \ - "$work/fixtures/fixture-evidence.txt" -python3 tests/erofs_fixture.py inspect "$work/fixtures/bad-magic.erofs" -wc -c "$work/fixtures/bad-magic.erofs" \ - "$work/fixtures/truncated-before-super.erofs" \ - "$work/fixtures/empty-provider.erofs" -(cd "$work/fixtures" && sha256 -c SHA256SUMS) -``` - -The bad-magic variant changes only absolute bytes 1024 through 1027 and -preserves media size. It intentionally retains the original checksum: the -production verifier's fixed-magic suffix calculation remains valid, while the -canonical checksum over the now-invalid magic does not. Magic is rejected -before checksum verification. The short and empty fixtures are intentional -provider-size cases, not same-size corruption variants. - -## FreeBSD Procedure - -Create a fresh mount directory. Execute each case separately and record whether -failure occurs at `mdconfig` or `nmount`; never assume a fixed md number. - -```sh -mkdir "$work/mnt" -for image in bad-magic.erofs truncated-before-super.erofs empty-provider.erofs -do - set +e - unit=$(mdconfig -a -t vnode -f "$work/fixtures/$image" \ - 2>"$work/$image.md.err") - md_status=$? - set -e - printf '%s md_status=%s unit=%s\n' "$image" "$md_status" "$unit" - if test "$md_status" -eq 0; then - set +e - mount -t erofs -o ro "/dev/$unit" "$work/mnt" \ - >"$work/$image.out" 2>"$work/$image.mount.err" - mount_status=$? - set -e - printf '%s mount_status=%s\n' "$image" "$mount_status" - test "$mount_status" -ne 0 - set +e - truss -o "$work/$image.mount.truss" mount -t erofs -o ro \ - "/dev/$unit" "$work/mnt" >/dev/null 2>&1 - set -e - grep -E 'nmount.*ERR#' "$work/$image.mount.truss" - mdconfig -d -u "${unit#md}" - else - cat "$work/$image.md.err" - fi - ! mount -p | awk -v path="$work/mnt" '$2 == path { found=1 } - END { exit found ? 0 : 1 }' -done -``` - -## Expected Results - -All three cases fail without panic or hang. Bad magic reaches `nmount` and -returns `EINVAL`. A 1023-byte vnode provider reaches `nmount` and returns -`ENXIO`; a zero-byte provider is rejected by `mdconfig` with `EINVAL`. Record -the observed stage and errno rather than fabricating a mount result. No md unit -or mount may remain after each case. diff --git a/tests/TC009-statfs-basic.md b/tests/TC009-statfs-basic.md deleted file mode 100644 index a4219e9..0000000 --- a/tests/TC009-statfs-basic.md +++ /dev/null @@ -1,42 +0,0 @@ -# Test Case: Basic statfs Information - -**Test ID**: TC009-statfs-basic - -## Objective - -Verify FreeBSD `statfs(2)` fields against the qualified image superblock. - -## Fixture - -Use `images/compact.erofs`. Before transfer, record `dump.erofs -s` and the -`compact` superblock row in `fixture-evidence.txt`. Follow -`tests/G1-MANUAL-SETUP.md` and compile `tests/statfs_probe.c` in the guest. - -## Procedure - -Host qualification: - -```sh -dump.erofs -s /work/build/repo22-g1/images/compact.erofs -python3 tests/erofs_fixture.py inspect /work/build/repo22-g1/images/compact.erofs -``` - -Guest dynamic check after mounting the image on `$mnt`: - -```sh -./statfs_probe "$mnt" -df -kT "$mnt" -df -iT "$mnt" -mount -p | awk -v p="$mnt" '$2 == p' -``` - -Unmount and detach the exact dynamic md unit. - -## Expected Results - -- `fstype=erofs`, `bsize=4096`, `iosize=4096`, and `readonly=1`. -- `blocks` and `files` equal the image's declared block and inode counts. -- `bfree`, `bavail`, and `ffree` are zero; `df` conversions agree. -- `mount -p` contains `ro`; cleanup succeeds. - -`stat -f` is not a `statfs(2)` probe and must not be substituted. diff --git a/tests/TC010-statfs-48bit.md b/tests/TC010-statfs-48bit.md deleted file mode 100644 index d2e5875..0000000 --- a/tests/TC010-statfs-48bit.md +++ /dev/null @@ -1,60 +0,0 @@ -# Test Case: statfs with 48-bit Block Count - -**Test ID**: TC010-statfs-48bit -**Category**: Filesystem Metadata -**Priority**: Medium - -## Objective - -Verify a nonzero 48-bit block-count high word survives mount validation and is -exported without truncation through FreeBSD `statfs(2)` and `df(1)`. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc010.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -python3 tests/erofs_fixture.py inspect \ - "$work/fixtures/48bit-statfs-prefix.erofs" -grep '^48bit-statfs-prefix ' "$work/fixtures/fixture-evidence.txt" -sha256 -q "$work/fixtures/48bit-statfs-prefix.erofs" -``` - -Read `total_blocks` and `required_provider_length` from the evidence. The -fixture sets incompat bit `0x80`, `rb.blocks_hi=1`, and a nonzero -`rootnid_8b`, then recomputes CRC32C. Its small prefix hash identifies the -metadata; the qualified provider is that exact prefix followed by sparse zeros -to the required media size. - -## FreeBSD Procedure - -```sh -provider="$work/48bit-statfs-provider.raw" -cp "$work/fixtures/48bit-statfs-prefix.erofs" "$provider" -truncate -s "$required_provider_length" "$provider" -stat -f 'size=%z blocks=%b' "$provider" -unit=$(mdconfig -a -t vnode -f "$provider") -diskinfo -v "/dev/$unit" -mkdir "$work/mnt" -mount -t erofs -o ro "/dev/$unit" "$work/mnt" -df -kT "$work/mnt" -df -iT "$work/mnt" -actual_1k=$(df -k "$work/mnt" | awk 'NR == 2 { print $2 }') -used_1k=$(df -k "$work/mnt" | awk 'NR == 2 { print $3 }') -avail_1k=$(df -k "$work/mnt" | awk 'NR == 2 { print $4 }') -expected_1k=$((total_blocks * 4)) -test "$actual_1k" -eq "$expected_1k" -test "$used_1k" -eq "$expected_1k" -test "$avail_1k" -eq 0 -umount "$work/mnt" -mdconfig -d -u "${unit#md}" -rm -f "$provider" -``` - -## Expected Results - -`diskinfo` reports at least the declared 16 TiB-plus media size. Mount succeeds, -the `df -k` total equals `total_blocks * 4`, Used equals total because -`f_bfree=0`, and Avail is zero. -If the filesystem cannot create or attach the sparse provider, record SHELVED -with all failed commands; a short-provider `ENXIO` is not PASS. diff --git a/tests/TC011-root-vnode.md b/tests/TC011-root-vnode.md deleted file mode 100644 index 7517eec..0000000 --- a/tests/TC011-root-vnode.md +++ /dev/null @@ -1,36 +0,0 @@ -# Test Case: Root Vnode Lookup - -**Test ID**: TC011-root-vnode - -## Objective - -Verify the mounted root vnode has the encoded root NID, directory attributes, -and the same top-level names as the fixture source. - -## Fixture - -Use `images/compact.erofs` and `source/compact`. The structured evidence records -the root NID and compact inode size. Follow `tests/G1-MANUAL-SETUP.md`. - -## Procedure - -After mounting on `$mnt`: - -```sh -stat -f 'ino=%i type=%HT mode=%Lp nlink=%l' "$mnt" -stat -f '%i' "$mnt" > /tmp/tc011-root-nid -ls -la "$mnt" -(cd /tmp/repo22-g1/source/compact && ls -1A | sort) > /tmp/tc011-source-names -(cd "$mnt" && ls -1A | sort) > /tmp/tc011-mount-names -diff -u /tmp/tc011-source-names /tmp/tc011-mount-names -cat "$mnt/root.txt" -``` - -Compare `/tmp/tc011-root-nid` with `root_nid` from `fixture-evidence.txt`, then -unmount, detach, and remove the two temporary name lists. - -## Expected Results - -- Root inode equals the encoded NID and reports a traversable directory. -- The complete top-level name lists match and `root.txt` is readable. -- All operations and cleanup return zero. diff --git a/tests/TC012-vget-normal.md b/tests/TC012-vget-normal.md deleted file mode 100644 index 1c5962a..0000000 --- a/tests/TC012-vget-normal.md +++ /dev/null @@ -1,39 +0,0 @@ -# Test Case: VFS Vget Normal Operation - -**Test ID**: TC012-vget-normal - -## Objective - -Exercise `VFS_VGET` through a real FreeBSD file handle and verify the returned -vnode identifies and reads the same EROFS inode as pathname lookup. - -## Fixture - -Use `images/compact.erofs`, `source/compact/testfile.txt`, and -`tests/nfs_fh_tool.c`. Compile the helper as described in -`tests/G1-MANUAL-SETUP.md`; run as root. - -## Procedure - -After mounting on `$mnt`: - -```sh -stat -f 'path_ino=%i gen=%v size=%z' "$mnt/testfile.txt" -./nfs_fh_tool capture "$mnt/testfile.txt" /tmp/tc012-a.fh -./nfs_fh_tool capture "$mnt/testfile.txt" /tmp/tc012-b.fh -./nfs_fh_tool compare /tmp/tc012-a.fh /tmp/tc012-b.fh -./nfs_fh_tool describe /tmp/tc012-a.fh -./nfs_fh_tool stat /tmp/tc012-a.fh -./nfs_fh_tool cat /tmp/tc012-a.fh /tmp/tc012-fh.out -cmp /tmp/repo22-g1/source/compact/testfile.txt /tmp/tc012-fh.out -sha256 /tmp/repo22-g1/source/compact/testfile.txt /tmp/tc012-fh.out -``` - -Remove the handles/output, then unmount and detach. - -## Expected Results - -- Both captures produce the same handle. -- Handle NID, `fhstat` inode, and pathname inode agree. -- `fhopen` output matches the deterministic source exactly. -- No stale/duplicate vnode error or cleanup failure occurs. diff --git a/tests/TC013-vget-invalid.md b/tests/TC013-vget-invalid.md deleted file mode 100644 index 313ad38..0000000 --- a/tests/TC013-vget-invalid.md +++ /dev/null @@ -1,39 +0,0 @@ -# Test Case: Vget with Invalid NID - -**Test ID**: TC013-vget-invalid - -## Objective - -Verify a checksum-valid directory entry that points outside the image reaches -the cold lookup/vget path and fails consistently with `EINTEGRITY`. - -## Fixture - -Use `images/invalid-dirent-nid.erofs`. `fixture-evidence.txt` records the exact -dirent offset, old NID, out-of-range NID, and valid recomputed CRC. Compile -`tests/read_probe.c` in the guest. - -## Procedure - -Mount the image without first listing the root, then run: - -```sh -./read_probe expect-error "$mnt/invalid-target.txt" 97 -./read_probe expect-error "$mnt/invalid-target.txt" 97 -set +e -truss -o /tmp/tc013.truss stat "$mnt/invalid-target.txt" -rc=$? -set -e -printf 'stat_rc=%d\n' "$rc" -tail -20 /tmp/tc013.truss -``` - -Record dmesg before/after, remove the trace, then unmount and detach. - -## Expected Results - -- Mount succeeds because only the child dirent is malformed. -- Both cold/repeated opens fail at pathname acquisition with FreeBSD errno 97 - (`EINTEGRITY`); `truss` shows the same syscall errno. -- The second failure is not converted to `ENOENT`; no panic or stale vnode - appears, and cleanup succeeds. diff --git a/tests/TC014-superblock-parse.md b/tests/TC014-superblock-parse.md deleted file mode 100644 index 34b0437..0000000 --- a/tests/TC014-superblock-parse.md +++ /dev/null @@ -1,42 +0,0 @@ -# Test Case: Superblock Parsing - -**Test ID**: TC014-superblock-parse - -## Objective - -Verify mandatory superblock fields at their real on-disk offsets and compare -the mounted values exposed by FreeBSD. - -## Fixture - -Use `images/compact.erofs`, its image hash, `fixture-evidence.txt`, and -`tests/statfs_probe.c`. - -## Procedure - -Host inspection: - -```sh -od -An -tx4 -j 1024 -N 4 /work/build/repo22-g1/images/compact.erofs -od -An -tu1 -j 1036 -N 1 /work/build/repo22-g1/images/compact.erofs -od -An -tu2 -j 1038 -N 2 /work/build/repo22-g1/images/compact.erofs -od -An -tu8 -j 1040 -N 8 /work/build/repo22-g1/images/compact.erofs -od -An -tu4 -j 1060 -N 8 /work/build/repo22-g1/images/compact.erofs -dump.erofs -s /work/build/repo22-g1/images/compact.erofs -python3 tests/erofs_fixture.py inspect /work/build/repo22-g1/images/compact.erofs -``` - -Mount the same hashed image in FreeBSD, then run: - -```sh -./statfs_probe "$mnt" -stat -f 'root_ino=%i type=%HT' "$mnt" -mount -p | awk -v p="$mnt" '$2 == p' -``` - -## Expected Results - -- Magic is `0xe0f5e1e2`, `blkszbits=12`, and checksum validation is true. -- Root NID, block count, and inode count agree with root `stat` and - `statfs_probe`; filesystem type is `erofs` and read-only. -- No parser error appears in the dmesg delta; cleanup succeeds. diff --git a/tests/TC015-superblock-corrupted.md b/tests/TC015-superblock-corrupted.md deleted file mode 100644 index 82f11c5..0000000 --- a/tests/TC015-superblock-corrupted.md +++ /dev/null @@ -1,63 +0,0 @@ -# Test Case: Corrupted Superblock Fields - -**Test ID**: TC015-superblock-corrupted -**Category**: Error Handling -**Priority**: High - -## Objective - -Verify ordered rejection of an unsupported block size, an image-bounds-invalid -48-bit root NID, and an unknown incompat feature. Every variant preserves -provider length and carries a checksum valid for its encoded fields. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc015.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -grep -E '^(bad-block-size|bad-root-nid|unsupported-feature) ' \ - "$work/fixtures/fixture-evidence.txt" -python3 tests/erofs_fixture.py inspect "$work/fixtures/bad-block-size.erofs" -python3 tests/erofs_fixture.py inspect "$work/fixtures/bad-root-nid.erofs" -python3 tests/erofs_fixture.py inspect "$work/fixtures/unsupported-feature.erofs" -(cd "$work/fixtures" && sha256 -c SHA256SUMS) -``` - -`bad-block-size` changes `blkszbits` at 1036 from 12 to 13 and recomputes the -checksum over `[1024,8192)`. `bad-root-nid` selects the 48-bit union form and -sets `rootnid_8b` beyond the declared inode area. `unsupported-feature` sets -unknown incompat bit `0x80000000`. - -## FreeBSD Procedure - -For each image, attach a dynamic vnode md unit, run mount directly and under -`truss`, assert nonzero status and no mount entry, then detach that exact unit. - -```sh -mkdir "$work/mnt" -for image in bad-block-size.erofs bad-root-nid.erofs unsupported-feature.erofs -do - unit=$(mdconfig -a -t vnode -f "$work/fixtures/$image") - set +e - mount -t erofs -o ro "/dev/$unit" "$work/mnt" \ - >"$work/$image.out" 2>"$work/$image.err" - status=$? - set -e - printf '%s status=%s\n' "$image" "$status" - test "$status" -ne 0 - set +e - truss -o "$work/$image.truss" mount -t erofs -o ro "/dev/$unit" \ - "$work/mnt" >/dev/null 2>&1 - set -e - grep -E 'nmount.*ERR#' "$work/$image.truss" - ! mount -p | awk -v path="$work/mnt" '$2 == path { found=1 } - END { exit found ? 0 : 1 }' - mdconfig -d -u "${unit#md}" -done -``` - -## Expected Results - -Bad block size returns `EINVAL`, bad root NID returns `EINTEGRITY`, and the -unknown incompat bit returns `EOPNOTSUPP`. Record syscall errno, image hash, -provider length, new dmesg lines, and cleanup for all three subcases. diff --git a/tests/TC016-superblock-crc-invalid.md b/tests/TC016-superblock-crc-invalid.md deleted file mode 100644 index a8120e8..0000000 --- a/tests/TC016-superblock-crc-invalid.md +++ /dev/null @@ -1,52 +0,0 @@ -# Test Case: Superblock CRC Field Mismatch - -**Test ID**: TC016-superblock-crc-invalid -**Category**: Error Handling -**Priority**: High - -## Objective - -Verify that changing only the stored superblock checksum field causes -`EINTEGRITY`, with all protected bytes and provider length unchanged. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc016.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -grep '^bad-checksum-field ' "$work/fixtures/fixture-evidence.txt" -python3 tests/erofs_fixture.py inspect \ - "$work/fixtures/bad-checksum-field.erofs" -wc -c "$work/fixtures/valid-plain.erofs" \ - "$work/fixtures/bad-checksum-field.erofs" -sha256 -q "$work/fixtures/bad-checksum-field.erofs" -``` - -The mutator flips bit 0 of the little-endian checksum at absolute byte 1028, -does not recompute it, asserts checksum invalidity, and changes no other byte. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode -f "$work/fixtures/bad-checksum-field.erofs") -set +e -mount -t erofs -o ro "/dev/$unit" "$work/mnt" \ - >"$work/mount.out" 2>"$work/mount.err" -status=$? -set -e -test "$status" -ne 0 -set +e -truss -o "$work/mount.truss" mount -t erofs -o ro "/dev/$unit" \ - "$work/mnt" >/dev/null 2>&1 -set -e -grep -E 'nmount.*ERR#97' "$work/mount.truss" -! mount -p | awk -v path="$work/mnt" '$2 == path { found=1 } - END { exit found ? 0 : 1 }' -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -Mount returns errno 97 (`EINTEGRITY`) with a checksum diagnostic. No mount or -md consumer remains. diff --git a/tests/TC017-48bit-blocks-parse.md b/tests/TC017-48bit-blocks-parse.md deleted file mode 100644 index a1ec6f4..0000000 --- a/tests/TC017-48bit-blocks-parse.md +++ /dev/null @@ -1,44 +0,0 @@ -# Test Case: 48-bit Block Count Parsing - -**Test ID**: TC017-48bit-blocks-parse -**Category**: Filesystem Metadata -**Priority**: Medium - -## Objective - -Verify the Linux EROFS union rule dynamically: with incompat bit `0x80` and a -nonzero `rootnid_8b`, combine `blocks_lo` with `rb.blocks_hi << 32` and preserve -the complete count through mount. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc017.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -python3 tests/erofs_fixture.py inspect \ - "$work/fixtures/48bit-statfs-prefix.erofs" -od -An -tx2 -j 1038 -N 2 "$work/fixtures/48bit-statfs-prefix.erofs" -od -An -tx4 -j 1060 -N 4 "$work/fixtures/48bit-statfs-prefix.erofs" -od -An -tx8 -j 1136 -N 8 "$work/fixtures/48bit-statfs-prefix.erofs" -grep '^48bit-statfs-prefix ' "$work/fixtures/fixture-evidence.txt" -``` - -Require `blocks_hi=1`, `rootnid_8b != 0`, a valid recomputed CRC, and a recorded -prefix SHA256. The expected count is `blocks_lo | (1 << 32)`. - -## FreeBSD Procedure - -Create and attach the sparse provider exactly as in TC010. Record `diskinfo`, -mount it read-only, compare `df -k` total with `total_blocks * 4`, read -`control.txt`, then unmount and detach the dynamic md unit. - -Also attach the unextended small prefix once and require mount failure with -`ENXIO`; record this only as the negative media-size guard, not the positive -parse result. - -## Expected Results - -The sparse provider mount succeeds and reports the full nonzero-high-word count. -The small prefix fails `ENXIO`. If no qualifying provider can be attached, -record the metadata parsing evidence separately and mark the dynamic portion -SHELVED, never PASS. diff --git a/tests/TC018-48bit-large-fs.md b/tests/TC018-48bit-large-fs.md deleted file mode 100644 index 5128c75..0000000 --- a/tests/TC018-48bit-large-fs.md +++ /dev/null @@ -1,66 +0,0 @@ -# Test Case: Large Filesystem with 48-bit Addressing - -**Test ID**: TC018-48bit-large-fs -**Category**: Filesystem Metadata -**Priority**: Low - -## Objective - -Verify a real FLAT_PLAIN read whose extended inode has `startblk_hi=1`, so the -provider I/O occurs above 16 TiB rather than only reporting a large `statfs` -total. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc018.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -python3 tests/erofs_fixture.py inspect \ - "$work/fixtures/48bit-high-file-prefix.erofs" --path /high-offset.txt -grep '^48bit-high-file-prefix ' "$work/fixtures/fixture-evidence.txt" -sha256 -q "$work/fixtures/48bit-high-file-prefix.erofs" -``` - -Require an extended nonempty FLAT_PLAIN inode, `startblk_hi=1`, a physical -`data_offset >= 17592186044416`, a valid CRC, and an end offset within -`required_provider_length`. The mutator zeros the original low-block payload, -so a reader that ignores `startblk_hi` cannot return the expected source bytes. -Read `required_provider_length`, `start_block`, `data_offset`, -`low_decoy_offset`, and `file_size` from the evidence line. - -## FreeBSD Procedure - -```sh -provider="$work/48bit-high-file-provider.raw" -cp "$work/fixtures/48bit-high-file-prefix.erofs" "$provider" -truncate -s "$required_provider_length" "$provider" -dd if="$work/fixtures/source/high-offset.txt" of="$provider" bs=4096 \ - seek="$start_block" conv=notrunc -cc -O2 -Wall -Wextra -std=c17 tests/read_probe.c -o "$work/read_probe" -"$work/read_probe" pread "$provider" "$low_decoy_offset" "$file_size" \ - "$work/raw-low-offset.txt" -"$work/read_probe" pread "$provider" "$data_offset" "$file_size" \ - "$work/raw-high-offset.txt" -dd if=/dev/zero of="$work/zero.bin" bs="$file_size" count=1 -cmp "$work/zero.bin" "$work/raw-low-offset.txt" -! cmp -s "$work/fixtures/source/high-offset.txt" "$work/raw-low-offset.txt" -cmp "$work/fixtures/source/high-offset.txt" "$work/raw-high-offset.txt" -unit=$(mdconfig -a -t vnode -f "$provider") -diskinfo -v "/dev/$unit" -mkdir "$work/mnt" -mount -t erofs -o ro "/dev/$unit" "$work/mnt" -cmp "$work/fixtures/source/high-offset.txt" "$work/mnt/high-offset.txt" -cmp "$work/fixtures/source/control.txt" "$work/mnt/control.txt" -df -h "$work/mnt" -umount "$work/mnt" -mdconfig -d -u "${unit#md}" -rm -f "$provider" -``` - -## Expected Results - -The raw provider probe and mounted file both return the exact source bytes from -above 16 TiB, while a low-offset control remains valid. Record the prefix hash, -sparse media size/allocation, high offset, source/data hash, dmesg delta, and -cleanup. If sparse high-offset I/O is unavailable, mark SHELVED with all -attempts; a large `df` result alone cannot pass TC018. diff --git a/tests/TC019-48bit-root-nid.md b/tests/TC019-48bit-root-nid.md deleted file mode 100644 index 82fffea..0000000 --- a/tests/TC019-48bit-root-nid.md +++ /dev/null @@ -1,44 +0,0 @@ -# Test Case: Nonzero 48-bit Root NID - -**Test ID**: TC019-48bit-root-nid - -## Objective - -Verify the `48BIT && rootnid_8b != 0` selector uses the complete 64-bit root -field and interprets the two-byte union as `blocks_hi`. - -## Fixture - -Use `images/root8-48bit.erofs`. The structured transformer requires incompat -bit `0x80`, nonzero image-bounded `rootnid_8b`, `blocks_hi=0`, valid CRC32C, -and unchanged provider-sized `blocks_lo`. Production fsck.erofs 1.8.6 does not -recognize this feature; FreeBSD mount/read is mandatory. - -## Procedure - -Host field check: - -```sh -python3 tests/erofs_fixture.py inspect /work/build/repo22-g1/images/root8-48bit.erofs --path=/root.txt -od -An -tu2 -j 1038 -N 2 /work/build/repo22-g1/images/root8-48bit.erofs -od -An -tx4 -j 1104 -N 4 /work/build/repo22-g1/images/root8-48bit.erofs -od -An -tu8 -j 1136 -N 8 /work/build/repo22-g1/images/root8-48bit.erofs -``` - -Mount the same hashed image in FreeBSD, then run: - -```sh -stat -f 'root_ino=%i type=%HT' "$mnt" -cmp /tmp/repo22-g1/source/compact/root.txt "$mnt/root.txt" -sha256 /tmp/repo22-g1/source/compact/root.txt "$mnt/root.txt" -./statfs_probe "$mnt" -``` - -## Expected Results - -- `feature_incompat & 0x80` is set, `rootnid_8b` is nonzero, and observed root - inode equals that complete field. -- Root content matches the source and statfs uses `blocks_lo` plus zero - `blocks_hi`; mount and cleanup succeed. -- TC150's zero-`rootnid_8b` fallback is a separate selector and is not accepted - as evidence for this TC. diff --git a/tests/TC020-compact-inode-basic.md b/tests/TC020-compact-inode-basic.md deleted file mode 100644 index 0104b91..0000000 --- a/tests/TC020-compact-inode-basic.md +++ /dev/null @@ -1,32 +0,0 @@ -# Test Case: Compact Inode Decoding - -**Test ID**: TC020-compact-inode-basic - -## Objective - -Verify a real 32-byte compact regular inode exposes correct mode, link count, -size, allocation, and source-exact data. - -## Fixture - -Use `images/compact.erofs` and `source/compact/small.txt`. The evidence records -the NID and asserts `inode_size=32`. - -## Procedure - -After mounting on `$mnt`: - -```sh -stat -f 'ino=%i size=%z mode=%p nlink=%l blocks=%b mtime=%m' "$mnt/small.txt" -wc -c /tmp/repo22-g1/source/compact/small.txt "$mnt/small.txt" -cmp /tmp/repo22-g1/source/compact/small.txt "$mnt/small.txt" -sha256 /tmp/repo22-g1/source/compact/small.txt "$mnt/small.txt" -``` - -Unmount and detach the exact md unit. - -## Expected Results - -- The inode is compact, regular mode `0644`, link count 1, timestamp 0, and - size equal to the source. -- Full content and hashes match; all operations and cleanup return zero. diff --git a/tests/TC021-compact-inode-nlink1.md b/tests/TC021-compact-inode-nlink1.md deleted file mode 100644 index c542749..0000000 --- a/tests/TC021-compact-inode-nlink1.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test Case: Compact Inode I_NLINK_1 - -**Test ID**: TC021-compact-inode-nlink1 - -## Objective - -Verify bit 4 on a non-directory compact inode forces `st_nlink=1` while the -`i_nb` union remains available for address high bits. - -## Fixture - -Use `images/compact-nlink1.erofs` and `source/compact/single.txt`. -`fixture-evidence.txt` asserts a 32-byte regular inode, bit 4 set, -`i_nb=0x1234`, and valid recomputed CRC32C. - -## Procedure - -```sh -stat -f 'ino=%i nlink=%l size=%z mode=%p' "$mnt/single.txt" -cmp /tmp/repo22-g1/source/compact/single.txt "$mnt/single.txt" -sha256 /tmp/repo22-g1/source/compact/single.txt "$mnt/single.txt" -``` - -## Expected Results - -- `st_nlink=1` despite on-disk `i_nb=0x1234`. -- Inline file data remains source-exact; mount and cleanup succeed. -- An `i_nb` value of zero is not used as a substitute for the bit-4 encoding. diff --git a/tests/TC022-compact-inode-special.md b/tests/TC022-compact-inode-special.md deleted file mode 100644 index 6f6b98d..0000000 --- a/tests/TC022-compact-inode-special.md +++ /dev/null @@ -1,31 +0,0 @@ -# Test Case: Compact Special Inodes and Linux dev_t Decode - -**Test ID**: TC022-compact-inode-special - -## Objective - -Verify compact char/block/FIFO inodes and Linux `new_decode_dev` conversion to -FreeBSD `dev_t`. - -## Fixture - -Use `images/compact.erofs`. Source metadata records real nodes -`char-large=2748:344865`, `block-large=2748:344865`, and `fifo`; structured -evidence asserts compact inodes and raw on-disk `i_u.rdev=0x543abc21` for both -devices. Compile `tests/stat_special.c` natively in FreeBSD. - -## Procedure - -```sh -./stat_special char "$mnt/char-large" 2748 344865 -./stat_special block "$mnt/block-large" 2748 344865 -./stat_special fifo "$mnt/fifo" -stat -f '%N mode=%p rdev=%r' "$mnt/char-large" "$mnt/block-large" "$mnt/fifo" -``` - -## Expected Results - -- Char and block helpers report major 2748, minor 344865, and raw FreeBSD - `st_rdev=0xa430005bc21`. -- FIFO type is correct and `st_rdev=NODEV`. -- A raw little-endian cast would differ; all helper checks and cleanup pass. diff --git a/tests/TC023-extended-inode-normal.md b/tests/TC023-extended-inode-normal.md deleted file mode 100644 index a5e8485..0000000 --- a/tests/TC023-extended-inode-normal.md +++ /dev/null @@ -1,27 +0,0 @@ -# Test Case: Extended Inode Decoding - -**Test ID**: TC023-extended-inode-normal - -## Objective - -Verify a real 64-byte extended regular inode, including size, full-width -metadata, timestamps, and source-exact content. - -## Fixture - -Use `images/extended.erofs` and `source/extended/large-file.bin`. -`fixture-evidence.txt` records the NID, byte offset, size, and `inode_size=64`. - -## Procedure - -```sh -stat -f 'ino=%i size=%z mode=%p nlink=%l mtime=%m ctime=%c blocks=%b' "$mnt/large-file.bin" -cmp /tmp/repo22-g1/source/extended/large-file.bin "$mnt/large-file.bin" -sha256 /tmp/repo22-g1/source/extended/large-file.bin "$mnt/large-file.bin" -``` - -## Expected Results - -- The file reports the evidence/source size, regular `0644`, link count 1, - deterministic timestamps, and a 64-byte on-disk inode. -- Full source compare and SHA256 match; cleanup succeeds. diff --git a/tests/TC024-extended-inode-large.md b/tests/TC024-extended-inode-large.md deleted file mode 100644 index 725f717..0000000 --- a/tests/TC024-extended-inode-large.md +++ /dev/null @@ -1,38 +0,0 @@ -# Test Case: Extended Inode File Above 4 GiB - -**Test ID**: TC024-extended-inode-large - -## Objective - -Verify a 64-bit extended `i_size` above 4 GiB without materializing a multi-GiB -payload, using deterministic FLAT_PLAIN hole reads at distant offsets. - -## Fixture - -Use `images/extended-large-hole.erofs` and -`expected/large-hole-zero-64k.bin`. Structured evidence asserts a 64-byte -inode, size `4294971393`, layout 0, `startblk=0xffffffffffff`, valid CRC, and -an all-zero content descriptor. - -## Procedure - -After mounting, run three bounded reads: - -```sh -stat -f 'size=%z blocks=%b mode=%p' "$mnt/huge-sparse.dat" -./read_probe pread "$mnt/huge-sparse.dat" 0 65536 /tmp/tc024-start -./read_probe pread "$mnt/huge-sparse.dat" 2147483648 65536 /tmp/tc024-middle -./read_probe pread "$mnt/huge-sparse.dat" 4294905857 65536 /tmp/tc024-end -cmp /tmp/repo22-g1/expected/large-hole-zero-64k.bin /tmp/tc024-start -cmp /tmp/repo22-g1/expected/large-hole-zero-64k.bin /tmp/tc024-middle -cmp /tmp/repo22-g1/expected/large-hole-zero-64k.bin /tmp/tc024-end -sha256 /tmp/tc024-start /tmp/tc024-middle /tmp/tc024-end -``` - -Remove outputs, unmount, and detach. - -## Expected Results - -- Size is exactly `4294971393`, not truncated to 32 bits. -- Start, middle, and final 64 KiB ranges match the deterministic zero source. -- Reads are bounded, no large allocation/panic occurs, and cleanup succeeds. diff --git a/tests/TC025-nlink1-handling.md b/tests/TC025-nlink1-handling.md deleted file mode 100644 index 9ea5208..0000000 --- a/tests/TC025-nlink1-handling.md +++ /dev/null @@ -1,35 +0,0 @@ -# Test Case: Compact Inode Link-Count Rules - -**Test ID**: TC025-nlink1-handling - -## Objective - -Compare explicit compact link counts, bit-4 single-link encoding, hard links, -and directory link counts without conflating directory dot omission. - -## Fixture - -Use `images/compact.erofs` and `images/compact-nlink1.erofs`. Evidence asserts -explicit `single.txt i_nb=1`, patched bit 4 plus `i_nb=0x1234`, identical -hard-link NID with `i_nb=2`, and explicit directory nlink. - -## Procedure - -Mount each image separately and record: - -```sh -stat -f '%N ino=%i nlink=%l mode=%p' \ - "$mnt/single.txt" "$mnt/hard-a.txt" "$mnt/hard-b.txt" "$mnt/dotdir" -cmp /tmp/repo22-g1/source/compact/single.txt "$mnt/single.txt" -cmp /tmp/repo22-g1/source/compact/hard-a.txt "$mnt/hard-a.txt" -cmp "$mnt/hard-a.txt" "$mnt/hard-b.txt" -``` - -Unmount and detach before attaching the second image. - -## Expected Results - -- Baseline and patched `single.txt` both report nlink 1 for different valid - encodings. -- `hard-a.txt` and `hard-b.txt` share one inode and report nlink 2. -- `dotdir` uses its explicit directory link count; all data compares pass. diff --git a/tests/TC026-dot-omitted-with.md b/tests/TC026-dot-omitted-with.md deleted file mode 100644 index 70a1580..0000000 --- a/tests/TC026-dot-omitted-with.md +++ /dev/null @@ -1,33 +0,0 @@ -# Test Case: Directory with dot_omitted - -**Test ID**: TC026-dot-omitted-with - -## Objective - -Verify a bit-4 directory omits only `.` on disk, retains explicit `..`, and -receives a synthetic `.` from FreeBSD readdir. - -## Fixture - -Use `images/compact-dot-omitted.erofs`. Structured evidence records bit 4, -on-disk names `..,child.txt`, 48-bit incompat selection with nonzero -`rootnid_8b`, `blocks_hi=0`, and valid CRC. Production fsck.erofs 1.8.6 cannot -qualify this newer format. - -## Procedure - -After mounting in FreeBSD: - -```sh -stat -f 'ino=%i nlink=%l mode=%p' "$mnt/dotdir" -ls -lai "$mnt/dotdir" -(cd "$mnt/dotdir" && test "$(pwd)" = "$mnt/dotdir") -cmp /tmp/repo22-g1/source/compact/dotdir/child.txt "$mnt/dotdir/child.txt" -``` - -## Expected Results - -- User-visible listing contains both `.` and `..`; `.` has the directory NID. -- Structured evidence, not the listing, proves `.` is absent on disk and bit 4 - is set; `..` remains an on-disk entry. -- Child data, navigation, mount, and cleanup succeed. diff --git a/tests/TC027-dot-omitted-without.md b/tests/TC027-dot-omitted-without.md deleted file mode 100644 index caa4c25..0000000 --- a/tests/TC027-dot-omitted-without.md +++ /dev/null @@ -1,35 +0,0 @@ -# Test Case: Directory without dot_omitted - -**Test ID**: TC027-dot-omitted-without - -## Objective - -Verify a bit-4-clear directory reads explicit on-disk `.` and `..` entries. - -## Fixture - -Use `images/compact.erofs` and `/dotdir`. Structured evidence asserts bit 4 is -clear and names are `.,..,child.txt`; production `dump.erofs --ls` also -qualifies this baseline. - -## Procedure - -Host: - -```sh -dump.erofs --ls --path=/dotdir /work/build/repo22-g1/images/compact.erofs -``` - -Guest after mount: - -```sh -stat -f 'ino=%i nlink=%l mode=%p' "$mnt/dotdir" -ls -lai "$mnt/dotdir" -cmp /tmp/repo22-g1/source/compact/dotdir/child.txt "$mnt/dotdir/child.txt" -``` - -## Expected Results - -- Host inspection and FreeBSD listing both contain explicit `.` and `..`. -- Child data matches the source, directory attributes are valid, and cleanup - succeeds. diff --git a/tests/TC028-flat-plain-small.md b/tests/TC028-flat-plain-small.md deleted file mode 100644 index 2d68bc2..0000000 --- a/tests/TC028-flat-plain-small.md +++ /dev/null @@ -1,26 +0,0 @@ -# Test Case: FLAT_PLAIN Small File - -**Test ID**: TC028-flat-plain-small - -## Objective - -Verify a sub-block regular file explicitly encoded as uncompressed -`FLAT_PLAIN` reads source-exactly. - -## Fixture - -Use `images/flat.erofs` and `source/flat/small.txt`. Structured evidence asserts -layout 0 and records NID and size. - -## Procedure - -```sh -stat -f 'size=%z blocks=%b mode=%p' "$mnt/small.txt" -cmp /tmp/repo22-g1/source/flat/small.txt "$mnt/small.txt" -sha256 /tmp/repo22-g1/source/flat/small.txt "$mnt/small.txt" -``` - -## Expected Results - -- Evidence proves `FLAT_PLAIN`; size and data match the source. -- The read, hash, mount, and cleanup all succeed. diff --git a/tests/TC029-flat-plain-medium.md b/tests/TC029-flat-plain-medium.md deleted file mode 100644 index cff6af5..0000000 --- a/tests/TC029-flat-plain-medium.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: FLAT_PLAIN Medium File - -**Test ID**: TC029-flat-plain-medium - -## Objective - -Verify complete and nonzero-offset reads of a multi-block `FLAT_PLAIN` file. - -## Fixture - -Use `images/flat.erofs`, `source/flat/medium.dat`, and -`expected/medium-10-5.bin`. Evidence asserts layout 0 and size 102417. - -## Procedure - -```sh -stat -f 'size=%z blocks=%b' "$mnt/medium.dat" -cmp /tmp/repo22-g1/source/flat/medium.dat "$mnt/medium.dat" -./read_probe pread "$mnt/medium.dat" 40960 20480 /tmp/tc029-range -cmp /tmp/repo22-g1/expected/medium-10-5.bin /tmp/tc029-range -sha256 /tmp/repo22-g1/source/flat/medium.dat "$mnt/medium.dat" \ - /tmp/repo22-g1/expected/medium-10-5.bin /tmp/tc029-range -``` - -Remove the range output, unmount, and detach. - -## Expected Results - -- Full file and the five-block range match their deterministic sources. -- No block-boundary truncation or cleanup failure occurs. diff --git a/tests/TC030-flat-plain-large.md b/tests/TC030-flat-plain-large.md deleted file mode 100644 index eef7e9a..0000000 --- a/tests/TC030-flat-plain-large.md +++ /dev/null @@ -1,33 +0,0 @@ -# Test Case: FLAT_PLAIN Large File - -**Test ID**: TC030-flat-plain-large - -## Objective - -Verify a greater-than-10-MiB contiguous `FLAT_PLAIN` file across thousands of -filesystem blocks. - -## Fixture - -Use `images/flat.erofs` and `source/flat/large.bin`. Evidence asserts layout 0 -and size 10485791. - -## Procedure - -```sh -stat -f 'size=%z blocks=%b' "$mnt/large.bin" -dd if="$mnt/large.bin" of=/tmp/tc030-full bs=1m -cmp /tmp/repo22-g1/source/flat/large.bin /tmp/tc030-full -sha256 /tmp/repo22-g1/source/flat/large.bin /tmp/tc030-full -./read_probe pread "$mnt/large.bin" 5242880 1048576 /tmp/tc030-seek -dd if=/tmp/repo22-g1/source/flat/large.bin of=/tmp/tc030-source-seek \ - bs=1m skip=5 count=1 -cmp /tmp/tc030-source-seek /tmp/tc030-seek -``` - -Remove all outputs, unmount, and detach. - -## Expected Results - -- Reported size is exact; full and seeked data match the source. -- No mapping error, panic, or cleanup failure occurs. diff --git a/tests/TC031-flat-inline-normal.md b/tests/TC031-flat-inline-normal.md deleted file mode 100644 index c299cb5..0000000 --- a/tests/TC031-flat-inline-normal.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test Case: FLAT_INLINE Small File - -**Test ID**: TC031-flat-inline-normal - -## Objective - -Verify a small uncompressed `FLAT_INLINE` payload stored in the inode metadata -area and bounded by its metadata block. - -## Fixture - -Use `images/inline.erofs` and `source/inline/tiny.txt`. Evidence asserts layout -2, compact inode, NID, and size. - -## Procedure - -```sh -stat -f 'size=%z blocks=%b mode=%p' "$mnt/tiny.txt" -cmp /tmp/repo22-g1/source/inline/tiny.txt "$mnt/tiny.txt" -sha256 /tmp/repo22-g1/source/inline/tiny.txt "$mnt/tiny.txt" -``` - -## Expected Results - -- Layout is 2 and size/content match the source. -- FreeBSD reports 8 allocated 512-byte sectors for the 4096-byte EROFS - allocation unit; zero blocks is not the current qualified semantic. -- Mount and cleanup succeed. diff --git a/tests/TC032-flat-inline-zero.md b/tests/TC032-flat-inline-zero.md deleted file mode 100644 index 83c678e..0000000 --- a/tests/TC032-flat-inline-zero.md +++ /dev/null @@ -1,29 +0,0 @@ -# Test Case: FLAT_INLINE Zero-Length File - -**Test ID**: TC032-flat-inline-zero - -## Objective - -Verify an explicitly layout-2 zero-length compact inode returns clean EOF and -zero allocation. - -## Fixture - -Use `images/inline-zero.erofs` and `source/inline/empty.txt`. The transformer -asserts compact `FLAT_INLINE`, size 0, valid CRC, and records the image/source -hashes. - -## Procedure - -```sh -stat -f 'size=%z blocks=%b mode=%p' "$mnt/empty.txt" -cmp /tmp/repo22-g1/source/inline/empty.txt "$mnt/empty.txt" -test "$(wc -c < "$mnt/empty.txt")" -eq 0 -sha256 /tmp/repo22-g1/source/inline/empty.txt "$mnt/empty.txt" -``` - -## Expected Results - -- Size and byte count are zero, `st_blocks=0`, hashes match the empty source, - and no read error occurs. -- Mount and cleanup succeed. diff --git a/tests/TC033-tailpacking-normal.md b/tests/TC033-tailpacking-normal.md deleted file mode 100644 index 8056f54..0000000 --- a/tests/TC033-tailpacking-normal.md +++ /dev/null @@ -1,33 +0,0 @@ -# Test Case: FLAT_INLINE Tailpacking - -**Test ID**: TC033-tailpacking-normal - -## Objective - -Verify a 5000-byte file reads exactly across its full data block and 904-byte -inline tail boundary. - -## Fixture - -Use `images/inline.erofs` and `source/inline/tailpacked.dat`. Evidence asserts -layout 2 and size 5000. - -## Procedure - -```sh -stat -f 'size=%z blocks=%b' "$mnt/tailpacked.dat" -cmp /tmp/repo22-g1/source/inline/tailpacked.dat "$mnt/tailpacked.dat" -sha256 /tmp/repo22-g1/source/inline/tailpacked.dat "$mnt/tailpacked.dat" -./read_probe pread "$mnt/tailpacked.dat" 4080 64 /tmp/tc033-mounted-range -dd if=/tmp/repo22-g1/source/inline/tailpacked.dat \ - of=/tmp/tc033-source-range bs=1 skip=4080 count=64 2>/dev/null -cmp /tmp/tc033-source-range /tmp/tc033-mounted-range -``` - -Remove outputs, unmount, and detach. - -## Expected Results - -- Full file and the range crossing logical offset 4096 match the source. -- The transition from plain block data to metadata tail is seamless and - cleanup succeeds. diff --git a/tests/TC034-tailpacking-boundary.md b/tests/TC034-tailpacking-boundary.md deleted file mode 100644 index c22f921..0000000 --- a/tests/TC034-tailpacking-boundary.md +++ /dev/null @@ -1,35 +0,0 @@ -# Test Case: Tailpacking Block Boundaries - -**Test ID**: TC034-tailpacking-boundary - -## Objective - -Verify the actual erofs-utils boundary choices at 4095, 4096, and 4097 bytes, -including the one-byte inline tail case. - -## Fixture - -Use `images/inline.erofs` and the three matching files under `source/inline`. -Structured evidence requires layouts `0,0,2` respectively. A 4095-byte payload -cannot fit beside a 32-byte inode in one 4096-byte metadata block, so it is not -claimed as fully inline. - -## Procedure - -```sh -for size in 4095 4096 4097; do - stat -f '%N size=%z blocks=%b' "$mnt/file-$size.dat" - cmp "/tmp/repo22-g1/source/inline/file-$size.dat" "$mnt/file-$size.dat" - sha256 "/tmp/repo22-g1/source/inline/file-$size.dat" "$mnt/file-$size.dat" -done -./read_probe pread "$mnt/file-4097.dat" 4088 9 /tmp/tc034-boundary -dd if=/tmp/repo22-g1/source/inline/file-4097.dat \ - of=/tmp/tc034-source bs=1 skip=4088 count=9 2>/dev/null -cmp /tmp/tc034-source /tmp/tc034-boundary -``` - -## Expected Results - -- Exact sizes and full hashes match for all three files. -- 4095 and 4096 are FLAT_PLAIN; 4097 is FLAT_INLINE with a one-byte tail. -- The nine-byte cross-boundary range matches; cleanup succeeds. diff --git a/tests/TC035-file-read-sequential.md b/tests/TC035-file-read-sequential.md deleted file mode 100644 index 8d8f185..0000000 --- a/tests/TC035-file-read-sequential.md +++ /dev/null @@ -1,29 +0,0 @@ -# Test Case: Sequential Regular-File Read - -**Test ID**: TC035-file-read-sequential - -## Objective - -Verify buffered sequential reads return every byte of a deterministic -multi-block regular file. - -## Fixture - -Use `images/flat.erofs` and `source/flat/data.txt`. Evidence asserts -`FLAT_PLAIN`, NID, and size 262163. - -## Procedure - -```sh -stat -f 'size=%z blocks=%b' "$mnt/data.txt" -dd if="$mnt/data.txt" of=/tmp/tc035.out bs=4096 -cmp /tmp/repo22-g1/source/flat/data.txt /tmp/tc035.out -sha256 /tmp/repo22-g1/source/flat/data.txt /tmp/tc035.out -``` - -Remove output, unmount, and detach. - -## Expected Results - -- `dd` reaches exact EOF and the complete output matches the source/hash. -- No short read, data corruption, dmesg error, or cleanup failure occurs. diff --git a/tests/TC036-file-read-random.md b/tests/TC036-file-read-random.md deleted file mode 100644 index 5022ce9..0000000 --- a/tests/TC036-file-read-random.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: Random Regular-File Reads - -**Test ID**: TC036-file-read-random - -## Objective - -Verify independent start, middle, and end `pread(2)` ranges from a deterministic -1 MiB regular file. - -## Fixture - -Use `images/flat.erofs`, `source/flat/random-test.bin`, and the three -`expected/random-*.bin` files. Evidence asserts layout 0 and exact size. - -## Procedure - -```sh -./read_probe pread "$mnt/random-test.bin" 0 10240 /tmp/tc036-start -./read_probe pread "$mnt/random-test.bin" 512000 10240 /tmp/tc036-mid -./read_probe pread "$mnt/random-test.bin" 1024000 24576 /tmp/tc036-end -cmp /tmp/repo22-g1/expected/random-start.bin /tmp/tc036-start -cmp /tmp/repo22-g1/expected/random-mid.bin /tmp/tc036-mid -cmp /tmp/repo22-g1/expected/random-end.bin /tmp/tc036-end -sha256 /tmp/tc036-start /tmp/tc036-mid /tmp/tc036-end -``` - -## Expected Results - -- Every bounded `pread` reports its requested byte count and exact offset. -- All three ranges match independently generated source ranges; cleanup passes. diff --git a/tests/TC037-file-read-large.md b/tests/TC037-file-read-large.md deleted file mode 100644 index 812e03a..0000000 --- a/tests/TC037-file-read-large.md +++ /dev/null @@ -1,33 +0,0 @@ -# Test Case: Large Regular-File Read - -**Test ID**: TC037-file-read-large - -## Objective - -Verify complete and distant-range reads of a deterministic file larger than -100 MiB without a correctness shortcut based only on throughput. - -## Fixture - -Use `images/flat.erofs`, `source/flat/huge-data.bin`, and -`expected/huge-sample.bin`. Evidence asserts layout 0 and size 104857857. - -## Procedure - -```sh -stat -f 'size=%z blocks=%b' "$mnt/huge-data.bin" -time dd if="$mnt/huge-data.bin" of=/tmp/tc037-full bs=1m -cmp /tmp/repo22-g1/source/flat/huge-data.bin /tmp/tc037-full -sha256 /tmp/repo22-g1/source/flat/huge-data.bin /tmp/tc037-full -./read_probe pread "$mnt/huge-data.bin" 52428800 5242880 /tmp/tc037-sample -cmp /tmp/repo22-g1/expected/huge-sample.bin /tmp/tc037-sample -sha256 /tmp/repo22-g1/expected/huge-sample.bin /tmp/tc037-sample -``` - -Remove outputs, unmount, and detach. - -## Expected Results - -- Exact size, full compare, and 50 MiB-offset sample compare all pass. -- Record elapsed time but impose no host-dependent QEMU throughput threshold. -- No memory exhaustion, panic, or cleanup failure occurs. diff --git a/tests/TC038-symlink-short.md b/tests/TC038-symlink-short.md deleted file mode 100644 index d747d23..0000000 --- a/tests/TC038-symlink-short.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: Short Symlink Target - -**Test ID**: TC038-symlink-short - -## Objective - -Verify a short `FLAT_INLINE` symlink returns the exact source target and follows -to source-exact file data. - -## Fixture - -Use `images/inline.erofs`, `source/inline/link1`, and -`source/inline/target.txt`. Evidence asserts symlink mode, layout 2, and target -length 10. - -## Procedure - -```sh -test -L "$mnt/link1" -readlink /tmp/repo22-g1/source/inline/link1 > /tmp/tc038-source-target -readlink "$mnt/link1" > /tmp/tc038-mount-target -cmp /tmp/tc038-source-target /tmp/tc038-mount-target -cmp /tmp/repo22-g1/source/inline/target.txt "$mnt/link1" -sha256 /tmp/repo22-g1/source/inline/target.txt "$mnt/link1" -``` - -## Expected Results - -- `readlink` returns exactly `target.txt`, without newline truncation ambiguity. -- Following the link yields the exact target source; cleanup succeeds. diff --git a/tests/TC039-symlink-long.md b/tests/TC039-symlink-long.md deleted file mode 100644 index 36561ef..0000000 --- a/tests/TC039-symlink-long.md +++ /dev/null @@ -1,36 +0,0 @@ -# Test Case: Long Symlink Target - -**Test ID**: TC039-symlink-long - -## Objective - -Verify a greater-than-60-byte EROFS symlink target is returned completely and -follows correctly. - -## Fixture - -Use `images/flat.erofs`, `source/flat/longlink`, and its nested target file. -Structured evidence records symlink size 73 and actual layout 2. EROFS does not -switch to a separate-data-block symlink merely at 60 bytes; that former fixture -assumption is invalid. - -## Procedure - -```sh -test -L "$mnt/longlink" -readlink /tmp/repo22-g1/source/flat/longlink > /tmp/tc039-source-target -readlink "$mnt/longlink" > /tmp/tc039-mount-target -cmp /tmp/tc039-source-target /tmp/tc039-mount-target -test "$(wc -c < /tmp/tc039-mount-target)" -eq 74 -cmp /tmp/repo22-g1/source/flat/very/long/path/to/a/deeply/nested/deterministic/target/directory/file.txt \ - "$mnt/longlink" -sha256 "$mnt/longlink" \ - /tmp/repo22-g1/source/flat/very/long/path/to/a/deeply/nested/deterministic/target/directory/file.txt -``` - -The `wc` count is 73 target bytes plus the output newline. - -## Expected Results - -- Source and mounted targets match exactly and exceed 60 bytes. -- The link follows to source-exact data with no truncation; cleanup succeeds. diff --git a/tests/TC040-symlink-broken.md b/tests/TC040-symlink-broken.md deleted file mode 100644 index 6833fb1..0000000 --- a/tests/TC040-symlink-broken.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: Broken Symlink - -**Test ID**: TC040-symlink-broken - -## Objective - -Verify `readlink(2)` succeeds for a deterministic broken link while following -the target fails with exact `ENOENT`. - -## Fixture - -Use `images/inline.erofs` and `source/inline/brokenlink`. Evidence asserts -symlink mode, layout 2, and target length 29. - -## Procedure - -```sh -test -L "$mnt/brokenlink" -readlink /tmp/repo22-g1/source/inline/brokenlink > /tmp/tc040-source-target -readlink "$mnt/brokenlink" > /tmp/tc040-mount-target -cmp /tmp/tc040-source-target /tmp/tc040-mount-target -./read_probe expect-error "$mnt/brokenlink" 2 -./read_probe expect-error "$mnt/brokenlink" 2 -``` - -## Expected Results - -- Both `readlink` targets equal `/nonexistent/repo22-g1-target`. -- Both follow attempts return errno 2 (`ENOENT`), while the symlink itself - remains visible; no kernel error or cleanup failure occurs. diff --git a/tests/TC041-dirent-decode-normal.md b/tests/TC041-dirent-decode-normal.md deleted file mode 100644 index 8ae87d1..0000000 --- a/tests/TC041-dirent-decode-normal.md +++ /dev/null @@ -1,29 +0,0 @@ -# Test Case: Directory Entry Decoding - Normal - -**Test ID**: TC041-dirent-decode-normal -**Category**: Directory Operations -**Priority**: High - -## Objective - -Decode every entry type and the 255-byte filename from the deterministic VFS -fixture through FreeBSD getdirentries(2), not inferred ls formatting. - -## Procedure - -Mount vfs/vfs-plain.erofs using G3-MANUAL-SETUP.md, then run: - - ./readdir_probe /tmp/repo22-g3/mnt/testdir 512 - find /tmp/repo22-g3/mnt/testdir -maxdepth 1 -print | - sed 's#.*/##' | sort > actual-testdir.txt - cmp fixtures/vfs/expected-testdir.txt actual-testdir.txt - name=$(find /tmp/repo22-g3/mnt/testdir -maxdepth 1 -type f \ - -name 'long-*') - test "$(basename "$name" | tr -d '\n' | wc -c)" -eq 255 - -## Expected Results - -The probe reports 36 unique entries, correct DT_DIR/DT_REG/DT_LNK counts, -strictly increasing restartable cookies, and one deterministic name/type hash. -The 34 real names match the manifest exactly and the longest name is 255 bytes. -Record probe output, manifest/image hashes, and mount/md cleanup. diff --git a/tests/TC042-dirent-large-dir.md b/tests/TC042-dirent-large-dir.md deleted file mode 100644 index 1bb06e5..0000000 --- a/tests/TC042-dirent-large-dir.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test Case: Directory Entry Decoding - Multi-Block Directory - -**Test ID**: TC042-dirent-large-dir -**Category**: Directory Operations -**Priority**: High - -## Objective - -Decode a real multi-block directory without missing, duplicating, or -truncating entries across directory-block boundaries. - -## Procedure - -Mount metadata/namei-base.erofs and run: - - ./readdir_probe /tmp/repo22-g3/mnt/wide 128 - find /tmp/repo22-g3/mnt/wide -maxdepth 1 -type f | - sed 's#.*/##' | sort > actual-wide.txt - cmp fixtures/metadata/expected-wide.txt actual-wide.txt - stat /tmp/repo22-g3/mnt/wide/entry-000-abcdefghijklmnopqrstuvwxyz.txt - stat /tmp/repo22-g3/mnt/wide/entry-159-abcdefghijklmnopqrstuvwxyz.txt - stat /tmp/repo22-g3/mnt/wide/entry-319-abcdefghijklmnopqrstuvwxyz.txt - -## Expected Results - -All 320 real files plus dot and dotdot are returned once. The helper reports -322 restartable kernel and libc positions and the three boundary samples stat -successfully. Record the image hash and cleanup. diff --git a/tests/TC043-lookup-found.md b/tests/TC043-lookup-found.md deleted file mode 100644 index 926dd94..0000000 --- a/tests/TC043-lookup-found.md +++ /dev/null @@ -1,29 +0,0 @@ -# Test Case: Lookup - Existing Entries - -**Test ID**: TC043-lookup-found -**Category**: Directory Operations -**Priority**: Critical - -## Objective - -Verify exact-name lookup of regular, directory, and symlink entries returns -stable vnode metadata and source-exact data. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/testdir/file.txt - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/testdir/subdir - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/testdir/shortlink - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/testdir/file.txt - cmp fixtures/vfs/source/testdir/file.txt \ - /tmp/repo22-g3/mnt/testdir/file.txt - sha256 fixtures/vfs/source/testdir/file.txt \ - /tmp/repo22-g3/mnt/testdir/file.txt - -## Expected Results - -All lookups succeed, repeated file lookup reports the same nonzero inode, each -type/mode/size matches fixture evidence, and source/mounted hashes are equal. -Record command statuses and cleanup. diff --git a/tests/TC044-lookup-not-found.md b/tests/TC044-lookup-not-found.md deleted file mode 100644 index 76c85c1..0000000 --- a/tests/TC044-lookup-not-found.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test Case: Lookup - Missing Entry - -**Test ID**: TC044-lookup-not-found -**Category**: Directory Operations -**Priority**: High - -## Objective - -Verify direct missing lookups return exact ENOENT without masking existing -entries or turning other errors into a miss. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/testdir/nonexistent.txt 2 - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/testdir/nonexistent.txt 2 - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/no-such-directory/file.txt 2 - ./g3_vfs_probe expect-success stat \ - /tmp/repo22-g3/mnt/testdir/file.txt - -## Expected Results - -Each missing syscall reports errno 2 and the positive control succeeds. -Wrapper text alone is not evidence; record the helper output and cleanup. diff --git a/tests/TC045-lookup-case-sensitive.md b/tests/TC045-lookup-case-sensitive.md deleted file mode 100644 index 2880b76..0000000 --- a/tests/TC045-lookup-case-sensitive.md +++ /dev/null @@ -1,27 +0,0 @@ -# Test Case: Lookup - Case Sensitivity - -**Test ID**: TC045-lookup-case-sensitive -**Category**: Directory Operations -**Priority**: High - -## Objective - -Verify four distinct case variants resolve independently and an unrecorded -case variant returns ENOENT. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - stat -f 'name=%N ino=%i size=%z' \ - /tmp/repo22-g3/mnt/testdir/file.txt \ - /tmp/repo22-g3/mnt/testdir/File.txt \ - /tmp/repo22-g3/mnt/testdir/FILE.TXT \ - /tmp/repo22-g3/mnt/testdir/FiLe.TxT - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/testdir/fILE.txt 2 - -## Expected Results - -The four exact variants have four nonzero, distinct NIDs and their recorded -sizes. The unmatched spelling returns errno 2. Record output and cleanup. diff --git a/tests/TC046-readdir-basic.md b/tests/TC046-readdir-basic.md deleted file mode 100644 index a0480fd..0000000 --- a/tests/TC046-readdir-basic.md +++ /dev/null @@ -1,25 +0,0 @@ -# Test Case: Readdir - Basic Completeness - -**Test ID**: TC046-readdir-basic -**Category**: Directory Operations -**Priority**: Critical - -## Objective - -Verify one complete getdirentries(2)/readdir(3) traversal returns the exact -fixture set, including dot and dotdot. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - ./readdir_probe /tmp/repo22-g3/mnt/testdir 4096 - find /tmp/repo22-g3/mnt/testdir -maxdepth 1 -mindepth 1 -print | - sed 's#.*/##' | sort > actual-testdir.txt - cmp fixtures/vfs/expected-testdir.txt actual-testdir.txt - -## Expected Results - -The manifest comparison is exact; the probe reports 36 unique entries and the -same ordered name/type hash through both APIs. Record the image hash and -cleanup. diff --git a/tests/TC047-readdir-large.md b/tests/TC047-readdir-large.md deleted file mode 100644 index dc1a220..0000000 --- a/tests/TC047-readdir-large.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test Case: Readdir - Multi-Block Completeness - -**Test ID**: TC047-readdir-large -**Category**: Directory Operations -**Priority**: High - -## Objective - -Verify repeated small-buffer reads cover a 320-file multi-block directory -exactly once. - -## Procedure - -Mount metadata/namei-base.erofs and run: - - ./readdir_probe /tmp/repo22-g3/mnt/wide 128 - find /tmp/repo22-g3/mnt/wide -maxdepth 1 -type f | - sed 's#.*/##' | sort > actual-wide.txt - test "$(wc -l < actual-wide.txt)" -eq 320 - test "$(sort actual-wide.txt | uniq -d | wc -l)" -eq 0 - -Compare actual-wide.txt with fixtures/metadata/expected-wide.txt. - -## Expected Results - -The helper reports 322 unique entries and 322 valid restart positions with a -128-byte userspace buffer; the exact manifest has 320 names and no duplicates. -Record the probe hash and cleanup. diff --git a/tests/TC048-readdir-seek.md b/tests/TC048-readdir-seek.md deleted file mode 100644 index 3ffbd98..0000000 --- a/tests/TC048-readdir-seek.md +++ /dev/null @@ -1,31 +0,0 @@ -# Test Case: Readdir Cookies and seekdir Resume - -**Test ID**: TC048-readdir-seek -**Category**: Directory Operations -**Priority**: Critical - -## Objective - -Verify every kernel d_off and libc telldir cookie resumes at the next exact -entry; complete second traversals are not substitutes. - -## Procedure - -Mount metadata/namei-base.erofs and run: - - ./readdir_probe /tmp/repo22-g3/mnt/wide 128 | - tee TC048-probe.out - -## Probe Contract - -The helper fails on non-increasing cookies, invalid records, duplicate names, -kernel/libc order or type differences, a restart at the wrong next name, or a -final cookie that does not reach EOF. Kernel d_off values are validated after -reopen/lseek; each libc telldir value is passed back to seekdir on the same -DIR stream that produced it, as required by the API contract. - -## Expected Results - -The helper exits zero and reports entries=322, d_off_restarts=322, -seekdir_restarts=322, buffer=128, the type counts, and one FNV-1a name/type -hash. Record literal output and cleanup. diff --git a/tests/TC049-dot-handling.md b/tests/TC049-dot-handling.md deleted file mode 100644 index 7a076a2..0000000 --- a/tests/TC049-dot-handling.md +++ /dev/null @@ -1,26 +0,0 @@ -# Test Case: Dot Entry Handling - -**Test ID**: TC049-dot-handling -**Category**: Directory Operations -**Priority**: High - -## Objective - -Verify dot is returned as a directory entry and explicit dot lookup resolves -to the same vnode as its directory. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - ./readdir_probe /tmp/repo22-g3/mnt/testdir 128 - stat -f '%i %HT' /tmp/repo22-g3/mnt/testdir - stat -f '%i %HT' /tmp/repo22-g3/mnt/testdir/. - cmp fixtures/vfs/source/testdir/file.txt \ - /tmp/repo22-g3/mnt/./testdir/./file.txt - -## Expected Results - -The traversal includes one dot entry with DT_DIR, both stat commands report -the same directory NID, and paths containing dot read source-exact data. -Record output and cleanup. diff --git a/tests/TC050-dotdot-handling.md b/tests/TC050-dotdot-handling.md deleted file mode 100644 index 39a9262..0000000 --- a/tests/TC050-dotdot-handling.md +++ /dev/null @@ -1,26 +0,0 @@ -# Test Case: Dotdot Entry Handling - -**Test ID**: TC050-dotdot-handling -**Category**: Directory Operations -**Priority**: High - -## Objective - -Verify dotdot is returned and resolves nested paths to the correct parent -vnode without lock-order failure. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - ./readdir_probe /tmp/repo22-g3/mnt/parent/child 128 - stat -f '%i %HT' /tmp/repo22-g3/mnt/parent - stat -f '%i %HT' /tmp/repo22-g3/mnt/parent/child/.. - cmp fixtures/vfs/source/parent/file.txt \ - /tmp/repo22-g3/mnt/parent/child/grandchild/../../file.txt - -## Expected Results - -The traversal contains one dotdot DT_DIR entry, both parent stat commands -report the same NID, and multiple dotdot components reach the expected file. -Record output, dmesg delta, and cleanup. diff --git a/tests/TC051-namecache-hit.md b/tests/TC051-namecache-hit.md deleted file mode 100644 index 9197734..0000000 --- a/tests/TC051-namecache-hit.md +++ /dev/null @@ -1,29 +0,0 @@ -# Test Case: Repeated Positive Lookup Behavior - -**Test ID**: TC051-namecache-hit -**Category**: VFS Integration -**Priority**: High - -## Objective - -Verify repeated positive pathname lookups remain stable within one mount. -This is behavior-level namecache integration coverage; timing differences do -not prove an internal cache hit. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - stat -f 'ino=%i mode=%p size=%z gen=%v' \ - /tmp/repo22-g3/mnt/testdir/file.txt > TC051-first.txt - stat -f 'ino=%i mode=%p size=%z gen=%v' \ - /tmp/repo22-g3/mnt/testdir/file.txt > TC051-second.txt - cmp TC051-first.txt TC051-second.txt - cmp fixtures/vfs/source/testdir/file.txt \ - /tmp/repo22-g3/mnt/testdir/file.txt - -## Expected Results - -Both lookups succeed with byte-identical metadata and data. Record only this -observable behavior as PASS; do not claim a measured cache hit without kernel -instrumentation. Record output and cleanup. diff --git a/tests/TC052-namecache-miss.md b/tests/TC052-namecache-miss.md deleted file mode 100644 index 7e36a1d..0000000 --- a/tests/TC052-namecache-miss.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test Case: Repeated Negative Lookup Behavior - -**Test ID**: TC052-namecache-miss -**Category**: VFS Integration -**Priority**: High - -## Objective - -Verify repeated missing-name lookups consistently return ENOENT and do not -poison a later positive lookup. This is behavior-level negative-namecache -coverage, not proof of an internal cache hit. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/testdir/cache-missing 2 - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/testdir/cache-missing 2 - ./g3_vfs_probe expect-success stat \ - /tmp/repo22-g3/mnt/testdir/file.txt - -## Expected Results - -Both negative syscalls report errno 2 and the positive control succeeds. -Record the stable behavior honestly, with no latency-based cache assertion, -then clean the mount and md. diff --git a/tests/TC053-vfs-hash-integration.md b/tests/TC053-vfs-hash-integration.md deleted file mode 100644 index c1fdf5b..0000000 --- a/tests/TC053-vfs-hash-integration.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: Stable Vnode Identity Behavior - -**Test ID**: TC053-vfs-hash-integration -**Category**: VFS Integration -**Priority**: High - -## Objective - -Verify repeated resolutions of one EROFS NID expose one stable vnode identity -and file handle within a mount. This is behavior-level coverage of the -vfs_hash contract; it does not directly observe the internal hash bucket. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - stat -f 'ino=%i gen=%v size=%z' \ - /tmp/repo22-g3/mnt/testdir/file.txt > TC053-a.txt - stat -f 'ino=%i gen=%v size=%z' \ - /tmp/repo22-g3/mnt/testdir/./file.txt > TC053-b.txt - cmp TC053-a.txt TC053-b.txt - ./nfs_fh_tool capture /tmp/repo22-g3/mnt/testdir/file.txt TC053-a.fh - ./nfs_fh_tool capture /tmp/repo22-g3/mnt/testdir/./file.txt TC053-b.fh - ./nfs_fh_tool compare TC053-a.fh TC053-b.fh - ./nfs_fh_tool describe TC053-a.fh - -## Expected Results - -Metadata and handles are identical and the handle NID/generation matches stat. -Record this observable identity behavior, handle hash, and cleanup. diff --git a/tests/TC054-vn-vget-ino.md b/tests/TC054-vn-vget-ino.md deleted file mode 100644 index fbd7987..0000000 --- a/tests/TC054-vn-vget-ino.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: Parent Vnode Lookup Behavior - -**Test ID**: TC054-vn-vget-ino -**Category**: VFS Integration -**Priority**: High - -## Objective - -Verify repeated dotdot lookups return the correct stable parent vnode without -deadlock. This is behavior-level coverage of the vn_vget_ino path; userspace -cannot prove the internal helper call by timing. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - stat -f 'ino=%i gen=%v type=%HT' \ - /tmp/repo22-g3/mnt/parent > TC054-parent.txt - stat -f 'ino=%i gen=%v type=%HT' \ - /tmp/repo22-g3/mnt/parent/child/.. > TC054-dotdot1.txt - stat -f 'ino=%i gen=%v type=%HT' \ - /tmp/repo22-g3/mnt/parent/child/.. > TC054-dotdot2.txt - cmp TC054-parent.txt TC054-dotdot1.txt - cmp TC054-dotdot1.txt TC054-dotdot2.txt - -## Expected Results - -All commands complete, metadata is identical, and dmesg has no lock-order, -trap, or panic report. Record behavior and cleanup without claiming direct -internal instrumentation. diff --git a/tests/TC055-getattr-basic.md b/tests/TC055-getattr-basic.md deleted file mode 100644 index 44c8726..0000000 --- a/tests/TC055-getattr-basic.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: getattr Basic Metadata - -**Test ID**: TC055-getattr-basic -**Category**: Vnode Operations -**Priority**: Critical - -## Objective - -Verify VOP_GETATTR returns fixture-exact type, mode, ownership, link count, -size, allocation, block size, inode, and generation for plain and compressed -regular files. - -## Procedure - -On separate fresh mounts of vfs/vfs-plain.erofs and vfs/vfs-lz4.erofs: - - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/pager.bin - stat -f 'ino=%i mode=%p uid=%u gid=%g nlink=%l size=%z blocks=%b blksize=%k gen=%v type=%HT' \ - /tmp/repo22-g3/mnt/pager.bin - sha256 /tmp/repo22-g3/mnt/pager.bin - -For the plain mount also compare the mounted hash with -fixtures/vfs/source/pager.bin and fixture evidence. - -## Expected Results - -Both files are regular mode 0644, uid/gid 0, size 21211, block size 4096, with -nonzero stable NID/generation. Both logical hashes equal the source hash; -st_blocks records each inode's reported allocated sectors. Record both image -hashes and per-mount cleanup. diff --git a/tests/TC056-getattr-special.md b/tests/TC056-getattr-special.md deleted file mode 100644 index 758c27d..0000000 --- a/tests/TC056-getattr-special.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test Case: getattr Special Files - -**Test ID**: TC056-getattr-special -**Category**: Vnode Operations -**Priority**: Critical - -## Objective - -Verify compact and extended char, block, and FIFO inodes expose exact types, -device numbers, zero size/allocation, and stable generation. - -## Procedure - -On fresh mounts of metadata/special-compact.erofs and -metadata/special-extended.erofs: - - ./stat_special char /tmp/repo22-g3/mnt/char-large 2748 344865 - ./stat_special block /tmp/repo22-g3/mnt/block-large 2748 344865 - ./stat_special fifo /tmp/repo22-g3/mnt/fifo - stat -f 'name=%N type=%HT size=%z blocks=%b gen=%v' \ - /tmp/repo22-g3/mnt/char-large \ - /tmp/repo22-g3/mnt/block-large \ - /tmp/repo22-g3/mnt/fifo - -## Expected Results - -All type/rdev probes pass; each special inode has size and st_blocks zero and -a nonzero generation. Record both fixture hashes and clean each mount/md. diff --git a/tests/TC057-access-allowed.md b/tests/TC057-access-allowed.md deleted file mode 100644 index 801fd3e..0000000 --- a/tests/TC057-access-allowed.md +++ /dev/null @@ -1,24 +0,0 @@ -# Test Case: access - Allowed Operations - -**Test ID**: TC057-access-allowed -**Category**: VFS Integration -**Priority**: High - -## Objective - -Verify a non-root credential receives read and execute access allowed by the -recorded mode bits on the read-only mount. - -## Procedure - -Mount vfs/vfs-plain.erofs and run with the native nobody credential: - - su -m nobody -c './g3_vfs_probe expect-success access-read /tmp/repo22-g3/mnt/testdir/file.txt' - su -m nobody -c './g3_vfs_probe expect-success access-exec /tmp/repo22-g3/mnt/testdir' - su -m nobody -c './g3_vfs_probe expect-success access-exec /tmp/repo22-g3/mnt/testdir/script.sh' - su -m nobody -c './g3_vfs_probe expect-success open-read /tmp/repo22-g3/mnt/testdir/file.txt' - -## Expected Results - -All four direct checks report success with errno 0. Record the nobody uid, -fixture modes, command output, and cleanup. diff --git a/tests/TC058-access-denied.md b/tests/TC058-access-denied.md deleted file mode 100644 index b8cc139..0000000 --- a/tests/TC058-access-denied.md +++ /dev/null @@ -1,24 +0,0 @@ -# Test Case: access - Denied Operations - -**Test ID**: TC058-access-denied -**Category**: VFS Integration -**Priority**: High - -## Objective - -Verify direct access checks distinguish permission denial from read-only -modification denial for a non-root credential. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - su -m nobody -c './g3_vfs_probe expect-error access-read /tmp/repo22-g3/mnt/testdir/rootonly.txt 13' - su -m nobody -c './g3_vfs_probe expect-error access-read /tmp/repo22-g3/mnt/testdir/writeonly.txt 13' - su -m nobody -c './g3_vfs_probe expect-error access-exec /tmp/repo22-g3/mnt/testdir/noexec.txt 13' - su -m nobody -c './g3_vfs_probe expect-error access-write /tmp/repo22-g3/mnt/testdir/file.txt 30' - -## Expected Results - -Mode-bit denials report EACCES (13); requested modification of the regular -file reports EROFS (30). Record exact helper output and cleanup. diff --git a/tests/TC059-readlink.md b/tests/TC059-readlink.md deleted file mode 100644 index e2f2deb..0000000 --- a/tests/TC059-readlink.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: readlink Exact Targets - -**Test ID**: TC059-readlink -**Category**: VFS Integration -**Priority**: High - -## Objective - -Verify short, relative, broken, and long symlink targets are returned exactly, -including the long inline target length and bytes. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - ./g3_vfs_probe readlink /tmp/repo22-g3/mnt/testdir/shortlink - ./g3_vfs_probe readlink /tmp/repo22-g3/mnt/testdir/relative-link - ./g3_vfs_probe readlink /tmp/repo22-g3/mnt/testdir/broken-link - ./g3_vfs_probe readlink /tmp/repo22-g3/mnt/testdir/long-link - cmp fixtures/vfs/source/testdir/file.txt \ - /tmp/repo22-g3/mnt/testdir/shortlink - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/testdir/broken-link 2 - -## Expected Results - -The helper reports exact targets and hashes: shortlink is file.txt, the -relative link is subdir/child.txt, broken-link is missing-target, and long-link -has the recorded 119-byte target. Following valid links matches source; the -broken target returns errno 2. Record output and cleanup. diff --git a/tests/TC060-pathconf.md b/tests/TC060-pathconf.md deleted file mode 100644 index 899c6e9..0000000 --- a/tests/TC060-pathconf.md +++ /dev/null @@ -1,25 +0,0 @@ -# Test Case: pathconf Queries - -**Test ID**: TC060-pathconf -**Category**: VFS Integration -**Priority**: High -**Latest Result**: PASS on exact cdba7e54f KLD; see -tests/results/manual/2026-08-09T1120Z-tc060-fix/manual-test-report.md and the -resolved issues/TC060-pathconf-standard-values.md. - -## Objective - -Verify direct pathconf(2) results for EROFS limits and read-only conventions. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - ./g3_vfs_probe pathconf /tmp/repo22-g3/mnt/testdir - -## Expected Results - -The helper must exit zero and report name_max=255, a positive path_max, -filesizebits=64, a positive link_max, no_trunc=1, and chown_restricted=1. -EINVAL for either standard query is a kernel failure, not an environmental -skip. Record every value/errno and cleanup. diff --git a/tests/TC061-setattr-chmod-reject.md b/tests/TC061-setattr-chmod-reject.md deleted file mode 100644 index e1d5eb4..0000000 --- a/tests/TC061-setattr-chmod-reject.md +++ /dev/null @@ -1,24 +0,0 @@ -# Test Case: chmod Rejection - -**Test ID**: TC061-setattr-chmod-reject -**Category**: Read-Only Enforcement -**Priority**: Critical - -## Objective - -Verify a direct chmod request returns EROFS and leaves metadata unchanged. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/testdir/file.txt > TC061-before.txt - ./g3_vfs_probe expect-error chmod \ - /tmp/repo22-g3/mnt/testdir/file.txt 30 - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/testdir/file.txt > TC061-after.txt - cmp TC061-before.txt TC061-after.txt - -## Expected Results - -chmod reports errno 30 and complete metadata remains unchanged. Record output, -fixture hash, and cleanup. diff --git a/tests/TC062-setattr-chown-reject.md b/tests/TC062-setattr-chown-reject.md deleted file mode 100644 index 563e682..0000000 --- a/tests/TC062-setattr-chown-reject.md +++ /dev/null @@ -1,25 +0,0 @@ -# Test Case: chown Rejection - -**Test ID**: TC062-setattr-chown-reject -**Category**: Read-Only Enforcement -**Priority**: Critical - -## Objective - -Verify a privileged direct chown request returns EROFS and leaves ownership -unchanged. - -## Procedure - -Mount vfs/vfs-plain.erofs as root and run: - - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/testdir/file.txt > TC062-before.txt - ./g3_vfs_probe expect-error chown \ - /tmp/repo22-g3/mnt/testdir/file.txt 30 - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/testdir/file.txt > TC062-after.txt - cmp TC062-before.txt TC062-after.txt - -## Expected Results - -chown to uid/gid 65534 reports errno 30 and metadata remains unchanged. Record -output and cleanup. diff --git a/tests/TC063-setattr-write-reject.md b/tests/TC063-setattr-write-reject.md deleted file mode 100644 index 96e963a..0000000 --- a/tests/TC063-setattr-write-reject.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: Write, Truncate, and Create Rejection - -**Test ID**: TC063-setattr-write-reject -**Category**: Read-Only Enforcement -**Priority**: Critical - -## Objective - -Verify direct open-for-write, truncate, and create requests return EROFS and -leave the mounted tree unchanged. - -## Procedure - -Mount vfs/vfs-plain.erofs and run: - - sha256 /tmp/repo22-g3/mnt/testdir/file.txt > TC063-before.txt - ./g3_vfs_probe expect-error open-rdwr \ - /tmp/repo22-g3/mnt/testdir/file.txt 30 - ./g3_vfs_probe expect-error truncate \ - /tmp/repo22-g3/mnt/testdir/file.txt 30 - ./g3_vfs_probe expect-error create \ - /tmp/repo22-g3/mnt/testdir/newfile.txt 30 - sha256 /tmp/repo22-g3/mnt/testdir/file.txt > TC063-after.txt - cmp TC063-before.txt TC063-after.txt - test ! -e /tmp/repo22-g3/mnt/testdir/newfile.txt - -## Expected Results - -All mutation syscalls report errno 30, the file hash is unchanged, and no new -entry exists. Record output and cleanup. diff --git a/tests/TC064-vop-getpages-normal.md b/tests/TC064-vop-getpages-normal.md deleted file mode 100644 index 4fed26f..0000000 --- a/tests/TC064-vop-getpages-normal.md +++ /dev/null @@ -1,25 +0,0 @@ -# Test Case: FreeBSD 15 Local Vnode Pager - -**Test ID**: TC064-vop-getpages-normal -**Category**: VM Integration -**Priority**: Critical - -## Objective - -Verify the exact KLD resolves FreeBSD 15 local pager entry points and real -plain/LZ4 vnode mappings fault data through the pager. - -## Procedure - -Record unresolved KLD symbols with nm -u. On separate fresh mounts of -vfs/vfs-plain.erofs and vfs/vfs-lz4.erofs, run: - - ./mmap_fault /tmp/repo22-g3/mnt/pager.bin - -## Expected Results - -The KLD loads with vnode_pager_local_getpages and -vnode_pager_local_getpages_async resolved. Both helpers report identical -21211-byte FNV hashes, six random faults, partial-EOF zeroing, expected child -SIGBUS, denied writable shared mapping, private COW, and O_RDWR EROFS. Record -both image hashes, dmesg, and per-mount cleanup. diff --git a/tests/TC065-vop-getpages-mmap.md b/tests/TC065-vop-getpages-mmap.md deleted file mode 100644 index ee78242..0000000 --- a/tests/TC065-vop-getpages-mmap.md +++ /dev/null @@ -1,24 +0,0 @@ -# Test Case: mmap Through VOP_GETPAGES - -**Test ID**: TC065-vop-getpages-mmap -**Category**: VM Integration -**Priority**: Critical - -## Objective - -Verify page-fault-driven reads, EOF semantics, and private/shared mapping -behavior for plain and compressed EROFS files. - -## Procedure - -On separate fresh mounts of vfs/vfs-plain.erofs and vfs/vfs-lz4.erofs: - - ./mmap_fault /tmp/repo22-g3/mnt/pager.bin - -## Expected Results - -Random and sequential mapped bytes match pread exactly; the partial EOF page -is zero, the next full page produces the helper's expected child SIGBUS, -writable MAP_SHARED is EACCES, MAP_PRIVATE COW succeeds, and O_RDWR is EROFS. -The plain and LZ4 logical FNV hashes are identical. Direct read utilities are -not acceptance evidence. Record dmesg and cleanup. diff --git a/tests/TC066-vop-bmap-unsupported.md b/tests/TC066-vop-bmap-unsupported.md deleted file mode 100644 index 21a4108..0000000 --- a/tests/TC066-vop-bmap-unsupported.md +++ /dev/null @@ -1,24 +0,0 @@ -# Test Case: Unsupported BMAP with Pager Fallback - -**Test ID**: TC066-vop-bmap-unsupported -**Category**: VM Integration -**Priority**: High - -## Objective - -Verify the exact source KLD keeps erofs_bmap as EOPNOTSUPP while FreeBSD's -local vnode pager successfully faults plain and compressed files through -VOP_READ. - -## Procedure - -Audit the exact archived source registration and record KLD symbols. Then, on -fresh mounts of vfs/vfs-plain.erofs and vfs/vfs-lz4.erofs, run: - - ./mmap_fault /tmp/repo22-g3/mnt/pager.bin - -## Expected Results - -The source returns EOPNOTSUPP from erofs_bmap and registers both local pager -entry points. Both real mmap tests pass; dmesg has no "No strategy for -buffer", assertion, trap, or panic. Record source/KLD/image hashes and cleanup. diff --git a/tests/TC067-shared-xattr-scan-found.md b/tests/TC067-shared-xattr-scan-found.md deleted file mode 100644 index 5843fb0..0000000 --- a/tests/TC067-shared-xattr-scan-found.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test Case: Shared xattr scan found - -**Test ID**: TC067-shared-xattr-scan-found -**Fixture**: `basic.erofs`, `/shared-a`, `/shared-b` - -## Objective - -Verify that shared IDs are scanned after inline entries and return exact data. -The generator manifest must show `shared_key` in the shared table and `local` -inline on `/shared-a`. - -## Manual steps - -Mount `basic.erofs` using `G4-MANUAL-SETUP.md`, then run: - -```sh -lsextattr user /mnt/repo22-g4/shared-a -getextattr -qq -x user shared_key /mnt/repo22-g4/shared-a -getextattr -qq -x user shared_key /mnt/repo22-g4/shared-b -getextattr -qq -x user local /mnt/repo22-g4/shared-a -stat -f 'size=%z inode=%i' /mnt/repo22-g4/shared-a -``` - -## Expected results - -Both shared reads equal `shared-value\0exact` byte-for-byte, the inline read is -`in-bounds-local`, and list output contains all four expected user names. -Unmount and detach before the next TC. diff --git a/tests/TC068-shared-xattr-scan-notfound.md b/tests/TC068-shared-xattr-scan-notfound.md deleted file mode 100644 index 3b1e49e..0000000 --- a/tests/TC068-shared-xattr-scan-notfound.md +++ /dev/null @@ -1,26 +0,0 @@ -# Test Case: Shared xattr scan not found - -**Test ID**: TC068-shared-xattr-scan-notfound -**Fixture**: `basic.erofs`, `/shared-a` - -## Objective - -Verify clean exhaustion of inline and shared scans and namespace isolation. - -## Manual steps - -Mount `basic.erofs`, list the user namespace, then capture both misses: - -```sh -lsextattr user /mnt/repo22-g4/shared-a -truss -o /tmp/tc068-name.truss \ - getextattr -qq user nonexistent /mnt/repo22-g4/shared-a -truss -o /tmp/tc068-ns.truss \ - getextattr -qq system shared_key /mnt/repo22-g4/shared-a -grep extattr_get_file /tmp/tc068-*.truss -``` - -## Expected results - -Both kernel calls return `ENOATTR` (87), with no `EINTEGRITY`, delay, or name -leak into the FreeBSD `system` namespace. Unmount and detach the image. diff --git a/tests/TC069-shared-xattr-list-multiple.md b/tests/TC069-shared-xattr-list-multiple.md deleted file mode 100644 index 61191ea..0000000 --- a/tests/TC069-shared-xattr-list-multiple.md +++ /dev/null @@ -1,23 +0,0 @@ -# Test Case: List multiple shared xattrs - -**Test ID**: TC069-shared-xattr-list-multiple -**Fixture**: `basic.erofs`, `/shared-multi` - -## Objective - -Verify FreeBSD's length-prefixed extattr list for several shared user entries. - -## Manual steps - -```sh -lsextattr user /mnt/repo22-g4/shared-multi -getextattr -qq -x user shared_key /mnt/repo22-g4/shared-multi -getextattr -qq -x user shared_comment /mnt/repo22-g4/shared-multi -getextattr -qq -x user shared_binary /mnt/repo22-g4/shared-multi -``` - -## Expected results - -The list contains exactly `shared_key`, `shared_comment`, and `shared_binary`, -without duplicates. Values are `shared-value\0exact`, `shared-comment`, and -bytes `00` through `0f`. Perform the standard unmount/md cleanup. diff --git a/tests/TC070-shared-trusted-xattr.md b/tests/TC070-shared-trusted-xattr.md deleted file mode 100644 index bab2cc6..0000000 --- a/tests/TC070-shared-trusted-xattr.md +++ /dev/null @@ -1,24 +0,0 @@ -# Test Case: Shared trusted xattr - -**Test ID**: TC070-shared-trusted-xattr -**Fixture**: `basic.erofs`, `/trusted-shared-a`, `/trusted-shared-b` - -## Objective - -Verify a shared Linux `trusted.*` entry through FreeBSD extattr semantics. -The transformer must record `e_name_index: 1 -> 4` for the shared record. - -## Manual steps - -```sh -lsextattr system /mnt/repo22-g4/trusted-shared-a -getextattr -qq -x system trusted.config /mnt/repo22-g4/trusted-shared-a -getextattr -qq -x system trusted.config /mnt/repo22-g4/trusted-shared-b -su -m nobody -c 'getextattr system trusted.config /mnt/repo22-g4/trusted-shared-a' -``` - -## Expected results - -Root receives `trusted-shared-value` from both files. FreeBSD exposes the full -`trusted.config` name in namespace `system`, not a namespace named `trusted`. -The unprivileged system-namespace request is denied. Clean up the mount/md. diff --git a/tests/TC071-shared-security-xattr.md b/tests/TC071-shared-security-xattr.md deleted file mode 100644 index 5f1ff67..0000000 --- a/tests/TC071-shared-security-xattr.md +++ /dev/null @@ -1,25 +0,0 @@ -# Test Case: Shared security xattr - -**Test ID**: TC071-shared-security-xattr -**Fixture**: `basic.erofs`, `/security-shared-a`, `/security-shared-b` - -## Objective - -Verify a shared Linux `security.*` entry mapped to FreeBSD namespace `system`. -The transformer must record `e_name_index: 1 -> 6`. - -## Manual steps - -```sh -lsextattr system /mnt/repo22-g4/security-shared-a -getextattr -qq -x system security.selinux /mnt/repo22-g4/security-shared-a -getextattr -qq -x system security.selinux /mnt/repo22-g4/security-shared-b -truss -o /tmp/tc071.truss getextattr -qq system security.missing \ - /mnt/repo22-g4/security-shared-a -grep extattr_get_file /tmp/tc071.truss -``` - -## Expected results - -Both values are exactly `system_u:object_r:shared_repo22_t:s0\0`; the missing -name returns `ENOATTR` (87). Unmount and detach. diff --git a/tests/TC072-shared-system-xattr-list.md b/tests/TC072-shared-system-xattr-list.md deleted file mode 100644 index d71e7f6..0000000 --- a/tests/TC072-shared-system-xattr-list.md +++ /dev/null @@ -1,26 +0,0 @@ -# Test Case: List the FreeBSD system namespace - -**Test ID**: TC072-shared-system-xattr-list -**Fixture**: `basic.erofs` - -## Objective - -Verify that trusted, security, and POSIX ACL indexes are listed only through -FreeBSD namespace `system` with their ABI names. - -## Manual steps - -```sh -lsextattr system /mnt/repo22-g4/trusted-shared-a -lsextattr system /mnt/repo22-g4/security-shared-a -lsextattr system /mnt/repo22-g4/acl-unordered -lsextattr user /mnt/repo22-g4/trusted-shared-a -stat -f '%HT %Sp' /mnt/repo22-g4/trusted-shared-a \ - /mnt/repo22-g4/security-shared-a /mnt/repo22-g4/acl-unordered -``` - -## Expected results - -The three system lists contain `trusted.config`, `security.selinux`, and -`posix_acl_access`, respectively. The trusted file's user list is empty. -Unmount and detach. diff --git a/tests/TC073-metabox-container.md b/tests/TC073-metabox-container.md deleted file mode 100644 index 56e8eff..0000000 --- a/tests/TC073-metabox-container.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Case: Plain metabox container - -**Test ID**: TC073-metabox-container -**Fixture**: `metabox-plain.erofs`, `/metabox-a`, `/metabox-b` - -## Objective - -Verify metadata and xattrs read from a plain regular metabox carrier. Before -guest use, require manifest fields `METABOX`, `sb_extslots=1`, relocated -`meta_blkaddr`, `metabox_nid`, bit-63 dirent NIDs, and image/source hashes. - -## Manual steps - -```sh -unit=$(mdconfig -a -t vnode -f /tmp/repo22-g4/images/metabox-plain.erofs) -mount -t erofs -o ro /dev/${unit} /mnt/repo22-g4 -stat -f 'mode=%Sp size=%z inode=%i' /mnt/repo22-g4/metabox-a \ - /mnt/repo22-g4/metabox-b -sha256 -q /mnt/repo22-g4/metabox-a /mnt/repo22-g4/metabox-b -lsextattr user /mnt/repo22-g4/metabox-a -getextattr -qq -x user metaboxshared /mnt/repo22-g4/metabox-a -getextattr -qq -x user metaboxshared /mnt/repo22-g4/metabox-b -umount /mnt/repo22-g4 -mdconfig -d -u "${unit#md}" -``` - -## Expected results - -The inode numbers retain bit 63; file hashes match the generated source; -`metaboxshared` is `nonzero-base` for both files. No mount/md remains. diff --git a/tests/TC074-inline-user-xattr-multiple.md b/tests/TC074-inline-user-xattr-multiple.md deleted file mode 100644 index dbd41d3..0000000 --- a/tests/TC074-inline-user-xattr-multiple.md +++ /dev/null @@ -1,24 +0,0 @@ -# Test Case: Multiple inline user xattrs - -**Test ID**: TC074-inline-user-xattr-multiple -**Fixture**: `basic.erofs`, `/inline-multi` - -## Objective - -Verify list/get across several inline user entries, including embedded NUL and -binary values. Qualify their inline offsets in `fixture-manifest.json` first. - -## Manual steps - -```sh -lsextattr user /mnt/repo22-g4/inline-multi -getextattr -qq -x user attr1 /mnt/repo22-g4/inline-multi -getextattr -qq -x user attr2 /mnt/repo22-g4/inline-multi -getextattr -qq -x user attr3 /mnt/repo22-g4/inline-multi -stat -f 'mode=%Sp size=%z' /mnt/repo22-g4/inline-multi -``` - -## Expected results - -Names are `attr1`, `attr2`, `attr3`; values are `one`, `two\0binary`, and -bytes `00` through `1f`. Unmount and detach. diff --git a/tests/TC075-inline-trusted-xattr.md b/tests/TC075-inline-trusted-xattr.md deleted file mode 100644 index 7056660..0000000 --- a/tests/TC075-inline-trusted-xattr.md +++ /dev/null @@ -1,24 +0,0 @@ -# Test Case: Inline trusted xattr - -**Test ID**: TC075-inline-trusted-xattr -**Fixture**: `basic.erofs`, `/inline-trusted` - -## Objective - -Verify an inline entry transformed to Linux index 4 and exposed through the -FreeBSD system namespace. - -## Manual steps - -```sh -lsextattr system /mnt/repo22-g4/inline-trusted -getextattr -qq -x system trusted.admin /mnt/repo22-g4/inline-trusted -truss -o /tmp/tc075.truss getextattr -qq user admin \ - /mnt/repo22-g4/inline-trusted -grep extattr_get_file /tmp/tc075.truss -``` - -## Expected results - -`trusted.admin` equals `trusted-inline-value`; the user-namespace lookup -returns `ENOATTR` (87). Perform standard cleanup. diff --git a/tests/TC076-inline-security-xattr.md b/tests/TC076-inline-security-xattr.md deleted file mode 100644 index 96f0e2a..0000000 --- a/tests/TC076-inline-security-xattr.md +++ /dev/null @@ -1,22 +0,0 @@ -# Test Case: Inline security xattrs - -**Test ID**: TC076-inline-security-xattr -**Fixture**: `basic.erofs`, `/inline-security` - -## Objective - -Verify inline Linux index 6 entries and binary-value preservation through -FreeBSD namespace `system`. - -## Manual steps - -```sh -lsextattr system /mnt/repo22-g4/inline-security -getextattr -qq -x system security.capability /mnt/repo22-g4/inline-security -getextattr -qq -x system security.selinux /mnt/repo22-g4/inline-security -``` - -## Expected results - -Capability bytes are `0100000200000000aabbccdd`; SELinux bytes are -`system_u:object_r:repo22_t:s0\0`. Unmount and detach. diff --git a/tests/TC077-long-prefix-user-xattr.md b/tests/TC077-long-prefix-user-xattr.md deleted file mode 100644 index f7f1db2..0000000 --- a/tests/TC077-long-prefix-user-xattr.md +++ /dev/null @@ -1,25 +0,0 @@ -# Test Case: Long-prefix user xattr - -**Test ID**: TC077-long-prefix-user-xattr -**Fixture**: `basic.erofs`, `/prefix-user-0`, `/prefix-user-1` - -## Objective - -Verify reconstruction of a long user name from packed prefix ID 0. The helper -must self-check prefix record base index 1 and infix -`repo22.application.component.`. - -## Manual steps - -```sh -lsextattr user /mnt/repo22-g4/prefix-user-0 -getextattr -qq -x user repo22.application.component.setting \ - /mnt/repo22-g4/prefix-user-0 -getextattr -qq -x user repo22.application.component.setting \ - /mnt/repo22-g4/prefix-user-1 -``` - -## Expected results - -The full name is listed once and the values are `prefix-value-0` and -`prefix-value-1`. Unmount and detach. diff --git a/tests/TC078-long-prefix-trusted-xattr.md b/tests/TC078-long-prefix-trusted-xattr.md deleted file mode 100644 index 1e9f0d1..0000000 --- a/tests/TC078-long-prefix-trusted-xattr.md +++ /dev/null @@ -1,24 +0,0 @@ -# Test Case: Long-prefix trusted xattr - -**Test ID**: TC078-long-prefix-trusted-xattr -**Fixture**: `basic.erofs`, `/prefix-trusted-0`, `/prefix-trusted-1` - -## Objective - -Verify packed long-prefix reconstruction after the transformer changes prefix -record 1 from user base index 1 to trusted base index 4. - -## Manual steps - -```sh -lsextattr system /mnt/repo22-g4/prefix-trusted-0 -getextattr -qq -x system trusted.repo22.trusted.deep.setting \ - /mnt/repo22-g4/prefix-trusted-0 -getextattr -qq -x system trusted.repo22.trusted.deep.setting \ - /mnt/repo22-g4/prefix-trusted-1 -``` - -## Expected results - -The full trusted name appears in FreeBSD namespace `system`; values are -`trusted-prefix-value-0` and `trusted-prefix-value-1`. Clean up. diff --git a/tests/TC079-packed-prefix-table.md b/tests/TC079-packed-prefix-table.md deleted file mode 100644 index 1ad6505..0000000 --- a/tests/TC079-packed-prefix-table.md +++ /dev/null @@ -1,33 +0,0 @@ -# Test Case: Packed prefix table - -**Test ID**: TC079-packed-prefix-table -**Fixture**: `basic.erofs` - -## Objective - -Verify a non-plain prefix table carried by the packed inode generated by -erofs-utils 1.8.6. - -## Layout qualification - -```sh -dump.erofs -s "$run/images/basic.erofs" -python3 tests/g4_fixtures.py verify --output "$run" -``` - -Require `xattr_prefix_count=2`, nonzero `packed_nid`, prefix start 0, user -record `(base=1, infix=repo22.application.component.)`, trusted record -`(base=4, infix=repo22.trusted.deep.)`, and matching image hash. - -## Guest observations - -```sh -getextattr -qq -x user repo22.application.component.setting \ - /mnt/repo22-g4/prefix-user-2 -getextattr -qq -x system trusted.repo22.trusted.deep.setting \ - /mnt/repo22-g4/prefix-trusted-2 -stat -f 'mode=%Sp inode=%i' /mnt/repo22-g4/prefix-user-2 \ - /mnt/repo22-g4/prefix-trusted-2 -``` - -Both values must end in `-2`; cleanup must be empty. diff --git a/tests/TC080-prefix-table-lookup.md b/tests/TC080-prefix-table-lookup.md deleted file mode 100644 index 39e1a12..0000000 --- a/tests/TC080-prefix-table-lookup.md +++ /dev/null @@ -1,25 +0,0 @@ -# Test Case: Prefix table lookup - -**Test ID**: TC080-prefix-table-lookup -**Fixture**: `basic.erofs`, `/prefix-user-0` through `/prefix-user-3` - -## Objective - -Verify repeated prefix-ID lookup and a clean missing-suffix result. - -## Manual steps - -Run the following lookup separately for suffix files 0, 1, 2, and 3: - -```sh -getextattr -qq -x user repo22.application.component.setting \ - /mnt/repo22-g4/prefix-user-N -truss -o /tmp/tc080.truss getextattr -qq user \ - repo22.application.component.missing /mnt/repo22-g4/prefix-user-0 -grep extattr_get_file /tmp/tc080.truss -``` - -## Expected results - -Values are `prefix-value-N` for each file; the missing suffix is `ENOATTR` -(87), not an integrity failure. Unmount and detach. diff --git a/tests/TC081-metabox-backed-xattr.md b/tests/TC081-metabox-backed-xattr.md deleted file mode 100644 index 0f8c7e6..0000000 --- a/tests/TC081-metabox-backed-xattr.md +++ /dev/null @@ -1,26 +0,0 @@ -# Test Case: Metabox-backed shared xattrs - -**Test ID**: TC081-metabox-backed-xattr -**Fixture**: `metabox-plain.erofs`, `/metabox-a`, `/metabox-b` - -## Objective - -Verify shared entries, inline entries, and long-prefix shared entries from a -metabox logical stream. Require `SHARED_EA_IN_METABOX`, `xattr_blkaddr=1`, -carrier size/bounds, shared IDs, and prefix fields in the generator manifest. - -## Manual steps - -```sh -lsextattr user /mnt/repo22-g4/metabox-a -getextattr -qq -x user metaboxshared /mnt/repo22-g4/metabox-a -getextattr -qq -x user metaboxshared /mnt/repo22-g4/metabox-b -getextattr -qq -x user shared-prefix-key /mnt/repo22-g4/metabox-a -getextattr -qq -x user repo22.application.component.setting \ - /mnt/repo22-g4/metabox-a -``` - -## Expected results - -Values are `nonzero-base`, `nonzero-base`, `shared-value`, and -`long-prefix-value`; all names list once. Clean up. diff --git a/tests/TC082-system-namespace-subset.md b/tests/TC082-system-namespace-subset.md deleted file mode 100644 index 301948e..0000000 --- a/tests/TC082-system-namespace-subset.md +++ /dev/null @@ -1,25 +0,0 @@ -# Test Case: FreeBSD system namespace subset - -**Test ID**: TC082-system-namespace-subset -**Fixture**: `basic.erofs`, `/acl-unordered` - -## Objective - -Verify POSIX ACL exposure in FreeBSD namespace `system` and isolation from -namespace `user`. - -## Manual steps - -```sh -lsextattr system /mnt/repo22-g4/acl-unordered -getextattr -qq -x system posix_acl_access /mnt/repo22-g4/acl-unordered -getfacl -n /mnt/repo22-g4/acl-unordered -truss -o /tmp/tc082.truss getextattr -qq user posix_acl_access \ - /mnt/repo22-g4/acl-unordered -grep extattr_get_file /tmp/tc082.truss -``` - -## Expected results - -Raw ACL starts with little-endian version 2; `getfacl` shows named users 3002 -then 2002. The user-namespace request returns `ENOATTR` (87). Clean up. diff --git a/tests/TC083-user-namespace-subset.md b/tests/TC083-user-namespace-subset.md deleted file mode 100644 index 2b16878..0000000 --- a/tests/TC083-user-namespace-subset.md +++ /dev/null @@ -1,25 +0,0 @@ -# Test Case: User namespace subset - -**Test ID**: TC083-user-namespace-subset -**Fixture**: `basic.erofs`, `/user-subset` - -## Objective - -Verify several application user xattrs without exposing them through FreeBSD -namespace `system`. - -## Manual steps - -```sh -lsextattr user /mnt/repo22-g4/user-subset -getextattr -qq -x user comment /mnt/repo22-g4/user-subset -getextattr -qq -x user author /mnt/repo22-g4/user-subset -getextattr -qq -x user checksum /mnt/repo22-g4/user-subset -getextattr -qq -x user com.repo22.app.setting /mnt/repo22-g4/user-subset -lsextattr system /mnt/repo22-g4/user-subset -``` - -## Expected results - -The user values are `subset-comment`, `repo22`, -`sha256:0123456789abcdef`, and `enabled`. The system list is empty. Clean up. diff --git a/tests/TC084-lz4-basic.md b/tests/TC084-lz4-basic.md deleted file mode 100644 index 5aee0e8..0000000 --- a/tests/TC084-lz4-basic.md +++ /dev/null @@ -1,71 +0,0 @@ -# Test Case: LZ4 Compression Basic Support - -**Test ID**: TC084-lz4-basic - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: Critical -**Regression**: None - -## Objective -Verify basic LZ4 compression support for reading compressed files. - -## Preconditions -- EROFS image with LZ4 compression: `test-lz4-basic.erofs` -- Test file: `/files/test-lz4.txt` (1MB, highly compressible) -- Mount point: `/mnt/test` - -## Test Steps -1. Mount the LZ4-compressed image: - ``` - mount -t erofs /dev/md0 /mnt/test - ``` - -2. Read entire compressed file: - ``` - cat /mnt/test/files/test-lz4.txt > /tmp/output.txt - ``` - -3. Verify file size: - ``` - stat -f %z /mnt/test/files/test-lz4.txt - ``` - -4. Compare with reference (uncompressed): - ``` - cmp /tmp/output.txt /tmp/reference.txt - ``` - -5. Test multiple LZ4-compressed files: - ``` - for f in /mnt/test/files/lz4-*.txt; do - cat "$f" > /dev/null - done - ``` - -## Expected Results -- Step 1: Mount succeeds -- Step 2: File read successfully, decompressed transparently -- Step 3: Size matches original uncompressed size (1048576 bytes) -- Step 4: Files identical (cmp returns 0) -- Step 5: All LZ4 files read without errors - -## Verification Method -- Verify decompression transparent to application -- Confirm data integrity after decompression -- Test LZ4 decompression performance acceptable -- Verify compressed file sizes smaller than uncompressed - -## Cleanup -``` -umount /mnt/test -mdconfig -d -u md0 -rm /tmp/output.txt -``` - -## Notes -- LZ4 provides fast decompression with moderate compression ratio -- EROFS uses LZ4 for general-purpose compression -- Related tests: TC003 (LZ4 compressed read), TC085 (large files) diff --git a/tests/TC085-lz4-large-file.md b/tests/TC085-lz4-large-file.md deleted file mode 100644 index 59368e8..0000000 --- a/tests/TC085-lz4-large-file.md +++ /dev/null @@ -1,78 +0,0 @@ -# Test Case: LZ4 Compression Large File Handling - -**Test ID**: TC085-lz4-large-file - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify LZ4 decompression correctness for large files (>100MB). - -## Preconditions -- Generated image: `images/lz4-large.erofs` -- Test file: `/large.bin`, exactly 268435456 bytes -- Reference: `sources/large/large.bin` and its generated SHA256 -- Mount point: `/mnt/test` - -## Test Steps -1. Mount the image: - ``` - mount -t erofs /dev/md0 /mnt/test - ``` - -2. Verify file size reported correctly: - ``` - stat -f %z /mnt/test/files/large-lz4.bin - ``` - -3. Read entire file and compute checksum: - ``` - sha256 -q /mnt/test/large.bin - sha256 -q sources/large/large.bin - cmp sources/large/large.bin /mnt/test/large.bin - ``` - -4. Test partial reads at various offsets: - ``` - read_probe pread /mnt/test/large.bin 52428800 10485760 guest.bin - read_probe pread sources/large/large.bin 52428800 10485760 source.bin - cmp source.bin guest.bin - read_probe pread /mnt/test/large.bin 209715200 10485760 guest.bin - read_probe pread sources/large/large.bin 209715200 10485760 source.bin - cmp source.bin guest.bin - ``` - -5. Test random access reads: - ``` - # Compare both guest ranges with the same source offsets. - read_probe pread /mnt/test/large.bin 4096000 4096 chunk1 - read_probe pread /mnt/test/large.bin 204800000 4096 chunk2 - ``` - -## Expected Results -- Step 2: Size is 268435456 bytes (256MB) -- Step 3: Checksum matches reference value -- Step 4: Partial reads succeed, no errors -- Step 5: Random access works correctly - -## Verification Method -- Verify data integrity via checksum -- Confirm partial/random reads decompress correctly -- Test memory usage during decompression reasonable -- Verify no memory leaks or buffer overflows - -## Cleanup -``` -umount /mnt/test -mdconfig -d -u "${unit#md}" -rm /tmp/chunk1 /tmp/chunk2 -``` - -## Notes -- Large file decompression tests pcluster handling -- LZ4 decompression should be streaming (low memory) -- Related tests: TC089 (pcluster mapping) diff --git a/tests/TC086-lz4-sequential-read.md b/tests/TC086-lz4-sequential-read.md deleted file mode 100644 index 3a0c47f..0000000 --- a/tests/TC086-lz4-sequential-read.md +++ /dev/null @@ -1,57 +0,0 @@ -# Test Case: LZ4 Sequential Read Correctness - -**Test ID**: TC086-lz4-sequential-read - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High - -## Objective - -Verify complete sequential reads of an LZ4-compressed file with two request -sizes against the original source bytes. Record timings only; repo22 exposes no -filesystem-specific DTrace provider and this test has no fixed throughput -threshold. - -## Fixture - -```sh -work=$(mktemp -d /tmp/repo22-tc086.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -dump.erofs -s "$work/fixtures/valid-lz4.erofs" -dump.erofs --path=/compressed.bin -e "$work/fixtures/valid-lz4.erofs" -``` - -The dump must identify `/compressed.bin` as compressed LZ4 data with real -physical extents. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode -f "$work/fixtures/valid-lz4.erofs") -mount -t erofs -o ro "/dev/$unit" "$work/mnt" - -expected=$(sha256 -q "$work/fixtures/source/compressed.bin") -actual=$(sha256 -q "$work/mnt/compressed.bin") -test "$actual" = "$expected" - -/usr/bin/time -p dd if="$work/mnt/compressed.bin" \ - of="$work/read-4k.bin" bs=4096 2>"$work/time-4k.txt" -cmp "$work/fixtures/source/compressed.bin" "$work/read-4k.bin" -/usr/bin/time -p dd if="$work/mnt/compressed.bin" \ - of="$work/read-1m.bin" bs=1048576 2>"$work/time-1m.txt" -cmp "$work/fixtures/source/compressed.bin" "$work/read-1m.bin" -cat "$work/time-4k.txt" "$work/time-1m.txt" - -umount "$work/mnt" -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -Both complete outputs compare byte-for-byte with the source and both hashes -match. Timing values are recorded with guest CPU, VM, and storage context; one -buffer size is not required to outperform the other. diff --git a/tests/TC087-lz4-random-read.md b/tests/TC087-lz4-random-read.md deleted file mode 100644 index c4275bf..0000000 --- a/tests/TC087-lz4-random-read.md +++ /dev/null @@ -1,53 +0,0 @@ -# Test Case: LZ4 Deterministic Random-Offset Reads - -**Test ID**: TC087-lz4-random-read - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High - -## Objective - -Verify non-sequential reads from deterministic offsets in an LZ4-compressed -file and compare every returned byte with the source. `$RANDOM` and timing-only -checks are not evidence of data correctness. - -## Fixture and Helper - -```sh -work=$(mktemp -d /tmp/repo22-tc087.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -cc -O2 -Wall -Wextra -std=c17 tests/read_probe.c -o "$work/read_probe" -dump.erofs --path=/compressed.bin -e "$work/fixtures/valid-lz4.erofs" -``` - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode -f "$work/fixtures/valid-lz4.erofs") -mount -t erofs -o ro "/dev/$unit" "$work/mnt" -size=$(stat -f %z "$work/fixtures/source/compressed.bin") -test "$size" -gt 1052672 - -index=0 -for offset in 0 4096 65536 1048576 $((size - 4096)); do - "$work/read_probe" pread "$work/mnt/compressed.bin" \ - "$offset" 4096 "$work/guest-$index.bin" - "$work/read_probe" pread "$work/fixtures/source/compressed.bin" \ - "$offset" 4096 "$work/source-$index.bin" - cmp "$work/source-$index.bin" "$work/guest-$index.bin" - index=$((index + 1)) -done - -umount "$work/mnt" -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -All five fixed reads, including the final 4 KiB of the file, return exactly -4096 bytes and compare equal to their corresponding source ranges. No -performance ordering is asserted. diff --git a/tests/TC088-lz4-config-handling.md b/tests/TC088-lz4-config-handling.md deleted file mode 100644 index 2027c01..0000000 --- a/tests/TC088-lz4-config-handling.md +++ /dev/null @@ -1,85 +0,0 @@ -# Test Case: LZ4 Compression Configuration Handling - -**Test ID**: TC088-lz4-config-handling - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: Medium -**Regression**: None - -## Objective -Verify correct handling of different LZ4 compression configurations. - -## Preconditions -- Multiple EROFS images with different LZ4 configs: - - `lz4-pcluster-4k.erofs` (4KB pcluster) - - `lz4-pcluster-64k.erofs` (64KB pcluster) - - `lz4-pcluster-256k.erofs` (256KB pcluster) -- Test file in each: `/files/test.dat` (identical content) -- Mount points: `/mnt/test1`, `/mnt/test2`, `/mnt/test3` -- Generate the images explicitly with `mkfs.erofs 1.8.6`: - ``` - mkfs.erofs -zlz4 -C4096 lz4-pcluster-4k.erofs source - mkfs.erofs -zlz4 -C65536 lz4-pcluster-64k.erofs source - mkfs.erofs -zlz4 -C262144 lz4-pcluster-256k.erofs source - ``` -- Confirm the requested feature/configuration with `dump.erofs -s` before - treating an image as a valid fixture. - -## Test Steps -1. Mount all three images: - ``` - mount -t erofs /dev/md0 /mnt/test1 - mount -t erofs /dev/md1 /mnt/test2 - mount -t erofs /dev/md2 /mnt/test3 - ``` - -2. Read from each and verify identical output: - ``` - sha256 /mnt/test1/files/test.dat - sha256 /mnt/test2/files/test.dat - sha256 /mnt/test3/files/test.dat - ``` - -3. Record image sizes without assuming a monotonic compression ratio: - ``` - stat -f %z /root/lz4-pcluster-4k.erofs - stat -f %z /root/lz4-pcluster-64k.erofs - stat -f %z /root/lz4-pcluster-256k.erofs - ``` - -4. Verify the same offset read with each pcluster size. Timing is diagnostic - only and is not a functional pass criterion: - ``` - time dd if=/mnt/test1/files/test.dat of=/dev/null bs=4K skip=100 count=10 - time dd if=/mnt/test2/files/test.dat of=/dev/null bs=4K skip=100 count=10 - time dd if=/mnt/test3/files/test.dat of=/dev/null bs=4K skip=100 count=10 - ``` - -## Expected Results -- Step 1: All mounts succeed -- Step 2: All three checksums identical (data integrity) -- Step 3: Image sizes are recorded for the exact corpus and mkfs version -- Step 4: All reads succeed and return identical data at the tested offset - -## Verification Method -- Verify all pcluster configurations work correctly -- Record pcluster size effects without assuming they are monotonic for every - corpus or under a QEMU TCG guest -- Test driver handles different configs transparently - -## Cleanup -``` -umount /mnt/test1 /mnt/test2 /mnt/test3 -mdconfig -d -u "${unit1#md}" -mdconfig -d -u "${unit2#md}" -mdconfig -d -u "${unit3#md}" -``` - -## Notes -- Pcluster size is compression unit (physical cluster) -- Larger pcluster -> better compression, worse random access -- Typical sizes: 4KB, 16KB, 64KB, 256KB -- Related tests: TC089 (pcluster mapping) diff --git a/tests/TC089-lz4-pcluster-4k.md b/tests/TC089-lz4-pcluster-4k.md deleted file mode 100644 index 43e1db5..0000000 --- a/tests/TC089-lz4-pcluster-4k.md +++ /dev/null @@ -1,69 +0,0 @@ -# Test Case: LZ4 Compressed Physical Cluster Mapping (4KB) - -**Test ID**: TC089-lz4-pcluster-4k - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify correct physical cluster (pcluster) mapping for LZ4 compression with 4KB pcluster size. - -## Preconditions -- EROFS image with LZ4, 4KB pcluster: `test-lz4-pcluster-4k.erofs` -- Test file: `/files/test-4k.dat` (aligned to 4KB boundaries) -- Mount point: `/mnt/test` - -## Test Steps -1. Mount the image: - ``` - mount -t erofs /dev/md0 /mnt/test - ``` - -2. Read aligned to pcluster boundary (offset 0): - ``` - dd if=/mnt/test/files/test-4k.dat of=/tmp/out1 bs=4K count=1 skip=0 - ``` - -3. Read aligned to pcluster boundary (offset 4K): - ``` - dd if=/mnt/test/files/test-4k.dat of=/tmp/out2 bs=4K count=1 skip=1 - ``` - -4. Read unaligned (crosses pcluster boundary): - ``` - dd if=/mnt/test/files/test-4k.dat of=/tmp/out3 bs=2K count=2 skip=1 - ``` - -5. Read spanning multiple pclusters: - ``` - dd if=/mnt/test/files/test-4k.dat of=/tmp/out4 bs=16K count=1 - ``` - -## Expected Results -- Step 2: Single pcluster decompressed, correct data -- Step 3: Adjacent pcluster decompressed, correct data -- Step 4: Two pclusters decompressed for unaligned read -- Step 5: Four pclusters decompressed (16K / 4K = 4) - -## Verification Method -- Verify pcluster boundaries handled correctly -- Confirm unaligned reads work (may decompress extra pclusters) -- Test each pcluster independently decompressible -- Verify correct data returned for all read patterns - -## Cleanup -``` -umount /mnt/test -mdconfig -d -u md0 -rm /tmp/out1 /tmp/out2 /tmp/out3 /tmp/out4 -``` - -## Notes -- 4KB pcluster provides good random access performance -- Each 4KB logical region compressed independently -- Small pcluster = less compression, better random access -- Related tests: TC090 (64KB pcluster) diff --git a/tests/TC090-lz4-pcluster-64k.md b/tests/TC090-lz4-pcluster-64k.md deleted file mode 100644 index 6094070..0000000 --- a/tests/TC090-lz4-pcluster-64k.md +++ /dev/null @@ -1,72 +0,0 @@ -# Test Case: LZ4 Compressed Physical Cluster Mapping (64KB) - -**Test ID**: TC090-lz4-pcluster-64k - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify correct physical cluster mapping for LZ4 compression with 64KB pcluster size. - -## Preconditions -- EROFS image with LZ4, 64KB pcluster: `test-lz4-pcluster-64k.erofs` -- Test file: `/files/test-64k.dat` (1MB, highly compressible) -- Mount point: `/mnt/test` -- Generate with `mkfs.erofs -zlz4 -C65536 test-lz4-pcluster-64k.erofs source` - and confirm `compr_cfgs big_pcluster` with `dump.erofs -s`. - -## Test Steps -1. Mount the image: - ``` - mount -t erofs /dev/md0 /mnt/test - ``` - -2. Read first pcluster (64KB): - ``` - dd if=/mnt/test/files/test-64k.dat of=/tmp/out1 bs=64K count=1 - ``` - -3. Read small portion requiring full pcluster decompression: - ``` - dd if=/mnt/test/files/test-64k.dat of=/tmp/out2 bs=4K count=1 skip=1 - ``` - -4. Read crossing pcluster boundary: - ``` - dd if=/mnt/test/files/test-64k.dat of=/tmp/cross bs=1 skip=65504 count=64 - dd if=/root/reference/test-64k.dat of=/tmp/cross.expected bs=1 skip=65504 count=64 - cmp /tmp/cross /tmp/cross.expected - ``` - -5. Record the physical extent layout: - ``` - dump.erofs --path=/files/test-64k.dat -e test-lz4-pcluster-64k.erofs - ``` - -## Expected Results -- Step 2: Full 64KB pcluster decompressed correctly -- Step 3: Reading 4KB requires decompressing entire 64KB pcluster -- Step 4: Boundary crossing handled correctly -- Step 5: The fixture contains a real compressed big-pcluster extent - -## Verification Method -- Verify large pcluster decompression works -- Confirm small reads still trigger full pcluster decompression -- Test memory usage reasonable for 64KB decompression -- Do not require a better compression ratio than 4KB for every corpus - -## Cleanup -``` -umount /mnt/test -mdconfig -d -u md0 -rm /tmp/out* /tmp/cross /tmp/cross.expected -``` - -## Notes -- 64KB pcluster provides better compression at cost of random access -- Common pcluster size for read-heavy workloads -- Related tests: TC089 (4KB pcluster), TC088 (config handling) diff --git a/tests/TC091-ztailpacking-basic.md b/tests/TC091-ztailpacking-basic.md deleted file mode 100644 index 1e8be76..0000000 --- a/tests/TC091-ztailpacking-basic.md +++ /dev/null @@ -1,76 +0,0 @@ -# Test Case: LZ4 ztailpacking Data Path - -**Test ID**: TC091-ztailpacking-basic - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify correct operation of ztailpacking (compressed tail packing) for LZ4-compressed files. - -## Preconditions -- EROFS image with ztailpacking enabled: `test-ztailpacking.erofs` -- Test file: `/inline.dat` (64KB deterministic compressible data) -- Mount point: `/mnt/test` -- Generate with: - ``` - mkfs.erofs -zlz4 -C4096 -Eztailpacking test-ztailpacking.erofs source - ``` -- Before mounting, require `dump.erofs -s` to report `ztailpacking` and - `dump.erofs --path=/inline.dat -e` to show the compressed physical extent - inside the metadata block. Enabling the mkfs option alone does not prove - that the target file was tail-packed. - -## Test Steps -1. Mount the image: - ``` - mount -t erofs /dev/md0 /mnt/test - ``` - -2. Read file with packed tail: - ``` - cat /mnt/test/inline.dat > /tmp/output.dat - ``` - -3. Verify file size correct: - ``` - stat -f %z /mnt/test/inline.dat - ``` - -4. Read file tail specifically: - ``` - tail -c 1024 /mnt/test/inline.dat > /tmp/tail.dat - ``` - -5. Verify data integrity: - ``` - sha256 /tmp/output.dat - ``` - -## Expected Results -- Step 2: File read successfully with tail decompressed -- Step 3: Size matches original uncompressed size -- Step 4: Tail data correct -- Step 5: Checksum matches reference - -## Verification Method -- Verify tail-packed data decompressed correctly -- Confirm storage efficiency (tail stored inline or in special area) -- Test tail packing transparent to reader -- Verify boundary between main data and packed tail handled correctly - -## Cleanup -``` -umount /mnt/test -mdconfig -d -u md0 -rm /tmp/output.dat /tmp/tail.dat -``` - -## Notes -- ztailpacking stores compressed file tail inline with inode or in special area -- Reduces fragmentation and improves small file efficiency -- Related tests: TC092 (edge cases), TC033 (uncompressed tailpacking) diff --git a/tests/TC092-ztailpacking-edge.md b/tests/TC092-ztailpacking-edge.md deleted file mode 100644 index c6563c5..0000000 --- a/tests/TC092-ztailpacking-edge.md +++ /dev/null @@ -1,79 +0,0 @@ -# Test Case: LZ4 ztailpacking Edge Cases - -**Test ID**: TC092-ztailpacking-edge - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: Medium -**Regression**: None - -## Objective -Verify correct handling of edge cases in ztailpacking (compressed tail packing) feature. - -## Preconditions -- Generated image: `images/lz4-ztail.erofs` -- Test files: - - `/exact-pcluster.dat` (4096-byte non-tail control) - - `/one-byte-tail.dat` (4097-byte logical boundary case) - - `/max-tail.dat` (8191-byte logical boundary case) - - `/zero-tail.dat` (empty-file control) -- `/inline.dat` is independently proven by the manifest to have - `Z_EROFS_ADVISE_INLINE_PCLUSTER` and nonzero `h_idata_size`. The four edge - controls must not be described as ztailpacked unless their own map headers - prove it. -- Mount point: `/mnt/test` - -## Test Steps -1. Mount the image: - ``` - mount -t erofs /dev/md0 /mnt/test - ``` - -2. Read file with exact pcluster size (no tail): - ``` - cat /mnt/test/files/exact-pcluster.dat > /tmp/out1 - sha256sum /tmp/out1 - ``` - -3. Read file with minimal tail (1 byte): - ``` - cat /mnt/test/files/one-byte-tail.dat > /tmp/out2 - tail -c 1 /tmp/out2 | od -A n -t x1 - ``` - -4. Read file with maximum tail size: - ``` - cat /mnt/test/files/max-tail.dat > /tmp/out3 - stat -f %z /tmp/out3 - ``` - -5. Read file with zero-length tail: - ``` - cat /mnt/test/files/zero-tail.dat > /tmp/out4 - ``` - -## Expected Results -- Step 2: File without tail read correctly -- Steps 2-5: Every exact logical size, full SHA256, and byte comparison matches - its source file - -## Verification Method -- Verify all tail size edge cases handled -- Confirm boundary conditions don't cause errors -- Test tail packing storage limits enforced correctly -- Verify data integrity for all edge cases - -## Cleanup -``` -umount /mnt/test -mdconfig -d -u "${unit#md}" -rm /tmp/out* -``` - -## Notes -- Tail size limits depend on inline storage capacity -- Zero-length tail is valid edge case -- Exact pcluster alignment means no tail needed -- Related tests: TC091 (basic ztailpacking) diff --git a/tests/TC093-chunk-single-device.md b/tests/TC093-chunk-single-device.md deleted file mode 100644 index 028f9b4..0000000 --- a/tests/TC093-chunk-single-device.md +++ /dev/null @@ -1,55 +0,0 @@ -# Test Case: Indexed Chunk Data on the Primary Device - -**Test ID**: TC093-chunk-single-device -**Category**: Chunk-Based -**Priority**: High - -## Objective - -Verify 8-byte chunk indexes whose device ID is zero without a device table. -This is not a 4-byte block-map substitute. - -## Fixture - -Use `single-indexed.erofs` from the fresh G6 generator. It is derived from the -mkfs blob baseline by asserting each original `(high=0, device=1, pblk)`, -folding the new blob bytes behind the metadata block, changing every index to -`device=0`, adding one to each pblk, and removing the device-table feature. -`verify` must report `extra_devices=0` and `entry_size=8`. - -## Manual procedure - -```sh -I=/root/repo22-g6/images -S=/root/repo22-g6/sources/chunk -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/single-indexed.erofs" -u 90 -mount -t erofs -o ro /dev/md90 /mnt/g6 -sha256 "$S/indexed.bin" /mnt/g6/indexed.bin -cmp "$S/indexed.bin" /mnt/g6/indexed.bin -dd if="$S/indexed.bin" of=/tmp/tc093.expected bs=1 skip=28672 count=8192 2>/dev/null -dd if=/mnt/g6/indexed.bin of=/tmp/tc093.actual bs=1 skip=28672 count=8192 2>/dev/null -cmp /tmp/tc093.expected /tmp/tc093.actual -``` - -## Expected results - -- Mount succeeds without any `device.N` option. -- Full-file and cross-32768-byte-chunk comparisons pass. -- No external GEOM provider or device-table lookup is attempted. - -## Cleanup and zero-state check - -```sh -umount /mnt/g6 -mdconfig -d -u 90 -kldunload erofs -rm -f /tmp/tc093.expected /tmp/tc093.actual -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md90|erofs' || true -``` - -All four final outputs must be empty. diff --git a/tests/TC094-chunk-multi-device.md b/tests/TC094-chunk-multi-device.md deleted file mode 100644 index 8ad6f70..0000000 --- a/tests/TC094-chunk-multi-device.md +++ /dev/null @@ -1,116 +0,0 @@ -# Test Case: Chunk Indexes and Compressed Pcluster on External Slots - -**Test ID**: TC094-chunk-multi-device -**Category**: Chunk-Based / Multi-Device -**Priority**: Critical - -## Objective - -Verify alternating external chunk slots, a nonzero device ID with -`uniaddr=0`, provider identity, and a real external two-block LZ4 pcluster. - -## Fixture qualification - -Use the fresh G6 artifacts and manifest: - -- `multi2-*`: `/cross-device.bin` has three 131072-byte chunks with device IDs - `1,2,1`; slot blocks differ, so provider swapping fails deterministically. -- `multi2-uniaddr0-*`: slot 1 has `uniaddr=0`, but indexes retain device ID 1. -- `lz4-external-pcluster-*`: the original fixed 8192-byte LZ4 pcluster is - fsck-qualified before relocation. The generated primary is only 8192 bytes, - contains metadata plus the table, and has HEAD pblk `2`; both compressed - blocks exist only in slot 1. - -## Alternating slots and provider identity - -```sh -I=/root/repo22-g6/images -S=/root/repo22-g6/sources/chunk -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/multi2-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2.blob" -u 92 -mount -t erofs -o ro -o device.2=/dev/md92 -o device.1=/dev/md91 \ - /dev/md90 /mnt/g6 -cmp "$S/cross-device.bin" /mnt/g6/cross-device.bin -sha256 "$S/cross-device.bin" /mnt/g6/cross-device.bin -dd if="$S/cross-device.bin" of=/tmp/tc094.expected bs=1 skip=126976 count=8192 2>/dev/null -dd if=/mnt/g6/cross-device.bin of=/tmp/tc094.actual bs=1 skip=126976 count=8192 2>/dev/null -cmp /tmp/tc094.expected /tmp/tc094.actual -for off in 0 131072 262144; do - dd if="$S/cross-device.bin" of=/tmp/tc094.src.$off bs=1 skip=$off count=4096 2>/dev/null - dd if=/mnt/g6/cross-device.bin of=/tmp/tc094.mnt.$off bs=1 skip=$off count=4096 2>/dev/null - cmp /tmp/tc094.src.$off /tmp/tc094.mnt.$off -done -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -Confirm swapped providers fail with `ENXIO`: - -```sh -mdconfig -a -t vnode -f "$I/multi2-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2.blob" -u 92 -truss -f -o /tmp/tc094-swap.truss mount -t erofs -o ro \ - -o device.1=/dev/md92 -o device.2=/dev/md91 /dev/md90 /mnt/g6 -tail -20 /tmp/tc094-swap.truss -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -## Zero unified address with explicit device ID - -```sh -mdconfig -a -t vnode -f "$I/multi2-uniaddr0-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-uniaddr0-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-uniaddr0-slot2.blob" -u 92 -mount -t erofs -o ro -o device.2=/dev/md92 -o device.1=/dev/md91 \ - /dev/md90 /mnt/g6 -cmp "$S/cross-device.bin" /mnt/g6/cross-device.bin -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -## External compressed pcluster - -```sh -P=/root/repo22-g6/sources/pcluster -mdconfig -a -t vnode -f "$I/lz4-external-pcluster-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/lz4-external-pcluster-slot1.blob" -u 91 -mount -t erofs -o ro -o device.1=/dev/md91 /dev/md90 /mnt/g6 -stat -f '%z' /mnt/g6/external-pcluster.bin -sha256 "$P/external-pcluster.bin" /mnt/g6/external-pcluster.bin -cmp "$P/external-pcluster.bin" /mnt/g6/external-pcluster.bin -dd if="$P/external-pcluster.bin" of=/tmp/tc094-p.src bs=1 skip=61440 count=8192 2>/dev/null -dd if=/mnt/g6/external-pcluster.bin of=/tmp/tc094-p.mnt bs=1 skip=61440 count=8192 2>/dev/null -cmp /tmp/tc094-p.src /tmp/tc094-p.mnt -truss -f -o /tmp/tc094-p-ebusy.truss mdconfig -d -u 91 -tail -20 /tmp/tc094-p-ebusy.truss -``` - -The 131072-byte output must match although the primary has no old compressed -extent; normal slot detach must return `EBUSY`. - -## Cleanup and zero-state check - -```sh -umount /mnt/g6 -mdconfig -d -u 91 -mdconfig -d -u 90 -kldunload erofs -rm -f /tmp/tc094.expected /tmp/tc094.actual /tmp/tc094.src.* \ - /tmp/tc094.mnt.* /tmp/tc094-p.src /tmp/tc094-p.mnt -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[0-2]|erofs' || true -``` - -All four final outputs must be empty, with no panic, trap, or hang. diff --git a/tests/TC095-chunk-index-read.md b/tests/TC095-chunk-index-read.md deleted file mode 100644 index b6b5e3c..0000000 --- a/tests/TC095-chunk-index-read.md +++ /dev/null @@ -1,113 +0,0 @@ -# Test Case: Chunk Index Decode and Mapped Device ID - -**Test ID**: TC095-chunk-index-read -**Category**: Chunk-Based -**Priority**: High - -## Objective - -Verify first/middle/last 8-byte indexes, zero-high and nonzero-high 48-bit -decode, the rounded device-ID mask, and `ENODEV` for mapped ID 3. - -## Field qualification - -After G6 generation, print the fields on the host that will be exercised: - -```sh -OUTPUT=/path/to/generated-fixtures -python3 -B - "$OUTPUT/manifest.json" <<'PY' -import json, sys -m = json.load(open(sys.argv[1])) -for fixture in ('multi2', 'multi2-unified48', 'bad-mapped-device3'): - c = m['fixtures'].get(fixture, {}).get('image', {}).get('chunks', {}) - if '/indexed.bin' in c: - print(fixture, c['/indexed.bin']) -print(m['mutations']) -PY -``` - -The normal `indexed.bin` has five 32768-byte indexes with IDs `1,2,1,2,1`. -The 48-bit control has format bit `0x40` with zero high words; its unified -target also contains a real nonzero high word. The negative second index is -exactly device ID 3 while `extra_devices=2`, whose mask is 3. - -## Normal and cross-index reads - -```sh -I=/root/repo22-g6/images -S=/root/repo22-g6/sources/chunk -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/multi2-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2.blob" -u 92 -mount -t erofs -o ro -o device.1=/dev/md91 -o device.2=/dev/md92 \ - /dev/md90 /mnt/g6 -cmp "$S/indexed.bin" /mnt/g6/indexed.bin -dd if="$S/indexed.bin" of=/tmp/tc095.first.src bs=1 skip=0 count=4096 2>/dev/null -dd if=/mnt/g6/indexed.bin of=/tmp/tc095.first.mnt bs=1 skip=0 count=4096 2>/dev/null -cmp /tmp/tc095.first.src /tmp/tc095.first.mnt -dd if="$S/indexed.bin" of=/tmp/tc095.middle.src bs=1 skip=65536 count=4096 2>/dev/null -dd if=/mnt/g6/indexed.bin of=/tmp/tc095.middle.mnt bs=1 skip=65536 count=4096 2>/dev/null -cmp /tmp/tc095.middle.src /tmp/tc095.middle.mnt -dd if="$S/indexed.bin" of=/tmp/tc095.last.src bs=1 skip=131072 count=4096 2>/dev/null -dd if=/mnt/g6/indexed.bin of=/tmp/tc095.last.mnt bs=1 skip=131072 count=4096 2>/dev/null -cmp /tmp/tc095.last.src /tmp/tc095.last.mnt -dd if="$S/indexed.bin" of=/tmp/tc095.cross.src bs=1 skip=28672 count=8192 2>/dev/null -dd if=/mnt/g6/indexed.bin of=/tmp/tc095.cross.mnt bs=1 skip=28672 count=8192 2>/dev/null -cmp /tmp/tc095.cross.src /tmp/tc095.cross.mnt -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -## 48-bit indexes and high unified address - -```sh -mdconfig -a -t vnode -f "$I/multi2-unified48-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-unified48-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-unified48-slot2.blob" -u 92 -mount -t erofs -o ro -o device.2=/dev/md92 -o device.1=/dev/md91 \ - /dev/md90 /mnt/g6 -cmp "$S/indexed.bin" /mnt/g6/indexed.bin -cmp "$S/unified-address.bin" /mnt/g6/unified-address.bin -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -## Mapped undeclared ID - -```sh -mdconfig -a -t vnode -f "$I/bad-mapped-device3.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2.blob" -u 92 -mount -t erofs -o ro -o device.1=/dev/md91 -o device.2=/dev/md92 \ - /dev/md90 /mnt/g6 -truss -f -o /tmp/tc095-enodev.truss dd if=/mnt/g6/indexed.bin \ - of=/dev/null bs=1 skip=32768 count=4096 -tail -20 /tmp/tc095-enodev.truss -cmp "$S/tc006.bin" /mnt/g6/tc006.bin -``` - -The affected read must return exactly `ENODEV`; the mount and unaffected file -remain usable. - -## Cleanup and zero-state check - -```sh -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -kldunload erofs -rm -f /tmp/tc095.*.src /tmp/tc095.*.mnt -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[0-2]|erofs' || true -``` - -All four final outputs must be empty. diff --git a/tests/TC096-device-table-parse.md b/tests/TC096-device-table-parse.md deleted file mode 100644 index f985466..0000000 --- a/tests/TC096-device-table-parse.md +++ /dev/null @@ -1,105 +0,0 @@ -# Test Case: Device Table Parse - -**Test ID**: TC096-device-table-parse -**Category**: Multi-Device -**Priority**: Critical - -## Objective - -Verify `extra_devices`, `devt_slotoff`, 48-bit `blocks`/`uniaddr`, table offset -zero, `uniaddr=0`, statfs aggregation, and out-of-primary table rejection. - -## Generator field self-check - -Run `python3 -B tests/g6_multidev_fixtures.py verify OUTPUT` on the host. The -manifest must contain these independently reparsed values: - -| Fixture | primary blocks | extra | slot offset | slots `(blocks, uniaddr)` | -| --- | ---: | ---: | ---: | --- | -| `multi2` | 2 | 2 | 32 | `(137,2)`, `(90,139)` | -| `multi2-devt0` | 2 | 2 | 0 | `(137,2)`, `(90,139)` | -| `multi2-uniaddr0` | 2 | 2 | 32 | `(137,0)`, `(90,139)` | -| `multi2-unified48` | 2 | 2 | 32 | high-word unified ranges | - -The exact slot sizes are generated values and must also match the current -manifest if the source changes. - -## Standard table and statfs - -```sh -I=/root/repo22-g6/images -S=/root/repo22-g6/sources/chunk -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/multi2-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2.blob" -u 92 -mount -t erofs -o ro -o device.2=/dev/md92 -o device.1=/dev/md91 \ - /dev/md90 /mnt/g6 -df -k /mnt/g6 -cmp "$S/cross-device.bin" /mnt/g6/cross-device.bin -``` - -The manifest `f_blocks` basis is `2 + 137 + 90 = 229` at 4096 bytes, so -`df -k` must report 916 one-kilobyte blocks. - -## Positive table variants - -Unmount and detach units 92, 91, and 90 before each replacement primary. -Mount each row with its two matching providers and compare the listed file: - -| Primary/provider prefix | Required check | -| --- | --- | -| `multi2-devt0` | table bytes at image offset 0; full `cross-device.bin` cmp | -| `multi2-uniaddr0` | slot 1 selected by nonzero ID; full `cross-device.bin` cmp | -| `multi2-unified48` | nonzero `uniaddr_hi` and `startblk_hi`; full `unified-address.bin` cmp | - -Use this direct command shape for each row, changing only `PREFIX`: - -```sh -PREFIX=multi2-devt0 -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -mdconfig -a -t vnode -f "$I/$PREFIX-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/$PREFIX-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/$PREFIX-slot2.blob" -u 92 -mount -t erofs -o ro -o device.1=/dev/md91 -o device.2=/dev/md92 \ - /dev/md90 /mnt/g6 -cmp "$S/cross-device.bin" /mnt/g6/cross-device.bin -``` - -For `multi2-unified48`, compare `unified-address.bin` instead. Execute the -block separately for all three prefixes; it is not a test runner. - -## Table outside the primary image - -After detaching the prior variant: - -```sh -mdconfig -a -t vnode -f "$I/bad-table-oob.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2.blob" -u 92 -truss -f -o /tmp/tc096-oob.truss mount -t erofs -o ro \ - -o device.1=/dev/md91 -o device.2=/dev/md92 /dev/md90 /mnt/g6 -tail -20 /tmp/tc096-oob.truss -``` - -Mount must return exactly `EINTEGRITY` before root vnode creation. - -## Cleanup and zero-state check - -```sh -umount /mnt/g6 2>/dev/null || true -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -kldunload erofs -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[0-2]|erofs' || true -``` - -All four final outputs must be empty. diff --git a/tests/TC097-device-table-invalid.md b/tests/TC097-device-table-invalid.md deleted file mode 100644 index c569ae8..0000000 --- a/tests/TC097-device-table-invalid.md +++ /dev/null @@ -1,91 +0,0 @@ -# Test Case: Invalid Device Table - -**Test ID**: TC097-device-table-invalid -**Category**: Multi-Device / Error Handling -**Priority**: Critical - -## Objective - -Verify fail-closed table validation and complete cleanup after parse failure or -after slot 1 has opened and slot 2 fails. - -## Deterministic negative fixtures - -The G6 generator makes one asserted change per checksum-valid image: - -| Image | Exact expected errno | -| --- | --- | -| `bad-table-oob.erofs` | `EINTEGRITY` | -| `bad-slot-zero-blocks.erofs` | `EINTEGRITY` | -| `bad-slot-inside-primary.erofs` | `EINTEGRITY` | -| `bad-slot-overlap.erofs` | `EINTEGRITY` | -| `bad-slot-32bit-limit.erofs` | `EINTEGRITY` | -| `bad-slot-48bit-limit.erofs` | `EINTEGRITY` | - -An on-disk slot has only 48-bit `blocks` and `uniaddr`; their sum cannot -overflow a 64-bit C integer. Therefore an alleged uint64-add-overflow fixture -is not representable. The last image tests the real format boundary: a -48-bit range ending above `2^48`. `devt_slotoff=0` and `uniaddr=0` are legal -TC096 positives, not invalid fixtures. - -## Manual procedure for each structural image - -For each row above, execute the following commands directly, replacing -`IMAGE` with that row. Do not put the rows in a runner or shell loop. - -```sh -I=/root/repo22-g6/images -IMAGE=bad-table-oob.erofs -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/$IMAGE" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2.blob" -u 92 -truss -f -o "/tmp/tc097-$IMAGE.truss" mount -t erofs -o ro \ - -o device.1=/dev/md91 -o device.2=/dev/md92 /dev/md90 /mnt/g6 -tail -20 "/tmp/tc097-$IMAGE.truss" -mount -p | awk '$3 == "erofs" { print }' -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -kldunload erofs -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[0-2]|erofs' || true -``` - -Each mount must return exactly `EINTEGRITY`. After each individual failure, -the mount, md, KLD, and matching GEOM consumer outputs must all be empty before -proceeding to the next image. - -## Reverse-open cleanup - -This valid table opens slot 1 and then fails the slot-2 pathname lookup: - -```sh -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/multi2-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -truss -f -o /tmp/tc097-reverse.truss mount -t erofs -o ro \ - -o device.1=/dev/md91 -o device.2=/no/such/repo22-g6-slot2 \ - /dev/md90 /mnt/g6 -tail -20 /tmp/tc097-reverse.truss -mdconfig -d -u 91 -mdconfig -d -u 90 -kldunload erofs -``` - -The mount returns `ENOENT`, and immediate detach of md91 proves reverse-order -cleanup released the already-open consumer. - -## Final zero-state check - -```sh -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[0-2]|erofs' || true -``` - -All four outputs must be empty, with no panic, trap, hang, leaked mount, or -stale GEOM consumer. diff --git a/tests/TC098-fragments-support.md b/tests/TC098-fragments-support.md deleted file mode 100644 index ae98a57..0000000 --- a/tests/TC098-fragments-support.md +++ /dev/null @@ -1,59 +0,0 @@ -# Test Case: Packed-Inode Fragments - -**Test ID**: TC098-fragments-support -**Category**: Compression -**Priority**: High - -## Objective - -Verify a deterministic whole-file fragment stored in the EROFS packed inode, -including full and offset reads. This is an in-image packed inode, not an -external fragment device. - -## Fixture qualification - -Use fresh `fragment.erofs`. The generator runs: - -```sh -mkfs.erofs -T0 --all-time --all-root --workers=1 \ - -U67360098-0000-0000-0000-000000000098 \ - -zlz4 -C65536 -E all-fragments fragment.erofs sources/fragment -``` - -The manifest and verifier must report a nonzero packed NID, fragment feature, -and compressed layout for `/fragment.dat`. Host fsck extracts and compares the -entire source before guest testing. - -## Manual procedure - -```sh -I=/root/repo22-g6/images -S=/root/repo22-g6/sources/fragment -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/fragment.erofs" -u 90 -mount -t erofs -o ro /dev/md90 /mnt/g6 -stat -f '%z' /mnt/g6/fragment.dat -sha256 "$S/fragment.dat" /mnt/g6/fragment.dat -cmp "$S/fragment.dat" /mnt/g6/fragment.dat -dd if="$S/fragment.dat" of=/tmp/tc098.src bs=1 skip=65536 count=8192 2>/dev/null -dd if=/mnt/g6/fragment.dat of=/tmp/tc098.mnt bs=1 skip=65536 count=8192 2>/dev/null -cmp /tmp/tc098.src /tmp/tc098.mnt -``` - -The size must be 100000 bytes, and all comparisons must pass. - -## Cleanup and zero-state check - -```sh -umount /mnt/g6 -mdconfig -d -u 90 -kldunload erofs -rm -f /tmp/tc098.src /tmp/tc098.mnt -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md90|erofs' || true -``` - -All four final outputs must be empty. diff --git a/tests/TC099-multidev-2devices.md b/tests/TC099-multidev-2devices.md deleted file mode 100644 index f422bb0..0000000 --- a/tests/TC099-multidev-2devices.md +++ /dev/null @@ -1,64 +0,0 @@ -# Test Case: Primary Plus One mkfs Blob Provider - -**Test ID**: TC099-multidev-2devices -**Category**: Multi-Device -**Priority**: Critical - -## Objective - -Verify the common two-provider layout emitted directly by erofs-utils 1.8.6, -including statfs aggregation and fail-closed omission of the blob provider. - -## Positive procedure - -```sh -I=/root/repo22-g6/images -S=/root/repo22-g6/sources/chunk -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/mkfs-blob-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/mkfs-blob-slot1.blob" -u 91 -mount -t erofs -o ro -o device.1=/dev/md91 /dev/md90 /mnt/g6 -mount -p | grep '/mnt/g6' -df -k /mnt/g6 -sha256 "$S/tc006.bin" /mnt/g6/tc006.bin -cmp "$S/tc006.bin" /mnt/g6/tc006.bin -dd if="$S/tc006.bin" of=/tmp/tc099.src bs=1 skip=28672 count=8192 2>/dev/null -dd if=/mnt/g6/tc006.bin of=/tmp/tc099.mnt bs=1 skip=28672 count=8192 2>/dev/null -cmp /tmp/tc099.src /tmp/tc099.mnt -``` - -The manifest reports primary blocks 1 and slot blocks 224, so `df -k` must -report 900 one-kilobyte blocks. Metadata comes from md90; all file bytes come -from md91. - -Unmount and detach both providers before the negative step. - -## Missing blob option - -```sh -umount /mnt/g6 -mdconfig -d -u 91 -mdconfig -d -u 90 -mdconfig -a -t vnode -f "$I/mkfs-blob-primary.erofs" -u 90 -truss -f -o /tmp/tc099-enxio.truss mount -t erofs -o ro \ - /dev/md90 /mnt/g6 -tail -20 /tmp/tc099-enxio.truss -``` - -Mount must return exactly `ENXIO`: the 4096-byte split primary is shorter than -the declared flatdev range. - -## Cleanup and zero-state check - -```sh -mdconfig -d -u 90 -kldunload erofs -rm -f /tmp/tc099.src /tmp/tc099.mnt -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[01]|erofs' || true -``` - -All four final outputs must be empty. diff --git a/tests/TC100-multidev-4devices.md b/tests/TC100-multidev-4devices.md deleted file mode 100644 index 5f9aff3..0000000 --- a/tests/TC100-multidev-4devices.md +++ /dev/null @@ -1,91 +0,0 @@ -# Test Case: Primary Plus Three External Providers - -**Test ID**: TC100-multidev-4devices -**Category**: Multi-Device -**Priority**: Critical - -## Objective - -Verify a four-provider filesystem, option-order independence, cross-slot and -concurrent reads, and missing, duplicate, and short slot cleanup. - -## Fixture qualification - -The G6 verifier reports `multi3` primary blocks 2 and slots `(89,2)`, `(90,91)`, -and `(51,181)`. `/cross-device.bin` has three 131072-byte indexes with device -IDs `1,2,3`. The generator host-fsck extracts all three providers and compares -the complete source tree. `multi3-slot3-short.blob` is exactly one block below -the declared slot-3 size. - -## Positive procedure - -```sh -I=/root/repo22-g6/images -S=/root/repo22-g6/sources/chunk -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/multi3-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi3-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi3-slot2.blob" -u 92 -mdconfig -a -t vnode -f "$I/multi3-slot3.blob" -u 93 -mount -t erofs -o ro -o device.3=/dev/md93 -o device.1=/dev/md91 \ - -o device.2=/dev/md92 /dev/md90 /mnt/g6 -sha256 "$S/cross-device.bin" /mnt/g6/cross-device.bin -cmp "$S/cross-device.bin" /mnt/g6/cross-device.bin -dd if="$S/cross-device.bin" of=/tmp/tc100.cross.src bs=1 skip=126976 count=262144 2>/dev/null -dd if=/mnt/g6/cross-device.bin of=/tmp/tc100.cross.mnt bs=1 skip=126976 count=262144 2>/dev/null -cmp /tmp/tc100.cross.src /tmp/tc100.cross.mnt -sha256 /mnt/g6/cross-device.bin >/tmp/tc100.concurrent.1 & -sha256 /mnt/g6/indexed.bin >/tmp/tc100.concurrent.2 & -sha256 /mnt/g6/cold-slot2.bin >/tmp/tc100.concurrent.3 & -wait -cat /tmp/tc100.concurrent.1 /tmp/tc100.concurrent.2 /tmp/tc100.concurrent.3 -``` - -The 262144-byte range starts before the slot-1/slot-2 transition and ends -after the slot-2/slot-3 transition. All full, range, and concurrent reads must -pass. - -Unmount and detach 93, 92, 91, 90 before each negative step. - -## Negative mount cases - -Run each case directly and capture `nmount` with `truss`: - -1. Omit `device.3`: expected `ENXIO`. -2. Map both `device.1` and `device.2` to md91: expected `EINVAL`. -3. Attach `multi3-slot3-short.blob` as md93: expected `ENXIO`. - -For the short case, the exact command is: - -```sh -mdconfig -a -t vnode -f "$I/multi3-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi3-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi3-slot2.blob" -u 92 -mdconfig -a -t vnode -f "$I/multi3-slot3-short.blob" -u 93 -truss -f -o /tmp/tc100-short.truss mount -t erofs -o ro \ - -o device.1=/dev/md91 -o device.2=/dev/md92 -o device.3=/dev/md93 \ - /dev/md90 /mnt/g6 -tail -20 /tmp/tc100-short.truss -``` - -After each failure, detach all attached md units in descending order and -verify no EROFS mount or matching GEOM consumer remains. - -## Cleanup and zero-state check - -```sh -umount /mnt/g6 2>/dev/null || true -mdconfig -d -u 93 2>/dev/null || true -mdconfig -d -u 92 2>/dev/null || true -mdconfig -d -u 91 2>/dev/null || true -mdconfig -d -u 90 2>/dev/null || true -kldunload erofs -rm -f /tmp/tc100.cross.* /tmp/tc100.concurrent.* -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[0-3]|erofs' || true -``` - -All four final outputs must be empty, with no panic, trap, or hang. diff --git a/tests/TC101-unified-address-mapping.md b/tests/TC101-unified-address-mapping.md deleted file mode 100644 index fbf1175..0000000 --- a/tests/TC101-unified-address-mapping.md +++ /dev/null @@ -1,151 +0,0 @@ -# Test Case: Unified Address and Flatdev Mapping - -**Test ID**: TC101-unified-address-mapping -**Category**: Multi-Device -**Priority**: Critical - -## Objective - -Verify both EROFS mapping forms and fail-closed complete-extent checks: - -1. device ID 0 plus a block in a nonzero slot `uniaddr` range selects that - explicit provider and subtracts the range base; -2. a nonzero device ID in flatdev mode adds `uniaddr` and reads the combined - primary provider; -3. gaps, cross-slot chunks/pclusters, and out-of-range 48-bit addresses return - `EINTEGRITY` before physical I/O fallback. - -## Positive explicit unified mapping - -```sh -I=/root/repo22-g6/images -S=/root/repo22-g6/sources/chunk -mkdir -p /mnt/g6 -kldload /root/repo22-g6/erofs.ko -mdconfig -a -t vnode -f "$I/multi2-unified-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-unified-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-unified-slot2.blob" -u 92 -mount -t erofs -o ro -o device.2=/dev/md92 -o device.1=/dev/md91 \ - /dev/md90 /mnt/g6 -sha256 "$S/unified-address.bin" /mnt/g6/unified-address.bin -cmp "$S/unified-address.bin" /mnt/g6/unified-address.bin -dd if="$S/unified-address.bin" of=/tmp/tc101.unified.src bs=1 skip=61440 count=8192 2>/dev/null -dd if=/mnt/g6/unified-address.bin of=/tmp/tc101.unified.mnt bs=1 skip=61440 count=8192 2>/dev/null -cmp /tmp/tc101.unified.src /tmp/tc101.unified.mnt -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -The manifest must show that this index is device ID 0 and pblk equals slot-1 -`uniaddr + local_pblk`, not a local primary address. - -## Positive flatdev mappings - -First test nonzero chunk device IDs in one combined provider: - -```sh -mdconfig -a -t vnode -f "$I/multi2-flatdev.erofs" -u 90 -mount -t erofs -o ro /dev/md90 /mnt/g6 -sha256 "$S/cross-device.bin" /mnt/g6/cross-device.bin -cmp "$S/cross-device.bin" /mnt/g6/cross-device.bin -dd if="$S/cross-device.bin" of=/tmp/tc101.flat.src bs=1 skip=126976 count=139264 2>/dev/null -dd if=/mnt/g6/cross-device.bin of=/tmp/tc101.flat.mnt bs=1 skip=126976 count=139264 2>/dev/null -cmp /tmp/tc101.flat.src /tmp/tc101.flat.mnt -umount /mnt/g6 -mdconfig -d -u 90 -``` - -Then test device-ID-0 unified addressing in the combined provider: - -```sh -mdconfig -a -t vnode -f "$I/multi2-unified-flatdev.erofs" -u 90 -mount -t erofs -o ro /dev/md90 /mnt/g6 -cmp "$S/unified-address.bin" /mnt/g6/unified-address.bin -umount /mnt/g6 -mdconfig -d -u 90 -``` - -The 139264-byte range spans the slot-1/slot-2 transition. erofs-utils 1.8.6 -cannot qualify these flatdev forms; the kernel full/range comparisons are the -qualification. - -## Positive 48-bit high mapping - -```sh -mdconfig -a -t vnode -f "$I/multi2-unified48-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-unified48-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-unified48-slot2.blob" -u 92 -mount -t erofs -o ro -o device.1=/dev/md91 -o device.2=/dev/md92 \ - /dev/md90 /mnt/g6 -cmp "$S/unified-address.bin" /mnt/g6/unified-address.bin -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -This requires nonzero `uniaddr_hi` and `startblk_hi`; zero-high truncation -would read the wrong bytes. - -## Fail-closed extent cases - -Each primary below is checksum-valid. Mount succeeds, and the named cold read -must return exactly `EINTEGRITY` in `truss`: - -| Primary and providers | Mode | Read target | -| --- | --- | --- | -| `bad-unified-gap-*` | explicit and `bad-unified-gap-flatdev.erofs` | `unified-address.bin` | -| `bad-cross-slot-explicit-*` | explicit | `indexed.bin` | -| `bad-cross-slot-unified-*` | explicit and `bad-cross-slot-unified-flatdev.erofs` | `indexed.bin` | -| `bad-lz4-cross-slot-*` | explicit and `bad-lz4-cross-slot-flatdev.erofs` | `external-pcluster.bin` | -| `bad-unified48-out-of-range-*` | explicit | `unified-address.bin` | - -For an explicit chunk case, use this direct command shape with the matching -prefix and target: - -```sh -PREFIX=bad-cross-slot-explicit -TARGET=indexed.bin -mdconfig -a -t vnode -f "$I/$PREFIX-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/$PREFIX-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/$PREFIX-slot2.blob" -u 92 -mount -t erofs -o ro -o device.1=/dev/md91 -o device.2=/dev/md92 \ - /dev/md90 /mnt/g6 -truss -f -o "/tmp/tc101-$PREFIX.truss" dd if="/mnt/g6/$TARGET" \ - of=/dev/null bs=131072 count=1 -tail -20 "/tmp/tc101-$PREFIX.truss" -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -For a flatdev row, attach only its `*-flatdev.erofs` as md90 and use no -`device.N` options. For the LZ4 explicit row, use its two one-block slot -providers. Execute every row and mode separately; do not use a runner. - -The explicit crossing providers are physically only as long as their declared -slots. Therefore both declared range and physical media would be exceeded; -`EINTEGRITY` must win before a possible `ENXIO` from I/O. The largest non-NULL -48-bit address (`0xfffffffffffe`; all ones is the hole sentinel) is -representable in 64-bit arithmetic but belongs to no declared range, so it -also returns `EINTEGRITY`. - -## Cleanup and zero-state check - -```sh -umount /mnt/g6 2>/dev/null || true -mdconfig -d -u 92 2>/dev/null || true -mdconfig -d -u 91 2>/dev/null || true -mdconfig -d -u 90 2>/dev/null || true -kldunload erofs -rm -f /tmp/tc101.*.src /tmp/tc101.*.mnt -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[0-2]|erofs' || true -``` - -All four final outputs must be empty, with no panic, trap, or hang. diff --git a/tests/TC102-deflate-level1.md b/tests/TC102-deflate-level1.md deleted file mode 100644 index 1e46e16..0000000 --- a/tests/TC102-deflate-level1.md +++ /dev/null @@ -1,54 +0,0 @@ -# Test Case: DEFLATE Level 1 Read Correctness - -**Test ID**: TC102-deflate-level1 - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High - -## Objective - -Verify full and offset reads from an erofs-utils 1.8.6 DEFLATE level 1 image. - -## Fixture and Helper - -```sh -work=$(mktemp -d /tmp/repo22-tc102.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -cc -O2 -Wall -Wextra -std=c17 tests/read_probe.c -o "$work/read_probe" -dump.erofs -s "$work/fixtures/valid-deflate-level1.erofs" -dump.erofs --path=/compressed.bin -e \ - "$work/fixtures/valid-deflate-level1.erofs" -``` - -The superblock and extent output must identify DEFLATE-compressed data. The -image is generated with `-zdeflate,level=1 -C65536`. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode \ - -f "$work/fixtures/valid-deflate-level1.erofs") -mount -t erofs -o ro "/dev/$unit" "$work/mnt" - -expected=$(sha256 -q "$work/fixtures/source/compressed.bin") -actual=$(sha256 -q "$work/mnt/compressed.bin") -test "$actual" = "$expected" -"$work/read_probe" pread "$work/mnt/compressed.bin" 32768 4096 \ - "$work/guest-range.bin" -"$work/read_probe" pread "$work/fixtures/source/compressed.bin" 32768 4096 \ - "$work/source-range.bin" -cmp "$work/source-range.bin" "$work/guest-range.bin" - -umount "$work/mnt" -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -The complete mounted-file hash equals the source hash, and the fixed 4 KiB -range compares byte-for-byte. The invalid former command -`sha256 output vs original` is not used. diff --git a/tests/TC103-deflate-level6.md b/tests/TC103-deflate-level6.md deleted file mode 100644 index f333698..0000000 --- a/tests/TC103-deflate-level6.md +++ /dev/null @@ -1,46 +0,0 @@ -# Test Case: DEFLATE Compression Level 6 - -**Test ID**: TC103-deflate-level6 - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify DEFLATE compressed data path with level 6 (default) compression. - -## Preconditions -- EROFS image created with DEFLATE level 6: `mkfs.erofs -zdeflate,level=6 test.img src/` -- Test files with various entropy levels -- FreeBSD 13.0+ with zlib support - -## Test Steps -1. Create image with DEFLATE level 6 compression -2. Attach and mount: `unit=$(mdconfig -a -t vnode -f test.img); mount -t erofs -o ro /dev/${unit} /mnt` -3. Read text files, binaries, and random data -4. Verify all content matches original -5. Test partial reads at various offsets - -## Expected Results -- All files decompress correctly -- Better compression ratio than level 1 -- Random access works at any offset -- Stable decompression performance - -## Verification Method -- SHA256 checksums match for all files -- Compression ratio >= level 1 -- No decompression failures -- Random read correctness verified - -## Cleanup -```sh -umount /mnt -mdconfig -d -u "${unit}" -``` - -## Notes -Tests feature 58: DEFLATE compressed data path with balanced compression. diff --git a/tests/TC104-deflate-level9.md b/tests/TC104-deflate-level9.md deleted file mode 100644 index 8716f33..0000000 --- a/tests/TC104-deflate-level9.md +++ /dev/null @@ -1,46 +0,0 @@ -# Test Case: DEFLATE Compression Level 9 - -**Test ID**: TC104-deflate-level9 - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify DEFLATE compressed data path with level 9 (maximum) compression. - -## Preconditions -- EROFS image created with DEFLATE level 9: `mkfs.erofs -zdeflate,level=9 test.img src/` -- Large test files (1MB+) for compression testing -- FreeBSD 13.0+ with zlib support - -## Test Steps -1. Create image with DEFLATE level 9 compression -2. Attach and mount: `unit=$(mdconfig -a -t vnode -f test.img); mount -t erofs -o ro /dev/${unit} /mnt` -3. Read large compressed files sequentially -4. Test random access throughout file -5. Measure decompression performance - -## Expected Results -- All files decompress correctly -- Best compression ratio among DEFLATE levels -- Slightly slower decompression acceptable -- No data corruption - -## Verification Method -- Verify checksums match original files -- Compression ratio > level 6 -- Decompression throughput > 50 MB/s -- Random access correctness validated - -## Cleanup -```sh -umount /mnt -mdconfig -d -u "${unit}" -``` - -## Notes -Tests feature 58: DEFLATE compressed data path with maximum compression. diff --git a/tests/TC105-zstd-level1.md b/tests/TC105-zstd-level1.md deleted file mode 100644 index 1122af6..0000000 --- a/tests/TC105-zstd-level1.md +++ /dev/null @@ -1,46 +0,0 @@ -# Test Case: zstd Compression Level 1 - -**Test ID**: TC105-zstd-level1 - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify zstd compressed data path with level 1 (fastest) compression. - -## Preconditions -- EROFS image created with zstd level 1: `mkfs.erofs -zzstd,level=1 test.img src/` -- Test files of various sizes -- FreeBSD 13.0+ with libzstd - -## Test Steps -1. Create image with zstd level 1 compression -2. Attach and mount: `unit=$(mdconfig -a -t vnode -f test.img); mount -t erofs -o ro /dev/${unit} /mnt` -3. Read files and verify content -4. Test random access patterns -5. Measure throughput - -## Expected Results -- All files decompress correctly -- Fast decompression speed -- Good compression ratio even at level 1 -- Random access works correctly - -## Verification Method -- SHA256 checksums match originals -- Decompression throughput > 200 MB/s -- No zstd decompression errors -- Random reads succeed - -## Cleanup -```sh -umount /mnt -mdconfig -d -u "${unit}" -``` - -## Notes -Tests feature 59: zstd compressed data path with fast compression. diff --git a/tests/TC106-zstd-level15.md b/tests/TC106-zstd-level15.md deleted file mode 100644 index 9402125..0000000 --- a/tests/TC106-zstd-level15.md +++ /dev/null @@ -1,46 +0,0 @@ -# Test Case: zstd Compression Level 15 - -**Test ID**: TC106-zstd-level15 - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify zstd compressed data path with level 15 (high) compression. - -## Preconditions -- EROFS image created with zstd level 15: `mkfs.erofs -zzstd,level=15 test.img src/` -- Large test files for compression testing -- FreeBSD 13.0+ with libzstd - -## Test Steps -1. Create image with zstd level 15 compression -2. Attach and mount: `unit=$(mdconfig -a -t vnode -f test.img); mount -t erofs -o ro /dev/${unit} /mnt` -3. Read all test files -4. Verify content integrity -5. Test partial reads - -## Expected Results -- All files decompress correctly -- Better compression than level 1 -- Decompression still fast -- No memory issues - -## Verification Method -- All checksums match -- Compression ratio > zstd level 1 -- Decompression works for all files -- Random access validated - -## Cleanup -```sh -umount /mnt -mdconfig -d -u "${unit}" -``` - -## Notes -Tests feature 59: zstd compressed data path with high compression. diff --git a/tests/TC107-zstd-level22.md b/tests/TC107-zstd-level22.md deleted file mode 100644 index 5491560..0000000 --- a/tests/TC107-zstd-level22.md +++ /dev/null @@ -1,46 +0,0 @@ -# Test Case: zstd Compression Level 22 - -**Test ID**: TC107-zstd-level22 - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: Medium -**Regression**: None - -## Objective -Verify zstd compressed data path with level 22 (maximum) compression. - -## Preconditions -- EROFS image created with zstd level 22: `mkfs.erofs -zzstd,level=22 test.img src/` -- Large test files (multi-MB) -- FreeBSD 13.0+ with libzstd - -## Test Steps -1. Create image with zstd level 22 compression -2. Attach and mount: `unit=$(mdconfig -a -t vnode -f test.img); mount -t erofs -o ro /dev/${unit} /mnt` -3. Read large files sequentially -4. Test random access -5. Monitor memory usage during decompression - -## Expected Results -- All files decompress correctly -- Maximum compression ratio for zstd -- Acceptable decompression speed -- Stable memory usage - -## Verification Method -- Checksums match original files -- Compression ratio > level 15 -- Decompression throughput > 100 MB/s -- No memory leaks - -## Cleanup -```sh -umount /mnt -mdconfig -d -u "${unit}" -``` - -## Notes -Tests feature 59: zstd compressed data path with maximum compression. diff --git a/tests/TC108-lzma-large-file.md b/tests/TC108-lzma-large-file.md deleted file mode 100644 index 61dccd4..0000000 --- a/tests/TC108-lzma-large-file.md +++ /dev/null @@ -1,49 +0,0 @@ -# Test Case: LZMA Large File Decompression - -**Test ID**: TC108-lzma-large-file - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify LZMA decompression works correctly for large files (100MB+). - -## Preconditions -- `images/lzma-large.erofs`, generated with LZMA level 6 -- `/large.bin` and `sources/lzma-large/large.bin`, exactly 104857601 bytes -- FreeBSD 13.0+ with liblzma - -## Test Steps -1. Create image with large LZMA-compressed file -2. Attach and mount: `unit=$(mdconfig -a -t vnode -f test.img); mount -t erofs -o ro /dev/${unit} /mnt` -3. Compare complete source and guest SHA256, then run - `timeout 1800 cmp SOURCE /mnt/large.bin`. -4. Compare 4096 bytes at offsets 0, 52428800, and 103809024 with the source. -5. Record `vmstat -m` while mounted and after cleanup; active EROFS - allocations must return to zero. - -## Expected Results -- Large file reads successfully -- Checksum matches original -- Random access works at any offset -- Memory usage < 256MB during decompression - -## Verification Method -- SHA256 checksum verification -- Random read correctness at offsets: 0, 50MB, 99MB -- Memory usage monitored via top -- No kernel panics or OOM - -## Cleanup -```sh -umount /mnt -mdconfig -d -u "${unit#md}" -rm -f /tmp/out -``` - -## Notes -Tests feature 60: LZMA decompression with large files. diff --git a/tests/TC109-microlzma.md b/tests/TC109-microlzma.md deleted file mode 100644 index b7c75c8..0000000 --- a/tests/TC109-microlzma.md +++ /dev/null @@ -1,48 +0,0 @@ -# Test Case: MicroLZMA Compression - -**Test ID**: TC109-microlzma - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: Medium -**Regression**: None - -## Objective -Verify MicroLZMA compressed data path works correctly. - -## Preconditions -- `images/microlzma-edge.erofs`, generated with LZMA level 6 -- Actual 1-byte, 4096-byte, and 16384-byte source files -- The 16384-byte inode must be compressed full-index with algorithm 1. The - smaller files are valid flat controls and must not be claimed as compressed. -- FreeBSD 13.0+ with liblzma - -## Test Steps -1. Create image with MicroLZMA-compressed small files -2. Attach and mount: `unit=$(mdconfig -a -t vnode -f test.img); mount -t erofs -o ro /dev/${unit} /mnt` -3. Read all small files -4. Verify content integrity -5. Test edge cases (1-byte, 16KB boundary) - -## Expected Results -- All files read correctly and match their source bytes -- MicroLZMA format handled properly -- Good compression for small files -- No format detection errors - -## Verification Method -- Checksums match for all files -- Verify LZMA format detection in dmesg -- Test files at boundaries: 1B, 4KB, 16KB -- No decompression failures - -## Cleanup -```sh -umount /mnt -mdconfig -d -u "${unit#md}" -``` - -## Notes -Tests feature 60: MicroLZMA variant of LZMA compression. diff --git a/tests/TC110-lzma-corrupt.md b/tests/TC110-lzma-corrupt.md deleted file mode 100644 index e840255..0000000 --- a/tests/TC110-lzma-corrupt.md +++ /dev/null @@ -1,50 +0,0 @@ -# Test Case: LZMA Corrupted Data Handling - -**Test ID**: TC110-lzma-corrupt - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: High -**Regression**: None - -## Objective -Verify proper error handling when LZMA compressed data is corrupted. - -## Preconditions -- EROFS image with LZMA-compressed files -- `g5_fixtures.py` manifest proving algorithm 1, the target pcluster range, - exact patch offset/length, and a valid superblock checksum -- FreeBSD 13.0+ - -## Test Steps -1. Create valid LZMA-compressed EROFS image -2. Use only the generated `lzma-partial-ref-corrupt.erofs`; do not apply blind - or random offsets. -3. Attach and mount: `unit=$(mdconfig -a -t vnode -f test.img); mount -t erofs -o ro /dev/${unit} /mnt` -4. Run `timeout 20 read_probe expect-error /mnt/a.dat 5`. -5. Compare `/mnt/control.bin` from the same corrupted image with its source, - then check cleanup, allocation count, dmesg, and responsiveness. - -## Expected Results -- Mount succeeds (corruption not in superblock) -- Read returns EIO error -- Error logged to dmesg -- System remains stable (no panic) -- Other uncorrupted files still readable - -## Verification Method -- Verify read fails with errno == EIO -- Check dmesg for LZMA decompression error -- Confirm system stability after error -- Verify other files still accessible - -## Cleanup -```sh -umount /mnt -mdconfig -d -u "${unit#md}" -``` - -## Notes -Tests feature 60: LZMA error path with corrupted compressed data. diff --git a/tests/TC111-manpage-accuracy.md b/tests/TC111-manpage-accuracy.md deleted file mode 100644 index ec3434f..0000000 --- a/tests/TC111-manpage-accuracy.md +++ /dev/null @@ -1,54 +0,0 @@ -# Test Case: Manual Page Accuracy - -**Test ID**: TC111-manpage-accuracy -**Category**: Documentation -**Priority**: Low -**Regression**: None - -## Objective -Verify that manual pages accurately document all mount options, supported features, and behavior. - -## Preconditions -- The repository `docs/erofs.5` source and `README.md` are available. -- Access to the current module source for cross-reference. -- A FreeBSD 15 guest with the current `erofs.ko` and a valid image. -- Do not assume that a filesystem-specific `mount_erofs(8)` exists. - -## Test Steps -1. Confirm whether `/sbin/mount_erofs` exists. If it does not, verify the - documented generic `/sbin/mount -t erofs` path directly. -2. Mount without `-o ro`, with `-o rw`, and with documented standard flags; - record the resulting mount flags and write behavior. -3. Verify every documented filesystem-specific option against `src/super.c` - and an applicable real image. -4. Submit an unknown filesystem-specific option and require `nmount(2)` to - reject it with `EINVAL` without leaving a mount. -5. Cross-reference `README.md`, `docs/features.md`, `docs/architecture.md`, - `docs/erofs.5`, and `tests/TEST-COVERAGE-MATRIX.md` with current source and - qualified manual results. -6. Validate all examples and SEE ALSO references that apply to the qualified - FreeBSD environment. - -## Expected Results -- The documented generic mount path works without `mount_erofs(8)`. -- Every successful EROFS mount is read-only, including a request containing - `-o rw`; mutating operations return `EROFS`. -- Documented filesystem-specific options match the current parser and work on - their applicable fixture types. -- Unknown filesystem-specific options return `EINVAL` and create no mount. -- Feature claims match the implementation and qualified coverage. -- Examples and SEE ALSO entries are accurate. - -## Verification Method -- Record direct command exit status separately from tracing-tool status. -- Verify feature claims against `TEST-COVERAGE-MATRIX.md`. -- Check documentation against source and actual `/sbin/mount`/`nmount(2)` - behavior. -- Validate all code examples on their applicable fixture type. - -## Cleanup -Unmount the test filesystem, detach its md provider, and unload the exact test -module. Assert that no EROFS mount, md provider, or EROFS KLD remains. - -## Notes -Tests feature 61: manual pages accuracy and completeness. diff --git a/tests/TC112-invalid-superblock.md b/tests/TC112-invalid-superblock.md deleted file mode 100644 index 03a471a..0000000 --- a/tests/TC112-invalid-superblock.md +++ /dev/null @@ -1,56 +0,0 @@ -# Test Case: Invalid Superblock Magic - -**Test ID**: TC112-invalid-superblock -**Category**: Error Handling -**Priority**: Critical - -## Objective - -Verify rejection of an invalid EROFS magic at the real field while provider -length and all suffix bytes covered by repo22's production checksum verifier -remain unchanged. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc112.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -od -An -tx4 -j 1024 -N 4 "$work/fixtures/bad-magic.erofs" -python3 tests/erofs_fixture.py inspect "$work/fixtures/bad-magic.erofs" -grep '^bad-magic ' "$work/fixtures/fixture-evidence.txt" -wc -c "$work/fixtures/valid-plain.erofs" \ - "$work/fixtures/bad-magic.erofs" -sha256 -q "$work/fixtures/bad-magic.erofs" -``` - -The mutator asserts magic `0xe0f5e1e2` at byte 1024 and replaces it with -`0x21444142`. It intentionally leaves the stored checksum unchanged. The -canonical checksum over the changed magic is invalid, but the production -fixed-magic suffix calculation over `[1032,4096)` remains valid because no -suffix byte changed. Magic validation occurs first. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode -f "$work/fixtures/bad-magic.erofs") -set +e -mount -t erofs -o ro "/dev/$unit" "$work/mnt" \ - >"$work/mount.out" 2>"$work/mount.err" -status=$? -set -e -test "$status" -ne 0 -set +e -truss -o "$work/mount.truss" mount -t erofs -o ro "/dev/$unit" \ - "$work/mnt" >/dev/null 2>&1 -set -e -grep -E 'nmount.*ERR#22' "$work/mount.truss" -! mount -p | awk -v path="$work/mnt" '$2 == path { found=1 } - END { exit found ? 0 : 1 }' -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -Mount returns errno 22 (`EINVAL`) before root vnode creation. No panic, mount, -or md consumer remains. diff --git a/tests/TC113-corrupted-inode.md b/tests/TC113-corrupted-inode.md deleted file mode 100644 index 50a60b1..0000000 --- a/tests/TC113-corrupted-inode.md +++ /dev/null @@ -1,48 +0,0 @@ -# Test Case: Reserved Inode Format Rejection - -**Test ID**: TC113-corrupted-inode -**Category**: Error Handling -**Priority**: Critical - -## Objective - -Verify a checksum-valid image mounts, but lookup/open of the resolved -`/inode-target.txt` inode with reserved `i_format` bit `0x8000` fails with -`EOPNOTSUPP`; an unaffected file must remain readable. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc113.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -cc -O2 -Wall -Wextra -std=c17 tests/read_probe.c -o "$work/read_probe" -python3 tests/erofs_fixture.py inspect \ - "$work/fixtures/bad-inode-format.erofs" --path /inode-target.txt -grep '^bad-inode-format ' "$work/fixtures/fixture-evidence.txt" -sha256 -q "$work/fixtures/bad-inode-format.erofs" -``` - -The mutator resolves the root dirent, calculates the inode byte offset from the -metadata block and NID, asserts the old format, sets only reserved bit -`0x8000`, recomputes CRC32C, and preserves provider length. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode -f "$work/fixtures/bad-inode-format.erofs") -mount -t erofs -o ro "/dev/$unit" "$work/mnt" -cmp "$work/fixtures/source/control.txt" "$work/mnt/control.txt" -truss -o "$work/read.truss" "$work/read_probe" expect-error \ - "$work/mnt/inode-target.txt" 45 | tee "$work/error.out" -grep -q 'expected_errno=45' "$work/error.out" -grep -E '(openat|read).*ERR#45' "$work/read.truss" -cmp "$work/fixtures/source/control.txt" "$work/mnt/control.txt" -umount "$work/mnt" -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -Only the target returns errno 45 (`EOPNOTSUPP`). The mount remains usable and -cleans up fully. diff --git a/tests/TC114-out-of-bounds-block.md b/tests/TC114-out-of-bounds-block.md deleted file mode 100644 index e4ce017..0000000 --- a/tests/TC114-out-of-bounds-block.md +++ /dev/null @@ -1,48 +0,0 @@ -# Test Case: Out-of-Bounds FLAT_PLAIN Start Block - -**Test ID**: TC114-out-of-bounds-block -**Category**: Error Handling -**Priority**: Critical - -## Objective - -Verify checked data mapping rejects `/plain.bin` when its resolved -`startblk_lo` equals the declared filesystem block count, while valid files in -the same mounted image remain readable. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc114.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -cc -O2 -Wall -Wextra -std=c17 tests/read_probe.c -o "$work/read_probe" -python3 tests/erofs_fixture.py inspect \ - "$work/fixtures/oob-start-block.erofs" --path /plain.bin -grep '^oob-start-block ' "$work/fixtures/fixture-evidence.txt" -sha256 -q "$work/fixtures/oob-start-block.erofs" -``` - -The mutator proves a nonempty FLAT_PLAIN path, records its NID and inode field -offset, sets `startblk_lo` to `blocks_lo`, recomputes CRC32C, and preserves -provider length. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode -f "$work/fixtures/oob-start-block.erofs") -mount -t erofs -o ro "/dev/$unit" "$work/mnt" -cmp "$work/fixtures/source/control.txt" "$work/mnt/control.txt" -truss -o "$work/read.truss" "$work/read_probe" expect-error \ - "$work/mnt/plain.bin" 97 | tee "$work/error.out" -grep -q 'expected_errno=97' "$work/error.out" -grep -E 'read.*ERR#97' "$work/read.truss" -cmp "$work/fixtures/source/control.txt" "$work/mnt/control.txt" -umount "$work/mnt" -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -The target read returns errno 97 (`EINTEGRITY`) without issuing bytes from a -different range. Control data remains exact and cleanup is complete. diff --git a/tests/TC115-unsupported-algorithm.md b/tests/TC115-unsupported-algorithm.md deleted file mode 100644 index ccfc79f..0000000 --- a/tests/TC115-unsupported-algorithm.md +++ /dev/null @@ -1,54 +0,0 @@ -# Test Case: Future Compression Algorithm Rejection - -**Test ID**: TC115-unsupported-algorithm -**Category**: Error Handling -**Priority**: High - -## Objective - -Verify mount-time `EOPNOTSUPP` for a legal compressed image whose -`available_compr_algs` field additionally advertises unknown bit `0x8000`. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc115.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -od -An -tx2 -j 1106 -N 2 "$work/fixtures/future-algorithm.erofs" -python3 tests/erofs_fixture.py inspect \ - "$work/fixtures/future-algorithm.erofs" -grep '^future-algorithm ' "$work/fixtures/fixture-evidence.txt" -wc -c "$work/fixtures/valid-lz4.erofs" \ - "$work/fixtures/future-algorithm.erofs" -sha256 -q "$work/fixtures/future-algorithm.erofs" -``` - -The mutator requires `EROFS_FEATURE_INCOMPAT_COMPR_CFGS`, asserts the old -16-bit field at absolute byte 1106, sets only future bit `0x8000`, recomputes -CRC32C, and preserves provider length. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode -f "$work/fixtures/future-algorithm.erofs") -set +e -mount -t erofs -o ro "/dev/$unit" "$work/mnt" \ - >"$work/mount.out" 2>"$work/mount.err" -status=$? -set -e -test "$status" -ne 0 -set +e -truss -o "$work/mount.truss" mount -t erofs -o ro "/dev/$unit" \ - "$work/mnt" >/dev/null 2>&1 -set -e -grep -E 'nmount.*ERR#45' "$work/mount.truss" -! mount -p | awk -v path="$work/mnt" '$2 == path { found=1 } - END { exit found ? 0 : 1 }' -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -Mount returns errno 45 (`EOPNOTSUPP`), not checksum or media-size failure. No -mount or md consumer remains. diff --git a/tests/TC116-truncated-compressed.md b/tests/TC116-truncated-compressed.md deleted file mode 100644 index 0454233..0000000 --- a/tests/TC116-truncated-compressed.md +++ /dev/null @@ -1,69 +0,0 @@ -# Test Case: Targeted Compressed-Stream Corruption - -**Test ID**: TC116-truncated-compressed -**Category**: Error Handling -**Priority**: Critical - -## Objective - -Verify targeted damage inside a proven LZ4 physical extent reaches -`z_erofs_decompress` and returns `EIO`. Provider length must remain identical; -this is not a short-media test. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc116.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -cc -O2 -Wall -Wextra -std=c17 tests/read_probe.c -o "$work/read_probe" -cat "$work/fixtures/compressed.extents" -grep '^compressed-stream-corrupt ' "$work/fixtures/fixture-evidence.txt" -wc -c "$work/fixtures/valid-lz4.erofs" \ - "$work/fixtures/compressed-stream-corrupt.erofs" -cat "$work/fixtures/compressed-corrupt-fsck.txt" -sha256 -q "$work/fixtures/compressed-stream-corrupt.erofs" -``` - -The generator parses extent 0 for `/compressed.bin`, proves the 64-byte patch -at extent offset +32 lies wholly inside that extent, changes nonzero bytes to -zero, remains outside the superblock checksum window, preserves provider -length, and requires `fsck.erofs --extract` failure. The superblock checksum -remains valid because payload bytes are outside its range. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode \ - -f "$work/fixtures/compressed-stream-corrupt.erofs") -mount -t erofs -o ro "/dev/$unit" "$work/mnt" -cmp "$work/fixtures/source/control.txt" "$work/mnt/control.txt" -probe_module=$(dtrace -l -f z_erofs_decompress | awk \ - '$4 == "z_erofs_decompress" && $5 == "entry" { print $3; exit }') -test -n "$probe_module" -dtrace -l -n "fbt:$probe_module:z_erofs_decompress:entry" -set +e -dtrace -q -n "fbt:$probe_module:z_erofs_decompress:entry { - @calls = count(); } END { printa(\"decompress_calls=%@d\\n\", @calls); }" \ - -c "$work/read_probe expect-error $work/mnt/compressed.bin 5" \ - >"$work/dtrace.out" 2>"$work/dtrace.err" -dtrace_status=$? -set -e -cat "$work/dtrace.out" -test "$dtrace_status" -eq 0 -grep -Eq 'decompress_calls=[1-9][0-9]*' "$work/dtrace.out" -truss -o "$work/read.truss" "$work/read_probe" expect-error \ - "$work/mnt/compressed.bin" 5 | tee "$work/error.out" -grep -q 'expected_errno=5' "$work/error.out" -grep -E 'read.*ERR#5' "$work/read.truss" -cmp "$work/fixtures/source/control.txt" "$work/mnt/control.txt" -umount "$work/mnt" -mdconfig -d -u "${unit#md}" -kldunload dtraceall -``` - -## Expected Results - -Mount succeeds, proving media-size validation passed. FBT records at least one -`z_erofs_decompress` entry for the target command, and the target read returns -errno 5 (`EIO`). Control data remains readable and cleanup is complete. diff --git a/tests/TC117-invalid-xattr-format.md b/tests/TC117-invalid-xattr-format.md deleted file mode 100644 index d2247ee..0000000 --- a/tests/TC117-invalid-xattr-format.md +++ /dev/null @@ -1,33 +0,0 @@ -# Test Case: Invalid xattr formats - -**Test ID**: TC117-invalid-xattr-format -**Fixtures**: `bad-inline-entry.erofs`, `bad-shared-entry.erofs` - -## Objective - -Verify that an oversized inline `e_value_size` and an oversized shared -`e_value_size` fail at xattr access without destabilizing the mount. The -manifest must identify each exact field offset and `before -> ffff` mutation. - -## Manual steps - -Test each image in a separate attach/mount/cleanup cycle: - -```sh -stat -f 'mode=%Sp size=%z' /mnt/repo22-g4/corrupt-inline -truss -o /tmp/tc117-inline.truss getextattr -qq user bad \ - /mnt/repo22-g4/corrupt-inline -grep extattr_get_file /tmp/tc117-inline.truss -``` - -```sh -getextattr -qq -x user local /mnt/repo22-g4/shared-a -truss -o /tmp/tc117-shared.truss getextattr -qq user shared_key \ - /mnt/repo22-g4/shared-a -grep extattr_get_file /tmp/tc117-shared.truss -``` - -## Expected results - -Both malformed gets return `EINTEGRITY` (97); mount/stat and the independent -inline `local` value still work. Neither image leaks a mount/md or hangs. diff --git a/tests/TC118-device-not-found.md b/tests/TC118-device-not-found.md deleted file mode 100644 index b066584..0000000 --- a/tests/TC118-device-not-found.md +++ /dev/null @@ -1,137 +0,0 @@ -# Test Case: Missing, Invalid, and Orphaned External Device - -**Test ID**: TC118-device-not-found -**Category**: Multi-Device / Error Handling -**Priority**: Critical - -## Objective - -Verify exact missing/short/duplicate/path errors, forced GEOM orphan behavior, -concurrent provider ownership, and complete vnode/cdev/consumer cleanup. - -## Fixture qualification - -Use the fresh `multi2` set. The manifest must show `/cold-slot2.bin` has one -chunk with device ID 2; do not read it before the orphan step. -`multi2-slot2-short.blob` is exactly one block below slot 2's declared size. - -## Mount-time errors - -Set up the test directory and load the exact KLD once: - -```sh -I=/root/repo22-g6/images -mkdir -p /mnt/g6 /mnt/g6b -kldload /root/repo22-g6/erofs.ko -``` - -Run each case separately with `truss`, detaching all attached md units after -each failure: - -1. Split primary with no `device.N`: `ENXIO`. -2. Only `device.1`: `ENXIO`. -3. Valid slot 1 plus `multi2-slot2-short.blob`: `ENXIO`. -4. The same md91 assigned to slots 1 and 2: `EINVAL`. -5. Primary md90 also assigned as slot 1, with valid md92 for slot 2: `EINVAL`. - -The short-media command is: - -```sh -mdconfig -a -t vnode -f "$I/multi2-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2-short.blob" -u 92 -truss -f -o /tmp/tc118-short.truss mount -t erofs -o ro \ - -o device.1=/dev/md91 -o device.2=/dev/md92 /dev/md90 /mnt/g6 -tail -20 /tmp/tc118-short.truss -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -After every case, `mount -p | awk '$3 == "erofs"'` and matching -`kern.geom.conftxt` consumer output must be empty. - -## Forced orphan - -Attach fresh providers and mount without reading `cold-slot2.bin`: - -```sh -mdconfig -a -t vnode -f "$I/multi2-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2.blob" -u 92 -mount -t erofs -o ro -o device.1=/dev/md91 -o device.2=/dev/md92 \ - /dev/md90 /mnt/g6 -truss -f -o /tmp/tc118-ebusy.truss mdconfig -d -u 92 -tail -20 /tmp/tc118-ebusy.truss -mdconfig -d -u 92 -o force -truss -f -o /tmp/tc118-orphan-read.truss sha256 \ - /mnt/g6/cold-slot2.bin -tail -20 /tmp/tc118-orphan-read.truss -``` - -Normal detach must return `EBUSY`; forced detach must complete; the first cold -slot-2 read must return exactly `ENXIO` without panic, hang, or leaked vnode. - -```sh -umount /mnt/g6 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -## Concurrent mount ownership - -```sh -mdconfig -a -t vnode -f "$I/multi2-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -mdconfig -a -t vnode -f "$I/multi2-slot2.blob" -u 92 -mount -t erofs -o ro -o device.1=/dev/md91 -o device.2=/dev/md92 \ - /dev/md90 /mnt/g6 -truss -f -o /tmp/tc118-concurrent.truss mount -t erofs -o ro \ - -o device.1=/dev/md91 -o device.2=/dev/md92 /dev/md90 /mnt/g6b -tail -20 /tmp/tc118-concurrent.truss -sha256 /mnt/g6/tc006.bin -``` - -The second mount must return exactly `EBUSY`, and the first mount must remain -readable. Record a kernel failure and do not edit `src/` if FreeBSD permits a -second mount of an already-open provider set. - -```sh -umount /mnt/g6 -mdconfig -d -u 92 -mdconfig -d -u 91 -mdconfig -d -u 90 -``` - -## Pathname preservation - -```sh -truss -f -o /tmp/tc118-primary-enoent.truss mount -t erofs -o ro \ - /no/such/repo22-g6-primary /mnt/g6 -tail -20 /tmp/tc118-primary-enoent.truss -mdconfig -a -t vnode -f "$I/multi2-primary.erofs" -u 90 -mdconfig -a -t vnode -f "$I/multi2-slot1.blob" -u 91 -truss -f -o /tmp/tc118-slot-enoent.truss mount -t erofs -o ro \ - -o device.1=/dev/md91 -o device.2=/no/such/repo22-g6-slot2 \ - /dev/md90 /mnt/g6 -tail -20 /tmp/tc118-slot-enoent.truss -``` - -Both operations must preserve exactly `ENOENT`; the external failure must also -release the already-open slot-1 consumer. - -## Cleanup and zero-state check - -```sh -mdconfig -d -u 91 -mdconfig -d -u 90 -kldunload erofs -mount -p | awk '$3 == "erofs" { print }' -mdconfig -l -kldstat -n erofs 2>/dev/null || true -sysctl -n kern.geom.conftxt | grep -E 'md9[0-2]|erofs' || true -dmesg | tail -120 -``` - -All four final lifecycle outputs must be empty. The dmesg suffix must contain -no panic, trap, assertion, watchdog, or unexpected EROFS error. diff --git a/tests/TC119-data-crc-mismatch.md b/tests/TC119-data-crc-mismatch.md deleted file mode 100644 index 0dbf22e..0000000 --- a/tests/TC119-data-crc-mismatch.md +++ /dev/null @@ -1,56 +0,0 @@ -# Test Case: Uncompressed Data Corruption Boundary - -**Test ID**: TC119-data-crc-mismatch -**Category**: Data Integrity Semantics -**Priority**: High - -## Objective - -Verify the actual EROFS integrity boundary. FLAT_PLAIN payload has no per-file -CRC, so one targeted raw-data bit flip is returned to the caller and detected -only by external byte/hash comparison. A fictional kernel data-CRC error is not -expected. - -## Fixture Qualification - -```sh -work=$(mktemp -d /tmp/repo22-tc119.XXXXXX) -tests/prepare_error_fixtures.sh "$work/fixtures" -cc -O2 -Wall -Wextra -std=c17 tests/read_probe.c -o "$work/read_probe" -python3 tests/erofs_fixture.py inspect \ - "$work/fixtures/raw-data-corrupt.erofs" --path /plain.bin -grep '^raw-data-corrupt ' "$work/fixtures/fixture-evidence.txt" -wc -c "$work/fixtures/valid-plain.erofs" \ - "$work/fixtures/raw-data-corrupt.erofs" -sha256 -q "$work/fixtures/raw-data-corrupt.erofs" -``` - -The mutator resolves `/plain.bin`, proves nonempty FLAT_PLAIN layout, flips bit -`0x80` at file offset 257, records the absolute provider byte, preserves media -size, and verifies that the unchanged superblock checksum remains valid. - -## FreeBSD Procedure - -```sh -mkdir "$work/mnt" -unit=$(mdconfig -a -t vnode -f "$work/fixtures/raw-data-corrupt.erofs") -mount -t erofs -o ro "/dev/$unit" "$work/mnt" -source_hash=$(sha256 -q "$work/fixtures/source/plain.bin") -mounted_hash=$(sha256 -q "$work/mnt/plain.bin") -test "$source_hash" != "$mounted_hash" -"$work/read_probe" pread "$work/fixtures/source/plain.bin" 257 1 \ - "$work/source-byte.bin" -"$work/read_probe" pread "$work/mnt/plain.bin" 257 1 \ - "$work/mounted-byte.bin" -! cmp -s "$work/source-byte.bin" "$work/mounted-byte.bin" -cmp "$work/fixtures/source/control.txt" "$work/mnt/control.txt" -umount "$work/mnt" -mdconfig -d -u "${unit#md}" -``` - -## Expected Results - -Mount and all reads succeed. The exact target byte and complete file hash differ -from source, while control data matches. No `EIO` or data-CRC diagnostic is -expected. Record source, image, mounted-file hashes, target bytes, dmesg delta, -and cleanup. diff --git a/tests/TC120-empty-file.md b/tests/TC120-empty-file.md deleted file mode 100644 index 82d7f36..0000000 --- a/tests/TC120-empty-file.md +++ /dev/null @@ -1,54 +0,0 @@ -# Test Case: Empty File Read - -**Test ID**: TC120-empty-file -**Category**: Boundary & Stress -**Priority**: Medium -**Regression**: None - -## Objective - -Verify exact zero-length inode, EOF, seek, hash, and close behavior. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md`. -- Load the exact G7 KLD by full path. -- Verify the guest copy of `SOURCE-SHA256SUMS` before mounting. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRC=$FIX/sources/boundaries/empty.bin -IMAGE=$FIX/images/boundaries.erofs -MNT=/mnt/repo22-g7-tc120 -E=$G7/evidence/TC120 -mkdir -p "$MNT" "$E" -unit=$(mdconfig -a -t vnode -f "$IMAGE") -printf 'md=%s\n' "$unit" | tee "$E/provider.txt" -mount -t erofs -o ro "/dev/$unit" "$MNT" -stat -f 'size=%z mode=%Sp name=%N' "$SRC" "$MNT/empty.bin" \ - | tee "$E/stat.txt" -source_hash=$(sha256 -q "$SRC") -target_hash=$(sha256 -q "$MNT/empty.bin") -printf 'source=%s\ntarget=%s\n' "$source_hash" "$target_hash" \ - | tee "$E/hashes.txt" -test "$source_hash" = "$target_hash" -cmp "$SRC" "$MNT/empty.bin" -"$G7/g7_probe" empty "$MNT/empty.bin" | tee "$E/probe.txt" -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- Both sizes are exactly 0 and both SHA256 values are the empty-file hash. -- `cmp` exits 0. -- The probe reports `read_bytes=0`, both seek offsets 0, and exit 0. -- The exact mount and md provider are absent after cleanup. - -Any kernel panic, trap, hang, or unexpected EROFS dmesg error is KFAIL and -requires an issue report without changing `src`. diff --git a/tests/TC121-maximum-file-size.md b/tests/TC121-maximum-file-size.md deleted file mode 100644 index c47ad86..0000000 --- a/tests/TC121-maximum-file-size.md +++ /dev/null @@ -1,64 +0,0 @@ -# Test Case: Sparse 64-bit File Boundary - -**Test ID**: TC121-maximum-file-size -**Category**: Boundary & Stress -**Priority**: High -**Regression**: None - -## Objective - -Verify a reproducible sparse file across 2 GiB and 4 GiB offset boundaries, -including exact 64-bit size, hole bytes, marker bytes, and EOF behavior. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md` and its fixture self-check. -- The source is exactly 4294971393 bytes and consumes less than 1 MiB of host - blocks; this is a practical sparse boundary, not a fabricated 16 TiB run. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRC=$FIX/sources/boundaries/maximum/sparse-boundary.bin -IMAGE=$FIX/images/boundaries.erofs -MNT=/mnt/repo22-g7-tc121 -E=$G7/evidence/TC121 -mkdir -p "$MNT" "$E" -stat -f 'source_size=%z source_blocks=%b blocksize=%k' "$SRC" \ - | tee "$E/source-stat.txt" -test "$(stat -f %z "$SRC")" -eq 4294971393 -test "$(stat -f %b "$SRC")" -lt 2048 -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -TARGET=$MNT/maximum/sparse-boundary.bin -stat -f 'target_size=%z target_blocks=%b blocksize=%k' "$TARGET" \ - | tee "$E/target-stat.txt" -test "$(stat -f %z "$TARGET")" -eq 4294971393 -: > "$E/ranges.txt" -while IFS="$(printf '\t')" read -r label offset length; do - "$G7/g7_probe" range "$SRC" "$TARGET" "$offset" "$length" \ - > "$E/range-$label.txt" - rc=$? - printf 'label=%s rc=%s\n' "$label" "$rc" | tee -a "$E/ranges.txt" - cat "$E/range-$label.txt" | tee -a "$E/ranges.txt" - test "$rc" -eq 0 || exit 1 -done < "$FIX/SPARSE-RANGES.tsv" -"$G7/g7_probe" eof "$TARGET" 4294971393 | tee "$E/eof.txt" -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- Source and mounted sizes are exactly 4294971393. -- Every listed range exits 0 with `mismatches=0`; the list covers a pure hole, - block edge, 2 GiB edge, 4 GiB edge, and final 64 bytes. -- A read at exact EOF returns 0 bytes with errno 0. -- The guest remains responsive and cleanup removes the exact mount/provider. - -A range mismatch, overflow errno, panic, trap, or hang is KFAIL. Do not change -kernel source during this regression group. diff --git a/tests/TC122-deep-directory-tree.md b/tests/TC122-deep-directory-tree.md deleted file mode 100644 index 9a04e70..0000000 --- a/tests/TC122-deep-directory-tree.md +++ /dev/null @@ -1,56 +0,0 @@ -# Test Case: Deep Directory Tree - -**Test ID**: TC122-deep-directory-tree -**Category**: Boundary & Stress -**Priority**: Medium -**Regression**: None - -## Objective - -Verify exact traversal, lookup, `getcwd`, and payload integrity through 128 -directory levels. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md`. -- `DEEP-PATH` came from the verified fixture, not a hand-written `a/b/...` - placeholder. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRCROOT=$FIX/sources/boundaries -IMAGE=$FIX/images/boundaries.erofs -MNT=/mnt/repo22-g7-tc122 -E=$G7/evidence/TC122 -mkdir -p "$MNT" "$E" -deep_path=$(cat "$FIX/DEEP-PATH") -levels=$(printf '%s\n' "$deep_path" | awk -F/ '{ print NF - 2 }') -printf 'levels=%s\npath=%s\n' "$levels" "$deep_path" \ - | tee "$E/path.txt" -test "$levels" -eq 128 -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -cmp "$SRCROOT/$deep_path" "$MNT/$deep_path" -source_hash=$(sha256 -q "$SRCROOT/$deep_path") -target_hash=$(sha256 -q "$MNT/$deep_path") -printf 'source=%s\ntarget=%s\n' "$source_hash" "$target_hash" \ - | tee "$E/hashes.txt" -test "$source_hash" = "$target_hash" -deep_dir=$(dirname "$deep_path") -(cd "$MNT/$deep_dir" && pwd -P) | tee "$E/getcwd.txt" -test "$(cat "$E/getcwd.txt")" = "$MNT/$deep_dir" -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- The metadata count is exactly 128 directory levels. -- Deepest source/mount `cmp` and SHA256 values match. -- `pwd -P` returns the exact mounted deepest directory. -- No stack warning, panic, hang, or leaked mount/provider occurs. diff --git a/tests/TC123-long-filename.md b/tests/TC123-long-filename.md deleted file mode 100644 index 2fc6969..0000000 --- a/tests/TC123-long-filename.md +++ /dev/null @@ -1,55 +0,0 @@ -# Test Case: 255-byte Filename - -**Test ID**: TC123-long-filename -**Category**: Boundary & Stress -**Priority**: Medium -**Regression**: None - -## Objective - -Verify exact `NAME_MAX` lookup, readdir name bytes, stat, and file content. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md`. -- Use the fixture's ASCII `LONG-NAME`; its byte count must be 255. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRCROOT=$FIX/sources/boundaries -IMAGE=$FIX/images/boundaries.erofs -MNT=/mnt/repo22-g7-tc123 -E=$G7/evidence/TC123 -mkdir -p "$MNT" "$E" -long_name=$(cat "$FIX/LONG-NAME") -name_bytes=$(printf '%s' "$long_name" | wc -c | tr -d ' ') -printf 'name_bytes=%s\nname=%s\n' "$name_bytes" "$long_name" \ - | tee "$E/name.txt" -test "$name_bytes" -eq 255 -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -(cd "$SRCROOT" && find longname -type f -print | sort) > "$E/source.names" -(cd "$MNT" && find longname -type f -print | sort) > "$E/target.names" -cmp "$E/source.names" "$E/target.names" -test "$(wc -l < "$E/target.names" | tr -d ' ')" -eq 1 -stat -f 'size=%z name=%N' "$MNT/longname/$long_name" | tee "$E/stat.txt" -cmp "$SRCROOT/longname/$long_name" "$MNT/longname/$long_name" -source_hash=$(sha256 -q "$SRCROOT/longname/$long_name") -target_hash=$(sha256 -q "$MNT/longname/$long_name") -printf 'source=%s\ntarget=%s\n' "$source_hash" "$target_hash" \ - | tee "$E/hashes.txt" -test "$source_hash" = "$target_hash" -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- The filename is exactly 255 bytes and readdir returns that one exact name. -- Stat succeeds; source/mount `cmp` and SHA256 values match. -- No truncation, extra entry, panic, hang, or cleanup leak occurs. diff --git a/tests/TC124-many-small-files.md b/tests/TC124-many-small-files.md deleted file mode 100644 index e25d802..0000000 --- a/tests/TC124-many-small-files.md +++ /dev/null @@ -1,60 +0,0 @@ -# Test Case: Many Small Files - -**Test ID**: TC124-many-small-files -**Category**: Boundary & Stress -**Priority**: High -**Regression**: None - -## Objective - -Verify exact names, count, sizes, and hashes for 12,000 one-KiB files spread -over 12 directories. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md` and verify the source inventory. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRCROOT=$FIX/sources/boundaries -IMAGE=$FIX/images/boundaries.erofs -MNT=/mnt/repo22-g7-tc124 -E=$G7/evidence/TC124 -mkdir -p "$MNT" "$E" -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -(cd "$SRCROOT" && find many-small -type f -print | sort) > "$E/source.names" -(cd "$MNT" && find many-small -type f -print | sort) > "$E/target.names" -source_count=$(wc -l < "$E/source.names" | tr -d ' ') -target_count=$(wc -l < "$E/target.names" | tr -d ' ') -printf 'source_count=%s\ntarget_count=%s\n' "$source_count" "$target_count" \ - | tee "$E/counts.txt" -test "$source_count" -eq 12000 -test "$target_count" -eq 12000 -cmp "$E/source.names" "$E/target.names" -(cd "$SRCROOT" && find many-small -type f -exec sha256sum {} + | sort) \ - > "$E/source.hashes" -(cd "$MNT" && find many-small -type f -exec sha256sum {} + | sort) \ - > "$E/target.hashes" -cmp "$E/source.hashes" "$E/target.hashes" -test "$(awk '$3 ~ /^sources\/boundaries\/many-small\// && $2 != 1024 \ - { bad++ } END { print bad + 0 }' \ - "$FIX/SOURCE-INVENTORY.tsv")" -eq 0 || \ - awk '$3 ~ /^sources\/boundaries\/many-small\// && $2 != 1024' \ - "$FIX/SOURCE-INVENTORY.tsv" -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- Source and mounted name lists are identical and contain exactly 12,000 - files. -- Every mounted SHA256 row equals the corresponding source row. -- Every many-small source file is exactly 1024 bytes. -- No missing inode, panic, hang, or cleanup leak occurs. diff --git a/tests/TC125-large-directory.md b/tests/TC125-large-directory.md deleted file mode 100644 index c220390..0000000 --- a/tests/TC125-large-directory.md +++ /dev/null @@ -1,64 +0,0 @@ -# Test Case: Large Directory - -**Test ID**: TC125-large-directory -**Category**: Boundary & Stress -**Priority**: High -**Regression**: None - -## Objective - -Verify exact readdir and lookup behavior for one directory containing 12,000 -direct child files. Record scan timing without a cross-machine time gate. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md` and verify the source inventory. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRCROOT=$FIX/sources/boundaries -IMAGE=$FIX/images/boundaries.erofs -MNT=/mnt/repo22-g7-tc125 -E=$G7/evidence/TC125 -mkdir -p "$MNT" "$E" -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -(cd "$SRCROOT" && find large-dir -type f -print | sort) > "$E/source.names" -/usr/bin/time -p sh -c \ - 'cd "$1" && find large-dir -type f -print | sort > "$2"' \ - sh "$MNT" "$E/target.names" 2> "$E/readdir.time" -cat "$E/readdir.time" -source_count=$(wc -l < "$E/source.names" | tr -d ' ') -target_count=$(wc -l < "$E/target.names" | tr -d ' ') -printf 'source_count=%s\ntarget_count=%s\n' "$source_count" "$target_count" \ - | tee "$E/counts.txt" -test "$source_count" -eq 12000 -test "$target_count" -eq 12000 -cmp "$E/source.names" "$E/target.names" -for name in entry-00000.txt entry-00001.txt entry-05999.txt \ - entry-11998.txt entry-11999.txt; do - cmp "$SRCROOT/large-dir/$name" "$MNT/large-dir/$name" - printf '%s source=%s target=%s\n' "$name" \ - "$(sha256 -q "$SRCROOT/large-dir/$name")" \ - "$(sha256 -q "$MNT/large-dir/$name")" -done | tee "$E/lookups.txt" -(cd "$SRCROOT" && find large-dir -type f -exec sha256sum {} + | sort) \ - > "$E/source.hashes" -(cd "$MNT" && find large-dir -type f -exec sha256sum {} + | sort) \ - > "$E/target.hashes" -cmp "$E/source.hashes" "$E/target.hashes" -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- Exact sorted names and all 12,000 source/mount hashes match. -- The five boundary/middle lookups match their sources. -- Readdir timing is recorded but has no fixed TCG or cross-machine limit. -- No readdir error, panic, hang, or cleanup leak occurs. diff --git a/tests/TC126-concurrent-reads.md b/tests/TC126-concurrent-reads.md deleted file mode 100644 index 15dad93..0000000 --- a/tests/TC126-concurrent-reads.md +++ /dev/null @@ -1,77 +0,0 @@ -# Test Case: Concurrent Reads - -**Test ID**: TC126-concurrent-reads -**Category**: Boundary & Stress -**Priority**: High -**Regression**: None - -## Objective - -Verify 16 simultaneous full-file reads, with every PID, exit code, expected -source hash, and mounted-file hash recorded and checkable. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md` and verify the workload source inventory. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRC=$FIX/sources/workloads/concurrent -IMAGE=$FIX/images/workloads.erofs -MNT=/mnt/repo22-g7-tc126 -E=$G7/evidence/TC126 -mkdir -p "$MNT" "$E" -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -: > "$E/pids.tsv" -index=0 -while [ "$index" -lt 16 ]; do - file=$(printf 'reader-%02d.bin' "$index") - expected=$(sha256 -q "$SRC/$file") - ( - actual=$(sha256 -q "$MNT/concurrent/$file") - printf 'worker=%02d file=%s expected=%s actual=%s\n' \ - "$index" "$file" "$expected" "$actual" - test "$actual" = "$expected" - ) > "$E/worker-$(printf '%02d' "$index").log" 2>&1 & - pid=$! - printf '%02d\t%s\n' "$index" "$pid" >> "$E/pids.tsv" - index=$((index + 1)) -done -failed=0 -: > "$E/waits.tsv" -while IFS="$(printf '\t')" read -r worker pid; do - wait "$pid" - rc=$? - printf '%s\t%s\t%s\n' "$worker" "$pid" "$rc" >> "$E/waits.tsv" - test "$rc" -eq 0 || failed=$((failed + 1)) -done < "$E/pids.tsv" -cat "$E/pids.tsv" "$E/waits.tsv" "$E"/worker-*.log -test "$failed" -eq 0 -test "$(wc -l < "$E/pids.tsv" | tr -d ' ')" -eq 16 -test "$(wc -l < "$E/waits.tsv" | tr -d ' ')" -eq 16 -alive=0 -while IFS="$(printf '\t')" read -r worker pid; do - if kill -0 "$pid" 2>/dev/null; then - printf 'still-alive worker=%s pid=%s\n' "$worker" "$pid" - alive=$((alive + 1)) - fi -done < "$E/pids.tsv" -test "$alive" -eq 0 -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- Exactly 16 PIDs and 16 waits are recorded; every wait exit code is 0. -- Every worker log contains identical expected/source and actual/mounted - SHA256 values. -- None of the recorded PIDs remains alive; no `killall` or unrelated-process - cleanup is used. -- No deadlock, panic, hang, dmesg error, mount leak, or provider leak occurs. diff --git a/tests/TC127-memory-pressure.md b/tests/TC127-memory-pressure.md deleted file mode 100644 index b970f7a..0000000 --- a/tests/TC127-memory-pressure.md +++ /dev/null @@ -1,114 +0,0 @@ -# Test Case: Controlled Memory Pressure - -**Test ID**: TC127-memory-pressure -**Category**: Boundary & Stress -**Priority**: High -**Regression**: None - -## Objective - -Verify correct EROFS reads while three synchronized FreeBSD processes touch -memory up to per-process RCTL virtual-memory denial, then prove allocation -failure, release, recovery, PID exit, and read integrity. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md`. -- `sysctl -n kern.racct.enable` must print 1. Record an ENV result instead of - inventing pressure if this loader facility cannot be enabled. -- Use `rctl vmemoryuse:deny`; do not use the invalid `jail -m 256M` command. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRC=$FIX/sources/workloads/pressure/pressure.bin -IMAGE=$FIX/images/workloads.erofs -MNT=/mnt/repo22-g7-tc127 -E=$G7/evidence/TC127 -mkdir -p "$MNT" "$E" -test "$(sysctl -n kern.racct.enable)" -eq 1 -sysctl kern.racct.enable | tee "$E/racct.txt" -rctl | tee "$E/rctl-before.txt" -vmstat -H 1 5 | tee "$E/vmstat-before.txt" -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -: > "$E/pids.tsv" -index=1 -while [ "$index" -le 3 ]; do - rm -f "$E/go-$index" "$E/stop-$index" "$E/pressure-$index.log" - "$G7/g7_probe" pressure "$E/go-$index" "$E/stop-$index" 1024 \ - > "$E/pressure-$index.log" 2>&1 & - pid=$! - printf '%s\t%s\n' "$index" "$pid" >> "$E/pids.tsv" - index=$((index + 1)) -done -attempt=0 -while [ "$attempt" -lt 30 ]; do - ready=$(grep -l '^state=waiting ' "$E"/pressure-*.log 2>/dev/null \ - | wc -l | tr -d ' ') - test "$ready" -eq 3 && break - sleep 1 - attempt=$((attempt + 1)) -done -test "$ready" -eq 3 -: > "$E/rctl-rules.txt" -while IFS="$(printf '\t')" read -r worker pid; do - rctl -a "process:$pid:vmemoryuse:deny=640M" - rctl -l "process:$pid" >> "$E/rctl-rules.txt" - touch "$E/go-$worker" -done < "$E/pids.tsv" -attempt=0 -while [ "$attempt" -lt 120 ]; do - holding=$(grep -l '^state=holding .*allocation_failure=ENOMEM ' \ - "$E"/pressure-*.log 2>/dev/null | wc -l | tr -d ' ') - test "$holding" -eq 3 && break - sleep 1 - attempt=$((attempt + 1)) -done -test "$holding" -eq 3 -vmstat -H 1 10 | tee "$E/vmstat-pressure.txt" -rctl -hu process:$(awk 'NR == 1 { print $2 }' "$E/pids.tsv") \ - | tee "$E/rctl-utilization.txt" -source_hash=$(sha256 -q "$SRC") -target_hash=$(sha256 -q "$MNT/pressure/pressure.bin") -printf 'source=%s\ntarget=%s\n' "$source_hash" "$target_hash" \ - | tee "$E/hashes.txt" -test "$source_hash" = "$target_hash" -cmp "$SRC" "$MNT/pressure/pressure.bin" -while IFS="$(printf '\t')" read -r worker pid; do - touch "$E/stop-$worker" -done < "$E/pids.tsv" -failed=0 -: > "$E/waits.tsv" -while IFS="$(printf '\t')" read -r worker pid; do - wait "$pid" - rc=$? - printf '%s\t%s\t%s\n' "$worker" "$pid" "$rc" >> "$E/waits.tsv" - test "$rc" -eq 0 || failed=$((failed + 1)) - rctl -r "process:$pid:vmemoryuse:deny=640M" 2>/dev/null || true -done < "$E/pids.tsv" -test "$failed" -eq 0 -grep '^state=released .*recovery=ok exit=0$' "$E"/pressure-*.log -vmstat -H 1 10 | tee "$E/vmstat-after.txt" -rctl | tee "$E/rctl-after.txt" -rm -f "$E"/go-* "$E"/stop-* -cat "$E/pids.tsv" "$E/waits.tsv" "$E"/pressure-*.log -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- RACCT/RCTL availability and before/pressure/after `vmstat` are recorded. -- Three persisted PIDs each report `allocation_failure=ENOMEM`, later report - exact released bytes and `recovery=ok`, and exit 0 when waited. -- The mounted 96 MiB file SHA256 and full bytes match the source while pressure - is held. -- G7 RCTL rules, synchronization files, mount, and provider are removed. - -Different page counts or allocation totals are recording-only. Read mismatch, -panic, hang, unreclaimed process, or unexpected kernel error is KFAIL. diff --git a/tests/TC128-sequential-throughput.md b/tests/TC128-sequential-throughput.md deleted file mode 100644 index e2d6436..0000000 --- a/tests/TC128-sequential-throughput.md +++ /dev/null @@ -1,62 +0,0 @@ -# Test Case: Sequential Throughput - -**Test ID**: TC128-sequential-throughput -**Category**: Boundary & Stress -**Priority**: Medium -**Regression**: None - -## Objective - -Record reproducible sequential-read metrics for the exact 256 MiB fixture -under QEMU TCG while gating PASS only on complete and correct reads. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md` and record the QEMU TCG configuration. -- Do not apply fixed MB/s thresholds across hosts. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRC=$FIX/sources/workloads/sequential/sequential.bin -IMAGE=$FIX/images/workloads.erofs -MNT=/mnt/repo22-g7-tc128 -E=$G7/evidence/TC128 -mkdir -p "$MNT" "$E" -test "$(stat -f %z "$SRC")" -eq 268435456 -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -TARGET=$MNT/sequential/sequential.bin -source_hash=$(sha256 -q "$SRC") -target_hash=$(sha256 -q "$TARGET") -printf 'bytes=268435456\nsource=%s\ntarget=%s\n' \ - "$source_hash" "$target_hash" | tee "$E/hashes.txt" -test "$source_hash" = "$target_hash" -cmp "$SRC" "$TARGET" -: > "$E/runs.tsv" -run=1 -while [ "$run" -le 3 ]; do - /usr/bin/time -p dd if="$TARGET" of=/dev/null bs=1m \ - > "$E/run-$run.log" 2>&1 - rc=$? - printf '%s\t%s\n' "$run" "$rc" >> "$E/runs.tsv" - cat "$E/run-$run.log" - test "$rc" -eq 0 || exit 1 - run=$((run + 1)) -done -cat "$E/runs.tsv" -test "$(awk '$2 != 0 { bad++ } END { print bad + 0 }' "$E/runs.tsv")" -eq 0 -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- Mounted size is exactly 268435456; SHA256 and full `cmp` match the source. -- All three `dd` reads transfer the full file and exit 0. -- Bytes/second and real/user/sys times are reported as TCG metrics only. -- No fixed speed gate, panic, hang, or cleanup leak occurs. diff --git a/tests/TC129-random-read-pattern.md b/tests/TC129-random-read-pattern.md deleted file mode 100644 index cd53f08..0000000 --- a/tests/TC129-random-read-pattern.md +++ /dev/null @@ -1,68 +0,0 @@ -# Test Case: Deterministic Random Reads - -**Test ID**: TC129-random-read-pattern -**Category**: Boundary & Stress -**Priority**: Medium -**Regression**: None - -## Objective - -Compare every 4 KiB random read to the exact source at deterministic offsets, -and record TCG IOPS/latency without fixed performance thresholds. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md` and compile the exact `g7_probe.c`. -- Do not use `fio` without data verification and do not impose 1000 IOPS or - 10 ms cross-machine gates. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRC=$FIX/sources/workloads/random/random.bin -IMAGE=$FIX/images/workloads.erofs -MNT=/mnt/repo22-g7-tc129 -E=$G7/evidence/TC129 -mkdir -p "$MNT" "$E" -test "$(stat -f %z "$SRC")" -eq 67108864 -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -TARGET=$MNT/random/random.bin -source_hash=$(sha256 -q "$SRC") -target_hash=$(sha256 -q "$TARGET") -printf 'source=%s\ntarget=%s\n' "$source_hash" "$target_hash" \ - | tee "$E/hashes.txt" -test "$source_hash" = "$target_hash" -cmp "$SRC" "$TARGET" -: > "$E/runs.tsv" -run=1 -while [ "$run" -le 3 ]; do - "$G7/g7_probe" random "$SRC" "$TARGET" \ - 0x6a09e667f3bcc909 16384 4096 > "$E/run-$run.log" - rc=$? - digest=$(awk '{ for (i=1; i<=NF; i++) if ($i ~ /^digest=/) print $i }' \ - "$E/run-$run.log") - printf '%s\t%s\t%s\n' "$run" "$rc" "$digest" >> "$E/runs.tsv" - cat "$E/run-$run.log" - test "$rc" -eq 0 || exit 1 - run=$((run + 1)) -done -cat "$E/runs.tsv" -test "$(awk '$2 != 0 { bad++ } END { print bad + 0 }' "$E/runs.tsv")" -eq 0 -test "$(awk '{ print $3 }' "$E/runs.tsv" | sort -u | wc -l | tr -d ' ')" -eq 1 -grep 'mismatches=0' "$E"/run-*.log -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- Full source/mount SHA256 and bytes match. -- Each run compares exactly 16,384 source/target 4 KiB reads, reports zero - mismatches, exits 0, and produces the same deterministic digest. -- IOPS, mean microseconds, and elapsed time are recorded only as TCG metrics. -- No fixed speed gate, panic, hang, or cleanup leak occurs. diff --git a/tests/TC130-mixed-workload.md b/tests/TC130-mixed-workload.md deleted file mode 100644 index 600d477..0000000 --- a/tests/TC130-mixed-workload.md +++ /dev/null @@ -1,121 +0,0 @@ -# Test Case: Mixed Workload - -**Test ID**: TC130-mixed-workload -**Category**: Boundary & Stress -**Priority**: High -**Regression**: None - -## Objective - -Run simultaneous sequential compare, full hash, two deterministic random-read -streams, multi-file hashes, recursive names, and stat-size inventory. Persist -and wait every PID, then prove complete process/mount/provider cleanup. - -## Preconditions - -- Complete `G7-MANUAL-SETUP.md` and TC126-TC129 prerequisites. - -## Manual Steps - -```sh -G7=/root/repo22-g7 -FIX=$G7/fixtures -SRC=$FIX/sources/workloads -IMAGE=$FIX/images/workloads.erofs -MNT=/mnt/repo22-g7-tc130 -E=$G7/evidence/TC130 -mkdir -p "$MNT" "$E" -rm -f "$E"/*.tmp "$E"/pids.tsv "$E"/waits.tsv -unit=$(mdconfig -a -t vnode -f "$IMAGE") -mount -t erofs -o ro "/dev/$unit" "$MNT" -: > "$E/pids.tsv" -( - cmp "$SRC/sequential/sequential.bin" \ - "$MNT/sequential/sequential.bin" -) > "$E/sequential.log" 2>&1 & -pid=$!; printf 'sequential\t%s\n' "$pid" >> "$E/pids.tsv" -( - expected=$(sha256 -q "$SRC/pressure/pressure.bin") - actual=$(sha256 -q "$MNT/pressure/pressure.bin") - printf 'expected=%s actual=%s\n' "$expected" "$actual" - test "$actual" = "$expected" -) > "$E/full-hash.log" 2>&1 & -pid=$!; printf 'full-hash\t%s\n' "$pid" >> "$E/pids.tsv" -( - "$G7/g7_probe" random "$SRC/random/random.bin" \ - "$MNT/random/random.bin" 0xbb67ae8584caa73b 8192 4096 -) > "$E/random-a.log" 2>&1 & -pid=$!; printf 'random-a\t%s\n' "$pid" >> "$E/pids.tsv" -( - "$G7/g7_probe" random "$SRC/random/random.bin" \ - "$MNT/random/random.bin" 0x3c6ef372fe94f82b 8192 4096 -) > "$E/random-b.log" 2>&1 & -pid=$!; printf 'random-b\t%s\n' "$pid" >> "$E/pids.tsv" -( - index=0 - while [ "$index" -lt 16 ]; do - file=$(printf 'reader-%02d.bin' "$index") - expected=$(sha256 -q "$SRC/concurrent/$file") - actual=$(sha256 -q "$MNT/concurrent/$file") - printf '%s expected=%s actual=%s\n' "$file" "$expected" "$actual" - test "$actual" = "$expected" || exit 1 - index=$((index + 1)) - done -) > "$E/multi-hash.log" 2>&1 & -pid=$!; printf 'multi-hash\t%s\n' "$pid" >> "$E/pids.tsv" -( - (cd "$SRC" && find . -type f -print | sort) > "$E/source.names.tmp" - (cd "$MNT" && find . -type f -print | sort) > "$E/target.names.tmp" - cmp "$E/source.names.tmp" "$E/target.names.tmp" -) > "$E/names.log" 2>&1 & -pid=$!; printf 'names\t%s\n' "$pid" >> "$E/pids.tsv" -( - (cd "$SRC" && find . -type f -exec stat -f '%N %z' {} + | sort) \ - > "$E/source.stat.tmp" - (cd "$MNT" && find . -type f -exec stat -f '%N %z' {} + | sort) \ - > "$E/target.stat.tmp" - cmp "$E/source.stat.tmp" "$E/target.stat.tmp" -) > "$E/stat.log" 2>&1 & -pid=$!; printf 'stat\t%s\n' "$pid" >> "$E/pids.tsv" -test "$(wc -l < "$E/pids.tsv" | tr -d ' ')" -eq 7 -failed=0 -: > "$E/waits.tsv" -while IFS="$(printf '\t')" read -r worker pid; do - wait "$pid" - rc=$? - printf '%s\t%s\t%s\n' "$worker" "$pid" "$rc" >> "$E/waits.tsv" - test "$rc" -eq 0 || failed=$((failed + 1)) -done < "$E/pids.tsv" -cat "$E/pids.tsv" "$E/waits.tsv" "$E"/*.log -test "$failed" -eq 0 -test "$(wc -l < "$E/waits.tsv" | tr -d ' ')" -eq 7 -alive=0 -while IFS="$(printf '\t')" read -r worker pid; do - if kill -0 "$pid" 2>/dev/null; then - printf 'still-alive worker=%s pid=%s\n' "$worker" "$pid" - alive=$((alive + 1)) - fi -done < "$E/pids.tsv" -test "$alive" -eq 0 -mv "$E/source.names.tmp" "$E/source.names" -mv "$E/target.names.tmp" "$E/target.names" -mv "$E/source.stat.tmp" "$E/source.stat" -mv "$E/target.stat.tmp" "$E/target.stat" -test -z "$(find "$E" -name '*.tmp' -print)" -test -z "$(find "$G7/evidence" \( -name 'go-*' -o -name 'stop-*' \) -print)" -umount "$MNT" -mdconfig -d -u "${unit#md}" -test -z "$(mount -p | awk -v m="$MNT" '$2 == m { print }')" -test -z "$(mdconfig -l | tr ' ' '\n' | grep -x "$unit")" -``` - -## PASS Gate - -- Exactly seven PIDs and seven waits are persisted; every child exits 0 and - no recorded PID remains alive. -- Sequential bytes, full hash, both random streams, all 16 file hashes, exact - names, and exact stat sizes match their sources. -- All temporary, synchronization, process, mount, and md resources are gone; - evidence logs remain for audit. -- No `killall`, timeout masking, deadlock, panic, hang, or unexpected dmesg - error occurs. diff --git a/tests/TC131-nfs-export-basic.md b/tests/TC131-nfs-export-basic.md deleted file mode 100644 index 73dc0bb..0000000 --- a/tests/TC131-nfs-export-basic.md +++ /dev/null @@ -1,143 +0,0 @@ -# Test Case: FreeBSD NFSv3 Export Basic Functionality - -**Test ID**: TC131-nfs-export-basic -**Category**: NFS Export -**Priority**: Critical -**Regression**: EROFS `VOP_VPTOFH` / `VFS_FHTOVP` / export updates - -## Objective - -Verify that FreeBSD 15 mountd can install an export on a read-only EROFS -mount, that nfsd can resolve EROFS file handles, and that regular files, -directories, symlinks, and FIFOs are visible through a real NFSv3 client. - -## Preconditions - -- FreeBSD 15 amd64 server/client, optionally the same host over loopback. -- Root privilege for module, md, NFS service, `getfh`, and `fhopen` operations. -- `erofs.ko`, `test-nfs.erofs`, and `tests/nfs_fh_tool.c` available in the - guest. -- The fixture contains `basic/regular.txt`, `basic/subdir`, - `basic/link-to-regular`, and `basic/test.fifo`. - -## Procedure - -1. Build the module and helper: - - ```sh - ./build.sh - cc -O2 -Wall -Wextra -std=c17 tests/nfs_fh_tool.c -o nfs_fh_tool - ``` - -2. Load and mount EROFS on a fixed md unit: - - ```sh - kldload ./erofs.ko - mdconfig -a -t vnode -f test-nfs.erofs -u 42 - mkdir -p /mnt/repo22-erofs - mount -t erofs -o ro /dev/md42 /mnt/repo22-erofs - mount -v | grep /mnt/repo22-erofs - ``` - - The line must show `read-only` and must not show `NFS exported` before - mountd processes `/etc/exports`. - -3. Configure the real FreeBSD mountd/nfsd flow: - - ```sh - cp -p /etc/exports /tmp/exports.before 2>/dev/null || true - printf '%s\n' \ - '/mnt/repo22-erofs -ro -maproot=root -network 127.0.0.0 -mask 255.0.0.0' \ - > /etc/exports - service rpcbind onestart - service mountd onestart - service nfsd onestart - service mountd onereload - rpcinfo -p 127.0.0.1 - showmount -e 127.0.0.1 - mount -v | grep /mnt/repo22-erofs - ``` - - After reload, the EROFS mount line must show `NFS exported`. The flag is - installed by the generic FreeBSD export layer after the filesystem accepts - the export-only `MNT_UPDATE`; EROFS must not set it during the initial mount. - -4. Mount the export through NFSv3 and confirm the negotiated options: - - ```sh - mkdir -p /mnt/repo22-nfs - mount_nfs -o nfsv3,tcp,rdirplus 127.0.0.1:/mnt/repo22-erofs \ - /mnt/repo22-nfs - nfsstat -m - ``` - -5. Verify all required vnode types and read-only behavior: - - ```sh - cmp /mnt/repo22-nfs/basic/regular.txt \ - /mnt/repo22-erofs/basic/regular.txt - test -d /mnt/repo22-nfs/basic/subdir - test "$(readlink /mnt/repo22-nfs/basic/link-to-regular)" = regular.txt - test -p /mnt/repo22-nfs/basic/test.fifo - test "$(stat -f %i /mnt/repo22-nfs/basic/regular.txt)" = \ - "$(stat -f %i /mnt/repo22-erofs/basic/regular.txt)" - ! touch /mnt/repo22-nfs/write-must-fail - ``` - -6. Exercise both shared- and exclusive-lock file-handle resolution locally: - - ```sh - ./nfs_fh_tool capture /mnt/repo22-erofs/basic/regular.txt regular.fh - ./nfs_fh_tool stat regular.fh - ./nfs_fh_tool cat regular.fh regular.out - cmp regular.out /mnt/repo22-erofs/basic/regular.txt - ``` - - `fhstat` requests a shared lock and `fhopen` requests an exclusive lock. - -7. Confirm that NFSv3 used READDIRPLUS and did not issue writes: - - ```sh - nfsstat -c - nfsstat -s - ``` - -## Expected Results - -- mountd accepts the EROFS export-only update. -- `showmount -e` lists the EROFS path and `mount -v` shows `NFS exported` only - after mountd reloads exports. -- NFSv3 regular reads, directory traversal, symlink lookup, FIFO metadata, and - inode numbers match the direct EROFS mount. -- Writes fail with `EROFS`/`Read-only file system`. -- `fhstat` and `fhopen` both resolve the same 64-bit EROFS NID. -- NFS statistics show READDIRPLUS traffic and zero successful write RPCs. - -## Cleanup - -Always remove clients before stopping the loopback server: - -```sh -umount /mnt/repo22-nfs -: > /etc/exports -service mountd onereload -service nfsd onestop -service mountd onestop -service rpcbind onestop -umount /mnt/repo22-erofs -mdconfig -d -u 42 -kldunload erofs -rm -f /etc/exports -``` - -Restore a pre-existing `/etc/exports` instead of removing it when applicable. - -## Notes - -- The EROFS FreeBSD file-handle payload is 16 bytes: - `len`, `pad`, `nid_hi`, `nid_lo`, and a nonzero superblock-seeded per-inode - generation matching `va_gen`. -- Do not use Linux `exportfs`; FreeBSD mountd installs exports through a mount - update carrying the `export` option. -- Throughput is recorded for information only; correctness has no fixed MB/s - threshold. diff --git a/tests/TC132-nfs-file-handle-stability.md b/tests/TC132-nfs-file-handle-stability.md deleted file mode 100644 index b782b59..0000000 --- a/tests/TC132-nfs-file-handle-stability.md +++ /dev/null @@ -1,97 +0,0 @@ -# Test Case: FreeBSD NFS File-Handle Stability and Image Replacement - -**Test ID**: TC132-nfs-file-handle-stability -**Category**: NFS Export -**Priority**: Critical -**Regression**: Constant generation could resolve an old handle in a replacement image - -## Objective - -Verify 16-byte EROFS FIDs with full 64-bit NIDs and stable superblock-seeded -per-inode generations. The same image remounted on the same explicit md unit -must preserve the complete `fhandle_t`; a different image with the same device -number and NID must reject the old handle with `ESTALE`. Verify `va_gen` and -handle generation are identical. - -## Preconditions - -- FreeBSD 15 and `tests/nfs_fh_tool.c` compiled as `nfs_fh_tool`. -- Deterministic `nfs-a.erofs` and `nfs-b.erofs` from the metadata/VFS fixture - generator. They contain the same NIDs/content but different UUIDs. -- Use one explicit md unit for every remount/replacement step because FreeBSD - stores the filesystem ID outside the filesystem-private FID. - -## Direct Handle Procedure - -Perform these steps **before starting NFS clients**, so the direct EROFS mount -can be unmounted and replaced without `EBUSY`. - -1. Attach `nfs-a.erofs` to md80 and mount it. -2. Capture regular, directory, symlink (`lcapture`), and FIFO handles. For the - regular file: - - ```sh - ./nfs_fh_tool describe a.fh - ./nfs_fh_tool stat a.fh - stat -f 'gen=%v ino=%i' /mnt/repo22-erofs/basic/regular.txt - ``` - - Assert `len=16`, `pad=0`, full NID preservation, nonzero generation, and - `describe gen == stat st_gen`. -3. Verify malformed versus stale classification: - - ```sh - ./nfs_fh_tool mutate a.fh bad-len.fh len 15 - ./nfs_fh_tool mutate a.fh bad-pad.fh pad 1 - ./nfs_fh_tool mutate a.fh bad-gen.fh gen_xor 1 - ./nfs_fh_tool mutate a.fh bad-nid.fh nid_hi 0xffffffff - - ./nfs_fh_tool expect-stat bad-len.fh EINVAL - ./nfs_fh_tool expect-open bad-len.fh EINVAL - ./nfs_fh_tool expect-stat bad-pad.fh EINVAL - ./nfs_fh_tool expect-open bad-pad.fh EINVAL - ./nfs_fh_tool expect-stat bad-gen.fh ESTALE - ./nfs_fh_tool expect-open bad-gen.fh ESTALE - ./nfs_fh_tool expect-stat bad-nid.fh ESTALE - ./nfs_fh_tool expect-open bad-nid.fh ESTALE - ``` - -4. Unmount, detach, reattach **the same image** to md80, remount, capture - `a-remount.fh`, compare complete handle bytes, and read through the old - handle. -5. Unmount/detach, attach `nfs-b.erofs` to the same md80, and remount. Capture - `b.fh`; it must have the same NID and fsid but a different generation. - Both `fhstat(a.fh)` and `fhopen(a.fh)` must return `ESTALE`, while `b.fh` - succeeds. - -## NFS Restart Procedure - -After the replacement test is cleaned up, mount/export the qualified NFS -fixture and mount the NFS client. Keep a client descriptor open across nfsd -restart, but prevent the restarted daemon from inheriting the test descriptor: - -```sh -exec 3< /mnt/repo22-nfs/basic/regular.txt -service nfsd onerestart 3<&- -cat <&3 > open-after-restart.out -exec 3<&- -cmp open-after-restart.out /mnt/repo22-erofs/basic/regular.txt -``` - -Without `3<&-` on the service command, nfsd inherits the NFS-client descriptor -and can keep the client mount busy during cleanup. - -## Metabox and Multidevice Regression - -- Capture/resolve a metabox bit-63 NID and mutate it beyond the metabox backing - range; expect `ESTALE` only for invalid NID/generation. -- For an external-data file, a valid handle must still resolve after the data - provider is detached; the subsequent read preserves `ENXIO`/I/O error rather - than rewriting it to `ESTALE`. - -## Cleanup - -Unmount NFS clients first while rpcbind/mountd/nfsd still run. Then clear and -reload `/etc/exports`, stop nfsd, mountd, and rpcbind, unmount EROFS, detach md -providers, and unload the module. Assert every mount, md, module, service PID, -and export entry is gone. diff --git a/tests/TC133-nfs-export-stress.md b/tests/TC133-nfs-export-stress.md deleted file mode 100644 index a82c2a1..0000000 --- a/tests/TC133-nfs-export-stress.md +++ /dev/null @@ -1,176 +0,0 @@ -# Test Case: FreeBSD NFSv3 READDIRPLUS and Export Stress - -**Test ID**: TC133-nfs-export-stress -**Category**: NFS Export -**Priority**: High -**Regression**: directory cookies, EOF, concurrency, and export stability - -## Objective - -Stress the FreeBSD NFSv3 export with a 10,000+ entry directory, small negotiated -readdir sizes, repeated cookie pagination, multiple concurrent traversals, and -parallel reads/stat operations. Verify no duplicates, omissions, premature EOF, -stale handles, panic, or resource leak. - -## Preconditions - -- Complete TC131 server setup. -- A fixture with at least 12,050 deterministically named files in `bigdir`, a - `concurrent` file set, and a larger sequential-read file. -- Four client mount points are available. - -## Procedure - -1. Mount four NFSv3 client views with READDIRPLUS. Request small readdir sizes - on three mounts and record the effective values: - - ```sh - mount_nfs -o nfsv3,tcp,rdirplus,readdirsize=512 \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-512 - mount_nfs -o nfsv3,tcp,rdirplus,readdirsize=1024 \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-1024 - mount_nfs -o nfsv3,tcp,rdirplus,readdirsize=4096 \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-4096 - mount_nfs -o nfsv3,tcp,rdirplus \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-default - nfsstat -m - ``` - - FreeBSD may clamp very small requests to its client minimum. Record both the - requested and effective values; a clamp is an environment characteristic, - not a test failure. - -2. Traverse each mount and verify exact names, counts, and uniqueness: - - ```sh - jot -w file-%05d 12050 0 > expected.names - for M in 512 1024 4096 default; do - find /mnt/repo22-nfs-$M/bigdir -type f -maxdepth 1 | \ - sed 's#.*/##' > names.$M - test "$(wc -l < names.$M)" -eq 12050 - sort names.$M | uniq -d > duplicates.$M - test ! -s duplicates.$M - sort names.$M > sorted.$M - cmp expected.names sorted.$M - done - sha256 expected.names sorted.* - ``` - -3. Verify `.` and `..` are each returned exactly once and EOF is reached only - after all real entries: - - ```sh - ls -a1 /mnt/repo22-nfs-default/bigdir > dot-list - test "$(wc -l < dot-list)" -eq 12052 - test "$(grep -cx '\.' dot-list)" -eq 1 - test "$(grep -cx '\.\.' dot-list)" -eq 1 - ``` - -4. Force cold cookie pagination by repeatedly unmounting/remounting the small - client and comparing every complete listing: - - ```sh - for round in 1 2 3 4 5; do - umount /mnt/repo22-nfs-512 - mount_nfs -o nfsv3,tcp,rdirplus,readdirsize=512 \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-512 - find /mnt/repo22-nfs-512/bigdir -type f -maxdepth 1 | \ - sed 's#.*/##' | sort > cold-$round.sorted - cmp expected.names cold-$round.sorted - done - ``` - -5. Run at least eight traversal workers, three rounds each, distributed across - the four mounts. Save every PID and wait for every PID individually; a bare - final `wait` does not prove that all background jobs succeeded: - - ```sh - pids="" - worker=0 - for round in 1 2 3; do - for M in 512 1024 4096 default; do - worker=$((worker + 1)) - (find /mnt/repo22-nfs-$M/bigdir -type f -maxdepth 1 | \ - sed 's#.*/##' | sort | cmp expected.names -) \ - > walk-$worker.log 2>&1 & - pids="$pids $!" - done - done - status=0 - for pid in $pids; do - wait "$pid" || status=1 - done - test "$status" -eq 0 - ``` - -6. Run parallel data and metadata operations: - - ```sh - pids="" - for M in 512 1024 4096 default; do - (find /mnt/repo22-nfs-$M/concurrent -type f -maxdepth 1 -print0 | \ - xargs -0 -n 1 -P 8 cat > /dev/null) > cat-$M.log 2>&1 & - pids="$pids $!" - (find /mnt/repo22-nfs-$M/concurrent -type f -maxdepth 1 -print0 | \ - xargs -0 -n 1 -P 8 stat -f '%i %z' > /dev/null) \ - > stat-$M.log 2>&1 & - pids="$pids $!" - done - status=0 - for pid in $pids; do - wait "$pid" || status=1 - done - test "$status" -eq 0 - ``` - -7. Record throughput as information only: - - ```sh - /usr/bin/time -p dd if=/mnt/repo22-nfs-default/throughput.dat \ - of=/dev/null bs=1m - ``` - -8. Record NFS and kernel state: - - ```sh - nfsstat -c - nfsstat -s - vmstat -H - dmesg - ``` - - READDIRPLUS must be non-zero. Timed-out RPCs, retries, successful write RPCs, - stale-handle messages, traps, and panics must be zero. - -## Expected Results - -- Every listing contains exactly 12,050 unique expected names. -- Cold remount and concurrent listings have identical SHA256 values. -- Cookie pagination has no duplicate or missing entry and no premature EOF. -- Dot-omitted EROFS directories expose one `.` and one `..` through NFS. -- Concurrent reads and stats complete without stale handles or deadlock. -- READDIRPLUS activity is visible in client/server statistics. -- Throughput is reported without a fixed pass/fail threshold. - -## Cleanup - -Unmount every NFS client while nfsd is still running, then reload an empty -export list, stop services, unmount EROFS, detach md, and unload the module: - -```sh -for M in 512 1024 4096 default; do - umount /mnt/repo22-nfs-$M -done -: > /etc/exports -service mountd onereload -service nfsd onestop -service mountd onestop -service rpcbind onestop -umount /mnt/repo22-erofs -mdconfig -d -u 42 -kldunload erofs -``` - -Final EROFS/NFS mount counts, md units, loaded EROFS modules, service PIDs, and -unexpected new dmesg lines must all be zero. Do not stop NFS services before -client unmounts; hard localhost NFS mounts can otherwise become uninterruptible. diff --git a/tests/TC134-metabox-shared-nonzero-base.md b/tests/TC134-metabox-shared-nonzero-base.md deleted file mode 100644 index c9c6e96..0000000 --- a/tests/TC134-metabox-shared-nonzero-base.md +++ /dev/null @@ -1,26 +0,0 @@ -# Test Case: Metabox shared xattr with nonzero base - -**Test ID**: TC134-metabox-shared-nonzero-base -**Fixture**: `metabox-plain.erofs` - -## Objective - -Verify that metabox shared IDs are relative to the nonzero -`xattr_blkaddr << blkszbits`. Require manifest fields `xattr_blkaddr=1`, -carrier xattr base 4096, shared record IDs/offsets, and bit-63 inode NIDs. - -## Manual steps - -```sh -getextattr -qq -x user metaboxshared /mnt/repo22-g4/metabox-a -getextattr -qq -x user metaboxshared /mnt/repo22-g4/metabox-b -sha256 -q /mnt/repo22-g4/metabox-a /mnt/repo22-g4/metabox-b -stat -f 'inode=%i size=%z' /mnt/repo22-g4/metabox-a \ - /mnt/repo22-g4/metabox-b -``` - -## Expected results - -Both values are `nonzero-base`; data hashes match source and inode numbers -retain bit 63. A read from logical offset `shared_id * 4` must not occur. -Clean up. diff --git a/tests/TC135-prefix-metabox-and-primary-fallback.md b/tests/TC135-prefix-metabox-and-primary-fallback.md deleted file mode 100644 index 87dab3b..0000000 --- a/tests/TC135-prefix-metabox-and-primary-fallback.md +++ /dev/null @@ -1,36 +0,0 @@ -# Test Case: Long-prefix backing selection - -**Test ID**: TC135-prefix-metabox-and-primary-fallback -**Fixtures**: `metabox-plain.erofs`, `basic.erofs`, `prefix-primary.erofs` - -## Objective - -Verify prefix table selection in order: metabox, packed inode, then primary -metadata. The manifest must prove each backing and record all changed super -fields and image hashes. - -## Manual steps - -Use a separate attach/mount/cleanup cycle for each image: - -```sh -# metabox-plain.erofs -lsextattr user /mnt/repo22-g4/metabox-a -getextattr -qq -x user repo22.application.component.setting \ - /mnt/repo22-g4/metabox-a - -# basic.erofs (packed backing) -lsextattr user /mnt/repo22-g4/prefix-user-0 -getextattr -qq -x user repo22.application.component.setting \ - /mnt/repo22-g4/prefix-user-0 - -# prefix-primary.erofs (PLAIN_XATTR_PFX) -lsextattr user /mnt/repo22-g4/prefix-user-0 -getextattr -qq -x user repo22.application.component.setting \ - /mnt/repo22-g4/prefix-user-0 -``` - -## Expected results - -Values are `long-prefix-value`, `prefix-value-0`, and `prefix-value-0`. -Every cycle cleans its mount and md provider. diff --git a/tests/TC136-metabox-truncated-super-extension.md b/tests/TC136-metabox-truncated-super-extension.md deleted file mode 100644 index 1a0a6d5..0000000 --- a/tests/TC136-metabox-truncated-super-extension.md +++ /dev/null @@ -1,26 +0,0 @@ -# Test Case: Truncated metabox super extension - -**Test ID**: TC136-metabox-truncated-super-extension -**Fixtures**: `bad-metabox-truncated-extension.erofs`, `bad-ishare-prefix-id.erofs` - -## Objective - -Reject `METABOX` when `sb_extslots=0` leaves `metabox_nid` undeclared, and -reject `ISHARE_XATTRS` when its prefix ID equals `xattr_prefix_count`. - -## Manual steps - -For each image, attach it and trace the read-only mount: - -```sh -truss -f -o /tmp/tc136.truss mount -t erofs -o ro \ - /dev/${unit} /mnt/repo22-g4 -grep nmount /tmp/tc136.truss -mount | grep repo22-g4 || true -mdconfig -d -u "${unit#md}" -``` - -## Expected results - -Both `nmount` calls return `EINTEGRITY` (97). The extension bytes are not -consumed, no mount appears, and detach succeeds without a vnode/md leak. diff --git a/tests/TC137-xattr-declared-image-bounds.md b/tests/TC137-xattr-declared-image-bounds.md deleted file mode 100644 index fc19eb1..0000000 --- a/tests/TC137-xattr-declared-image-bounds.md +++ /dev/null @@ -1,37 +0,0 @@ -# Test Case: Xattr declared-image bounds - -**Test ID**: TC137-xattr-declared-image-bounds -**Fixtures**: `basic.erofs`, `bad-shared-declared-bounds.erofs`, `bad-prefix-declared-bounds.erofs` - -## Objective - -Ensure shared and prefix reads are bounded by superblock `blocks`, even when -the provider contains a valid-looking appended sentinel. Require manifest -provider/declared sizes, sentinel offsets, redirected ID/start, and hashes. - -## Manual steps - -Run three independent cycles: - -```sh -# basic.erofs control -getextattr -qq -x user shared_key /mnt/repo22-g4/shared-a -getextattr -qq -x user local /mnt/repo22-g4/shared-a - -# bad-shared-declared-bounds.erofs -stat -f 'size=%z' /mnt/repo22-g4/shared-a -getextattr -qq -x user local /mnt/repo22-g4/shared-a -truss -o /tmp/tc137-shared.truss getextattr -qq user shared_key \ - /mnt/repo22-g4/shared-a -grep extattr_get_file /tmp/tc137-shared.truss - -# bad-prefix-declared-bounds.erofs -truss -f -o /tmp/tc137-prefix.truss mount -t erofs -o ro \ - /dev/${unit} /mnt/repo22-g4 -grep nmount /tmp/tc137-prefix.truss -``` - -## Expected results - -Control passes. The shared sentinel get and prefix mount return `EINTEGRITY` -(97); normal data and `local` remain readable. Sentinel bytes never appear. diff --git a/tests/TC138-acl-linux-order-and-empty-header.md b/tests/TC138-acl-linux-order-and-empty-header.md deleted file mode 100644 index 84a7237..0000000 --- a/tests/TC138-acl-linux-order-and-empty-header.md +++ /dev/null @@ -1,27 +0,0 @@ -# Test Case: Linux ACL order and empty header - -**Test ID**: TC138-acl-linux-order-and-empty-header -**Fixture**: `basic.erofs` - -## Objective - -Accept Linux ACL tag phases without sorting named IDs, treat a version-only -header as mode fallback, and reject duplicate IDs, bad phase, and bad perms. -The manifest must record rebuilt xattr bodies and `i_xattr_icount` fields. - -## Manual steps - -```sh -getfacl -n /mnt/repo22-g4/acl-unordered -getfacl -n /mnt/repo22-g4/acl-header -truss -o /tmp/tc138-duplicate.truss getfacl -n \ - /mnt/repo22-g4/acl-duplicate -truss -o /tmp/tc138-phase.truss getfacl -n /mnt/repo22-g4/acl-phase -truss -o /tmp/tc138-perm.truss getfacl -n /mnt/repo22-g4/acl-perm -grep __acl_get_file /tmp/tc138-*.truss -``` - -## Expected results - -The first ACL lists user 3002 before 2002. Header-only output is mode-derived -`rw-/r--/r--`. All three malformed ACLs return `EINTEGRITY` (97). Clean up. diff --git a/tests/TC139-fifo-acl-xattr-readonly.md b/tests/TC139-fifo-acl-xattr-readonly.md deleted file mode 100644 index 56a8edb..0000000 --- a/tests/TC139-fifo-acl-xattr-readonly.md +++ /dev/null @@ -1,31 +0,0 @@ -# Test Case: FIFO ACL and xattr read-only VOPs - -**Test ID**: TC139-fifo-acl-xattr-readonly -**Fixture**: `basic.erofs`, `/fifo-node` - -## Objective - -Verify FIFO stat, ACL, and xattr reads without opening FIFO data, and require -read-only errors for all mutations. The transformer must record the regular -placeholder to FIFO `i_mode` change and ACL body fields. - -## Manual steps - -```sh -stat -f 'type=%HT mode=%Sp size=%z inode=%i' /mnt/repo22-g4/fifo-node -getfacl -n /mnt/repo22-g4/fifo-node -lsextattr system /mnt/repo22-g4/fifo-node -lsextattr user /mnt/repo22-g4/fifo-node -getextattr -qq -x system posix_acl_access /mnt/repo22-g4/fifo-node -getextattr -qq -x user fifo_note /mnt/repo22-g4/fifo-node -truss -o /tmp/tc139-acl.truss setfacl -m u::rwx /mnt/repo22-g4/fifo-node -truss -o /tmp/tc139-set.truss setextattr user repo22 changed \ - /mnt/repo22-g4/fifo-node -truss -o /tmp/tc139-del.truss rmextattr system posix_acl_access \ - /mnt/repo22-g4/fifo-node -``` - -## Expected results - -Stat reports FIFO, reads return the exact ACL and `fifo-readonly-xattr`, and -all three mutations return `EROFS` (30). Nothing blocks or logs a trap. Clean up. diff --git a/tests/TC140-compressed-metabox-carrier.md b/tests/TC140-compressed-metabox-carrier.md deleted file mode 100644 index 1d78a90..0000000 --- a/tests/TC140-compressed-metabox-carrier.md +++ /dev/null @@ -1,37 +0,0 @@ -# Test Case: Compressed metabox carrier - -**Test ID**: TC140-compressed-metabox-carrier -**Fixtures**: `metabox-compressed.erofs`, `metabox-fragment.erofs`, four `bad-*` fragment images - -## Objective - -Verify metadata/xattr reads from compressed and fragment-backed compressed -metabox carriers, then reject loop, range, and recursive carrier relations. -Require manifest carrier layout, NIDs, packed size, fragment offset/range, -relocated metadata block, transformed fields, and image/source hashes. - -## Positive steps - -Mount each positive image separately and run: - -```sh -stat -f 'mode=%Sp size=%z inode=%i' /mnt/repo22-g4/metabox-a -sha256 -q /mnt/repo22-g4/metabox-a -lsextattr user /mnt/repo22-g4/metabox-a -getextattr -qq -x user metaboxshared /mnt/repo22-g4/metabox-a -getextattr -qq -x user repo22.application.component.setting \ - /mnt/repo22-g4/metabox-a -getextattr -qq -x user shared-prefix-key /mnt/repo22-g4/metabox-a -``` - -## Negative steps - -Attach and trace-mount `bad-fragment-self-loop.erofs`, -`bad-fragment-range.erofs`, `bad-metabox-recursive-nid.erofs`, and -`bad-packed-recursive-nid.erofs` separately. Record `nmount`, dmesg, -`vmstat -m | grep erofs`, responsiveness, and cleanup after each. - -## Expected results - -Both positives return source hash and exact values. All four negatives return -`EINTEGRITY` (97), with active EROFS allocations, mounts, and md units at zero. diff --git a/tests/TC141-cold-nested-namei.md b/tests/TC141-cold-nested-namei.md deleted file mode 100644 index 1d6de7c..0000000 --- a/tests/TC141-cold-nested-namei.md +++ /dev/null @@ -1,44 +0,0 @@ -# Test Case: Cold Nested Namei and Corruption - -**Test ID**: TC141-cold-nested-namei -**Category**: Directory Lookup -**Priority**: Critical - -## Objective - -Observe nested positive and missing lookup on fresh mounts, complete multi- -block cookies, Linux-compatible padding, and stable EINTEGRITY for three -structured directory corruptions. - -## Procedure - -On a fresh metadata/namei-base.erofs mount, make the first pathname operation: - - ./g3_vfs_probe expect-success open-read \ - /tmp/repo22-g3/mnt/alpha/bravo/charlie/payload.txt - -Compare the payload with the recorded source, require two direct missing -lookups to return errno 2, and run: - - ./readdir_probe /tmp/repo22-g3/mnt/wide 128 - -Repeat the cold positive read and cookie probe on -metadata/namei-padding-nonzero.erofs. - -Finally mount each corruption independently and issue two direct stat calls: - -| Image | Path | Expected errno | -|---|---|---:| -| namei-corrupt-short.erofs | /alpha | 97 | -| namei-corrupt-nameoff.erofs | /alpha | 97 | -| namei-corrupt-name.erofs | /wide/entry-077-abcdefghijklmnopqrstuvwxyz.txt | 97 | - -Run readdir_probe on the affected directory and require its direct syscall -failure to report integrity failure. Unmount and detach between every image. - -## Expected Results - -Fresh positive lookup reads exact data; repeated missing lookup is ENOENT; both -valid variants report all 322 restartable entries. Every corruption remains -EINTEGRITY on both lookup attempts and readdir, without cache masking, panic, -or hang. Record all five image hashes and cleanup. diff --git a/tests/TC142-fragment-backed-compressed-metabox.md b/tests/TC142-fragment-backed-compressed-metabox.md deleted file mode 100644 index 309a0a2..0000000 --- a/tests/TC142-fragment-backed-compressed-metabox.md +++ /dev/null @@ -1,42 +0,0 @@ -# Test Case: Fragment-backed compressed metabox - -**Test ID**: TC142-fragment-backed-compressed-metabox -**Fixture**: `metabox-fragment.erofs` and its four negative variants - -## Objective - -Independently regress packed-carrier initialization before a fragment-backed -metabox read and recursion/range rejection. Do not reuse TC140 guest results. - -## Layout qualification - -`g4_fixtures.py verify` must prove a compressed metabox carrier, whole-file -fragment header with bit 63 set, `fragment_offset + carrier_size <= packed_size`, -ordinary metabox/packed NIDs, bit-63 synthetic inode NIDs, CRC-valid images, -and one-field mutations for self-loop, range, metabox recursion, and packed -recursion. Record source and all image SHA256 values. - -## Positive steps - -```sh -unit=$(mdconfig -a -t vnode -f /tmp/repo22-g4/images/metabox-fragment.erofs) -mount -t erofs -o ro /dev/${unit} /mnt/repo22-g4 -stat -f 'mode=%Sp size=%z inode=%i' /mnt/repo22-g4/metabox-a -sha256 -q /mnt/repo22-g4/metabox-a -lsextattr user /mnt/repo22-g4/metabox-a -getextattr -qq -x user shared-prefix-key /mnt/repo22-g4/metabox-a -getextattr -qq -x user item /mnt/repo22-g4/metabox-a -umount /mnt/repo22-g4 -mdconfig -d -u "${unit#md}" -``` - -## Negative steps - -Trace-mount each of the four negative images in a fresh md cycle. Capture the -exact `nmount` errno/text, pre/post dmesg, `vmstat -m`, guest responsiveness, -and zero mount/md cleanup. - -## Expected results - -Positive file SHA256 and values `shared-value`/`value-000` match source. Every -negative returns `EINTEGRITY` (97), with no leak, hang, panic, trap, or residue. diff --git a/tests/TC143-deflate-zstd-partial-reference.md b/tests/TC143-deflate-zstd-partial-reference.md deleted file mode 100644 index 99fcadc..0000000 --- a/tests/TC143-deflate-zstd-partial-reference.md +++ /dev/null @@ -1,81 +0,0 @@ -# Test Case: DEFLATE and ZSTD Partial References - -**Test ID**: TC143-deflate-zstd-partial-reference - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: Critical -**Regression**: Non-LZ4 partial-reference decoding - -## Objective -Verify full and partial-reference decoding for raw DEFLATE and ZSTD streams, -including exact full-file SHA256, cross-extent reads, random reads, and -corrupted-stream `EIO` behavior. - -## Preconditions -- FreeBSD 15 guest. -- ZSTDIO-enabled repo22 module for the ZSTD cases. -- Deterministic full-index fixtures whose layout parser proves: - - the full file points at a complete compressed stream; - - the partial file reuses that physical stream; - - the reused HEAD record has `Z_EROFS_LI_PARTIAL_REF` (`0x8000`) set; - - the reused first NONHEAD record has the complete source pcluster's - `D0_CBLKCNT`, not the partial file's original compressed-block count; - - the partial file's logical length is shorter than the source stream's - decompressed length. -- Corrupted copies produced only after locating the target compressed extent; - do not use a blind image offset. - -## Layout Proof -```sh -dump.erofs -s deflate-partial-ref.erofs -dump.erofs --path=/a.dat -e deflate-partial-ref.erofs -dump.erofs --path=/b.dat -e deflate-partial-ref.erofs -dump.erofs -s zstd-partial-ref.erofs -dump.erofs --path=/a.dat -e zstd-partial-ref.erofs -dump.erofs --path=/b.dat -e zstd-partial-ref.erofs -``` - -The byte-level fixture record must assert the original NID, map-header offset, -HEAD record, pblk, and checksum before setting the partial-reference bit or -redirecting a pblk. - -## Test Steps -1. Mount the DEFLATE image and hash `/a.dat` and `/b.dat` completely. -2. Read a range crossing a known DEFLATE logical extent boundary. -3. Read another non-aligned random range and compare it byte-for-byte with the - source. -4. Repeat steps 1-3 for ZSTD. -5. Mount each corrupted copy and read the targeted file through a small C - errno probe. -6. Compare pre/post `dmesg` and active `vmstat -m` EROFS allocations. - -The G5 DEFLATE partial is 100000 bytes because erofs-utils 1.8.6 splits the -1 MiB DEFLATE source according to the 32 KiB DEFLATE window. It reuses the -first 17-block source pcluster and uses offsets 65500/2048 and 90000/4096. -The ZSTD partial is 700000 bytes and uses these checks: - -```sh -dd if=/mnt/repo22/b.dat of=/tmp/guest bs=1 skip=122900 count=512 status=none -dd if=source-b.dat of=/tmp/source bs=1 skip=122900 count=512 status=none -cmp /tmp/guest /tmp/source - -dd if=/mnt/repo22/b.dat of=/tmp/guest bs=1 skip=524287 count=4097 status=none -dd if=source-prefix-b.dat of=/tmp/source bs=1 skip=524287 count=4097 status=none -cmp /tmp/guest /tmp/source -``` - -## Expected Results -- Full DEFLATE and ZSTD streams require complete output and stream completion. -- Partial references succeed after producing the requested logical prefix; - they do not require the reused source frame to finish. -- Complete SHA256 and every boundary/random byte comparison match. -- Targeted corruption returns read errno 5 (`EIO`), with no leaked active - allocation, panic, or trap. -- `control.bin` in each corrupted image remains byte-identical to its source. - -## Cleanup -Unmount each image, detach every md unit, remove temporary range outputs, and -unload the module by its `kldstat` ID. diff --git a/tests/TC144-microlzma-consumption-and-corruption.md b/tests/TC144-microlzma-consumption-and-corruption.md deleted file mode 100644 index 103beb6..0000000 --- a/tests/TC144-microlzma-consumption-and-corruption.md +++ /dev/null @@ -1,52 +0,0 @@ -# Test Case: MicroLZMA Consumption and Corruption - -**Test ID**: TC144-microlzma-consumption-and-corruption - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression -**Priority**: Critical -**Regression**: MicroLZMA full/partial completion semantics - -## Objective -Verify MicroLZMA input-consumption rules for complete streams, early-success -rules for partial references, and deterministic rejection of damaged streams. - -## Preconditions -- FreeBSD 15 guest and repo22 module. -- A full-index LZMA image with: - - `/a.dat`: 1048576-byte complete MicroLZMA stream; - - `/b.dat`: 700000-byte partial reference to `/a.dat`'s pcluster; - - the `/b.dat` HEAD record marked `Z_EROFS_LI_PARTIAL_REF`. - - the reused pblk and `D0_CBLKCNT` both equal `/a.dat`'s values. -- A corrupted copy with a byte changed inside the proven compressed byte range. -- Source files and their SHA256 values available on the guest. - -## Test Steps -1. Prove the pcluster and partial-reference record with `dump.erofs` and a - byte-level parser. -2. Mount the valid image and hash `/a.dat` completely. -3. Hash `/b.dat` completely; this requests only a prefix of the reused stream. -4. Compare reads at offsets 65500 and 524287 against source bytes. -5. Mount the corrupted image and read `/a.dat` with an errno-reporting helper. -6. Confirm the implementation's complete-stream condition includes both - `XZ_STREAM_END` and `buffer.in_pos == srclen`. -7. Compare `vmstat -m` active EROFS allocations and pre/post `dmesg`. - -## Expected Results -- `/a.dat` succeeds only after exact output, `XZ_STREAM_END`, and complete - compressed-input consumption. -- `/b.dat` accepts `XZ_OK` after the requested partial output is filled. -- Both complete SHA256 values and random ranges match. -- The corrupted full stream returns errno 5 (`EIO`). -- `control.bin` in the same corrupted image still matches its source. -- Decoder state is freed on success and error; active EROFS allocations return - to zero after unmount/unload. - -## Cleanup -```sh -umount /mnt/repo22 2>/dev/null || true -mdconfig -d -u "${unit#md}" 2>/dev/null || true -kldunload -i "${module_id}" -``` diff --git a/tests/TC145-zstdio-build-gate.md b/tests/TC145-zstdio-build-gate.md deleted file mode 100644 index 4241ed0..0000000 --- a/tests/TC145-zstdio-build-gate.md +++ /dev/null @@ -1,54 +0,0 @@ -# Test Case: FreeBSD ZSTDIO Build Gate - -**Test ID**: TC145-zstdio-build-gate - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Build / Compression -**Priority**: Critical -**Regression**: Optional-kernel ZSTD symbol references - -## Objective -Verify that repo22 uses FreeBSD's formal ZSTD header/API only when `ZSTDIO` is -enabled, emits no ZSTD symbol references otherwise, rejects ZSTD images -accurately in the disabled build, and reads them in the enabled build. - -## Preconditions -- FreeBSD 15 source tree used by `build.sh`. -- FreeBSD guest kernel built with ZSTDIO, so the enabled module can resolve the - official kernel symbols. -- One LZ4 image and one ZSTD image with known SHA256 output. - -## Test Steps -1. Build without ZSTDIO and save the module outside `build/`: - ```sh - WITH_ZSTDIO=0 ./build.sh - cp build/erofs.ko /tmp/erofs-nozstd.ko - ``` -2. Verify no unresolved `ZSTD_*` or `bcmp` symbol: - ```sh - nm -u /tmp/erofs-nozstd.ko | grep 'ZSTD_' && exit 1 || true - nm -u /tmp/erofs-nozstd.ko | grep -w bcmp && exit 1 || true - ``` -3. Load the disabled module, read the LZ4 control image, then attempt to mount - the ZSTD image. -4. Unload by the exact module ID reported by `kldstat`. -5. Build with ZSTDIO: - ```sh - WITH_ZSTDIO=1 ./build.sh - cp build/erofs.ko /tmp/erofs-zstdio.ko - ``` -6. Verify the undefined ZSTD symbols are official API names from - `` and that `bcmp` is absent. -7. Load the enabled module, mount the same ZSTD image, and verify full SHA256. -8. Unload by exact ID and audit final state. - -## Expected Results -- Disabled build: no `ZSTD_*` references, KLD load succeeds, LZ4 reads, and a - ZSTD image fails mount with `EOPNOTSUPP` and the message - `ZSTD compression requires ZSTDIO support`. -- Enabled build: KLD load succeeds and the ZSTD SHA256 matches. -- Both modules have no unresolved `bcmp`. -- The build rejects `WITH_ZSTDIO` values other than `0` or `1`. -- No module, mount, or md unit remains after cleanup. diff --git a/tests/TC146-head2-interlaced-extent-mapping.md b/tests/TC146-head2-interlaced-extent-mapping.md deleted file mode 100644 index a2be291..0000000 --- a/tests/TC146-head2-interlaced-extent-mapping.md +++ /dev/null @@ -1,85 +0,0 @@ -# Test Case: HEAD2, Interlaced, and Extent Mapping - -**Overall Status Rule**: **PARTIAL** when HEAD2 and interlaced pass but the -explicit mapped-payload subscenario remains **SHELVED**; see -`issues/extent-metadata-fixture-unavailable.md` - -**Test ID**: TC146-head2-interlaced-extent-mapping - -**G5 fixture contract**: Use [G5-MANUAL-SETUP.md](G5-MANUAL-SETUP.md). -Its generated paths, source comparisons, structured corruption offsets, and -cleanup rules supersede placeholder examples in this file. -**Category**: Compression Mapping -**Priority**: Critical -**Regression**: New compressed mapping formats - -## Objective -Verify FreeBSD kernel reads for HEAD2 and interlaced pclusters, and independently -review extent-record mapping when erofs-utils 1.8.6 cannot generate that format. - -## Preconditions -- FreeBSD 15 guest and repo22 module. -- HEAD2 fixture with byte-level assertions: - - incompat bit `EROFS_FEATURE_INCOMPAT_COMPR_HEAD2` is set; - - `Z_EROFS_ADVISE_BIG_PCLUSTER_2` is set; - - a proven full-index HEAD record type is changed from HEAD1 to HEAD2; - - the high algorithm nibble names the decoder used by HEAD2; - - CRC32C is recomputed. -- Interlaced fixture generated by erofs-utils 1.8.6: - ```sh - mkfs.erofs -zlz4 -C4096 -Efragments -T0 \ - interlaced.erofs source-dir - ``` - `dump.erofs -e` must show real 4096-byte plain extents interspersed with - compressed extents. - -## HEAD2 and Interlaced Steps -1. Prove the HEAD2 feature, map-header advise bits, algorithm nibbles, and HEAD2 - record before guest transfer. -2. Mount the HEAD2 image, hash the full file, and compare a read crossing the - first logical pcluster boundary recorded by the manifest. Do not assume the - logical boundary is 65536 merely because the physical pcluster limit is - 65536. -3. Read the targeted corrupted HEAD2 copy and record errno. -4. Mount the erofs-utils 1.8.6 interlaced image, hash the full file, and compare - a range crossing the first compressed/plain transition. -5. Compare pre/post `dmesg` and clean all resources. - -## Extent Metadata Static Review -Run these checks against exactly erofs-utils 1.8.6 and both reference trees: - -```sh -mkfs.erofs -V -grep -R -E 'Z_EROFS_ADVISE_EXTENTS|z_erofs_extent_recsize|struct z_erofs_extent[[:space:]]*\{' \ - /work/build/erofs-utils-v1.8.6-source/include \ - /work/build/erofs-utils-v1.8.6-source/lib -grep -n "struct z_erofs_extent\|z_erofs_extent_recsize" \ - src/erofs_fs.h /work/dev-src-linux/fs/erofs/erofs_fs.h -grep -n "z_erofs_map_blocks_ext" \ - src/zmap.c /work/dev-src-linux/fs/erofs/zmap.c -``` - -Review all four record sizes (4, 8, 16, 32 bytes): -- 4-byte records use the initial 64-bit physical base and accumulated `plen`; -- 8-byte records carry per-record 32-bit physical starts; -- 16/32-byte records use explicit extent counts and binary search by logical - start, with 32-byte records adding `lstart_hi`; -- `plen` format, partial-reference, interlaced/shifted, and final fragment - encodings are decoded in Linux order; -- metadata reads route through the metabox when the inode NID has bit 63; -- malformed zero-count explicit tables and fragment bounds fail closed. - -## Expected Results -- HEAD2 and interlaced full SHA256 and boundary reads match source bytes. -- Targeted HEAD2 corruption returns errno 5 (`EIO`) without a panic/trap. -- Explicit mapped-payload result is **SHELVED**, not PASS, because erofs-utils - 1.8.6 has no extent-record generator or on-disk extent structure and the - structured helper cannot validate a relocated mapped payload. -- The static FreeBSD/Linux format and control-flow review is recorded - separately from guest results. -- TC146 overall is **PARTIAL** unless all three subscenarios have dynamic - positive coverage. - -## Cleanup -Unmount all images, detach md units, unload by exact module ID, and verify no -new `dmesg` lines. diff --git a/tests/TC147-flat-inline-block-bounds.md b/tests/TC147-flat-inline-block-bounds.md deleted file mode 100644 index f8129db..0000000 --- a/tests/TC147-flat-inline-block-bounds.md +++ /dev/null @@ -1,46 +0,0 @@ -# Test Case: FLAT_INLINE Metadata-Block Bounds - -**Test ID**: TC147-flat-inline-block-bounds - -## Objective - -Verify a valid inline payload reads exactly and a checksum-valid inline range -crossing its inode metadata block fails repeatedly with `EINTEGRITY`. - -## Fixture - -Use `images/inline.erofs`, `images/inline-cross-block.erofs`, and -`source/inline/inline.txt`. Evidence records the NID/inode offset, patched -`i_xattr_icount`, inline start at block offset 4068, 31-byte size, valid CRC, -and both image/source hashes. - -## Procedure - -Mount `inline.erofs` on a fresh md unit: - -```sh -stat -f 'size=%z blocks=%b' "$mnt/inline.txt" -cmp /tmp/repo22-g1/source/inline/inline.txt "$mnt/inline.txt" -sha256 /tmp/repo22-g1/source/inline/inline.txt "$mnt/inline.txt" -``` - -Clean it up, then mount `inline-cross-block.erofs` without listing the root: - -```sh -./read_probe expect-error "$mnt/inline.txt" 97 -./read_probe expect-error "$mnt/inline.txt" 97 -set +e -truss -o /tmp/tc147.truss stat "$mnt/inline.txt" -rc=$? -set -e -printf 'stat_rc=%d\n' "$rc" -tail -20 /tmp/tc147.truss -``` - -Record dmesg delta and clean the second mount/provider. - -## Expected Results - -- Positive data matches source and reports 8 sectors. -- Every corrupt lookup/open returns errno 97, never `ENOENT` or stale-vnode - errors; no next-block read, panic, dmesg error, or cleanup residue occurs. diff --git a/tests/TC148-linux-directory-tail-padding.md b/tests/TC148-linux-directory-tail-padding.md deleted file mode 100644 index 65558c5..0000000 --- a/tests/TC148-linux-directory-tail-padding.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test Case: Nonzero Directory Tail Padding - -**Test ID**: TC148-linux-directory-tail-padding -**Category**: Directory Compatibility -**Priority**: Critical - -## Objective - -Verify FreeBSD accepts the structured eight-byte PAD!ERO! patch after the final -directory-name NUL and still returns complete names and restartable cookies. - -## Procedure - -Record metadata/fixture-evidence.txt and the -metadata/namei-padding-nonzero.erofs hash. On a fresh mount: - - printf 'wide entry 079\n' > expected-079.txt - cmp expected-079.txt \ - /tmp/repo22-g3/mnt/wide/entry-079-abcdefghijklmnopqrstuvwxyz.txt - test "$(find /tmp/repo22-g3/mnt/wide -maxdepth 1 -type f | - wc -l)" -eq 320 - ./readdir_probe /tmp/repo22-g3/mnt/wide 128 - -## Expected Results - -The actual patched image mounts, reads exact data, exposes 320 real files, and -reports 322 valid kernel/libc restart positions. Userspace fixture inspection -alone is not PASS. Record image/evidence hashes and cleanup. diff --git a/tests/TC149-vnode-pager-real-faults.md b/tests/TC149-vnode-pager-real-faults.md deleted file mode 100644 index 5f81b71..0000000 --- a/tests/TC149-vnode-pager-real-faults.md +++ /dev/null @@ -1,23 +0,0 @@ -# Test Case: Vnode Pager Real Faults - -**Test ID**: TC149-vnode-pager-real-faults -**Category**: VM Integration -**Priority**: Critical - -## Objective - -Exercise real sequential/randomized mmap faults, partial and full pages around -EOF, read-only shared mappings, and private COW on plain and LZ4 EROFS data. - -## Procedure - -On separate fresh mounts of vfs/vfs-plain.erofs and vfs/vfs-lz4.erofs: - - ./mmap_fault /tmp/repo22-g3/mnt/pager.bin - -## Expected Results - -Both runs print the same 21211-byte logical FNV hash and pass six random page -faults, sequential comparison, EOF zeroing, expected child SIGBUS, EACCES for -writable MAP_SHARED, private COW, and EROFS for O_RDWR. The expected child -SIGBUS dmesg line is evidence, not a kernel failure. Record hashes and cleanup. diff --git a/tests/TC150-48bit-fallback-root.md b/tests/TC150-48bit-fallback-root.md deleted file mode 100644 index 592ded6..0000000 --- a/tests/TC150-48bit-fallback-root.md +++ /dev/null @@ -1,43 +0,0 @@ -# Test Case: 48-bit Fallback Root Union - -**Test ID**: TC150-48bit-fallback-root - -## Objective - -Rerun the Linux-compatible `48BIT=1 && rootnid_8b=0` fallback on the current -exact KLD: the two-byte union remains `rootnid_2b`, not `blocks_hi`. - -## Fixture - -Use `images/fallback-48bit-root2.erofs` and `source/compact/root.txt`. -Structured evidence asserts incompat `0x80`, `rootnid_8b=0`, nonzero -`rootnid_2b`, provider-sized `blocks_lo`, and valid CRC. fsck.erofs 1.8.6 does -not recognize this newer incompat bit. - -## Procedure - -Host: - -```sh -python3 tests/erofs_fixture.py inspect /work/build/repo22-g1/images/fallback-48bit-root2.erofs --path=/root.txt -od -An -tu2 -j 1038 -N 2 /work/build/repo22-g1/images/fallback-48bit-root2.erofs -od -An -tx4 -j 1104 -N 4 /work/build/repo22-g1/images/fallback-48bit-root2.erofs -od -An -tu8 -j 1136 -N 8 /work/build/repo22-g1/images/fallback-48bit-root2.erofs -``` - -Guest after mount: - -```sh -stat -f 'root_ino=%i type=%HT' "$mnt" -cmp /tmp/repo22-g1/source/compact/root.txt "$mnt/root.txt" -sha256 /tmp/repo22-g1/source/compact/root.txt "$mnt/root.txt" -./statfs_probe "$mnt" -df -kT "$mnt" -``` - -## Expected Results - -- Root inode equals `rootnid_2b`; source data matches. -- `statfs` block count equals `blocks_lo` only, with no multi-terabyte size - error. Current exact KLD mount and cleanup pass. -- Historical TC150 evidence cannot substitute for this rerun. diff --git a/tests/TC151-extended-inode-off-max.md b/tests/TC151-extended-inode-off-max.md deleted file mode 100644 index 7d6c12f..0000000 --- a/tests/TC151-extended-inode-off-max.md +++ /dev/null @@ -1,42 +0,0 @@ -# Test Case: Extended Inode Size Above OFF_MAX - -**Test ID**: TC151-extended-inode-off-max - -## Objective - -Rerun rejection of an extended inode with bit 63 set in `i_size` on the current -exact KLD, before open, read, mmap, or pager setup. - -## Fixture - -Use `images/extended-size-bit63.erofs`. Evidence asserts a 64-byte regular -inode, exact NID/offset, `i_size=0x8000000000000000`, valid CRC, and image hash. -Compile `read_probe` and `mmap_fault` natively in FreeBSD. - -## Procedure - -Mount the image, then run every acquisition twice: - -```sh -./read_probe expect-error "$mnt/oversize.dat" 97 -./read_probe expect-error "$mnt/oversize.dat" 97 -set +e -truss -o /tmp/tc151-stat.truss stat "$mnt/oversize.dat"; stat_rc=$? -./mmap_fault "$mnt/oversize.dat" > /tmp/tc151-mmap1 2>&1; mmap1_rc=$? -./mmap_fault "$mnt/oversize.dat" > /tmp/tc151-mmap2 2>&1; mmap2_rc=$? -set -e -printf 'stat_rc=%d mmap_rc=%d,%d\n' "$stat_rc" "$mmap1_rc" "$mmap2_rc" -tail -20 /tmp/tc151-stat.truss -cat /tmp/tc151-mmap1 /tmp/tc151-mmap2 -``` - -Record dmesg delta, remove outputs, unmount, detach, and unload the exact KLD -at the end of G1. - -## Expected Results - -- All direct opens and traced `stat` fail with errno 97 (`EINTEGRITY`). -- Both mmap helpers exit 1 at `open` with the same error; no fd reaches mmap or - pager setup. -- No negative pager size, trap, panic, stale vnode, or cleanup residue occurs. -- Historical TC151 output cannot substitute for this current-KLD rerun. diff --git a/tests/TC152-extent-hole-bounded-read.md b/tests/TC152-extent-hole-bounded-read.md deleted file mode 100644 index c63f719..0000000 --- a/tests/TC152-extent-hole-bounded-read.md +++ /dev/null @@ -1,26 +0,0 @@ -# Test Case: Bounded Read of 5 GiB Extent Hole - -**Test ID**: TC152-extent-hole-bounded-read -**Category**: Compression Extent Mapping -**Priority**: Critical - -## Objective - -Verify a 5 GiB plus one page unmapped compression extent clears only the -requested byte/page, with bounded allocation. - -## Procedure - -Record final/fixture-evidence.txt and the extent-hole-5g.erofs hash. On a fresh -mount, record stat and the erofs vmstat -m row, then run twice: - - /usr/bin/time -l ./sparse_hole_probe \ - /tmp/repo22-g3/mnt/hole.dat 3221225472 - -Record the post-run allocation row, dmesg, and cleanup. - -## Expected Results - -The mounted size is 5368713216. Both one-byte pread calls and real one-page mmap -faults at 3 GiB return zero promptly; allocation does not scale with the 5 GiB -logical hole. No OOM, hang, trap, or panic occurs. diff --git a/tests/TC153-large-directory-block-index.md b/tests/TC153-large-directory-block-index.md deleted file mode 100644 index c57ad6f..0000000 --- a/tests/TC153-large-directory-block-index.md +++ /dev/null @@ -1,43 +0,0 @@ -# Test Case: Large Directory Block Index Width - -**Test ID**: TC153-large-directory-block-index -**Category**: Directory Corruption -**Priority**: Critical -**Latest Result**: PASS on exact 9ae22009f FreeBSD 15 KLD; see the G3 report. - -## Objective - -Verify a valid extended FLAT_PLAIN/Layout0 directory with final block index -2147483648 enters the 64-bit lookup path and consistently reports sparse -midpoint corruption as EINTEGRITY. - -## Procedure - -Record final/tc153-sparse-evidence.txt and the sparse-prefix hash. Copy the -90112-byte prefix, extend the copy sparsely to 8796093091840 bytes, and verify -the prefix hash remains unchanged: - - cp large-dir-intmax-sparse-prefix.erofs TC153-provider.erofs - truncate -s 8796093091840 TC153-provider.erofs - stat -f 'provider_size=%z allocated_sectors=%b' TC153-provider.erofs - unit=$(mdconfig -a -t vnode -f TC153-provider.erofs) - diskinfo "/dev/$unit" - mount -t erofs -o ro "/dev/$unit" /tmp/repo22-g3/mnt - ./g3_vfs_probe stat /tmp/repo22-g3/mnt/huge - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/huge/missing 97 - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/huge/missing 97 - ./readdir_probe /tmp/repo22-g3/mnt 128 - ./g3_vfs_probe expect-error stat \ - /tmp/repo22-g3/mnt/huge/missing 97 - -Unmount, detach, and remove the sparse copy. - -## Expected Results - -GEOM reports 8796093091840 bytes while host allocation remains sparse. -The /huge inode reports size 8796093026304 and Layout0 qualification records -directory_blocks=2147483649, last_block=2147483648. Both cold lookups and the -post-root-readdir lookup return errno 97, with no ENOENT cache masking, wrap, -hang, trap, or panic. Record prefix/KLD hashes and complete cleanup. diff --git a/tests/TC154-nfs-per-inode-generation.md b/tests/TC154-nfs-per-inode-generation.md deleted file mode 100644 index 9d33533..0000000 --- a/tests/TC154-nfs-per-inode-generation.md +++ /dev/null @@ -1,77 +0,0 @@ -# Test Case: NFS Per-Inode Generation on Same-Superblock Replacement - -**Test ID**: TC154-nfs-per-inode-generation -**Category**: NFS Export -**Priority**: Critical -**Regression**: A mount-wide superblock hash aliases a replacement inode at the same NID - -## Objective - -Verify that an EROFS file handle keeps the existing 16-byte FreeBSD FID ABI but -uses a stable per-inode generation. Remounting an unchanged image must preserve -the complete handle. Replacing it with an image whose superblock block, UUID, -NID, and file content are identical but whose raw inode metadata differs must -make the old handle return `ESTALE`. - -Also verify that a valid-NID handle resolving to a malformed replacement inode -returns the positive inode-read errno instead of a negative Linux errno or a -successful alias. - -## Fixture - -Generate all review fixtures on the host: - -```sh -python3 tests/review_fixtures.py make \ - --output /work/build/repo22-review-fixes-fixtures -``` - -For `nfs-inode-a.erofs`, `nfs-inode-b.erofs`, and -`nfs-inode-corrupt.erofs`, require the helper evidence to prove: - -- target `/zz-identity.txt` has the same NID and inode offset; -- the inode lies after the complete superblock checksum block; -- the complete checksum block and declared superblock are byte-identical; -- A and B differ only in the compact inode `i_mtime` field; -- the helper's superblock-seeded raw-inode FNV generations are nonzero and - different; -- all three images retain a valid superblock checksum. - -## Procedure - -1. Compile `tests/nfs_fh_tool.c` natively on FreeBSD 15. -2. Attach image A to an explicit md unit, mount it, capture `a.fh`, and compare - `nfs_fh_tool describe`, `nfs_fh_tool stat`, and `stat -f '%v'`. The FID and - `st_gen` values must equal `generation_a` in `fixture-manifest.json`. -3. Unmount and detach image A, then reattach the unchanged image to the same md - unit. Capture `a-remount.fh`, require a byte-for-byte handle match, and read - through `a.fh`. -4. Replace A with image B on the same md unit. Capture `b.fh`, require the same - fsid and NID but `generation_b`, then run: - - ```sh - nfs_fh_tool expect-stat a.fh ESTALE - nfs_fh_tool expect-open a.fh ESTALE - nfs_fh_tool stat b.fh - ``` - -5. Replace B with `nfs-inode-corrupt.erofs` on the same md unit. Resolving - `a.fh` must return positive FreeBSD `EOPNOTSUPP` (45) from inode decoding: - - ```sh - nfs_fh_tool expect-stat a.fh 45 - nfs_fh_tool expect-open a.fh 45 - ``` - -6. Compare pre/post dmesg, unmount, detach the md unit, and unload the exact - module under test. - -## Expected Results - -- Image A has an identical handle and generation across remounts. -- Image B's same-NID replacement has a different generation and both old-handle - operations return `ESTALE`. -- The new handle resolves successfully and `va_gen == FID.gen`. -- The malformed inode propagates errno 45 as a positive error and never returns - a vnode. -- The FID remains exactly 16 bytes with unchanged field offsets. diff --git a/tests/TC155-explicit-extent-pa-wrap.md b/tests/TC155-explicit-extent-pa-wrap.md deleted file mode 100644 index d7f975e..0000000 --- a/tests/TC155-explicit-extent-pa-wrap.md +++ /dev/null @@ -1,60 +0,0 @@ -# Test Case: 4-Byte Explicit Extent Physical Address Wrap - -**Test ID**: TC155-explicit-extent-pa-wrap -**Category**: Compression Mapping -**Priority**: Critical -**Regression**: Unchecked `pa += plen` can wrap and alias the next extent - -## Objective - -Verify that every address/index increment used while walking 4-byte explicit -extent records is checked and that a `uint64_t` physical-address wrap returns -`EINTEGRITY` before the wrapped address can be used for I/O. - -## Fixture - -Generate `extent-pa-wrap.erofs` with: - -```sh -python3 tests/review_fixtures.py make \ - --output /work/build/repo22-review-fixes-fixtures -``` - -erofs-utils 1.8.6 cannot emit explicit extent records, so the helper performs a -minimal structured conversion of a real extended legacy-compressed inode. It -parses the root directory and target inode, then self-checks these fields: - -- target `/extent.bin` remains an extended `COMPRESSED_FULL` inode; -- map header has `Z_EROFS_ADVISE_EXTENTS` and record size 4; -- the 64-bit initial physical base is `0xfffffffffffff000`; -- the first two `plen` records are 8192 and 4096; -- `initial_pa + first_plen > UINT64_MAX`; -- the second logical cluster starts at offset 4096; -- the transformed image has a valid recomputed superblock CRC32C. - -The second cluster is essential: an unchecked implementation wraps the first -accumulation to 4096 and can present that low address as the second extent. - -## Procedure - -1. Attach and mount the fixture read-only on FreeBSD 15. -2. Confirm `stat /mnt/repo22-review/extent.bin` succeeds, proving vnode creation - and map-header parsing completed. -3. Read one byte from logical offset 4096 and capture the syscall with `truss`: - - ```sh - truss -o extent-wrap.truss \ - dd if=/mnt/repo22-review/extent.bin of=/dev/null bs=1 skip=4096 count=1 - ``` - -4. Require nonzero `dd` status and a read/pread result of `ERR#97` in the trace. - Repeat the probe to exercise vnode-cache reuse. -5. Confirm there is no physical read at wrapped offset 4096, panic, trap, or - assertion in dmesg; then unmount, detach, and unload the module. - -## Expected Results - -- Both reads fail promptly with FreeBSD `EINTEGRITY` (97). -- No wrapped physical address reaches decompression or device I/O. -- The guest remains responsive and cleanup leaves no EROFS mount, md provider, - or EROFS module. diff --git a/tests/TC156-inode-timestamp-validation.md b/tests/TC156-inode-timestamp-validation.md deleted file mode 100644 index 2af0ea5..0000000 --- a/tests/TC156-inode-timestamp-validation.md +++ /dev/null @@ -1,51 +0,0 @@ -# Test Case: Compact and Extended Timestamp Validation - -**Test ID**: TC156-inode-timestamp-validation -**Category**: Inode Corruption -**Priority**: Critical -**Regression**: Timestamp arithmetic and conversion accepted malformed values - -## Objective - -Verify checked compact-inode epoch addition, nanoseconds below one billion for -both inode layouts, and rejection of unsigned seconds that cannot be represented -by FreeBSD 15 amd64 `time_t`. - -## Fixtures - -Generate the review fixtures with `tests/review_fixtures.py`. Its structured -field assertions produce: - -| Image | Malformed field | Trigger | -|---|---|---| -| `compact-epoch-wrap.erofs` | `epoch=UINT64_MAX`, root `i_mtime=1` | `uint64_t` addition overflow | -| `compact-epoch-range.erofs` | `epoch=INT64_MAX`, target `i_mtime=1` | result exceeds signed 64-bit `time_t` | -| `compact-nsec-invalid.erofs` | `fixed_nsec=1000000000` | invalid compact nanoseconds | -| `extended-nsec-invalid.erofs` | target `i_mtime_nsec=1000000000` | invalid extended nanoseconds | -| `extended-seconds-range.erofs` | target `i_mtime=INT64_MAX+1` | seconds exceed `time_t` | - -Every image must pass the helper's EROFS field-location and CRC32C checks before -guest transfer. - -## Procedure - -1. For `compact-epoch-wrap.erofs`, attach the image and trace the mount. The - mount must fail with `ERR#97` when the root inode is decoded. -2. For `compact-nsec-invalid.erofs`, trace the mount. Superblock timestamp - validation must fail with `ERR#97` before a vnode is returned. -3. Mount `compact-epoch-range.erofs`. The root timestamp at `INT64_MAX` remains - representable, but `stat /zz-identity.txt` must fail with `ERR#97`. -4. Mount each extended fixture. The root remains usable, while - `stat /extended-time.txt` must fail with `ERR#97` for the targeted field. -5. Repeat every failing access, compare dmesg, and clean the mount and md unit - after each image. - -## Expected Results - -- Compact epoch addition overflow returns `EINTEGRITY`, not a wrapped time. -- Exactly `999999999` remains the maximum accepted nanosecond value; - `1000000000` is rejected for compact and extended timestamps. -- `INT64_MAX` seconds is accepted on the qualified amd64 ABI, while - `INT64_MAX+1` is rejected before assignment to `timespec.tv_sec`. -- No malformed inode creates a vnode with normalized, negative, or wrapped - timestamps, and no panic or assertion occurs. diff --git a/tests/TC157-explicit-extent-order-validation.md b/tests/TC157-explicit-extent-order-validation.md deleted file mode 100644 index ba0bc55..0000000 --- a/tests/TC157-explicit-extent-order-validation.md +++ /dev/null @@ -1,55 +0,0 @@ -# Test Case: Explicit Extent Global Ordering Validation - -**Test ID**: TC157-explicit-extent-order-validation -**Category**: Compression Mapping -**Priority**: Critical -**Regression**: Binary search over an unvalidated explicit extent table can silently select the wrong mapping - -## Objective - -Verify that every 16-byte and 32-byte explicit extent table is validated once, -before binary search, for globally strict `lstart` ordering and logical bounds. -Descending, duplicate, and cross-search-branch violations must return positive -FreeBSD `EINTEGRITY` without reading the referenced compressed payload. - -## Fixtures - -Generate and self-check all final-review fixtures on the host: - -```sh -python3 tests/final_review_fixtures.py make \ - --output /work/build/repo22-final-review-fixtures -``` - -For each record size, the helper emits `descending`, `duplicate`, and -`cross-branch` images. The manifest records the complete `lstart` list, old -binary-search indices, table offsets, payload offset, and SHA256. The -cross-branch shape `[0, 8192, 4096, 12288]` is queried at 4096; the old search -visits records 2 and 3 but never record 1, so a hit-neighbor-only check is not -sufficient. - -## Procedure - -1. Build and load the exact `WITH_ZSTDIO=0` module on FreeBSD 15. -2. For each of the six images, attach an md provider and mount it read-only. -3. Before any lookup of `/extent.bin`, run: - - ```sh - final_review_probe expect-stat-error /mnt/repo22-final/extent.bin 97 - ``` - -4. Wrap the command with `io:::start`, filtered to that md unit. Require a raw - `dd` positive control to report the manifest payload offset 4096, then require - zero events at offset 4096 during every failing target lookup. -5. Repeat one case to confirm the same positive errno, compare dmesg, and clean - the mount, md unit, EROFS KLD, and DTrace modules. - -## Expected Results - -- All six target lookups return `EINTEGRITY` (97). -- Both record sizes reject descending and duplicate `lstart` values globally. -- Cross-branch corruption is rejected even though the old binary search would - not visit the preceding offending record. -- The payload-offset positive control fires and all six failing operations have - zero payload-offset GEOM events. -- No wrapped, negative, or Linux-style errno is returned. diff --git a/tests/TC158-48bit-compressed-blocks-hi.md b/tests/TC158-48bit-compressed-blocks-hi.md deleted file mode 100644 index d9a31dd..0000000 --- a/tests/TC158-48bit-compressed-blocks-hi.md +++ /dev/null @@ -1,45 +0,0 @@ -# Test Case: 48-Bit Extended Compressed Block Count - -**Test ID**: TC158-48bit-compressed-blocks-hi -**Category**: Inode and VFS Metadata -**Priority**: Critical -**Regression**: Extended compressed inodes ignored `i_nb.blocks_hi` - -## Objective - -Verify Linux-compatible union decoding for an extended compressed inode with -`blocks_hi=1`, and exact FreeBSD `va_bytes`/`st_blocks` values through direct and -file-handle/NFS-style getattr paths. - -## Fixture - -`final_review_fixtures.py` emits `compressed-blocks-hi.erofs` as a small seed. -Its manifest records `blocks_lo`, `blocks_hi`, block size, exact `va_bytes`, -`st_blocks`, and the required sparse provider size. Verify the seed SHA256 before -extending it in the guest. - -## Procedure - -1. Copy the seed in the guest and create the qualified sparse provider: - - ```sh - cp compressed-blocks-hi.erofs compressed-blocks-hi-sparse.erofs - truncate -s 17592186056704 compressed-blocks-hi-sparse.erofs - ``` - -2. Attach and mount it read-only with the exact module. -3. Run `final_review_probe stat-blocks` with the manifest's expected value. The - helper compares `stat` and `getfh`/`fhstat` results. -4. Use an FBT entry/return probe on `erofs_getattr` to capture - `a_vap->va_bytes`; require the manifest value. -5. Read at least one block of `/compressed-blocks.bin`, then unmount, detach, - shrink/remove the sparse file, unload DTrace, and unload EROFS. - -## Expected Results - -- `data_blocks == (1ULL << 32) | blocks_lo`. -- `va_bytes == 17592186052608`. -- Direct `stat` and file-handle `fhstat` both report - `st_blocks == 34359738384`. -- The compressed file remains readable; the high block count is allocation - metadata, not a physical read address. diff --git a/tests/TC159-special-setattr-combinations.md b/tests/TC159-special-setattr-combinations.md deleted file mode 100644 index c78ad57..0000000 --- a/tests/TC159-special-setattr-combinations.md +++ /dev/null @@ -1,44 +0,0 @@ -# Test Case: Special Vnode Combined Setattr Rejection - -**Test ID**: TC159-special-setattr-combinations -**Category**: Read-Only Vnode Operations -**Priority**: Critical -**Regression**: A special-vnode size field caused early success and hid other requested mutations - -## Objective - -Verify that size-only setattr remains an ignored no-op for special vnodes, while -a single `VOP_SETATTR` containing size plus mode, ownership, timestamps, or all -three classes returns `EROFS`. - -## Fixture and Probe - -Use `special-setattr.erofs` and its FIFO `/special.fifo`. Build the kernel-side -probe against the same FreeBSD 15 source tree: - -```sh -mkdir setattr-probe -cp tests/final_review_setattr_probe.c setattr-probe/ -cp tests/final_review_setattr_probe.mk setattr-probe/Makefile -make -C setattr-probe SYSDIR=/path/to/freebsd/sys -``` - -The probe performs one locked vnode lookup and one `VOP_SETATTR` per sysctl -trigger, avoiding userspace syscalls that split attributes across operations. - -## Procedure - -1. Mount the fixture read-only and record mode, uid, gid, atime, and mtime. -2. Load `erofs_setattr_probe.ko` and set - `debug.erofs_setattr_probe.path` to the FIFO. -3. Trigger operation 1 (size only); require result 0. -4. Trigger operations 2 through 5 (size+mode, size+owner, size+times, all); - require result 30 (`EROFS`) after each trigger. -5. Require the metadata snapshot to remain unchanged, then unload the probe, - unmount, detach, and unload EROFS. - -## Expected Results - -- Size-only special-vnode setattr returns 0. -- Every combined mutation returns `EROFS`, never false success. -- No mode, ownership, or timestamp changes are observable. diff --git a/tests/TC160-dot-omitted-offmax-cookie.md b/tests/TC160-dot-omitted-offmax-cookie.md deleted file mode 100644 index fbacc4f..0000000 --- a/tests/TC160-dot-omitted-offmax-cookie.md +++ /dev/null @@ -1,39 +0,0 @@ -# Test Case: Dot-Omitted OFF_MAX Cookie Rejection - -**Test ID**: TC160-dot-omitted-offmax-cookie -**Category**: Directory Corruption -**Priority**: Critical -**Regression**: Synthetic dot at `i_size + 1` overflowed signed directory offsets - -## Objective - -Verify that a `dot_omitted` directory with `i_size == OFF_MAX` is rejected before -the synthetic dot entry can create an unrepresentable cookie or negative offset. - -## Fixture - -`dot-omitted-offmax.erofs` contains an extended directory inode with a valid -backing block, flat-plain layout, the dot-omitted format bit, and -`i_size=9223372036854775807`. The structured helper records every mutated field -and the image hash. - -## Procedure - -1. Mount the fixture and require `stat` on `/offmax-dir` to report `OFF_MAX`. -2. Run: - - ```sh - truss -o tc160.truss \ - final_review_probe readdir-offmax /mnt/repo22-final/offmax-dir 97 - ``` - -3. The helper first calls `getdirentries` at `OFF_MAX`, then at offset 0. Require - `EINTEGRITY` both times and require the descriptor offset to remain exactly - `OFF_MAX` and 0 respectively. -4. Require both syscalls to show `ERR#97`, compare dmesg, and clean all resources. - -## Expected Results - -- No synthetic dot dirent or cookie is returned. -- Both reads fail with positive errno 97. -- No file offset becomes negative or advances after the error. diff --git a/tests/TC161-build-nm-failure.md b/tests/TC161-build-nm-failure.md deleted file mode 100644 index bfe60a1..0000000 --- a/tests/TC161-build-nm-failure.md +++ /dev/null @@ -1,38 +0,0 @@ -# Test Case: Build Undefined-Symbol Tool Failure - -**Test ID**: TC161-build-nm-failure -**Category**: Build Qualification -**Priority**: Critical -**Regression**: `nm -u | awk` could pass when `nm` itself failed - -## Objective - -Verify that `build.sh` treats `nm` failure as fatal independently of the awk -`bcmp` check, safely removes its temporary output, preserves prior module output, -and still builds both supported configurations normally. - -## Procedure - -1. On FreeBSD 15, run normal exact-source builds with `WITH_ZSTDIO=0` and 1. - Require both SUCCESS lines and save each module hash. -2. Create a private executable `nm` shim containing: - - ```sh - #!/bin/sh - exit 73 - ``` - -3. Put only that shim directory before the normal system paths and run - `WITH_ZSTDIO=0 ./build.sh`, capturing stdout/stderr and exit status. -4. Require nonzero status, `ERROR: nm failed while checking erofs.ko`, no SUCCESS - line, unchanged hash for the previously published `build/erofs.ko`, and no - `build/obj/nm-undef.*` file. -5. Run one final normal build to prove the trap and shim did not contaminate the - environment. - -## Expected Results - -- The shimmed build fails after module linkage and before publication. -- Awk cannot convert an `nm` execution failure into PASS. -- Temporary output is removed on failure and normal dual-configuration builds - remain successful. diff --git a/tests/TEST-COVERAGE-MATRIX.md b/tests/TEST-COVERAGE-MATRIX.md deleted file mode 100644 index f79bd26..0000000 --- a/tests/TEST-COVERAGE-MATRIX.md +++ /dev/null @@ -1,304 +0,0 @@ -# Test Coverage Matrix - repo-pre-15 - -## Overview -Test specifications for the implemented FreeBSD EROFS paths. A listed test case -is not proof that every environment has executed it. Historical TC000-TC161 -evidence remains under `tests/results/manual/`; Pre15 bounded runs use immutable -manifests from `tests/pre15/`. - -## Coverage Summary -- **Specification IDs**: TC000-TC161; TC000 is a template -- **Executable results**: 161 total, 160 PASS, TC146 PARTIAL -- **Open kernel failures**: none -- **Approved gap**: positive explicit mapped-payload fixture for TC146 -- **Pre15 test-only addition**: TC162 legacy xattr prefix carrier contract -- **No automated coverage percentage is claimed** - -## Feature-to-Test Mapping - -### 1. Mount Operations (Features 1-2) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| mount / umount | TC001, TC007, TC008 | Normal, concurrent, with-errors | -| statfs | TC009, TC010 | Basic, 48-bit blocks | - -### 2. Root & VNode (Features 3-4) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| root vnode lookup | TC011 | Normal | -| vget | TC012, TC013 | Normal, invalid nid | - -### 3. Superblock (Features 5-7) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| superblock parsing | TC014, TC015 | Normal, corrupted | -| superblock CRC32C verification | TC002, TC016 | Valid, invalid | -| 48-bit block count parsing | TC017, TC018 | Normal, large FS | -| 48-bit root nid parsing | TC019 | Normal | - -### 4. Inode Formats (Features 8-15) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| compact inode decoding | TC020, TC021, TC022 | Basic, flag-based nlink=1, Linux dev decode | -| extended inode decoding | TC023, TC024 | Normal, large file | -| compact inode link-count handling | TC025 | Flagged single link, explicit hard links, directories | -| dot_omitted handling | TC026, TC027 | With/without | -| uncompressed FLAT_PLAIN read | TC028, TC029, TC030 | Small, medium, large | -| uncompressed FLAT_INLINE read | TC031, TC032 | Normal, zero-length | -| inline tailpacking read | TC033, TC034, TC147 | Normal, boundary, metadata-block/image bounds | - -### 5. File Operations (Features 16-18) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| regular file read | TC035, TC036, TC037 | Sequential, random, large | -| symlink read | TC038, TC039, TC040 | Short, long, broken | -| directory entry decoding | TC041, TC042, TC141, TC148 | Normal, large, shared validator, Linux tail padding | - -### 6. Directory Operations (Features 19-22) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| lookup | TC043, TC044, TC045 | Found, not-found, case | -| readdir | TC046, TC047, TC048, TC160 | Basic, large, restart cookies, `OFF_MAX` corruption | -| . handling | TC049 | Dot entry | -| .. handling | TC050 | Dotdot entry | - -### 7. VFS Integration (Features 23-28) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| namecache integration | TC051, TC052 | Hit, miss | -| vfs_hash integration | TC053 | Normal | -| vn_vget_ino integration | TC054 | Normal | -| getattr | TC055, TC056, TC158 | Compressed 48-bit allocation, generation, decoded special rdev | -| access | TC057, TC058 | Allowed, denied | -| readlink | TC059 | Normal | -| pathconf | TC060 | All queries | - -### 8. Read-Only (Features 29) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| read-only setattr rejection | TC061, TC062, TC063, TC159 | chmod, chown, write, combined special-vnode attributes | - -### 9. VM Integration (Features 30-31) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| FreeBSD local vnode pager | TC064, TC065, TC149 | Sync/async ABI, real randomized mmap faults | -| explicit vop_bmap unsupported | TC066, TC149 | Local pager fallback without strategy assumptions | - -### 10. Extended Attributes - Shared (Features 32-36) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| shared user.* xattr scan | TC067, TC068 | Found, not-found | -| shared user.* xattr list | TC069 | Multiple | -| shared trusted.* xattr get | TC070 | Normal | -| shared security.* xattr get | TC071 | Normal | -| shared system xattr list | TC072 | Normal | - -### 11. Extended Attributes - Container (Features 37-40) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| shared-ea-in-metabox container | TC073 | Normal | -| inline user.* xattr | TC005, TC074 | Normal, multiple | -| inline trusted.* xattr | TC075 | Normal | -| inline security.* xattr | TC076 | Normal | - -### 12. Extended Attributes - Advanced (Features 41-46) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| long-prefix user.* xattr | TC077 | Long name | -| long-prefix trusted.* xattr | TC078 | Long name | -| packed prefix table | TC079, TC080 | Normal, lookup | -| legacy prefix primary fallback | TC162 | Checksum-valid legacy/plain/packed/metabox and exact negatives | -| metabox-backed shared xattr | TC081 | Normal | -| system namespace xattr subset | TC082 | Specific attrs | -| user namespace xattr subset | TC083 | Filtering | - -### 13. Compression - LZ4 (Features 47-51) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| LZ4 compression support | TC003, TC084, TC085 | Small, large, corrupt | -| LZ4 compressed data path | TC086, TC087 | Full source compare, deterministic offset compares | -| LZ4 compression config handling | TC088 | Different configs | -| LZ4 compressed pcluster mapping | TC089, TC090 | 4KB, 64KB | -| ztailpacking data path | TC091, TC092 | Normal, edge | - -### 14. Chunk-Based (Features 52-53) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| chunk-based inode data path | TC093, TC094 | Single-dev, multi-dev | -| chunk index read path | TC095 | Normal | - -### 15. Multi-Device (Features 54-57) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| device table support | TC096, TC097 | Parse, invalid | -| fragments support | TC098 | Normal | -| multi-device support | TC006, TC099, TC100 | 2-dev, 4-dev, errors | -| unified address to device mapping | TC101 | All devices | - -### 16. Compression - DEFLATE/zstd (Features 58-59) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| DEFLATE compressed data path | TC102, TC103, TC104 | Level 1 full/range compare, levels 6 and 9 | -| zstd compressed data path | TC105, TC106, TC107 | Level 1, 15, 22 | - -### 17. Compression - LZMA (Feature 60) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| LZMA/MicroLZMA compressed data path | TC004, TC108, TC109, TC110 | Normal, large, micro, corrupt | - -### 18. Documentation (Feature 61) -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| manual pages | TC111 | Accuracy check | - -### 19. Error Handling (Additional Paths) -| Scenario | Test Cases | Coverage | -|---------|-----------|----------| -| Invalid superblock magic | TC112 | Mount failure | -| Corrupted inode | TC113 | Read error | -| Out-of-bounds block | TC114 | Access error | -| Unsupported algorithm | TC115 | EOPNOTSUPP | -| Targeted compressed-stream corruption | TC116 | Equal provider length, proven extent, EIO | -| Invalid xattr format | TC117 | Parse error | -| Device not found | TC118 | Multi-dev error | -| Uncompressed payload integrity boundary | TC119 | No per-file CRC; source hash detects mutation | - -### 20. Boundary & Stress (Additional Paths) -| Scenario | Test Cases | Coverage | -|---------|-----------|----------| -| Empty file read | TC120 | Zero-length | -| Maximum file size | TC121 | 16TB limit | -| Deep directory tree | TC122 | 100+ levels | -| Long filename | TC123 | 255 chars | -| Many small files | TC124 | 10000+ files | -| Large directory | TC125 | 10000+ entries | -| Concurrent reads | TC126 | Multi-threaded | -| Memory pressure | TC127 | Low memory | -| Sequential throughput | TC128 | Performance | -| Random read pattern | TC129 | Performance | -| Mixed workload | TC130 | Realistic | - -## Execution Path Coverage - -### Normal Paths (Green) -The matrix maps implemented paths to manual procedures. See dated reports for -the subset actually executed on a given FreeBSD 15 environment. - -### Error Paths (Red) -- Mount errors: TC007, TC008, TC112 -- Read errors: TC113, TC114, TC116 -- Lookup errors: TC044, TC117 -- Permission errors: TC061-TC063 -- Device errors: TC097, TC118 -- Compression errors: TC085, TC104, TC110, TC116 -- CRC errors: TC002, TC016 -- End-to-end payload authenticity boundary: TC119 - -### Boundary Conditions (Yellow) -- Zero-length: TC032, TC120 -- Maximum size: TC018, TC121 -- Deep nesting: TC122 -- Long names: TC077, TC078, TC123 -- Large collections: TC042, TC048, TC124, TC125 -- Edge alignments: TC034, TC090, TC092 -- Inline metadata block: TC147 -- Directory tail padding: TC148 -- VM EOF/private mapping: TC149 -- 48-bit fallback superblock union: TC150 -- Extended inode `OFF_MAX` rejection: TC151 -- Multi-GiB compressed extent holes: TC152 -- Directory block indexes above `INT_MAX`: TC153 -- Explicit extent global order: TC157 -- Extended compressed allocation high bits: TC158 -- Combined special-vnode setattr: TC159 -- Dot-omitted `OFF_MAX` cookie: TC160 -- Build-tool failure propagation: TC161 - -## Test Execution Order -1. Basic mount (TC001-TC002) -2. Core functionality (TC003-TC060) -3. Extended features (TC061-TC111) -4. Error handling (TC112-TC119) -5. Stress and NFS tests (TC120-TC133) -6. Metadata/compression completion (TC134-TC149) -7. Final correctness findings and build qualification (TC150-TC161) - -## Coverage Metrics - -No line, branch, path, or feature percentage is asserted. The repository does -not contain an instrumented kernel coverage run or CI harness. - -The historical TC000-TC161 suite remains procedure-driven. -`tests/pre15/run-host.sh`, `run-build.sh`, `run-qemu.sh`, and `run-smoke.sh` -provide the bounded Pre15 adapter. They discover exact case files, preserve raw -stdout/stderr and hashes, separate DUT/runner/infrastructure failures, and make -cleanup success part of PASS. This does not claim full-suite automation. - -## Test Case Status - -- [x] Specifications TC000-TC161 are present exactly once. -- [x] Execution checklist and result templates are present. -- [x] Dated manual reports record qualified runs and real limitations. -- [x] TC150-TC152 have final-module FreeBSD 15 evidence. -- [x] TC153 has exact-source sparse-provider FreeBSD 15 evidence. -- [x] TC154-TC156 have final-module FreeBSD 15 evidence. -- [x] TC157-TC161 have independent final-source FreeBSD 15 evidence. -- [x] Final result is 160 PASS and TC146 PARTIAL. -- [ ] Automated kernel regression/coverage infrastructure is not implemented. -- [x] Pre15 B01 runner controls and TC162 host contract are implemented. - -### 21. Compression P0 Completion -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| fragment-backed compressed metabox | TC142 | Positive, loop, recursive NID, bounds | -| DEFLATE/ZSTD partial reference | TC143 | Full, partial, random, corrupt | -| MicroLZMA completion semantics | TC144 | Input consumption, partial, corrupt | -| FreeBSD ZSTDIO build gate | TC145 | Disabled/enabled build and KLD | -| HEAD2/interlaced/extent mapping | TC146 | HEAD2/interlaced PASS; explicit mapped payload PARTIAL | - -### 22. Metadata and VFS Semantics -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| FLAT_INLINE metadata-block and declared bounds | TC147 | Positive and checksum-valid cross-block corruption | -| Linux-compatible directory tail padding | TC141, TC148 | Cold lookup, readdir, cookies, strict corruptions | -| Real FreeBSD vnode pager faults | TC064-TC066, TC149 | Plain/LZ4, random faults, EOF, SIGBUS, private COW | -| Linux special-device decode | TC022, TC056 | Compact/extended char, block, FIFO `st_rdev` | -| Real compressed allocation accounting | TC055, TC158 | Algorithms/shapes and extended 48-bit block count | -| Stable per-inode NFS generation | TC132, TC154 | Stable metadata; changed inode metadata makes old handles stale | -| NFS background exit and cleanup | TC133 | Per-PID waits, fd inheritance guard, client-first cleanup | - -### 23. Final Correctness Findings -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| Linux-compatible 48-bit superblock union fallback | TC017, TC019, TC150 | `rootnid_8b == 0` selects `rootnid_2b` and low blocks | -| Extended inode signed-size boundary | TC151 | Bit-63 `i_size` fails before pager setup | -| Bounded compressed extent holes | TC152 | 5 GiB hole, one-byte pread and one-page mmap | -| 64-bit directory block search | TC153 | Multi-TiB Layout 0 provider returned repeated `EINTEGRITY` | - -### 24. Review Fixes -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| Per-inode NFS replacement detection | TC154 | Same superblock, UUID, NID and content; changed raw inode | -| Explicit extent physical-address checked add | TC155 | Four-byte records with `pa + plen` overflow | -| Inode timestamp range validation | TC156 | Compact epoch overflow, signed `time_t`, compact/extended nanoseconds | - -### 25. Independent Final Review -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| Explicit table global ordering | TC157 | 16/32-byte descending, duplicate, cross-branch rejection | -| Extended compressed `blocks_hi` | TC158 | Direct/file-handle allocation and FBT `va_bytes` | -| Special-vnode setattr combinations | TC159 | Size-only no-op and combined mutation rejection | -| Dot-omitted `OFF_MAX` cookie | TC160 | Both initial offsets return `EINTEGRITY` without advancement | -| Fatal `nm` failure | TC161 | Publication preservation, temp cleanup, post-shim rebuild | - -### 26. Pre15 Test Foundation -| Feature | Test Cases | Coverage | -|---------|-----------|----------| -| G3 stable fixture migration | B01-g3-equivalence | Archived hash, canonical inventory, byte hash, expectation, repeat generation | -| Runner failure separation | B01-runner-selftest | Good, DUT mismatch, pre-target failure, SSH, QEMU exit, timeout, cleanup | -| Legacy xattr prefix fallback | TC162 | Primary legacy, explicit plain, packed, metabox, exact `EINTEGRITY` negatives | - -Latest final-review results are recorded in -`tests/results/manual/2026-08-09T1804Z-final-review-independent/manual-test-report.md`. -TC010, TC060, and TC153 are resolved historical issues. The only approved -remaining gap is TC146's positive explicit mapped-payload fixture. diff --git a/tests/erofs_fixture.py b/tests/erofs_fixture.py deleted file mode 100755 index c632885..0000000 --- a/tests/erofs_fixture.py +++ /dev/null @@ -1,764 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect and make assertion-driven EROFS manual-test fixtures.""" - -from __future__ import annotations - -import argparse -import hashlib -import shutil -import struct -from dataclasses import dataclass -from pathlib import Path - - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -FEATURE_COMPAT_SB_CHKSUM = 0x00000001 -FEATURE_INCOMPAT_COMPR_CFGS = 0x00000002 -FEATURE_INCOMPAT_48BIT = 0x00000080 -CRC32C_POLYNOMIAL = 0x82F63B78 -CRC32C_INITIAL = 0xFFFFFFFF -KERNEL_CRC32C_SEED = 0x5045B54A - - -def crc32c(data: bytes | bytearray, initial: int = CRC32C_INITIAL) -> int: - checksum = initial - for byte in data: - checksum ^= byte - for _ in range(8): - checksum = (checksum >> 1) ^ ( - CRC32C_POLYNOMIAL if checksum & 1 else 0 - ) - return checksum & 0xFFFFFFFF - - -@dataclass(frozen=True) -class Inode: - nid: int - offset: int - inode_format: int - inode_size: int - xattr_size: int - layout: int - mode: int - size: int - start_block_low: int - start_block_high: int - start_block: int - - -@dataclass(frozen=True) -class DirectoryEntry: - nid: int - offset: int - name_offset: int - end_offset: int - name: bytes - - -class ErofsImage: - def __init__(self, data: bytearray, source: Path): - self.data = data - self.source = source - if len(data) < SUPER + 128: - raise ValueError(f"{source}: shorter than the EROFS superblock") - - @classmethod - def load(cls, path: Path) -> "ErofsImage": - return cls(bytearray(path.read_bytes()), path) - - def clone(self) -> "ErofsImage": - return ErofsImage(bytearray(self.data), self.source) - - def u16(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - struct.pack_into(" None: - struct.pack_into(" None: - struct.pack_into(" int: - return self.data[SUPER + 12] - - @property - def block_size(self) -> int: - if not 9 <= self.block_bits <= 16: - raise ValueError(f"invalid block bits {self.block_bits}") - return 1 << self.block_bits - - @property - def checksum_end(self) -> int: - span = self.block_size - if span > SUPER: - span -= SUPER - end = SUPER + span - if end > len(self.data): - raise ValueError( - f"checksum window ends at {end}, image size is {len(self.data)}" - ) - return end - - @property - def feature_compat(self) -> int: - return self.u32(SUPER + 8) - - @property - def feature_incompat(self) -> int: - return self.u32(SUPER + 80) - - @property - def blocks(self) -> int: - blocks = self.u32(SUPER + 36) - root8 = self.u64(SUPER + 112) - if self.feature_incompat & FEATURE_INCOMPAT_48BIT and root8 != 0: - blocks |= self.u16(SUPER + 14) << 32 - return blocks - - @property - def root_nid(self) -> int: - root8 = self.u64(SUPER + 112) - if self.feature_incompat & FEATURE_INCOMPAT_48BIT and root8 != 0: - return root8 - return self.u16(SUPER + 14) - - def calculated_checksum(self) -> int: - window = bytearray(self.data[SUPER : self.checksum_end]) - struct.pack_into(" int: - return crc32c( - self.data[SUPER + 8 : self.checksum_end], KERNEL_CRC32C_SEED - ) - - def checksum_valid(self) -> bool: - if not self.feature_compat & FEATURE_COMPAT_SB_CHKSUM: - return True - return self.u32(SUPER + 4) == self.calculated_checksum() - - def kernel_checksum_valid(self) -> bool: - if not self.feature_compat & FEATURE_COMPAT_SB_CHKSUM: - return True - return self.u32(SUPER + 4) == self.kernel_calculated_checksum() - - def update_checksum(self) -> None: - if not self.feature_compat & FEATURE_COMPAT_SB_CHKSUM: - raise ValueError("image does not advertise superblock checksum") - self.put_u32(SUPER + 4, 0) - self.put_u32(SUPER + 4, self.calculated_checksum()) - if not self.checksum_valid(): - raise AssertionError("updated checksum does not verify") - if ( - self.u32(SUPER) == MAGIC - and self.calculated_checksum() != self.kernel_calculated_checksum() - ): - raise AssertionError("canonical and kernel checksum forms differ") - - def validate_superblock(self) -> None: - if self.u32(SUPER) != MAGIC: - raise ValueError(f"unexpected magic {self.u32(SUPER):#010x}") - if self.blocks == 0: - raise ValueError("declared block count is zero") - if not self.checksum_valid(): - raise ValueError("superblock checksum is invalid") - - def inode(self, nid: int) -> Inode: - metadata = self.u32(SUPER + 40) << self.block_bits - offset = metadata + (nid << 5) - if offset > len(self.data) - 32: - raise ValueError(f"nid {nid} maps outside the image at {offset}") - inode_format = self.u16(offset) - inode_size = 64 if inode_format & 1 else 32 - if offset > len(self.data) - inode_size: - raise ValueError(f"nid {nid} extended inode is truncated") - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) - size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8) - start_block_low = self.u32(offset + 16) - start_block_high = ( - self.u16(offset + 6) - if inode_size == 64 - and self.feature_incompat & FEATURE_INCOMPAT_48BIT - else 0 - ) - return Inode( - nid=nid, - offset=offset, - inode_format=inode_format, - inode_size=inode_size, - xattr_size=xattr_size, - layout=(inode_format >> 1) & 7, - mode=self.u16(offset + 4), - size=size, - start_block_low=start_block_low, - start_block_high=start_block_high, - start_block=start_block_low | (start_block_high << 32), - ) - - def directory_entries(self, inode: Inode) -> list[DirectoryEntry]: - if inode.size == 0 or inode.size > self.block_size: - raise ValueError("helper resolves only one-block directories") - if inode.layout == 2: - data_offset = inode.offset + inode.inode_size + inode.xattr_size - elif inode.layout == 0: - data_offset = inode.start_block << self.block_bits - else: - raise ValueError(f"unsupported directory layout {inode.layout}") - if data_offset > len(self.data) - inode.size: - raise ValueError("directory data lies outside the image") - first_name_offset = self.u16(data_offset + 8) - if ( - first_name_offset < 12 - or first_name_offset % 12 != 0 - or first_name_offset >= inode.size - ): - raise ValueError("invalid first directory name offset") - count = first_name_offset // 12 - entries = [] - previous = 0 - for index in range(count): - entry_offset = data_offset + index * 12 - name_offset = self.u16(entry_offset + 8) - end_offset = ( - self.u16(entry_offset + 20) - if index + 1 < count - else inode.size - ) - if ( - name_offset < first_name_offset - or name_offset <= previous - or end_offset <= name_offset - or end_offset > inode.size - ): - raise ValueError("invalid directory name offsets") - slot = bytes( - self.data[ - data_offset + name_offset : data_offset + end_offset - ] - ) - name = slot.split(b"\0", 1)[0] - if not name: - raise ValueError("empty directory name") - entries.append( - DirectoryEntry( - nid=self.u64(entry_offset), - offset=entry_offset, - name_offset=name_offset, - end_offset=end_offset, - name=name, - ) - ) - previous = name_offset - return entries - - def resolve_root_entry(self, path: str) -> tuple[Inode, DirectoryEntry]: - name = path.removeprefix("/").encode("ascii") - if not name or b"/" in name: - raise ValueError("helper accepts one root-level path component") - root = self.inode(self.root_nid) - for entry in self.directory_entries(root): - if entry.name == name: - return root, entry - raise ValueError(f"path not found in root directory: {path}") - - def save(self, path: Path) -> None: - path.write_bytes(self.data) - - -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def write_checksums(output: Path) -> None: - paths = sorted(output.glob("*.erofs")) - with (output / "SHA256SUMS").open("w", encoding="ascii") as sums: - for path in paths: - sums.write(f"{sha256(path)} {path.name}\n") - - -def save_checked(image: ErofsImage, path: Path) -> None: - image.update_checksum() - image.validate_superblock() - image.save(path) - - -def assert_same_length(original: ErofsImage, changed: ErofsImage) -> None: - if len(original.data) != len(changed.data): - raise AssertionError("fixture mutation changed provider length") - - -def make_error_fixtures(args: argparse.Namespace) -> None: - output = args.output - if output.exists(): - raise ValueError(f"output already exists: {output}") - output.mkdir(parents=True) - plain = ErofsImage.load(args.plain) - compressed = ErofsImage.load(args.compressed) - deflate = ErofsImage.load(args.deflate) - plain.validate_superblock() - compressed.validate_superblock() - deflate.validate_superblock() - shutil.copy2(args.plain, output / "valid-plain.erofs") - shutil.copy2(args.compressed, output / "valid-lz4.erofs") - shutil.copy2(args.deflate, output / "valid-deflate-level1.erofs") - evidence = [] - - bad_crc = plain.clone() - patch_offset = SUPER + 64 - old_byte = bad_crc.data[patch_offset] - bad_crc.data[patch_offset] ^= 0x01 - assert_same_length(plain, bad_crc) - if bad_crc.checksum_valid(): - raise AssertionError("bad checksum fixture still verifies") - bad_crc.save(output / "bad-super-crc.erofs") - evidence.append( - f"bad-super-crc patch_offset={patch_offset} checksum_offset={SUPER + 4} " - f"old={old_byte:#04x} new={bad_crc.data[patch_offset]:#04x} " - f"coverage={SUPER}:{plain.checksum_end} recomputed=no " - f"provider_length={len(plain.data)}" - ) - - bad_checksum_field = plain.clone() - old_checksum = bad_checksum_field.u32(SUPER + 4) - bad_checksum_field.put_u32(SUPER + 4, old_checksum ^ 0x00000001) - assert_same_length(plain, bad_checksum_field) - if bad_checksum_field.checksum_valid(): - raise AssertionError("altered checksum field still verifies") - bad_checksum_field.save(output / "bad-checksum-field.erofs") - evidence.append( - f"bad-checksum-field field_offset={SUPER + 4} " - f"old={old_checksum:#010x} new={old_checksum ^ 1:#010x} " - f"coverage={SUPER}:{plain.checksum_end} recomputed=no " - f"provider_length={len(plain.data)}" - ) - - bad_magic = plain.clone() - if bad_magic.u32(SUPER) != MAGIC: - raise AssertionError("unexpected source magic") - bad_magic.put_u32(SUPER, 0x21444142) - assert_same_length(plain, bad_magic) - if bad_magic.checksum_valid() or not bad_magic.kernel_checksum_valid(): - raise AssertionError("bad magic checksum qualification failed") - bad_magic.save(output / "bad-magic.erofs") - evidence.append( - f"bad-magic patch_offset={SUPER} old={MAGIC:#010x} new=0x21444142 " - f"crc=unchanged canonical_valid=no kernel_suffix_valid=yes " - f"provider_length={len(plain.data)}" - ) - - bad_block_size = plain.clone() - old_block_bits = bad_block_size.block_bits - bad_block_size.data[SUPER + 12] = 13 - bad_block_size.update_checksum() - assert_same_length(plain, bad_block_size) - bad_block_size.save(output / "bad-block-size.erofs") - evidence.append( - f"bad-block-size field_offset={SUPER + 12} old={old_block_bits} new=13 " - f"coverage={SUPER}:{bad_block_size.checksum_end} crc=recomputed " - f"provider_length={len(plain.data)}" - ) - - bad_root = plain.clone() - if bad_root.feature_incompat & FEATURE_INCOMPAT_48BIT: - raise AssertionError("plain source unexpectedly has 48-bit feature") - metadata_offset = bad_root.u32(SUPER + 40) << bad_root.block_bits - invalid_root = ( - ((bad_root.blocks << bad_root.block_bits) - metadata_offset) // 32 - + 1024 - ) - bad_root.put_u32( - SUPER + 80, bad_root.feature_incompat | FEATURE_INCOMPAT_48BIT - ) - bad_root.put_u16(SUPER + 14, 0) - bad_root.put_u64(SUPER + 112, invalid_root) - save_checked(bad_root, output / "bad-root-nid.erofs") - assert_same_length(plain, bad_root) - evidence.append( - f"bad-root-nid feature_offset={SUPER + 80} " - f"blocks_hi_offset={SUPER + 14} rootnid_8b_offset={SUPER + 112} " - f"old_root={plain.root_nid} new_root={invalid_root} " - f"crc=recomputed provider_length={len(plain.data)}" - ) - - unsupported_feature = plain.clone() - old_incompat = unsupported_feature.feature_incompat - unsupported_feature.put_u32(SUPER + 80, old_incompat | 0x80000000) - save_checked(unsupported_feature, output / "unsupported-feature.erofs") - assert_same_length(plain, unsupported_feature) - evidence.append( - f"unsupported-feature field_offset={SUPER + 80} " - f"old={old_incompat:#010x} new={old_incompat | 0x80000000:#010x} " - f"crc=recomputed provider_length={len(plain.data)}" - ) - - short_path = output / "truncated-before-super.erofs" - short_path.write_bytes(plain.data[: SUPER - 1]) - evidence.append( - f"truncated-before-super source_length={len(plain.data)} " - f"provider_length={SUPER - 1} required_super_offset={SUPER}" - ) - empty_path = output / "empty-provider.erofs" - empty_path.write_bytes(b"") - evidence.append( - f"empty-provider source_length={len(plain.data)} provider_length=0" - ) - - bad_dirent = plain.clone() - _, entry = bad_dirent.resolve_root_entry("/bad-entry.txt") - invalid_nid = (bad_dirent.blocks << bad_dirent.block_bits) // 32 + 1024 - bad_dirent.put_u64(entry.offset, invalid_nid) - save_checked(bad_dirent, output / "bad-dirent-nid.erofs") - assert_same_length(plain, bad_dirent) - evidence.append( - f"bad-dirent-nid dirent_offset={entry.offset} old_nid={entry.nid} " - f"new_nid={invalid_nid} crc=recomputed " - f"provider_length={len(plain.data)}" - ) - - bad_inode = plain.clone() - _, entry = bad_inode.resolve_root_entry("/inode-target.txt") - inode = bad_inode.inode(entry.nid) - if inode.inode_format & 0x8000: - raise AssertionError("reserved i_format bit already set") - bad_inode.put_u16(inode.offset, inode.inode_format | 0x8000) - save_checked(bad_inode, output / "bad-inode-format.erofs") - assert_same_length(plain, bad_inode) - evidence.append( - f"bad-inode-format nid={inode.nid} inode_offset={inode.offset} " - f"i_format={inode.inode_format:#06x}->{inode.inode_format | 0x8000:#06x} " - f"crc=recomputed provider_length={len(plain.data)}" - ) - - out_of_bounds = plain.clone() - _, entry = out_of_bounds.resolve_root_entry("/plain.bin") - inode = out_of_bounds.inode(entry.nid) - if inode.layout != 0 or inode.size == 0: - raise AssertionError("plain.bin is not nonempty FLAT_PLAIN") - out_of_bounds.put_u32(inode.offset + 16, out_of_bounds.blocks) - save_checked(out_of_bounds, output / "oob-start-block.erofs") - assert_same_length(plain, out_of_bounds) - evidence.append( - f"oob-start-block nid={inode.nid} field_offset={inode.offset + 16} " - f"old={inode.start_block} new={out_of_bounds.blocks} crc=recomputed " - f"provider_length={len(plain.data)}" - ) - - future = compressed.clone() - if not future.feature_incompat & FEATURE_INCOMPAT_COMPR_CFGS: - raise AssertionError("compressed image has no compression config feature") - available_offset = SUPER + 82 - old_algorithms = future.u16(available_offset) - if old_algorithms & 0x8000: - raise AssertionError("future algorithm bit already set") - future.put_u16(available_offset, old_algorithms | 0x8000) - save_checked(future, output / "future-algorithm.erofs") - assert_same_length(compressed, future) - evidence.append( - f"future-algorithm field_offset={available_offset} " - f"old={old_algorithms:#06x} new={old_algorithms | 0x8000:#06x} " - f"crc=recomputed provider_length={len(compressed.data)}" - ) - - compressed_corrupt = compressed.clone() - corrupt_offset = args.compressed_offset + 32 - corrupt_length = 64 - if args.compressed_length < 32 + corrupt_length: - raise ValueError("compressed extent is too short for targeted mutation") - if corrupt_offset + corrupt_length > ( - args.compressed_offset + args.compressed_length - ): - raise ValueError("compressed mutation exceeds selected extent") - if corrupt_offset < compressed_corrupt.checksum_end: - raise ValueError("compressed corruption overlaps checksum window") - if corrupt_offset + corrupt_length > len(compressed_corrupt.data): - raise ValueError("compressed corruption exceeds provider") - old_bytes = bytes( - compressed_corrupt.data[corrupt_offset : corrupt_offset + corrupt_length] - ) - if old_bytes == bytes(corrupt_length): - raise ValueError("compressed mutation would not change bytes") - compressed_corrupt.data[ - corrupt_offset : corrupt_offset + corrupt_length - ] = bytes(corrupt_length) - if not compressed_corrupt.checksum_valid(): - raise AssertionError("payload mutation changed superblock checksum") - assert_same_length(compressed, compressed_corrupt) - compressed_corrupt.save(output / "compressed-stream-corrupt.erofs") - evidence.append( - f"compressed-stream-corrupt extent_offset={args.compressed_offset} " - f"extent_length={args.compressed_length} patch_offset={corrupt_offset} " - f"patch_length={corrupt_length} provider_length={len(compressed.data)} " - "crc=unchanged-valid" - ) - - raw_corrupt = plain.clone() - _, entry = raw_corrupt.resolve_root_entry("/plain.bin") - inode = raw_corrupt.inode(entry.nid) - if inode.layout != 0 or inode.size <= 257: - raise ValueError("plain.bin is not a qualifying FLAT_PLAIN target") - raw_offset = (inode.start_block << raw_corrupt.block_bits) + 257 - if raw_offset < raw_corrupt.checksum_end or raw_offset >= len(raw_corrupt.data): - raise ValueError("raw payload patch offset is invalid") - old_byte = raw_corrupt.data[raw_offset] - raw_corrupt.data[raw_offset] ^= 0x80 - assert_same_length(plain, raw_corrupt) - if not raw_corrupt.checksum_valid(): - raise AssertionError("raw payload mutation changed superblock checksum") - raw_corrupt.save(output / "raw-data-corrupt.erofs") - evidence.append( - f"raw-data-corrupt path=/plain.bin nid={inode.nid} " - f"start_block={inode.start_block} file_offset=257 " - f"patch_offset={raw_offset} old={old_byte:#04x} " - f"new={raw_corrupt.data[raw_offset]:#04x} crc=unchanged-valid " - f"provider_length={len(plain.data)}" - ) - - if plain.feature_incompat & FEATURE_INCOMPAT_48BIT: - raise AssertionError("plain source unexpectedly has 48-bit feature") - if plain.root_nid == 0: - raise AssertionError("48-bit fixture needs a nonzero root nid") - blocks_lo = plain.u32(SUPER + 36) - blocks_hi = 1 - total_blocks = blocks_lo | (blocks_hi << 32) - provider_length = total_blocks << plain.block_bits - - large = plain.clone() - large.put_u32(SUPER + 80, large.feature_incompat | FEATURE_INCOMPAT_48BIT) - large.put_u16(SUPER + 14, blocks_hi) - large.put_u64(SUPER + 112, plain.root_nid) - save_checked(large, output / "48bit-statfs-prefix.erofs") - assert_same_length(plain, large) - evidence.append( - f"48bit-statfs-prefix feature_offset={SUPER + 80} " - f"blocks_lo_offset={SUPER + 36} blocks_lo={blocks_lo} " - f"blocks_hi_offset={SUPER + 14} blocks_hi={blocks_hi} " - f"rootnid_8b_offset={SUPER + 112} rootnid_8b={plain.root_nid} " - f"total_blocks={total_blocks} required_provider_length={provider_length} " - f"prefix_length={len(plain.data)} crc=recomputed" - ) - - high = large.clone() - _, high_entry = high.resolve_root_entry("/high-offset.txt") - high_inode = high.inode(high_entry.nid) - if high_inode.inode_size != 64 or high_inode.layout != 0 or high_inode.size == 0: - raise ValueError("high-offset.txt is not a nonempty extended FLAT_PLAIN file") - if high_inode.start_block_high != 0: - raise ValueError("high-offset.txt already has a high start block") - low_data_offset = high_inode.start_block_low << high.block_bits - low_data = bytes( - high.data[low_data_offset : low_data_offset + high_inode.size] - ) - if low_data == bytes(high_inode.size): - raise ValueError("high-offset.txt low payload is already zero") - high.data[low_data_offset : low_data_offset + high_inode.size] = bytes( - high_inode.size - ) - high.put_u16(high_inode.offset + 6, 1) - save_checked(high, output / "48bit-high-file-prefix.erofs") - assert_same_length(plain, high) - high_inode = high.inode(high_entry.nid) - high_data_offset = high_inode.start_block << high.block_bits - if high_data_offset + high_inode.size > provider_length: - raise ValueError("high-offset target exceeds declared provider") - evidence.append( - f"48bit-high-file-prefix path=/high-offset.txt nid={high_inode.nid} " - f"inode_offset={high_inode.offset} startblk_hi_offset={high_inode.offset + 6} " - f"startblk_lo={high_inode.start_block_low} startblk_hi=1 " - f"start_block={high_inode.start_block} data_offset={high_data_offset} " - f"low_decoy_offset={low_data_offset} low_decoy=zero " - f"file_size={high_inode.size} required_provider_length={provider_length} " - f"prefix_length={len(plain.data)} crc=recomputed" - ) - - (output / "fixture-evidence.txt").write_text( - "\n".join(evidence) + "\n", encoding="ascii" - ) - write_checksums(output) - - -def validate_full_directory_block( - image: ErofsImage, offset: int -) -> list[DirectoryEntry]: - first_name_offset = image.u16(offset + 8) - if first_name_offset < 12 or first_name_offset % 12 != 0: - raise ValueError("invalid first name offset in full directory block") - count = first_name_offset // 12 - entries = [] - previous = 0 - for index in range(count): - entry_offset = offset + index * 12 - name_offset = image.u16(entry_offset + 8) - end_offset = ( - image.u16(entry_offset + 20) - if index + 1 < count - else image.block_size - ) - if ( - name_offset < first_name_offset - or name_offset <= previous - or end_offset <= name_offset - or end_offset > image.block_size - ): - raise ValueError("invalid full-block directory offsets") - name = bytes( - image.data[offset + name_offset : offset + end_offset] - ).split(b"\0", 1)[0] - if not name: - raise ValueError("empty full-block directory name") - entries.append( - DirectoryEntry( - nid=image.u64(entry_offset), - offset=entry_offset, - name_offset=name_offset, - end_offset=end_offset, - name=name, - ) - ) - previous = name_offset - return entries - - -def make_directory_fixtures(args: argparse.Namespace) -> None: - output = args.output - if output.exists(): - raise ValueError(f"output already exists: {output}") - output.mkdir(parents=True) - base = ErofsImage.load(args.base) - base.validate_superblock() - shutil.copy2(args.base, output / "namei-base.erofs") - _, wide_entry = base.resolve_root_entry("/wide") - wide = base.inode(wide_entry.nid) - if wide.layout != 2 or wide.size <= base.block_size: - raise ValueError("wide must be a multi-block FLAT_INLINE directory") - wide_block = wide.start_block << base.block_bits - entries = validate_full_directory_block(base, wide_block) - last = entries[-1] - slot_start = wide_block + last.name_offset - slot_end = wide_block + last.end_offset - padding_nul = base.data.index(0, slot_start, slot_end) - if padding_nul + 9 > wide_block + base.block_size: - raise ValueError("not enough final-name padding to patch") - evidence = [ - f"wide_nid={wide.nid} inode_offset={wide.offset} block_offset={wide_block} " - f"dirents={len(entries)} final_name={last.name.decode('ascii')} " - f"padding_patch={padding_nul + 1}:{padding_nul + 9}" - ] - - nonzero = base.clone() - nonzero.data[padding_nul + 1 : padding_nul + 9] = b"PAD!ERO!" - save_checked(nonzero, output / "namei-padding-nonzero.erofs") - - short = base.clone() - root = short.inode(short.root_nid) - if root.inode_size != 32: - raise ValueError("expected compact root inode") - short.put_u32(root.offset + 8, 8) - save_checked(short, output / "namei-corrupt-short.erofs") - evidence.append(f"short-root inode_offset={root.offset} size=8") - - bad_offset = base.clone() - root = bad_offset.inode(bad_offset.root_nid) - root_entries = bad_offset.directory_entries(root) - if len(root_entries) < 2: - raise ValueError("root needs at least two entries") - bad_offset.put_u16( - root_entries[1].offset + 8, root_entries[0].name_offset - ) - save_checked(bad_offset, output / "namei-corrupt-nameoff.erofs") - evidence.append( - f"bad-nameoff field_offset={root_entries[1].offset + 8} " - f"new={root_entries[0].name_offset}" - ) - - bad_name = base.clone() - slash_offset = slot_start + 5 - if bad_name.data[slash_offset] in (0, ord("/")): - raise ValueError("chosen name byte cannot be patched") - bad_name.data[slash_offset] = ord("/") - save_checked(bad_name, output / "namei-corrupt-name.erofs") - evidence.append(f"bad-name patch_offset={slash_offset} new=0x2f") - - (output / "fixture-evidence.txt").write_text( - "\n".join(evidence) + "\n", encoding="ascii" - ) - write_checksums(output) - - -def inspect_image(args: argparse.Namespace) -> None: - image = ErofsImage.load(args.image) - print(f"image={args.image}") - print(f"provider_bytes={len(image.data)}") - print(f"magic={image.u32(SUPER):#010x} offset={SUPER}") - print(f"block_size={image.block_size} block_bits={image.block_bits}") - print(f"blocks={image.blocks} blocks_lo_offset={SUPER + 36}") - print(f"root_nid={image.root_nid}") - print(f"feature_compat={image.feature_compat:#010x} offset={SUPER + 8}") - print(f"feature_incompat={image.feature_incompat:#010x} offset={SUPER + 80}") - print( - f"checksum={image.u32(SUPER + 4):#010x} offset={SUPER + 4} " - f"calculated={image.calculated_checksum():#010x} " - f"coverage={SUPER}:{image.checksum_end} valid={image.checksum_valid()}" - ) - print( - f"kernel_calculated={image.kernel_calculated_checksum():#010x} " - f"kernel_seed={KERNEL_CRC32C_SEED:#010x} " - f"kernel_coverage={SUPER + 8}:{image.checksum_end} " - f"kernel_valid={image.kernel_checksum_valid()} " - f"equivalent={image.calculated_checksum() == image.kernel_calculated_checksum()}" - ) - if args.path: - _, entry = image.resolve_root_entry(args.path) - inode = image.inode(entry.nid) - print( - f"path={args.path} nid={inode.nid} inode_offset={inode.offset} " - f"i_format={inode.inode_format:#06x} layout={inode.layout} " - f"size={inode.size} start_block_low={inode.start_block_low} " - f"start_block_high={inode.start_block_high} " - f"start_block={inode.start_block}" - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - - inspect_parser = subparsers.add_parser("inspect") - inspect_parser.add_argument("image", type=Path) - inspect_parser.add_argument("--path") - inspect_parser.set_defaults(function=inspect_image) - - error_parser = subparsers.add_parser("make-error-fixtures") - error_parser.add_argument("--plain", type=Path, required=True) - error_parser.add_argument("--compressed", type=Path, required=True) - error_parser.add_argument("--deflate", type=Path, required=True) - error_parser.add_argument("--compressed-offset", type=int, required=True) - error_parser.add_argument("--compressed-length", type=int, required=True) - error_parser.add_argument("--output", type=Path, required=True) - error_parser.set_defaults(function=make_error_fixtures) - - directory_parser = subparsers.add_parser("make-directory-fixtures") - directory_parser.add_argument("--base", type=Path, required=True) - directory_parser.add_argument("--output", type=Path, required=True) - directory_parser.set_defaults(function=make_directory_fixtures) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - args.function(args) - - -if __name__ == "__main__": - main() diff --git a/tests/final_review_fixtures.py b/tests/final_review_fixtures.py deleted file mode 100644 index 06e5eff..0000000 --- a/tests/final_review_fixtures.py +++ /dev/null @@ -1,412 +0,0 @@ -#!/usr/bin/env python3 -"""Build and self-check fixtures for TC157-TC160.""" - -from __future__ import annotations - -import argparse -import json -import os -from pathlib import Path -import shutil -import stat -import struct -import subprocess - -from review_fixtures import ( - EROFS_INODE_COMPRESSED_FULL, - ErofsImage, - FixtureError, - SUPER, - align, - normalize_times, - run_mkfs, - sha256, -) - - -FEATURE_INCOMPAT_48BIT = 0x80 -EROFS_INODE_FLAT_PLAIN = 0 -EROFS_I_DOT_OMITTED = 1 << 4 -Z_EROFS_ADVISE_EXTENTS = 0x1 -OFF_MAX = (1 << 63) - 1 - - -def map_header_offset(inode) -> int: - return align(inode.offset + inode.inode_size + inode.xattr_size, 8) - - -def record_size(advise: int) -> int: - return 4 << ((advise >> 1) & 3) - - -def binary_search_indices(lstarts: list[int], logical: int) -> list[int]: - visited: list[int] = [] - left = 0 - right = len(lstarts) - while left < right: - middle = left + (right - left) // 2 - visited.append(middle) - if lstarts[middle] > logical: - right = middle - else: - left = middle + 1 - if lstarts[middle] == logical: - right = min(left + 1, right) - return visited - - -def convert_extent_table( - base: ErofsImage, - inode, - recsz: int, - lstarts: list[int], - payload: int, -) -> tuple[ErofsImage, int, int]: - image = base.clone() - header = map_header_offset(inode) - table = align(header + 8, recsz) - table_end = table + recsz * len(lstarts) - if table_end > len(image.data): - raise FixtureError("explicit extent table exceeds the base image") - root = image.inode(image.root_nid) - if not ( - table_end <= root.offset - or root.offset + root.inode_size <= header - ): - raise FixtureError("explicit extent conversion overlaps the root inode") - - advise = Z_EROFS_ADVISE_EXTENTS | ({16: 2, 32: 3}[recsz] << 1) - struct.pack_into("> 32, - lstart & 0xFFFFFFFF, - ) - if recsz == 32: - struct.pack_into("> 32) - image.update_checksum() - return image, header, payload - - -def verify_extent_fixture( - image: ErofsImage, - inode, - recsz: int, - expected: list[int], - kind: str, - logical_probe: int, -) -> tuple[int, list[int]]: - header = map_header_offset(inode) - count = image.u32(header) | (image.u16(header + 6) << 32) - advise = image.u16(header + 4) - if count != len(expected) or record_size(advise) != recsz: - raise FixtureError("explicit extent header self-check failed") - table = align(header + 8, recsz) - observed: list[int] = [] - for index in range(count): - offset = table + index * recsz - value = image.u32(offset + 12) - if recsz == 32: - value |= image.u32(offset + 16) << 32 - observed.append(value) - if observed != expected or any(value >= inode.size for value in observed): - raise FixtureError("explicit extent lstart self-check failed") - violations = [ - index - for index in range(1, len(observed)) - if observed[index] <= observed[index - 1] - ] - if len(violations) != 1: - raise FixtureError("fixture must contain exactly one ordering violation") - violation = violations[0] - if kind == "duplicate" and observed[violation] != observed[violation - 1]: - raise FixtureError("duplicate fixture is not a duplicate") - if kind != "duplicate" and observed[violation] >= observed[violation - 1]: - raise FixtureError("descending fixture is not descending") - visited = binary_search_indices(observed, logical_probe) - if kind == "cross-branch" and violation - 1 in visited: - raise FixtureError("cross-branch violation is visible to the old search") - return table, visited - - -def make_extent_fixtures(output: Path, source: Path) -> tuple[list[str], dict]: - target_source = source / "extent.bin" - target_source.write_bytes(b"extent-ordering-payload\n" * 65536) - normalize_times(source) - base_path = output / ".extent-base.erofs" - run_mkfs( - source, - base_path, - "77777777-1111-4222-8333-000000000157", - "-E", - "legacy-compress,force-inode-extended", - "-z", - "lz4", - "-C4096", - ) - base = ErofsImage.load(base_path) - inode = base.resolve_root_entry("extent.bin") - if inode.inode_size != 64 or inode.layout != EROFS_INODE_COMPRESSED_FULL: - raise FixtureError("extent target is not extended COMPRESSED_FULL") - header = map_header_offset(inode) - payload = base.u32(header + 20) << base.block_bits - if payload == 0 or payload + base.block_size > header: - raise FixtureError("original compressed payload is not isolated from metadata") - - cases = { - "descending": ([0, 4096, 12288, 8192], 8192), - "duplicate": ([0, 4096, 4096, 12288], 4096), - "cross-branch": ([0, 8192, 4096, 12288], 4096), - } - evidence: list[str] = [] - manifest_cases: dict[str, object] = {} - for recsz in (16, 32): - for kind, (lstarts, logical_probe) in cases.items(): - image, header, payload = convert_extent_table( - base, inode, recsz, lstarts, payload - ) - table, visited = verify_extent_fixture( - image, inode, recsz, lstarts, kind, logical_probe - ) - name = f"extent-{recsz}-{kind}.erofs" - path = output / name - image.save(path) - evidence.append( - "extent-order " - f"image={name} nid={inode.nid} recsize={recsz} " - f"header={header} table={table} lstarts={lstarts} " - f"probe={logical_probe} old_search_indices={visited} " - f"payload_offset={payload} violation={kind}" - ) - manifest_cases[name] = { - "path": "/extent.bin", - "nid": inode.nid, - "record_size": recsz, - "header_offset": header, - "table_offset": table, - "lstarts": lstarts, - "logical_probe": logical_probe, - "old_search_indices": visited, - "payload_offset": payload, - "payload_length": image.block_size, - "expected_errno": 97, - } - base_path.unlink() - return evidence, {"cases": manifest_cases} - - -def make_blocks_fixture(output: Path, source: Path) -> tuple[str, dict]: - target_source = source / "compressed-blocks.bin" - target_source.write_bytes(bytes(range(256)) * 4096) - normalize_times(source) - base_path = output / ".blocks-base.erofs" - run_mkfs( - source, - base_path, - "77777777-1111-4222-8333-000000000158", - "-E", - "legacy-compress,force-inode-extended", - "-z", - "lz4", - "-C4096", - ) - image = ErofsImage.load(base_path) - inode = image.resolve_root_entry("compressed-blocks.bin") - if inode.inode_size != 64 or inode.layout != EROFS_INODE_COMPRESSED_FULL: - raise FixtureError("compressed block target has the wrong inode layout") - root_nid = image.root_nid - inode_blocks_lo = image.u32(inode.offset + 16) - if inode_blocks_lo == 0: - raise FixtureError("compressed target has zero blocks_lo") - super_blocks_lo = image.u32(SUPER + 36) - image.put_u64(SUPER + 112, root_nid) - image.put_u32( - SUPER + 80, image.u32(SUPER + 80) | FEATURE_INCOMPAT_48BIT - ) - image.put_u16(SUPER + 14, 1) - image.put_u16(inode.offset + 6, 1) - image.update_checksum() - path = output / "compressed-blocks-hi.erofs" - image.save(path) - base_path.unlink() - - data_blocks = (1 << 32) | inode_blocks_lo - va_bytes = data_blocks << image.block_bits - st_blocks = va_bytes // 512 - provider_blocks = (1 << 32) | super_blocks_lo - provider_size = provider_blocks << image.block_bits - if image.root_nid != root_nid or image.u16(inode.offset + 6) != 1: - raise FixtureError("48-bit inode/superblock self-check failed") - if provider_size <= 16 * 1024**4: - raise FixtureError("sparse provider is not larger than 16 TiB") - evidence = ( - "compressed-blocks-hi " - f"nid={inode.nid} inode_offset={inode.offset} blocks_lo={inode_blocks_lo} " - f"blocks_hi=1 data_blocks={data_blocks} block_size={image.block_size} " - f"va_bytes={va_bytes} st_blocks={st_blocks} " - f"provider_blocks={provider_blocks} provider_size={provider_size}" - ) - manifest = { - "image": path.name, - "path": "/compressed-blocks.bin", - "nid": inode.nid, - "inode_offset": inode.offset, - "blocks_lo": inode_blocks_lo, - "blocks_hi": 1, - "data_blocks": data_blocks, - "block_size": image.block_size, - "va_bytes": va_bytes, - "st_blocks": st_blocks, - "provider_size": provider_size, - "seed_size": len(image.data), - } - return evidence, manifest - - -def make_special_fixture(output: Path, source: Path) -> tuple[str, dict]: - fifo = source / "special.fifo" - os.mkfifo(fifo, 0o640) - normalize_times(source) - path = output / "special-setattr.erofs" - run_mkfs( - source, - path, - "77777777-1111-4222-8333-000000000159", - "-E", - "force-inode-extended", - ) - image = ErofsImage.load(path) - inode = image.resolve_root_entry("special.fifo") - mode = image.u16(inode.offset + 4) - if not stat.S_ISFIFO(mode): - raise FixtureError("special setattr target is not a FIFO") - evidence = ( - f"special-setattr nid={inode.nid} inode_offset={inode.offset} " - f"mode={mode:#o} type=fifo" - ) - return evidence, { - "image": path.name, - "path": "/special.fifo", - "nid": inode.nid, - "mode": mode, - } - - -def make_offmax_fixture(output: Path, source: Path) -> tuple[str, dict]: - directory = source / "offmax-dir" - directory.mkdir() - for index in range(384): - (directory / f"entry-{index:03d}").write_text( - f"entry {index}\n", encoding="ascii" - ) - normalize_times(source) - base_path = output / ".offmax-base.erofs" - run_mkfs( - source, - base_path, - "77777777-1111-4222-8333-000000000160", - "-E", - "force-inode-extended", - ) - image = ErofsImage.load(base_path) - inode = image.resolve_root_entry("offmax-dir") - mode = image.u16(inode.offset + 4) - if inode.inode_size != 64 or not stat.S_ISDIR(mode) or inode.start_block == 0: - raise FixtureError("OFF_MAX target is not an extended backed directory") - inode_format = image.u16(inode.offset) - inode_format &= ~0x0E - inode_format |= EROFS_I_DOT_OMITTED - image.put_u16(inode.offset, inode_format) - image.put_u64(inode.offset + 8, OFF_MAX) - image.update_checksum() - path = output / "dot-omitted-offmax.erofs" - image.save(path) - base_path.unlink() - mutated = image.resolve_root_entry("offmax-dir") - if ( - mutated.inode_size != 64 - or mutated.layout != EROFS_INODE_FLAT_PLAIN - or mutated.size != OFF_MAX - or not image.u16(mutated.offset) & EROFS_I_DOT_OMITTED - ): - raise FixtureError("OFF_MAX directory field self-check failed") - evidence = ( - f"dot-omitted-offmax nid={mutated.nid} inode_offset={mutated.offset} " - f"format={image.u16(mutated.offset):#x} size={mutated.size} " - f"start_block={mutated.start_block} expected_errno=97" - ) - return evidence, { - "image": path.name, - "path": "/offmax-dir", - "nid": mutated.nid, - "inode_offset": mutated.offset, - "size": mutated.size, - "expected_errno": 97, - } - - -def make_fixtures(output: Path) -> None: - if shutil.which("mkfs.erofs") is None: - raise FixtureError("mkfs.erofs is required") - if output.exists() and any(output.iterdir()): - raise FixtureError(f"output directory is not empty: {output}") - output.mkdir(parents=True, exist_ok=True) - sources = output / ".sources" - extent_source = sources / "extent" - blocks_source = sources / "blocks" - special_source = sources / "special" - offmax_source = sources / "offmax" - for source in (extent_source, blocks_source, special_source, offmax_source): - source.mkdir(parents=True) - - evidence, extent_manifest = make_extent_fixtures(output, extent_source) - blocks_evidence, blocks_manifest = make_blocks_fixture(output, blocks_source) - special_evidence, special_manifest = make_special_fixture(output, special_source) - offmax_evidence, offmax_manifest = make_offmax_fixture(output, offmax_source) - evidence.extend([blocks_evidence, special_evidence, offmax_evidence]) - manifest = { - "extent_order": extent_manifest, - "compressed_blocks_hi": blocks_manifest, - "special_setattr": special_manifest, - "dot_omitted_offmax": offmax_manifest, - } - shutil.rmtree(sources) - (output / "fixture-evidence.txt").write_text( - "\n".join(evidence) + "\n", encoding="ascii" - ) - (output / "fixture-manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - with (output / "SHA256SUMS").open("w", encoding="ascii") as sums: - for path in sorted(output.glob("*.erofs")): - sums.write(f"{sha256(path)} {path.name}\n") - print((output / "fixture-evidence.txt").read_text(encoding="ascii"), end="") - print((output / "SHA256SUMS").read_text(encoding="ascii"), end="") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - make_parser = subparsers.add_parser("make") - make_parser.add_argument("--output", type=Path, required=True) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - if args.command == "make": - make_fixtures(args.output) - - -if __name__ == "__main__": - try: - main() - except (FixtureError, OSError, subprocess.CalledProcessError) as error: - raise SystemExit(f"final_review_fixtures.py: {error}") from error diff --git a/tests/final_review_probe.c b/tests/final_review_probe.c deleted file mode 100644 index 6797734..0000000 --- a/tests/final_review_probe.c +++ /dev/null @@ -1,127 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static uint64_t -parse_u64(const char *text) -{ - char *end; - uintmax_t value; - - errno = 0; - value = strtoumax(text, &end, 0); - if (errno != 0 || *text == '\0' || *end != '\0' || value > UINT64_MAX) - errx(2, "invalid integer: %s", text); - return ((uint64_t)value); -} - -static void -stat_blocks(const char *path, const char *expected_text) -{ - fhandle_t handle; - struct stat direct, by_handle; - uint64_t expected; - - expected = parse_u64(expected_text); - if (stat(path, &direct) != 0) - err(1, "stat %s", path); - if (getfh(path, &handle) != 0) - err(1, "getfh %s", path); - if (fhstat(&handle, &by_handle) != 0) - err(1, "fhstat %s", path); - if ((uint64_t)direct.st_blocks != expected || - (uint64_t)by_handle.st_blocks != expected) - errx(1, "st_blocks direct=%jd handle=%jd expected=%" PRIu64, - (intmax_t)direct.st_blocks, (intmax_t)by_handle.st_blocks, - expected); - printf("direct_blocks=%jd handle_blocks=%jd size=%jd expected=%" PRIu64 - "\n", (intmax_t)direct.st_blocks, (intmax_t)by_handle.st_blocks, - (intmax_t)direct.st_size, expected); -} - -static void -expect_stat_error(const char *path, const char *expected_text) -{ - struct stat status; - int expected; - - expected = (int)parse_u64(expected_text); - errno = 0; - if (stat(path, &status) != -1) - errx(1, "stat unexpectedly succeeded: %s", path); - if (errno != expected) - errx(1, "stat errno=%d expected=%d", errno, expected); - printf("stat_errno=%d path=%s\n", errno, path); -} - -static void -readdir_offmax(const char *path, const char *expected_text) -{ - char buffer[512]; - off_t base, current; - ssize_t bytes; - int descriptor, expected; - - expected = (int)parse_u64(expected_text); - descriptor = open(path, O_RDONLY | O_DIRECTORY); - if (descriptor < 0) - err(1, "open %s", path); - if (lseek(descriptor, OFF_MAX, SEEK_SET) != OFF_MAX) - err(1, "lseek OFF_MAX"); - base = 0; - errno = 0; - bytes = getdirentries(descriptor, buffer, sizeof(buffer), &base); - if (bytes != -1 || errno != expected) - errx(1, "OFF_MAX getdirentries bytes=%zd errno=%d expected=%d", - bytes, errno, expected); - current = lseek(descriptor, 0, SEEK_CUR); - if (current < 0 || current != OFF_MAX) - errx(1, "negative or changed post-error offset: %jd", - (intmax_t)current); - if (lseek(descriptor, 0, SEEK_SET) != 0) - err(1, "lseek zero"); - base = 0; - errno = 0; - bytes = getdirentries(descriptor, buffer, sizeof(buffer), &base); - if (bytes != -1 || errno != expected) - errx(1, "zero-offset getdirentries bytes=%zd errno=%d expected=%d", - bytes, errno, expected); - current = lseek(descriptor, 0, SEEK_CUR); - if (current < 0 || current != 0) - errx(1, "negative or changed zero offset: %jd", (intmax_t)current); - if (close(descriptor) != 0) - err(1, "close %s", path); - printf("offmax_errno=%d offmax_offset=%jd zero_errno=%d zero_offset=%jd\n", - expected, (intmax_t)OFF_MAX, expected, (intmax_t)current); -} - -int -main(int argc, char **argv) -{ - if (argc == 4 && strcmp(argv[1], "stat-blocks") == 0) { - stat_blocks(argv[2], argv[3]); - return (0); - } - if (argc == 4 && strcmp(argv[1], "expect-stat-error") == 0) { - expect_stat_error(argv[2], argv[3]); - return (0); - } - if (argc == 4 && strcmp(argv[1], "readdir-offmax") == 0) { - readdir_offmax(argv[2], argv[3]); - return (0); - } - errx(2, "usage: %s stat-blocks PATH EXPECTED | " - "expect-stat-error PATH ERRNO | readdir-offmax PATH ERRNO", argv[0]); -} diff --git a/tests/final_review_setattr_probe.c b/tests/final_review_setattr_probe.c deleted file mode 100644 index 7ef1856..0000000 --- a/tests/final_review_setattr_probe.c +++ /dev/null @@ -1,87 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -static char probe_path[MAXPATHLEN]; -static int probe_result = -1; - -static int -probe_run(SYSCTL_HANDLER_ARGS) -{ - struct nameidata nd; - struct vattr attributes; - struct vnode *vnode; - int error, operation; - - operation = 0; - error = sysctl_handle_int(oidp, &operation, 0, req); - if (error != 0 || req->newptr == NULL) - return (error); - if (operation < 1 || operation > 5 || probe_path[0] == '\0') - return (EINVAL); - - NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, probe_path); - error = namei(&nd); - if (error != 0) { - probe_result = error; - return (0); - } - vnode = nd.ni_vp; - NDFREE_PNBUF(&nd); - VATTR_NULL(&attributes); - attributes.va_size = 0; - if (operation == 2 || operation == 5) - attributes.va_mode = 0600; - if (operation == 3 || operation == 5) { - attributes.va_uid = 1; - attributes.va_gid = 1; - } - if (operation == 4 || operation == 5) { - attributes.va_atime.tv_sec = 1; - attributes.va_atime.tv_nsec = 0; - attributes.va_mtime.tv_sec = 1; - attributes.va_mtime.tv_nsec = 0; - } - probe_result = VOP_SETATTR(vnode, &attributes, curthread->td_ucred); - vput(vnode); - return (0); -} - -static int -probe_modevent(module_t module, int event, void *arg) -{ - (void)module; - (void)arg; - switch (event) { - case MOD_LOAD: - case MOD_UNLOAD: - return (0); - default: - return (EOPNOTSUPP); - } -} - -SYSCTL_NODE(_debug, OID_AUTO, erofs_setattr_probe, - CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "EROFS setattr regression probe"); -SYSCTL_STRING(_debug_erofs_setattr_probe, OID_AUTO, path, CTLFLAG_RW, - probe_path, sizeof(probe_path), "Target vnode path"); -SYSCTL_PROC(_debug_erofs_setattr_probe, OID_AUTO, run, - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0, probe_run, "I", - "1=size; 2=size+mode; 3=size+owner; 4=size+times; 5=all"); -SYSCTL_INT(_debug_erofs_setattr_probe, OID_AUTO, result, CTLFLAG_RD, - &probe_result, 0, "Last VOP_SETATTR errno"); - -static moduledata_t probe_module = { - "erofs_setattr_probe", - probe_modevent, - NULL, -}; - -DECLARE_MODULE(erofs_setattr_probe, probe_module, SI_SUB_DRIVERS, - SI_ORDER_MIDDLE); -MODULE_VERSION(erofs_setattr_probe, 1); diff --git a/tests/final_review_setattr_probe.mk b/tests/final_review_setattr_probe.mk deleted file mode 100644 index eda58e1..0000000 --- a/tests/final_review_setattr_probe.mk +++ /dev/null @@ -1,4 +0,0 @@ -KMOD=erofs_setattr_probe -SRCS=final_review_setattr_probe.c vnode_if.h - -.include diff --git a/tests/g1_fixtures.py b/tests/g1_fixtures.py deleted file mode 100755 index caa366b..0000000 --- a/tests/g1_fixtures.py +++ /dev/null @@ -1,372 +0,0 @@ -#!/usr/bin/env python3 -"""Build assertion-driven structured variants for the G1 manual tests.""" - -from __future__ import annotations - -import argparse -import hashlib -import shutil -import struct -from pathlib import Path - -from erofs_fixture import ErofsImage, FEATURE_INCOMPAT_48BIT, SUPER - - -S_IFMT = 0o170000 -S_IFDIR = 0o040000 -S_IFREG = 0o100000 -S_IFLNK = 0o120000 -S_IFCHR = 0o020000 -S_IFBLK = 0o060000 -S_IFIFO = 0o010000 -FLAT_PLAIN = 0 -FLAT_INLINE = 2 -I_NLINK_1 = 0x10 -NULL_ADDR_48 = (1 << 48) - 1 -LARGE_FILE_SIZE = (1 << 32) + 4097 - - -def inode_for(image: ErofsImage, path: str): - _, entry = image.resolve_root_entry(path) - return image.inode(entry.nid) - - -def assert_inode( - image: ErofsImage, - path: str, - *, - inode_size: int | None = None, - layout: int | None = None, - file_type: int | None = None, -): - inode = inode_for(image, path) - if inode_size is not None and inode.inode_size != inode_size: - raise ValueError(f"{path}: inode size {inode.inode_size}, expected {inode_size}") - if layout is not None and inode.layout != layout: - raise ValueError(f"{path}: layout {inode.layout}, expected {layout}") - if file_type is not None and inode.mode & S_IFMT != file_type: - raise ValueError(f"{path}: mode {inode.mode:#o}, expected type {file_type:#o}") - return inode - - -def save_checked(image: ErofsImage, path: Path) -> None: - image.update_checksum() - image.validate_superblock() - image.save(path) - - -def select_48bit_root(image: ErofsImage) -> int: - root_nid = image.root_nid - if root_nid == 0 or root_nid >= 1 << 48: - raise ValueError(f"root NID {root_nid} is unsuitable for the 48-bit fixture") - image.put_u32(SUPER + 80, image.feature_incompat | FEATURE_INCOMPAT_48BIT) - image.put_u16(SUPER + 14, 0) - image.put_u64(SUPER + 112, root_nid) - return root_nid - - -def rewrite_without_dot(image: ErofsImage, path: str) -> tuple[int, int, int]: - inode = assert_inode( - image, path, inode_size=32, layout=FLAT_INLINE, file_type=S_IFDIR - ) - entries = image.directory_entries(inode) - names = [entry.name for entry in entries] - if names.count(b".") != 1 or b".." not in names: - raise ValueError(f"{path}: expected one dot and an explicit dotdot entry") - kept = [entry for entry in entries if entry.name != b"."] - data_offset = inode.offset + inode.inode_size + inode.xattr_size - data = bytearray(len(kept) * 12) - names_blob = bytearray() - for index, entry in enumerate(kept): - name_offset = len(data) + len(names_blob) - file_type = image.data[entry.offset + 10] - struct.pack_into("= inode.size: - raise ValueError(f"{path}: rebuilt directory did not shrink") - image.data[data_offset : data_offset + inode.size] = bytes(inode.size) - image.data[data_offset : data_offset + len(rebuilt)] = rebuilt - image.put_u32(inode.offset + 8, len(rebuilt)) - image.put_u16(inode.offset, inode.inode_format | I_NLINK_1) - updated = image.inode(inode.nid) - updated_names = [entry.name for entry in image.directory_entries(updated)] - if b"." in updated_names or b".." not in updated_names: - raise AssertionError(f"{path}: dot omission rewrite failed") - return inode.nid, inode.size, len(rebuilt) - - -def image_hash(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def write_hashes(output: Path) -> None: - with (output / "IMAGE-SHA256SUMS").open("w", encoding="ascii") as sums: - for path in sorted(output.glob("*.erofs")): - sums.write(f"{image_hash(path)} {path.name}\n") - - -def copy_validated(source: Path, destination: Path) -> ErofsImage: - image = ErofsImage.load(source) - image.validate_superblock() - shutil.copy2(source, destination) - return image - - -def make(args: argparse.Namespace) -> None: - output = args.output - if output.exists(): - raise ValueError(f"output already exists: {output}") - output.mkdir(parents=True) - - compact = copy_validated(args.compact, output / "compact.erofs") - extended = copy_validated(args.extended, output / "extended.erofs") - flat = copy_validated(args.flat, output / "flat.erofs") - inline = copy_validated(args.inline, output / "inline.erofs") - evidence = [] - - compact_root = compact.inode(compact.root_nid) - if compact_root.inode_size != 32: - raise ValueError("compact image root is not a compact inode") - explicit_dir = assert_inode( - compact, "/dotdir", inode_size=32, layout=FLAT_INLINE, file_type=S_IFDIR - ) - explicit_names = [entry.name for entry in compact.directory_entries(explicit_dir)] - if b"." not in explicit_names or b".." not in explicit_names: - raise ValueError("compact /dotdir does not contain explicit dot entries") - if explicit_dir.inode_format & I_NLINK_1: - raise ValueError("compact /dotdir unexpectedly advertises dot omission") - evidence.append( - f"compact root_nid={compact.root_nid} inode_size=32 " - f"blocks={compact.blocks} inos={compact.u64(SUPER + 16)} " - f"dotdir_nid={explicit_dir.nid} dotdir_i_format={explicit_dir.inode_format:#06x} " - f"dotdir_names={','.join(name.decode('ascii') for name in explicit_names)}" - ) - - small = assert_inode(compact, "/small.txt", inode_size=32, file_type=S_IFREG) - single = assert_inode(compact, "/single.txt", inode_size=32, file_type=S_IFREG) - hard_a = assert_inode(compact, "/hard-a.txt", inode_size=32, file_type=S_IFREG) - hard_b = assert_inode(compact, "/hard-b.txt", inode_size=32, file_type=S_IFREG) - if hard_a.nid != hard_b.nid or compact.u16(hard_a.offset + 6) != 2: - raise ValueError("compact hard-link fixture does not encode nlink=2") - if compact.u16(single.offset + 6) != 1 or single.inode_format & I_NLINK_1: - raise ValueError("compact single-link baseline is not explicit nlink=1") - evidence.append( - f"compact small_nid={small.nid} single_nid={single.nid} " - f"single_i_format={single.inode_format:#06x} single_i_nb=1 " - f"hardlink_nid={hard_a.nid} hardlink_i_nb=2" - ) - - expected_rdev = 0x543ABC21 - special_fields = [] - for path, expected_type in ( - ("/char-large", S_IFCHR), - ("/block-large", S_IFBLK), - ("/fifo", S_IFIFO), - ): - inode = assert_inode( - compact, path, inode_size=32, file_type=expected_type - ) - raw = compact.u32(inode.offset + 16) - if expected_type in (S_IFCHR, S_IFBLK) and raw != expected_rdev: - raise ValueError(f"{path}: raw rdev {raw:#x}, expected {expected_rdev:#x}") - special_fields.append(f"{path[1:]}:nid={inode.nid}:raw={raw:#010x}") - evidence.append("compact special=" + ",".join(special_fields)) - - bad_dirent = compact.clone() - _, invalid_entry = bad_dirent.resolve_root_entry("/invalid-target.txt") - invalid_nid = (bad_dirent.blocks << bad_dirent.block_bits) // 32 + 1024 - bad_dirent.put_u64(invalid_entry.offset, invalid_nid) - save_checked(bad_dirent, output / "invalid-dirent-nid.erofs") - evidence.append( - f"invalid-dirent entry_offset={invalid_entry.offset} " - f"old_nid={invalid_entry.nid} new_nid={invalid_nid} crc=valid" - ) - - nlink1 = compact.clone() - single = assert_inode(nlink1, "/single.txt", inode_size=32, file_type=S_IFREG) - nlink1.put_u16(single.offset, single.inode_format | I_NLINK_1) - nlink1.put_u16(single.offset + 6, 0x1234) - save_checked(nlink1, output / "compact-nlink1.erofs") - evidence.append( - f"nlink1 nid={single.nid} i_format={single.inode_format | I_NLINK_1:#06x} " - "i_nb=0x1234 crc=valid" - ) - - dot_omitted = compact.clone() - dot_nid, old_size, new_size = rewrite_without_dot(dot_omitted, "/dotdir") - dot_root = select_48bit_root(dot_omitted) - save_checked(dot_omitted, output / "compact-dot-omitted.erofs") - evidence.append( - f"dot-omitted nid={dot_nid} i_format_bit4=1 old_size={old_size} " - f"new_size={new_size} ondisk_names=..,child.txt " - f"feature=0x80 rootnid_8b={dot_root} blocks_hi=0 crc=valid" - ) - - root8 = compact.clone() - root_nid = select_48bit_root(root8) - save_checked(root8, output / "root8-48bit.erofs") - if root8.root_nid != root_nid or root8.blocks != compact.blocks: - raise AssertionError("48-bit root selector self-check failed") - evidence.append( - f"root8 feature={root8.feature_incompat:#x} rootnid_8b={root_nid} " - f"blocks_hi={root8.u16(SUPER + 14)} blocks_lo={root8.u32(SUPER + 36)}" - ) - - fallback = compact.clone() - fallback_root = fallback.root_nid - fallback.put_u32( - SUPER + 80, fallback.feature_incompat | FEATURE_INCOMPAT_48BIT - ) - fallback.put_u64(SUPER + 112, 0) - save_checked(fallback, output / "fallback-48bit-root2.erofs") - if fallback.root_nid != fallback_root or fallback.blocks != compact.blocks: - raise AssertionError("48-bit fallback selector self-check failed") - evidence.append( - f"fallback feature={fallback.feature_incompat:#x} " - f"rootnid_2b={fallback_root} rootnid_8b=0 blocks_lo={fallback.blocks}" - ) - - normal = assert_inode( - extended, "/large-file.bin", inode_size=64, file_type=S_IFREG - ) - evidence.append( - f"extended large-file_nid={normal.nid} inode_off={normal.offset} " - f"size={normal.size} inode_size=64" - ) - - large_hole = extended.clone() - huge = assert_inode( - large_hole, "/huge-sparse.dat", inode_size=64, file_type=S_IFREG - ) - large_hole.put_u16(huge.offset, (huge.inode_format & ~(7 << 1)) | (FLAT_PLAIN << 1)) - large_hole.put_u16(huge.offset + 6, (NULL_ADDR_48 >> 32) & 0xFFFF) - large_hole.put_u32(huge.offset + 16, NULL_ADDR_48 & 0xFFFFFFFF) - large_hole.put_u64(huge.offset + 8, LARGE_FILE_SIZE) - save_checked(large_hole, output / "extended-large-hole.erofs") - evidence.append( - f"extended-large-hole nid={huge.nid} inode_off={huge.offset} " - f"size={LARGE_FILE_SIZE} layout=0 startblk=0xffffffffffff crc=valid" - ) - - oversize = extended.clone() - oversize_inode = assert_inode( - oversize, "/oversize.dat", inode_size=64, file_type=S_IFREG - ) - oversize.put_u64(oversize_inode.offset + 8, 1 << 63) - save_checked(oversize, output / "extended-size-bit63.erofs") - evidence.append( - f"extended-size-bit63 nid={oversize_inode.nid} " - f"inode_off={oversize_inode.offset} i_size=0x8000000000000000 crc=valid" - ) - - for path in ( - "/small.txt", - "/medium.dat", - "/large.bin", - "/data.txt", - "/random-test.bin", - "/huge-data.bin", - ): - inode = assert_inode(flat, path, layout=FLAT_PLAIN, file_type=S_IFREG) - evidence.append( - f"flat path={path} nid={inode.nid} layout={inode.layout} size={inode.size}" - ) - longlink = assert_inode( - flat, "/longlink", layout=FLAT_INLINE, file_type=S_IFLNK - ) - if longlink.size <= 60: - raise ValueError("longlink target is not longer than 60 bytes") - evidence.append( - f"flat path=/longlink nid={longlink.nid} layout={longlink.layout} " - f"size={longlink.size}" - ) - - inline_zero = inline.clone() - empty = assert_inode(inline_zero, "/empty.txt", inode_size=32, file_type=S_IFREG) - inline_zero.put_u16( - empty.offset, (empty.inode_format & ~(7 << 1)) | (FLAT_INLINE << 1) - ) - save_checked(inline_zero, output / "inline-zero.erofs") - empty_after = assert_inode( - inline_zero, - "/empty.txt", - inode_size=32, - layout=FLAT_INLINE, - file_type=S_IFREG, - ) - if empty_after.size != 0: - raise AssertionError("inline zero fixture size changed") - evidence.append( - f"inline-zero nid={empty_after.nid} i_format={empty_after.inode_format:#06x} size=0" - ) - - inline_expectations = { - "/tiny.txt": FLAT_INLINE, - "/inline.txt": FLAT_INLINE, - "/tailpacked.dat": FLAT_INLINE, - "/file-4095.dat": FLAT_PLAIN, - "/file-4096.dat": FLAT_PLAIN, - "/file-4097.dat": FLAT_INLINE, - "/link1": FLAT_INLINE, - "/brokenlink": FLAT_INLINE, - } - for path, layout in inline_expectations.items(): - expected_type = S_IFLNK if path in ("/link1", "/brokenlink") else S_IFREG - inode = assert_inode(inline, path, layout=layout, file_type=expected_type) - evidence.append( - f"inline path={path} nid={inode.nid} layout={inode.layout} size={inode.size}" - ) - - inline_cross = inline.clone() - cross = assert_inode( - inline_cross, - "/inline.txt", - inode_size=32, - layout=FLAT_INLINE, - file_type=S_IFREG, - ) - if inline_cross.u16(cross.offset + 2) != 0: - raise ValueError("inline.txt unexpectedly has inline xattrs") - chosen_count = None - chosen_offset = None - for count in range(1, 65536): - xattr_size = 12 + 4 * (count - 1) - data_offset = cross.offset + cross.inode_size + xattr_size - block_offset = data_offset % inline_cross.block_size - if block_offset + cross.size > inline_cross.block_size: - chosen_count = count - chosen_offset = data_offset - break - if chosen_count is None or chosen_offset is None: - raise AssertionError("could not construct cross-block inline range") - inline_cross.put_u16(cross.offset + 2, chosen_count) - save_checked(inline_cross, output / "inline-cross-block.erofs") - evidence.append( - f"inline-cross nid={cross.nid} inode_off={cross.offset} " - f"xattr_icount=0->{chosen_count} data_blockoff={chosen_offset % inline_cross.block_size} " - f"size={cross.size} crc=valid" - ) - - (output / "fixture-evidence.txt").write_text( - "\n".join(evidence) + "\n", encoding="ascii" - ) - write_hashes(output) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--compact", type=Path, required=True) - parser.add_argument("--extended", type=Path, required=True) - parser.add_argument("--flat", type=Path, required=True) - parser.add_argument("--inline", dest="inline", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - return parser.parse_args() - - -def main() -> None: - make(parse_args()) - - -if __name__ == "__main__": - main() diff --git a/tests/g3_fixtures.py b/tests/g3_fixtures.py deleted file mode 100755 index 291fe12..0000000 --- a/tests/g3_fixtures.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 -"""Compatibility entry point for the maintained Pre15 G3 fixture helper.""" - -from __future__ import annotations - -import importlib.util -from pathlib import Path -import sys - - -HELPER = Path(__file__).resolve().parent / "pre15/fixtures/g3.py" - - -def main() -> None: - spec = importlib.util.spec_from_file_location("pre15_g3", HELPER) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load {HELPER}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - module.main(sys.argv[1:]) - - -if __name__ == "__main__": - main() diff --git a/tests/g3_vfs_probe.c b/tests/g3_vfs_probe.c deleted file mode 100644 index 1679af6..0000000 --- a/tests/g3_vfs_probe.c +++ /dev/null @@ -1,208 +0,0 @@ -#define _POSIX_C_SOURCE 200809L - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static int -parse_errno(const char *text) -{ - char *end; - long value; - - errno = 0; - value = strtol(text, &end, 0); - if (errno != 0 || *text == '\0' || *end != '\0' || value <= 0 || - value > 255) - errx(2, "invalid errno: %s", text); - return ((int)value); -} - -static int -run_operation(const char *operation, const char *path) -{ - struct stat sb; - int fd; - - if (strcmp(operation, "stat") == 0) - return (stat(path, &sb)); - if (strcmp(operation, "lstat") == 0) - return (lstat(path, &sb)); - if (strcmp(operation, "open-read") == 0) { - fd = open(path, O_RDONLY); - if (fd < 0) - return (-1); - return (close(fd)); - } - if (strcmp(operation, "open-rdwr") == 0) { - fd = open(path, O_RDWR); - if (fd < 0) - return (-1); - return (close(fd)); - } - if (strcmp(operation, "access-read") == 0) - return (access(path, R_OK)); - if (strcmp(operation, "access-write") == 0) - return (access(path, W_OK)); - if (strcmp(operation, "access-exec") == 0) - return (access(path, X_OK)); - if (strcmp(operation, "chmod") == 0) - return (chmod(path, 0777)); - if (strcmp(operation, "chown") == 0) - return (chown(path, 65534, 65534)); - if (strcmp(operation, "truncate") == 0) - return (truncate(path, 0)); - if (strcmp(operation, "create") == 0) { - fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0600); - if (fd < 0) - return (-1); - if (close(fd) != 0) - return (-1); - return (unlink(path)); - } - errx(2, "unknown operation: %s", operation); -} - -static void -expect_result(const char *operation, const char *path, int expected_errno) -{ - int error, result; - - errno = 0; - result = run_operation(operation, path); - error = errno; - if (expected_errno == 0) { - if (result != 0) - errx(1, "%s %s returned errno %d (%s)", operation, path, - error, strerror(error)); - printf("op=%s result=success errno=0\n", operation); - return; - } - if (result == 0) - errx(1, "%s %s unexpectedly succeeded", operation, path); - if (error != expected_errno) - errx(1, "%s %s returned errno %d (%s), expected %d", operation, - path, error, strerror(error), expected_errno); - printf("op=%s result=error errno=%d message=%s\n", operation, - error, strerror(error)); -} - -static long -checked_pathconf(const char *path, int name, const char *label, int *errorp) -{ - long value; - - errno = 0; - value = pathconf(path, name); - *errorp = value == -1 ? errno : 0; - if (*errorp != 0) - printf("%s=error:%d ", label, *errorp); - else - printf("%s=%ld ", label, value); - return (value); -} - -static void -show_pathconf(const char *path) -{ - int chown_error, filesizebits_error, link_error, name_error, - no_trunc_error, path_error; - long chown_restricted, filesizebits, link_max, name_max, no_trunc, - path_max; - - name_max = checked_pathconf(path, _PC_NAME_MAX, "name_max", - &name_error); - path_max = checked_pathconf(path, _PC_PATH_MAX, "path_max", - &path_error); - filesizebits = checked_pathconf(path, _PC_FILESIZEBITS, "filesizebits", - &filesizebits_error); - link_max = checked_pathconf(path, _PC_LINK_MAX, "link_max", - &link_error); - no_trunc = checked_pathconf(path, _PC_NO_TRUNC, "no_trunc", - &no_trunc_error); - chown_restricted = checked_pathconf(path, _PC_CHOWN_RESTRICTED, - "chown_restricted", &chown_error); - putchar('\n'); - if (name_error != 0 || path_error != 0 || filesizebits_error != 0 || - link_error != 0 || no_trunc_error != 0 || chown_error != 0 || - name_max != 255 || path_max <= 0 || filesizebits != 64 || - link_max <= 0 || no_trunc != 1 || chown_restricted != 1) - errx(1, "unexpected EROFS pathconf values"); -} - -static void -show_readlink(const char *path) -{ - unsigned char target[4096]; - uint64_t hash; - ssize_t length; - - length = readlink(path, (char *)target, sizeof(target)); - if (length < 0) - err(1, "readlink %s", path); - if ((size_t)length == sizeof(target)) - errx(1, "symlink target is too long"); - hash = UINT64_C(14695981039346656037); - for (ssize_t index = 0; index < length; index++) { - hash ^= target[index]; - hash *= UINT64_C(1099511628211); - } - target[length] = '\0'; - printf("length=%zd fnv1a64=%016" PRIx64 " target=%s\n", length, - hash, target); -} - -static void -show_stat(const char *path) -{ - struct stat sb; - - if (lstat(path, &sb) != 0) - err(1, "lstat %s", path); - printf("ino=%ju mode=%#jo uid=%ju gid=%ju nlink=%ju size=%jd " - "blocks=%jd blksize=%jd", - (uintmax_t)sb.st_ino, (uintmax_t)sb.st_mode, - (uintmax_t)sb.st_uid, (uintmax_t)sb.st_gid, - (uintmax_t)sb.st_nlink, (intmax_t)sb.st_size, - (intmax_t)sb.st_blocks, (intmax_t)sb.st_blksize); -#ifdef __FreeBSD__ - printf(" gen=%ju", (uintmax_t)sb.st_gen); -#endif - putchar('\n'); -} - -int -main(int argc, char **argv) -{ - if (argc == 5 && strcmp(argv[1], "expect-error") == 0) { - expect_result(argv[2], argv[3], parse_errno(argv[4])); - return (0); - } - if (argc == 4 && strcmp(argv[1], "expect-success") == 0) { - expect_result(argv[2], argv[3], 0); - return (0); - } - if (argc == 3 && strcmp(argv[1], "pathconf") == 0) { - show_pathconf(argv[2]); - return (0); - } - if (argc == 3 && strcmp(argv[1], "readlink") == 0) { - show_readlink(argv[2]); - return (0); - } - if (argc == 3 && strcmp(argv[1], "stat") == 0) { - show_stat(argv[2]); - return (0); - } - errx(2, "usage: %s expect-error OP PATH ERRNO | " - "expect-success OP PATH | pathconf PATH | readlink PATH | stat PATH", - argv[0]); -} diff --git a/tests/g4_fixtures.py b/tests/g4_fixtures.py deleted file mode 100755 index f3cb8a0..0000000 --- a/tests/g4_fixtures.py +++ /dev/null @@ -1,1277 +0,0 @@ -#!/usr/bin/env python3 -"""Build and verify the deterministic xattr, ACL, and metabox G4 fixtures.""" - -from __future__ import annotations - -import argparse -from dataclasses import dataclass -import hashlib -import json -import os -from pathlib import Path -import shutil -import stat -import struct -import subprocess - - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -FEATURE_COMPAT_SB_CHKSUM = 0x00000001 -FEATURE_COMPAT_SHARED_EA_IN_METABOX = 0x00000008 -FEATURE_COMPAT_PLAIN_XATTR_PFX = 0x00000010 -FEATURE_COMPAT_ISHARE_XATTRS = 0x00000020 -FEATURE_INCOMPAT_FRAGMENTS = 0x00000020 -FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040 -FEATURE_INCOMPAT_METABOX = 0x00000100 -XATTR_INDEX_USER = 1 -XATTR_INDEX_ACL_ACCESS = 2 -XATTR_INDEX_ACL_DEFAULT = 3 -XATTR_INDEX_TRUSTED = 4 -XATTR_INDEX_SECURITY = 6 -XATTR_LONG_PREFIX = 0x80 -METABOX_NID_BIT = 1 << 63 -LAYOUT_FLAT_PLAIN = 0 -LAYOUT_COMPRESSED_FULL = 1 -LAYOUT_FLAT_INLINE = 2 -LAYOUT_COMPRESSED_COMPACT = 3 -ACL_VERSION = 2 -ACL_USER_OBJ = 0x01 -ACL_USER = 0x02 -ACL_GROUP_OBJ = 0x04 -ACL_GROUP = 0x08 -ACL_MASK = 0x10 -ACL_OTHER = 0x20 -ACL_UNDEFINED_ID = 0xFFFFFFFF -BLOCK_SIZE = 4096 -METABOX_SIZE = 32768 -METABOX_INODE_A = 16 -METABOX_INODE_B = 32 -METABOX_PREFIX_OFFSET = 8192 -METABOX_XATTR_BASE = 4096 -CRC32C_POLYNOMIAL = 0x82F63B78 - - -class FixtureError(RuntimeError): - pass - - -@dataclass(frozen=True) -class Inode: - nid: int - offset: int - inode_format: int - inode_size: int - xattr_size: int - layout: int - mode: int - size: int - start_block: int - - -@dataclass(frozen=True) -class DirectoryEntry: - name: bytes - nid: int - offset: int - - -@dataclass(frozen=True) -class XattrEntry: - offset: int - size: int - name_index: int - name: bytes - value: bytes - - -def align(value: int, alignment: int) -> int: - return (value + alignment - 1) & -alignment - - -def sha256_bytes(data: bytes | bytearray) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def crc32c(data: bytes | bytearray) -> int: - checksum = 0xFFFFFFFF - for byte in data: - checksum ^= byte - for _ in range(8): - checksum = (checksum >> 1) ^ ( - CRC32C_POLYNOMIAL if checksum & 1 else 0 - ) - return checksum & 0xFFFFFFFF - - -class ErofsImage: - def __init__(self, data: bytes | bytearray, source: Path) -> None: - self.data = bytearray(data) - self.source = source - self.validate_superblock() - - @classmethod - def load(cls, path: Path) -> "ErofsImage": - return cls(path.read_bytes(), path) - - def clone(self) -> "ErofsImage": - return ErofsImage(self.data, self.source) - - def u16(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - struct.pack_into(" None: - struct.pack_into(" None: - struct.pack_into(" int: - return self.data[SUPER + 12] - - @property - def block_size(self) -> int: - return 1 << self.block_bits - - @property - def feature_compat(self) -> int: - return self.u32(SUPER + 8) - - @property - def feature_incompat(self) -> int: - return self.u32(SUPER + 80) - - @property - def blocks(self) -> int: - return self.u32(SUPER + 36) - - @property - def declared_size(self) -> int: - return self.blocks << self.block_bits - - @property - def root_nid(self) -> int: - return self.u16(SUPER + 14) - - @property - def meta_offset(self) -> int: - return self.u32(SUPER + 40) << self.block_bits - - @property - def packed_nid(self) -> int: - return self.u64(SUPER + 96) - - @property - def checksum_end(self) -> int: - return SUPER + self.block_size - SUPER - - def calculated_checksum(self) -> int: - window = bytearray(self.data[SUPER : self.checksum_end]) - struct.pack_into(" None: - if not self.feature_compat & FEATURE_COMPAT_SB_CHKSUM: - raise FixtureError(f"{self.source}: checksum feature is required") - self.put_u32(SUPER + 4, 0) - self.put_u32(SUPER + 4, self.calculated_checksum()) - if self.u32(SUPER + 4) != self.calculated_checksum(): - raise AssertionError("updated checksum does not verify") - - def validate_superblock(self) -> None: - if len(self.data) < SUPER + 144: - raise FixtureError(f"{self.source}: image is shorter than 1168 bytes") - if self.u32(SUPER) != MAGIC: - raise FixtureError(f"{self.source}: bad magic") - if not 9 <= self.block_bits <= 16: - raise FixtureError(f"{self.source}: invalid block bits") - if self.blocks == 0 or self.declared_size > len(self.data): - raise FixtureError(f"{self.source}: invalid declared size") - if ( - self.feature_compat & FEATURE_COMPAT_SB_CHKSUM - and self.u32(SUPER + 4) != self.calculated_checksum() - ): - raise FixtureError(f"{self.source}: bad checksum") - - def inode(self, nid: int) -> Inode: - offset = self.meta_offset + (nid << 5) - if offset > self.declared_size - 32: - raise FixtureError(f"{self.source}: nid {nid} is outside the image") - inode_format = self.u16(offset) - inode_size = 64 if inode_format & 1 else 32 - count = self.u16(offset + 2) - xattr_size = 0 if count == 0 else 12 + 4 * (count - 1) - size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8) - return Inode( - nid=nid, - offset=offset, - inode_format=inode_format, - inode_size=inode_size, - xattr_size=xattr_size, - layout=(inode_format >> 1) & 7, - mode=self.u16(offset + 4), - size=size, - start_block=self.u32(offset + 16), - ) - - def inode_data(self, inode: Inode) -> bytes: - if inode.layout == LAYOUT_FLAT_PLAIN: - start = inode.start_block << self.block_bits - end = start + inode.size - if end > self.declared_size: - raise FixtureError("plain inode data exceeds the declared image") - return bytes(self.data[start:end]) - if inode.layout != LAYOUT_FLAT_INLINE: - raise FixtureError(f"inode {inode.nid}: unsupported host layout {inode.layout}") - full_size = inode.size // self.block_size * self.block_size - if inode.size and inode.size % self.block_size == 0: - full_size -= self.block_size - full_start = inode.start_block << self.block_bits - inline_start = inode.offset + inode.inode_size + inode.xattr_size - full = self.data[full_start : full_start + full_size] - tail = self.data[inline_start : inline_start + inode.size - full_size] - if len(full) + len(tail) != inode.size: - raise FixtureError("inline inode data is truncated") - return bytes(full + tail) - - def directory_entries(self, inode: Inode) -> list[DirectoryEntry]: - data = self.inode_data(inode) - entries: list[DirectoryEntry] = [] - data_base = ( - inode.start_block << self.block_bits - if inode.layout == LAYOUT_FLAT_PLAIN - else inode.offset + inode.inode_size + inode.xattr_size - ) - for block_start in range(0, len(data), self.block_size): - block = data[block_start : block_start + self.block_size] - if len(block) < 12: - raise FixtureError("short directory block") - first_name = struct.unpack_from(" len(block): - raise FixtureError("invalid directory first-name offset") - count = first_name // 12 - for index in range(count): - item = index * 12 - name_start = struct.unpack_from(" tuple[DirectoryEntry, Inode]: - encoded = name.removeprefix("/").encode("ascii") - root = self.inode(self.root_nid) - for entry in self.directory_entries(root): - if entry.name == encoded: - return entry, self.inode(entry.nid) - raise FixtureError(f"{self.source}: root entry not found: {name}") - - def inline_xattrs(self, inode: Inode) -> tuple[int, list[XattrEntry]]: - if inode.xattr_size < 12: - raise FixtureError(f"inode {inode.nid} has no xattr header") - body = inode.offset + inode.inode_size - shared_count = self.data[body + 4] - cursor = body + 12 + shared_count * 4 - end = body + inode.xattr_size - if cursor > end: - raise FixtureError("shared xattr array exceeds inode body") - entries = [] - while cursor < end: - if cursor > end - 4: - raise FixtureError("truncated inline xattr header") - name_len = self.data[cursor] - name_index = self.data[cursor + 1] - value_len = self.u16(cursor + 2) - size = align(4 + name_len + value_len, 4) - if size > end - cursor: - raise FixtureError("inline xattr entry exceeds inode body") - entries.append( - XattrEntry( - offset=cursor, - size=size, - name_index=name_index, - name=bytes(self.data[cursor + 4 : cursor + 4 + name_len]), - value=bytes( - self.data[ - cursor + 4 + name_len : cursor + 4 + name_len + value_len - ] - ), - ) - ) - cursor += size - return shared_count, entries - - def shared_ids(self, inode: Inode) -> list[int]: - body = inode.offset + inode.inode_size - count = self.data[body + 4] - return [self.u32(body + 12 + index * 4) for index in range(count)] - - def shared_xattr(self, shared_id: int) -> XattrEntry: - offset = (self.u32(SUPER + 44) << self.block_bits) + shared_id * 4 - if offset > self.declared_size - 4: - raise FixtureError(f"shared id {shared_id} starts outside the image") - name_len = self.data[offset] - value_len = self.u16(offset + 2) - size = align(4 + name_len + value_len, 4) - if size > self.declared_size - offset: - raise FixtureError(f"shared id {shared_id} exceeds the image") - return XattrEntry( - offset=offset, - size=size, - name_index=self.data[offset + 1], - name=bytes(self.data[offset + 4 : offset + 4 + name_len]), - value=bytes( - self.data[offset + 4 + name_len : offset + 4 + name_len + value_len] - ), - ) - - def find_xattr(self, path: str, name: bytes) -> tuple[str, XattrEntry, int | None]: - _, inode = self.resolve_root(path) - _, inline = self.inline_xattrs(inode) - for entry in inline: - if entry.name == name: - return "inline", entry, None - for index, shared_id in enumerate(self.shared_ids(inode)): - entry = self.shared_xattr(shared_id) - if entry.name == name: - return "shared", entry, index - raise FixtureError(f"{path}: xattr {name!r} not found") - - def prefix_records(self) -> list[tuple[int, int, bytes]]: - count = self.data[SUPER + 91] - if count == 0: - return [] - if self.feature_compat & FEATURE_COMPAT_PLAIN_XATTR_PFX: - backing = bytes(self.data[: self.declared_size]) - else: - if self.packed_nid == 0: - raise FixtureError("prefix table has no packed backing") - backing = self.inode_data(self.inode(self.packed_nid)) - cursor = self.u32(SUPER + 92) << 2 - records = [] - for _ in range(count): - cursor = align(cursor, 4) - if cursor > len(backing) - 3: - raise FixtureError("prefix record header exceeds backing") - length = struct.unpack_from(" len(backing): - raise FixtureError("invalid prefix record") - records.append((cursor, backing[cursor + 2], backing[cursor + 3 : cursor + 2 + length])) - cursor += 2 + length - return records - - def save(self, path: Path) -> None: - path.write_bytes(self.data) - - -def xattr_entry(name_index: int, name: bytes, value: bytes) -> bytes: - raw = struct.pack(" bytes: - return struct.pack(" bytes: - entries = [(ACL_USER_OBJ, 7, ACL_UNDEFINED_ID)] - entries.extend((ACL_USER, 5, identifier) for identifier in named_users) - entries.extend( - [ - (ACL_GROUP_OBJ, 5, ACL_UNDEFINED_ID), - (ACL_MASK, 5, ACL_UNDEFINED_ID), - (ACL_OTHER, 1, ACL_UNDEFINED_ID), - ] - ) - return acl_value(entries) - - -def prefix_record(base_index: int, infix: bytes) -> bytes: - payload = bytes([base_index]) + infix - return struct.pack(" int: - start = align(offset, 4) - blob[start : start + len(payload)] = payload - return start - - -def synthetic_inode(content: bytes, shared_ids: list[int], item_value: bytes) -> bytes: - header = bytearray(12 + 4 * len(shared_ids)) - header[4] = len(shared_ids) - for index, shared_id in enumerate(shared_ids): - struct.pack_into(" tuple[bytes, dict[str, object]]: - payload = bytearray(METABOX_SIZE) - shared_offset = METABOX_XATTR_BASE - shared_records = [] - for name_index, name, value in ( - (XATTR_INDEX_USER, b"metaboxshared", b"nonzero-base"), - (XATTR_LONG_PREFIX | 0, b"setting", b"long-prefix-value"), - (XATTR_INDEX_USER, b"shared-prefix-key", b"shared-value"), - ): - raw = xattr_entry(name_index, name, value) - start = add_aligned(payload, shared_offset, raw) - shared_id = (start - METABOX_XATTR_BASE) // 4 - shared_records.append((shared_id, start, raw)) - shared_offset = start + len(raw) - - prefix_cursor = METABOX_PREFIX_OFFSET - prefix_fields = [] - for base_index, infix in ( - (XATTR_INDEX_USER, b"repo22.application.component."), - (XATTR_INDEX_TRUSTED, b"repo22.trusted.deep."), - ): - raw = prefix_record(base_index, infix) - start = add_aligned(payload, prefix_cursor, raw) - prefix_fields.append((start, base_index, infix)) - prefix_cursor = start + len(raw) - - inode_a = synthetic_inode( - b"metabox-file-a\n", - [shared_records[0][0], shared_records[1][0], shared_records[2][0]], - b"value-000", - ) - inode_b = synthetic_inode( - b"metabox-file-b\n", - [shared_records[0][0]], - b"value-001", - ) - offset_a = METABOX_INODE_A << 5 - offset_b = METABOX_INODE_B << 5 - if offset_a + len(inode_a) >= offset_b: - raise AssertionError("synthetic metabox inodes overlap") - payload[offset_a : offset_a + len(inode_a)] = inode_a - payload[offset_b : offset_b + len(inode_b)] = inode_b - - manifest = { - "size": len(payload), - "sha256": sha256_bytes(payload), - "inode_a": {"nid": METABOX_INODE_A, "offset": offset_a}, - "inode_b": {"nid": METABOX_INODE_B, "offset": offset_b}, - "xattr_base": METABOX_XATTR_BASE, - "shared": [ - { - "id": shared_id, - "offset": offset, - "sha256": sha256_bytes(raw), - } - for shared_id, offset, raw in shared_records - ], - "prefix_start": METABOX_PREFIX_OFFSET // 4, - "prefixes": [ - {"offset": offset, "base_index": base_index, "infix": infix.decode("ascii")} - for offset, base_index, infix in prefix_fields - ], - } - return bytes(payload), manifest - - -def write_file(path: Path, data: bytes = b"") -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - - -def set_user_xattr(path: Path, name: str, value: bytes) -> None: - os.setxattr(path, f"user.{name}", value) - - -def normalize_times(root: Path) -> None: - for path in sorted(root.rglob("*"), reverse=True): - os.utime(path, (0, 0), follow_symlinks=False) - os.utime(root, (0, 0), follow_symlinks=False) - - -def make_sources(output: Path) -> dict[str, object]: - source_root = output / "source" - basic = source_root / "basic" - carrier = source_root / "carrier" - basic.mkdir(parents=True) - carrier.mkdir(parents=True) - metabox_payload, metabox_manifest = make_metabox_payload() - - basic_files: dict[str, dict[str, bytes]] = { - "inline-user": { - "comment": b"inline-user-value\x00tail", - "special.chars": b"special-value", - }, - "inline-multi": { - "attr1": b"one", - "attr2": b"two\x00binary", - "attr3": bytes(range(32)), - }, - "inline-trusted": {"admin": b"trusted-inline-value"}, - "inline-security": { - "capability": bytes.fromhex("0100000200000000aabbccdd"), - "selinux": b"system_u:object_r:repo22_t:s0\x00", - }, - "corrupt-inline": {"bad": b"bad-format-target"}, - "user-subset": { - "comment": b"subset-comment", - "author": b"repo22", - "checksum": b"sha256:0123456789abcdef", - "com.repo22.app.setting": b"enabled", - }, - } - for name, attrs in basic_files.items(): - path = basic / name - write_file(path, f"{name}\n".encode("ascii")) - for xattr_name, value in attrs.items(): - set_user_xattr(path, xattr_name, value) - - for name in ("shared-a", "shared-b", "shared-multi"): - path = basic / name - write_file(path, f"{name}\n".encode("ascii")) - set_user_xattr(path, "shared_key", b"shared-value\x00exact") - set_user_xattr(path, "shared_comment", b"shared-comment") - set_user_xattr(path, "shared_binary", bytes(range(16))) - set_user_xattr(basic / "shared-a", "local", b"in-bounds-local") - - for name in ("trusted-shared-a", "trusted-shared-b", "trusted-shared-c"): - path = basic / name - write_file(path, f"{name}\n".encode("ascii")) - set_user_xattr(path, "config", b"trusted-shared-value") - - for name in ("security-shared-a", "security-shared-b", "security-shared-c"): - path = basic / name - write_file(path, f"{name}\n".encode("ascii")) - set_user_xattr(path, "selinux", b"system_u:object_r:shared_repo22_t:s0\x00") - - for index in range(4): - path = basic / f"prefix-user-{index}" - write_file(path, f"prefix user {index}\n".encode("ascii")) - set_user_xattr( - path, - "repo22.application.component.setting", - f"prefix-value-{index}".encode("ascii"), - ) - for index in range(3): - path = basic / f"prefix-trusted-{index}" - write_file(path, f"prefix trusted {index}\n".encode("ascii")) - set_user_xattr( - path, - "repo22.trusted.deep.setting", - f"trusted-prefix-value-{index}".encode("ascii"), - ) - - acl_values = { - "acl-unordered": canonical_acl(3002, 2002), - "acl-header": struct.pack(" dict[str, object]: - entries = [] - for path in sorted([root, *root.rglob("*")]): - info = path.lstat() - relative = path.relative_to(root).as_posix() or "." - item: dict[str, object] = { - "path": relative, - "mode": stat.S_IFMT(info.st_mode) | stat.S_IMODE(info.st_mode), - "uid": info.st_uid, - "gid": info.st_gid, - "size": info.st_size, - } - if path.is_file(): - item["content_sha256"] = sha256_path(path) - names = sorted(os.listxattr(path, follow_symlinks=False)) - item["xattrs"] = { - name: os.getxattr(path, name, follow_symlinks=False).hex() - for name in names - } - entries.append(item) - canonical = json.dumps(entries, separators=(",", ":"), sort_keys=True).encode("ascii") - return {"sha256": sha256_bytes(canonical), "entries": entries} - - -def tool_version() -> str: - result = subprocess.run( - ["mkfs.erofs", "-V"], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - version = result.stdout.strip() - if "erofs-utils) 1.8.6" not in version: - raise FixtureError(f"mkfs.erofs 1.8.6 is required, got: {version}") - return version - - -def run_mkfs(source: Path, image: Path, uuid: str, *extra: str) -> list[str]: - command = [ - "mkfs.erofs", - "-d0", - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "--sort=path", - "-U", - uuid, - "--xattr-prefix=user.repo22.application.component.", - "--xattr-prefix=user.repo22.trusted.deep.", - *extra, - str(image), - str(source), - ] - subprocess.run(command, check=True) - return command - - -class TransformLog: - def __init__(self) -> None: - self.records: list[dict[str, object]] = [] - - def bytes(self, image: ErofsImage, field: str, offset: int, after: bytes) -> None: - before = bytes(image.data[offset : offset + len(after)]) - if len(before) != len(after): - raise FixtureError(f"{field}: patch exceeds image") - image.data[offset : offset + len(after)] = after - self.records.append( - { - "field": field, - "offset": offset, - "size": len(after), - "before_hex": before.hex(), - "after_hex": after.hex(), - } - ) - - def u8(self, image: ErofsImage, field: str, offset: int, value: int) -> None: - self.bytes(image, field, offset, bytes([value])) - - def u16(self, image: ErofsImage, field: str, offset: int, value: int) -> None: - self.bytes(image, field, offset, struct.pack(" None: - self.bytes(image, field, offset, struct.pack(" None: - self.bytes(image, field, offset, struct.pack(" int: - for prefix_id, (_, _, record_infix) in enumerate(image.prefix_records()): - if record_infix == infix: - return prefix_id - raise FixtureError(f"prefix not found: {infix!r}") - - -def patch_namespace_entries(image: ErofsImage, log: TransformLog) -> None: - for path, name, expected_storage, new_index in ( - ("/inline-trusted", b"admin", "inline", XATTR_INDEX_TRUSTED), - ("/inline-security", b"capability", "inline", XATTR_INDEX_SECURITY), - ("/inline-security", b"selinux", "inline", XATTR_INDEX_SECURITY), - ("/trusted-shared-a", b"config", "shared", XATTR_INDEX_TRUSTED), - ("/security-shared-a", b"selinux", "shared", XATTR_INDEX_SECURITY), - ): - storage, entry, _ = image.find_xattr(path, name) - if storage != expected_storage or entry.name_index != XATTR_INDEX_USER: - raise FixtureError(f"{path}:{name!r} has unexpected storage/index") - log.u8(image, f"{path}:{name.decode()}.e_name_index", entry.offset + 1, new_index) - - trusted_prefix_id = find_prefix_id(image, b"repo22.trusted.deep.") - packed = image.inode(image.packed_nid) - packed_data = image.inode_data(packed) - record_offset = image.prefix_records()[trusted_prefix_id][0] - if packed.layout != LAYOUT_FLAT_INLINE: - raise FixtureError("expected inline packed prefix carrier") - physical = packed.offset + packed.inode_size + packed.xattr_size + record_offset + 2 - if packed_data[record_offset + 2] != XATTR_INDEX_USER: - raise FixtureError("trusted prefix base is not the user placeholder") - log.u8(image, "packed_prefix.trusted.base_index", physical, XATTR_INDEX_TRUSTED) - - -def replace_acl_entry(image: ErofsImage, path: str, value: bytes, log: TransformLog) -> None: - _, inode = image.resolve_root(path) - shared_count, entries = image.inline_xattrs(inode) - placeholders = [entry for entry in entries if entry.name == b"acl_access_placeholder"] - if shared_count != 0 or len(placeholders) != 1: - raise FixtureError(f"{path}: ACL placeholder is not one inline xattr") - body = bytearray(12) - for entry in entries: - if entry.name == b"acl_access_placeholder": - body.extend(xattr_entry(XATTR_INDEX_ACL_ACCESS, b"", value)) - else: - body.extend(xattr_entry(entry.name_index, entry.name, entry.value)) - body.extend(bytes(align(len(body), 4) - len(body))) - if len(body) > inode.xattr_size: - raise FixtureError(f"{path}: rebuilt ACL body grew") - padded = body + bytes(inode.xattr_size - len(body)) - log.bytes(image, f"{path}.xattr_body", inode.offset + inode.inode_size, padded) - icount = 1 + (len(body) - 12) // 4 - log.u16(image, f"{path}.i_xattr_icount", inode.offset + 2, icount) - - -def patch_acls(image: ErofsImage, log: TransformLog) -> None: - acl_values = { - "/acl-unordered": canonical_acl(3002, 2002), - "/acl-header": struct.pack(" None: - image.update_checksum() - image.validate_superblock() - image.save(path) - - -def transformed_basic(base: Path) -> tuple[ErofsImage, TransformLog]: - image = ErofsImage.load(base) - log = TransformLog() - patch_namespace_entries(image, log) - patch_acls(image, log) - image.update_checksum() - return image, log - - -def add_metabox_fields( - image: ErofsImage, - log: TransformLog, - carrier_nid: int, - clear_packed: bool, -) -> None: - if image.u32(SUPER + 40) != 0: - raise FixtureError("metabox conversion expects metadata in block zero") - metadata_copy = bytes(image.data[: image.block_size]) - relocated_offset = align(len(image.data), image.block_size) - image.data.extend(bytes(relocated_offset - len(image.data))) - image.data.extend(bytes(image.block_size)) - image.data[relocated_offset : relocated_offset + image.block_size] = metadata_copy - log.u32(image, "super.blocks_lo", SUPER + 36, len(image.data) // image.block_size) - log.u32(image, "super.meta_blkaddr", SUPER + 40, relocated_offset // image.block_size) - log.u32( - image, - "super.feature_compat", - SUPER + 8, - image.feature_compat | FEATURE_COMPAT_SHARED_EA_IN_METABOX, - ) - log.u8(image, "super.sb_extslots", SUPER + 13, 1) - log.u32( - image, - "super.feature_incompat", - SUPER + 80, - image.feature_incompat | FEATURE_INCOMPAT_METABOX | FEATURE_INCOMPAT_XATTR_PREFIXES, - ) - log.u32(image, "super.xattr_blkaddr", SUPER + 44, METABOX_XATTR_BASE // BLOCK_SIZE) - log.u8(image, "super.xattr_prefix_count", SUPER + 91, 2) - log.u32(image, "super.xattr_prefix_start", SUPER + 92, METABOX_PREFIX_OFFSET // 4) - log.u64(image, "super.metabox_nid", SUPER + 128, carrier_nid) - if clear_packed: - log.u64(image, "super.packed_nid", SUPER + 96, 0) - for path, nid in (("/metabox-a", METABOX_INODE_A), ("/metabox-b", METABOX_INODE_B)): - entry, _ = image.resolve_root(path) - log.u64(image, f"{path}.dirent_nid", entry.offset, METABOX_NID_BIT | nid) - - -def make_primary_prefix(image: ErofsImage, log: TransformLog) -> None: - packed_data = image.inode_data(image.inode(image.packed_nid)) - start = align(len(image.data), image.block_size) - if len(image.data) < start: - image.data.extend(bytes(start - len(image.data))) - image.data.extend(packed_data) - image.data.extend(bytes(align(len(image.data), image.block_size) - len(image.data))) - new_blocks = len(image.data) // image.block_size - log.u32(image, "super.blocks_lo", SUPER + 36, new_blocks) - log.u32( - image, - "super.feature_compat", - SUPER + 8, - image.feature_compat | FEATURE_COMPAT_PLAIN_XATTR_PFX, - ) - log.u32(image, "super.xattr_prefix_start", SUPER + 92, start // 4) - log.u64(image, "super.packed_nid", SUPER + 96, 0) - - -def save_variant( - output: Path, - name: str, - image: ErofsImage, - log: TransformLog, - transforms: dict[str, object], - base_name: str, - semantics: dict[str, object] | None = None, -) -> None: - path = output / "images" / name - checksum_and_save(image, path) - transforms[name] = { - "base": base_name, - "image_sha256": sha256_path(path), - "provider_size": path.stat().st_size, - "declared_size": image.declared_size, - "fields": log.records, - "semantics": semantics or {}, - } - - -def make_variants(output: Path, base_commands: dict[str, list[str]]) -> dict[str, object]: - bases = output / "bases" - images = output / "images" - images.mkdir() - transforms: dict[str, object] = {} - - basic, basic_log = transformed_basic(bases / "basic-base.erofs") - save_variant(output, "basic.erofs", basic.clone(), basic_log, transforms, "basic-base.erofs") - - primary = basic.clone() - primary_log = TransformLog() - make_primary_prefix(primary, primary_log) - save_variant(output, "prefix-primary.erofs", primary, primary_log, transforms, "basic.erofs") - - metabox_plain = ErofsImage.load(bases / "carrier-plain-base.erofs") - metabox_plain_log = TransformLog() - _, carrier_inode = metabox_plain.resolve_root("/metabox-carrier.bin") - if carrier_inode.layout != LAYOUT_FLAT_PLAIN or carrier_inode.size != METABOX_SIZE: - raise FixtureError("plain metabox carrier has the wrong layout or size") - add_metabox_fields(metabox_plain, metabox_plain_log, carrier_inode.nid, True) - save_variant( - output, - "metabox-plain.erofs", - metabox_plain, - metabox_plain_log, - transforms, - "carrier-plain-base.erofs", - {"carrier_nid": carrier_inode.nid, "carrier_layout": carrier_inode.layout}, - ) - - compressed = ErofsImage.load(bases / "carrier-compressed-base.erofs") - compressed_log = TransformLog() - _, compressed_carrier = compressed.resolve_root("/metabox-carrier.bin") - if compressed_carrier.layout not in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT): - raise FixtureError("compressed metabox carrier is not compressed") - add_metabox_fields(compressed, compressed_log, compressed_carrier.nid, True) - save_variant( - output, - "metabox-compressed.erofs", - compressed, - compressed_log, - transforms, - "carrier-compressed-base.erofs", - {"carrier_nid": compressed_carrier.nid, "carrier_layout": compressed_carrier.layout}, - ) - - fragment = ErofsImage.load(bases / "carrier-fragment-base.erofs") - fragment_log = TransformLog() - _, fragment_carrier = fragment.resolve_root("/metabox-carrier.bin") - if fragment_carrier.layout not in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT): - raise FixtureError("fragment metabox carrier is not compressed") - add_metabox_fields(fragment, fragment_log, fragment_carrier.nid, False) - _, fragment_carrier = fragment.resolve_root("/metabox-carrier.bin") - header_offset = align( - fragment_carrier.offset + fragment_carrier.inode_size + fragment_carrier.xattr_size, - 8, - ) - fragment_header = fragment.u64(header_offset) - if not fragment_header & METABOX_NID_BIT: - raise FixtureError("metabox carrier is not a whole-file fragment") - fragment_offset = fragment_header ^ METABOX_NID_BIT - packed_inode = fragment.inode(fragment.packed_nid) - if fragment_offset + fragment_carrier.size > packed_inode.size: - raise FixtureError("positive fragment range exceeds packed inode") - fragment_semantics = { - "carrier_nid": fragment_carrier.nid, - "carrier_layout": fragment_carrier.layout, - "fragment_header_offset": header_offset, - "fragment_header": fragment_header, - "fragment_offset": fragment_offset, - "carrier_size": fragment_carrier.size, - "packed_nid": fragment.packed_nid, - "packed_size": packed_inode.size, - } - save_variant( - output, - "metabox-fragment.erofs", - fragment, - fragment_log, - transforms, - "carrier-fragment-base.erofs", - fragment_semantics, - ) - - bad_inline = basic.clone() - bad_inline_log = TransformLog() - storage, entry, _ = bad_inline.find_xattr("/corrupt-inline", b"bad") - if storage != "inline": - raise FixtureError("corrupt-inline target is not inline") - bad_inline_log.u16(bad_inline, "/corrupt-inline.e_value_size", entry.offset + 2, 0xFFFF) - save_variant(output, "bad-inline-entry.erofs", bad_inline, bad_inline_log, transforms, "basic.erofs") - - bad_shared = basic.clone() - bad_shared_log = TransformLog() - storage, shared_entry, _ = bad_shared.find_xattr("/shared-a", b"shared_key") - if storage != "shared": - raise FixtureError("shared corruption target is not shared") - bad_shared_log.u16(bad_shared, "shared_key.e_value_size", shared_entry.offset + 2, 0xFFFF) - save_variant(output, "bad-shared-entry.erofs", bad_shared, bad_shared_log, transforms, "basic.erofs") - - shared_oob = basic.clone() - shared_oob_log = TransformLog() - _, shared_inode = shared_oob.resolve_root("/shared-a") - target_id = None - target_array_index = None - for array_index, shared_id in enumerate(shared_oob.shared_ids(shared_inode)): - if shared_oob.shared_xattr(shared_id).name == b"shared_key": - target_id = shared_id - target_array_index = array_index - break - if target_id is None or target_array_index is None: - raise FixtureError("shared OOB target ID not found") - declared_end = shared_oob.declared_size - sentinel = xattr_entry(XATTR_INDEX_USER, b"shared_key", b"sentinel-must-not-leak") - shared_oob.data.extend(bytes(BLOCK_SIZE)) - shared_oob.data[declared_end : declared_end + len(sentinel)] = sentinel - body = shared_inode.offset + shared_inode.inode_size - shared_oob_log.u32( - shared_oob, - "/shared-a.shared_key_id", - body + 12 + target_array_index * 4, - declared_end // 4, - ) - save_variant( - output, - "bad-shared-declared-bounds.erofs", - shared_oob, - shared_oob_log, - transforms, - "basic.erofs", - {"sentinel_offset": declared_end, "old_shared_id": target_id}, - ) - - prefix_oob = basic.clone() - prefix_oob_log = TransformLog() - prefix_declared_end = prefix_oob.declared_size - sentinel_prefix = prefix_record(XATTR_INDEX_USER, b"sentinel-never-visible.") - prefix_oob.data.extend(bytes(BLOCK_SIZE)) - prefix_oob.data[prefix_declared_end : prefix_declared_end + len(sentinel_prefix)] = sentinel_prefix - prefix_oob_log.u32( - prefix_oob, - "super.feature_compat", - SUPER + 8, - prefix_oob.feature_compat | FEATURE_COMPAT_PLAIN_XATTR_PFX, - ) - prefix_oob_log.u8(prefix_oob, "super.xattr_prefix_count", SUPER + 91, 1) - prefix_oob_log.u32( - prefix_oob, - "super.xattr_prefix_start", - SUPER + 92, - prefix_declared_end // 4, - ) - prefix_oob_log.u64(prefix_oob, "super.packed_nid", SUPER + 96, 0) - save_variant( - output, - "bad-prefix-declared-bounds.erofs", - prefix_oob, - prefix_oob_log, - transforms, - "basic.erofs", - {"sentinel_offset": prefix_declared_end}, - ) - - truncated = basic.clone() - truncated_log = TransformLog() - _, truncated_carrier = truncated.resolve_root("/metabox-carrier.bin") - truncated_log.u32( - truncated, - "super.feature_incompat", - SUPER + 80, - truncated.feature_incompat | FEATURE_INCOMPAT_METABOX, - ) - truncated_log.u64(truncated, "super.metabox_nid", SUPER + 128, truncated_carrier.nid) - save_variant(output, "bad-metabox-truncated-extension.erofs", truncated, truncated_log, transforms, "basic.erofs") - - ishare = basic.clone() - ishare_log = TransformLog() - ishare_log.u32( - ishare, - "super.feature_compat", - SUPER + 8, - ishare.feature_compat | FEATURE_COMPAT_ISHARE_XATTRS, - ) - ishare_log.u8( - ishare, - "super.ishare_xattr_prefix_id", - SUPER + 105, - ishare.data[SUPER + 91], - ) - save_variant(output, "bad-ishare-prefix-id.erofs", ishare, ishare_log, transforms, "basic.erofs") - - positive_fragment_path = output / "images" / "metabox-fragment.erofs" - positive_fragment = ErofsImage.load(positive_fragment_path) - positive_semantics = transforms["metabox-fragment.erofs"]["semantics"] - - self_loop = positive_fragment.clone() - self_loop_log = TransformLog() - self_loop_log.u64( - self_loop, - "super.packed_nid", - SUPER + 96, - int(positive_semantics["carrier_nid"]), - ) - save_variant(output, "bad-fragment-self-loop.erofs", self_loop, self_loop_log, transforms, "metabox-fragment.erofs") - - range_bad = positive_fragment.clone() - range_log = TransformLog() - range_log.u64( - range_bad, - "metabox.fragment_header", - int(positive_semantics["fragment_header_offset"]), - METABOX_NID_BIT | int(positive_semantics["packed_size"]), - ) - save_variant(output, "bad-fragment-range.erofs", range_bad, range_log, transforms, "metabox-fragment.erofs") - - recursive_metabox = positive_fragment.clone() - recursive_metabox_log = TransformLog() - recursive_metabox_log.u64( - recursive_metabox, - "super.metabox_nid", - SUPER + 128, - METABOX_NID_BIT | int(positive_semantics["carrier_nid"]), - ) - save_variant( - output, - "bad-metabox-recursive-nid.erofs", - recursive_metabox, - recursive_metabox_log, - transforms, - "metabox-fragment.erofs", - ) - - recursive_packed = positive_fragment.clone() - recursive_packed_log = TransformLog() - recursive_packed_log.u64( - recursive_packed, - "super.packed_nid", - SUPER + 96, - METABOX_NID_BIT | int(positive_semantics["packed_nid"]), - ) - save_variant( - output, - "bad-packed-recursive-nid.erofs", - recursive_packed, - recursive_packed_log, - transforms, - "metabox-fragment.erofs", - ) - - return {"base_commands": base_commands, "images": transforms} - - -def write_checksums(output: Path) -> None: - paths = sorted((output / "bases").glob("*.erofs")) + sorted((output / "images").glob("*.erofs")) - with (output / "IMAGE-SHA256SUMS").open("w", encoding="ascii") as sums: - for path in paths: - sums.write(f"{sha256_path(path)} {path.relative_to(output)}\n") - - -def verify_output(output: Path) -> None: - manifest = json.loads((output / "fixture-manifest.json").read_text(encoding="ascii")) - inventory = source_inventory(output / "source") - if inventory["sha256"] != manifest["source"]["sha256"]: - raise FixtureError("source inventory hash changed") - for name, item in manifest["transforms"]["images"].items(): - path = output / "images" / name - if sha256_path(path) != item["image_sha256"]: - raise FixtureError(f"{name}: image hash changed") - data = path.read_bytes() - for field in item["fields"]: - offset = field["offset"] - size = field["size"] - if data[offset : offset + size].hex() != field["after_hex"]: - raise FixtureError(f"{name}: field check failed: {field['field']}") - image = ErofsImage.load(path) - if image.declared_size != item["declared_size"]: - raise FixtureError(f"{name}: declared size changed") - if len(data) != item["provider_size"]: - raise FixtureError(f"{name}: provider size changed") - checksums = {} - for line in (output / "IMAGE-SHA256SUMS").read_text(encoding="ascii").splitlines(): - digest, relative = line.split(" ", 1) - checksums[relative] = digest - for relative, digest in checksums.items(): - if sha256_path(output / relative) != digest: - raise FixtureError(f"checksum list mismatch: {relative}") - - -def make(output: Path) -> None: - if shutil.which("mkfs.erofs") is None: - raise FixtureError("mkfs.erofs is required") - if output.exists(): - raise FixtureError(f"output already exists: {output}") - output.mkdir(parents=True) - version = tool_version() - source_details = make_sources(output) - source = source_inventory(output / "source") - bases = output / "bases" - bases.mkdir() - base_commands = { - "basic-base.erofs": run_mkfs( - output / "source" / "basic", - bases / "basic-base.erofs", - "44444444-4444-4444-8444-444444444404", - ), - "carrier-plain-base.erofs": run_mkfs( - output / "source" / "carrier", - bases / "carrier-plain-base.erofs", - "44444444-4444-4444-8444-444444444434", - ), - "carrier-compressed-base.erofs": run_mkfs( - output / "source" / "carrier", - bases / "carrier-compressed-base.erofs", - "44444444-4444-4444-8444-444444444440", - "-zlz4", - "-C4096", - "-Elegacy-compress", - ), - "carrier-fragment-base.erofs": run_mkfs( - output / "source" / "carrier", - bases / "carrier-fragment-base.erofs", - "44444444-4444-4444-8444-444444444442", - "-zlz4", - "-C4096", - "-Eall-fragments", - ), - } - transforms = make_variants(output, base_commands) - write_checksums(output) - manifest = { - "generator": "g4_fixtures.py", - "mkfs_version": version, - "source": source, - "source_details": source_details, - "transforms": transforms, - } - (output / "fixture-manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - (output / "SOURCE-SHA256").write_text(f"{source['sha256']} source-inventory\n", encoding="ascii") - (output / "mkfs-version.txt").write_text(version + "\n", encoding="ascii") - verify_output(output) - print(f"source_sha256={source['sha256']}") - print((output / "IMAGE-SHA256SUMS").read_text(encoding="ascii"), end="") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - make_parser = subparsers.add_parser("make") - make_parser.add_argument("--output", required=True, type=Path) - verify_parser = subparsers.add_parser("verify") - verify_parser.add_argument("--output", required=True, type=Path) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - if args.command == "make": - make(args.output) - else: - verify_output(args.output) - - -if __name__ == "__main__": - try: - main() - except (FixtureError, OSError, subprocess.CalledProcessError) as error: - raise SystemExit(f"g4_fixtures.py: {error}") from error diff --git a/tests/g5_fixtures.py b/tests/g5_fixtures.py deleted file mode 100644 index e633e42..0000000 --- a/tests/g5_fixtures.py +++ /dev/null @@ -1,1109 +0,0 @@ -#!/usr/bin/env python3 -"""Generate and verify deterministic repo22 G5 compression fixtures.""" - -from __future__ import annotations - -import argparse -from collections import Counter -import hashlib -import json -from pathlib import Path -import re -import struct -import subprocess -from typing import Any - -from erofs_fixture import ErofsImage, Inode, SUPER, sha256 - - -BLOCK_SIZE = 4096 -FIXED_UUID = "00000000-0000-0000-0000-000000000000" -FEATURE_INCOMPAT_COMPR_HEAD2 = 0x00000008 -LAYOUT_COMPRESSED_FULL = 1 -LAYOUT_COMPRESSED_COMPACT = 3 -Z_EROFS_ADVISE_COMPACTED_2B = 0x0001 -Z_EROFS_ADVISE_BIG_PCLUSTER_1 = 0x0002 -Z_EROFS_ADVISE_BIG_PCLUSTER_2 = 0x0004 -Z_EROFS_ADVISE_INLINE_PCLUSTER = 0x0008 -Z_EROFS_ADVISE_INTERLACED_PCLUSTER = 0x0010 -Z_EROFS_LCLUSTER_TYPE_MASK = 0x0003 -Z_EROFS_LCLUSTER_TYPE_PLAIN = 0 -Z_EROFS_LCLUSTER_TYPE_HEAD1 = 1 -Z_EROFS_LCLUSTER_TYPE_NONHEAD = 2 -Z_EROFS_LCLUSTER_TYPE_HEAD2 = 3 -Z_EROFS_LI_PARTIAL_REF = 0x8000 -Z_EROFS_LI_D0_CBLKCNT = 0x0800 -ALGORITHMS = { - "lz4": 0, - "lzma": 1, - "deflate": 2, - "zstd": 3, -} - - -class FixtureError(RuntimeError): - pass - - -def align(value: int, alignment: int) -> int: - return (value + alignment - 1) & -alignment - - -def relative(path: Path, root: Path) -> str: - return str(path.relative_to(root)) - - -def run( - command: list[str], - *, - log: Path | None = None, - check: bool = True, -) -> subprocess.CompletedProcess[str]: - result = subprocess.run( - command, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - if log is not None: - log.write_text( - "$ " + " ".join(command) + "\n" + result.stdout, - encoding="utf-8", - ) - if check and result.returncode != 0: - raise FixtureError( - f"command failed ({result.returncode}): {' '.join(command)}\n" - f"{result.stdout}" - ) - return result - - -def write_repeated(path: Path, size: int, label: str) -> None: - token = (label.encode("ascii") + b"\n") * 64 - chunk = (token * ((1024 * 1024 + len(token) - 1) // len(token)))[ - : 1024 * 1024 - ] - remaining = size - with path.open("wb") as output: - while remaining: - amount = min(remaining, len(chunk)) - output.write(chunk[:amount]) - remaining -= amount - - -def mixed_chunk(seed: str, chunk_index: int, size: int) -> bytes: - output = bytearray() - block_index = 0 - while len(output) < size: - amount = min(BLOCK_SIZE, size - len(output)) - if block_index % 4 == 0: - material = hashlib.shake_256( - f"{seed}:{chunk_index}:{block_index}".encode("ascii") - ).digest(amount) - else: - token = ( - f"repo22-g5:{seed}:{chunk_index % 7}:{block_index % 11}\n" - ).encode("ascii") - material = (token * ((amount + len(token) - 1) // len(token)))[ - :amount - ] - output.extend(material) - block_index += 1 - return bytes(output) - - -def write_mixed(path: Path, size: int, seed: str) -> None: - chunk_size = 1024 * 1024 - remaining = size - index = 0 - with path.open("wb") as output: - while remaining: - amount = min(remaining, chunk_size) - output.write(mixed_chunk(seed, index, amount)) - remaining -= amount - index += 1 - - -def write_interlaced(path: Path, size: int) -> None: - remaining = size - extent = 0 - with path.open("wb") as output: - while remaining: - compressible_size = min(remaining, 16384) - token = f"repo22-interlaced-{extent % 13:02d}\n".encode("ascii") - output.write( - (token * ((compressible_size + len(token) - 1) // len(token)))[ - :compressible_size - ] - ) - remaining -= compressible_size - if not remaining: - break - random_size = min(remaining, BLOCK_SIZE) - output.write( - hashlib.shake_256( - f"repo22-interlaced-random-{extent}".encode("ascii") - ).digest(random_size) - ) - remaining -= random_size - extent += 1 - - -def create_sources(source_root: Path) -> None: - directories = { - "lz4": source_root / "lz4", - "large": source_root / "large", - "levels": source_root / "levels", - "lzma-large": source_root / "lzma-large", - "microlzma": source_root / "microlzma", - "partial": source_root / "partial", - "partial-deflate": source_root / "partial-deflate", - "shape": source_root / "shape", - "ztail": source_root / "ztail", - } - for directory in directories.values(): - directory.mkdir(parents=True) - - write_mixed(directories["lz4"] / "compressed.bin", 8 * 1024 * 1024, "lz4") - write_mixed( - directories["large"] / "large.bin", 256 * 1024 * 1024, "large" - ) - write_mixed( - directories["levels"] / "level.dat", 8 * 1024 * 1024, "levels" - ) - write_mixed( - directories["lzma-large"] / "large.bin", - 100 * 1024 * 1024 + 1, - "lzma-large", - ) - (directories["microlzma"] / "one-byte.bin").write_bytes(b"G") - write_repeated( - directories["microlzma"] / "block-4k.bin", BLOCK_SIZE, "microlzma-4k" - ) - write_repeated( - directories["microlzma"] / "boundary-16k.bin", - 4 * BLOCK_SIZE, - "microlzma-16k", - ) - write_repeated( - directories["partial"] / "a.dat", 1024 * 1024, "partial-stream" - ) - source_a = directories["partial"] / "a.dat" - source_b = directories["partial"] / "b.dat" - with source_a.open("rb") as source, source_b.open("wb") as target: - target.write(source.read(700000)) - write_mixed( - directories["partial"] / "control.bin", - 32768, - "partial-control", - ) - write_repeated( - directories["partial-deflate"] / "a.dat", - 1024 * 1024, - "partial-stream", - ) - with (directories["partial-deflate"] / "a.dat").open("rb") as source, ( - directories["partial-deflate"] / "b.dat" - ).open("wb") as target: - target.write(source.read(100000)) - write_mixed( - directories["partial-deflate"] / "control.bin", - 32768, - "partial-control", - ) - write_interlaced(directories["shape"] / "shape.dat", 1024 * 1024) - write_repeated(directories["ztail"] / "inline.dat", 131071, "ztail-inline") - write_mixed( - directories["ztail"] / "exact-pcluster.dat", BLOCK_SIZE, "ztail-exact" - ) - write_repeated( - directories["ztail"] / "one-byte-tail.dat", - BLOCK_SIZE + 1, - "ztail-one-byte", - ) - write_repeated( - directories["ztail"] / "max-tail.dat", - BLOCK_SIZE * 2 - 1, - "ztail-max", - ) - (directories["ztail"] / "zero-tail.dat").write_bytes(b"") - - -def source_inventory(source_root: Path) -> list[dict[str, Any]]: - inventory = [] - for path in sorted(item for item in source_root.rglob("*") if item.is_file()): - inventory.append( - { - "path": relative(path, source_root.parent), - "size": path.stat().st_size, - "sha256": sha256(path), - } - ) - return inventory - - -def mkfs_image( - *, - output_root: Path, - image_name: str, - source: Path, - options: list[str], - command_log: list[dict[str, Any]], -) -> Path: - image = output_root / "images" / image_name - command = [ - "mkfs.erofs", - "--workers=1", - "--sort=path", - "-T0", - "--all-time", - f"-U{FIXED_UUID}", - "--all-root", - *options, - str(image), - str(source), - ] - log = output_root / "logs" / f"mkfs-{image.stem}.log" - run(command, log=log) - command_log.append( - { - "image": relative(image, output_root), - "source": relative(source, output_root), - "options": options, - "log": relative(log, output_root), - } - ) - parsed = ErofsImage.load(image) - parsed.validate_superblock() - return image - - -def inode_for(image: ErofsImage, path: str) -> Inode: - _, entry = image.resolve_root_entry(path) - return image.inode(entry.nid) - - -def map_header(image: ErofsImage, inode: Inode) -> dict[str, int]: - if inode.layout not in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT): - raise FixtureError(f"inode layout {inode.layout} is not compressed") - offset = align(inode.offset + inode.inode_size + inode.xattr_size, 8) - if offset > len(image.data) - 8: - raise FixtureError("compressed map header exceeds image") - raw0, advise, algorithm, clusterbits = struct.unpack_from( - "> 4, - "clusterbits_raw": clusterbits, - "lcluster_bits": image.block_bits + (clusterbits & 0x07), - "idata_size": raw0 >> 16, - } - - -def full_index_offset(image: ErofsImage, inode: Inode) -> int: - header = map_header(image, inode) - return align(header["offset"] + 8, 8) + 8 - - -def full_record(image: ErofsImage, inode: Inode, lcn: int) -> dict[str, int]: - if inode.layout != LAYOUT_COMPRESSED_FULL: - raise FixtureError("full record requested from a non-full inode") - offset = full_index_offset(image, inode) + lcn * 8 - if offset > len(image.data) - 8: - raise FixtureError("full index record exceeds image") - advise, clusterofs, word = struct.unpack_from("> 16, - } - - -def full_index_summary(image: ErofsImage, inode: Inode) -> dict[str, Any]: - header = map_header(image, inode) - count = (inode.size + (1 << header["lcluster_bits"]) - 1) >> header[ - "lcluster_bits" - ] - types: Counter[int] = Counter() - heads = [] - partial_heads = [] - for lcn in range(count): - record = full_record(image, inode, lcn) - types[record["type"]] += 1 - if record["type"] in ( - Z_EROFS_LCLUSTER_TYPE_PLAIN, - Z_EROFS_LCLUSTER_TYPE_HEAD1, - Z_EROFS_LCLUSTER_TYPE_HEAD2, - ): - if len(heads) < 16: - heads.append( - { - "lcn": lcn, - "offset": record["offset"], - "type": record["type"], - "pblk": record["pblk"], - "clusterofs": record["clusterofs"], - } - ) - if record["partial_ref"]: - partial_heads.append( - { - "lcn": lcn, - "offset": record["offset"], - "pblk": record["pblk"], - } - ) - return { - "record_count": count, - "type_counts": {str(key): value for key, value in sorted(types.items())}, - "first_heads": heads, - "partial_heads": partial_heads, - } - - -EXTENT_RE = re.compile( - r"^\s*\d+:\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:" - r"\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$" -) - - -def dump_extents(image: Path, path: str, log: Path | None = None) -> list[dict[str, int]]: - result = run( - ["dump.erofs", f"--path={path}", "-e", str(image)], - log=log, - ) - extents = [] - for line in result.stdout.splitlines(): - match = EXTENT_RE.match(line) - if match is None: - continue - logical_start, logical_end, logical_length, physical_start, physical_end, physical_length = ( - int(value) for value in match.groups() - ) - if logical_end - logical_start != logical_length: - raise FixtureError("dump.erofs reported an inconsistent logical extent") - if physical_end - physical_start != physical_length: - raise FixtureError("dump.erofs reported an inconsistent physical extent") - extents.append( - { - "logical_start": logical_start, - "logical_end": logical_end, - "logical_length": logical_length, - "physical_start": physical_start, - "physical_end": physical_end, - "physical_length": physical_length, - } - ) - if not extents: - raise FixtureError(f"dump.erofs reported no extents for {image}:{path}") - return extents - - -def extent_summary(extents: list[dict[str, int]]) -> dict[str, Any]: - transitions = [] - for index in range(1, len(extents)): - previous = extents[index - 1] - current = extents[index] - previous_plain = previous["logical_length"] == previous["physical_length"] - current_plain = current["logical_length"] == current["physical_length"] - if previous_plain != current_plain: - transitions.append( - { - "logical_offset": current["logical_start"], - "previous_plain": previous_plain, - "current_plain": current_plain, - } - ) - return { - "count": len(extents), - "first": extents[:8], - "last": extents[-1], - "max_logical_length": max(item["logical_length"] for item in extents), - "max_physical_length": max(item["physical_length"] for item in extents), - "plain_count": sum( - item["physical_length"] != 0 - and item["logical_length"] == item["physical_length"] - for item in extents - ), - "compressed_count": sum( - item["physical_length"] != 0 - and item["logical_length"] > item["physical_length"] - for item in extents - ), - "hole_or_fragment_count": sum( - item["physical_length"] == 0 for item in extents - ), - "transitions": transitions[:8], - } - - -def inspect_path( - output_root: Path, - image_path: Path, - path: str, - *, - write_log: bool = True, -) -> dict[str, Any]: - image = ErofsImage.load(image_path) - image.validate_superblock() - inode = inode_for(image, path) - details: dict[str, Any] = { - "path": path, - "nid": inode.nid, - "inode_offset": inode.offset, - "inode_size": inode.inode_size, - "xattr_size": inode.xattr_size, - "layout": inode.layout, - "size": inode.size, - } - if inode.layout in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT): - details["map_header"] = map_header(image, inode) - log = None - if write_log: - safe_path = path.removeprefix("/").replace("/", "-") - log = output_root / "logs" / f"dump-{image_path.stem}-{safe_path}.log" - details["dump_log"] = relative(log, output_root) - details["extents"] = extent_summary(dump_extents(image_path, path, log)) - if inode.layout == LAYOUT_COMPRESSED_FULL: - details["full_index"] = full_index_summary(image, inode) - return details - - -def assert_compressed( - details: dict[str, Any], - *, - algorithm: str, - layout: int | None = None, -) -> None: - if details["layout"] not in (LAYOUT_COMPRESSED_FULL, LAYOUT_COMPRESSED_COMPACT): - raise FixtureError(f"{details['path']} was not compressed") - if layout is not None and details["layout"] != layout: - raise FixtureError( - f"{details['path']} layout {details['layout']} != expected {layout}" - ) - actual = details["map_header"]["head1_algorithm"] - if actual != ALGORITHMS[algorithm]: - raise FixtureError( - f"{details['path']} algorithm {actual} != {ALGORITHMS[algorithm]}" - ) - - -def patch_partial_reference( - base: Path, - output: Path, - *, - algorithm: str, -) -> dict[str, Any]: - image = ErofsImage.load(base) - inode_a = inode_for(image, "/a.dat") - inode_b = inode_for(image, "/b.dat") - if inode_a.layout != LAYOUT_COMPRESSED_FULL or inode_b.layout != LAYOUT_COMPRESSED_FULL: - raise FixtureError("partial-reference transform requires full indexes") - header_a = map_header(image, inode_a) - header_b = map_header(image, inode_b) - expected_algorithm = ALGORITHMS[algorithm] - if ( - header_a["head1_algorithm"] != expected_algorithm - or header_b["head1_algorithm"] != expected_algorithm - ): - raise FixtureError("partial-reference source algorithm mismatch") - record_a = full_record(image, inode_a, 0) - record_b = full_record(image, inode_b, 0) - record_a_next = full_record(image, inode_a, 1) - record_b_next = full_record(image, inode_b, 1) - if record_a["type"] != Z_EROFS_LCLUSTER_TYPE_HEAD1: - raise FixtureError("a.dat first record is not HEAD1") - if record_b["type"] != Z_EROFS_LCLUSTER_TYPE_HEAD1: - raise FixtureError("b.dat first record is not HEAD1") - if record_a["pblk"] == record_b["pblk"]: - raise FixtureError("mkfs unexpectedly reused the partial pcluster") - if record_b["advise"] & Z_EROFS_LI_PARTIAL_REF: - raise FixtureError("b.dat was already marked partial") - if inode_b.size >= inode_a.size: - raise FixtureError("partial file is not shorter than the complete stream") - if ( - record_a_next["type"] != Z_EROFS_LCLUSTER_TYPE_NONHEAD - or record_b_next["type"] != Z_EROFS_LCLUSTER_TYPE_NONHEAD - or not record_a_next["delta0"] & Z_EROFS_LI_D0_CBLKCNT - or not record_b_next["delta0"] & Z_EROFS_LI_D0_CBLKCNT - ): - raise FixtureError("partial source lacks a first-pcluster block count") - - struct.pack_into( - " dict[str, Any]: - image = ErofsImage.load(base) - inode = inode_for(image, "/shape.dat") - if inode.layout != LAYOUT_COMPRESSED_FULL: - raise FixtureError("HEAD2 transform requires full indexes") - header = map_header(image, inode) - if header["head1_algorithm"] != ALGORITHMS["lz4"]: - raise FixtureError("HEAD2 base is not LZ4") - if not header["advise"] & Z_EROFS_ADVISE_BIG_PCLUSTER_1: - raise FixtureError("HEAD2 base lacks HEAD1 big-pcluster advise") - - count = (inode.size + (1 << header["lcluster_bits"]) - 1) >> header[ - "lcluster_bits" - ] - target_lcn = None - target = None - for lcn in range(count): - record = full_record(image, inode, lcn) - if record["type"] == Z_EROFS_LCLUSTER_TYPE_HEAD1: - target_lcn = lcn - target = record - break - if target_lcn is None or target is None: - raise FixtureError("HEAD2 base has no HEAD1 record") - - old_incompat = image.feature_incompat - old_advise = header["advise"] - old_algorithm = header["algorithm_raw"] - new_advise = old_advise | Z_EROFS_ADVISE_BIG_PCLUSTER_2 - new_algorithm = (old_algorithm & 0x0F) | ((old_algorithm & 0x0F) << 4) - new_record_advise = ( - target["advise"] & ~Z_EROFS_LCLUSTER_TYPE_MASK - ) | Z_EROFS_LCLUSTER_TYPE_HEAD2 - image.put_u32(SUPER + 80, old_incompat | FEATURE_INCOMPAT_COMPR_HEAD2) - struct.pack_into(" dict[str, Any]: - source = ErofsImage.load(image_path) - inode = inode_for(source, path) - header = map_header(source, inode) - if header["head1_algorithm"] != ALGORITHMS[algorithm]: - raise FixtureError("corruption target algorithm mismatch") - extents = dump_extents(image_path, path) - target = next( - ( - extent - for extent in extents - if extent["physical_length"] > 0 - and extent["logical_length"] > extent["physical_length"] - ), - None, - ) - if target is None: - raise FixtureError("no compressed extent available for corruption") - start = target["physical_start"] - end = target["physical_end"] - if start < source.checksum_end or end > len(source.data): - raise FixtureError("target compressed extent is outside payload bounds") - payload = source.data[start:end] - nonzero = [index for index, value in enumerate(payload) if value != 0] - if len(nonzero) < 16: - raise FixtureError("target compressed extent has too little encoded data") - patch_start = start + nonzero[0] - patch_length = min(64, end - patch_start) - old = bytes(source.data[patch_start : patch_start + patch_length]) - for index in range(patch_start, patch_start + patch_length): - source.data[index] ^= 0xA5 - new = bytes(source.data[patch_start : patch_start + patch_length]) - if old == new: - raise AssertionError("compressed corruption did not change bytes") - if not source.checksum_valid(): - raise FixtureError("payload corruption invalidated the superblock checksum") - source.save(output) - reopened = ErofsImage.load(output) - reopened.validate_superblock() - return { - "path": path, - "algorithm": algorithm, - "algorithm_id": ALGORITHMS[algorithm], - "nid": inode.nid, - "map_header_offset": header["offset"], - "extent_logical_start": target["logical_start"], - "extent_logical_length": target["logical_length"], - "pcluster_start": start, - "pcluster_length": target["physical_length"], - "patch_offset": patch_start, - "patch_length": patch_length, - "old_sha256": hashlib.sha256(old).hexdigest(), - "new_sha256": hashlib.sha256(new).hexdigest(), - "superblock_checksum_valid": True, - } - - -def write_checksum_file(root: Path, paths: list[Path], name: str) -> None: - with (root / name).open("w", encoding="ascii") as output: - for path in sorted(paths): - output.write(f"{sha256(path)} {relative(path, root)}\n") - - -def explicit_probe( - output_root: Path, - attempt_image: Path, - utils_source: Path | None, -) -> dict[str, Any]: - attempt = inspect_path(output_root, attempt_image, "/shape.dat") - header = attempt["map_header"] - tokens = [ - "Z_EROFS_ADVISE_EXTENTS", - "z_erofs_extent_recsize", - "struct z_erofs_extent {", - ] - matches: dict[str, list[str]] = {token: [] for token in tokens} - if utils_source is not None: - for base in (utils_source / "include", utils_source / "lib"): - if not base.is_dir(): - continue - for path in sorted(item for item in base.rglob("*") if item.is_file()): - try: - content = path.read_text(encoding="utf-8", errors="ignore") - except OSError: - continue - for token in tokens: - if token in content: - matches[token].append(str(path)) - return { - "status": "SHELVED", - "attempt_image": relative(attempt_image, output_root), - "attempt_layout": attempt["layout"], - "attempt_map_header_offset": header["offset"], - "attempt_h_advise": header["advise"], - "attempt_explicit_bit": bool( - attempt["layout"] == LAYOUT_COMPRESSED_FULL - and header["advise"] & Z_EROFS_ADVISE_COMPACTED_2B - ), - "mapped_payload_generated": False, - "erofs_utils_source": str(utils_source) if utils_source else None, - "source_token_matches": matches, - "structured_transform_attempt": ( - "Refused: converting legacy lcluster indexes into variable-size " - "extent records requires relocating subsequent metadata and payload; " - "the helper cannot validate a mapped result with erofs-utils 1.8.6." - ), - } - - -def build_fixtures(args: argparse.Namespace) -> None: - output_root = args.output.resolve() - if output_root.exists(): - raise FixtureError(f"output already exists: {output_root}") - (output_root / "images").mkdir(parents=True) - (output_root / "logs").mkdir() - (output_root / "sources").mkdir() - (output_root / "base-images").mkdir() - create_sources(output_root / "sources") - - version = run(["mkfs.erofs", "-V"]).stdout.strip() - if "1.8.6" not in version: - raise FixtureError(f"mkfs.erofs 1.8.6 is required, got: {version}") - - commands: list[dict[str, Any]] = [] - images: dict[str, Path] = {} - targets: dict[str, dict[str, Any]] = {} - - def generate( - name: str, - source_dir: str, - options: list[str], - path: str, - algorithm: str, - layout: int | None = None, - ) -> dict[str, Any]: - image = mkfs_image( - output_root=output_root, - image_name=name, - source=output_root / "sources" / source_dir, - options=options, - command_log=commands, - ) - images[name] = image - details = inspect_path(output_root, image, path) - assert_compressed(details, algorithm=algorithm, layout=layout) - targets[f"{name}:{path}"] = details - return details - - compact_4k = generate( - "lz4-compact-4k.erofs", - "lz4", - ["-zlz4", "-C4096"], - "/compressed.bin", - "lz4", - LAYOUT_COMPRESSED_COMPACT, - ) - full_4k = generate( - "lz4-full-4k.erofs", - "lz4", - ["-zlz4", "-C4096", "-Elegacy-compress"], - "/compressed.bin", - "lz4", - LAYOUT_COMPRESSED_FULL, - ) - compact_64k = generate( - "lz4-compact-64k.erofs", - "lz4", - ["-zlz4", "-C65536"], - "/compressed.bin", - "lz4", - LAYOUT_COMPRESSED_COMPACT, - ) - compact_256k = generate( - "lz4-compact-256k.erofs", - "lz4", - ["-zlz4", "-C262144"], - "/compressed.bin", - "lz4", - LAYOUT_COMPRESSED_COMPACT, - ) - generate( - "lz4-large.erofs", - "large", - ["-zlz4", "-C65536"], - "/large.bin", - "lz4", - ) - ztail = generate( - "lz4-ztail.erofs", - "ztail", - ["-zlz4", "-C4096", "-Eztailpacking"], - "/inline.dat", - "lz4", - ) - for path in ( - "/exact-pcluster.dat", - "/one-byte-tail.dat", - "/max-tail.dat", - "/zero-tail.dat", - ): - details = inspect_path(output_root, images["lz4-ztail.erofs"], path) - targets[f"lz4-ztail.erofs:{path}"] = details - - if not compact_4k["map_header"]["advise"] & Z_EROFS_ADVISE_COMPACTED_2B: - raise FixtureError("4K compact image lacks compacted index advise") - if full_4k["map_header"]["advise"] & Z_EROFS_ADVISE_COMPACTED_2B: - raise FixtureError("full-index image advertises compacted indexes") - for details, requested in ((compact_64k, 65536), (compact_256k, 262144)): - header = details["map_header"] - if not header["advise"] & Z_EROFS_ADVISE_BIG_PCLUSTER_1: - raise FixtureError(f"{requested} image lacks big-pcluster advise") - if details["extents"]["max_physical_length"] <= BLOCK_SIZE: - raise FixtureError(f"{requested} image has no multi-block pcluster") - if not ztail["map_header"]["advise"] & Z_EROFS_ADVISE_INLINE_PCLUSTER: - raise FixtureError("ztailpacking target lacks inline-pcluster advise") - if ztail["map_header"]["idata_size"] == 0: - raise FixtureError("ztailpacking target has zero inline encoded size") - - for level in (1, 6, 9): - generate( - f"deflate-level{level}.erofs", - "levels", - [f"-zdeflate,level={level}", "-C65536", "-Elegacy-compress"], - "/level.dat", - "deflate", - LAYOUT_COMPRESSED_FULL, - ) - for level in (1, 15, 22): - generate( - f"zstd-level{level}.erofs", - "levels", - [f"-zzstd,level={level}", "-C65536", "-Elegacy-compress"], - "/level.dat", - "zstd", - LAYOUT_COMPRESSED_FULL, - ) - generate( - "lzma-level6.erofs", - "levels", - ["-zlzma,level=6", "-C65536", "-Elegacy-compress"], - "/level.dat", - "lzma", - LAYOUT_COMPRESSED_FULL, - ) - generate( - "lzma-large.erofs", - "lzma-large", - ["-zlzma,level=6", "-C65536", "-Elegacy-compress"], - "/large.bin", - "lzma", - LAYOUT_COMPRESSED_FULL, - ) - generate( - "microlzma-edge.erofs", - "microlzma", - ["-zlzma,level=6", "-C4096", "-Elegacy-compress"], - "/boundary-16k.bin", - "lzma", - LAYOUT_COMPRESSED_FULL, - ) - for path in ("/one-byte.bin", "/block-4k.bin"): - details = inspect_path(output_root, images["microlzma-edge.erofs"], path) - targets[f"microlzma-edge.erofs:{path}"] = details - - partials: dict[str, Any] = {} - corruptions: dict[str, Any] = {} - for algorithm, compressor in ( - ("deflate", "-zdeflate,level=1"), - ("zstd", "-zzstd,level=1"), - ("lzma", "-zlzma,level=6"), - ): - base_name = f"{algorithm}-partial-base.erofs" - base = mkfs_image( - output_root=output_root, - image_name=f"../base-images/{base_name}", - source=output_root - / "sources" - / ("partial-deflate" if algorithm == "deflate" else "partial"), - options=[compressor, "-C1048576", "-Elegacy-compress"], - command_log=commands, - ) - valid = output_root / "images" / f"{algorithm}-partial-ref.erofs" - partials[algorithm] = patch_partial_reference( - base, valid, algorithm=algorithm - ) - images[valid.name] = valid - for path in ("/a.dat", "/b.dat"): - details = inspect_path(output_root, valid, path) - assert_compressed( - details, - algorithm=algorithm, - layout=LAYOUT_COMPRESSED_FULL, - ) - targets[f"{valid.name}:{path}"] = details - corrupt = output_root / "images" / f"{algorithm}-partial-ref-corrupt.erofs" - corruptions[algorithm] = corrupt_target_extent( - valid, - corrupt, - path="/a.dat", - algorithm=algorithm, - ) - images[corrupt.name] = corrupt - - head2_base = mkfs_image( - output_root=output_root, - image_name="../base-images/head2-base.erofs", - source=output_root / "sources" / "shape", - options=["-zlz4", "-C65536", "-Elegacy-compress"], - command_log=commands, - ) - head2 = output_root / "images" / "head2.erofs" - head2_transform = patch_head2(head2_base, head2) - images[head2.name] = head2 - head2_details = inspect_path(output_root, head2, "/shape.dat") - assert_compressed( - head2_details, - algorithm="lz4", - layout=LAYOUT_COMPRESSED_FULL, - ) - targets[f"{head2.name}:/shape.dat"] = head2_details - head2_corrupt = output_root / "images" / "head2-corrupt.erofs" - corruptions["head2"] = corrupt_target_extent( - head2, - head2_corrupt, - path="/shape.dat", - algorithm="lz4", - ) - images[head2_corrupt.name] = head2_corrupt - - interlaced = generate( - "interlaced.erofs", - "shape", - ["-zlz4", "-C4096", "-Efragments"], - "/shape.dat", - "lz4", - ) - if not interlaced["map_header"]["advise"] & Z_EROFS_ADVISE_INTERLACED_PCLUSTER: - raise FixtureError("interlaced target lacks interlaced advise") - if not interlaced["extents"]["transitions"]: - raise FixtureError("interlaced target has no compressed/plain transition") - - extent_attempt = generate( - "extent-attempt.erofs", - "shape", - [ - "-zlz4", - "-C65536", - "-Elegacy-compress", - "--max-extent-bytes=65536", - ], - "/shape.dat", - "lz4", - LAYOUT_COMPRESSED_FULL, - ) - if extent_attempt["map_header"]["advise"] & Z_EROFS_ADVISE_COMPACTED_2B: - raise FixtureError("extent attempt unexpectedly selected the explicit bit") - - image_paths = sorted(set(images.values())) - source_paths = sorted( - path for path in (output_root / "sources").rglob("*") if path.is_file() - ) - write_checksum_file(output_root, image_paths, "SHA256SUMS") - write_checksum_file(output_root, source_paths, "SOURCE-SHA256SUMS") - - manifest = { - "schema": 1, - "mkfs_version": version, - "fixed_uuid": FIXED_UUID, - "source_inventory": source_inventory(output_root / "sources"), - "commands": commands, - "images": { - relative(path, output_root): { - "size": path.stat().st_size, - "sha256": sha256(path), - } - for path in image_paths - }, - "targets": targets, - "partial_references": partials, - "corruptions": corruptions, - "head2_transform": head2_transform, - "explicit_extent": explicit_probe( - output_root, - images["extent-attempt.erofs"], - args.erofs_utils_source.resolve() - if args.erofs_utils_source is not None - else None, - ), - } - (output_root / "fixture-manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="ascii", - ) - verify_output(output_root) - - -def verify_output(output_root: Path) -> None: - output_root = output_root.resolve() - manifest_path = output_root / "fixture-manifest.json" - manifest = json.loads(manifest_path.read_text(encoding="ascii")) - for path_text, expected in manifest["images"].items(): - path = output_root / path_text - if not path.is_file(): - raise FixtureError(f"missing image: {path_text}") - if path.stat().st_size != expected["size"]: - raise FixtureError(f"image size changed: {path_text}") - if sha256(path) != expected["sha256"]: - raise FixtureError(f"image hash changed: {path_text}") - ErofsImage.load(path).validate_superblock() - for expected in manifest["source_inventory"]: - path = output_root / expected["path"] - if path.stat().st_size != expected["size"] or sha256(path) != expected["sha256"]: - raise FixtureError(f"source changed: {expected['path']}") - for key, expected in manifest["targets"].items(): - image_name, path = key.split(":", 1) - actual = inspect_path( - output_root, - output_root / "images" / image_name, - path, - write_log=False, - ) - actual.pop("dump_log", None) - comparable = dict(expected) - comparable.pop("dump_log", None) - if actual != comparable: - raise FixtureError(f"structured target fields changed: {key}") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - create = subparsers.add_parser("create") - create.add_argument("--output", type=Path, required=True) - create.add_argument("--erofs-utils-source", type=Path) - verify = subparsers.add_parser("verify") - verify.add_argument("--output", type=Path, required=True) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - if args.command == "create": - build_fixtures(args) - else: - verify_output(args.output) - - -if __name__ == "__main__": - main() diff --git a/tests/g6_multidev_fixtures.py b/tests/g6_multidev_fixtures.py deleted file mode 100644 index cb3ccfa..0000000 --- a/tests/g6_multidev_fixtures.py +++ /dev/null @@ -1,1218 +0,0 @@ -#!/usr/bin/env python3 -"""Generate and verify assertion-driven G6 chunk/multidevice fixtures.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import math -import os -import shutil -import struct -import subprocess -import tempfile -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Callable - -from erofs_fixture import ErofsImage, SUPER - - -FEATURE_INCOMPAT_CHUNKED_FILE = 0x00000004 -FEATURE_INCOMPAT_DEVICE_TABLE = 0x00000008 -FEATURE_INCOMPAT_48BIT = 0x00000080 -CHUNK_FORMAT_BLOCK_BITS_MASK = 0x001F -CHUNK_FORMAT_INDEXES = 0x0020 -CHUNK_FORMAT_48BIT = 0x0040 -DEVICE_SLOT_SIZE = 128 -DEVICE_SLOT_FIELDS = 64 -BLOCK_SIZE = 4096 - -CHUNK_UUID = "67360006-0093-0094-0095-000000000101" -FRAGMENT_UUID = "67360098-0000-0000-0000-000000000098" -PCLUSTER_UUID = "67360094-0101-0000-0000-000000000094" - -CHUNK_FILES = { - "tc006.bin": 24, - "cross-device.bin": 96, - "indexed.bin": 40, - "cold-slot2.bin": 32, - "unified-address.bin": 32, -} - - -class FixtureError(RuntimeError): - pass - - -@dataclass(frozen=True) -class ChunkEntry: - offset: int - start_block_high: int - device_id: int - start_block_low: int - - @property - def start_block(self) -> int: - return self.start_block_low | (self.start_block_high << 32) - - -@dataclass(frozen=True) -class ChunkLayout: - path: str - nid: int - inode_offset: int - format_offset: int - chunk_format: int - chunk_bits: int - chunk_size: int - entry_size: int - index_base: int - entries: tuple[ChunkEntry, ...] - - -@dataclass(frozen=True) -class DeviceSlot: - blocks: int - uniaddr: int - - -@dataclass -class MultiFixture: - primary: ErofsImage - providers: list[bytearray] - slots: list[DeviceSlot] - - -def align_up(value: int, alignment: int) -> int: - return (value + alignment - 1) & ~(alignment - 1) - - -def sha256_bytes(data: bytes | bytearray) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def deterministic_block(path: str, block: int) -> bytes: - seed = f"repo22-g6:{path}:{block}".encode("ascii") - return b"".join( - hashlib.sha256(seed + index.to_bytes(4, "little")).digest() - for index in range(128) - )[:BLOCK_SIZE] - - -def run( - command: list[str], *, stdout: bool = False, cwd: Path | None = None -) -> str: - result = subprocess.run( - command, - check=True, - cwd=cwd, - stdout=subprocess.PIPE if stdout else subprocess.DEVNULL, - stderr=subprocess.STDOUT if stdout else None, - text=True, - ) - return result.stdout if stdout else "" - - -def tool_version(tool: str) -> str: - output = run([tool, "-V"], stdout=True).strip().splitlines() - if not output: - raise FixtureError(f"{tool} returned no version") - return output[0] - - -def write_chunk_sources(root: Path) -> None: - root.mkdir(parents=True) - for name, blocks in CHUNK_FILES.items(): - with (root / name).open("wb") as stream: - for block in range(blocks): - stream.write(deterministic_block(name, block)) - - -def write_fragment_source(root: Path) -> None: - root.mkdir(parents=True) - seed = deterministic_block("fragment.dat", 0) - data = (seed * math.ceil(100000 / len(seed)))[:100000] - (root / "fragment.dat").write_bytes(data) - - -def write_pcluster_source(root: Path) -> None: - root.mkdir(parents=True) - seed = b"".join( - hashlib.sha256(f"repo22-g6:pcluster:{index}".encode("ascii")).digest() - for index in range(188) - )[:6000] - data = (seed * math.ceil(131072 / len(seed)))[:131072] - (root / "external-pcluster.bin").write_bytes(data) - - -def make_image( - mkfs: str, - output: Path, - source: Path, - uuid: str, - options: list[str], -) -> None: - run( - [ - mkfs, - "-T0", - "--all-time", - "--all-root", - "--workers=1", - f"-U{uuid}", - *options, - str(output), - str(source), - ] - ) - - -def chunk_layout(image: ErofsImage, path: str) -> ChunkLayout: - _, entry = image.resolve_root_entry(path) - inode = image.inode(entry.nid) - if inode.layout != 4: - raise FixtureError(f"{path}: expected chunk layout, got {inode.layout}") - chunk_format, reserved = struct.unpack_from( - " image.blocks * image.block_size: - raise FixtureError(f"{path}: chunk index array exceeds declared image") - entries = [] - for index in range(count): - offset = index_base + index * entry_size - if entry_size == 8: - high, device_id, low = struct.unpack_from( - " None: - for path in paths: - layout = chunk_layout(image, path) - if layout.entry_size != 8: - raise FixtureError(f"{path}: blob baseline has no 8-byte indexes") - minimum = 3 if path in ("/cross-device.bin", "/indexed.bin") else 1 - if len(layout.entries) < minimum: - raise FixtureError( - f"{path}: need at least {minimum} chunk indexes" - ) - for entry in layout.entries: - if entry.start_block_high != 0 or entry.device_id != 1: - raise FixtureError(f"{path}: unexpected mkfs chunk index {entry}") - - -def patch_chunk_entry( - image: ErofsImage, - entry: ChunkEntry, - *, - expected: tuple[int, int, int], - replacement: tuple[int, int, int], -) -> None: - current = struct.unpack_from(" None: - for path in paths: - layout = chunk_layout(image, path) - if layout.entry_size != 8: - raise FixtureError(f"{path}: 48-bit conversion requires indexes") - expected = layout.chunk_format - if expected & CHUNK_FORMAT_48BIT: - raise FixtureError(f"{path}: already has 48-bit indexes") - current = image.u16(layout.format_offset) - if current != expected: - raise FixtureError(f"{path}: chunk format changed before patch") - image.put_u16(layout.format_offset, expected | CHUNK_FORMAT_48BIT) - - -def enable_48bit(image: ErofsImage) -> None: - if image.feature_incompat & FEATURE_INCOMPAT_48BIT: - raise FixtureError("image already has 48-bit feature") - root_nid = image.root_nid - image.put_u32( - SUPER + 80, image.feature_incompat | FEATURE_INCOMPAT_48BIT - ) - image.put_u16(SUPER + 14, 0) - image.put_u64(SUPER + 112, root_nid) - - -def decode_slots(image: ErofsImage) -> list[DeviceSlot]: - extra, slot_offset = struct.unpack_from(" len(image.data): - raise FixtureError("device table lies outside provider bytes") - slots = [] - for index in range(extra): - offset = table_offset + index * DEVICE_SLOT_SIZE + DEVICE_SLOT_FIELDS - blocks_low, uniaddr_low, blocks_high, uniaddr_high = struct.unpack_from( - " None: - if not slots: - raise FixtureError("device table needs at least one slot") - if table_offset % DEVICE_SLOT_SIZE != 0: - raise FixtureError("device table is not 128-byte aligned") - required = max( - primary_blocks * image.block_size, - table_offset + len(slots) * DEVICE_SLOT_SIZE, - ) - if len(image.data) > required: - raise FixtureError("primary metadata exceeds requested primary size") - image.data.extend(bytes(required - len(image.data))) - image.put_u32(SUPER + 36, primary_blocks) - image.put_u32( - SUPER + 80, image.feature_incompat | FEATURE_INCOMPAT_DEVICE_TABLE - ) - image.put_u16(SUPER + 86, len(slots)) - image.put_u16(SUPER + 88, table_offset // DEVICE_SLOT_SIZE) - image.data[ - table_offset : table_offset + len(slots) * DEVICE_SLOT_SIZE - ] = bytes(len(slots) * DEVICE_SLOT_SIZE) - for index, slot in enumerate(slots): - if slot.blocks <= 0 or slot.blocks >= 1 << 48: - raise FixtureError(f"slot {index + 1}: invalid blocks {slot.blocks}") - if slot.uniaddr < 0 or slot.uniaddr >= 1 << 48: - raise FixtureError(f"slot {index + 1}: invalid uniaddr {slot.uniaddr}") - offset = table_offset + index * DEVICE_SLOT_SIZE - tag = f"repo22-g6-slot-{index + 1}".encode("ascii") - image.data[offset : offset + len(tag)] = tag - struct.pack_into( - "> 32, - slot.uniaddr >> 32, - ) - - -def finalize_image(image: ErofsImage, path: Path) -> None: - image.update_checksum() - image.validate_superblock() - image.save(path) - - -def build_single_indexed( - baseline: ErofsImage, blob: bytes, paths: list[str] -) -> ErofsImage: - image = baseline.clone() - if len(image.data) != BLOCK_SIZE or len(blob) % BLOCK_SIZE != 0: - raise FixtureError("single indexed folding requires aligned providers") - for path in paths: - layout = chunk_layout(image, path) - for entry in layout.entries: - patch_chunk_entry( - image, - entry, - expected=(0, 1, entry.start_block_low), - replacement=(0, 0, entry.start_block_low + 1), - ) - image.data.extend(blob) - image.put_u32(SUPER + 36, len(image.data) // BLOCK_SIZE) - image.put_u32( - SUPER + 80, image.feature_incompat & ~FEATURE_INCOMPAT_DEVICE_TABLE - ) - image.put_u16(SUPER + 86, 0) - image.put_u16(SUPER + 88, 0) - return image - - -def build_multislot( - baseline: ErofsImage, - blob: bytes, - paths: list[str], - slot_count: int, -) -> MultiFixture: - image = baseline.clone() - providers = [bytearray() for _ in range(slot_count)] - for path in paths: - layout = chunk_layout(image, path) - blocks_per_chunk = layout.chunk_size // BLOCK_SIZE - for index, entry in enumerate(layout.entries): - if entry.start_block_high != 0 or entry.device_id != 1: - raise FixtureError(f"{path}: unexpected source index {entry}") - source_start = entry.start_block * BLOCK_SIZE - source_end = source_start + blocks_per_chunk * BLOCK_SIZE - if source_end > len(blob): - raise FixtureError(f"{path}: source chunk exceeds blob provider") - slot = 1 if path == "/cold-slot2.bin" and slot_count >= 2 else ( - index % slot_count - ) - local_block = len(providers[slot]) // BLOCK_SIZE - providers[slot].extend(blob[source_start:source_end]) - patch_chunk_entry( - image, - entry, - expected=(0, 1, entry.start_block_low), - replacement=(0, slot + 1, local_block), - ) - for index, provider in enumerate(providers): - provider.extend(bytes((index + 1) * BLOCK_SIZE)) - slots = [] - uniaddr = 2 - for provider in providers: - blocks = len(provider) // BLOCK_SIZE - slots.append(DeviceSlot(blocks, uniaddr)) - uniaddr += blocks - write_device_table(image, slots) - return MultiFixture(image, providers, slots) - - -def clone_multifixture(fixture: MultiFixture) -> MultiFixture: - return MultiFixture( - fixture.primary.clone(), - [bytearray(provider) for provider in fixture.providers], - list(fixture.slots), - ) - - -def rewrite_slots( - fixture: MultiFixture, - slots: list[DeviceSlot], - *, - table_offset: int = BLOCK_SIZE, -) -> None: - fixture.slots = slots - write_device_table(fixture.primary, slots, table_offset=table_offset) - - -def build_flatdev(fixture: MultiFixture) -> bytes: - end_block = max( - [fixture.primary.blocks] - + [slot.uniaddr + slot.blocks for slot in fixture.slots] - ) - data = bytearray(end_block * BLOCK_SIZE) - data[: len(fixture.primary.data)] = fixture.primary.data - for slot, provider in zip(fixture.slots, fixture.providers, strict=True): - if len(provider) != slot.blocks * BLOCK_SIZE: - raise FixtureError("provider length does not match slot declaration") - start = slot.uniaddr * BLOCK_SIZE - data[start : start + len(provider)] = provider - return bytes(data) - - -def first_entry(image: ErofsImage, path: str, index: int = 0) -> ChunkEntry: - layout = chunk_layout(image, path) - try: - return layout.entries[index] - except IndexError as error: - raise FixtureError(f"{path}: missing chunk index {index}") from error - - -def patch_unified_entry( - fixture: MultiFixture, - path: str, - *, - index: int = 0, -) -> None: - entry = first_entry(fixture.primary, path, index) - if entry.device_id < 1 or entry.device_id > len(fixture.slots): - raise FixtureError(f"{path}: source index has no external slot") - slot = fixture.slots[entry.device_id - 1] - global_block = slot.uniaddr + entry.start_block - patch_chunk_entry( - fixture.primary, - entry, - expected=(entry.start_block_high, entry.device_id, entry.start_block_low), - replacement=(global_block >> 32, 0, global_block & 0xFFFFFFFF), - ) - - -def patch_table_field( - image: ErofsImage, - slot: int, - field_offset: int, - fmt: str, - value: int, -) -> None: - _, slot_offset = struct.unpack_from(" tuple[MultiFixture, int]: - _, entry = lz4_image.resolve_root_entry("/external-pcluster.bin") - inode = lz4_image.inode(entry.nid) - if inode.layout != 1 or inode.size != 131072: - raise FixtureError("LZ4 source is not a legacy full-index inode") - header = align_up(inode.offset + inode.inode_size + inode.xattr_size, 8) - index_offset = header + 16 - advise, cluster_offset, pblk = struct.unpack_from( - " MultiFixture: - image = positive.primary.clone() - providers = [ - bytearray(positive.providers[0][:BLOCK_SIZE]), - bytearray(positive.providers[0][BLOCK_SIZE:]), - ] - slots = [DeviceSlot(1, 2), DeviceSlot(1, 3)] - write_device_table(image, slots) - current = struct.unpack_from(" dict[str, Any]: - extra, slot_offset = struct.unpack_from(" None: - source_files = sorted( - path.relative_to(source) for path in source.rglob("*") if path.is_file() - ) - extracted_files = sorted( - path.relative_to(extracted) - for path in extracted.rglob("*") - if path.is_file() - ) - if source_files != extracted_files: - raise FixtureError( - f"extracted paths differ: {source_files} != {extracted_files}" - ) - for relative in source_files: - if sha256_file(source / relative) != sha256_file(extracted / relative): - raise FixtureError(f"extracted checksum differs: {relative}") - - -def fsck_extract( - fsck: str, - primary: Path, - devices: list[Path], - source: Path, - scratch_parent: Path, -) -> None: - with tempfile.TemporaryDirectory(prefix="repo22-g6-fsck-", dir=scratch_parent) as tmp: - extracted = Path(tmp) / "extracted" - command = [fsck] - command.extend(f"--device={device}" for device in devices) - command.extend([f"--extract={extracted}", str(primary)]) - run(command) - compare_trees(source, extracted) - - -def write_provider(path: Path, data: bytes | bytearray) -> None: - if len(data) == 0 or len(data) % BLOCK_SIZE != 0: - raise FixtureError(f"{path.name}: provider is not block aligned") - path.write_bytes(data) - - -def save_multifixture( - output: Path, - name: str, - fixture: MultiFixture, - paths: list[str], - manifest: dict[str, Any], -) -> tuple[Path, list[Path]]: - primary_path = output / "images" / f"{name}-primary.erofs" - finalize_image(fixture.primary, primary_path) - provider_paths = [] - for index, provider in enumerate(fixture.providers, 1): - provider_path = output / "images" / f"{name}-slot{index}.blob" - write_provider(provider_path, provider) - provider_paths.append(provider_path) - manifest["fixtures"][name] = { - "primary": str(primary_path.relative_to(output)), - "providers": [str(path.relative_to(output)) for path in provider_paths], - "image": image_description(fixture.primary, paths), - } - return primary_path, provider_paths - - -def save_image_fixture( - output: Path, - name: str, - image: ErofsImage, - paths: list[str], - manifest: dict[str, Any], -) -> Path: - path = output / "images" / f"{name}.erofs" - finalize_image(image, path) - manifest["fixtures"][name] = { - "primary": str(path.relative_to(output)), - "providers": [], - "image": image_description(image, paths), - } - return path - - -def add_mutation( - manifest: dict[str, Any], - artifact: Path, - output: Path, - label: str, - offset: int, - fmt: str, - value: int, -) -> None: - manifest["mutations"].append( - { - "artifact": str(artifact.relative_to(output)), - "label": label, - "offset": offset, - "format": fmt, - "value": value, - } - ) - - -def save_negative( - output: Path, - name: str, - image: ErofsImage, - manifest: dict[str, Any], -) -> Path: - path = output / "images" / f"{name}.erofs" - finalize_image(image, path) - manifest["negative_images"][name] = str(path.relative_to(output)) - return path - - -def generate(args: argparse.Namespace) -> None: - output: Path = args.output.resolve() - if output.exists(): - raise FixtureError(f"output already exists: {output}") - output.mkdir(parents=True) - (output / "images").mkdir() - chunk_source = output / "sources" / "chunk" - fragment_source = output / "sources" / "fragment" - pcluster_source = output / "sources" / "pcluster" - write_chunk_sources(chunk_source) - write_fragment_source(fragment_source) - write_pcluster_source(pcluster_source) - - mkfs_version = tool_version(args.mkfs) - fsck_version = tool_version(args.fsck) - if "1.8.6" not in mkfs_version or "1.8.6" not in fsck_version: - raise FixtureError( - f"G6 requires erofs-utils 1.8.6, got {mkfs_version!r}, {fsck_version!r}" - ) - - base_primary_path = output / "images" / "mkfs-blob-primary.erofs" - base_blob_path = output / "images" / "mkfs-blob-slot1.blob" - base_blob_path.write_bytes(b"") - make_image( - args.mkfs, - base_primary_path, - chunk_source, - CHUNK_UUID, - ["--chunksize=4096", f"--blobdev={base_blob_path}"], - ) - baseline = ErofsImage.load(base_primary_path) - baseline.validate_superblock() - paths = [f"/{name}" for name in CHUNK_FILES] - assert_chunk_baseline(baseline, paths) - base_blob = base_blob_path.read_bytes() - if len(baseline.data) != BLOCK_SIZE or len(base_blob) % BLOCK_SIZE != 0: - raise FixtureError("mkfs blob baseline has unexpected provider sizes") - - manifest: dict[str, Any] = { - "schema": 1, - "generator": "tests/g6_multidev_fixtures.py", - "erofs_utils": {"mkfs": mkfs_version, "fsck": fsck_version}, - "fixtures": {}, - "negative_images": {}, - "mutations": [], - "artifacts": {}, - "host_qualification": { - "flatdev": ( - "kernel-only: erofs-utils 1.8.6 requires --device for " - "nonzero chunk device IDs" - ), - "external_pcluster": ( - "kernel-only after relocation: erofs-utils 1.8.6 does not " - "apply the device-ID-0 unified mapping used by the kernel" - ), - }, - } - manifest["fixtures"]["mkfs-blob"] = { - "primary": str(base_primary_path.relative_to(output)), - "providers": [str(base_blob_path.relative_to(output))], - "image": image_description(baseline, paths), - } - - single = build_single_indexed(baseline, base_blob, paths) - single_path = save_image_fixture( - output, "single-indexed", single, paths, manifest - ) - - multi2 = build_multislot(baseline, base_blob, paths, 2) - multi2_primary, multi2_providers = save_multifixture( - output, "multi2", multi2, paths, manifest - ) - flatdev_path = output / "images" / "multi2-flatdev.erofs" - write_provider(flatdev_path, build_flatdev(multi2)) - manifest["fixtures"]["multi2-flatdev"] = { - "primary": str(flatdev_path.relative_to(output)), - "providers": [], - "image": image_description(ErofsImage.load(flatdev_path), paths), - } - - multi3 = build_multislot(baseline, base_blob, paths, 3) - multi3_primary, multi3_providers = save_multifixture( - output, "multi3", multi3, paths, manifest - ) - - slot_zero = clone_multifixture(multi2) - zero_slots = [DeviceSlot(slot_zero.slots[0].blocks, 0), slot_zero.slots[1]] - rewrite_slots(slot_zero, zero_slots) - slot_zero_primary, slot_zero_providers = save_multifixture( - output, "multi2-uniaddr0", slot_zero, paths, manifest - ) - - table_zero = clone_multifixture(multi2) - table_bytes = bytes( - table_zero.primary.data[ - BLOCK_SIZE : BLOCK_SIZE + 2 * DEVICE_SLOT_SIZE - ] - ) - table_zero.primary.data[: 2 * DEVICE_SLOT_SIZE] = table_bytes - table_zero.primary.put_u16(SUPER + 88, 0) - table_zero_primary, table_zero_providers = save_multifixture( - output, "multi2-devt0", table_zero, paths, manifest - ) - - unified = clone_multifixture(multi2) - patch_unified_entry(unified, "/unified-address.bin") - unified_primary, unified_providers = save_multifixture( - output, "multi2-unified", unified, paths, manifest - ) - unified_flatdev_path = output / "images" / "multi2-unified-flatdev.erofs" - write_provider(unified_flatdev_path, build_flatdev(unified)) - manifest["fixtures"]["multi2-unified-flatdev"] = { - "primary": str(unified_flatdev_path.relative_to(output)), - "providers": [], - "image": image_description( - ErofsImage.load(unified_flatdev_path), paths - ), - } - - unified48 = clone_multifixture(multi2) - enable_48bit(unified48.primary) - set_chunk_48bit(unified48.primary, paths) - high_start = (1 << 32) + 2 - high_slots = [] - cursor = high_start - for slot in unified48.slots: - high_slots.append(DeviceSlot(slot.blocks, cursor)) - cursor += slot.blocks - rewrite_slots(unified48, high_slots) - patch_unified_entry(unified48, "/unified-address.bin") - unified48_primary, unified48_providers = save_multifixture( - output, "multi2-unified48", unified48, paths, manifest - ) - - unified48_oob = clone_multifixture(unified48) - unified48_oob_entry = first_entry( - unified48_oob.primary, "/unified-address.bin" - ) - patch_chunk_entry( - unified48_oob.primary, - unified48_oob_entry, - expected=( - unified48_oob_entry.start_block_high, - 0, - unified48_oob_entry.start_block_low, - ), - replacement=(0xFFFF, 0, 0xFFFFFFFE), - ) - unified48_oob_primary, unified48_oob_providers = save_multifixture( - output, "bad-unified48-out-of-range", unified48_oob, paths, manifest - ) - add_mutation( - manifest, - unified48_oob_primary, - output, - "device-ID-0 chunk address is the largest non-NULL 48-bit block", - unified48_oob_entry.offset, - " tuple[int, str, int, str]: - value = image.blocks * BLOCK_SIZE // DEVICE_SLOT_SIZE - image.put_u16(SUPER + 88, value) - return SUPER + 88, " tuple[int, str, int, str]: - patch_table_field(image, 1, 0, " tuple[int, str, int, str]: - patch_table_field(image, 1, 4, " tuple[int, str, int, str]: - value = multi2.slots[0].uniaddr - patch_table_field(image, 2, 4, " tuple[int, str, int, str]: - patch_table_field(image, 1, 0, " tuple[int, str, int, str]: - enable_48bit(image) - patch_table_field(image, 1, 0, " Any: - size = struct.calcsize(fmt) - with path.open("rb") as stream: - stream.seek(offset) - data = stream.read(size) - if len(data) != size: - raise FixtureError(f"{path}: field at {offset} is truncated") - values = struct.unpack(fmt, data) - return values[0] if len(values) == 1 else list(values) - - -def verify_image_description(path: Path, expected: dict[str, Any]) -> None: - image = ErofsImage.load(path) - paths = list(expected["chunks"]) - actual = image_description(image, paths) - if actual != expected: - raise FixtureError(f"{path}: parsed image fields differ from manifest") - - -def verify_output(output: Path) -> None: - manifest_path = output / "manifest.json" - manifest = json.loads(manifest_path.read_text(encoding="ascii")) - if manifest.get("schema") != 1: - raise FixtureError("unsupported G6 manifest schema") - for relative, expected in manifest["artifacts"].items(): - path = output / relative - if not path.is_file(): - raise FixtureError(f"missing artifact: {relative}") - if path.stat().st_size != expected["bytes"]: - raise FixtureError(f"size differs: {relative}") - if sha256_file(path) != expected["sha256"]: - raise FixtureError(f"SHA256 differs: {relative}") - for fixture in manifest["fixtures"].values(): - image = fixture.get("image") - if image is not None: - verify_image_description(output / fixture["primary"], image) - for mutation in manifest["mutations"]: - actual = unpack_assertion( - output / mutation["artifact"], - mutation["offset"], - mutation["format"], - ) - if actual != mutation["value"]: - raise FixtureError( - f"{mutation['label']}: expected {mutation['value']}, got {actual}" - ) - run(["sha256sum", "-c", "SHA256SUMS"], stdout=True, cwd=output) - print( - f"verified={output} artifacts={len(manifest['artifacts'])} " - f"field_assertions={len(manifest['mutations'])}" - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - - generate_parser = subparsers.add_parser("generate") - generate_parser.add_argument("--output", required=True, type=Path) - generate_parser.add_argument("--mkfs", default=shutil.which("mkfs.erofs")) - generate_parser.add_argument("--fsck", default=shutil.which("fsck.erofs")) - generate_parser.set_defaults(function=generate) - - verify_parser = subparsers.add_parser("verify") - verify_parser.add_argument("output", type=Path) - verify_parser.set_defaults(function=lambda args: verify_output(args.output.resolve())) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - if args.command == "generate" and (args.mkfs is None or args.fsck is None): - raise FixtureError("mkfs.erofs and fsck.erofs are required") - args.function(args) - - -if __name__ == "__main__": - os.environ.setdefault("PYTHONDONTWRITEBYTECODE", "1") - main() diff --git a/tests/g7_fixtures.py b/tests/g7_fixtures.py deleted file mode 100644 index 24a2f8e..0000000 --- a/tests/g7_fixtures.py +++ /dev/null @@ -1,495 +0,0 @@ -#!/usr/bin/env python3 -"""Generate and verify deterministic repo22 G7 stress fixtures.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import subprocess -from typing import Any - - -BLOCK_SIZE = 4096 -MIB = 1024 * 1024 -SPARSE_SIZE = 4 * 1024 * MIB + BLOCK_SIZE + 1 -DEEP_LEVELS = 128 -MANY_SMALL_COUNT = 12000 -LARGE_DIRECTORY_COUNT = 12000 -CONCURRENT_FILE_COUNT = 16 -CONCURRENT_FILE_SIZE = 2 * MIB -PRESSURE_FILE_SIZE = 96 * MIB -SEQUENTIAL_FILE_SIZE = 256 * MIB -RANDOM_FILE_SIZE = 64 * MIB -FIXED_UUIDS = { - "boundaries.erofs": "00000000-0000-0000-0000-000000000071", - "workloads.erofs": "00000000-0000-0000-0000-000000000072", -} -SPARSE_MARKERS = ( - (0, b"repo22-g7-start"), - (BLOCK_SIZE - 8, b"g7-block-edge"), - (2**31 - 8, b"g7-2gib-edge"), - (2**32 - 8, b"g7-4gib-edge"), - (SPARSE_SIZE - 16, b"repo22-g7-end!!"), -) -SPARSE_RANGES = ( - ("start", 0, 64), - ("block-edge", BLOCK_SIZE - 32, 64), - ("one-gib-hole", 1024 * MIB, BLOCK_SIZE), - ("two-gib-edge", 2**31 - 32, 64), - ("four-gib-edge", 2**32 - 32, 64), - ("eof-edge", SPARSE_SIZE - 64, 64), -) - - -class FixtureError(RuntimeError): - pass - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - while chunk := source.read(MIB): - digest.update(chunk) - return digest.hexdigest() - - -def run(command: list[str], log: Path | None = None) -> str: - result = subprocess.run( - command, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - if log is not None: - log.write_text( - "$ " + " ".join(command) + "\n" + result.stdout, - encoding="utf-8", - ) - if result.returncode != 0: - raise FixtureError( - f"command failed ({result.returncode}): {' '.join(command)}\n" - f"{result.stdout}" - ) - return result.stdout - - -def repeated_bytes(label: str, size: int) -> bytes: - token = (label + "\n").encode("ascii") - return (token * ((size + len(token) - 1) // len(token)))[:size] - - -def write_pattern( - path: Path, - size: int, - seed: str, - random_period: int, -) -> None: - remaining = size - block_index = 0 - with path.open("wb") as output: - while remaining: - amount = min(BLOCK_SIZE, remaining) - if block_index % random_period == 0: - block = hashlib.shake_256( - f"repo22-g7:{seed}:{block_index}".encode("ascii") - ).digest(amount) - else: - block = repeated_bytes( - f"repo22-g7:{seed}:{block_index % 97:02d}", amount - ) - output.write(block) - remaining -= amount - block_index += 1 - - -def create_sparse(path: Path) -> None: - with path.open("wb") as output: - output.truncate(SPARSE_SIZE) - for offset, marker in SPARSE_MARKERS: - output.seek(offset) - output.write(marker) - - -def create_boundaries(root: Path) -> dict[str, Any]: - root.mkdir(parents=True) - (root / "empty.bin").write_bytes(b"") - - maximum = root / "maximum" - maximum.mkdir() - create_sparse(maximum / "sparse-boundary.bin") - - deep = root / "deep" - deep.mkdir() - current = deep - components = [] - for level in range(DEEP_LEVELS): - component = f"d{level:03d}" - components.append(component) - current /= component - current.mkdir() - deep_payload = current / "payload.bin" - deep_payload.write_bytes(repeated_bytes("repo22-g7-deep", BLOCK_SIZE)) - deep_path = str(Path("deep", *components, "payload.bin")) - - long_directory = root / "longname" - long_directory.mkdir() - long_name = "n" * 255 - (long_directory / long_name).write_bytes( - repeated_bytes("repo22-g7-long-name", BLOCK_SIZE) - ) - - many_small = root / "many-small" - many_small.mkdir() - for index in range(MANY_SMALL_COUNT): - shard = many_small / f"shard-{index // 1000:02d}" - shard.mkdir(exist_ok=True) - (shard / f"file-{index:05d}.bin").write_bytes( - repeated_bytes(f"repo22-g7-small:{index:05d}", 1024) - ) - - large_directory = root / "large-dir" - large_directory.mkdir() - for index in range(LARGE_DIRECTORY_COUNT): - (large_directory / f"entry-{index:05d}.txt").write_bytes( - f"repo22-g7-large-dir:{index:05d}\n".encode("ascii") - ) - - return { - "deep_path": deep_path, - "long_name": long_name, - } - - -def create_workloads(root: Path) -> None: - root.mkdir(parents=True) - concurrent = root / "concurrent" - concurrent.mkdir() - for index in range(CONCURRENT_FILE_COUNT): - write_pattern( - concurrent / f"reader-{index:02d}.bin", - CONCURRENT_FILE_SIZE, - f"concurrent-{index:02d}", - 8, - ) - - pressure = root / "pressure" - pressure.mkdir() - write_pattern( - pressure / "pressure.bin", - PRESSURE_FILE_SIZE, - "pressure", - 8, - ) - - sequential = root / "sequential" - sequential.mkdir() - write_pattern( - sequential / "sequential.bin", - SEQUENTIAL_FILE_SIZE, - "sequential", - 4, - ) - - random_directory = root / "random" - random_directory.mkdir() - write_pattern( - random_directory / "random.bin", - RANDOM_FILE_SIZE, - "random", - 1, - ) - - -def normalize_modes(root: Path) -> None: - for path in sorted(root.rglob("*")): - os.chmod(path, 0o755 if path.is_dir() else 0o644) - os.chmod(root, 0o755) - - -def inventory(source_root: Path, output_root: Path) -> list[dict[str, Any]]: - records = [] - for path in sorted(item for item in source_root.rglob("*") if item.is_file()): - records.append( - { - "sha256": sha256(path), - "size": path.stat().st_size, - "path": str(path.relative_to(output_root)), - } - ) - return records - - -def inventory_text(records: list[dict[str, Any]]) -> str: - lines = ["sha256\tsize\tpath"] - lines.extend( - f"{record['sha256']}\t{record['size']}\t{record['path']}" - for record in records - ) - return "\n".join(lines) + "\n" - - -def checksum_text(records: list[dict[str, Any]]) -> str: - return "".join( - f"{record['sha256']} {record['path']}\n" for record in records - ) - - -def create_images(output_root: Path) -> list[dict[str, Any]]: - images = output_root / "images" - logs = output_root / "logs" - images.mkdir() - logs.mkdir() - records = [] - for image_name, source_name in ( - ("boundaries.erofs", "boundaries"), - ("workloads.erofs", "workloads"), - ): - image = images / image_name - command = [ - "mkfs.erofs", - "--workers=1", - "--sort=path", - "--all-root", - "-T0", - "--all-time", - f"-U{FIXED_UUIDS[image_name]}", - "-z", - "lz4", - str(image), - str(output_root / "sources" / source_name), - ] - run(command, logs / f"mkfs-{source_name}.log") - run( - ["fsck.erofs", "-d0", str(image)], - logs / f"fsck-{source_name}.log", - ) - records.append( - { - "sha256": sha256(image), - "size": image.stat().st_size, - "path": str(image.relative_to(output_root)), - "command": command, - } - ) - return records - - -def write_metadata( - output_root: Path, - source_records: list[dict[str, Any]], - image_records: list[dict[str, Any]], - boundary_metadata: dict[str, Any], -) -> None: - source_inventory = inventory_text(source_records) - image_checksums = checksum_text(image_records) - (output_root / "SOURCE-INVENTORY.tsv").write_text( - source_inventory, encoding="utf-8" - ) - (output_root / "SOURCE-SHA256SUMS").write_text( - checksum_text(source_records), encoding="utf-8" - ) - (output_root / "SHA256SUMS").write_text( - image_checksums, encoding="utf-8" - ) - (output_root / "DEEP-PATH").write_text( - boundary_metadata["deep_path"] + "\n", encoding="ascii" - ) - (output_root / "LONG-NAME").write_text( - boundary_metadata["long_name"] + "\n", encoding="ascii" - ) - (output_root / "SPARSE-RANGES.tsv").write_text( - "".join( - f"{label}\t{offset}\t{length}\n" - for label, offset, length in SPARSE_RANGES - ), - encoding="ascii", - ) - manifest = { - "format": 1, - "block_size": BLOCK_SIZE, - "deep_levels": DEEP_LEVELS, - "deep_path": boundary_metadata["deep_path"], - "long_name_bytes": len(boundary_metadata["long_name"].encode("ascii")), - "many_small_count": MANY_SMALL_COUNT, - "large_directory_count": LARGE_DIRECTORY_COUNT, - "concurrent_file_count": CONCURRENT_FILE_COUNT, - "concurrent_file_size": CONCURRENT_FILE_SIZE, - "pressure_file_size": PRESSURE_FILE_SIZE, - "sequential_file_size": SEQUENTIAL_FILE_SIZE, - "random_file_size": RANDOM_FILE_SIZE, - "sparse_size": SPARSE_SIZE, - "sparse_markers": [ - {"offset": offset, "hex": marker.hex()} - for offset, marker in SPARSE_MARKERS - ], - "sparse_ranges": [ - {"label": label, "offset": offset, "length": length} - for label, offset, length in SPARSE_RANGES - ], - "source_count": len(source_records), - "source_inventory_sha256": hashlib.sha256( - source_inventory.encode("utf-8") - ).hexdigest(), - "image_count": len(image_records), - "images": image_records, - } - (output_root / "fixture-manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - - -def create(output_root: Path) -> None: - if output_root.exists(): - raise FixtureError(f"output already exists: {output_root}") - sources = output_root / "sources" - sources.mkdir(parents=True) - boundary_metadata = create_boundaries(sources / "boundaries") - create_workloads(sources / "workloads") - normalize_modes(sources) - source_records = inventory(sources, output_root) - image_records = create_images(output_root) - write_metadata( - output_root, - source_records, - image_records, - boundary_metadata, - ) - verify(output_root) - - -def require(condition: bool, message: str) -> None: - if not condition: - raise FixtureError(message) - - -def verify(output_root: Path) -> None: - manifest_path = output_root / "fixture-manifest.json" - require(manifest_path.is_file(), "missing fixture-manifest.json") - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - sources = output_root / "sources" - source_records = inventory(sources, output_root) - source_text = inventory_text(source_records) - require( - source_text == (output_root / "SOURCE-INVENTORY.tsv").read_text( - encoding="utf-8" - ), - "source inventory mismatch", - ) - require( - checksum_text(source_records) - == (output_root / "SOURCE-SHA256SUMS").read_text(encoding="utf-8"), - "source checksum list mismatch", - ) - require(len(source_records) == manifest["source_count"], "source count mismatch") - require( - hashlib.sha256(source_text.encode("utf-8")).hexdigest() - == manifest["source_inventory_sha256"], - "source inventory hash mismatch", - ) - - sparse = sources / "boundaries" / "maximum" / "sparse-boundary.bin" - require(sparse.stat().st_size == SPARSE_SIZE, "sparse file size mismatch") - require( - sparse.stat().st_blocks * 512 < MIB, - "sparse source unexpectedly consumes at least 1 MiB", - ) - with sparse.open("rb") as source: - for offset, marker in SPARSE_MARKERS: - source.seek(offset) - require(source.read(len(marker)) == marker, f"marker mismatch at {offset}") - - deep_path = (output_root / "DEEP-PATH").read_text(encoding="ascii").strip() - require(deep_path == manifest["deep_path"], "deep path metadata mismatch") - require((sources / "boundaries" / deep_path).is_file(), "deep payload missing") - require( - len(Path(deep_path).parts) - 2 == DEEP_LEVELS, - "deep directory level mismatch", - ) - long_name = (output_root / "LONG-NAME").read_text(encoding="ascii").strip() - require(len(long_name.encode("ascii")) == 255, "long name is not 255 bytes") - require( - (sources / "boundaries" / "longname" / long_name).is_file(), - "long-name source missing", - ) - require( - sum(1 for path in (sources / "boundaries" / "many-small").rglob("*") if path.is_file()) - == MANY_SMALL_COUNT, - "many-small count mismatch", - ) - require( - sum(1 for path in (sources / "boundaries" / "large-dir").iterdir() if path.is_file()) - == LARGE_DIRECTORY_COUNT, - "large-directory count mismatch", - ) - require( - sum(1 for path in (sources / "workloads" / "concurrent").iterdir() if path.is_file()) - == CONCURRENT_FILE_COUNT, - "concurrent source count mismatch", - ) - - image_records = [] - logs = output_root / "logs" - for image in sorted((output_root / "images").glob("*.erofs")): - run(["fsck.erofs", "-d0", str(image)], logs / f"verify-{image.stem}.log") - image_records.append( - { - "sha256": sha256(image), - "size": image.stat().st_size, - "path": str(image.relative_to(output_root)), - } - ) - require(len(image_records) == manifest["image_count"], "image count mismatch") - expected_images = [ - {key: record[key] for key in ("sha256", "size", "path")} - for record in manifest["images"] - ] - require(image_records == expected_images, "image inventory mismatch") - require( - checksum_text(image_records) - == (output_root / "SHA256SUMS").read_text(encoding="utf-8"), - "image checksum list mismatch", - ) - sparse_dump = run( - [ - "dump.erofs", - "--path=/maximum/sparse-boundary.bin", - str(output_root / "images" / "boundaries.erofs"), - ] - ) - require(f"Size: {SPARSE_SIZE} " in sparse_dump, "image sparse size mismatch") - print( - "verified " - f"sources={len(source_records)} images={len(image_records)} " - f"inventory_sha256={manifest['source_inventory_sha256']}" - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest="command", required=True) - for command in ("create", "verify"): - subparser = subparsers.add_parser(command) - subparser.add_argument("--output", type=Path, required=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - try: - if args.command == "create": - create(args.output.resolve()) - else: - verify(args.output.resolve()) - except FixtureError as error: - print(f"error: {error}") - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/g7_probe.c b/tests/g7_probe.c deleted file mode 100644 index 522a733..0000000 --- a/tests/g7_probe.c +++ /dev/null @@ -1,356 +0,0 @@ -#define _POSIX_C_SOURCE 200809L - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define PRESSURE_CHUNK (8U * 1024U * 1024U) - -static void -usage(void) -{ - fprintf(stderr, - "usage:\n" - " g7_probe empty FILE\n" - " g7_probe eof FILE OFFSET\n" - " g7_probe range SOURCE TARGET OFFSET LENGTH\n" - " g7_probe random SOURCE TARGET SEED OPERATIONS BLOCK_SIZE\n" - " g7_probe pressure GO_FILE STOP_FILE REQUEST_MIB\n"); -} - -static int -parse_u64(const char *value, uint64_t *result) -{ - char *end; - unsigned long long parsed; - - errno = 0; - parsed = strtoull(value, &end, 0); - if (errno != 0 || *value == '\0' || *end != '\0') - return (-1); - *result = parsed; - return (0); -} - -static int -read_exact_at(int descriptor, unsigned char *buffer, size_t length, - off_t offset) -{ - size_t total; - ssize_t amount; - - total = 0; - while (total < length) { - amount = pread(descriptor, buffer + total, length - total, - offset + (off_t)total); - if (amount < 0 && errno == EINTR) - continue; - if (amount <= 0) - return (-1); - total += (size_t)amount; - } - return (0); -} - -static uint64_t -digest_bytes(uint64_t digest, const unsigned char *buffer, size_t length) -{ - size_t index; - - for (index = 0; index < length; index++) { - digest ^= buffer[index]; - digest *= UINT64_C(1099511628211); - } - return (digest); -} - -static int -empty_probe(const char *path) -{ - struct stat status; - unsigned char byte; - ssize_t amount; - int descriptor; - - descriptor = open(path, O_RDONLY); - if (descriptor < 0) - return (1); - if (fstat(descriptor, &status) != 0 || status.st_size != 0) { - close(descriptor); - return (1); - } - amount = read(descriptor, &byte, sizeof(byte)); - if (amount != 0 || lseek(descriptor, 0, SEEK_SET) != 0 || - lseek(descriptor, 0, SEEK_END) != 0) { - close(descriptor); - return (1); - } - close(descriptor); - printf("size=0 read_bytes=0 seek_set=0 seek_end=0 exit=0\n"); - return (0); -} - -static int -eof_probe(const char *path, const char *offset_value) -{ - unsigned char byte; - uint64_t offset; - ssize_t amount; - int descriptor; - - if (parse_u64(offset_value, &offset) != 0 || offset > INT64_MAX) - return (1); - descriptor = open(path, O_RDONLY); - if (descriptor < 0) - return (1); - amount = pread(descriptor, &byte, sizeof(byte), (off_t)offset); - close(descriptor); - if (amount != 0) - return (1); - printf("offset=%" PRIu64 " read_bytes=0 errno=0 exit=0\n", offset); - return (0); -} - -static int -range_probe(const char *source_path, const char *target_path, - const char *offset_value, const char *length_value) -{ - unsigned char *source_buffer, *target_buffer; - uint64_t digest, length, offset; - int source, target, result; - - if (parse_u64(offset_value, &offset) != 0 || - parse_u64(length_value, &length) != 0 || length == 0 || - length > SIZE_MAX || offset > INT64_MAX) - return (1); - source_buffer = malloc((size_t)length); - target_buffer = malloc((size_t)length); - if (source_buffer == NULL || target_buffer == NULL) { - free(source_buffer); - free(target_buffer); - return (1); - } - source = open(source_path, O_RDONLY); - target = open(target_path, O_RDONLY); - result = 1; - if (source >= 0 && target >= 0 && - read_exact_at(source, source_buffer, (size_t)length, - (off_t)offset) == 0 && - read_exact_at(target, target_buffer, (size_t)length, - (off_t)offset) == 0 && - memcmp(source_buffer, target_buffer, (size_t)length) == 0) { - digest = digest_bytes(UINT64_C(1469598103934665603), - source_buffer, (size_t)length); - printf("offset=%" PRIu64 " length=%" PRIu64 - " mismatches=0 digest=%016" PRIx64 " exit=0\n", - offset, length, digest); - result = 0; - } - if (source >= 0) - close(source); - if (target >= 0) - close(target); - free(source_buffer); - free(target_buffer); - return (result); -} - -static uint64_t -xorshift64star(uint64_t *state) -{ - uint64_t value; - - value = *state; - value ^= value >> 12; - value ^= value << 25; - value ^= value >> 27; - *state = value; - return (value * UINT64_C(2685821657736338717)); -} - -static double -elapsed_seconds(const struct timespec *start, const struct timespec *end) -{ - return ((double)(end->tv_sec - start->tv_sec) + - (double)(end->tv_nsec - start->tv_nsec) / 1000000000.0); -} - -static int -random_probe(int argc, char **argv) -{ - struct stat source_status, target_status; - struct timespec start, end; - unsigned char *source_buffer, *target_buffer; - uint64_t block_size, digest, index, input_seed, operations, offset, seed; - uint64_t slots; - double elapsed; - int source, target, result; - - if (argc != 7 || parse_u64(argv[4], &seed) != 0 || seed == 0 || - parse_u64(argv[5], &operations) != 0 || operations == 0 || - parse_u64(argv[6], &block_size) != 0 || block_size == 0 || - block_size > SIZE_MAX) - return (1); - input_seed = seed; - source = open(argv[2], O_RDONLY); - target = open(argv[3], O_RDONLY); - if (source < 0 || target < 0 || fstat(source, &source_status) != 0 || - fstat(target, &target_status) != 0 || source_status.st_size <= 0 || - source_status.st_size != target_status.st_size || - (uint64_t)source_status.st_size < block_size) { - if (source >= 0) - close(source); - if (target >= 0) - close(target); - return (1); - } - source_buffer = malloc((size_t)block_size); - target_buffer = malloc((size_t)block_size); - if (source_buffer == NULL || target_buffer == NULL) { - close(source); - close(target); - free(source_buffer); - free(target_buffer); - return (1); - } - slots = (uint64_t)source_status.st_size / block_size; - digest = UINT64_C(1469598103934665603); - result = 1; - clock_gettime(CLOCK_MONOTONIC, &start); - for (index = 0; index < operations; index++) { - offset = (xorshift64star(&seed) % slots) * block_size; - if (read_exact_at(source, source_buffer, (size_t)block_size, - (off_t)offset) != 0 || - read_exact_at(target, target_buffer, (size_t)block_size, - (off_t)offset) != 0 || - memcmp(source_buffer, target_buffer, (size_t)block_size) != 0) - goto out; - digest = digest_bytes(digest, target_buffer, (size_t)block_size); - } - clock_gettime(CLOCK_MONOTONIC, &end); - elapsed = elapsed_seconds(&start, &end); - printf("seed=0x%016" PRIx64 " operations=%" PRIu64 - " block_size=%" PRIu64 - " bytes=%" PRIu64 " mismatches=0 digest=%016" PRIx64 - " elapsed_seconds=%.6f iops=%.2f mean_us=%.2f exit=0\n", - input_seed, operations, block_size, operations * block_size, digest, elapsed, - (double)operations / elapsed, elapsed * 1000000.0 / operations); - result = 0; -out: - close(source); - close(target); - free(source_buffer); - free(target_buffer); - return (result); -} - -static int -wait_for_path(const char *path, unsigned int attempts) -{ - struct timespec delay; - unsigned int attempt; - - delay.tv_sec = 0; - delay.tv_nsec = 100000000; - for (attempt = 0; attempt < attempts; attempt++) { - if (access(path, F_OK) == 0) - return (0); - nanosleep(&delay, NULL); - } - return (-1); -} - -static int -pressure_probe(const char *go_path, const char *stop_path, - const char *request_value) -{ - struct rusage usage; - unsigned char *recovery; - unsigned char **chunks; - uint64_t allocated, index, page_size, request_mib, requested; - int failure_errno, recovery_ok; - - if (parse_u64(request_value, &request_mib) != 0 || request_mib == 0 || - request_mib > 4096) - return (1); - requested = request_mib * 1024 * 1024; - chunks = calloc((size_t)(requested / PRESSURE_CHUNK + 1), sizeof(*chunks)); - if (chunks == NULL) - return (1); - printf("state=waiting pid=%ld requested_bytes=%" PRIu64 "\n", - (long)getpid(), requested); - fflush(stdout); - if (wait_for_path(go_path, 600) != 0) { - free(chunks); - return (1); - } - page_size = (uint64_t)sysconf(_SC_PAGESIZE); - allocated = 0; - failure_errno = 0; - for (index = 0; allocated < requested; index++) { - size_t offset; - - errno = 0; - chunks[index] = malloc(PRESSURE_CHUNK); - if (chunks[index] == NULL) { - failure_errno = errno; - break; - } - for (offset = 0; offset < PRESSURE_CHUNK; offset += page_size) - chunks[index][offset] = (unsigned char)(index + offset); - allocated += PRESSURE_CHUNK; - } - getrusage(RUSAGE_SELF, &usage); - printf("state=holding pid=%ld allocated_bytes=%" PRIu64 - " allocation_failure=%s failure_errno=%d maxrss=%ld\n", - (long)getpid(), allocated, - failure_errno == ENOMEM ? "ENOMEM" : "none", failure_errno, - usage.ru_maxrss); - fflush(stdout); - if (wait_for_path(stop_path, 3000) != 0) { - for (index = 0; index < allocated / PRESSURE_CHUNK; index++) - free(chunks[index]); - free(chunks); - return (1); - } - for (index = 0; index < allocated / PRESSURE_CHUNK; index++) - free(chunks[index]); - free(chunks); - recovery = malloc(1024 * 1024); - recovery_ok = recovery != NULL; - if (recovery != NULL) { - recovery[0] = 1; - free(recovery); - } - printf("state=released pid=%ld released_bytes=%" PRIu64 - " recovery=%s exit=%d\n", - (long)getpid(), allocated, recovery_ok ? "ok" : "failed", - failure_errno == ENOMEM && recovery_ok ? 0 : 1); - return (failure_errno == ENOMEM && recovery_ok ? 0 : 1); -} - -int -main(int argc, char **argv) -{ - if (argc == 3 && strcmp(argv[1], "empty") == 0) - return (empty_probe(argv[2])); - if (argc == 4 && strcmp(argv[1], "eof") == 0) - return (eof_probe(argv[2], argv[3])); - if (argc == 6 && strcmp(argv[1], "range") == 0) - return (range_probe(argv[2], argv[3], argv[4], argv[5])); - if (argc == 7 && strcmp(argv[1], "random") == 0) - return (random_probe(argc, argv)); - if (argc == 5 && strcmp(argv[1], "pressure") == 0) - return (pressure_probe(argv[2], argv[3], argv[4])); - usage(); - return (64); -} diff --git a/tests/mmap_fault.c b/tests/mmap_fault.c deleted file mode 100644 index f3ca7ca..0000000 --- a/tests/mmap_fault.c +++ /dev/null @@ -1,200 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static void -pread_all(int fd, unsigned char *buf, size_t len) -{ - size_t done; - ssize_t nr; - - for (done = 0; done < len; done += nr) { - nr = pread(fd, buf + done, len - done, done); - if (nr < 0) - err(1, "pread"); - if (nr == 0) - errx(1, "unexpected EOF at %zu", done); - } -} - -static uint64_t -fnv1a64(const unsigned char *buf, size_t len) -{ - uint64_t hash; - - hash = UINT64_C(14695981039346656037); - for (size_t i = 0; i < len; i++) { - hash ^= buf[i]; - hash *= UINT64_C(1099511628211); - } - return (hash); -} - -static uint32_t -xorshift32(uint32_t *state) -{ - uint32_t value; - - value = *state; - value ^= value << 13; - value ^= value >> 17; - value ^= value << 5; - *state = value; - return (value); -} - -static void -verify_random_faults(const unsigned char *map, const unsigned char *expected, - size_t size, size_t pagesize) -{ - size_t *order; - size_t npages, page, span; - uint32_t state; - - npages = (size + pagesize - 1) / pagesize; - order = calloc(npages, sizeof(*order)); - if (order == NULL) - err(1, "calloc page order"); - for (size_t i = 0; i < npages; i++) - order[i] = i; - state = UINT32_C(0x5eed22); - for (size_t i = npages; i > 1; i--) { - size_t other; - - other = xorshift32(&state) % i; - page = order[i - 1]; - order[i - 1] = order[other]; - order[other] = page; - } - for (size_t i = 0; i < npages; i++) { - page = order[i] * pagesize; - span = size - page < pagesize ? size - page : pagesize; - if (memcmp(map + page, expected + page, span) != 0) - errx(1, "random page mismatch at offset %zu", page); - } - free(order); -} - -static void -verify_sigbus(const unsigned char *map, size_t eof_page) -{ - pid_t child; - int status; - - child = fork(); - if (child < 0) - err(1, "fork"); - if (child == 0) { - volatile unsigned char value; - - value = map[eof_page]; - _exit(value == 0 ? 0 : 1); - } - if (waitpid(child, &status, 0) != child) - err(1, "waitpid"); - if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGBUS) - errx(1, "full page beyond EOF did not raise SIGBUS (status=%#x)", - status); -} - -int -main(int argc, char **argv) -{ - unsigned char *expected, *map, *private_map, byte; - struct stat sb; - size_t eof_page, map_len, pagesize, private_offset; - uint64_t hash; - int fd, rwfd, shared_errno; - - if (argc != 2) - errx(2, "usage: mmap_fault file"); - pagesize = (size_t)getpagesize(); - fd = open(argv[1], O_RDONLY); - if (fd < 0) - err(1, "open %s", argv[1]); - if (fstat(fd, &sb) != 0) - err(1, "fstat %s", argv[1]); - if (sb.st_size <= (off_t)(pagesize * 3)) - errx(1, "fixture must exceed three VM pages"); - if ((uintmax_t)sb.st_size > SIZE_MAX - pagesize * 2) - errx(1, "fixture is too large"); - eof_page = ((size_t)sb.st_size + pagesize - 1) / pagesize * pagesize; - map_len = eof_page + pagesize; - expected = malloc((size_t)sb.st_size); - if (expected == NULL) - err(1, "malloc expected data"); - pread_all(fd, expected, (size_t)sb.st_size); - - map = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fd, 0); - if (map == MAP_FAILED) - err(1, "mmap read-only private"); - if (madvise(map, map_len, MADV_DONTNEED) != 0) - err(1, "madvise MADV_DONTNEED"); - verify_random_faults(map, expected, (size_t)sb.st_size, pagesize); - if (memcmp(map, expected, (size_t)sb.st_size) != 0) - errx(1, "sequential mmap data differs from pread"); - for (size_t i = (size_t)sb.st_size; i < eof_page; i++) { - if (map[i] != 0) - errx(1, "non-zero byte in partial EOF page at %zu", i); - } - verify_sigbus(map, eof_page); - hash = fnv1a64(map, (size_t)sb.st_size); - - errno = 0; - private_map = mmap(NULL, eof_page, PROT_READ | PROT_WRITE, MAP_SHARED, - fd, 0); - shared_errno = errno; - if (private_map != MAP_FAILED) { - munmap(private_map, eof_page); - errx(1, "writable MAP_SHARED unexpectedly succeeded"); - } - if (shared_errno != EACCES) - errx(1, "writable MAP_SHARED returned %s, expected Permission denied", - strerror(shared_errno)); - - private_map = mmap(NULL, eof_page, PROT_READ | PROT_WRITE, MAP_PRIVATE, - fd, 0); - if (private_map == MAP_FAILED) - err(1, "mmap writable private"); - private_offset = (size_t)sb.st_size / 2; - byte = expected[private_offset]; - private_map[private_offset] ^= UINT8_C(0xff); - if (private_map[private_offset] == byte) - errx(1, "private mapping did not change"); - if (pread(fd, &byte, 1, private_offset) != 1) - err(1, "pread private verification"); - if (byte != expected[private_offset]) - errx(1, "MAP_PRIVATE modified the EROFS file"); - - errno = 0; - rwfd = open(argv[1], O_RDWR); - if (rwfd >= 0) { - close(rwfd); - errx(1, "O_RDWR unexpectedly succeeded"); - } - if (errno != EROFS) - errx(1, "O_RDWR returned %s, expected Read-only file system", - strerror(errno)); - - if (munmap(private_map, eof_page) != 0 || munmap(map, map_len) != 0) - err(1, "munmap"); - free(expected); - if (close(fd) != 0) - err(1, "close"); - printf("PASS size=%jd pages=%zu fnv1a64=%016" PRIx64 - " random-faults=%zu eof-zero=PASS sigbus=PASS private-cow=PASS\n", - (intmax_t)sb.st_size, eof_page / pagesize, hash, - ((size_t)sb.st_size + pagesize - 1) / pagesize); - return (0); -} diff --git a/tests/nfs_fh_tool.c b/tests/nfs_fh_tool.c deleted file mode 100644 index 2f252d0..0000000 --- a/tests/nfs_fh_tool.c +++ /dev/null @@ -1,283 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct erofs_test_fid { - uint16_t len; - uint16_t pad; - uint32_t nid_hi; - uint32_t nid_lo; - uint32_t gen; -}; - -_Static_assert(sizeof(struct erofs_test_fid) == 16, - "unexpected EROFS test file handle size"); -_Static_assert(sizeof(struct erofs_test_fid) <= sizeof(struct fid), - "EROFS test file handle does not fit struct fid"); - -static void -usage(void) -{ - fprintf(stderr, - "usage:\n" - " nfs_fh_tool capture path handle\n" - " nfs_fh_tool lcapture path handle\n" - " nfs_fh_tool describe handle\n" - " nfs_fh_tool compare handle1 handle2\n" - " nfs_fh_tool stat handle\n" - " nfs_fh_tool cat handle output\n" - " nfs_fh_tool mutate input output field value\n" - " nfs_fh_tool expect-stat handle errno\n" - " nfs_fh_tool expect-open handle errno\n"); - exit(2); -} - -static void -read_handle(const char *path, fhandle_t *fh) -{ - FILE *fp; - - fp = fopen(path, "rb"); - if (fp == NULL) - err(1, "fopen %s", path); - if (fread(fh, sizeof(*fh), 1, fp) != 1) - err(1, "fread %s", path); - if (fgetc(fp) != EOF) - errx(1, "%s has trailing data", path); - if (fclose(fp) != 0) - err(1, "fclose %s", path); -} - -static void -write_handle(const char *path, const fhandle_t *fh) -{ - FILE *fp; - - fp = fopen(path, "wb"); - if (fp == NULL) - err(1, "fopen %s", path); - if (fwrite(fh, sizeof(*fh), 1, fp) != 1) - err(1, "fwrite %s", path); - if (fclose(fp) != 0) - err(1, "fclose %s", path); -} - -static int -parse_errno(const char *name) -{ - char *end; - long value; - - if (strcmp(name, "EINVAL") == 0) - return (EINVAL); - if (strcmp(name, "ESTALE") == 0) - return (ESTALE); - errno = 0; - value = strtol(name, &end, 0); - if (errno != 0 || *end != '\0' || value <= 0 || value > INT_MAX) - errx(2, "invalid errno: %s", name); - return ((int)value); -} - -static uint64_t -parse_value(const char *text) -{ - char *end; - uintmax_t value; - - errno = 0; - value = strtoumax(text, &end, 0); - if (errno != 0 || *end != '\0' || value > UINT64_MAX) - errx(2, "invalid value: %s", text); - return ((uint64_t)value); -} - -static void -capture(const char *path, const char *output, int nofollow) -{ - fhandle_t fh; - int error; - - bzero(&fh, sizeof(fh)); - error = nofollow ? lgetfh(path, &fh) : getfh(path, &fh); - if (error != 0) - err(1, "%s %s", nofollow ? "lgetfh" : "getfh", path); - write_handle(output, &fh); -} - -static void -describe(const char *path) -{ - struct erofs_test_fid efid; - fhandle_t fh; - uint64_t nid; - - read_handle(path, &fh); - bzero(&efid, sizeof(efid)); - memcpy(&efid, &fh.fh_fid, sizeof(efid)); - nid = ((uint64_t)efid.nid_hi << 32) | efid.nid_lo; - printf("fsid=%08x:%08x len=%u pad=%u nid=%016" PRIx64 - " gen=%u\n", (unsigned int)fh.fh_fsid.val[0], - (unsigned int)fh.fh_fsid.val[1], efid.len, efid.pad, nid, efid.gen); -} - -static void -compare(const char *left, const char *right) -{ - fhandle_t a, b; - - read_handle(left, &a); - read_handle(right, &b); - if (memcmp(&a, &b, sizeof(a)) != 0) - errx(1, "file handles differ"); -} - -static void -stat_handle(const char *path) -{ - fhandle_t fh; - struct stat sb; - - read_handle(path, &fh); - if (fhstat(&fh, &sb) != 0) - err(1, "fhstat %s", path); - printf("mode=%#o ino=%ju gen=%u size=%jd\n", (unsigned int)sb.st_mode, - (uintmax_t)sb.st_ino, (unsigned int)sb.st_gen, - (intmax_t)sb.st_size); -} - -static void -cat_handle(const char *handle, const char *output) -{ - char buf[65536]; - fhandle_t fh; - ssize_t nr, nw; - int fd, outfd; - - read_handle(handle, &fh); - fd = fhopen(&fh, O_RDONLY); - if (fd < 0) - err(1, "fhopen %s", handle); - outfd = open(output, O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (outfd < 0) - err(1, "open %s", output); - while ((nr = read(fd, buf, sizeof(buf))) > 0) { - for (ssize_t done = 0; done < nr; done += nw) { - nw = write(outfd, buf + done, nr - done); - if (nw < 0) - err(1, "write %s", output); - } - } - if (nr < 0) - err(1, "read %s", handle); - if (close(outfd) != 0) - err(1, "close %s", output); - if (close(fd) != 0) - err(1, "close fhopen"); -} - -static void -mutate(const char *input, const char *output, const char *field, - const char *text) -{ - struct erofs_test_fid efid; - fhandle_t fh; - uint64_t value; - - read_handle(input, &fh); - bzero(&efid, sizeof(efid)); - memcpy(&efid, &fh.fh_fid, sizeof(efid)); - value = parse_value(text); - if (strcmp(field, "len") == 0) { - if (value > UINT16_MAX) - errx(2, "len is too large"); - efid.len = value; - } else if (strcmp(field, "pad") == 0) { - if (value > UINT16_MAX) - errx(2, "pad is too large"); - efid.pad = value; - } else if (strcmp(field, "nid_hi") == 0) { - if (value > UINT32_MAX) - errx(2, "nid_hi is too large"); - efid.nid_hi = value; - } else if (strcmp(field, "nid_lo") == 0) { - if (value > UINT32_MAX) - errx(2, "nid_lo is too large"); - efid.nid_lo = value; - } else if (strcmp(field, "gen") == 0) { - if (value > UINT32_MAX) - errx(2, "gen is too large"); - efid.gen = value; - } else if (strcmp(field, "gen_xor") == 0) { - if (value == 0 || value > UINT32_MAX) - errx(2, "gen_xor must be a non-zero uint32_t"); - efid.gen ^= value; - } else { - errx(2, "unknown field: %s", field); - } - memcpy(&fh.fh_fid, &efid, sizeof(efid)); - write_handle(output, &fh); -} - -static void -expect_failure(const char *path, const char *error_name, int use_open) -{ - fhandle_t fh; - struct stat sb; - int expected, fd, result; - - read_handle(path, &fh); - expected = parse_errno(error_name); - errno = 0; - if (use_open) { - fd = fhopen(&fh, O_RDONLY); - result = fd; - if (fd >= 0) - close(fd); - } else { - result = fhstat(&fh, &sb); - } - if (result != -1) - errx(1, "%s unexpectedly succeeded", use_open ? "fhopen" : "fhstat"); - if (errno != expected) - errx(1, "%s returned %s, expected %s", - use_open ? "fhopen" : "fhstat", strerror(errno), - strerror(expected)); -} - -int -main(int argc, char **argv) -{ - if (argc == 4 && strcmp(argv[1], "capture") == 0) - capture(argv[2], argv[3], 0); - else if (argc == 4 && strcmp(argv[1], "lcapture") == 0) - capture(argv[2], argv[3], 1); - else if (argc == 3 && strcmp(argv[1], "describe") == 0) - describe(argv[2]); - else if (argc == 4 && strcmp(argv[1], "compare") == 0) - compare(argv[2], argv[3]); - else if (argc == 3 && strcmp(argv[1], "stat") == 0) - stat_handle(argv[2]); - else if (argc == 4 && strcmp(argv[1], "cat") == 0) - cat_handle(argv[2], argv[3]); - else if (argc == 6 && strcmp(argv[1], "mutate") == 0) - mutate(argv[2], argv[3], argv[4], argv[5]); - else if (argc == 4 && strcmp(argv[1], "expect-stat") == 0) - expect_failure(argv[2], argv[3], 0); - else if (argc == 4 && strcmp(argv[1], "expect-open") == 0) - expect_failure(argv[2], argv[3], 1); - else - usage(); - return (0); -} diff --git a/tests/pre13_ondisk_layout_probe.c b/tests/pre13_ondisk_layout_probe.c deleted file mode 100644 index 3a046b8..0000000 --- a/tests/pre13_ondisk_layout_probe.c +++ /dev/null @@ -1,244 +0,0 @@ -#include -#include -#include - -#include "../src/erofs_fs.h" - -struct legacy_erofs_super_block { - uint32_t magic; - uint32_t checksum; - uint32_t feature_compat; - uint8_t blkszbits; - uint8_t sb_extslots; - union { - uint16_t rootnid_2b; - uint16_t blocks_hi; - } __packed rb; - uint64_t inos; - uint64_t epoch; - uint32_t fixed_nsec; - uint32_t blocks_lo; - uint32_t meta_blkaddr; - uint32_t xattr_blkaddr; - uint8_t uuid[16]; - uint8_t volume_name[16]; - uint32_t feature_incompat; - union { - uint16_t available_compr_algs; - uint16_t lz4_max_distance; - } __packed u1; - uint16_t extra_devices; - uint16_t devt_slotoff; - uint8_t dirblkbits; - uint8_t xattr_prefix_count; - uint32_t xattr_prefix_start; - uint64_t packed_nid; - uint8_t xattr_filter_reserved; - uint8_t ishare_xattr_prefix_id; - uint8_t reserved[2]; - uint32_t build_time; - uint64_t rootnid_8b; - uint64_t reserved2; - uint64_t metabox_nid; - uint64_t reserved3; -} __packed; - -union legacy_erofs_inode_i_nb { - uint16_t nlink; - uint16_t blocks_hi; - uint16_t startblk_hi; -} __packed; - -struct legacy_erofs_inode_compact { - uint16_t i_format; - uint16_t i_xattr_icount; - uint16_t i_mode; - union legacy_erofs_inode_i_nb i_nb; - uint32_t i_size; - uint32_t i_mtime; - union erofs_inode_i_u i_u; - uint32_t i_ino; - uint16_t i_uid; - uint16_t i_gid; - uint32_t i_reserved; -} __packed; - -struct legacy_erofs_inode_extended { - uint16_t i_format; - uint16_t i_xattr_icount; - uint16_t i_mode; - union legacy_erofs_inode_i_nb i_nb; - uint64_t i_size; - union erofs_inode_i_u i_u; - uint32_t i_ino; - uint32_t i_uid; - uint32_t i_gid; - uint64_t i_mtime; - uint32_t i_mtime_nsec; - uint32_t i_nlink; - uint8_t i_reserved2[16]; -} __packed; - -struct legacy_erofs_dirent { - uint64_t nid; - uint16_t nameoff; - uint8_t file_type; - uint8_t reserved; -} __packed; - -struct legacy_erofs_xattr_ibody_header { - uint32_t h_name_filter; - uint8_t h_shared_count; - uint8_t h_reserved2[7]; - uint32_t h_shared_xattrs[0]; -} __packed; - -struct legacy_erofs_xattr_long_prefix { - uint8_t base_index; - char infix[0]; -} __packed; - -struct legacy_erofs_xattr_entry { - uint8_t e_name_len; - uint8_t e_name_index; - uint16_t e_value_size; - char e_name[]; -} __packed; - -#define ASSERT_LAYOUT(current, legacy) \ - _Static_assert(sizeof(current) == sizeof(legacy), "size " #current); \ - _Static_assert(_Alignof(current) == _Alignof(legacy), "align " #current) -#define ASSERT_OFFSET(current, legacy, field) \ - _Static_assert(offsetof(current, field) == offsetof(legacy, field), \ - "offset " #current "." #field) - -ASSERT_LAYOUT(struct erofs_super_block, struct legacy_erofs_super_block); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, magic); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, checksum); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - feature_compat); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - blkszbits); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - sb_extslots); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, rb); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, inos); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, epoch); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - fixed_nsec); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - blocks_lo); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - meta_blkaddr); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - xattr_blkaddr); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, uuid); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - volume_name); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - feature_incompat); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, u1); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - extra_devices); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - devt_slotoff); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - dirblkbits); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - xattr_prefix_count); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - xattr_prefix_start); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - packed_nid); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - xattr_filter_reserved); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - ishare_xattr_prefix_id); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - reserved); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - build_time); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - rootnid_8b); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - reserved2); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - metabox_nid); -ASSERT_OFFSET(struct erofs_super_block, struct legacy_erofs_super_block, - reserved3); - -ASSERT_LAYOUT(union erofs_inode_i_nb, union legacy_erofs_inode_i_nb); -ASSERT_OFFSET(union erofs_inode_i_nb, union legacy_erofs_inode_i_nb, nlink); -ASSERT_OFFSET(union erofs_inode_i_nb, union legacy_erofs_inode_i_nb, blocks_hi); -ASSERT_OFFSET(union erofs_inode_i_nb, union legacy_erofs_inode_i_nb, - startblk_hi); - -#define ASSERT_INODE_LAYOUT(current, legacy) \ - ASSERT_LAYOUT(current, legacy); \ - ASSERT_OFFSET(current, legacy, i_format); \ - ASSERT_OFFSET(current, legacy, i_xattr_icount); \ - ASSERT_OFFSET(current, legacy, i_mode); \ - ASSERT_OFFSET(current, legacy, i_nb); \ - ASSERT_OFFSET(current, legacy, i_size); \ - ASSERT_OFFSET(current, legacy, i_u); \ - ASSERT_OFFSET(current, legacy, i_ino); \ - ASSERT_OFFSET(current, legacy, i_uid); \ - ASSERT_OFFSET(current, legacy, i_gid) - -ASSERT_INODE_LAYOUT(struct erofs_inode_compact, - struct legacy_erofs_inode_compact); -ASSERT_OFFSET(struct erofs_inode_compact, struct legacy_erofs_inode_compact, - i_mtime); -ASSERT_OFFSET(struct erofs_inode_compact, struct legacy_erofs_inode_compact, - i_reserved); - -ASSERT_INODE_LAYOUT(struct erofs_inode_extended, - struct legacy_erofs_inode_extended); -ASSERT_OFFSET(struct erofs_inode_extended, struct legacy_erofs_inode_extended, - i_mtime); -ASSERT_OFFSET(struct erofs_inode_extended, struct legacy_erofs_inode_extended, - i_mtime_nsec); -ASSERT_OFFSET(struct erofs_inode_extended, struct legacy_erofs_inode_extended, - i_nlink); -ASSERT_OFFSET(struct erofs_inode_extended, struct legacy_erofs_inode_extended, - i_reserved2); - -ASSERT_LAYOUT(struct erofs_dirent, struct legacy_erofs_dirent); -ASSERT_OFFSET(struct erofs_dirent, struct legacy_erofs_dirent, nid); -ASSERT_OFFSET(struct erofs_dirent, struct legacy_erofs_dirent, nameoff); -ASSERT_OFFSET(struct erofs_dirent, struct legacy_erofs_dirent, file_type); -ASSERT_OFFSET(struct erofs_dirent, struct legacy_erofs_dirent, reserved); - -ASSERT_LAYOUT(struct erofs_xattr_ibody_header, - struct legacy_erofs_xattr_ibody_header); -ASSERT_OFFSET(struct erofs_xattr_ibody_header, - struct legacy_erofs_xattr_ibody_header, h_name_filter); -ASSERT_OFFSET(struct erofs_xattr_ibody_header, - struct legacy_erofs_xattr_ibody_header, h_shared_count); -ASSERT_OFFSET(struct erofs_xattr_ibody_header, - struct legacy_erofs_xattr_ibody_header, h_reserved2); -ASSERT_OFFSET(struct erofs_xattr_ibody_header, - struct legacy_erofs_xattr_ibody_header, h_shared_xattrs); - -ASSERT_LAYOUT(struct erofs_xattr_long_prefix, - struct legacy_erofs_xattr_long_prefix); -ASSERT_OFFSET(struct erofs_xattr_long_prefix, - struct legacy_erofs_xattr_long_prefix, base_index); -ASSERT_OFFSET(struct erofs_xattr_long_prefix, - struct legacy_erofs_xattr_long_prefix, infix); - -ASSERT_LAYOUT(struct erofs_xattr_entry, struct legacy_erofs_xattr_entry); -ASSERT_OFFSET(struct erofs_xattr_entry, struct legacy_erofs_xattr_entry, - e_name_len); -ASSERT_OFFSET(struct erofs_xattr_entry, struct legacy_erofs_xattr_entry, - e_name_index); -ASSERT_OFFSET(struct erofs_xattr_entry, struct legacy_erofs_xattr_entry, - e_value_size); -ASSERT_OFFSET(struct erofs_xattr_entry, struct legacy_erofs_xattr_entry, - e_name); - -int -main(void) -{ - return (0); -} diff --git a/tests/pre15/EVIDENCE-SCHEMA.json b/tests/pre15/EVIDENCE-SCHEMA.json deleted file mode 100644 index f092dc6..0000000 --- a/tests/pre15/EVIDENCE-SCHEMA.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://erofs.example/pre15/evidence.schema.json", - "title": "EROFS FreeBSD Pre15 immutable run evidence", - "type": "object", - "additionalProperties": false, - "required": [ - "schema_version", - "run_id", - "mode", - "case_id", - "dut_head", - "dut_tree", - "worktree_diff_sha256", - "freebsd_source", - "linux_source", - "module_sha256", - "fixture_sha256", - "command_argv", - "start_utc", - "end_utc", - "deadline_seconds", - "exit_code", - "timed_out", - "target_marker", - "status", - "reason", - "failure_origin", - "cleanup", - "stdout", - "stderr", - "raw_sha256", - "ownership_manifest", - "cleanup_log", - "identity", - "fixtures", - "module" - ], - "properties": { - "schema_version": { "const": 1 }, - "run_id": { "type": "string", "minLength": 1 }, - "mode": { "enum": ["host", "build", "qemu", "smoke", "control"] }, - "case_id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" }, - "dut_head": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, - "dut_tree": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, - "worktree_diff_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, - "freebsd_source": { "type": "string", "minLength": 1 }, - "linux_source": { "type": "string", "minLength": 1 }, - "module_sha256": { - "oneOf": [ - { "type": "null" }, - { "type": "string", "pattern": "^[0-9a-f]{64}$" } - ] - }, - "fixture_sha256": { - "type": "array", - "items": { "type": "string", "pattern": "^[0-9a-f]{64}$" } - }, - "command_argv": { - "type": "array", - "minItems": 1, - "items": { "type": "string" } - }, - "start_utc": { "type": "string", "format": "date-time" }, - "end_utc": { "type": "string", "format": "date-time" }, - "deadline_seconds": { "type": "integer", "minimum": 1 }, - "exit_code": { "type": ["integer", "null"] }, - "timed_out": { "type": "boolean" }, - "target_marker": { "enum": ["reached", "not_reached"] }, - "status": { - "enum": ["PASS", "DUT_FAIL", "RUNNER_FAIL", "INFRA_BLOCKED", "STOP", "NOT_RUN"] - }, - "reason": { "type": "string", "minLength": 1 }, - "failure_origin": { "enum": ["none", "dut", "runner", "infrastructure", "gate", "plan"] }, - "cleanup": { "enum": ["PASS", "FAIL"] }, - "stdout": { "type": "string", "minLength": 1 }, - "stderr": { "type": "string", "minLength": 1 }, - "raw_sha256": { - "type": "object", - "additionalProperties": false, - "required": ["stdout", "stderr", "ownership", "cleanup"], - "properties": { - "stdout": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, - "stderr": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, - "ownership": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, - "cleanup": { "type": "string", "pattern": "^[0-9a-f]{64}$" } - } - }, - "ownership_manifest": { "type": "string", "minLength": 1 }, - "cleanup_log": { "type": "string", "minLength": 1 }, - "identity": { "type": "object" }, - "fixtures": { "type": "array" }, - "module": { "type": ["object", "null"] } - } -} diff --git a/tests/pre15/cases/B01-g3-equivalence.sh b/tests/pre15/cases/B01-g3-equivalence.sh deleted file mode 100755 index 78072c1..0000000 --- a/tests/pre15/cases/B01-g3-equivalence.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" -umask 022 - -spec=$PRE15_DUT/tests/pre15/fixtures/B01-g3-archives.json -helper=$PRE15_DUT/tests/pre15/fixtures/g3.py -metadata_archive=$PRE15_DUT/tests/results/manual/2026-08-08T2337Z-metadata-vfs/prepare-fixtures.sh -final_archive=$PRE15_DUT/tests/results/manual/2026-08-09T0124Z-final-review/prepare-fixtures.sh - -for tool in mkfs.erofs fsck.erofs dump.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing G3 equivalence tool: $tool" -done -test "$(id -u)" -eq 0 || \ - pre15_infra_blocked 'root is required for deterministic device-node fixtures' - -python3 - "$spec" "$PRE15_DUT" <<'PY' -import hashlib -import json -from pathlib import Path -import sys - -spec = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -dut = Path(sys.argv[2]) -for item in spec["archives"]: - path = dut / item["path"] - data = path.read_bytes() - digest = hashlib.sha256(data).hexdigest() - lines = len(data.splitlines()) - if digest != item["sha256"] or lines != item["lines"]: - raise SystemExit( - f"{item['id']}: hash/line mismatch {digest}/{lines}, " - f"expected {item['sha256']}/{item['lines']}" - ) -PY - -pre15_record_fixture g3-archive-spec "$spec" -pre15_record_fixture g3-stable-helper "$helper" -pre15_record_fixture g3-archive-metadata "$metadata_archive" -pre15_record_fixture g3-archive-final "$final_archive" - -mkdir "$PRE15_CASE_TMP/archive-metadata-source" "$PRE15_CASE_TMP/archive-metadata" -rmdir "$PRE15_CASE_TMP/archive-metadata-source" "$PRE15_CASE_TMP/archive-metadata" -FIXTURE_DIR="$PRE15_CASE_TMP/archive-metadata-source" \ -ARTIFACT_DIR="$PRE15_CASE_TMP/archive-metadata" "$metadata_archive" -find "$PRE15_CASE_TMP/archive-metadata-source/namei/wide" \ - -mindepth 1 -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort \ - > "$PRE15_CASE_TMP/archive-metadata/expected-wide.txt" - -python3 -B "$helper" make-metadata \ - --source "$PRE15_CASE_TMP/stable-a-metadata-source" \ - --output "$PRE15_CASE_TMP/stable-a-metadata" -python3 -B "$helper" make-metadata \ - --source "$PRE15_CASE_TMP/stable-b-metadata-source" \ - --output "$PRE15_CASE_TMP/stable-b-metadata" -mkdir -p "$PRE15_RUN_DIR/artifacts" -python3 -B "$helper" compare-set \ - --left-source "$PRE15_CASE_TMP/archive-metadata-source" \ - --left-artifacts "$PRE15_CASE_TMP/archive-metadata" \ - --right-source "$PRE15_CASE_TMP/stable-a-metadata-source" \ - --right-artifacts "$PRE15_CASE_TMP/stable-a-metadata" \ - --output "$PRE15_RUN_DIR/artifacts/metadata-archive-equivalence.json" -python3 -B "$helper" compare-set \ - --left-source "$PRE15_CASE_TMP/stable-a-metadata-source" \ - --left-artifacts "$PRE15_CASE_TMP/stable-a-metadata" \ - --right-source "$PRE15_CASE_TMP/stable-b-metadata-source" \ - --right-artifacts "$PRE15_CASE_TMP/stable-b-metadata" \ - --output "$PRE15_RUN_DIR/artifacts/metadata-repeat-equivalence.json" - -FIXTURE_DIR="$PRE15_CASE_TMP/archive-final-source" \ -ARTIFACT_DIR="$PRE15_CASE_TMP/archive-final" "$final_archive" -python3 -B "$helper" make-final \ - --source "$PRE15_CASE_TMP/stable-a-final-source" \ - --output "$PRE15_CASE_TMP/stable-a-final" -python3 -B "$helper" make-final \ - --source "$PRE15_CASE_TMP/stable-b-final-source" \ - --output "$PRE15_CASE_TMP/stable-b-final" -python3 -B "$helper" compare-set \ - --left-source "$PRE15_CASE_TMP/archive-final-source" \ - --left-artifacts "$PRE15_CASE_TMP/archive-final" \ - --right-source "$PRE15_CASE_TMP/stable-a-final-source" \ - --right-artifacts "$PRE15_CASE_TMP/stable-a-final" \ - --output "$PRE15_RUN_DIR/artifacts/final-archive-equivalence.json" -python3 -B "$helper" compare-set \ - --left-source "$PRE15_CASE_TMP/stable-a-final-source" \ - --left-artifacts "$PRE15_CASE_TMP/stable-a-final" \ - --right-source "$PRE15_CASE_TMP/stable-b-final-source" \ - --right-artifacts "$PRE15_CASE_TMP/stable-b-final" \ - --output "$PRE15_RUN_DIR/artifacts/final-repeat-equivalence.json" - -for evidence in \ - metadata-archive-equivalence.json metadata-repeat-equivalence.json \ - final-archive-equivalence.json final-repeat-equivalence.json; do - pre15_record_fixture "g3-$evidence" "$PRE15_RUN_DIR/artifacts/$evidence" -done - -python3 - "$PRE15_RUN_DIR/artifacts" <<'PY' -import json -from pathlib import Path -import sys - -root = Path(sys.argv[1]) -names = ( - "metadata-archive-equivalence.json", - "metadata-repeat-equivalence.json", - "final-archive-equivalence.json", - "final-repeat-equivalence.json", -) -for name in names: - result = json.loads((root / name).read_text(encoding="ascii")) - if result["status"] != "PASS": - raise SystemExit(f"{name}: {result['status']}") -print("G3 archived=548 lines; source/artifact bytes and expectations are equivalent") -PY - -pre15_target_reached diff --git a/tests/pre15/cases/B01-runner-selftest.sh b/tests/pre15/cases/B01-runner-selftest.sh deleted file mode 100755 index 066da8c..0000000 --- a/tests/pre15/cases/B01-runner-selftest.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -controls=$PRE15_DUT/tests/pre15/fixtures/B01-runner-controls.json -pre15_record_fixture runner-control-contract "$controls" -pre15_record_fixture evidence-schema "$PRE15_DUT/tests/pre15/EVIDENCE-SCHEMA.json" - -command -v qemu-system-x86_64 >/dev/null 2>&1 || \ - pre15_infra_blocked 'qemu-system-x86_64 is required for the early-exit control' -command -v ssh >/dev/null 2>&1 || \ - pre15_infra_blocked 'ssh is required for the connection-failure control' - -pre15_control_run "$PRE15_CASE_TMP/known-good" 5 \ - sh -c 'printf "reached\n" > "$PRE15_TARGET_MARKER_FILE"' -pre15_control_run "$PRE15_CASE_TMP/dut-mismatch" 5 \ - sh -c 'printf "reached\n" > "$PRE15_TARGET_MARKER_FILE"; exit 10' -pre15_control_run "$PRE15_CASE_TMP/guest-command-failure" 5 sh -c 'exit 1' -pre15_control_run "$PRE15_CASE_TMP/ssh-failure" 5 sh -c \ - 'if ssh -o BatchMode=yes -o ConnectTimeout=1 -p 1 root@127.0.0.1 true >/dev/null 2>&1; then exit 20; fi; exit 21' -pre15_control_run "$PRE15_CASE_TMP/qemu-early-exit" 5 sh -c \ - 'qemu-system-x86_64 --pre15-invalid-option >/dev/null 2>&1; qemu_rc=$?; test "$qemu_rc" -ne 0 || exit 20; exit 21' - -timeout_pid_file=$PRE15_CASE_TMP/timeout.pid -export timeout_pid_file -pre15_control_run "$PRE15_CASE_TMP/timeout" 1 sh -c \ - 'printf "%s\n" "$$" > "$timeout_pid_file"; exec sleep 30' -timeout_pid=$(cat "$timeout_pid_file") -if kill -0 "$timeout_pid" 2>/dev/null; then - pre15_runner_fail "timeout control left PID $timeout_pid alive" -fi - -pre15_control_run "$PRE15_CASE_TMP/owned-cleanup" 5 sh -c ' - sleep 30 >/dev/null 2>&1 & - owned_pid=$! - printf "pid\t%s\tselftest-owned-sleep\n" "$owned_pid" >> "$PRE15_OWNERSHIP_FILE" - printf "%s\n" "$owned_pid" > "$PRE15_RUN_DIR/owned.pid" - printf "reached\n" > "$PRE15_TARGET_MARKER_FILE" -' -owned_pid=$(cat "$PRE15_CASE_TMP/owned-cleanup/owned.pid") -if kill -0 "$owned_pid" 2>/dev/null; then - pre15_runner_fail "owned cleanup left PID $owned_pid alive" -fi - -outside_path=$PRE15_CASE_TMP/cleanup-must-refuse -printf 'owned by outer selftest\n' > "$outside_path" -export outside_path -pre15_control_run "$PRE15_CASE_TMP/cleanup-failure" 5 sh -c ' - printf "path\t%s\toutside-control-boundary\n" "$outside_path" >> "$PRE15_OWNERSHIP_FILE" - printf "reached\n" > "$PRE15_TARGET_MARKER_FILE" -' -test -f "$outside_path" || \ - pre15_runner_fail 'cleanup safety control deleted a non-owned path' - -python3 - "$controls" "$PRE15_CASE_TMP" <<'PY' -import json -from pathlib import Path -import sys - -contract = json.loads(Path(sys.argv[1]).read_text(encoding="ascii"))["controls"] -root = Path(sys.argv[2]) -for name, expected in contract.items(): - actual = (root / name / "result.tsv").read_text(encoding="ascii").rstrip("\n").split("\t") - if actual != expected: - raise SystemExit(f"{name}: {actual!r}, expected {expected!r}") - print(f"{name}: status={actual[0]} origin={actual[1]} cleanup={actual[2]} timeout={actual[3]}") -PY - -mkdir -p "$PRE15_RUN_DIR/artifacts/controls" -for control in \ - cleanup-failure dut-mismatch guest-command-failure known-good \ - owned-cleanup qemu-early-exit ssh-failure timeout; do - mkdir "$PRE15_RUN_DIR/artifacts/controls/$control" - cp "$PRE15_CASE_TMP/$control/result.tsv" \ - "$PRE15_CASE_TMP/$control/stdout.log" \ - "$PRE15_CASE_TMP/$control/stderr.log" \ - "$PRE15_CASE_TMP/$control/cleanup.log" \ - "$PRE15_RUN_DIR/artifacts/controls/$control/" -done - -pre15_target_reached diff --git a/tests/pre15/cases/B02-layout.sh b/tests/pre15/cases/B02-layout.sh deleted file mode 100755 index 7490a95..0000000 --- a/tests/pre15/cases/B02-layout.sh +++ /dev/null @@ -1,319 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_FREEBSD_SRC:?PRE15_FREEBSD_SRC is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -probe=$PRE15_DUT/tests/pre15/probes/ondisk_layout.c -freebsd_header=$PRE15_DUT/src/erofs_fs.h -internal_header=$PRE15_DUT/src/internal.h -linux_header=$PRE15_ROOT/src-linux/erofs_fs.h -sys=$PRE15_FREEBSD_SRC/sys -target=x86_64-unknown-freebsd15.0 -artifacts=$PRE15_RUN_DIR/artifacts -include_root=$PRE15_CASE_TMP/include - -for tool in clang python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B02 host tool: $tool" -done -test -d "$sys/amd64/include" || \ - pre15_infra_blocked "FreeBSD amd64 headers are absent: $sys" - -pre15_record_fixture b02-layout-probe "$probe" -pre15_record_fixture b02-freebsd-ondisk-header "$freebsd_header" -pre15_record_fixture b02-freebsd-internal-header "$internal_header" -pre15_record_fixture b02-linux-ondisk-header "$linux_header" - -mkdir -p "$artifacts" "$include_root" -ln -s "$sys/amd64/include" "$include_root/machine" -ln -s "$sys/x86/include" "$include_root/x86" -resource_include=$(clang -print-resource-dir)/include -pre15_target_reached - -if clang --target="$target" -fsyntax-only -std=gnu17 -Wall -Wextra -Werror \ - -nostdinc -isystem "$resource_include" \ - -isystem "$PRE15_FREEBSD_SRC/include" -isystem "$include_root" \ - -isystem "$sys" -Xclang -fdump-record-layouts "$probe" \ - > "$artifacts/freebsd-record-layouts.txt" \ - 2> "$artifacts/freebsd-layout.stderr"; then - : -else - pre15_dut_fail 'FreeBSD ondisk layout/type/macro oracle failed' -fi -if clang -DB02_LINUX_REFERENCE -fsyntax-only -std=gnu17 \ - -Wall -Wextra -Werror -Xclang -fdump-record-layouts "$probe" \ - > "$artifacts/linux-record-layouts.txt" \ - 2> "$artifacts/linux-layout.stderr"; then - : -else - pre15_dut_fail 'Linux reference layout/type/macro oracle failed' -fi - -clang --target="$target" -E -dM -std=gnu17 -nostdinc \ - -isystem "$resource_include" -isystem "$PRE15_FREEBSD_SRC/include" \ - -isystem "$include_root" -isystem "$sys" "$probe" | \ - awk '$2 ~ /^(EROFS_|Z_EROFS_)/ { print }' | LC_ALL=C sort \ - > "$artifacts/freebsd-macros.txt" -clang -DB02_LINUX_REFERENCE -E -dM -std=gnu17 "$probe" | \ - awk '$2 ~ /^(EROFS_|Z_EROFS_)/ { print }' | LC_ALL=C sort \ - > "$artifacts/linux-macros.txt" - -if python3 - "$PRE15_ROOT" "$artifacts/static-check.json" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import sys - - -root = Path(sys.argv[1]) -output = Path(sys.argv[2]) -dut = root / "repo-pre-15" -src = dut / "src" -ondisk = (src / "erofs_fs.h").read_text(encoding="utf-8") -internal = (src / "internal.h").read_text(encoding="utf-8") -linux = (root / "src-linux/erofs_fs.h").read_text(encoding="utf-8") - - -def require_once(source: str, token: str) -> None: - count = source.count(token) - if count != 1: - raise SystemExit(f"expected one occurrence of {token!r}, found {count}") - - -def require_order(source: str, tokens: tuple[str, ...], label: str) -> None: - positions = [] - for token in tokens: - require_once(source, token) - positions.append(source.index(token)) - if positions != sorted(positions): - raise SystemExit(f"{label} section order mismatch: {tokens!r}") - - -ondisk_markers = ( - "struct erofs_deviceslot {", - "struct erofs_super_block {", - "EROFS_INODE_CHUNK_BASED", - "#define EROFS_CHUNK_FORMAT_BLKBITS_MASK", - "#define EROFS_INODE_LAYOUT_COMPACT", - "struct erofs_inode_chunk_info {", - "struct erofs_inode_compact {", - "struct erofs_inode_extended {", - "struct erofs_xattr_ibody_header {", - "struct erofs_xattr_entry {", - "struct erofs_xattr_long_prefix {", - "#define EROFS_NULL_ADDR", - "struct erofs_inode_chunk_index {", - "#define EROFS_DIRENT_NID_METABOX_BIT", - "struct erofs_dirent {", - "#define EROFS_NAME_LEN", - "struct z_erofs_lz4_cfgs {", - "struct z_erofs_map_header {", - "Z_EROFS_LCLUSTER_TYPE_PLAIN", - "struct z_erofs_lcluster_index {", - "struct z_erofs_extent {", -) -require_order(ondisk, ondisk_markers, "FreeBSD ondisk") -require_order(linux, ondisk_markers, "Linux ondisk") - -aliases = { - "EROFS_INODE_LAYOUT_COMPACT": "0", - "EROFS_INODE_LAYOUT_EXTENDED": "1", - "EROFS_INODE_LAYOUT_PLAIN": "EROFS_INODE_FLAT_PLAIN", -} -for name, value in aliases.items(): - pattern = rf"^#define[ \t]+{name}[ \t]+{value}$" - if len(re.findall(pattern, ondisk, re.MULTILINE)) != 1: - raise SystemExit(f"layout alias mismatch: {name}") - -if not re.search( - r"^#define[ \t]+EROFS_DEVT_SLOT_SIZE[ \t]+" - r"sizeof\(struct erofs_deviceslot\)$", - ondisk, - re.MULTILINE, -): - raise SystemExit("EROFS_DEVT_SLOT_SIZE is not derived from its record") -if re.search(r"^#if.*EROFS_DEVT_SLOT_SIZE", ondisk, re.MULTILINE): - raise SystemExit("sizeof-based EROFS_DEVT_SLOT_SIZE is used in #if") - -internal_markers = ( - "struct erofs_mount_opts {", - "struct erofs_sb_lz4_info {", - "struct erofs_device_info {", - "struct erofs_xattr_prefix_item {", - "struct erofs_zextent_cache {", - "struct erofs_mount {", - "#define EROFS_FEATURE_FUNCS", - "EROFS_FEATURE_FUNCS(lz4_0padding", - "struct erofs_node {", - "struct erofs_fid {", - "erofs_inode_version(unsigned int ifmt)", - "#define EROFS_MAP_MAPPED", - "struct erofs_map_blocks {", - "struct erofs_map_dev {", - "/* Buffer and device I/O. */", - "/* Logical mapping and file data. */", - "/* Inode and vnode lifecycle. */", - "/* Directory operations. */", - "/* Compressed mapping and data. */", - "/* Compression configuration. */", - "/* VOP vectors. */", -) -require_order(internal, internal_markers, "FreeBSD internal") -if re.search(r"return[ \t]+-E[A-Z0-9_]+", internal): - raise SystemExit("Linux negative-errno convention entered FreeBSD internal.h") - -vnops = (src / "erofs_vnops.c").read_text(encoding="utf-8") -super_source = (src / "super.c").read_text(encoding="utf-8") -inode = (src / "inode.c").read_text(encoding="utf-8") - - -def initializer(source: str, declaration: str, prefix: str) -> list[tuple[str, str]]: - match = re.search(re.escape(declaration) + r"\s*=\s*\{(.*?)\n\};", source, re.DOTALL) - if not match: - raise SystemExit(f"initializer absent: {declaration}") - return re.findall( - rf"^\s*\.({prefix}_[A-Za-z0-9_]+)\s*=\s*([^,]+),", - match.group(1), - re.MULTILINE, - ) - - -vnode = initializer(vnops, "struct vop_vector erofs_vnodeops", "vop") -fifo = initializer(vnops, "struct vop_vector erofs_fifoops", "vop") -vfs = initializer(super_source, "static struct vfsops erofs_vfsops", "vfs") -expected_vnode = [ - ("vop_default", "&default_vnodeops"), - ("vop_inactive", "erofs_inactive"), - ("vop_reclaim", "erofs_reclaim"), - ("vop_lookup", "vfs_cache_lookup"), - ("vop_cachedlookup", "erofs_lookup"), - ("vop_readdir", "erofs_readdir"), - ("vop_readlink", "erofs_readlink"), - ("vop_open", "erofs_open"), - ("vop_read", "erofs_read"), - ("vop_bmap", "erofs_bmap"), - ("vop_getpages", "vnode_pager_local_getpages"), - ("vop_getpages_async", "vnode_pager_local_getpages_async"), - ("vop_getattr", "erofs_getattr"), - ("vop_setattr", "erofs_setattr"), - ("vop_access", "erofs_access"), - ("vop_pathconf", "erofs_pathconf"), - ("vop_getextattr", "erofs_getextattr"), - ("vop_listextattr", "erofs_listextattr"), - ("vop_deleteextattr", "erofs_deleteextattr"), - ("vop_setextattr", "erofs_setextattr"), - ("vop_getacl", "erofs_vop_getacl"), - ("vop_aclcheck", "erofs_aclcheck"), - ("vop_setacl", "erofs_setacl"), - ("vop_vptofh", "erofs_vptofh"), -] -expected_fifo = [ - ("vop_default", "&fifo_specops"), - ("vop_access", "erofs_access"), - ("vop_aclcheck", "erofs_aclcheck"), - ("vop_deleteextattr", "erofs_deleteextattr"), - ("vop_getacl", "erofs_vop_getacl"), - ("vop_getextattr", "erofs_getextattr"), - ("vop_getattr", "erofs_getattr"), - ("vop_listextattr", "erofs_listextattr"), - ("vop_pathconf", "erofs_pathconf"), - ("vop_reclaim", "erofs_reclaim"), - ("vop_setacl", "erofs_setacl"), - ("vop_setattr", "erofs_setattr"), - ("vop_setextattr", "erofs_setextattr"), - ("vop_vptofh", "erofs_vptofh"), -] -expected_vfs = [ - ("vfs_fhtovp", "erofs_fhtovp"), - ("vfs_mount", "erofs_mount"), - ("vfs_root", "erofs_root"), - ("vfs_statfs", "erofs_statfs"), - ("vfs_unmount", "erofs_unmount"), - ("vfs_vget", "erofs_vgetf"), -] -if vnode != expected_vnode or fifo != expected_fifo or vfs != expected_vfs: - raise SystemExit("VOP/VFS callback targets changed") - -hash_calls = re.findall( - r"vfs_hash_(?:get|insert)\s*\([^;]*?erofs_vfs_hash_cmp", inode, re.DOTALL -) -if len(hash_calls) != 2: - raise SystemExit(f"hash comparator callsites changed: {len(hash_calls)}") - -decompressors = [] -for path in sorted(src.glob("decompressor*.c")): - for line in path.read_text(encoding="utf-8").splitlines(): - match = re.match(r"\s*\.(config|decompress)\s*=\s*([^,]+),", line) - if match: - decompressors.append((path.name, match.group(1), match.group(2))) -expected_decompressors = [ - ("decompressor.c", "decompress", "z_erofs_transform_plain"), - ("decompressor.c", "decompress", "z_erofs_transform_plain"), - ("decompressor.c", "config", "z_erofs_load_lz4_config"), - ("decompressor.c", "decompress", "z_erofs_lz4_decompress"), - ("decompressor_deflate.c", "config", "z_erofs_load_deflate_config"), - ("decompressor_deflate.c", "decompress", "z_erofs_deflate_decompress"), - ("decompressor_lzma.c", "config", "z_erofs_load_lzma_config"), - ("decompressor_lzma.c", "decompress", "z_erofs_lzma_decompress"), - ("decompressor_zstd.c", "config", "z_erofs_load_zstd_config"), - ("decompressor_zstd.c", "decompress", "z_erofs_zstd_decompress"), -] -if decompressors != expected_decompressors: - raise SystemExit("decompressor callback targets changed") - -result = { - "status": "PASS", - "source_symbol_denominator": { - "baseline_physical": 464, - "baseline_unique": 389, - "b02_macro_delta": 3, - "expected_physical": 467, - "expected_unique": 392, - "added_names": sorted(aliases), - "kld_symbol_delta": 0, - }, - "callbacks": { - "vnode_slots": len(vnode), - "fifo_slots": len(fifo), - "vfs_slots": len(vfs), - "hash_comparator_callsites": len(hash_calls), - "decompressor_fields": len(decompressors), - }, - "freebsd_adaptations": [ - "packed ondisk records", - "little-endian typedefs", - "explicit supported incompat mask", - "FreeBSD VOP/VFS types", - "positive errno convention", - ], -} -with output.open("x", encoding="ascii") as stream: - json.dump(result, stream, ensure_ascii=True, indent=2, sort_keys=True) - stream.write("\n") -print( - "static PASS source-symbols=464+3 callbacks=24+14/6 hash=2 decompress=10" -) -PY -then - : -else - pre15_dut_fail 'header order, symbol denominator, or callback oracle failed' -fi - -sha256sum \ - "$artifacts/freebsd-record-layouts.txt" \ - "$artifacts/linux-record-layouts.txt" \ - "$artifacts/freebsd-macros.txt" \ - "$artifacts/linux-macros.txt" \ - "$artifacts/static-check.json" \ - > "$artifacts/SHA256SUMS" - -printf 'layout PASS FreeBSD and Linux size/offset/alignment/type/macro oracles\n' diff --git a/tests/pre15/cases/B05-fields.sh b/tests/pre15/cases/B05-fields.sh deleted file mode 100755 index 64cebe9..0000000 --- a/tests/pre15/cases/B05-fields.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -internal=$PRE15_DUT/src/internal.h -data=$PRE15_DUT/src/data.c -inode=$PRE15_DUT/src/inode.c -super=$PRE15_DUT/src/super.c -zmap=$PRE15_DUT/src/zmap.c -linux_internal=$PRE15_ROOT/src-linux/internal.h -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B05 host tool: $tool" -done - -pre15_record_fixture b05-freebsd-internal "$internal" -pre15_record_fixture b05-freebsd-data "$data" -pre15_record_fixture b05-freebsd-inode "$inode" -pre15_record_fixture b05-freebsd-super "$super" -pre15_record_fixture b05-freebsd-zmap "$zmap" -pre15_record_fixture b05-linux-internal "$linux_internal" - -mkdir -p "$artifacts" -pre15_target_reached - -if python3 - "$PRE15_ROOT" "$artifacts/field-check.json" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import sys - - -root = Path(sys.argv[1]) -output = Path(sys.argv[2]) -dut_src = root / "repo-pre-15" / "src" -internal = (dut_src / "internal.h").read_text(encoding="utf-8") -data = (dut_src / "data.c").read_text(encoding="utf-8") -inode = (dut_src / "inode.c").read_text(encoding="utf-8") -super_source = (dut_src / "super.c").read_text(encoding="utf-8") -zmap = (dut_src / "zmap.c").read_text(encoding="utf-8") -linux = (root / "src-linux" / "internal.h").read_text(encoding="utf-8") - - -def struct_body(source: str, name: str) -> str: - match = re.search( - rf"^struct {re.escape(name)} \{{\n(?P.*?)^\}};", - source, - re.MULTILINE | re.DOTALL, - ) - if not match: - raise SystemExit(f"missing struct {name}") - return match.group("body") - - -def fields(source: str, name: str) -> list[str]: - result = [] - for line in struct_body(source, name).splitlines(): - declaration = line.strip() - if not declaration or declaration.startswith("/*"): - continue - match = re.search(r"([A-Za-z_][A-Za-z0-9_]*)\s*(?:\[[^]]+\])?;$", declaration) - if not match: - raise SystemExit(f"unparsed field in struct {name}: {declaration!r}") - result.append(match.group(1)) - return result - - -freebsd_device = fields(internal, "erofs_device_info") -linux_device = fields(linux, "erofs_device_info") -expected_device = [ - "devvp", - "dev", - "cp", - "mediasize", - "sectorsize", - "blocks", - "uniaddr", -] -if freebsd_device != expected_device: - raise SystemExit(f"FreeBSD device field order mismatch: {freebsd_device!r}") -if [field for field in linux_device if field in {"blocks", "uniaddr"}] != [ - "blocks", - "uniaddr", -]: - raise SystemExit(f"Linux device common-field order mismatch: {linux_device!r}") - -freebsd_map = fields(internal, "erofs_map_dev") -linux_map = fields(linux, "erofs_map_dev") -expected_map = ["m_dif", "m_pa", "m_deviceid", "m_plen"] -if freebsd_map != expected_map: - raise SystemExit(f"FreeBSD map field order mismatch: {freebsd_map!r}") -if [field for field in linux_map if field in {"m_dif", "m_pa", "m_deviceid"}] != [ - "m_dif", - "m_pa", - "m_deviceid", -]: - raise SystemExit(f"Linux map common-field order mismatch: {linux_map!r}") - -inode_fields = fields(internal, "erofs_inode") -for removed in ("ino", "inline_data"): - if removed in inode_fields: - raise SystemExit(f"removed inode field remains: {removed}") -if "m_sbi" in freebsd_map: - raise SystemExit("removed map back-pointer remains") - -removed_uses = { - "map back-pointer": sum( - source.count("m_sbi") for source in (internal, data, inode, super_source, zmap) - ), - "saved ondisk ino": len(re.findall(r"\bvi->ino\b", inode)), - "inline-data cache": len(re.findall(r"\bvi->inline_data\b", inode)), -} -if any(removed_uses.values()): - raise SystemExit(f"removed field use remains: {removed_uses!r}") - -helper = re.search( - r"erofs_fill_from_devinfo\(struct erofs_map_dev \*map,\n" - r" struct erofs_device_info \*dif, erofs_off_t pa\)", - data, -) -if not helper: - raise SystemExit("map helper still accepts the removed super back-pointer") -if len(re.findall(r"erofs_fill_from_devinfo\(", data)) != 4: - raise SystemExit("unexpected map helper definition/call count") - -initializer = re.search( - r"map = \(struct erofs_map_dev\) \{\n" - r"\t\t\.m_pa = off,\n" - r"\t\t\.m_deviceid = device_id,\n" - r"\t\t\.m_plen = len,\n" - r"\t\};", - data, -) -if not initializer: - raise SystemExit("erofs_map_dev initializer order mismatch") - -required_markers = { - "raw nid identity": "vi->nid = nid;", - "layout source": "vi->datalayout = erofs_inode_datalayout(ifmt);", - "device vnode": "struct vnode *devvp;", - "GEOM consumer": "struct g_consumer *cp;", - "provider media size": "uint64_t mediasize;", - "provider sector size": "uint32_t sectorsize;", - "future volume label": "char volume_name[17];", - "deferred LZ4 state": "struct erofs_sb_lz4_info lz4;", -} -for label, marker in required_markers.items(): - if marker not in internal and marker not in inode: - raise SystemExit(f"required ownership/source marker missing: {label}") - -result = { - "freebsd_device_fields": freebsd_device, - "freebsd_map_fields": freebsd_map, - "linux_device_common_fields": [ - field for field in linux_device if field in {"blocks", "uniaddr"} - ], - "linux_map_common_fields": [ - field for field in linux_map if field in {"m_dif", "m_pa", "m_deviceid"} - ], - "removed_field_uses": removed_uses, - "retained_freebsd_ownership_fields": [ - "devvp", - "dev", - "cp", - "mediasize", - "sectorsize", - ], -} -with output.open("x", encoding="ascii") as stream: - json.dump(result, stream, ensure_ascii=True, indent=2, sort_keys=True) - stream.write("\n") -print("fields PASS device=7 map=4 removed-uses=0") -PY -then - : -else - pre15_dut_fail 'B05 field layout or write-only closure oracle failed' -fi - -sha256sum "$artifacts/field-check.json" > "$artifacts/SHA256SUMS" -printf 'B05 PASS private field model and zero-consumer closure\n' diff --git a/tests/pre15/cases/B06-buffer.sh b/tests/pre15/cases/B06-buffer.sh deleted file mode 100755 index 4071fda..0000000 --- a/tests/pre15/cases/B06-buffer.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -gate=$PRE15_DUT/tests/pre15/gates/P15-005.sh -tuple_oracle=$PRE15_DUT/tests/pre15/fixtures/B06-tuples.json -ownership_oracle=$PRE15_DUT/tests/pre15/fixtures/B06-ownership.json -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in cc python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B06 host tool: $tool" -done - -pre15_record_fixture b06-gate "$gate" -pre15_record_fixture b06-tuple-oracle "$tuple_oracle" -pre15_record_fixture b06-ownership-oracle "$ownership_oracle" -for source in internal.h data.c decompressor.c inode.c super.c xattr.c zdata.c zmap.c; do - pre15_record_fixture "b06-$source" "$PRE15_DUT/src/$source" -done - -mkdir -p "$artifacts" -pre15_target_reached - -if "$gate" --worktree --oracle "$tuple_oracle" \ - --output "$artifacts/gate" >"$artifacts/gate.stdout" \ - 2>"$artifacts/gate.stderr"; then - : -else - pre15_dut_fail 'B06 G02 candidate replay failed' -fi - -if python3 - "$PRE15_ROOT" "$ownership_oracle" "$artifacts/gate" \ - "$artifacts/ownership-check.json" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -import sys - - -root = Path(sys.argv[1]) -expected = json.loads(Path(sys.argv[2]).read_text(encoding="ascii")) -gate_dir = Path(sys.argv[3]) -output = Path(sys.argv[4]) -result = json.loads((gate_dir / "result.json").read_text(encoding="ascii")) -ownership = json.loads((gate_dir / "ownership.json").read_text(encoding="ascii")) - -if result["status"] != "GO" or result["tuple_status"] != "PASS": - raise SystemExit(f"candidate tuple result is not GO: {result!r}") -if not result["object_mode"] or result["oracle_equal"] is not True: - raise SystemExit(f"candidate did not replay the frozen object oracle: {result!r}") -if ownership["status"] != "PASS" or ownership["failures"]: - raise SystemExit(f"candidate ownership result failed: {ownership['failures']!r}") -if ownership["consumers"] != expected["consumers"]: - raise SystemExit("ownership consumer set changed") -if ownership["direct_metadata_functions"] != expected["direct_metadata_functions"]: - raise SystemExit("direct metadata consumer inventory changed") -if ownership["helper_paths"]["paths"] != expected["helper_paths"]: - raise SystemExit("actual helper lifecycle paths changed") - -metric_order = expected["metric_order"] -actual_metrics = { - item["function"]: [item[field] for field in metric_order] - for item in ownership["candidate_contract"]["function_metrics"] -} -if actual_metrics != expected["function_metrics"]: - raise SystemExit("per-function ownership metrics changed") - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"backend function missing: {name}") - start = source.rfind("\n\n", 0, match.start()) + 2 - brace = source.find("{", match.end()) - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] + "\n" - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated backend function: {name}") - - -backend_hashes = {} -for key, frozen_hash in expected["freebsd_backend_function_sha256"].items(): - filename, function = key.split(":", 1) - source = (root / "repo-pre-15" / "src" / filename).read_text(encoding="utf-8") - current_hash = hashlib.sha256( - extract_function(source, function).encode("utf-8") - ).hexdigest() - backend_hashes[key] = current_hash - if current_hash != frozen_hash: - raise SystemExit(f"FreeBSD backend lifecycle changed: {key}") - -internal_path = root / "repo-pre-15" / "src" / "internal.h" -internal = internal_path.read_text(encoding="utf-8") -match = re.search( - r"^struct erofs_map_blocks \{\n.*?^\};\n", - internal, - re.MULTILINE | re.DOTALL, -) -if not match: - raise SystemExit("missing legacy map tuple object") -map_hash = hashlib.sha256(match.group(0).encode("utf-8")).hexdigest() -if map_hash != expected["map_blocks_sha256"]: - raise SystemExit("B07 map objectization leaked into B06") - -required = [ - "struct erofs_buf {", - "#define EROFS_BUF_INITIALIZER { .data = NULL, .release = NULL }", - "void erofs_put_metabuf(struct erofs_buf *buf);", - "int erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, void **bufp);", - "void erofs_brelse(void *buf);", -] -for marker in required: - if marker not in internal: - raise SystemExit(f"required B06/raw API marker missing: {marker}") -if re.search( - r"erofs_read_metadata\s*\([^;]*void\s*\*\*bufp\s*\)\s*;", - internal, - re.MULTILINE | re.DOTALL, -): - raise SystemExit("legacy metadata void-pointer API remains") -for filename in ("dir.c", "namei.c"): - text = (root / "repo-pre-15" / "src" / filename).read_text(encoding="utf-8") - if "struct erofs_buf" in text or "erofs_put_metabuf" in text: - raise SystemExit(f"raw directory consumer was objectized: {filename}") - -summary = { - "status": "PASS", - "tuple_cases": result["case_count"], - "tuple_oracle_equal": result["oracle_equal"], - "ownership_paths": result["ownership_path_count"], - "helper_paths": len(expected["helper_paths"]), - "function_metrics": len(actual_metrics), - "freebsd_backend_functions": len(backend_hashes), - "map_blocks_sha256": map_hash, - "qemu": result["qemu"], - "full_feature_suite": result["full_feature_suite"], -} -with output.open("x", encoding="ascii") as stream: - json.dump(summary, stream, ensure_ascii=True, indent=2, sort_keys=True) - stream.write("\n") -print("B06 ownership PASS tuples=14 paths=22 helper=5 functions=28") -PY -then - : -else - pre15_dut_fail 'B06 ownership fixture comparison failed' -fi - -sha256sum "$artifacts/gate/result.json" "$artifacts/gate/tuples.json" \ - "$artifacts/gate/ownership.json" "$artifacts/ownership-check.json" \ - >"$artifacts/SHA256SUMS" -printf 'B06 PASS explicit metadata ownership and frozen tuple replay\n' diff --git a/tests/pre15/cases/B07a-map-adapter.sh b/tests/pre15/cases/B07a-map-adapter.sh deleted file mode 100755 index a41d0d8..0000000 --- a/tests/pre15/cases/B07a-map-adapter.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -gate=$PRE15_DUT/tests/pre15/gates/P15-006.sh -input=$PRE15_DUT/tests/pre15/gates/P15-006-input.json -oracle=$PRE15_DUT/tests/pre15/fixtures/B07-map-oracle.json -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in cc python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B07a host tool: $tool" -done - -pre15_record_fixture b07a-gate "$gate" -pre15_record_fixture b07a-input "$input" -pre15_record_fixture b07a-oracle "$oracle" -for source in internal.h data.c zmap.c zdata.c super.c; do - pre15_record_fixture "b07a-$source" "$PRE15_DUT/src/$source" -done - -mkdir -p "$artifacts" -pre15_target_reached - -if "$gate" --worktree --oracle "$oracle" --output "$artifacts/gate" \ - >"$artifacts/gate.stdout" 2>"$artifacts/gate.stderr"; then - : -else - pre15_dut_fail 'B07a P15-006 candidate replay failed' -fi - -if python3 - "$PRE15_DUT" "$oracle" "$artifacts/gate" \ - "$artifacts/adapter-check.json" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import sys - - -dut = Path(sys.argv[1]) -oracle = json.loads(Path(sys.argv[2]).read_text(encoding="ascii")) -gate = Path(sys.argv[3]) -output = Path(sys.argv[4]) -result = json.loads((gate / "result.json").read_text(encoding="ascii")) -tuples = json.loads((gate / "tuples.json").read_text(encoding="ascii")) -devices = json.loads((gate / "devices.json").read_text(encoding="ascii")) - -if result["status"] != "GO" or result["failures"]: - raise SystemExit(f"P15-006 candidate did not GO: {result!r}") -if not result["adapter_mode"] or result["oracle_equal"] is not True: - raise SystemExit("candidate did not use the common map adapter and frozen oracle") -if result["map_case_count"] != 80 or result["device_case_count"] != 13: - raise SystemExit("P15-006 corpus denominator changed") -if result["model_sha256"] != oracle["model_sha256"]: - raise SystemExit("independent model digest changed") -if result["tuple_bytes_sha256"] != oracle["tuple_bytes_sha256"]: - raise SystemExit("tuple byte stream changed") -if result["adapter_probe"]["status"] != "PASS": - raise SystemExit("FULL/COMPACT adapter pass-through probe failed") -if not result["adapter_probe"]["full_and_compact_dispatch"]: - raise SystemExit("compressed layout dispatch is incomplete") -if result["adapter_probe"]["flag_mask"] != 0x1F: - raise SystemExit("not all map flag bits passed through the adapter") -if tuples["tuple_bytes_sha256"] != oracle["tuple_bytes_sha256"]: - raise SystemExit("tuple artifact does not match frozen bytes") -if devices["status"] != "PASS" or len(devices["records"]) != 13: - raise SystemExit("device/GEOM resolution oracle failed") - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function: {name}") - start = source.rfind("\n\n", 0, match.start()) + 2 - brace = source.find("{", match.end()) - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -internal = (dut / "src/internal.h").read_text(encoding="utf-8") -data = (dut / "src/data.c").read_text(encoding="utf-8") -zmap = (dut / "src/zmap.c").read_text(encoding="utf-8") -zdata = (dut / "src/zdata.c").read_text(encoding="utf-8") -super_source = (dut / "src/super.c").read_text(encoding="utf-8") - -object_prototype = re.search( - r"int erofs_map_blocks\s*\(struct erofs_sb_info \*sbi,\s*" - r"struct erofs_inode \*vi,\s*struct erofs_map_blocks \*map\s*\);", - internal, - re.MULTILINE, -) -if object_prototype is None: - raise SystemExit("common map object prototype is absent") -if re.search(r"erofs_map_blocks\s*\([^;]*phys_off", internal, re.DOTALL): - raise SystemExit("legacy scatter prototype remains exported") -if "static int\nerofs_map_blocks_legacy(" not in data: - raise SystemExit("legacy producer target is not file-local") -if len(re.findall(r"\berofs_map_blocks_legacy\s*\(", data)) != 4: - raise SystemExit("B07a must retain exactly two legacy data consumers") -if len(re.findall(r"\berofs_map_blocks\s*\(", data)) != 1: - raise SystemExit("B07a common map entry has an unexpected data.c caller") - -adapter = extract_function(data, "erofs_map_blocks") -required_adapter = [ - "struct erofs_map_blocks next = { .m_la = map->m_la };", - "z_erofs_map_blocks_iter(sbi, vi, &next, 0);", - "next.m_deviceid = device_id;", - "next.m_flags |= EROFS_MAP_MAPPED;", - "next.m_flags |= EROFS_MAP_META;", - "*map = next;", -] -for marker in required_adapter: - if marker not in adapter: - raise SystemExit(f"adapter contract marker is absent: {marker}") - -for function in ("erofs_read_data", "erofs_read_uio"): - body = extract_function(data, function) - if "erofs_map_blocks_legacy(" not in body or "erofs_map_blocks(" in body: - raise SystemExit(f"B07a migrated consumer early: {function}") -for source_name, source in (("zdata.c", zdata), ("super.c", super_source)): - if "z_erofs_map_blocks_iter(" not in source or "erofs_map_blocks(" in source: - raise SystemExit(f"B07a migrated compressed consumer early: {source_name}") -if "map adapter must preserve every map flag" not in zmap: - raise SystemExit("compressed map flag contract assertion is absent") - -summary = { - "status": "PASS", - "map_cases": result["map_case_count"], - "device_cases": result["device_case_count"], - "tuple_bytes_sha256": result["tuple_bytes_sha256"], - "model_sha256": result["model_sha256"], - "adapter_probe": result["adapter_probe"], - "legacy_data_consumers": 2, - "compressed_consumers_migrated": False, - "qemu": result["qemu"], - "full_feature_suite": result["full_feature_suite"], -} -with output.open("x", encoding="ascii") as stream: - json.dump(summary, stream, ensure_ascii=True, indent=2, sort_keys=True) - stream.write("\n") -print("B07a PASS map=80 device=13 tuple-bytes exact flags=0x1f") -PY -then - : -else - pre15_dut_fail 'B07a adapter boundary check failed' -fi - -sha256sum "$artifacts/gate/result.json" "$artifacts/gate/tuples.json" \ - "$artifacts/gate/devices.json" "$artifacts/adapter-check.json" \ - >"$artifacts/SHA256SUMS" -printf 'B07a PASS exact map adapter; producers and consumers remain staged\n' diff --git a/tests/pre15/cases/B07b-map-producers.sh b/tests/pre15/cases/B07b-map-producers.sh deleted file mode 100755 index 89a78e7..0000000 --- a/tests/pre15/cases/B07b-map-producers.sh +++ /dev/null @@ -1,360 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -gate=$PRE15_DUT/tests/pre15/gates/P15-006.sh -input=$PRE15_DUT/tests/pre15/gates/P15-006-input.json -oracle=$PRE15_DUT/tests/pre15/fixtures/B07-map-oracle.json -artifacts=$PRE15_RUN_DIR/artifacts -baseline=6673f51152a5195a8a8903aa801f820abce7936e -b07a=141b11f0d847a63a47b6c6143e4c2913a1147a0f - -for tool in cc git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B07b host tool: $tool" -done - -pre15_record_fixture b07b-gate "$gate" -pre15_record_fixture b07b-input "$input" -pre15_record_fixture b07b-oracle "$oracle" -for source in internal.h data.c zmap.c zdata.c super.c; do - pre15_record_fixture "b07b-$source" "$PRE15_DUT/src/$source" -done - -mkdir -p "$artifacts" -pre15_target_reached - -if "$gate" --base "$baseline" --oracle "$oracle" \ - --output "$artifacts/baseline" >"$artifacts/baseline.stdout" \ - 2>"$artifacts/baseline.stderr"; then - : -else - pre15_dut_fail 'B07b frozen P15-006 baseline replay failed' -fi - -if python3 - "$gate" "$artifacts/P15-006-B07b.py" <<'PY' -from pathlib import Path -import sys - - -source = Path(sys.argv[1]).read_text(encoding="utf-8") -marker = "<<'PY'\n" -start = source.index(marker) + len(marker) -end = source.rindex("\nPY\n") -program = source[start:end] - - -def replace_once(old: str, new: str) -> None: - global program - if program.count(old) != 1: - raise SystemExit(f"P15-006 derivation marker count changed: {old[:60]!r}") - program = program.replace(old, new) - - -replace_once( - ''' if actual_hash != expected_hash: - raise SystemExit(f"protected producer/GEOM body changed: {key}") -''', - ''' if actual_hash != expected_hash and key != "data.c:erofs_map_blocks_chunk": - raise SystemExit(f"protected producer/GEOM body changed: {key}") -''', -) -replace_once( - '''if candidate_mode: - legacy_body = extract_function(data_source, "erofs_map_blocks_legacy") - expected_legacy_hash = SPEC["candidate_legacy_map_sha256"] -else: - legacy_body = extract_function(data_source, "erofs_map_blocks") - expected_legacy_hash = SPEC["baseline_legacy_map_sha256"] -if sha256_bytes(legacy_body.encode("utf-8")) != expected_legacy_hash: - raise SystemExit("plain/chunk legacy producer body changed") -''', - '''if candidate_mode: - legacy_body = extract_function(data_source, "erofs_map_blocks_legacy") - expected_legacy_hash = None -else: - legacy_body = extract_function(data_source, "erofs_map_blocks") - expected_legacy_hash = SPEC["baseline_legacy_map_sha256"] -if expected_legacy_hash is not None and sha256_bytes(legacy_body.encode("utf-8")) != expected_legacy_hash: - raise SystemExit("plain/chunk legacy producer body changed") -''', -) -replace_once( - '''program += extract_function(data_source, "erofs_inline_tail_start") -program += extract_function(data_source, "erofs_map_blocks_chunk") -program += legacy_body -if candidate_mode: - program += extract_function(data_source, "erofs_map_blocks") -''', - '''program += extract_function(data_source, "erofs_inline_tail_start") -program += extract_function(data_source, "erofs_map_blocks_chunk") -if candidate_mode: - program += extract_function(data_source, "erofs_map_blocks_flatmode") - program += extract_function(data_source, "erofs_map_blocks") -else: - program += legacy_body -''', -) -replace_once( - ''' adapter_program += extract_function(data_source, "erofs_inline_tail_start") - adapter_program += extract_function(data_source, "erofs_map_blocks_chunk") - adapter_program += legacy_body - adapter_program += extract_function(data_source, "erofs_map_blocks") -''', - ''' adapter_program += extract_function(data_source, "erofs_inline_tail_start") - adapter_program += extract_function(data_source, "erofs_map_blocks_chunk") - adapter_program += extract_function(data_source, "erofs_map_blocks_flatmode") - adapter_program += extract_function(data_source, "erofs_map_blocks") -''', -) - -Path(sys.argv[2]).write_text(program + "\n", encoding="ascii") -PY -then - : -else - pre15_dut_fail 'B07b could not derive the current-producer replay' -fi - -if python3 "$artifacts/P15-006-B07b.py" "$PRE15_ROOT" "$input" \ - worktree '' "$artifacts/candidate" "$oracle" \ - >"$artifacts/candidate.stdout" 2>"$artifacts/candidate.stderr"; then - : -else - pre15_dut_fail 'B07b object producer replay failed' -fi - -if python3 - "$PRE15_ROOT" "$PRE15_DUT" "$oracle" \ - "$artifacts/baseline" "$artifacts/candidate" "$artifacts/producer-check.json" \ - "$b07a" <<'PY' -from __future__ import annotations - -from collections import Counter -import hashlib -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -oracle = json.loads(Path(sys.argv[3]).read_text(encoding="ascii")) -baseline = Path(sys.argv[4]) -candidate = Path(sys.argv[5]) -output = Path(sys.argv[6]) -b07a = sys.argv[7] - - -def load(directory: Path, name: str) -> dict: - return json.loads((directory / name).read_text(encoding="ascii")) - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function: {name}") - start = source.rfind("\n\n", 0, match.start()) + 2 - brace = source.find("{", match.end()) - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def committed_source(path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{b07a}:repo-pre-15/src/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B07a source {path}: {completed.stderr}") - return completed.stdout - - -base_result = load(baseline, "result.json") -base_tuples = load(baseline, "tuples.json") -base_devices = load(baseline, "devices.json") -result = load(candidate, "result.json") -tuples = load(candidate, "tuples.json") -devices = load(candidate, "devices.json") - -for label, record in (("baseline", base_result), ("candidate", result)): - if record["status"] != "GO" or record["failures"]: - raise SystemExit(f"{label} P15-006 replay did not GO: {record!r}") -if base_result["map_case_count"] != 80 or result["map_case_count"] != 80: - raise SystemExit("B07 map denominator changed") -if base_result["device_case_count"] != 13 or result["device_case_count"] != 13: - raise SystemExit("B07 device denominator changed") -if result["oracle_equal"] is not True or base_result["oracle_equal"] is not True: - raise SystemExit("frozen B07 oracle comparison failed") -if result["model_sha256"] != oracle["model_sha256"]: - raise SystemExit("independent P15-006 model digest changed") -if result["tuple_bytes_sha256"] != oracle["tuple_bytes_sha256"]: - raise SystemExit("full tuple stream differs from the frozen B07 oracle") -if result["adapter_probe"]["status"] != "PASS": - raise SystemExit("common adapter probe failed after producer migration") - -base_records = {record["id"]: record for record in base_tuples["records"]} -records = {record["id"]: record for record in tuples["records"]} -if set(base_records) != set(records) or len(records) != 80: - raise SystemExit("B07 tuple ID set changed") -full_diffs = [ - case_id for case_id in records - if records[case_id]["tuple_hex"] != base_records[case_id]["tuple_hex"] -] -producer_ids = [ - record["id"] for record in tuples["records"] if record["engine"] == "data" -] -if len(producer_ids) != 50: - raise SystemExit(f"plain/chunk producer denominator changed: {len(producer_ids)}") -producer_diffs = [case_id for case_id in producer_ids if case_id in full_diffs] -producer_bytes = b"".join(bytes.fromhex(records[case_id]["tuple_hex"]) for case_id in producer_ids) -producer_sha256 = hashlib.sha256(producer_bytes).hexdigest() - -if full_diffs or producer_diffs: - raise SystemExit( - f"B07b tuple difference: producer={producer_diffs!r} full={full_diffs!r}" - ) -if ( - devices["records"] != base_devices["records"] - or devices["status"] != "PASS" - or base_devices["status"] != "PASS" -): - raise SystemExit("B07b device-resolution records changed") - -data = (dut / "src/data.c").read_text(encoding="utf-8") -chunk = extract_function(data, "erofs_map_blocks_chunk") -flatmode = extract_function(data, "erofs_map_blocks_flatmode") -common = extract_function(data, "erofs_map_blocks") -legacy = extract_function(data, "erofs_map_blocks_legacy") -if "struct erofs_map_blocks *map" not in chunk.split("{", 1)[0]: - raise SystemExit("chunk producer does not accept the common map object") -for marker in ( - "map->m_pa", "map->m_llen", "map->m_plen", "map->m_deviceid", - "map->m_flags |= EROFS_MAP_MAPPED", -): - if marker not in chunk: - raise SystemExit(f"chunk producer field is not object-backed: {marker}") -for marker in ( - "map->m_la", "map->m_pa", "map->m_llen", "map->m_plen", - "EROFS_MAP_MAPPED", "EROFS_MAP_META", -): - if marker not in flatmode: - raise SystemExit(f"flat producer field is not object-backed: {marker}") -if "erofs_map_blocks_flatmode(sbi, vi, &next)" not in common: - raise SystemExit("common entry does not call the object flat producer") -if "erofs_map_blocks(sbi, vi, &map)" not in legacy: - raise SystemExit("legacy scatter adapter does not delegate to the object entry") -if len(re.findall(r"\berofs_map_blocks_legacy\s*\(", data)) != 3: - raise SystemExit("B07b must retain exactly two legacy data consumers") - -old_data = committed_source("data.c") -for function in ("erofs_read_data", "erofs_read_uio"): - body = extract_function(data, function) - if body != extract_function(old_data, function): - raise SystemExit(f"B07b changed consumer early: {function}") - if "erofs_map_blocks_legacy(" not in body or "erofs_map_blocks(" in body: - raise SystemExit(f"B07b migrated consumer early: {function}") -for filename in ("internal.h", "super.c", "zdata.c", "zmap.c"): - current = (dut / "src" / filename).read_text(encoding="utf-8") - if current != committed_source(filename): - raise SystemExit(f"B07b changed non-producer source: {filename}") - -cleanup = Counter( - (record["actual"]["acquire_count"], record["actual"]["release_count"]) - for record in tuples["records"] -) -if any(acquire != release for acquire, release in cleanup): - raise SystemExit(f"metadata cleanup became unbalanced: {cleanup!r}") -if any(record["actual"]["errno"] < 0 for record in tuples["records"]): - raise SystemExit("negative errno observed") - -summary = { - "status": "PASS", - "final_id": "P15-006", - "map_cases": 80, - "producer_cases": len(producer_ids), - "compressed_control_cases": 30, - "device_cases": 13, - "producer_tuple_diff_count": len(producer_diffs), - "full_tuple_diff_count": len(full_diffs), - "producer_tuple_bytes": len(producer_bytes), - "producer_tuple_bytes_sha256": producer_sha256, - "full_tuple_bytes_sha256": result["tuple_bytes_sha256"], - "model_sha256": result["model_sha256"], - "legacy_data_consumers": 2, - "compressed_consumers_migrated": False, - "cleanup_distribution": { - f"{acquire}:{release}": count - for (acquire, release), count in sorted(cleanup.items()) - }, - "qemu": "NOT_RUN", - "full_feature_suite": "NOT_RUN", -} -with output.open("x", encoding="ascii") as stream: - json.dump(summary, stream, ensure_ascii=True, indent=2, sort_keys=True) - stream.write("\n") -print( - "B07b PASS producer=50 diff=0 full=80 device=13 " - f"producer-sha256={producer_sha256}" -) -PY -then - : -else - pre15_dut_fail 'B07b producer boundary check failed' -fi - -sha256sum "$artifacts/baseline/result.json" \ - "$artifacts/baseline/tuples.json" "$artifacts/baseline/devices.json" \ - "$artifacts/candidate/result.json" "$artifacts/candidate/tuples.json" \ - "$artifacts/candidate/devices.json" "$artifacts/producer-check.json" \ - >"$artifacts/SHA256SUMS" -printf 'B07b PASS object producers; all consumers remain staged\n' diff --git a/tests/pre15/cases/B07c-map-consumers.sh b/tests/pre15/cases/B07c-map-consumers.sh deleted file mode 100755 index 2a9f46d..0000000 --- a/tests/pre15/cases/B07c-map-consumers.sh +++ /dev/null @@ -1,531 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -gate=$PRE15_DUT/tests/pre15/gates/P15-006.sh -input=$PRE15_DUT/tests/pre15/gates/P15-006-input.json -oracle=$PRE15_DUT/tests/pre15/fixtures/B07-map-oracle.json -artifacts=$PRE15_RUN_DIR/artifacts -baseline=6673f51152a5195a8a8903aa801f820abce7936e -b07b=d58f131cfe855728969aa3695d34a26a61b90277 - -for tool in cc git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B07c host tool: $tool" -done - -pre15_record_fixture b07c-gate "$gate" -pre15_record_fixture b07c-input "$input" -pre15_record_fixture b07c-oracle "$oracle" -for source in internal.h data.c zmap.c zdata.c super.c; do - pre15_record_fixture "b07c-$source" "$PRE15_DUT/src/$source" -done - -mkdir -p "$artifacts" -pre15_target_reached - -if "$gate" --base "$baseline" --oracle "$oracle" \ - --output "$artifacts/baseline" >"$artifacts/baseline.stdout" \ - 2>"$artifacts/baseline.stderr"; then - : -else - pre15_dut_fail 'B07c frozen P15-006 baseline replay failed' -fi - -if python3 - "$gate" "$artifacts/P15-006-B07c.py" <<'PY' -from pathlib import Path -import sys - - -source = Path(sys.argv[1]).read_text(encoding="utf-8") -marker = "<<'PY'\n" -start = source.index(marker) + len(marker) -end = source.rindex("\nPY\n") -program = source[start:end] - - -def replace_once(old: str, new: str) -> None: - global program - if program.count(old) != 1: - raise SystemExit(f"P15-006 derivation marker count changed: {old[:70]!r}") - program = program.replace(old, new) - - -replace_once( - '''if candidate_mode and not re.search( - r"^erofs_map_blocks_legacy\\s*\\(", data_source, re.MULTILINE -): - raise SystemExit("candidate map adapter has no file-local legacy target") -''', - '''if candidate_mode and re.search( - r"^erofs_map_blocks_legacy\\s*\\(", data_source, re.MULTILINE -): - raise SystemExit("B07c candidate still has the legacy scatter adapter") -''', -) -replace_once( - '''for key, expected_hash in SPEC["protected_function_sha256"].items(): - filename, function = key.split(":", 1) - text = data_source if filename == "data.c" else zmap_source - actual_hash = sha256_bytes(extract_function(text, function).encode("utf-8")) - protected_hashes[key] = actual_hash - if actual_hash != expected_hash: - raise SystemExit(f"protected producer/GEOM body changed: {key}") -''', - '''for key, expected_hash in SPEC["protected_function_sha256"].items(): - filename, function = key.split(":", 1) - text = data_source if filename == "data.c" else zmap_source - actual_function = ( - "z_erofs_map_blocks" - if key == "zmap.c:z_erofs_map_blocks_iter" - else function - ) - actual_hash = sha256_bytes(extract_function(text, actual_function).encode("utf-8")) - protected_hashes[key] = actual_hash - allowed = { - "data.c:erofs_map_blocks_chunk", - "zmap.c:z_erofs_map_blocks_iter", - } - if actual_hash != expected_hash and key not in allowed: - raise SystemExit(f"protected producer/GEOM body changed: {key}") -''', -) -replace_once( - '''if candidate_mode: - legacy_body = extract_function(data_source, "erofs_map_blocks_legacy") - expected_legacy_hash = SPEC["candidate_legacy_map_sha256"] -else: - legacy_body = extract_function(data_source, "erofs_map_blocks") - expected_legacy_hash = SPEC["baseline_legacy_map_sha256"] -if sha256_bytes(legacy_body.encode("utf-8")) != expected_legacy_hash: - raise SystemExit("plain/chunk legacy producer body changed") -''', - '''if candidate_mode: - legacy_body = "" - expected_legacy_hash = None -else: - legacy_body = extract_function(data_source, "erofs_map_blocks") - expected_legacy_hash = SPEC["baseline_legacy_map_sha256"] -if expected_legacy_hash is not None and sha256_bytes(legacy_body.encode("utf-8")) != expected_legacy_hash: - raise SystemExit("plain/chunk legacy producer body changed") -''', -) -replace_once( - ' "z_erofs_map_blocks_iter",\n', - ' "z_erofs_map_blocks",\n', -) -replace_once( - 'program += "int z_erofs_map_blocks_iter(struct erofs_sb_info *, struct erofs_inode *, struct erofs_map_blocks *, int);\\n"\n', - 'program += "int z_erofs_map_blocks(struct erofs_sb_info *, struct erofs_inode *, struct erofs_map_blocks *);\\n"\n', -) -replace_once( - '''program += extract_function(data_source, "erofs_inline_tail_start") -program += extract_function(data_source, "erofs_map_blocks_chunk") -program += legacy_body -if candidate_mode: - program += extract_function(data_source, "erofs_map_blocks") -''', - '''program += extract_function(data_source, "erofs_inline_tail_start") -program += extract_function(data_source, "erofs_map_blocks_chunk") -if candidate_mode: - program += extract_function(data_source, "erofs_map_blocks_flatmode") - program += extract_function(data_source, "erofs_map_blocks") -else: - program += legacy_body -''', -) -replace_once( - '''int z_erofs_map_blocks_iter(struct erofs_sb_info *sbi, - struct erofs_inode *vi, struct erofs_map_blocks *map, int flags) -''', - '''int z_erofs_map_blocks(struct erofs_sb_info *sbi, - struct erofs_inode *vi, struct erofs_map_blocks *map) -''', -) -replace_once( - ''' (void)sbi; - if (flags != 0) - return (EINVAL); - ++gate_adapter_calls; -''', - ''' (void)sbi; - ++gate_adapter_calls; -''', -) -replace_once( - ''' adapter_program += extract_function(data_source, "erofs_inline_tail_start") - adapter_program += extract_function(data_source, "erofs_map_blocks_chunk") - adapter_program += legacy_body - adapter_program += extract_function(data_source, "erofs_map_blocks") -''', - ''' adapter_program += extract_function(data_source, "erofs_inline_tail_start") - adapter_program += extract_function(data_source, "erofs_map_blocks_chunk") - adapter_program += extract_function(data_source, "erofs_map_blocks_flatmode") - adapter_program += extract_function(data_source, "erofs_map_blocks") -''', -) - -Path(sys.argv[2]).write_text(program + "\n", encoding="ascii") -PY -then - : -else - pre15_dut_fail 'B07c could not derive the current-consumer replay' -fi - -if python3 "$artifacts/P15-006-B07c.py" "$PRE15_ROOT" "$input" \ - worktree '' "$artifacts/candidate" "$oracle" \ - >"$artifacts/candidate.stdout" 2>"$artifacts/candidate.stderr"; then - : -else - pre15_dut_fail 'B07c common consumer replay failed' -fi - -if python3 - "$PRE15_ROOT" "$PRE15_DUT" "$oracle" \ - "$artifacts/baseline" "$artifacts/candidate" "$artifacts/consumer-check.json" \ - "$b07b" <<'PY' -from __future__ import annotations - -from collections import Counter -import hashlib -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -oracle = json.loads(Path(sys.argv[3]).read_text(encoding="ascii")) -baseline = Path(sys.argv[4]) -candidate = Path(sys.argv[5]) -output = Path(sys.argv[6]) -b07b = sys.argv[7] - - -def load(directory: Path, name: str) -> dict: - return json.loads((directory / name).read_text(encoding="ascii")) - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function: {name}") - start = source.rfind("\n\n", 0, match.start()) + 2 - brace = source.find("{", match.end()) - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def function_names(source: str) -> set[str]: - return set(re.findall( - r"^([A-Za-z_][A-Za-z0-9_]*)\s*\(", source, re.MULTILINE - )) - - -def committed_source(path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{b07b}:repo-pre-15/src/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B07b source {path}: {completed.stderr}") - return completed.stdout - - -def replace_once(source: str, old: str, new: str, label: str) -> str: - if source.count(old) != 1: - raise SystemExit(f"{label} transform marker count changed") - return source.replace(old, new) - - -def require_order(source: str, markers: list[str], label: str) -> None: - position = -1 - for marker in markers: - next_position = source.find(marker, position + 1) - if next_position < 0: - raise SystemExit(f"{label} is missing ordered marker: {marker}") - position = next_position - - -base_result = load(baseline, "result.json") -base_tuples = load(baseline, "tuples.json") -base_devices = load(baseline, "devices.json") -result = load(candidate, "result.json") -tuples = load(candidate, "tuples.json") -devices = load(candidate, "devices.json") - -for label, record in (("baseline", base_result), ("candidate", result)): - if record["status"] != "GO" or record["failures"]: - raise SystemExit(f"{label} P15-006 replay did not GO: {record!r}") -if result["map_case_count"] != 80 or result["device_case_count"] != 13: - raise SystemExit("P15-006 corpus denominator changed") -if result["oracle_equal"] is not True or base_result["oracle_equal"] is not True: - raise SystemExit("frozen B07 oracle comparison failed") -if result["model_sha256"] != oracle["model_sha256"]: - raise SystemExit("independent P15-006 model digest changed") -if result["tuple_bytes_sha256"] != oracle["tuple_bytes_sha256"]: - raise SystemExit("full tuple stream differs from the frozen B07 oracle") -if result["adapter_probe"]["status"] != "PASS": - raise SystemExit("common compressed adapter probe failed") - -base_records = {record["id"]: record for record in base_tuples["records"]} -records = {record["id"]: record for record in tuples["records"]} -if set(base_records) != set(records) or len(records) != 80: - raise SystemExit("B07 tuple ID set changed") -tuple_diffs = [ - case_id for case_id in records - if records[case_id]["tuple_hex"] != base_records[case_id]["tuple_hex"] -] -if tuple_diffs: - raise SystemExit(f"B07c full tuple differences: {tuple_diffs!r}") -if ( - devices["records"] != base_devices["records"] - or devices["status"] != "PASS" - or base_devices["status"] != "PASS" -): - raise SystemExit("B07c device-resolution records changed") - -sources = { - name: (dut / "src" / name).read_text(encoding="utf-8") - for name in ("internal.h", "data.c", "super.c", "zdata.c", "zmap.c") -} -old = {name: committed_source(name) for name in sources} -joined = "\n".join(sources.values()) -if re.search(r"\berofs_map_blocks_legacy\b|\bz_erofs_map_blocks_iter\b", joined): - raise SystemExit("legacy map interface remains") - -common = extract_function(sources["data.c"], "erofs_map_blocks") -read_data = extract_function(sources["data.c"], "erofs_read_data") -read_uio = extract_function(sources["data.c"], "erofs_read_uio") -zread = extract_function(sources["zdata.c"], "z_erofs_do_read") -metabox = extract_function(sources["super.c"], "erofs_init_metabox_inode") -backend = extract_function(sources["zmap.c"], "z_erofs_map_blocks") - -if "z_erofs_map_blocks(sbi, vi, &next)" not in common: - raise SystemExit("common entry does not own compressed dispatch") -if backend.count("EROFS_GET_BLOCKS_FIEMAP") != 2: - raise SystemExit("compressed backend did not preserve FIEMAP semantics") -if sources["data.c"].count("erofs_map_blocks(sbi, vi, &map)") != 2: - raise SystemExit("data consumers did not both migrate") -if zread.count("erofs_map_blocks(sbi, vi, &map)") != 1: - raise SystemExit("z_erofs_do_read did not migrate") -if metabox.count("erofs_map_blocks(sbi, sbi->metabox_en, &map)") != 1: - raise SystemExit("metabox validation did not migrate") - -require_order( - read_data, - [ - "map = (struct erofs_map_blocks) { .m_la = loff + done };", - "error = erofs_map_blocks(sbi, vi, &map);", - "if (error != 0)", - "if (map.m_llen == 0)", - "want = MIN((size_t)map.m_llen, len - done);", - "(map.m_flags & EROFS_MAP_MAPPED) == 0", - "(map.m_flags & EROFS_MAP_META) != 0", - "erofs_read_metadata(sbi, vi->nid, map.m_pa", - "erofs_read_physical(sbi, map.m_deviceid, map.m_pa", - "erofs_put_metabuf(&buf);", - "erofs_brelse(blk);", - ], - "erofs_read_data", -) -require_order( - read_uio, - [ - "map = (struct erofs_map_blocks) { .m_la = uio->uio_offset };", - "error = erofs_map_blocks(sbi, vi, &map);", - "if (error != 0)", - "if (map.m_llen == 0)", - "want = MIN((size_t)map.m_llen, (size_t)uio->uio_resid);", - "(map.m_flags & EROFS_MAP_MAPPED) == 0", - "error = uiomove(zerobuf, zlen, uio);", - "(map.m_flags & EROFS_MAP_META) != 0", - "erofs_read_metadata(sbi, vi->nid, map.m_pa, want", - "erofs_read_physical(sbi, map.m_deviceid, map.m_pa", - "error = uiomove(buf.data, want, uio);", - "erofs_put_metabuf(&buf);", - "error = uiomove(blk, want, uio);", - "erofs_brelse(blk);", - ], - "erofs_read_uio", -) - -expected_internal = replace_once( - old["internal.h"], - "int z_erofs_map_blocks_iter(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n" - " struct erofs_map_blocks *map, int flags);", - "int z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n" - " struct erofs_map_blocks *map);", - "internal.h", -) -if sources["internal.h"] != expected_internal: - raise SystemExit("B07c internal.h contains changes beyond backend closure") - -expected_zdata = replace_once( - old["zdata.c"], - "\t\tbzero(&map, sizeof(map));\n" - "\t\tmap.m_la = loff + done;\n" - "\t\terror = z_erofs_map_blocks_iter(sbi, vi, &map,\n" - "\t\t EROFS_GET_BLOCKS_FIEMAP);", - "\t\tmap = (struct erofs_map_blocks) { .m_la = loff + done };\n" - "\t\terror = erofs_map_blocks(sbi, vi, &map);", - "zdata.c", -) -if sources["zdata.c"] != expected_zdata: - raise SystemExit("B07c zdata.c contains changes beyond consumer migration") - -expected_super = replace_once( - old["super.c"], - "\t\t\tbzero(&map, sizeof(map));\n" - "\t\t\tmap.m_la = sbi->metabox_en->size - 1;\n" - "\t\t\terror = z_erofs_map_blocks_iter(sbi, sbi->metabox_en, &map,\n" - "\t\t\t EROFS_GET_BLOCKS_FIEMAP);", - "\t\t\tmap = (struct erofs_map_blocks) {\n" - "\t\t\t\t.m_la = sbi->metabox_en->size - 1,\n" - "\t\t\t};\n" - "\t\t\terror = erofs_map_blocks(sbi, sbi->metabox_en, &map);", - "super.c", -) -if sources["super.c"] != expected_super: - raise SystemExit("B07c super.c contains changes beyond metabox migration") - -old_backend = extract_function(old["zmap.c"], "z_erofs_map_blocks_iter") -expected_backend = replace_once( - old_backend, - "z_erofs_map_blocks_iter(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n" - " struct erofs_map_blocks *map, int flags)", - "z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n" - " struct erofs_map_blocks *map)", - "zmap signature", -) -expected_backend = expected_backend.replace( - "z_erofs_map_blocks_ext(sbi, vi, map, flags)", - "z_erofs_map_blocks_ext(sbi, vi, map,\n\t\t\t EROFS_GET_BLOCKS_FIEMAP)", -) -expected_backend = expected_backend.replace( - "z_erofs_map_blocks_fo(sbi, vi, map, flags)", - "z_erofs_map_blocks_fo(sbi, vi, map,\n\t\t\t EROFS_GET_BLOCKS_FIEMAP)", -) -if backend != expected_backend: - raise SystemExit("compressed backend changed beyond FIEMAP interface closure") -expected_zmap = old["zmap.c"].replace(old_backend, expected_backend) -if sources["zmap.c"] != expected_zmap: - raise SystemExit("B07c zmap.c contains changes outside the backend wrapper") - -old_data_functions = function_names(old["data.c"]) -data_functions = function_names(sources["data.c"]) -if old_data_functions - data_functions != {"erofs_map_blocks_legacy"}: - raise SystemExit("B07c data.c removed functions beyond the legacy adapter") -if data_functions - old_data_functions: - raise SystemExit("B07c data.c added an unexpected function") -changed_data_functions = { - "erofs_map_blocks", - "erofs_read_data", - "erofs_read_uio", -} -for function in sorted(data_functions - changed_data_functions): - if extract_function(old["data.c"], function) != extract_function( - sources["data.c"], function - ): - raise SystemExit(f"B07c changed protected data.c function: {function}") - -cleanup = Counter( - (record["actual"]["acquire_count"], record["actual"]["release_count"]) - for record in tuples["records"] -) -if any(acquire != release for acquire, release in cleanup): - raise SystemExit(f"metadata cleanup became unbalanced: {cleanup!r}") -if any(record["actual"]["errno"] < 0 for record in tuples["records"]): - raise SystemExit("negative errno observed") - -tuple_bytes = b"".join( - bytes.fromhex(record["tuple_hex"]) for record in tuples["records"] -) -summary = { - "status": "PASS", - "final_id": "P15-006", - "map_cases": 80, - "producer_cases": 50, - "compressed_cases": 30, - "device_cases": 13, - "full_tuple_diff_count": len(tuple_diffs), - "tuple_bytes": len(tuple_bytes), - "tuple_bytes_sha256": hashlib.sha256(tuple_bytes).hexdigest(), - "model_sha256": result["model_sha256"], - "legacy_callsite_count": 0, - "data_consumers": 2, - "compressed_consumers": 2, - "backend_fiemap_dispatches": 2, - "cleanup_distribution": { - f"{acquire}:{release}": count - for (acquire, release), count in sorted(cleanup.items()) - }, - "qemu": "NOT_RUN", - "full_feature_suite": "NOT_RUN", -} -with output.open("x", encoding="ascii") as stream: - json.dump(summary, stream, ensure_ascii=True, indent=2, sort_keys=True) - stream.write("\n") -print("B07c PASS full=80 diff=0 device=13 legacy=0 consumers=4") -PY -then - : -else - pre15_dut_fail 'B07c consumer boundary check failed' -fi - -sha256sum "$artifacts/baseline/result.json" \ - "$artifacts/baseline/tuples.json" "$artifacts/baseline/devices.json" \ - "$artifacts/candidate/result.json" "$artifacts/candidate/tuples.json" \ - "$artifacts/candidate/devices.json" "$artifacts/consumer-check.json" \ - >"$artifacts/SHA256SUMS" -printf 'B07c PASS common consumers; legacy map interfaces absent\n' diff --git a/tests/pre15/cases/B08-plain-run.sh b/tests/pre15/cases/B08-plain-run.sh deleted file mode 100755 index bd4b914..0000000 --- a/tests/pre15/cases/B08-plain-run.sh +++ /dev/null @@ -1,907 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -gate=$PRE15_DUT/tests/pre15/gates/P15-006.sh -input=$PRE15_DUT/tests/pre15/gates/P15-006-input.json -oracle=$PRE15_DUT/tests/pre15/fixtures/B07-map-oracle.json -map_fixture=$PRE15_DUT/tests/pre15/fixtures/B08-map-runs.json -io_fixture=$PRE15_DUT/tests/pre15/fixtures/B08-io-caps.json -b07c_case=$PRE15_DUT/tests/pre15/cases/B07c-map-consumers.sh -artifacts=$PRE15_RUN_DIR/artifacts -baseline=6673f51152a5195a8a8903aa801f820abce7936e -b07c=55c609db13fc9b0f21a3c7c9ec5cb9574fd18276 - -for tool in cc git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B08 host tool: $tool" -done - -pre15_record_fixture b08-gate "$gate" -pre15_record_fixture b08-input "$input" -pre15_record_fixture b08-b07-oracle "$oracle" -pre15_record_fixture b08-map-runs "$map_fixture" -pre15_record_fixture b08-io-caps "$io_fixture" -pre15_record_fixture b08-b07c-case "$b07c_case" -pre15_record_fixture b08-data "$PRE15_DUT/src/data.c" - -mkdir -p "$artifacts" -pre15_target_reached - -if "$gate" --base "$baseline" --oracle "$oracle" \ - --output "$artifacts/baseline" >"$artifacts/baseline.stdout" \ - 2>"$artifacts/baseline.stderr"; then - : -else - pre15_dut_fail 'B08 frozen P15-006 baseline replay failed' -fi - -if python3 - "$b07c_case" "$artifacts/derive-B07c.py" <<'PY' -from pathlib import Path -import sys - - -source = Path(sys.argv[1]).read_text(encoding="utf-8") -marker = 'if python3 - "$gate" "$artifacts/P15-006-B07c.py" <<\'PY\'\n' -start = source.index(marker) + len(marker) -end = source.index("\nPY\nthen", start) -Path(sys.argv[2]).write_text(source[start:end] + "\n", encoding="ascii") -PY -then - : -else - pre15_dut_fail 'B08 could not recover the proven B07c replay adapter' -fi - -if python3 "$artifacts/derive-B07c.py" "$gate" \ - "$artifacts/P15-006-B08.py" >"$artifacts/derive.stdout" \ - 2>"$artifacts/derive.stderr"; then - : -else - pre15_dut_fail 'B08 could not derive the current map replay' -fi - -candidate_rc=0 -python3 "$artifacts/P15-006-B08.py" "$PRE15_ROOT" "$input" \ - worktree '' "$artifacts/candidate" "$oracle" \ - >"$artifacts/candidate.stdout" 2>"$artifacts/candidate.stderr" || \ - candidate_rc=$? -if test "$candidate_rc" -ne 1; then - pre15_dut_fail 'B08 old-run oracle did not reject exactly the intended change' -fi - -if python3 - "$PRE15_ROOT" "$PRE15_DUT" "$oracle" "$map_fixture" \ - "$io_fixture" "$artifacts/baseline" "$artifacts/candidate" \ - "$artifacts/B08-result.json" "$b07c" "$artifacts" <<'PY' -from __future__ import annotations - -from collections import Counter -import hashlib -import json -from pathlib import Path -import re -import struct -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -oracle_path = Path(sys.argv[3]) -map_fixture_path = Path(sys.argv[4]) -io_fixture_path = Path(sys.argv[5]) -baseline = Path(sys.argv[6]) -candidate = Path(sys.argv[7]) -output = Path(sys.argv[8]) -b07c = sys.argv[9] -artifacts = Path(sys.argv[10]) - - -def load(path: Path) -> dict: - return json.loads(path.read_text(encoding="ascii")) - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def function_names(source: str) -> set[str]: - return set(re.findall( - r"^([A-Za-z_][A-Za-z0-9_]*)\s*\(", source, re.MULTILINE - )) - - -def committed_data() -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{b07c}:repo-pre-15/src/data.c"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B07c data.c: {completed.stderr}") - return completed.stdout - - -def require_order(source: str, markers: list[str], label: str) -> None: - position = -1 - for marker in markers: - next_position = source.find(marker, position + 1) - if next_position < 0: - raise SystemExit(f"{label} is missing ordered marker: {marker}") - position = next_position - - -def compile_and_run(name: str, program: str) -> list[str]: - source_path = artifacts / f"{name}.c" - binary_path = artifacts / name - source_path.write_text(program, encoding="ascii") - compiled = subprocess.run( - [ - "cc", "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", - str(source_path), "-o", str(binary_path), - ], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - (artifacts / f"{name}.compile.stdout").write_text( - compiled.stdout, encoding="utf-8" - ) - (artifacts / f"{name}.compile.stderr").write_text( - compiled.stderr, encoding="utf-8" - ) - if compiled.returncode != 0: - raise SystemExit(f"{name} compilation failed") - executed = subprocess.run( - [str(binary_path)], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - (artifacts / f"{name}.stdout").write_text( - executed.stdout, encoding="utf-8" - ) - (artifacts / f"{name}.stderr").write_text( - executed.stderr, encoding="utf-8" - ) - if executed.returncode != 0: - raise SystemExit(f"{name} execution failed") - return executed.stdout.splitlines() - - -oracle = load(oracle_path) -map_fixture = load(map_fixture_path) -io_fixture = load(io_fixture_path) -base_result = load(baseline / "result.json") -base_tuples = load(baseline / "tuples.json") -base_devices = load(baseline / "devices.json") -result = load(candidate / "result.json") -tuples = load(candidate / "tuples.json") -devices = load(candidate / "devices.json") - -if sha256_bytes(oracle_path.read_bytes()) != map_fixture["b07_oracle_sha256"]: - raise SystemExit("B08 map fixture is not anchored to the frozen B07 oracle") -if map_fixture["schema"] != 1 or map_fixture["candidate"] != "P15-074": - raise SystemExit("invalid B08 map fixture identity") -if io_fixture["schema"] != 1 or io_fixture["candidate"] != "P15-074": - raise SystemExit("invalid B08 I/O fixture identity") -if ( - base_result["status"] != "GO" - or base_result["oracle_equal"] is not True - or base_result["map_case_count"] != 80 - or base_result["device_case_count"] != 13 -): - raise SystemExit("frozen B07 control replay did not GO") -if result["status"] != "STOP" or result["map_case_count"] != 80: - raise SystemExit("B08 candidate did not expose the intended old-run delta") -if result["device_case_count"] != 13 or result["model_sha256"] != oracle["model_sha256"]: - raise SystemExit("B08 candidate changed the independent corpus denominator") -if result["adapter_probe"]["status"] != "PASS": - raise SystemExit("B08 common map adapter probe failed") -if devices["status"] != "PASS" or devices["records"] != base_devices["records"]: - raise SystemExit("B08 changed one of the 13 frozen device records") - -tuple_fields = ( - "m_la", "m_pa", "m_llen", "m_plen", "m_deviceid", "m_flags", - "m_algorithmformat", "errno", "acquire_count", "release_count", -) -tuple_layout = map_fixture["b07_tuple_layout"] -if tuple_layout != oracle["tuple_byte_layout"]: - raise SystemExit("B08 tuple byte layout drifted") -oracle_records = {record["id"]: record for record in oracle["records"]} -actual_records = {record["id"]: record for record in tuples["records"]} -base_records = {record["id"]: record for record in base_tuples["records"]} -transforms = {record["id"]: record for record in map_fixture["b07_transforms"]} -if len(oracle_records) != 80 or set(actual_records) != set(oracle_records): - raise SystemExit("B08 map tuple ID set changed") -if set(base_records) != set(oracle_records): - raise SystemExit("B08 baseline tuple ID set changed") - -expected_bytes = bytearray() -actual_bytes = bytearray() -changed_ids = [] -for record in oracle["records"]: - case_id = record["id"] - if base_records[case_id]["tuple_hex"] != record["tuple_hex"]: - raise SystemExit(f"frozen B07 baseline drifted: {case_id}") - expected = dict(zip( - tuple_fields, - struct.unpack(tuple_layout, bytes.fromhex(record["tuple_hex"])), - strict=True, - )) - if case_id in transforms: - expected["m_llen"] = transforms[case_id]["m_llen"] - expected["m_plen"] = transforms[case_id]["m_plen"] - expected_tuple = struct.pack( - tuple_layout, *(expected[field] for field in tuple_fields) - ) - actual_tuple = bytes.fromhex(actual_records[case_id]["tuple_hex"]) - if actual_tuple != expected_tuple: - raise SystemExit(f"B08 transformed tuple mismatch: {case_id}") - if actual_tuple != bytes.fromhex(record["tuple_hex"]): - changed_ids.append(case_id) - expected_bytes.extend(expected_tuple) - actual_bytes.extend(actual_tuple) - -if set(changed_ids) != set(transforms): - raise SystemExit(f"B08 changed unexpected B07 tuples: {changed_ids!r}") -transformed_hash = sha256_bytes(bytes(expected_bytes)) -if bytes(actual_bytes) != bytes(expected_bytes): - raise SystemExit("B08 full transformed tuple stream mismatch") -if transformed_hash != map_fixture["expected_transformed_tuple_sha256"]: - raise SystemExit("B08 transformed tuple fixture digest mismatch") -allowed_failures = [] -for failure in result["failures"]: - if failure.get("field") == "frozen-byte-oracle" and "id" not in failure: - continue - if failure.get("id") in transforms and failure.get("field") in { - "m_llen", "m_plen" - }: - continue - allowed_failures.append(failure) -if allowed_failures: - raise SystemExit(f"B08 old oracle reported unrelated failures: {allowed_failures!r}") - -data_source = (dut / "src/data.c").read_text(encoding="utf-8") -old_data = committed_data() -expected_preamble = old_data[:old_data.index("static erofs_off_t\n")].replace( - "#include \n", - "#include \n#include \n", -) -if data_source[:data_source.index("static erofs_off_t\n")] != expected_preamble: - raise SystemExit("B08 data.c preamble changed beyond the MAXPHYS header") -old_functions = function_names(old_data) -data_functions = function_names(data_source) -if data_functions != old_functions: - raise SystemExit("B08 added or removed a data.c function") -changed_functions = { - "erofs_map_blocks_flatmode", "erofs_read_data", "erofs_read_uio" -} -for function in sorted(data_functions - changed_functions): - if extract_function(data_source, function) != extract_function( - old_data, function - ): - raise SystemExit(f"B08 changed out-of-scope data.c function: {function}") -for function in changed_functions: - if extract_function(data_source, function) == extract_function( - old_data, function - ): - raise SystemExit(f"B08 did not change required function: {function}") - -flatmode = extract_function(data_source, "erofs_map_blocks_flatmode") -read_data = extract_function(data_source, "erofs_read_data") -read_uio = extract_function(data_source, "erofs_read_uio") -if flatmode.count("map->m_llen = remain;") != 1: - raise SystemExit("plain mapping does not expose the EOF run") -if flatmode.count("map->m_llen = MIN(remain, tail_start - loff);") != 1: - raise SystemExit("pre-inline mapping does not stop at the inline tail") -if "MAXPHYS" in flatmode: - raise SystemExit("B08 incorrectly capped the map contract") -if read_data.count("(uint64_t)MAXPHYS") != 1: - raise SystemExit("erofs_read_data lacks one MAXPHYS cap") -if read_uio.count("(uint64_t)MAXPHYS") != 1: - raise SystemExit("erofs_read_uio lacks one MAXPHYS cap") -require_order( - read_data, - [ - "if (erofs_inode_is_data_compressed(vi->datalayout))", - "return (z_erofs_read_data(sbi, vi, loff, len, bufp));", - "error = erofs_map_blocks(sbi, vi, &map);", - "MIN(map.m_llen, (uint64_t)MAXPHYS)", - "erofs_read_physical(sbi, map.m_deviceid, map.m_pa", - "erofs_brelse(blk);", - ], - "erofs_read_data", -) -require_order( - read_uio, - [ - "if (erofs_inode_is_data_compressed(vi->datalayout))", - "return (z_erofs_read_uio(sbi, vi, uio));", - "error = erofs_map_blocks(sbi, vi, &map);", - "MIN(map.m_llen, (uint64_t)MAXPHYS)", - "erofs_read_physical(sbi, map.m_deviceid, map.m_pa", - "error = uiomove(blk, want, uio);", - "erofs_brelse(blk);", - ], - "erofs_read_uio", -) - -map_common = r''' -#include -#include -#include -#include - -#define EOPNOTSUPP 45 -#define EOVERFLOW 84 -#define EINTEGRITY 97 -#define EROFS_NULL_ADDR UINT32_MAX -#define EROFS_INODE_FLAT_PLAIN 0 -#define EROFS_INODE_COMPRESSED_FULL 1 -#define EROFS_INODE_FLAT_INLINE 2 -#define EROFS_INODE_COMPRESSED_COMPACT 3 -#define EROFS_INODE_CHUNK_BASED 4 -#define EROFS_MAP_MAPPED 0x0001 -#define EROFS_MAP_META 0x0002 -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define roundup2(x, y) (((x) + ((y) - 1)) & ~((y) - 1)) - -typedef uint64_t erofs_off_t; -typedef uint64_t erofs_blk_t; - -struct erofs_sb_info { - uint64_t block_size; - unsigned int blkszbits; -}; - -struct erofs_inode { - uint64_t size; - erofs_off_t inode_off; - erofs_blk_t startblk; - uint8_t datalayout; - uint8_t inode_isize; - uint32_t xattr_isize; -}; - -struct erofs_map_blocks { - erofs_off_t m_pa, m_la; - uint64_t m_plen, m_llen; - unsigned short m_deviceid; - char m_algorithmformat; - unsigned int m_flags; -}; - -static bool -erofs_inode_is_data_compressed(unsigned int layout) -{ - return (layout == EROFS_INODE_COMPRESSED_FULL || - layout == EROFS_INODE_COMPRESSED_COMPACT); -} - -static int -erofs_map_blocks_chunk(struct erofs_sb_info *sbi, struct erofs_inode *vi, - struct erofs_map_blocks *map) -{ - (void)sbi; - (void)vi; - (void)map; - return (EOPNOTSUPP); -} - -static int -z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, - struct erofs_map_blocks *map) -{ - (void)sbi; - (void)vi; - (void)map; - return (EOPNOTSUPP); -} -''' -map_program = map_common -map_program += extract_function(data_source, "erofs_inline_tail_start") + "\n" -map_program += flatmode + "\n" -map_program += extract_function(data_source, "erofs_map_blocks") + "\n" -map_program += "\nint\nmain(void)\n{\n\tint error;\n" -for case in map_fixture["map_cases"]: - inode = case["inode"] - map_program += f''' - {{ - struct erofs_sb_info sbi = {{ .block_size = 4096, .blkszbits = 12 }}; - struct erofs_inode vi = {{ - .size = UINT64_C({inode['size']}), - .inode_off = UINT64_C({inode.get('inode_off', 0)}), - .startblk = UINT64_C({inode['startblk']}), - .datalayout = {inode['layout']}, - .inode_isize = {inode.get('inode_isize', 0)}, - .xattr_isize = {inode.get('xattr_isize', 0)}, - }}; - struct erofs_map_blocks map = {{ .m_la = UINT64_C({case['request']}) }}; - error = erofs_map_blocks(&sbi, &vi, &map); - printf("map\\t{case['id']}\\t%llu\\t%llu\\t%llu\\t%llu\\t%u\\t%u\\t%d\\t%d\\n", - (unsigned long long)map.m_la, (unsigned long long)map.m_pa, - (unsigned long long)map.m_llen, (unsigned long long)map.m_plen, - map.m_deviceid, map.m_flags, (int)map.m_algorithmformat, error); - }} -''' -map_program += "\treturn (0);\n}\n" -map_lines = compile_and_run("B08-map-extractor", map_program) -map_actual = {} -for line in map_lines: - fields = line.split("\t") - if len(fields) != 10 or fields[0] != "map": - raise SystemExit(f"invalid B08 map extractor line: {line!r}") - map_actual[fields[1]] = dict(zip( - ( - "m_la", "m_pa", "m_llen", "m_plen", "m_deviceid", - "m_flags", "m_algorithmformat", "errno", - ), - [int(value) for value in fields[2:]], - strict=True, - )) -if set(map_actual) != {case["id"] for case in map_fixture["map_cases"]}: - raise SystemExit("B08 custom map case set changed") -positive_runs = 0 -for case in map_fixture["map_cases"]: - actual = map_actual[case["id"]] - expected = {**case["expected"], "m_algorithmformat": 0} - if actual != expected: - raise SystemExit( - f"B08 custom map mismatch {case['id']}: {actual!r} != {expected!r}" - ) - if ( - actual["errno"] == 0 - and case["request"] < case["inode"]["size"] - ): - if actual["m_llen"] == 0: - raise SystemExit(f"B08 non-EOF zero run: {case['id']}") - positive_runs += 1 - -maxphys = io_fixture["harness_maxphys"] -consumer_common = f''' -#include -#include -#include -#include -#include -#include - -#define EIO 5 -#define ENOMEM 12 -#define EINVAL 22 -#define EOPNOTSUPP 45 -#define EOVERFLOW 84 -#define EINTEGRITY 97 -#define PAGE_SIZE 4096 -#define MAXPHYS {maxphys} -#define M_EROFS 0 -#define M_WAITOK 0 -#define EROFS_INODE_FLAT_PLAIN 0 -#define EROFS_MAP_MAPPED 0x0001 -#define EROFS_MAP_META 0x0002 -#define EROFS_BUF_INITIALIZER {{ .data = NULL, .release = NULL }} -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define bzero(ptr, len) memset((ptr), 0, (len)) - -typedef uint64_t erofs_off_t; -typedef uint64_t erofs_nid_t; - -struct erofs_sb_info {{ int unused; }}; -struct erofs_inode {{ - erofs_nid_t nid; - uint64_t size; - uint8_t datalayout; -}}; -struct erofs_map_blocks {{ - erofs_off_t m_pa, m_la; - uint64_t m_plen, m_llen; - unsigned short m_deviceid; - char m_algorithmformat; - unsigned int m_flags; -}}; -struct erofs_buf {{ - void *data; - void (*release)(void *); -}}; -struct uio {{ - int64_t uio_offset; - size_t uio_resid; - unsigned char *buffer; - size_t moved; -}}; - -static size_t gate_allocations; - -static void * -gate_malloc(size_t size) -{{ - void *buffer = malloc(size); - if (buffer != NULL) - ++gate_allocations; - return (buffer); -}} - -static void -gate_free(void *buffer) -{{ - if (buffer != NULL) {{ - --gate_allocations; - free(buffer); - }} -}} - -#define malloc(size, type, flags) gate_malloc(size) -#define free(buffer, type) gate_free(buffer) - -#define GATE_MAX_CALLS 32 -#define GATE_PHYSICAL_BASE UINT64_C(1048576) -static bool gate_hole; -static unsigned int gate_map_calls; -static unsigned int gate_physical_calls; -static erofs_off_t gate_physical_offsets[GATE_MAX_CALLS]; -static size_t gate_physical_lengths[GATE_MAX_CALLS]; - -static void -gate_reset(bool hole) -{{ - gate_hole = hole; - gate_map_calls = 0; - gate_physical_calls = 0; - memset(gate_physical_offsets, 0, sizeof(gate_physical_offsets)); - memset(gate_physical_lengths, 0, sizeof(gate_physical_lengths)); -}} - -static bool -erofs_inode_is_data_compressed(unsigned int layout) -{{ - (void)layout; - return (false); -}} - -static int -erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, - struct erofs_map_blocks *map) -{{ - struct erofs_map_blocks next = {{ .m_la = map->m_la }}; - (void)sbi; - ++gate_map_calls; - if (next.m_la < vi->size) {{ - next.m_pa = GATE_PHYSICAL_BASE + next.m_la; - next.m_llen = vi->size - next.m_la; - next.m_plen = next.m_llen; - if (!gate_hole) - next.m_flags = EROFS_MAP_MAPPED; - }} - *map = next; - return (0); -}} - -static int -erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id, - erofs_off_t off, size_t len, void **bufp) -{{ - unsigned char *buffer; - size_t index; - (void)sbi; - (void)device_id; - if (len > MAXPHYS || gate_physical_calls == GATE_MAX_CALLS) - return (EINVAL); - gate_physical_offsets[gate_physical_calls] = off; - gate_physical_lengths[gate_physical_calls] = len; - ++gate_physical_calls; - buffer = gate_malloc(len); - if (buffer == NULL) - return (ENOMEM); - for (index = 0; index < len; ++index) - buffer[index] = (unsigned char)(off - GATE_PHYSICAL_BASE + index); - *bufp = buffer; - return (0); -}} - -static int -erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid, - erofs_off_t off, size_t len, struct erofs_buf *buf) -{{ - (void)sbi; - (void)nid; - (void)off; - (void)len; - (void)buf; - return (EIO); -}} - -static void -erofs_put_metabuf(struct erofs_buf *buf) -{{ - (void)buf; -}} - -static void -erofs_brelse(void *buffer) -{{ - gate_free(buffer); -}} - -static int -uiomove(const void *source, size_t len, struct uio *uio) -{{ - if (len > uio->uio_resid) - return (EINVAL); - memcpy(uio->buffer + uio->moved, source, len); - uio->moved += len; - uio->uio_resid -= len; - uio->uio_offset += (int64_t)len; - return (0); -}} - -static int -z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, - erofs_off_t loff, size_t len, void **bufp) -{{ - (void)sbi; - (void)vi; - (void)loff; - (void)len; - (void)bufp; - return (EOPNOTSUPP); -}} - -static int -z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, - struct uio *uio) -{{ - (void)sbi; - (void)vi; - (void)uio; - return (EOPNOTSUPP); -}} - -static int -gate_check_data(const unsigned char *buffer, size_t length, - uint64_t offset, bool hole) -{{ - size_t index; - for (index = 0; index < length; ++index) {{ - unsigned char expected = hole ? 0 : (unsigned char)(offset + index); - if (buffer[index] != expected) - return (1); - }} - return (0); -}} - -static int -gate_fail(const char *id, const char *reason) -{{ - fprintf(stderr, "%s: %s\\n", id, reason); - return (1); -}} -''' -consumer_program = consumer_common -consumer_program += read_data + "\n" -consumer_program += read_uio + "\n" -consumer_program += "\nint\nmain(void)\n{\n" -for scenario in io_fixture["scenarios"]: - scenario_id = scenario["id"] - hole = scenario["mapping"] == "hole" - expected_offsets = scenario["expected_physical_offsets"] - expected_lengths = scenario["expected_physical_lengths"] - checks = [] - for index, (offset, length) in enumerate(zip( - expected_offsets, expected_lengths, strict=True - )): - checks.append( - f"\t\tif (gate_physical_offsets[{index}] != UINT64_C({offset}) || " - f"gate_physical_lengths[{index}] != {length})\n" - f"\t\t\treturn (gate_fail(\"{scenario_id}\", \"physical call tuple\"));\n" - ) - call_checks = "".join(checks) - if scenario["api"] == "read_data": - success_check = "" - if scenario["expected_errno"] == 0: - success_check = f''' - if (buffer == NULL || gate_check_data(buffer, {scenario['length']}, - UINT64_C({scenario['offset']}), {'true' if hole else 'false'}) != 0) - return (gate_fail("{scenario_id}", "returned data")); -''' - else: - success_check = f''' - if (buffer != NULL) - return (gate_fail("{scenario_id}", "error buffer ownership")); -''' - consumer_program += f''' - {{ - struct erofs_sb_info sbi = {{ 0 }}; - struct erofs_inode vi = {{ - .nid = 1, .size = UINT64_C({scenario['inode_size']}), - .datalayout = EROFS_INODE_FLAT_PLAIN, - }}; - void *buffer = NULL; - int error; - - gate_reset({'true' if hole else 'false'}); - error = erofs_read_data(&sbi, &vi, UINT64_C({scenario['offset']}), - {scenario['length']}, &buffer); - if (error != {scenario['expected_errno']}) - return (gate_fail("{scenario_id}", "errno")); - if (gate_map_calls != {scenario['expected_map_calls']} || - gate_physical_calls != {len(expected_lengths)}) - return (gate_fail("{scenario_id}", "call count")); -{call_checks}{success_check} gate_free(buffer); - if (gate_allocations != 0) - return (gate_fail("{scenario_id}", "allocation balance")); - printf("io\\t{scenario_id}\\tPASS\\t%u\\t%u\\n", - gate_map_calls, gate_physical_calls); - }} -''' - elif scenario["api"] == "read_uio": - consumer_program += f''' - {{ - struct erofs_sb_info sbi = {{ 0 }}; - struct erofs_inode vi = {{ - .nid = 1, .size = UINT64_C({scenario['inode_size']}), - .datalayout = EROFS_INODE_FLAT_PLAIN, - }}; - unsigned char *buffer = gate_malloc({scenario['length']}); - struct uio uio = {{ - .uio_offset = {scenario['offset']}, .uio_resid = {scenario['length']}, - .buffer = buffer, .moved = 0, - }}; - int error; - - if (buffer == NULL) - return (gate_fail("{scenario_id}", "test allocation")); - memset(buffer, 0xa5, {scenario['length']}); - gate_reset({'true' if hole else 'false'}); - error = erofs_read_uio(&sbi, &vi, &uio); - if (error != {scenario['expected_errno']} || - uio.uio_offset != {scenario['expected_final_offset']} || - uio.uio_resid != {scenario['expected_resid']}) - return (gate_fail("{scenario_id}", "uio result")); - if (gate_map_calls != {scenario['expected_map_calls']} || - gate_physical_calls != {len(expected_lengths)}) - return (gate_fail("{scenario_id}", "call count")); -{call_checks} if (uio.moved != {scenario['length'] - scenario['expected_resid']} || - gate_check_data(buffer, uio.moved, UINT64_C({scenario['offset']}), - {'true' if hole else 'false'}) != 0) - return (gate_fail("{scenario_id}", "moved data")); - gate_free(buffer); - if (gate_allocations != 0) - return (gate_fail("{scenario_id}", "allocation balance")); - printf("io\\t{scenario_id}\\tPASS\\t%u\\t%u\\n", - gate_map_calls, gate_physical_calls); - }} -''' - else: - raise SystemExit(f"unknown B08 I/O API: {scenario['api']}") -consumer_program += "\treturn (0);\n}\n" -io_lines = compile_and_run("B08-io-extractor", consumer_program) -expected_io_ids = [scenario["id"] for scenario in io_fixture["scenarios"]] -actual_io_ids = [] -for line in io_lines: - fields = line.split("\t") - if len(fields) != 5 or fields[0] != "io" or fields[2] != "PASS": - raise SystemExit(f"invalid B08 I/O extractor line: {line!r}") - actual_io_ids.append(fields[1]) -if actual_io_ids != expected_io_ids: - raise SystemExit("B08 I/O scenario order or denominator changed") - -cleanup = Counter( - (record["actual"]["acquire_count"], record["actual"]["release_count"]) - for record in tuples["records"] -) -if any(acquire != release for acquire, release in cleanup): - raise SystemExit(f"B08 metadata cleanup became unbalanced: {cleanup!r}") -if any(record["actual"]["errno"] < 0 for record in tuples["records"]): - raise SystemExit("B08 observed a negative errno") - -summary = { - "status": "PASS", - "final_id": "P15-074", - "b07_map_cases": 80, - "b07_unchanged_tuples": 80 - len(changed_ids), - "b07_intended_run_tuples": len(changed_ids), - "b07_unexpected_tuple_differences": 0, - "b07_device_cases": 13, - "b07_device_differences": 0, - "b07_model_sha256": result["model_sha256"], - "transformed_tuple_bytes": len(actual_bytes), - "transformed_tuple_sha256": sha256_bytes(bytes(actual_bytes)), - "custom_map_cases": len(map_fixture["map_cases"]), - "custom_positive_runs": positive_runs, - "io_scenarios": len(io_fixture["scenarios"]), - "harness_maxphys": maxphys, - "max_observed_physical_io": max( - length - for scenario in io_fixture["scenarios"] - for length in scenario["expected_physical_lengths"] - ), - "cleanup_distribution": { - f"{acquire}:{release}": count - for (acquire, release), count in sorted(cleanup.items()) - }, - "compressed_scope": "UNCHANGED", - "vnode_backed_scope": "NOT_ADDED", - "qemu": "NOT_RUN", - "full_feature_suite": "NOT_RUN", -} -with output.open("x", encoding="ascii") as stream: - json.dump(summary, stream, ensure_ascii=True, indent=2, sort_keys=True) - stream.write("\n") -print( - "B08 PASS map=80 unchanged=73 intended=7 device=13 " - f"custom={len(map_fixture['map_cases'])} io={len(io_fixture['scenarios'])}" -) -PY -then - : -else - pre15_dut_fail 'B08 plain-run/MAXPHYS boundary check failed' -fi - -sha256sum "$artifacts/baseline/result.json" \ - "$artifacts/baseline/tuples.json" "$artifacts/baseline/devices.json" \ - "$artifacts/candidate/result.json" "$artifacts/candidate/tuples.json" \ - "$artifacts/candidate/devices.json" "$artifacts/B08-result.json" \ - "$artifacts/B08-map-extractor" "$artifacts/B08-io-extractor" \ - >"$artifacts/SHA256SUMS" -printf 'B08 PASS contiguous plain/pre-inline runs with MAXPHYS-capped I/O\n' diff --git a/tests/pre15/cases/B09-vnode.sh b/tests/pre15/cases/B09-vnode.sh deleted file mode 100755 index d0b3643..0000000 --- a/tests/pre15/cases/B09-vnode.sh +++ /dev/null @@ -1,375 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=8e85a8b64dd1a956376ff1cb31d2157bc73df8ad -artifacts=$PRE15_RUN_DIR/artifacts -inode=$PRE15_DUT/src/inode.c -vnops=$PRE15_DUT/src/erofs_vnops.c -super=$PRE15_DUT/src/super.c -internal=$PRE15_DUT/src/internal.h -namei=$PRE15_DUT/src/namei.c - -for tool in git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B09 host tool: $tool" -done - -pre15_record_fixture b09-inode "$inode" -pre15_record_fixture b09-vnops "$vnops" -pre15_record_fixture b09-super "$super" -pre15_record_fixture b09-internal "$internal" -pre15_record_fixture b09-namei "$namei" -pre15_record_fixture b09-case "$PRE15_DUT/tests/pre15/cases/B09-vnode.sh" - -mkdir -p "$artifacts" -pre15_target_reached - -if python3 - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" "$artifacts" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -artifacts = Path(sys.argv[4]) -src = dut / "src" - - -def committed(path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/src/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B09 baseline {path}: {completed.stderr}") - return completed.stdout - - -def replace_once(source: str, old: str, new: str, label: str) -> str: - count = source.count(old) - if count != 1: - raise SystemExit(f"{label}: expected one transform source, found {count}") - return source.replace(old, new, 1) - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - if brace < 0: - raise SystemExit(f"missing function body: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def initializer(source: str, declaration: str, prefix: str) -> list[tuple[str, str]]: - match = re.search( - re.escape(declaration) + r"\s*=\s*\{(.*?)\n\};", source, re.DOTALL - ) - if not match: - raise SystemExit(f"initializer absent: {declaration}") - return re.findall( - rf"^\s*\.({prefix}_[A-Za-z0-9_]+)\s*=\s*([^,]+),", - match.group(1), - re.MULTILINE, - ) - - -def require_order(source: str, markers: list[str], label: str) -> None: - position = -1 - for marker in markers: - position = source.find(marker, position + 1) - if position < 0: - raise SystemExit(f"{label} is missing ordered marker: {marker}") - - -current = { - name: (src / name).read_text(encoding="utf-8") - for name in ("inode.c", "erofs_vnops.c", "super.c", "internal.h", "namei.c") -} -base = {name: committed(name) for name in current} - -moved_marker = "static u_int\nerofs_vfs_hash(erofs_nid_t nid)" -moved_offset = base["inode.c"].find(moved_marker) -if moved_offset < 0: - raise SystemExit("B09 baseline vnode adapter block is absent") -moved_block = base["inode.c"][moved_offset:] -expected_inode = base["inode.c"][:moved_offset].rstrip("\n") + "\n" -if current["inode.c"] != expected_inode: - raise SystemExit("inode.c changed outside the exact vnode adapter movement") -if current["internal.h"] != base["internal.h"]: - raise SystemExit("internal.h changed despite no B09 interface delta") -if current["namei.c"] != base["namei.c"]: - raise SystemExit("namecache/lookup code changed outside the B09 write set") - -expected_vnops = replace_once( - base["erofs_vnops.c"], - "#include \n", - "#include \n#include \n", - "vnode hash include", -) -old_open = """\tif (vp->v_type == VREG) { -\t\tif (vnode_create_vobject(vp, vi->size, ap->a_td) != 0) -\t\t\treturn (ENOMEM); -\t} -""" -new_open = """\tif (vp->v_type == VREG) -\t\tvnode_create_vobject(vp, vi->size, ap->a_td); -""" -expected_vnops = replace_once(expected_vnops, old_open, new_open, "pager dead branch") -expected_vnops = replace_once( - expected_vnops, - "static int\nerofs_reclaim(struct vop_reclaim_args *ap)", - moved_block + "\nstatic int\nerofs_reclaim(struct vop_reclaim_args *ap)", - "vnode adapter placement", -) -if current["erofs_vnops.c"] != expected_vnops: - raise SystemExit("erofs_vnops.c differs from the two declared B09 transforms") - -expected_super = replace_once( - base["super.c"], "static vfs_vget_t erofs_vgetf;\n", "", "vget declaration" -) -expected_super = replace_once( - expected_super, - """static int -erofs_vgetf(struct mount *mp, ino_t ino, int flags, struct vnode **vpp) -{ -\treturn (erofs_vget(mp, ino, flags, vpp)); -} - -""", - "", - "vget wrapper", -) -expected_super = replace_once( - expected_super, - "\t.vfs_vget = erofs_vgetf,", - "\t.vfs_vget = erofs_vget,", - "vfs_vget slot", -) -if current["super.c"] != expected_super: - raise SystemExit("super.c differs from the exact direct-vget transform") - -moved_functions = ( - "erofs_vfs_hash", - "erofs_vfs_hash_cmp", - "erofs_fill_vnode", - "erofs_vget", -) -for name in moved_functions: - if extract_function(base["inode.c"], name) != extract_function( - current["erofs_vnops.c"], name - ): - raise SystemExit(f"moved function changed semantics: {name}") - definitions = { - path: len(re.findall(r"^" + re.escape(name) + r"\s*\(", text, re.MULTILINE)) - for path, text in current.items() - } - if definitions["erofs_vnops.c"] != 1 or sum(definitions.values()) != 1: - raise SystemExit(f"B09 function ownership is not unique: {name} {definitions}") - -base_vnode_slots = initializer( - base["erofs_vnops.c"], "struct vop_vector erofs_vnodeops", "vop" -) -base_fifo_slots = initializer( - base["erofs_vnops.c"], "struct vop_vector erofs_fifoops", "vop" -) -vnode_slots = initializer( - current["erofs_vnops.c"], "struct vop_vector erofs_vnodeops", "vop" -) -fifo_slots = initializer( - current["erofs_vnops.c"], "struct vop_vector erofs_fifoops", "vop" -) -base_vfs_slots = initializer(base["super.c"], "static struct vfsops erofs_vfsops", "vfs") -vfs_slots = initializer(current["super.c"], "static struct vfsops erofs_vfsops", "vfs") -expected_vfs_slots = [ - (slot, "erofs_vget" if slot == "vfs_vget" else target) - for slot, target in base_vfs_slots -] -if vnode_slots != base_vnode_slots or fifo_slots != base_fifo_slots: - raise SystemExit("FreeBSD KOBJ VOP slot order or targets changed") -if vfs_slots != expected_vfs_slots: - raise SystemExit("VFS slots changed beyond direct erofs_vget registration") -if ("vop_lookup", "vfs_cache_lookup") not in vnode_slots or ( - "vop_cachedlookup", "erofs_lookup" -) not in vnode_slots: - raise SystemExit("FreeBSD namecache slots were not preserved") -if "erofs_vgetf" in current["super.c"]: - raise SystemExit("trivial erofs_vgetf wrapper remains") - -vget = extract_function(current["erofs_vnops.c"], "erofs_vget") -require_order( - vget, - [ - "td = curthread;", - "nid = (uint64_t)ino;", - "shared = (flags & LK_TYPE_MASK) == LK_SHARED;", - "hash = erofs_vfs_hash(nid);", - "error = vfs_hash_get(", - "if (error != 0 || *vpp != NULL)", - "sbi = MTOE(mp);", - "vi = malloc(sizeof(*vi), M_EROFS, M_WAITOK | M_ZERO);", - "error = getnewvnode(\"erofs\", mp, &erofs_vnodeops, &vp);", - "vp->v_data = vi;", - "vi->nid = nid;", - "lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL);", - "error = insmntque(vp, mp);", - "error = vfs_hash_insert(", - "if (error != 0 || *vpp != NULL)", - "error = erofs_read_inode(sbi, nid, vi);", - "vgone(vp);", - "vput(vp);", - "erofs_fill_vnode(sbi, vp, vi);", - "vn_set_state(vp, VSTATE_CONSTRUCTED);", - "if (shared)", - "VOP_LOCK(vp, LK_DOWNGRADE);", - "*vpp = vp;", - ], - "erofs_vget lifecycle", -) -if vget.count("erofs_vfs_hash_cmp") != 2: - raise SystemExit("vget no longer uses one comparator at both hash callsites") -if vget.count("free(vi, M_EROFS);") != 2 or vget.count("*vpp = NULL;") != 3: - raise SystemExit("vget allocation/insmntque/read failure cleanup changed") -if vget.count("vgone(vp);") != 1 or vget.count("vput(vp);") != 1: - raise SystemExit("vget failed-inode vnode cleanup changed") -if re.search(r"return\s*\(\s*-", vget): - raise SystemExit("negative errno entered the FreeBSD vget path") - -reclaim = extract_function(current["erofs_vnops.c"], "erofs_reclaim") -if reclaim != extract_function(base["erofs_vnops.c"], "erofs_reclaim"): - raise SystemExit("reclaim/hash removal semantics changed") -require_order( - reclaim, - ["vi = VTOE(vp);", "vfs_hash_remove(vp);", "free(vi, M_EROFS);", "vp->v_data = NULL;"], - "erofs_reclaim lifecycle", -) - -open_body = extract_function(current["erofs_vnops.c"], "erofs_open") -if open_body.count("vnode_create_vobject(vp, vi->size, ap->a_td);") != 1: - raise SystemExit("regular-vnode pager setup is not a single direct call") -if "ENOMEM" in open_body or "vnode_create_vobject" not in open_body: - raise SystemExit("dead pager errno folding remains") - -slot_report = { - "status": "PASS", - "vnode": [{"slot": slot, "target": target} for slot, target in vnode_slots], - "fifo": [{"slot": slot, "target": target} for slot, target in fifo_slots], - "vfs": [{"slot": slot, "target": target} for slot, target in vfs_slots], -} -(artifacts / "B09-vop-slots.json").write_text( - json.dumps(slot_report, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", -) - -callgraph = [ - "VFS root -> erofs_vget", - "namecache cachedlookup -> erofs_lookup -> erofs_vget", - "VFS vget slot -> erofs_vget", - "erofs_vget -> vfs_hash_get[erofs_vfs_hash_cmp]", - "erofs_vget -> getnewvnode -> LK_EXCLUSIVE -> insmntque", - "erofs_vget -> vfs_hash_insert[erofs_vfs_hash_cmp]", - "erofs_vget -> erofs_read_inode -> erofs_fill_vnode", - "erofs_vget -> VSTATE_CONSTRUCTED -> optional LK_DOWNGRADE", - "erofs_reclaim -> vfs_hash_remove -> free inode", -] -(artifacts / "B09-vget-callgraph.txt").write_text( - "\n".join(callgraph) + "\n", encoding="ascii" -) - -result = { - "status": "PASS", - "final_ids": ["P15-008", "P15-050", "P15-088"], - "baseline": baseline, - "moved_functions": list(moved_functions), - "moved_functions_byte_identical": True, - "vnode_slots": len(vnode_slots), - "fifo_slots": len(fifo_slots), - "vfs_slots": len(vfs_slots), - "hash_comparator_callsites": 2, - "exclusive_construct_lock": True, - "shared_result_downgrade": True, - "insmntque_cleanup_preserved": True, - "reclaim_hash_removal_preserved": True, - "namecache_slots_preserved": True, - "positive_errno_preserved": True, - "b11_b15_scope": "NOT_INCLUDED", - "qemu": "NOT_RUN", - "full_feature_suite": "NOT_RUN", -} -(artifacts / "B09-result.json").write_text( - json.dumps(result, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", -) -print( - f"B09 PASS moved={len(moved_functions)} " - f"slots={len(vnode_slots)}+{len(fifo_slots)}/{len(vfs_slots)} hash=2" -) -PY -then - : -else - pre15_dut_fail 'B09 vnode ownership, VOP slots, or vget graph check failed' -fi - -sha256sum "$artifacts/B09-vop-slots.json" \ - "$artifacts/B09-vget-callgraph.txt" "$artifacts/B09-result.json" \ - > "$artifacts/SHA256SUMS" -printf 'B09 PASS vnode adapter ownership and FreeBSD lifecycle preserved\n' diff --git a/tests/pre15/cases/B10-directory.sh b/tests/pre15/cases/B10-directory.sh deleted file mode 100755 index 55cb369..0000000 --- a/tests/pre15/cases/B10-directory.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=22965aea267c3fcaafb67c14f9a246a7bc6195b1 -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -spec=$fixture_dir/B10-directory-spec.json -oracle=$fixture_dir/B10-directory-oracle.py -artifacts=$PRE15_RUN_DIR/artifacts -report=$artifacts/B10-directory-results.json - -for tool in git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B10 host tool: $tool" -done - -pre15_record_fixture b10-spec "$spec" -pre15_record_fixture b10-oracle "$oracle" -pre15_record_fixture b10-case "$PRE15_DUT/tests/pre15/cases/B10-directory.sh" -pre15_record_fixture b10-dir "$PRE15_DUT/src/dir.c" -pre15_record_fixture b10-namei "$PRE15_DUT/src/namei.c" -mkdir -p "$artifacts" -pre15_target_reached - -if python3 -B "$oracle" --root "$PRE15_ROOT" --dut "$PRE15_DUT" \ - --baseline "$baseline" --spec "$spec" --report "$report" -then - pre15_record_fixture b10-report "$report" -else - pre15_dut_fail 'B10 directory trust-boundary oracle failed' -fi diff --git a/tests/pre15/cases/B11-dtype.sh b/tests/pre15/cases/B11-dtype.sh deleted file mode 100755 index 8c33a86..0000000 --- a/tests/pre15/cases/B11-dtype.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=e2e3fb86b6fffcb01d6fd29c17dd95628ad070de -gate=$PRE15_DUT/tests/pre15/gates/P15-081.sh -input=$PRE15_DUT/tests/pre15/gates/P15-081-input.json -spec=$PRE15_DUT/tests/pre15/fixtures/B11-dtype-spec.json -oracle=$PRE15_DUT/tests/pre15/fixtures/B11-dtype-oracle.py -artifacts=$PRE15_RUN_DIR/artifacts -gate_output=$artifacts/gate - -for tool in cc dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B11 host tool: $tool" -done - -pre15_record_fixture b11-gate "$gate" -pre15_record_fixture b11-gate-input "$input" -pre15_record_fixture b11-spec "$spec" -pre15_record_fixture b11-oracle "$oracle" -for source in inode.c internal.h namei.c dir.c erofs_vnops.c; do - pre15_record_fixture "b11-$source" "$PRE15_DUT/src/$source" -done - -mkdir -p "$artifacts" -if "$gate" --base "$baseline" --output "$gate_output" \ - >"$artifacts/gate.stdout" 2>"$artifacts/gate.stderr"; then - : -else - pre15_runner_fail 'P15-081 frozen gate replay failed' -fi - -pre15_target_reached -if python3 -B "$oracle" --root "$PRE15_ROOT" --dut "$PRE15_DUT" \ - --spec "$spec" --gate-output "$gate_output" --artifacts "$artifacts" -then - pre15_record_fixture b11-report "$artifacts/B11-dtype-report.json" -else - pre15_dut_fail 'B11 dtype source or fixture oracle failed' -fi diff --git a/tests/pre15/cases/B12-readahead.sh b/tests/pre15/cases/B12-readahead.sh deleted file mode 100755 index e9a85a6..0000000 --- a/tests/pre15/cases/B12-readahead.sh +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -policy=$fixture_dir/B12-readahead-policy.json -kld_builder=$fixture_dir/B12-build-kld.sh -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in awk clang file git nm python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B12 host tool: $tool" -done - -pre15_record_fixture b12-policy "$policy" -pre15_record_fixture b12-kld-builder "$kld_builder" -pre15_record_fixture b12-case "$PRE15_DUT/tests/pre15/cases/B12-readahead.sh" -for source in data.c dir.c internal.h; do - pre15_record_fixture "b12-$source" "$PRE15_DUT/src/$source" -done -mkdir -p "$artifacts" -pre15_target_reached - -if python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$policy" \ - "$artifacts/B12-policy-report.json" <<'PY' -from __future__ import annotations - -import csv -import hashlib -import json -from pathlib import Path -import re -import statistics -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -policy_path = Path(sys.argv[3]) -report_path = Path(sys.argv[4]) -policy = json.loads(policy_path.read_text(encoding="ascii")) -if policy.get("schema") != 1 or policy.get("candidate") != "P15-087": - raise SystemExit("invalid B12 policy identity") - - -def digest(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -source_hashes = { - relative: digest(dut / relative) - for relative in policy["source_sha256"] -} -if source_hashes != policy["source_sha256"]: - raise SystemExit("B12 production source differs from the accepted source set") - -changed = subprocess.run( - [ - "git", "-C", str(root), "diff", "--name-only", - policy["baseline"], "--", "repo-pre-15/src", - ], - check=True, - text=True, - stdout=subprocess.PIPE, -).stdout.splitlines() -expected_changed = [ - "repo-pre-15/src/data.c", - "repo-pre-15/src/dir.c", - "repo-pre-15/src/internal.h", -] -if sorted(changed) != expected_changed: - raise SystemExit(f"B12 production write set mismatch: {changed}") - -data = (dut / "src/data.c").read_text(encoding="utf-8") -directory = (dut / "src/dir.c").read_text(encoding="utf-8") -internal = (dut / "src/internal.h").read_text(encoding="utf-8") -if "cluster_read(" in data + directory + internal: - raise SystemExit("B12 must not introduce cluster_read") -if data.count("breadn(") != 1: - raise SystemExit("B12 must contain one backing-vnode breadn call") -if not re.search(r"breadn\(dif->devvp,", data): - raise SystemExit("B12 breadn is not issued on erofs_device_info.devvp") -required_data = ( - "#define EROFS_DIR_READAHEAD_BYTES\t(1024 * 1024)", - "#define EROFS_DIR_READAHEAD_SLOTS\t(EROFS_DIR_READAHEAD_BYTES / PAGE_SIZE)", - "future.m_dif != current.m_dif", - "future.m_pa != current.m_pa + step", - "vi->datalayout == EROFS_INODE_FLAT_PLAIN", - "return (erofs_read_data_impl(sbi, vi, loff, len, 0, bufp));", -) -for marker in required_data: - if marker not in data: - raise SystemExit(f"missing B12 data policy marker: {marker}") -if directory.count("erofs_read_data_readahead(") != 1: - raise SystemExit("B12 directory hint is not limited to readdir") -if "sequential = logical_off == 0;" not in directory: - raise SystemExit("B12 random seek no-op predicate is absent") -if internal.count("erofs_read_data_readahead(") != 1: - raise SystemExit("B12 internal prototype is absent or duplicated") - -evidence = root / policy["gate_evidence"] -result_path = evidence / "result.json" -rows_path = evidence / "runs.tsv" -if digest(result_path) != policy["gate_result_sha256"]: - raise SystemExit("B12 gate result hash mismatch") -if digest(rows_path) != policy["gate_rows_sha256"]: - raise SystemExit("B12 gate rows hash mismatch") -result = json.loads(result_path.read_text(encoding="ascii")) -if result.get("status") != "GO" or result.get("b12") != "AUTHORIZED": - raise SystemExit("P15-087 did not authorize B12") -if result.get("qemu") != "PASS" or not all(result.get("checks", {}).values()): - raise SystemExit("P15-087 runtime or semantic checks are incomplete") -thresholds = result.get("thresholds", {}) -if thresholds != { - "maximum_extra_provider_reads_percent": policy["maximum_extra_provider_reads_percent"], - "minimum_cold_median_improvement_percent": policy["minimum_cold_median_improvement_percent"], -}: - raise SystemExit("P15-087 thresholds differ from the frozen B12 policy") - -rows = [] -with rows_path.open(encoding="ascii", newline="") as source: - for raw in csv.reader(source, delimiter="\t"): - if len(raw) != 10: - raise SystemExit(f"invalid B12 gate row: {raw}") - rows.append({ - "variant": raw[0], - "run": raw[1], - "mode": raw[3], - "elapsed_ns": int(raw[4]), - "provider_reads": int(raw[5]), - "entry_count": int(raw[7]), - "hash": raw[8], - "final_cookie": int(raw[9]), - }) -sequential = [row for row in rows if row["mode"] == "sequential"] -random_rows = [row for row in rows if row["mode"] == "random"] -baseline = [row for row in sequential if row["variant"] == "baseline"] -candidate = [row for row in sequential if row["variant"] == "candidate"] -if len(baseline) != 5 or len(candidate) != 5 or len(random_rows) != 2: - raise SystemExit("B12 gate does not contain the complete 5+5 and 1+1 sample set") -baseline_ns = statistics.median(row["elapsed_ns"] for row in baseline) -candidate_ns = statistics.median(row["elapsed_ns"] for row in candidate) -improvement = (baseline_ns - candidate_ns) * 100.0 / baseline_ns -baseline_reads = statistics.median(row["provider_reads"] for row in baseline) -candidate_reads = statistics.median(row["provider_reads"] for row in candidate) -extra_reads = (candidate_reads - baseline_reads) * 100.0 / baseline_reads -if improvement < policy["minimum_cold_median_improvement_percent"]: - raise SystemExit("B12 recomputed cold median improvement misses the gate") -if extra_reads > policy["maximum_extra_provider_reads_percent"]: - raise SystemExit("B12 recomputed provider-read delta misses the gate") -if any( - (row["entry_count"], row["hash"], row["final_cookie"]) - != (18002, "427bb414efa99dc0", 3880368) - for row in sequential -): - raise SystemExit("B12 sequential correctness rows differ from the oracle") -if { - (row["provider_reads"], row["entry_count"], row["hash"], row["final_cookie"]) - for row in random_rows -} != {(19, 292, "9805eadece9f500e", 1998932)}: - raise SystemExit("B12 random path is not an exact no-op pair") - -report = { - "baseline_samples": len(baseline), - "candidate_samples": len(candidate), - "baseline_median_elapsed_ns": baseline_ns, - "candidate_median_elapsed_ns": candidate_ns, - "cold_median_improvement_percent": improvement, - "extra_provider_reads_percent": extra_reads, - "gate": "GO", - "qemu_runtime": "PASS", - "random_samples": len(random_rows), - "source_sha256": source_hashes, - "status": "PASS", - "window_bytes": policy["maximum_readahead_bytes"], -} -report_path.write_text( - json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii" -) -PY -then - : -else - pre15_dut_fail 'B12 source, write-set, or frozen G11 policy replay failed' -fi -pre15_record_fixture b12-policy-report "$artifacts/B12-policy-report.json" - -module=$PRE15_CASE_TMP/B12-erofs-zstdio0.ko -if ! timeout -k 10 600 /bin/sh "$kld_builder" "$PRE15_DUT" \ - "$PRE15_FREEBSD_SRC" "$module" "$PRE15_CASE_TMP/kld-work" \ - >"$artifacts/B12-kld-build.stdout" 2>"$artifacts/B12-kld-build.stderr"; then - pre15_dut_fail 'B12 exact-ABI zstdio0 KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/B12-kld-file.txt" -sha256sum "$module" >"$artifacts/B12-kld-sha256.txt" -nm -g "$module" | LC_ALL=C sort >"$artifacts/B12-kld-nm-global.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/B12-kld-nm-u.txt" -if ! awk '$NF == "erofs_read_data_readahead" { found = 1 } END { exit !found }' \ - "$artifacts/B12-kld-nm-global.txt"; then - pre15_dut_fail 'B12 exact-ABI KLD lacks the directory readahead symbol' -fi -if grep -q ' ZSTD_' "$artifacts/B12-kld-nm-u.txt"; then - pre15_dut_fail 'B12 zstdio0 KLD contains an unexpected Zstd symbol' -fi - -printf '%s\n' \ - 'B12-policy PASS: exact source/write set and frozen 5+5 G11 statistics' \ - 'K zstdio0 PASS: FreeBSD 15 exact-ABI cross-KLD and symbol contract' \ - 'TC183 correctness/performance PASS from immutable P15-087 QEMU evidence' diff --git a/tests/pre15/cases/B13-time.sh b/tests/pre15/cases/B13-time.sh deleted file mode 100755 index 04e60eb..0000000 --- a/tests/pre15/cases/B13-time.sh +++ /dev/null @@ -1,604 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=08a567554881f19bcec933a92a46d18cfb73d22e -artifacts=$PRE15_RUN_DIR/artifacts -compact_fixture=$PRE15_DUT/tests/pre15/fixtures/B13-compact-time.json -extended_fixture=$PRE15_DUT/tests/pre15/fixtures/B13-extended-time.json -inode=$PRE15_DUT/src/inode.c -internal=$PRE15_DUT/src/internal.h -super=$PRE15_DUT/src/super.c -ondisk=$PRE15_DUT/src/erofs_fs.h -vnops=$PRE15_DUT/src/erofs_vnops.c - -for tool in cc git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B13 host tool: $tool" -done - -pre15_record_fixture b13-compact "$compact_fixture" -pre15_record_fixture b13-extended "$extended_fixture" -pre15_record_fixture b13-inode "$inode" -pre15_record_fixture b13-internal "$internal" -pre15_record_fixture b13-super "$super" -pre15_record_fixture b13-ondisk "$ondisk" -pre15_record_fixture b13-vnops "$vnops" -pre15_record_fixture b13-case "$PRE15_DUT/tests/pre15/cases/B13-time.sh" - -mkdir -p "$artifacts" -pre15_target_reached - -if python3 - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" "$compact_fixture" \ - "$extended_fixture" "$artifacts" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -compact_path = Path(sys.argv[4]) -extended_path = Path(sys.argv[5]) -artifacts = Path(sys.argv[6]) -src = dut / "src" -eintegrity = 97 -int64_min = -(1 << 63) -int64_max = (1 << 63) - 1 -int32_min = -(1 << 31) -int32_max = (1 << 31) - 1 - - -def committed(path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/src/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B13 baseline {path}: {completed.stderr}") - return completed.stdout - - -def replace_once(source: str, old: str, new: str, label: str) -> str: - count = source.count(old) - if count != 1: - raise SystemExit(f"{label}: expected one transform source, found {count}") - return source.replace(old, new, 1) - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - if brace < 0: - raise SystemExit(f"missing function body: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def load_fixture(path: Path, expected_encoding: set[str]) -> dict: - fixture = json.loads(path.read_text(encoding="ascii")) - if fixture.get("schema") != 1 or fixture.get("candidate") != "P15-018": - raise SystemExit(f"invalid B13 fixture identity: {path.name}") - if set(fixture.get("encoding", {})) != expected_encoding: - raise SystemExit(f"invalid B13 fixture encoding keys: {path.name}") - if not isinstance(fixture.get("cases"), list) or not fixture["cases"]: - raise SystemExit(f"empty B13 fixture: {path.name}") - return fixture - - -def decode_le(raw: str, width: int, *, signed: bool) -> int: - if not re.fullmatch(r"[0-9a-f]+", raw) or len(raw) != width * 2: - raise SystemExit(f"invalid {width}-byte lowercase hex value: {raw!r}") - return int.from_bytes(bytes.fromhex(raw), byteorder="little", signed=signed) - - -def status_for_range(value: int, minimum: int, maximum: int) -> str: - return "PASS" if minimum <= value <= maximum else "EINTEGRITY" - - -current = { - name: (src / name).read_text(encoding="utf-8") - for name in ("internal.h", "inode.c", "super.c", "erofs_fs.h", "erofs_vnops.c") -} -base = {name: committed(name) for name in current} - -expected_internal = replace_once( - base["internal.h"], "\tuint64_t epoch;", "\tint64_t epoch;", "signed epoch field" -) -expected_internal = replace_once( - expected_internal, "\tuint64_t mtime;", "\ttime_t mtime;", "time_t inode field" -) -if current["internal.h"] != expected_internal: - raise SystemExit("internal.h differs from the two declared B13 type changes") - -old_set_timestamp = """static int -erofs_set_timestamp(struct erofs_inode *vi, uint64_t seconds, - uint32_t nanoseconds) -{ -\tif (nanoseconds >= 1000000000 || seconds > (uint64_t)INT64_MAX) -\t\treturn (EINTEGRITY); -\tvi->mtime = seconds; -\tvi->mtime_nsec = nanoseconds; -\treturn (0); -} -""" -new_set_timestamp = """static int -erofs_set_timestamp(struct erofs_inode *vi, int64_t seconds, - uint32_t nanoseconds) -{ -\ttime_t mtime; - -\tif (nanoseconds >= 1000000000 || -\t __builtin_add_overflow(seconds, 0, &mtime)) -\t\treturn (EINTEGRITY); -\tvi->mtime = mtime; -\tvi->mtime_nsec = nanoseconds; -\treturn (0); -} -""" -expected_inode = replace_once( - base["inode.c"], old_set_timestamp, new_set_timestamp, "timestamp conversion" -) -expected_inode = replace_once( - expected_inode, - "\tuint64_t addrmask, mtime;", - "\tuint64_t addrmask;\n\tint64_t mtime;", - "signed decoded mtime", -) -expected_inode = replace_once( - expected_inode, - "\t\t (uint64_t)le32toh(dic->i_mtime), &mtime)", - "\t\t (int64_t)le32toh(dic->i_mtime), &mtime)", - "compact signed checked add", -) -expected_inode = replace_once( - expected_inode, - "\t\terror = erofs_set_timestamp(vi, le64toh(die->i_mtime),\n" - "\t\t le32toh(die->i_mtime_nsec));", - "\t\terror = erofs_set_timestamp(vi,\n" - "\t\t (int64_t)le64toh(die->i_mtime),\n" - "\t\t le32toh(die->i_mtime_nsec));", - "extended signed decode", -) -if current["inode.c"] != expected_inode: - raise SystemExit("inode.c differs from the four declared B13 transforms") - -expected_super = replace_once( - base["super.c"], - "\tsbi->epoch = le64toh(dsb->epoch);", - "\tsbi->epoch = (int64_t)le64toh(dsb->epoch);", - "super signed epoch decode", -) -if current["super.c"] != expected_super: - raise SystemExit("super.c differs from the exact signed epoch transform") -if current["erofs_fs.h"] != base["erofs_fs.h"]: - raise SystemExit("B13 changed ondisk endian types, widths, or layout") -if current["erofs_vnops.c"] != base["erofs_vnops.c"]: - raise SystemExit("B13 changed FreeBSD VOP/timespec publication code") - -ondisk_required = ( - "\t__le64 epoch;", - "\t__le32 i_mtime;", - "\t__le64 i_mtime;", - "\t__le32 i_mtime_nsec;", -) -for declaration in ondisk_required: - if current["erofs_fs.h"].count(declaration) != 1: - raise SystemExit(f"ondisk timestamp declaration changed: {declaration}") - -timestamp_helper = extract_function(current["inode.c"], "erofs_set_timestamp") -read_inode = extract_function(current["inode.c"], "erofs_read_inode") -if timestamp_helper.count("__builtin_add_overflow(seconds, 0, &mtime)") != 1: - raise SystemExit("time_t checked conversion is absent or duplicated") -compact_pattern = re.compile( - r"__builtin_add_overflow\(sbi->epoch,\s*" - r"\(int64_t\)le32toh\(dic->i_mtime\), &mtime\)" -) -if len(compact_pattern.findall(read_inode)) != 1: - raise SystemExit("compact signed checked addition is absent or duplicated") -if read_inode.count("(int64_t)le64toh(die->i_mtime)") != 1: - raise SystemExit("extended signed seconds decode is absent or duplicated") -if "(uint64_t)le32toh(dic->i_mtime)" in read_inode: - raise SystemExit("unsigned compact timestamp arithmetic remains") - -changed_sources = current["internal.h"] + current["inode.c"] + current["super.c"] -if "ckd_add" in changed_sources or "stdckdint" in changed_sources: - raise SystemExit("unsupported checked-arithmetic API entered B13") -if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", changed_sources): - raise SystemExit("negative errno entered the FreeBSD timestamp path") - -overflow_lines = [] -for path in ("inode.c", "super.c"): - for line_number, line in enumerate(current[path].splitlines(), start=1): - if "__builtin_" in line and "overflow" in line: - overflow_lines.append(f"{path}:{line_number}:{line.strip()}") -(artifacts / "B13-overflow-scan.txt").write_text( - "\n".join(overflow_lines) + "\n", encoding="ascii" -) - -compact = load_fixture( - compact_path, {"epoch", "fixed_nsec", "mtime_delta"} -) -extended = load_fixture(extended_path, {"mtime", "mtime_nsec"}) -seen_ids: set[str] = set() -decoded_compact = [] -decoded_extended = [] - -for case in compact["cases"]: - case_id = case.get("id") - if not isinstance(case_id, str) or case_id in seen_ids: - raise SystemExit(f"invalid or duplicate B13 case id: {case_id!r}") - seen_ids.add(case_id) - epoch = decode_le(case["epoch_le"], 8, signed=True) - delta = decode_le(case["mtime_delta_le"], 4, signed=False) - nanoseconds = decode_le(case["fixed_nsec_le"], 4, signed=False) - mathematical = epoch + delta - add_status = status_for_range(mathematical, int64_min, int64_max) - if add_status != case["expected_add"]: - raise SystemExit(f"compact add oracle mismatch: {case_id}") - if add_status == "PASS": - if mathematical != case["expected_seconds"]: - raise SystemExit(f"compact seconds oracle mismatch: {case_id}") - if status_for_range(mathematical, int64_min, int64_max) != case["expected_time64"]: - raise SystemExit(f"compact time64 oracle mismatch: {case_id}") - if status_for_range(mathematical, int32_min, int32_max) != case["expected_time32"]: - raise SystemExit(f"compact time32 oracle mismatch: {case_id}") - nsec_status = "PASS" if nanoseconds < 1_000_000_000 else "EINTEGRITY" - if nsec_status != case["expected_nsec"]: - raise SystemExit(f"compact nsec oracle mismatch: {case_id}") - elif any(case[key] != "NOT_REACHED" for key in ( - "expected_time32", "expected_time64", "expected_nsec" - )) or case["expected_seconds"] is not None: - raise SystemExit(f"compact overflow must stop before conversion: {case_id}") - decoded_compact.append( - { - "id": case_id, - "epoch": epoch, - "delta": delta, - "nanoseconds": nanoseconds, - "mathematical_seconds": mathematical, - "add_status": add_status, - } - ) - -for case in extended["cases"]: - case_id = case.get("id") - if not isinstance(case_id, str) or case_id in seen_ids: - raise SystemExit(f"invalid or duplicate B13 case id: {case_id!r}") - seen_ids.add(case_id) - seconds = decode_le(case["mtime_le"], 8, signed=True) - nanoseconds = decode_le(case["mtime_nsec_le"], 4, signed=False) - if seconds != case["expected_seconds"]: - raise SystemExit(f"extended signed decode mismatch: {case_id}") - if status_for_range(seconds, int64_min, int64_max) != case["expected_time64"]: - raise SystemExit(f"extended time64 oracle mismatch: {case_id}") - if status_for_range(seconds, int32_min, int32_max) != case["expected_time32"]: - raise SystemExit(f"extended time32 oracle mismatch: {case_id}") - nsec_status = "PASS" if nanoseconds < 1_000_000_000 else "EINTEGRITY" - if nsec_status != case["expected_nsec"]: - raise SystemExit(f"extended nsec oracle mismatch: {case_id}") - decoded_extended.append( - {"id": case_id, "seconds": seconds, "nanoseconds": nanoseconds} - ) - -fixture_result = { - "status": "PASS", - "compact": decoded_compact, - "extended": decoded_extended, -} -(artifacts / "B13-decoded-fixtures.json").write_text( - json.dumps(fixture_result, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", -) - - -def c_i64(value: int) -> str: - if value == int64_min: - return "INT64_MIN" - if value == int64_max: - return "INT64_MAX" - if value < 0: - return f"(-INT64_C({-value}))" - return f"INT64_C({value})" - - -def c_status(value: str) -> str: - if value == "PASS": - return "0" - if value == "EINTEGRITY": - return "EINTEGRITY" - raise SystemExit(f"cannot emit non-terminal status: {value}") - - -program = f'''#include -#include -#include -#include - -#define EINTEGRITY {eintegrity} - -struct erofs_inode {{ -\ttime_t mtime; -\tuint32_t mtime_nsec; -}}; - -{timestamp_helper} - -static int -checked_compact(int64_t epoch, uint32_t delta, int64_t *seconds) -{{ -\tif (__builtin_add_overflow(epoch, (int64_t)delta, seconds)) -\t\treturn (EINTEGRITY); -\treturn (0); -}} - -static int -checked_time32(int64_t seconds, int32_t *result) -{{ -\treturn (__builtin_add_overflow(seconds, 0, result) ? EINTEGRITY : 0); -}} - -static int -checked_time64(int64_t seconds, int64_t *result) -{{ -\treturn (__builtin_add_overflow(seconds, 0, result) ? EINTEGRITY : 0); -}} - -static int -check_timestamp(const char *id, int64_t seconds, uint32_t nanoseconds, - int expected_time32, int expected_time64, int expected_nsec) -{{ -\tstruct erofs_inode inode = {{ 0 }}; -\tint32_t time32; -\tint64_t time64; -\tint error, expected; - -\tif (checked_time32(seconds, &time32) != expected_time32 || -\t checked_time64(seconds, &time64) != expected_time64) {{ -\t\tfprintf(stderr, "%s: representability\\n", id); -\t\treturn (1); -\t}} -\texpected = expected_nsec != 0 ? expected_nsec : -\t (sizeof(time_t) == sizeof(int32_t) ? expected_time32 : expected_time64); -\terror = erofs_set_timestamp(&inode, seconds, nanoseconds); -\tif (error != expected) {{ -\t\tfprintf(stderr, "%s: timestamp errno %d != %d\\n", id, error, expected); -\t\treturn (1); -\t}} -\tif (error == 0 && ((int64_t)inode.mtime != seconds || -\t inode.mtime_nsec != nanoseconds)) {{ -\t\tfprintf(stderr, "%s: timestamp value\\n", id); -\t\treturn (1); -\t}} -\treturn (0); -}} - -int -main(void) -{{ -\tint64_t seconds; -\tint error; - -\tif (sizeof(time_t) != sizeof(int32_t) && sizeof(time_t) != sizeof(int64_t)) -\t\treturn (2); -''' -for case, decoded in zip(compact["cases"], decoded_compact, strict=True): - epoch = decoded["epoch"] - delta = decoded["delta"] - expected_add = c_status(case["expected_add"]) - program += f'''\terror = checked_compact({c_i64(epoch)}, UINT32_C({delta}), &seconds); -\tif (error != {expected_add}) {{ -\t\tfprintf(stderr, "{case['id']}: compact add errno\\n"); -\t\treturn (1); -\t}} -''' - if case["expected_add"] == "PASS": - program += f'''\tif (seconds != {c_i64(case['expected_seconds'])} || -\t check_timestamp("{case['id']}", seconds, UINT32_C({decoded['nanoseconds']}), -\t {c_status(case['expected_time32'])}, {c_status(case['expected_time64'])}, -\t {c_status(case['expected_nsec'])}) != 0) -\t\treturn (1); -''' -for case, decoded in zip(extended["cases"], decoded_extended, strict=True): - program += f'''\tif (check_timestamp("{case['id']}", {c_i64(decoded['seconds'])}, -\t UINT32_C({decoded['nanoseconds']}), {c_status(case['expected_time32'])}, -\t {c_status(case['expected_time64'])}, {c_status(case['expected_nsec'])}) != 0) -\t\treturn (1); -''' -program += f'''\tprintf("B13 boundary PASS compact={len(compact['cases'])} extended={len(extended['cases'])} time_t=%zu\\n", -\t sizeof(time_t) * 8); -\treturn (0); -}} -''' - -program_path = artifacts / "B13-time-boundary.c" -binary_path = artifacts / "B13-time-boundary" -program_path.write_text(program, encoding="ascii") -compiled = subprocess.run( - [ - "cc", "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", - str(program_path), "-o", str(binary_path), - ], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -(artifacts / "B13-time-boundary.compile.stdout").write_text( - compiled.stdout, encoding="utf-8" -) -(artifacts / "B13-time-boundary.compile.stderr").write_text( - compiled.stderr, encoding="utf-8" -) -if compiled.returncode != 0: - raise SystemExit("B13 boundary extractor compilation failed") -executed = subprocess.run( - [str(binary_path)], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -(artifacts / "B13-time-boundary.stdout").write_text( - executed.stdout, encoding="utf-8" -) -(artifacts / "B13-time-boundary.stderr").write_text( - executed.stderr, encoding="utf-8" -) -if executed.returncode != 0: - raise SystemExit("B13 boundary extractor execution failed") - -program32 = program.replace("#include \n", "#define time_t int32_t\n", 1) -program32_path = artifacts / "B13-time-boundary32.c" -binary32_path = artifacts / "B13-time-boundary32" -program32_path.write_text(program32, encoding="ascii") -compiled32 = subprocess.run( - [ - "cc", "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", - str(program32_path), "-o", str(binary32_path), - ], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -(artifacts / "B13-time-boundary32.compile.stdout").write_text( - compiled32.stdout, encoding="utf-8" -) -(artifacts / "B13-time-boundary32.compile.stderr").write_text( - compiled32.stderr, encoding="utf-8" -) -if compiled32.returncode != 0: - raise SystemExit("B13 32-bit time_t extractor compilation failed") -executed32 = subprocess.run( - [str(binary32_path)], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -(artifacts / "B13-time-boundary32.stdout").write_text( - executed32.stdout, encoding="utf-8" -) -(artifacts / "B13-time-boundary32.stderr").write_text( - executed32.stderr, encoding="utf-8" -) -if executed32.returncode != 0: - raise SystemExit("B13 32-bit time_t extractor execution failed") - -result = { - "status": "PASS", - "final_id": "P15-018", - "baseline": baseline, - "compact_cases": len(compact["cases"]), - "extended_cases": len(extended["cases"]), - "signed_add_overflows": sum( - case["expected_add"] == "EINTEGRITY" for case in compact["cases"] - ), - "negative_seconds_passes": sum( - case["expected_seconds"] is not None and case["expected_seconds"] < 0 - for case in compact["cases"] + extended["cases"] - ), - "time32_lower_rejections": sum( - case["expected_seconds"] is not None - and case["expected_seconds"] < int32_min - and case["expected_time32"] == "EINTEGRITY" - for case in compact["cases"] + extended["cases"] - ), - "time32_upper_rejections": sum( - case["expected_seconds"] is not None - and case["expected_seconds"] > int32_max - and case["expected_time32"] == "EINTEGRITY" - for case in compact["cases"] + extended["cases"] - ), - "nanosecond_rejections": sum( - case["expected_nsec"] == "EINTEGRITY" - for case in compact["cases"] + extended["cases"] - ), - "ondisk_header_byte_identical": True, - "ondisk_seconds_width": 64, - "ondisk_compact_delta_width": 32, - "time_t_checked_conversion": True, - "extracted_helper_time_widths": [32, 64], - "unsupported_ckd_api": False, - "positive_errno_preserved": True, - "qemu": "NOT_RUN", - "full_feature_suite": "NOT_RUN", -} -(artifacts / "B13-result.json").write_text( - json.dumps(result, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", -) -print( - f"B13 PASS compact={len(compact['cases'])} extended={len(extended['cases'])} " - f"signed-overflow={result['signed_add_overflows']}" -) -PY -then - : -else - pre15_dut_fail 'B13 signed timestamp source or boundary check failed' -fi - -sha256sum "$artifacts/B13-overflow-scan.txt" \ - "$artifacts/B13-decoded-fixtures.json" \ - "$artifacts/B13-time-boundary.c" "$artifacts/B13-time-boundary" \ - "$artifacts/B13-time-boundary32.c" "$artifacts/B13-time-boundary32" \ - "$artifacts/B13-result.json" > "$artifacts/SHA256SUMS" -printf 'B13 PASS signed timestamps and checked time_t boundaries\n' diff --git a/tests/pre15/cases/B14-chunk-types.sh b/tests/pre15/cases/B14-chunk-types.sh deleted file mode 100755 index bed1bf7..0000000 --- a/tests/pre15/cases/B14-chunk-types.sh +++ /dev/null @@ -1,226 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=bf72cdffbc2b58c2b6468e20391eea14096b5a83 -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -generator=$fixture_dir/B14-chunk-fixtures.py -kld_builder=$fixture_dir/B14-build-kld.sh -probe=$fixture_dir/B14-chunk-probe.c -spec=$fixture_dir/B14-chunk-spec.json -artifacts=$PRE15_RUN_DIR/artifacts -first=$PRE15_CASE_TMP/first -second=$PRE15_CASE_TMP/second - -for tool in cmp fsck.erofs git mkfs.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B14 host tool: $tool" -done - -pre15_record_fixture b14-generator "$generator" -pre15_record_fixture b14-kld-builder "$kld_builder" -pre15_record_fixture b14-probe "$probe" -pre15_record_fixture b14-spec "$spec" -pre15_record_fixture b14-case "$PRE15_DUT/tests/pre15/cases/B14-chunk-types.sh" -pre15_record_fixture b14-inode "$PRE15_DUT/src/inode.c" -pre15_record_fixture b14-data "$PRE15_DUT/src/data.c" -pre15_record_fixture b14-vnops "$PRE15_DUT/src/erofs_vnops.c" -pre15_record_fixture b14-linux-inode "$PRE15_ROOT/src-linux/inode.c" - -mkdir -p "$artifacts" -if ! python3 -B "$generator" generate --output "$first" --spec "$spec" \ - >"$artifacts/generate-first.stdout" 2>"$artifacts/generate-first.stderr"; then - pre15_runner_fail 'B14 first fixture generation failed' -fi -if ! python3 -B "$generator" generate --output "$second" --spec "$spec" \ - >"$artifacts/generate-second.stdout" 2>"$artifacts/generate-second.stderr"; then - pre15_runner_fail 'B14 repeat fixture generation failed' -fi -if ! cmp "$first/manifest.json" "$second/manifest.json" || \ - ! cmp "$first/SHA256SUMS" "$second/SHA256SUMS"; then - pre15_runner_fail 'B14 repeat generation is not byte deterministic' -fi - -for name in B14-slot1.blob B14-good32.erofs B14-good48.erofs \ - B14-bad-index.erofs B14-bad-index48.erofs B14-bad-reserved.erofs \ - B14-bad-format.erofs B14-bad-48-no-index.erofs; do - pre15_record_fixture "b14-$name" "$first/$name" -done -pre15_target_reached - -if ! python3 -B "$generator" audit --root "$PRE15_ROOT" --dut "$PRE15_DUT" \ - --baseline "$baseline" --spec "$spec" --report "$artifacts/B14-source-report.json" \ - >"$artifacts/audit.stdout" 2>"$artifacts/audit.stderr"; then - pre15_dut_fail 'B14 source or Linux/FreeBSD semantic audit failed' -fi -if ! python3 -B "$generator" verify --output "$first" --spec "$spec" \ - >"$artifacts/verify-first.stdout" 2>"$artifacts/verify-first.stderr"; then - pre15_dut_fail 'B14 fixture oracle failed' -fi -if ! python3 -B "$generator" verify --output "$second" --spec "$spec" \ - >"$artifacts/verify-second.stdout" 2>"$artifacts/verify-second.stderr"; then - pre15_runner_fail 'B14 repeat fixture verification failed' -fi - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' 'TC174 host fixture/source oracle PASS' \ - 'QEMU runtime NOT_RUN in host mode' - exit 0 -fi - -for tool in awk cc clang file nm scp tar; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B14 QEMU tool: $tool" -done - -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -fixture_archive=$PRE15_CASE_TMP/B14-fixtures.tar.gz -module=$PRE15_CASE_TMP/B14-erofs.ko -if ! /bin/sh "$kld_builder" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work" >"$artifacts/B14-kld-build.stdout" \ - 2>"$artifacts/B14-kld-build.stderr"; then - pre15_dut_fail 'B14 cross-target KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/B14-kld-file.txt" -sha256sum "$module" >"$artifacts/B14-kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/B14-kld-nm-u.txt" -tar -C "$first" -czf "$fixture_archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -if ! pre15_scp "$fixture_archive" /root/B14-fixtures.tar.gz || \ - ! pre15_scp "$probe" /root/B14-chunk-probe.c || \ - ! pre15_scp "$module" /root/B14-erofs.ko; then - pre15_infra_blocked 'could not transfer B14 module or fixtures to the guest' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/pre15-b14-fixtures && mkdir /root/pre15-b14-fixtures && tar -xzf /root/B14-fixtures.tar.gz -C /root/pre15-b14-fixtures'; then - pre15_infra_blocked 'could not prepare B14 guest directories' -fi -pre15_guest_ssh_bounded sha256 /root/B14-erofs.ko \ - >"$artifacts/B14-guest-module-sha256.txt" -if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then - pre15_infra_blocked 'FreeBSD guest already has an EROFS module loaded' -fi -if ! pre15_guest_ssh_bounded kldload /root/B14-erofs.ko \ - >"$artifacts/B14-kldload.stdout" 2>"$artifacts/B14-kldload.stderr"; then - if grep -q 'module already loaded or in kernel' \ - "$artifacts/B14-kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - pre15_dut_fail 'B14 exact-source KLD failed to load' -fi -pre15_own_guest_kld erofs 'B14 exact-source KLD' -if ! pre15_guest_ssh_bounded cc -Wall -Wextra -Werror \ - -o /root/B14-chunk-probe /root/B14-chunk-probe.c; then - pre15_infra_blocked 'could not compile the B14 guest errno probe' -fi - -pre15_attach_md() -{ - pre15_md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode -f "$1") || \ - pre15_dut_fail "could not attach B14 provider: $1" - case "$pre15_md" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected mdconfig output: $pre15_md" ;; - esac - pre15_md=${pre15_md#md} - pre15_own_guest_md "$pre15_md" "$2" - printf '%s\n' "$pre15_md" -} - -pre15_mount_image() -{ - pre15_image=$1 - pre15_mountpoint=$2 - pre15_label=$3 - pre15_primary=$(pre15_attach_md "/root/pre15-b14-fixtures/$pre15_image" \ - "$pre15_label primary") - pre15_guest_ssh_bounded mkdir -p "$pre15_mountpoint" - if ! pre15_guest_ssh_bounded mount -t erofs -o ro \ - -o "device.1=/dev/md$blob_md" "/dev/md$pre15_primary" \ - "$pre15_mountpoint"; then - pre15_dut_fail "$pre15_label mount failed" - fi - pre15_own_guest_mount "$pre15_mountpoint" "$pre15_label mount" -} - -blob_md=$(pre15_attach_md /root/pre15-b14-fixtures/B14-slot1.blob \ - 'B14 external slot 1') -pre15_mount_image B14-good32.erofs /mnt/pre15-b14-good32 'B14 good32' -pre15_guest_ssh_bounded cmp /root/pre15-b14-fixtures/source/real-dir/entry.txt \ - /mnt/pre15-b14-good32/chunk-dir/entry.txt -pre15_guest_ssh_bounded cmp /root/pre15-b14-fixtures/source/target.txt \ - /mnt/pre15-b14-good32/chunk-link -pre15_guest_ssh_bounded /root/B14-chunk-probe readlink-pass \ - /mnt/pre15-b14-good32/chunk-link target.txt -pre15_guest_ssh_bounded stat -f '%HT %z' /mnt/pre15-b14-good32/chunk-char \ - >"$artifacts/B14-char-stat.txt" -pre15_guest_ssh_bounded /root/B14-chunk-probe open-unsupported \ - /mnt/pre15-b14-good32/chunk-char -pre15_guest_ssh_bounded ls -1A /mnt/pre15-b14-good32/chunk-dir \ - >"$artifacts/B14-directory-list.txt" -if test "$(cat "$artifacts/B14-directory-list.txt")" != entry.txt; then - pre15_dut_fail 'chunk directory readdir returned the wrong set' -fi - -pre15_mount_image B14-good48.erofs /mnt/pre15-b14-good48 'B14 good48' -pre15_guest_ssh_bounded cmp /root/pre15-b14-fixtures/source/real-dir/entry.txt \ - /mnt/pre15-b14-good48/chunk-dir/entry.txt -pre15_guest_ssh_bounded /root/B14-chunk-probe readlink-pass \ - /mnt/pre15-b14-good48/chunk-link target.txt - -pre15_mount_image B14-bad-index.erofs /mnt/pre15-b14-bad-index 'B14 bad index' -pre15_guest_ssh_bounded /root/B14-chunk-probe readlink-integrity \ - /mnt/pre15-b14-bad-index/chunk-link - -pre15_mount_image B14-bad-index48.erofs /mnt/pre15-b14-bad-index48 \ - 'B14 bad 48-bit index' -pre15_guest_ssh_bounded /root/B14-chunk-probe readlink-integrity \ - /mnt/pre15-b14-bad-index48/chunk-link - -pre15_mount_image B14-bad-reserved.erofs /mnt/pre15-b14-bad-reserved \ - 'B14 bad reserved' -pre15_guest_ssh_bounded /root/B14-chunk-probe stat-integrity \ - /mnt/pre15-b14-bad-reserved/chunk-dir - -pre15_mount_image B14-bad-format.erofs /mnt/pre15-b14-bad-format \ - 'B14 bad format' -pre15_guest_ssh_bounded /root/B14-chunk-probe stat-unsupported \ - /mnt/pre15-b14-bad-format/chunk-link - -pre15_mount_image B14-bad-48-no-index.erofs /mnt/pre15-b14-bad-48-no-index \ - 'B14 bad 48-bit without indexes' -pre15_guest_ssh_bounded /root/B14-chunk-probe stat-integrity \ - /mnt/pre15-b14-bad-48-no-index/chunk-link - -if pre15_guest_ssh_bounded truss -f -o /root/B14-md-ebusy.truss \ - mdconfig -d -u "$blob_md"; then - pre15_dut_fail 'external GEOM provider detached while B14 mounts were active' -fi -if ! pre15_guest_ssh_bounded grep -Eq 'ERR#16' /root/B14-md-ebusy.truss; then - pre15_dut_fail 'external GEOM detach did not return exact EBUSY' -fi -pre15_guest_ssh_bounded cat /root/B14-md-ebusy.truss \ - >"$artifacts/B14-md-ebusy.truss" -pre15_guest_ssh_bounded dmesg >"$artifacts/B14-dmesg.txt" -printf '%s\n' 'TC174 QEMU directory/symlink/multidevice/48-bit PASS' diff --git a/tests/pre15/cases/B16-symlink.sh b/tests/pre15/cases/B16-symlink.sh deleted file mode 100755 index 20f24af..0000000 --- a/tests/pre15/cases/B16-symlink.sh +++ /dev/null @@ -1,343 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -spec=$fixture_dir/B16-symlink-spec.json -probe=$fixture_dir/B16-symlink-probe.c -kld_builder=$fixture_dir/B16-build-kld.sh -gate_script=$PRE15_DUT/tests/pre15/gates/P15-019.sh -gate_input=$PRE15_DUT/tests/pre15/gates/P15-019-input.json -artifacts=$PRE15_RUN_DIR/artifacts -gate_clone=$PRE15_CASE_TMP/gate-clone -first=$PRE15_CASE_TMP/first -second=$PRE15_CASE_TMP/second - -for tool in cmp fsck.erofs git mkfs.erofs python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B16 host tool: $tool" -done - -pre15_record_fixture b16-spec "$spec" -pre15_record_fixture b16-probe "$probe" -pre15_record_fixture b16-kld-builder "$kld_builder" -pre15_record_fixture b16-case "$PRE15_DUT/tests/pre15/cases/B16-symlink.sh" -pre15_record_fixture b16-gate "$gate_script" -pre15_record_fixture b16-gate-input "$gate_input" -pre15_record_fixture b16-internal "$PRE15_DUT/src/internal.h" -pre15_record_fixture b16-inode "$PRE15_DUT/src/inode.c" -pre15_record_fixture b16-data "$PRE15_DUT/src/data.c" -pre15_record_fixture b16-vnops "$PRE15_DUT/src/erofs_vnops.c" -pre15_record_fixture b16-linux-inode "$PRE15_ROOT/src-linux/inode.c" -mkdir -p "$artifacts" - -if ! git clone -q --shared --no-checkout "$PRE15_ROOT" "$gate_clone" \ - >"$artifacts/gate-clone.stdout" 2>"$artifacts/gate-clone.stderr" || \ - ! git -C "$gate_clone" checkout -q --detach \ - aea34aba38a298d493d0a584f3985cf3589f358e \ - >"$artifacts/gate-checkout.stdout" 2>"$artifacts/gate-checkout.stderr"; then - pre15_runner_fail 'could not materialize the frozen P15-019 gate commit' -fi - -frozen_gate=$gate_clone/repo-pre-15/tests/pre15/gates/P15-019.sh -if ! timeout -k 10 240 "$frozen_gate" \ - --base d645feb720c7022d2138d2a62eb72c022eb75351 --output "$first" \ - >"$artifacts/gate-first.stdout" 2>"$artifacts/gate-first.stderr"; then - pre15_runner_fail 'first frozen P15-019 gate replay failed' -fi -if ! timeout -k 10 240 "$frozen_gate" \ - --base d645feb720c7022d2138d2a62eb72c022eb75351 --output "$second" \ - >"$artifacts/gate-second.stdout" 2>"$artifacts/gate-second.stderr"; then - pre15_runner_fail 'second frozen P15-019 gate replay failed' -fi -for name in result.json cases.json commands.json semantics.json cleanup.json \ - fixtures/SHA256SUMS; do - if ! cmp "$first/host/$name" "$second/host/$name"; then - pre15_runner_fail "P15-019 repeat output differs: $name" - fi -done - -if ! python3 -B - "$spec" "$gate_script" "$gate_input" \ - "$first/host/result.json" "$first/host/cases.json" \ - "$first/host/fixtures/SHA256SUMS" \ - "$PRE15_DUT/src/internal.h" "$PRE15_DUT/src/inode.c" \ - "$PRE15_DUT/src/data.c" "$PRE15_DUT/src/erofs_vnops.c" \ - "$PRE15_ROOT/src-linux/inode.c" \ - >"$artifacts/B16-host-audit.json" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -import sys - - -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def function(source: str, name: str) -> str: - match = re.search(rf"\n{name}\([^;]*?\n\{{", source, re.DOTALL) - if match is None: - raise SystemExit(f"missing function: {name}") - start = match.start() + 1 - brace = source.index("{", match.start()) - depth = 0 - for index in range(brace, len(source)): - if source[index] == "{": - depth += 1 - elif source[index] == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - raise SystemExit(f"unterminated function: {name}") - - -spec = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -gate = Path(sys.argv[2]) -gate_input = Path(sys.argv[3]) -result = json.loads(Path(sys.argv[4]).read_text(encoding="ascii")) -cases = json.loads(Path(sys.argv[5]).read_text(encoding="ascii")) -sums = Path(sys.argv[6]) -internal = Path(sys.argv[7]).read_text(encoding="utf-8") -inode = Path(sys.argv[8]).read_text(encoding="utf-8") -data = Path(sys.argv[9]).read_text(encoding="utf-8") -vnops = Path(sys.argv[10]).read_text(encoding="utf-8") -linux = Path(sys.argv[11]).read_text(encoding="utf-8") - -if spec.get("schema") != 1 or spec.get("batch") != "B16" or spec.get("test") != "TC166-symlink-layouts": - raise SystemExit("B16 spec identity changed") -if sha256(gate) != spec["gate"]["script_sha256"] or sha256(gate_input) != spec["gate"]["input_sha256"]: - raise SystemExit("tracked P15-019 gate differs from the GO decision") -if result.get("decision") != "GO" or result.get("case_count") != 35: - raise SystemExit("frozen P15-019 gate did not reproduce GO/35") -if result.get("fixture_set_sha256") != spec["fixture_set_sha256"] or sha256(sums) != spec["fixture_set_sha256"]: - raise SystemExit("B16 fixture-set hash differs from G03") - -expected_pairs = { - (layout, case) - for layout in spec["layouts"] - for case in spec["cases"] -} -actual_pairs = {(item["layout"], item["case"]) for item in cases} -if actual_pairs != expected_pairs: - raise SystemExit("B16 fixture matrix is incomplete") -for item in cases: - layout = spec["layouts"][item["layout"]] - if item["inode"]["layout"] != layout["layout"]: - raise SystemExit("B16 fixture layout differs") - -validator = function(data, "erofs_validate_symlink_target") -readlink = function(data, "erofs_readlink_target") -read_inode = function(inode, "erofs_read_inode") -markers = ( - "vi->vtype != VLNK", - "vi->size == 0", - "vi->size > MAXPATHLEN", - "erofs_read_data(sbi, vi, 0, (size_t)vi->size, &target)", - "memchr(target, '\\0', (size_t)vi->size)", - "erofs_brelse(target)", -) -if not all(marker in validator for marker in markers): - raise SystemExit("B16 bounded validator is incomplete") -if "uiomove" in validator: - raise SystemExit("B16 validator performs partial caller transfer") -if not ( - read_inode.index("vi->size == 0") - < read_inode.index("z_erofs_fill_inode(sbi, vi)") - and read_inode.index("vi->size > MAXPATHLEN") - < read_inode.index("z_erofs_fill_inode(sbi, vi)") - and read_inode.index("erofs_validate_symlink_target(sbi, vi)") - > read_inode.index("z_erofs_fill_inode(sbi, vi)") -): - raise SystemExit("B16 size/content validation order changed") -if "erofs_validate_symlink_target" not in internal: - raise SystemExit("B16 validator prototype is missing") -if "vi->size == 0" not in readlink or "vi->size > MAXPATHLEN" not in readlink or "erofs_read_uio" not in readlink: - raise SystemExit("B16 VOP readlink guard/path changed") -if "return (erofs_readlink_target(ap->a_vp, ap->a_uio));" not in vnops: - raise SystemExit("FreeBSD VOP_READLINK adapter changed") -if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", internal + inode + data + vnops): - raise SystemExit("B16 introduced Linux negative errno") -if "i_link" in internal + inode + data + vnops or "page_get_link" in internal + inode + data + vnops: - raise SystemExit("B16 copied the Linux page/i_link implementation") -for marker in ( - "vi->datalayout == EROFS_INODE_FLAT_INLINE", - "kmemdup_nul(bptr + ofs, inode->i_size, GFP_KERNEL)", - ".get_link = page_get_link", - ".get_link = simple_get_link", - "return -EFSCORRUPTED", -): - if marker not in linux: - raise SystemExit("Linux symlink semantic anchor changed") - -report = { - "status": "PASS", - "candidate": "P15-019", - "test": "TC166-symlink-layouts", - "fixture_count": len(cases), - "fixture_set_sha256": result["fixture_set_sha256"], - "layouts": sorted(spec["layouts"]), - "errno": spec["errno"], - "linux_path": "fast inline i_link validation; page_get_link otherwise; negative errno", - "freebsd_path": "vnode VOP_READLINK; bounded pre-publication scan; GEOM/map; positive errno", -} -print(json.dumps(report, indent=2, sort_keys=True)) -PY -then - pre15_runner_fail 'B16 independent fixture/source audit failed' -fi - -cp "$first/host/result.json" "$artifacts/P15-019-result.json" -cp "$first/host/cases.json" "$artifacts/P15-019-cases.json" -cp "$first/host/fixtures/SHA256SUMS" "$artifacts/B16-SHA256SUMS" -pre15_record_fixture b16-generated-sums "$artifacts/B16-SHA256SUMS" -pre15_target_reached - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' \ - 'TC166 host all-layout fixture and independent source oracle PASS' \ - 'QEMU runtime NOT_RUN in host mode' - exit 0 -fi - -for tool in awk cc clang file nm scp tar; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B16 QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -fixture_archive=$PRE15_CASE_TMP/B16-fixtures.tar.gz -module=$PRE15_CASE_TMP/B16-erofs.ko -if ! /bin/sh "$kld_builder" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work" >"$artifacts/B16-kld-build.stdout" \ - 2>"$artifacts/B16-kld-build.stderr"; then - pre15_dut_fail 'B16 cross-target KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/B16-kld-file.txt" -sha256sum "$module" >"$artifacts/B16-kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/B16-kld-nm-u.txt" -tar -C "$first/host/fixtures" -czf "$fixture_archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -if ! pre15_scp "$fixture_archive" /root/B16-fixtures.tar.gz || \ - ! pre15_scp "$probe" /root/B16-symlink-probe.c || \ - ! pre15_scp "$module" /root/B16-erofs.ko; then - pre15_infra_blocked 'could not transfer B16 module or fixtures to the guest' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/pre15-b16-fixtures && mkdir /root/pre15-b16-fixtures && tar -xzf /root/B16-fixtures.tar.gz -C /root/pre15-b16-fixtures'; then - pre15_infra_blocked 'could not prepare B16 guest fixtures' -fi -if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then - pre15_infra_blocked 'FreeBSD guest already has an EROFS module loaded' -fi -if ! pre15_guest_ssh_bounded kldload /root/B16-erofs.ko \ - >"$artifacts/B16-kldload.stdout" 2>"$artifacts/B16-kldload.stderr"; then - if grep -q 'module already loaded or in kernel' "$artifacts/B16-kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - pre15_dut_fail 'B16 exact-source KLD failed to load' -fi -pre15_own_guest_kld erofs 'B16 exact-source KLD' -if ! pre15_guest_ssh_bounded cc -std=c11 -Wall -Wextra -Werror \ - -o /root/B16-symlink-probe /root/B16-symlink-probe.c; then - pre15_infra_blocked 'could not compile the B16 guest probe' -fi - -pre15_attach_md() -{ - pre15_md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode -f "$1") || \ - pre15_dut_fail "could not attach B16 provider: $1" - case "$pre15_md" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected mdconfig output: $pre15_md" ;; - esac - pre15_md=${pre15_md#md} - pre15_own_guest_md "$pre15_md" "$2" - printf '%s\n' "$pre15_md" -} - -pre15_mount_case() -{ - pre15_layout=$1 - pre15_case=$2 - pre15_mountpoint=/mnt/pre15-b16-$pre15_layout-$pre15_case - pre15_options= - if test "$pre15_layout" = chunk; then - pre15_blob=$(pre15_attach_md \ - "/root/pre15-b16-fixtures/$pre15_layout-$pre15_case.blob" \ - "B16 $pre15_layout $pre15_case blob") - pre15_options="-o device.1=/dev/md$pre15_blob" - fi - pre15_primary=$(pre15_attach_md \ - "/root/pre15-b16-fixtures/$pre15_layout-$pre15_case.erofs" \ - "B16 $pre15_layout $pre15_case primary") - pre15_guest_ssh_bounded mkdir -p "$pre15_mountpoint" - if ! pre15_guest_ssh_bounded mount -t erofs -o ro $pre15_options \ - "/dev/md$pre15_primary" "$pre15_mountpoint"; then - pre15_dut_fail "B16 $pre15_layout $pre15_case mount failed" - fi - pre15_own_guest_mount "$pre15_mountpoint" \ - "B16 $pre15_layout $pre15_case mount" - printf '%s\n' "$pre15_mountpoint/link" -} - -for layout in chunk compressed fragment inline plain; do - case "$layout" in - compressed) short_length=64 ;; - *) short_length=31 ;; - esac - for case_id in normal-short normal-max empty nul-middle too-long; do - path=$(pre15_mount_case "$layout" "$case_id") - case "$case_id" in - normal-short) - pre15_guest_ssh_bounded /root/B16-symlink-probe \ - readlink-pass "$path" "$short_length" - pre15_guest_ssh_bounded /root/B16-symlink-probe \ - concurrent "$path" "$short_length" 16 20 - ;; - normal-max) - pre15_guest_ssh_bounded /root/B16-symlink-probe \ - readlink-pass "$path" 1024 - pre15_guest_ssh_bounded /root/B16-symlink-probe \ - follow-errno "$path" 63 - ;; - empty|nul-middle) - pre15_guest_ssh_bounded /root/B16-symlink-probe \ - readlink-errno "$path" 97 - ;; - too-long) - pre15_guest_ssh_bounded /root/B16-symlink-probe \ - readlink-errno "$path" 63 - ;; - esac - done -done - -pre15_guest_ssh_bounded dmesg >"$artifacts/B16-dmesg.txt" -if grep -Eq 'panic:|Fatal trap|lock order reversal|KDB: stack backtrace' \ - "$artifacts/B16-dmesg.txt"; then - pre15_dut_fail 'B16 runtime produced a kernel diagnostic' -fi -printf '%s\n' 'TC166 QEMU all-layout symlink validation PASS' diff --git a/tests/pre15/cases/B17-xattr-integrity.sh b/tests/pre15/cases/B17-xattr-integrity.sh deleted file mode 100755 index 8482c38..0000000 --- a/tests/pre15/cases/B17-xattr-integrity.sh +++ /dev/null @@ -1,275 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -fixtures_only=0 -if test "${1:-}" = --fixtures-only; then - fixtures_only=1 - shift -fi -test "$#" -eq 0 || pre15_runner_fail 'B17 accepts only --fixtures-only' - -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -spec=$fixture_dir/B17-xattr-spec.json -generator=$fixture_dir/B17-xattr-generate.py -oracle=$fixture_dir/B17-xattr-oracle.py -seed=$fixture_dir/B17-xattr-seed.erofs -artifacts=$PRE15_RUN_DIR/artifacts -first=$PRE15_CASE_TMP/first -second=$PRE15_CASE_TMP/second -rebuilt=$PRE15_CASE_TMP/rebuilt-seed.erofs -rebuild_work=$PRE15_CASE_TMP/rebuild-work - -for tool in fsck.erofs git mkfs.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B17 host tool: $tool" -done - -pre15_record_fixture b17-spec "$spec" -pre15_record_fixture b17-generator "$generator" -pre15_record_fixture b17-oracle "$oracle" -pre15_record_fixture b17-seed "$seed" -pre15_record_fixture b17-case "$PRE15_DUT/tests/pre15/cases/B17-xattr-integrity.sh" -mkdir -p "$artifacts" - -if python3 -B "$generator" rebuild-seed --spec "$spec" --seed "$seed" \ - --output "$rebuilt" --work "$rebuild_work" \ - >"$artifacts/rebuild-seed.json" 2>"$artifacts/rebuild-seed.stderr"; then - : -else - pre15_runner_fail 'B17 tracked seed is not reproducible' -fi - -if python3 -B "$generator" generate --spec "$spec" --seed "$seed" \ - --output "$first" >"$artifacts/generate-first.json" \ - 2>"$artifacts/generate-first.stderr"; then - : -else - pre15_runner_fail 'B17 first fixture generation failed' -fi -if python3 -B "$generator" generate --spec "$spec" --seed "$seed" \ - --output "$second" >"$artifacts/generate-second.json" \ - 2>"$artifacts/generate-second.stderr"; then - : -else - pre15_runner_fail 'B17 second fixture generation failed' -fi - -if cmp "$first/fixture-index.json" "$second/fixture-index.json" && \ - find "$first" -type f ! -name fixture-index.json -printf '%f\n' | sort | \ - while IFS= read -r name; do cmp "$first/$name" "$second/$name" || exit 1; done -then - : -else - pre15_runner_fail 'B17 fixture generation is not byte reproducible' -fi - -if python3 -B "$oracle" --spec "$spec" --fixtures "$first" \ - --report "$artifacts/oracle-first.json" \ - >"$artifacts/oracle-first.stdout" 2>"$artifacts/oracle-first.stderr"; then - : -else - pre15_runner_fail 'B17 independent first oracle replay failed' -fi -if python3 -B "$oracle" --spec "$spec" --fixtures "$second" \ - --report "$artifacts/oracle-second.json" \ - >"$artifacts/oracle-second.stdout" 2>"$artifacts/oracle-second.stderr"; then - : -else - pre15_runner_fail 'B17 independent second oracle replay failed' -fi - -cp "$first/fixture-index.json" "$artifacts/B17-fixture-index.json" -pre15_record_fixture b17-generated-index "$artifacts/B17-fixture-index.json" - -if python3 - "$artifacts/B17-fixture-index.json" \ - "$artifacts/oracle-first.json" "$artifacts/B17-matrix.tsv" <<'PY' -import json -from pathlib import Path -import sys - -index = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -report = json.loads(Path(sys.argv[2]).read_text(encoding="ascii")) -if index["fixture_count"] != 25 or index["legal_count"] != 10 or index["damaged_count"] != 15: - raise SystemExit("B17 fixture cardinality changed") -if report["status"] != "PASS" or report["legal_passed"] != 10 or report["damaged_passed"] != 15: - raise SystemExit("B17 oracle matrix is incomplete") -lines = ["id\tclass\terrno\treject"] -for item in report["results"]: - lines.append( - f"{item['id']}\t{item['class']}\t{item['actual_errno']}\t{item['actual_reject']}" - ) -Path(sys.argv[3]).write_text("\n".join(lines) + "\n", encoding="ascii") -print( - f"B17 fixtures: {index['fixture_count']} " - f"({index['legal_count']} legal, {index['damaged_count']} damaged)" -) -print(f"B17 fixture-set SHA256: {index['fixture_set_sha256']}") -PY -then - : -else - pre15_runner_fail 'B17 fixture matrix summary failed' -fi - -pre15_target_reached -if test "$fixtures_only" -eq 1; then - printf '%s\n' 'B17 fixture gate: READY' - exit 0 -fi - -if python3 - "$PRE15_ROOT" "$PRE15_DUT" "$artifacts/B17-source-check.json" <<'PY' -from pathlib import Path -import json -import subprocess -import sys - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -output = Path(sys.argv[3]) -baseline = "98d76e4e73ed9a760b7f684d453b301c73167c22" - - -def committed(name: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/src/{name}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B17 baseline {name}: {completed.stderr}") - return completed.stdout - - -def replace_once(source: str, old: str, new: str, label: str) -> str: - count = source.count(old) - if count != 1: - raise SystemExit(f"{label}: expected one baseline transform, found {count}") - return source.replace(old, new, 1) - - -current = { - name: (dut / "src" / name).read_text(encoding="utf-8") - for name in ("internal.h", "super.c", "xattr.c") -} -base = {name: committed(name) for name in current} - -expected_internal = replace_once( - base["internal.h"], - "\tuint8_t xattr_prefix_count;\n", - "\tuint8_t xattr_prefix_count;\n\tuint8_t xattr_filter_reserved;\n", - "raw xattr filter state", -) - -expected_super = replace_once( - base["super.c"], - "\t/* A non-zero reserved value disables the current name-filter format. */\n" - "\tif (erofs_sb_has_xattr_filter(sbi) && dsb->xattr_filter_reserved != 0)\n" - "\t\tsbi->feature_compat &= ~EROFS_FEATURE_COMPAT_XATTR_FILTER;\n", - "\t/* Preserve the raw feature declaration and gate its format at use sites. */\n" - "\tsbi->xattr_filter_reserved = dsb->xattr_filter_reserved;\n", - "raw feature preservation", -) - -expected_xattr = replace_once( - base["xattr.c"], - "\tif (entry_size > remaining)\n\t\treturn (EINTEGRITY);\n", - "\tif (entry_size > remaining)\n\t\treturn (EINTEGRITY);\n" - "\tif (memchr(entry->e_name, '\\0', entry->e_name_len) != NULL)\n" - "\t\treturn (EINTEGRITY);\n", - "inline name NUL validation", -) -expected_xattr = replace_once( - expected_xattr, - "\terror = erofs_xattr_read_backing(sbi, backing_en, off, entry_size,\n" - "\t entrybuf);\n" - "\tif (error != 0)\n" - "\t\treturn (error);\n" - "\tif (entry_sizep != NULL)\n", - "\terror = erofs_xattr_read_backing(sbi, backing_en, off, entry_size,\n" - "\t entrybuf);\n" - "\tif (error != 0)\n" - "\t\treturn (error);\n" - "\terror = erofs_xattr_validate_entry(entrybuf->data, entry_size, NULL,\n" - "\t value_sizep);\n" - "\tif (error != 0) {\n" - "\t\terofs_put_metabuf(entrybuf);\n" - "\t\treturn (error);\n" - "\t}\n" - "\tif (entry_sizep != NULL)\n", - "shared name NUL validation", -) -expected_xattr = replace_once( - expected_xattr, - "\tif (!erofs_sb_has_xattr_filter(sbi))\n\t\treturn (0);\n", - "\tif (!erofs_sb_has_xattr_filter(sbi) ||\n" - "\t sbi->xattr_filter_reserved != 0)\n" - "\t\treturn (0);\n", - "xattr filter use-site gate", -) -expected_xattr = replace_once( - expected_xattr, - "\t\tprefix = buf.data;\n" - "\t\tinfix_len = len - sizeof(*prefix);\n" - "\t\tsbi->xattr_prefixes[i].base_index = prefix->base_index;\n", - "\t\tprefix = buf.data;\n" - "\t\tinfix_len = len - sizeof(*prefix);\n" - "\t\tif (memchr(prefix->infix, '\\0', infix_len) != NULL) {\n" - "\t\t\terror = EINTEGRITY;\n" - "\t\t\tgoto fail;\n" - "\t\t}\n" - "\t\tsbi->xattr_prefixes[i].base_index = prefix->base_index;\n", - "long-prefix NUL validation", -) - -expected = { - "internal.h": expected_internal, - "super.c": expected_super, - "xattr.c": expected_xattr, -} -for name in expected: - if current[name] != expected[name]: - raise SystemExit(f"{name} differs from the exact B17 ledger transforms") - -diff = subprocess.run( - ["git", "-C", str(root), "diff", baseline, "--", "repo-pre-15/src/internal.h", - "repo-pre-15/src/super.c", "repo-pre-15/src/xattr.c"], - check=True, - text=True, - stdout=subprocess.PIPE, -).stdout -for forbidden in ("xxh32", "bloom", "cache owner", "cache waiter"): - if forbidden in diff.lower(): - raise SystemExit(f"B17 source diff contains out-of-scope marker: {forbidden}") -if "return (-E" in "".join(current.values()): - raise SystemExit("negative errno entered the FreeBSD B17 source") - -result = { - "status": "PASS", - "baseline": baseline, - "changed_sources": sorted(expected), - "raw_feature_preserved": True, - "reserved_gated_at_use": True, - "inline_name_nul": True, - "shared_name_nul": True, - "prefix_infix_nul": True, - "acl_empty_suffix_preserved": True, -} -output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii") -print("B17 source: exact raw-feature and name-integrity transforms present") -PY -then - : -else - pre15_dut_fail 'B17 source does not match the fixture-authorized transforms' -fi - -printf '%s\n' 'B17 source gate: PASS' diff --git a/tests/pre15/cases/B18-xattr-order.sh b/tests/pre15/cases/B18-xattr-order.sh deleted file mode 100755 index ff780f2..0000000 --- a/tests/pre15/cases/B18-xattr-order.sh +++ /dev/null @@ -1,359 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=145d892afc8a425294029d4349f03f1b17cf8f92 -artifacts=$PRE15_RUN_DIR/artifacts -xattr=$PRE15_DUT/src/xattr.c -linux_xattr=$PRE15_ROOT/src-linux/xattr.c -xattr_header=$PRE15_DUT/src/xattr.h -vnops=$PRE15_DUT/src/erofs_vnops.c -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -fixture_spec=$fixture_dir/B17-xattr-spec.json -fixture_generator=$fixture_dir/B17-xattr-generate.py -fixture_oracle=$fixture_dir/B17-xattr-oracle.py -fixture_seed=$fixture_dir/B17-xattr-seed.erofs - -for tool in git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B18 host tool: $tool" -done - -pre15_record_fixture b18-freebsd-xattr "$xattr" -pre15_record_fixture b18-linux-xattr "$linux_xattr" -pre15_record_fixture b18-xattr-header "$xattr_header" -pre15_record_fixture b18-vnops "$vnops" -pre15_record_fixture b18-b17-spec "$fixture_spec" -pre15_record_fixture b18-b17-generator "$fixture_generator" -pre15_record_fixture b18-b17-oracle "$fixture_oracle" -pre15_record_fixture b18-b17-seed "$fixture_seed" -pre15_record_fixture b18-case \ - "$PRE15_DUT/tests/pre15/cases/B18-xattr-order.sh" - -mkdir -p "$artifacts" -pre15_target_reached - -if python3 - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" \ - "$artifacts/B18-xattr-order.json" <<'PY' -from __future__ import annotations - -from collections import Counter -import hashlib -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -output = Path(sys.argv[4]) -xattr_path = dut / "src/xattr.c" -current = xattr_path.read_text(encoding="utf-8") -linux = (root / "src-linux/xattr.c").read_text(encoding="utf-8") - - -def committed_bytes(path: str) -> bytes: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit( - f"cannot read B18 baseline {path}: " - f"{completed.stderr.decode(errors='replace')}" - ) - return completed.stdout - - -def require_once(source: str, marker: str, label: str) -> int: - count = source.count(marker) - if count != 1: - raise SystemExit(f"{label}: expected one marker, found {count}: {marker!r}") - return source.index(marker) - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - if brace < 0: - raise SystemExit(f"missing function body: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def require_order(source: str, markers: tuple[str, ...], label: str) -> None: - position = -1 - for marker in markers: - position = source.find(marker, position + 1) - if position < 0: - raise SystemExit(f"{label}: missing ordered marker: {marker}") - - -base = committed_bytes("src/xattr.c").decode("utf-8") -iterator_start = require_once(base, "struct erofs_xattr_iter {", "iterator") -iterator_end_marker = "\n};\n\n" -iterator_end = base.find(iterator_end_marker, iterator_start) -if iterator_end < 0: - raise SystemExit("B18 baseline iterator terminator is absent") -iterator_end += len(iterator_end_marker) -iterator_block = base[iterator_start:iterator_end] - -without_iterator = base[:iterator_start] + base[iterator_end:] -acl_start = require_once( - without_iterator, - "static int\nerofs_inode_has_noacl(", - "ACL/filter block", -) -acl_end = require_once( - without_iterator, - "static int\nerofs_listxattr_foreach(", - "common iterator block", -) -if acl_end <= acl_start: - raise SystemExit("B18 baseline ACL/filter block is not before the iterator core") -acl_block = without_iterator[acl_start:acl_end] - -expected = without_iterator[:acl_start] + without_iterator[acl_end:] -core_start = require_once( - expected, - "static int\nerofs_xattr_backing_size(", - "backing core", -) -expected = expected[:core_start] + iterator_block + expected[core_start:] -acl_adapter = require_once(expected, "int\nerofs_get_acl(", "ACL adapter") -expected = expected[:acl_adapter] + acl_block + expected[acl_adapter:] -if current != expected: - raise SystemExit("xattr.c differs from the two exact B18 definition moves") - -base_names = re.findall(r"^(erofs_[A-Za-z0-9_]+)\s*\(", base, re.MULTILINE) -current_names = re.findall( - r"^(erofs_[A-Za-z0-9_]+)\s*\(", current, re.MULTILINE -) -if Counter(base_names) != Counter(current_names): - raise SystemExit("B18 changed the xattr function definition multiset") - -base_functions = {name: extract_function(base, name) for name in base_names} -current_functions = {name: extract_function(current, name) for name in current_names} -for name in base_functions: - if current_functions[name] != base_functions[name]: - raise SystemExit(f"B18 changed function text instead of moving it: {name}") - if current_functions[name].startswith("static ") != base_functions[name].startswith( - "static " - ): - raise SystemExit(f"B18 changed symbol visibility: {name}") - - -def direct_calls(functions: dict[str, str]) -> Counter[tuple[str, str]]: - calls: Counter[tuple[str, str]] = Counter() - for caller, function in functions.items(): - brace = function.find("{") - for callee in re.findall( - r"\b(erofs_[A-Za-z0-9_]+)\s*\(", function[brace + 1 :] - ): - calls[(caller, callee)] += 1 - return calls - - -base_calls = direct_calls(base_functions) -current_calls = direct_calls(current_functions) -if current_calls != base_calls: - raise SystemExit("B18 changed the direct-call multiset") - -expected_order = [ - "erofs_xattr_backing_size", - "erofs_xattr_read_backing", - "erofs_xattr_read_metadata", - "erofs_xattr_move", - "erofs_xattr_load_body", - "erofs_xattr_validate_entry", - "erofs_xattr_prefix", - "erofs_xattr_namespace_prefix", - "erofs_xattr_list_move", - "erofs_xattr_resolve_name", - "erofs_xattr_name_match", - "erofs_xattr_shared_entry_offset", - "erofs_xattr_load_shared_entry", - "erofs_listxattr_foreach", - "erofs_getxattr_foreach", - "erofs_xattr_iter_inline", - "erofs_xattr_iter_shared", - "erofs_getxattr", - "erofs_listxattr", - "erofs_xattr_prefixes_cleanup", - "erofs_xattr_prefixes_init", - "erofs_inode_has_noacl", - "erofs_acl_from_mode", - "erofs_posix_acl_from_xattr", - "erofs_get_acl", -] -if current_names != expected_order: - raise SystemExit(f"B18 xattr definition order mismatch: {current_names}") -if current.index("struct erofs_xattr_iter {") > current.index( - "erofs_xattr_backing_size(" -): - raise SystemExit("xattr iterator contract is not before the backing/core helpers") - -linux_aligned_core = ( - "erofs_listxattr_foreach(", - "erofs_getxattr_foreach(", - "erofs_xattr_iter_inline(", - "erofs_xattr_iter_shared(", - "erofs_getxattr(", - "erofs_listxattr(", - "erofs_xattr_prefixes_cleanup(", - "erofs_xattr_prefixes_init(", -) -require_order(current, linux_aligned_core, "FreeBSD common xattr core") -require_order(linux, linux_aligned_core, "Linux common xattr core") - -readonly_paths = ( - "src/xattr.h", - "src/erofs_vnops.c", - "tests/pre15/fixtures/B17-xattr-spec.json", - "tests/pre15/fixtures/B17-xattr-generate.py", - "tests/pre15/fixtures/B17-xattr-oracle.py", - "tests/pre15/fixtures/B17-xattr-seed.erofs", -) -for path in readonly_paths: - if (dut / path).read_bytes() != committed_bytes(path): - raise SystemExit(f"B18 changed a read-only xattr/VOP/fixture contract: {path}") - -vnops = (dut / "src/erofs_vnops.c").read_text(encoding="utf-8") -getextattr = extract_function(vnops, "erofs_getextattr") -listextattr = extract_function(vnops, "erofs_listextattr") -require_order( - getextattr, - ("extattr_check_cred(", "switch (ap->a_attrnamespace)", "erofs_getxattr("), - "FreeBSD getextattr adapter", -) -require_order( - listextattr, - ("extattr_check_cred(", "switch (ap->a_attrnamespace)", "erofs_listxattr("), - "FreeBSD listextattr adapter", -) -if vnops.count("extattr_check_cred(") != 2: - raise SystemExit("FreeBSD extattr credential boundary changed") -if "ERANGE" in current: - raise SystemExit("Linux all-or-nothing xattr buffer semantics entered FreeBSD") -if "uiomove(value, value_size, uio)" not in current: - raise SystemExit("FreeBSD partial extattr transfer path is absent") -if re.search(r"return\s*(?:\(\s*)?-E[A-Z0-9_]+", current): - raise SystemExit("Linux negative errno entered FreeBSD xattr.c") - -load_body = extract_function(current, "erofs_xattr_load_body") -require_order( - load_body, - ( - "vi->xattr_isize < sizeof(*ih)", - "vi->xattr_isize == sizeof(*ih)", - "error = EOPNOTSUPP", - "header_size = sizeof(*ih)", - ), - "exact-header compatibility", -) - -spec_path = dut / "tests/pre15/fixtures/B17-xattr-spec.json" -spec = json.loads(spec_path.read_text(encoding="ascii")) -cases = spec.get("cases", []) -legal = sum(item.get("class") == "legal" for item in cases) -damaged = sum(item.get("class") == "damaged" for item in cases) -if len(cases) != 25 or legal != 10 or damaged != 15: - raise SystemExit("B17 xattr fixture order/cardinality changed") -seed_path = dut / "tests/pre15/fixtures" / spec["seed"]["path"] -seed_hash = hashlib.sha256(seed_path.read_bytes()).hexdigest() -if seed_hash != spec["seed"]["sha256"]: - raise SystemExit("B17 xattr seed no longer matches its frozen specification") - -result = { - "status": "PASS", - "baseline": baseline, - "source_sha256": hashlib.sha256(current.encode("utf-8")).hexdigest(), - "definition_count": len(current_names), - "direct_call_edges": len(current_calls), - "direct_call_sites": sum(current_calls.values()), - "static_visibility_unchanged": True, - "function_text_unchanged": True, - "linux_core_order": list(linux_aligned_core), - "freebsd_contracts": [ - "positive errno", - "partial uiomove", - "extattr_check_cred", - "USER and SYSTEM namespaces", - "exact-header EOPNOTSUPP", - "uncached xattr lookup", - ], - "b17_fixture_cases": len(cases), - "b17_fixture_legal": legal, - "b17_fixture_damaged": damaged, - "b17_seed_sha256": seed_hash, -} -output.write_text( - json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii" -) -print( - "B18 xattr order: exact moves, " - f"{len(current_names)} definitions, " - f"{sum(current_calls.values())} direct calls" -) -print(f"B18 B17 fixture contract: {len(cases)} cases, seed {seed_hash}") -PY -then - : -else - pre15_dut_fail 'B18 xattr ordering or FreeBSD contract equivalence failed' -fi - -printf '%s\n' 'B18 xattr order: PASS' diff --git a/tests/pre15/cases/B19a-xattr-cache.sh b/tests/pre15/cases/B19a-xattr-cache.sh deleted file mode 100644 index 26c3f28..0000000 --- a/tests/pre15/cases/B19a-xattr-cache.sh +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/sh -set -eu - -: "$PRE15_DUT" -: "$PRE15_ROOT" -: "$PRE15_CASE_TMP" -: "$PRE15_RUN_DIR" -: "$PRE15_LIB_DIR" -. "$PRE15_LIB_DIR/runner.sh" - -test "$PRE15_MODE" = qemu || pre15_infra_blocked \ - 'B19a focused case requires exact-ABI QEMU mode' -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -generator=$fixture_dir/B19a-xattr-generate.py -spec=$fixture_dir/B19a-xattr-spec.json -probe=$fixture_dir/B19a-xattr-probe.c -kld_builder=$fixture_dir/B28-build-kld.sh -artifacts=$PRE15_RUN_DIR/artifacts -first=$PRE15_CASE_TMP/fixtures-first -second=$PRE15_CASE_TMP/fixtures-second -module=$PRE15_CASE_TMP/B19a-erofs.ko - -for tool in cc cmp diff file mkfs.erofs nm python3 scp sha256sum tar timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B19a QEMU tool: $tool" -done -for value in PRE15_QEMU_CONTROL_PATH PRE15_QEMU_SSH_KEY \ - PRE15_QEMU_SSH_PORT PRE15_QEMU_SSH_USER; do - test -n "$(eval "printf '%s' \"\$$value\"")" || \ - pre15_runner_fail "$value is required" -done - -for fixture in "$generator" "$spec" "$probe" "$kld_builder"; do - pre15_record_fixture "b19a-$(basename "$fixture")" "$fixture" -done -for source in erofs_vnops.c internal.h xattr.c; do - pre15_record_fixture "b19a-$source" "$PRE15_DUT/src/$source" -done -mkdir -p "$artifacts" - -if ! timeout -k 5 180 python3 -B "$generator" --spec "$spec" \ - --output "$first" --work "$PRE15_CASE_TMP/generate-first" \ - >"$artifacts/generate-first.json" 2>"$artifacts/generate-first.stderr"; then - pre15_runner_fail 'B19a first fixture generation failed' -fi -if ! timeout -k 5 180 python3 -B "$generator" --spec "$spec" \ - --output "$second" --work "$PRE15_CASE_TMP/generate-second" \ - >"$artifacts/generate-second.json" 2>"$artifacts/generate-second.stderr"; then - pre15_runner_fail 'B19a second fixture generation failed' -fi -if ! diff -qr "$first" "$second" >"$artifacts/fixture-repeat.diff"; then - pre15_runner_fail 'B19a focused fixtures are not byte reproducible' -fi -pre15_record_fixture b19a-valid-image "$first/valid.erofs" -tar -C "$first" -czf "$PRE15_CASE_TMP/B19a-fixtures.tar.gz" . - -if ! timeout -k 10 600 /bin/sh "$kld_builder" "$PRE15_DUT" \ - "$PRE15_FREEBSD_SRC" "$module" "$PRE15_CASE_TMP/kld-work" 0 \ - >"$artifacts/kld-build.stdout" 2>"$artifacts/kld-build.stderr"; then - pre15_dut_fail 'B19a exact-ABI KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/kld-file.txt" -sha256sum "$module" >"$artifacts/kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/kld-nm-u.txt" - -pre15_scp() -{ - timeout -k 5 60 scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -if ! pre15_scp "$PRE15_CASE_TMP/B19a-fixtures.tar.gz" \ - /root/B19a-fixtures.tar.gz || ! pre15_scp "$probe" /root/B19a-xattr-probe.c || - ! pre15_scp "$module" /root/B19a-erofs.ko; then - pre15_infra_blocked 'could not transfer B19a module, probe, or fixtures' -fi -pre15_guest_ssh_bounded \ - 'rm -rf /root/pre15-b19a-fixtures && mkdir /root/pre15-b19a-fixtures && tar -xzf /root/B19a-fixtures.tar.gz -C /root/pre15-b19a-fixtures' || - pre15_infra_blocked 'could not prepare B19a guest fixtures' -pre15_guest_ssh_bounded cc -std=c11 -Wall -Wextra -Werror \ - -o /root/B19a-xattr-probe /root/B19a-xattr-probe.c || - pre15_infra_blocked 'could not compile the B19a guest probe' -if pre15_guest_ssh_bounded kldstat -n B19a-erofs >/dev/null 2>&1; then - pre15_infra_blocked 'guest already has an EROFS KLD loaded' -fi -pre15_guest_ssh_bounded dmesg >"$artifacts/dmesg-before-load.txt" -if ! pre15_guest_ssh_bounded kldload /root/B19a-erofs.ko \ - >"$artifacts/kldload.stdout" 2>"$artifacts/kldload.stderr"; then - pre15_dut_fail 'B19a exact-ABI KLD load failed' -fi -pre15_own_guest_kld B19a-erofs 'B19a exact-source KLD' -pre15_guest_ssh_bounded kldstat >"$artifacts/kldstat-after-load.txt" - -b19a_unown() -{ - tmp=$PRE15_CASE_TMP/ownership.$$ - awk -F ' ' -v kind="$1" -v value="$2" \ - '!($1 == kind && $2 == value)' "$PRE15_OWNERSHIP_FILE" >"$tmp" - mv "$tmp" "$PRE15_OWNERSHIP_FILE" -} - -b19a_mount() -{ - image=$1 - mountpoint=$2 - label=$3 - md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/pre15-b19a-fixtures/$image") || - pre15_dut_fail "$label provider attach failed" - case "$md" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected B19a md unit: $md" ;; - esac - pre15_own_guest_md "$md" "$label provider" - pre15_guest_ssh_bounded mkdir -p "$mountpoint" - if ! pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$md" \ - "$mountpoint"; then - pre15_dut_fail "$label mount failed" - fi - pre15_own_guest_mount "$mountpoint" "$label mount" - printf '%s\n' "$md" -} - -b19a_detach() -{ - mountpoint=$1 - md=$2 - pre15_guest_ssh_bounded umount "$mountpoint" || - pre15_dut_fail "B19a unmount failed: $mountpoint" - b19a_unown guest-mount "$mountpoint" - pre15_guest_ssh_bounded mdconfig -d -u "$md" || - pre15_dut_fail "B19a md detach failed: $md" - b19a_unown guest-md "$md" -} - -valid_md=$(b19a_mount valid.erofs /mnt/pre15-b19a-valid 'B19a valid') -valid=/mnt/pre15-b19a-valid/target.bin -pre15_guest_ssh_bounded /root/B19a-xattr-probe valid "$valid" \ - >"$artifacts/valid-first.txt" -pre15_guest_ssh_bounded /root/B19a-xattr-probe concurrent "$valid" 16 10 \ - >"$artifacts/valid-concurrent.txt" -pre15_guest_ssh_bounded /root/B19a-xattr-probe valid "$valid" \ - >"$artifacts/valid-repeat.txt" -b19a_detach /mnt/pre15-b19a-valid "$valid_md" - -for image in corrupt-shared-count.erofs corrupt-shared-id.erofs corrupt-inline-name.erofs; do - label=$(basename "$image" .erofs) - mountpoint=/mnt/pre15-b19a-$label - md=$(b19a_mount "$image" "$mountpoint" "B19a $label") - if ! pre15_guest_ssh_bounded /root/B19a-xattr-probe corrupt \ - "$mountpoint/target.bin" >"$artifacts/$label.txt" 2>&1; then - pre15_dut_fail "$label did not return EINTEGRITY" - fi - pre15_guest_ssh_bounded umount "$mountpoint" || - pre15_dut_fail "$label unmount failed" - b19a_unown guest-mount "$mountpoint" - pre15_guest_ssh_bounded mdconfig -d -u "$md" || - pre15_dut_fail "$label md detach failed" - b19a_unown guest-md "$md" -done - -normal_md=$(b19a_mount valid.erofs /mnt/pre15-b19a-normal \ - 'B19a normal-unmount') -normal_pid=$(pre15_guest_ssh_bounded \ - 'cd /mnt/pre15-b19a-normal && sleep 60 >/tmp/B19a-normal.sleep 2>&1 & echo $!') -if pre15_guest_ssh_bounded umount /mnt/pre15-b19a-normal \ - >"$artifacts/normal-unmount.stdout" 2>"$artifacts/normal-unmount.stderr"; then - pre15_dut_fail 'normal unmount unexpectedly ignored a held vnode' -fi -pre15_guest_ssh_bounded kill "$normal_pid" || true -pre15_guest_ssh_bounded umount /mnt/pre15-b19a-normal || - pre15_dut_fail 'normal unmount failed after releasing held vnode' -b19a_unown guest-mount /mnt/pre15-b19a-normal -pre15_guest_ssh_bounded mdconfig -d -u "$normal_md" || - pre15_dut_fail 'normal-unmount md detach failed' -b19a_unown guest-md "$normal_md" - -forced_md=$(b19a_mount valid.erofs /mnt/pre15-b19a-forced \ - 'B19a forced-unmount') -forced_pid=$(pre15_guest_ssh_bounded \ - 'cd /mnt/pre15-b19a-forced && sleep 60 >/tmp/B19a-forced.sleep 2>&1 & echo $!') -pre15_guest_ssh_bounded umount -f /mnt/pre15-b19a-forced || - pre15_dut_fail 'forced unmount failed' -b19a_unown guest-mount /mnt/pre15-b19a-forced -pre15_guest_ssh_bounded kill "$forced_pid" || true -pre15_guest_ssh_bounded mdconfig -d -u "$forced_md" || - pre15_dut_fail 'forced-unmount md detach failed' -b19a_unown guest-md "$forced_md" - -pre15_guest_ssh_bounded dmesg >"$artifacts/dmesg-after.txt" -diff -u "$artifacts/dmesg-before-load.txt" "$artifacts/dmesg-after.txt" \ - >"$artifacts/dmesg.diff" || true -sed -n '/^+++ /d; /^+/s/^+/ /p' "$artifacts/dmesg.diff" \ - >"$artifacts/dmesg-added.txt" -if grep -Eqi 'panic:|fatal trap|lock order reversal|witness.*warning|use-after-free|pager fault|undefined symbol|linker.*error' \ - "$artifacts/dmesg-added.txt"; then - pre15_dut_fail 'B19a QEMU produced a kernel diagnostic' -fi -pre15_guest_ssh_bounded kldunload B19a-erofs || - pre15_dut_fail 'B19a KLD unload failed' -b19a_unown guest-kld B19a-erofs -if pre15_guest_ssh_bounded kldstat -n B19a-erofs >/dev/null 2>&1; then - pre15_dut_fail 'B19a KLD remained loaded after cleanup' -fi -pre15_guest_ssh_bounded mount >"$artifacts/mount-after.txt" -pre15_guest_ssh_bounded mdconfig -l >"$artifacts/md-after.txt" -if grep -q 'pre15-b19a-' "$artifacts/mount-after.txt" || - grep -q 'B19a' "$artifacts/md-after.txt"; then - pre15_dut_fail 'B19a guest mount or md resource remained after cleanup' -fi -pre15_target_reached -printf '%s\n' 'B19a exact-ABI xattr cache lifecycle PASS' diff --git a/tests/pre15/cases/B19b-xattr-bloom.sh b/tests/pre15/cases/B19b-xattr-bloom.sh deleted file mode 100755 index 9f08c56..0000000 --- a/tests/pre15/cases/B19b-xattr-bloom.sh +++ /dev/null @@ -1,220 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -spec=$fixture_dir/B19b-xattr-spec.json -generator=$fixture_dir/B19b-xattr-generate.py -oracle=$fixture_dir/B19b-xattr-oracle.py -probe=$fixture_dir/B19b-xattr-probe.c -kld_builder=$fixture_dir/B19b-build-kld.sh -gate_script=$PRE15_DUT/tests/pre15/gates/P15-021.sh -gate_input=$PRE15_DUT/tests/pre15/gates/P15-021-input.json -gate_commit=675ed9b650b3bc157b38bcc165a0cb215a2aa989 -gate_base=666e52f710363df07f7c93919eb835d41092d011 -artifacts=$PRE15_RUN_DIR/artifacts -first=$PRE15_CASE_TMP/first -second=$PRE15_CASE_TMP/second -gate_clone=$PRE15_CASE_TMP/gate-clone -gate_output=$PRE15_CASE_TMP/gate-output - -for tool in cmp fsck.erofs git mkfs.erofs python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B19b host tool: $tool" -done - -pre15_record_fixture b19b-spec "$spec" -pre15_record_fixture b19b-generator "$generator" -pre15_record_fixture b19b-oracle "$oracle" -pre15_record_fixture b19b-probe "$probe" -pre15_record_fixture b19b-kld-builder "$kld_builder" -pre15_record_fixture b19b-case "$PRE15_DUT/tests/pre15/cases/B19b-xattr-bloom.sh" -pre15_record_fixture b19b-gate "$gate_script" -pre15_record_fixture b19b-gate-input "$gate_input" -pre15_record_fixture b19b-erofs-fs "$PRE15_DUT/src/erofs_fs.h" -pre15_record_fixture b19b-internal "$PRE15_DUT/src/internal.h" -pre15_record_fixture b19b-xattr "$PRE15_DUT/src/xattr.c" -pre15_record_fixture b19b-linux-erofs-fs "$PRE15_ROOT/src-linux/erofs_fs.h" -pre15_record_fixture b19b-linux-xattr "$PRE15_ROOT/src-linux/xattr.c" -mkdir -p "$artifacts" - -if ! git clone -q --shared --no-checkout "$PRE15_ROOT" "$gate_clone" \ - >"$artifacts/gate-clone.stdout" 2>"$artifacts/gate-clone.stderr" || \ - ! git -C "$gate_clone" checkout -q --detach "$gate_commit" \ - >"$artifacts/gate-checkout.stdout" 2>"$artifacts/gate-checkout.stderr"; then - pre15_runner_fail 'could not materialize the committed P15-021 gate' -fi -frozen_gate=$gate_clone/repo-pre-15/tests/pre15/gates/P15-021.sh -if ! timeout -k 10 240 "$frozen_gate" --base "$gate_base" \ - --output "$gate_output" >"$artifacts/gate.stdout" \ - 2>"$artifacts/gate.stderr"; then - pre15_runner_fail 'committed P15-021 gate did not replay GO' -fi - -if ! python3 -B "$generator" generate --spec "$spec" --output "$first" \ - --work "$PRE15_CASE_TMP/first-work" >"$artifacts/generate-first.stdout" \ - 2>"$artifacts/generate-first.stderr"; then - pre15_runner_fail 'B19b first real-fixture generation failed' -fi -if ! python3 -B "$generator" generate --spec "$spec" --output "$second" \ - --work "$PRE15_CASE_TMP/second-work" >"$artifacts/generate-second.stdout" \ - 2>"$artifacts/generate-second.stderr"; then - pre15_runner_fail 'B19b repeat real-fixture generation failed' -fi -if ! cmp "$first/SHA256SUMS" "$second/SHA256SUMS" || \ - ! cmp "$first/manifest.json" "$second/manifest.json"; then - pre15_runner_fail 'B19b fixture generation is not byte reproducible' -fi -for image in valid.erofs unknown-filter.erofs feature-off.erofs \ - corrupt-shared-count.erofs corrupt-shared-id.erofs; do - if ! cmp "$first/$image" "$second/$image"; then - pre15_runner_fail "B19b repeat image differs: $image" - fi - pre15_record_fixture "b19b-$image" "$first/$image" -done - -if ! python3 -B "$oracle" --spec "$spec" --fixtures "$first" \ - --root "$PRE15_ROOT" --dut "$PRE15_DUT" \ - --report "$artifacts/B19b-oracle.json" \ - >"$artifacts/oracle.stdout" 2>"$artifacts/oracle.stderr"; then - pre15_dut_fail 'B19b fixture/source/Linux oracle failed' -fi -cp "$gate_output/result.json" "$artifacts/P15-021-gate-result.json" -cp "$gate_output/million.json" "$artifacts/P15-021-million.json" -cp "$gate_output/benchmark.json" "$artifacts/P15-021-benchmark.json" -cp "$first/SHA256SUMS" "$artifacts/B19b-SHA256SUMS" -pre15_record_fixture b19b-generated-sums "$artifacts/B19b-SHA256SUMS" -pre15_target_reached - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' \ - 'TC169 host real fixture/hash/filter/integrity/concurrency oracle PASS' \ - 'QEMU runtime NOT_RUN in host mode' - exit 0 -fi - -for tool in awk clang file nm scp tar; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B19b QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -fixture_archive=$PRE15_CASE_TMP/B19b-fixtures.tar.gz -module=$PRE15_CASE_TMP/B19b-erofs.ko -if ! /bin/sh "$kld_builder" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work" >"$artifacts/B19b-kld-build.stdout" \ - 2>"$artifacts/B19b-kld-build.stderr"; then - pre15_dut_fail 'B19b cross-target KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/B19b-kld-file.txt" -sha256sum "$module" >"$artifacts/B19b-kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/B19b-kld-nm-u.txt" -tar -C "$first" -czf "$fixture_archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -if ! pre15_scp "$fixture_archive" /root/B19b-fixtures.tar.gz || \ - ! pre15_scp "$probe" /root/B19b-xattr-probe.c || \ - ! pre15_scp "$module" /root/B19b-erofs.ko; then - pre15_infra_blocked 'could not transfer B19b module, probe, or fixtures' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/pre15-b19b-fixtures && mkdir /root/pre15-b19b-fixtures && tar -xzf /root/B19b-fixtures.tar.gz -C /root/pre15-b19b-fixtures'; then - pre15_infra_blocked 'could not prepare B19b guest fixtures' -fi -if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then - pre15_infra_blocked 'FreeBSD guest already has an EROFS module loaded' -fi -if ! pre15_guest_ssh_bounded kldload /root/B19b-erofs.ko \ - >"$artifacts/B19b-kldload.stdout" 2>"$artifacts/B19b-kldload.stderr"; then - if grep -q 'module already loaded or in kernel' "$artifacts/B19b-kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - pre15_dut_fail 'B19b exact-source KLD failed to load' -fi -pre15_own_guest_kld erofs 'B19b exact-source KLD' -if ! pre15_guest_ssh_bounded cc -std=c11 -Wall -Wextra -Werror \ - -o /root/B19b-xattr-probe /root/B19b-xattr-probe.c; then - pre15_infra_blocked 'could not compile the B19b guest probe' -fi - -pre15_mount_image() -{ - pre15_image=$1 - pre15_mountpoint=$2 - pre15_label=$3 - pre15_md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/pre15-b19b-fixtures/$pre15_image") || \ - pre15_dut_fail "could not attach B19b provider: $pre15_image" - case "$pre15_md" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected mdconfig output: $pre15_md" ;; - esac - pre15_own_guest_md "$pre15_md" "$pre15_label provider" - pre15_guest_ssh_bounded mkdir -p "$pre15_mountpoint" - if ! pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$pre15_md" \ - "$pre15_mountpoint"; then - pre15_dut_fail "$pre15_label mount failed" - fi - pre15_own_guest_mount "$pre15_mountpoint" "$pre15_label mount" -} - -pre15_mount_image valid.erofs /mnt/pre15-b19b-valid 'B19b valid' -valid=/mnt/pre15-b19b-valid/target.bin -pre15_guest_ssh_bounded /root/B19b-xattr-probe hit "$valid" attr00 0 -pre15_guest_ssh_bounded /root/B19b-xattr-probe errno "$valid" user \ - absent-00001 87 -pre15_guest_ssh_bounded /root/B19b-xattr-probe errno "$valid" user \ - absent-00000 87 -pre15_guest_ssh_bounded /root/B19b-xattr-probe list "$valid" -pre15_guest_ssh_bounded /root/B19b-xattr-probe concurrent "$valid" attr00 \ - absent-00001 absent-00000 64 10 - -pre15_mount_image unknown-filter.erofs /mnt/pre15-b19b-unknown \ - 'B19b unknown filter' -unknown=/mnt/pre15-b19b-unknown/target.bin -pre15_guest_ssh_bounded /root/B19b-xattr-probe hit "$unknown" attr00 0 -pre15_guest_ssh_bounded /root/B19b-xattr-probe errno "$unknown" user \ - absent-00001 87 - -pre15_mount_image feature-off.erofs /mnt/pre15-b19b-feature-off \ - 'B19b feature off' -feature_off=/mnt/pre15-b19b-feature-off/target.bin -pre15_guest_ssh_bounded /root/B19b-xattr-probe hit "$feature_off" attr00 0 -pre15_guest_ssh_bounded /root/B19b-xattr-probe errno "$feature_off" user \ - absent-00001 87 - -pre15_mount_image corrupt-shared-count.erofs /mnt/pre15-b19b-bad-count \ - 'B19b corrupt shared count' -pre15_guest_ssh_bounded /root/B19b-xattr-probe errno \ - /mnt/pre15-b19b-bad-count/target.bin user absent-00001 97 - -pre15_mount_image corrupt-shared-id.erofs /mnt/pre15-b19b-bad-id \ - 'B19b corrupt shared ID' -pre15_guest_ssh_bounded /root/B19b-xattr-probe errno \ - /mnt/pre15-b19b-bad-id/target.bin user attr00 97 - -pre15_guest_ssh_bounded dmesg >"$artifacts/B19b-dmesg.txt" -if grep -Eq 'panic:|Fatal trap|lock order reversal|KDB: stack backtrace' \ - "$artifacts/B19b-dmesg.txt"; then - pre15_dut_fail 'B19b runtime produced a kernel diagnostic' -fi -printf '%s\n' 'TC169 QEMU hit/miss/collision/fallback/integrity/concurrency PASS' diff --git a/tests/pre15/cases/B21-super.sh b/tests/pre15/cases/B21-super.sh deleted file mode 100644 index 38f0b46..0000000 --- a/tests/pre15/cases/B21-super.sh +++ /dev/null @@ -1,540 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=e769e4ae32d4967c1f69067dc92628b60b62b648 -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -spec=$fixture_dir/B21-super-spec.json -generator=$fixture_dir/B21-super-generate.py -oracle=$fixture_dir/B21-super-oracle.py -qemu_probe=$fixture_dir/B21-qemu-probe.c -kld_builder=$fixture_dir/B28-build-kld.sh -artifacts=$PRE15_RUN_DIR/artifacts -first=$PRE15_CASE_TMP/first -second=$PRE15_CASE_TMP/second - -for tool in cmp fsck.erofs git mkfs.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B21 host tool: $tool" -done - -pre15_record_fixture b21-spec "$spec" -pre15_record_fixture b21-generator "$generator" -pre15_record_fixture b21-oracle "$oracle" -pre15_record_fixture b21-qemu-probe "$qemu_probe" -pre15_record_fixture b21-kld-builder "$kld_builder" -pre15_record_fixture b21-case "$PRE15_DUT/tests/pre15/cases/B21-super.sh" -pre15_record_fixture b21-super "$PRE15_DUT/src/super.c" -pre15_record_fixture b21-internal "$PRE15_DUT/src/internal.h" -pre15_record_fixture b21-xattr "$PRE15_DUT/src/xattr.c" -mkdir -p "$artifacts" - -if python3 -B "$generator" --spec "$spec" --output "$first" \ - --work "$PRE15_CASE_TMP/first-work" \ - >"$artifacts/generate-first.json" \ - 2>"$artifacts/generate-first.stderr"; then - : -else - pre15_runner_fail 'B21 first fixture generation failed' -fi -if python3 -B "$generator" --spec "$spec" --output "$second" \ - --work "$PRE15_CASE_TMP/second-work" \ - >"$artifacts/generate-second.json" \ - 2>"$artifacts/generate-second.stderr"; then - : -else - pre15_runner_fail 'B21 second fixture generation failed' -fi - -if cmp "$first/fixture-index.json" "$second/fixture-index.json" && \ - find "$first" -type f ! -name fixture-index.json -printf '%f\n' | sort | \ - while IFS= read -r name; do cmp "$first/$name" "$second/$name" || exit 1; done -then - : -else - pre15_runner_fail 'B21 fixture generation is not byte reproducible' -fi - -if python3 -B "$oracle" --fixtures "$first" \ - --report "$artifacts/oracle-first.json" \ - >"$artifacts/oracle-first.stdout" 2>"$artifacts/oracle-first.stderr" && \ - python3 -B "$oracle" --fixtures "$second" \ - --report "$artifacts/oracle-second.json" \ - >"$artifacts/oracle-second.stdout" 2>"$artifacts/oracle-second.stderr" && \ - cmp "$artifacts/oracle-first.json" "$artifacts/oracle-second.json" -then - : -else - pre15_runner_fail 'B21 independent oracle replay failed' -fi - -if fsck.erofs -d0 "$first/legal-control.erofs" \ - >"$artifacts/legal-fsck.stdout" 2>"$artifacts/legal-fsck.stderr"; then - : -else - pre15_runner_fail 'B21 legal control is not accepted by fsck.erofs' -fi - -cp "$first/fixture-index.json" "$artifacts/B21-fixture-index.json" -pre15_record_fixture b21-generated-index "$artifacts/B21-fixture-index.json" -pre15_target_reached - -if test "${PRE15_QEMU_TARGET_ONLY:-0}" = 1; then - : -else -if python3 - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" \ - "$artifacts/B21-source-check.json" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -output = Path(sys.argv[4]) -src = dut / "src" - - -def committed(name: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/src/{name}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B21 baseline {name}: {completed.stderr}") - return completed.stdout - - -def replace_once(source: str, old: str, new: str, label: str) -> str: - count = source.count(old) - if count != 1: - raise SystemExit(f"{label}: expected one transform source, found {count}") - return source.replace(old, new, 1) - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def require_order(source: str, markers: list[str], label: str) -> None: - position = -1 - for marker in markers: - position = source.find(marker, position + 1) - if position < 0: - raise SystemExit(f"{label} is missing ordered marker: {marker}") - - -current = { - name: (src / name).read_text(encoding="utf-8") - for name in ("internal.h", "super.c", "xattr.c") -} -base = {name: committed(name) for name in current} - -if current["internal.h"] != base["internal.h"]: - raise SystemExit("B21 changed internal.h despite all required helpers existing") -for helper in ( - "EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)", - "EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)", - "EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX)", -): - if current["internal.h"].count(helper) != 1: - raise SystemExit(f"B21 feature helper is missing or duplicated: {helper}") - -expected_super = replace_once( - base["super.c"], - "\tif ((le32toh(dsb->feature_compat) & EROFS_FEATURE_COMPAT_SB_CHKSUM) == 0)\n", - "\tif (!erofs_sb_has_sb_chksum(sbi))\n", - "checksum helper", -) -expected_super = replace_once( - expected_super, - "\tif ((sbi->feature_incompat & EROFS_FEATURE_INCOMPAT_FRAGMENTS) != 0 &&\n" - "\t sbi->packed_nid > 0) {\n", - "\tif (erofs_sb_has_fragments(sbi) && sbi->packed_nid > 0) {\n", - "fragments helper", -) -old_read = extract_function(expected_super, "erofs_read_superblock") -new_read = extract_function(current["super.c"], "erofs_read_superblock") -expected_read = replace_once( - old_read, - "\tif (dsb->dirblkbits != 0)\n" - "\t\treturn (EOPNOTSUPP);\n" - "\tsbi->feature_compat = le32toh(dsb->feature_compat);\n", - "\tsbi->blkszbits = dsb->blkszbits;\n" - "\tsbi->block_size = 1u << sbi->blkszbits;\n" - "\tsbi->feature_compat = le32toh(dsb->feature_compat);\n" - "\terror = erofs_superblock_csum_verify(sbi, dsb);\n" - "\tif (error != 0)\n" - "\t\treturn (error);\n\n" - "\tif (dsb->dirblkbits != 0)\n" - "\t\treturn (EOPNOTSUPP);\n", - "checksum trust order", -) -dead_exception = """\t/* -\t * Narrowly allow one extra combination: long xattr prefixes enabled -\t * with non-plain prefix table stored in a packed inode, which adds -\t * the FRAGMENTS (0x20) incompat bit. This is NOT a declaration of -\t * general fragments support; per-inode data layout is still gated -\t * by plain/inline checks in erofs_read_inode(). -\t */ -\tif (unsupported != 0) { -\t\tif (unsupported != EROFS_FEATURE_INCOMPAT_FRAGMENTS || -\t\t (sbi->feature_incompat & -\t\t\tEROFS_FEATURE_INCOMPAT_XATTR_PREFIXES) == 0 || -\t\t (sbi->feature_compat & -\t\t\tEROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX) != 0 || -\t\t sbi->packed_nid == 0) -\t\t\treturn (EOPNOTSUPP); -\t} -\tsbi->blkszbits = dsb->blkszbits; -\tsbi->block_size = 1u << sbi->blkszbits; -""" -expected_read = replace_once( - expected_read, - dead_exception, - "\tif (unsupported != 0)\n\t\treturn (EOPNOTSUPP);\n", - "dead fragments exception", -) -old_checksum_site = ( - "\terror = erofs_superblock_csum_verify(sbi, dsb);\n" - "\tif (error != 0)\n" - "\t\treturn (error);\n" -) -if expected_read.count(old_checksum_site) != 2: - raise SystemExit("checksum trust-order transform did not produce two sites") -old_checksum_offset = expected_read.rfind(old_checksum_site) -expected_read = ( - expected_read[:old_checksum_offset] - + expected_read[old_checksum_offset + len(old_checksum_site) :] -) -if new_read != expected_read: - raise SystemExit("erofs_read_superblock differs from exact B21 transforms") -expected_super = expected_super.replace(old_read, expected_read, 1) -if current["super.c"] != expected_super: - raise SystemExit("super.c changed outside exact B21 transforms") - -expected_xattr = replace_once( - base["xattr.c"], - "\tif ((sbi->feature_incompat & EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES) ==\n" - "\t\t0 ||\n" - "\t sbi->xattr_prefix_count == 0)\n", - "\tif (!erofs_sb_has_xattr_prefixes(sbi) || sbi->xattr_prefix_count == 0)\n", - "xattr-prefix helper", -) -expected_xattr = replace_once( - expected_xattr, - "\tif ((sbi->feature_compat & EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX) == 0) {\n", - "\tif (!erofs_sb_has_plain_xattr_pfx(sbi)) {\n", - "plain-prefix helper", -) -if current["xattr.c"] != expected_xattr: - raise SystemExit("xattr.c changed outside exact B21 helper conversions") - -read_super = new_read -checksum_call = read_super.index("erofs_superblock_csum_verify(sbi, dsb)") -before_checksum = read_super[:checksum_call] -for protected in ( - "dirblkbits", - "feature_incompat", - "packed_nid", - "extra_devices", - "sb_extslots", - "meta_blkaddr", - "xattr_blkaddr", - "xattr_prefix_start", - "xattr_prefix_count", - "blocks_root", -): - if protected in before_checksum: - raise SystemExit(f"protected field used before checksum: {protected}") -require_order( - read_super, - [ - "dsb->magic", - "dsb->blkszbits < 9", - "sbi->feature_compat = le32toh(dsb->feature_compat)", - "erofs_superblock_csum_verify(sbi, dsb)", - "dsb->dirblkbits", - "sbi->feature_incompat = le32toh(dsb->feature_incompat)", - "unsupported = sbi->feature_incompat", - "sbi->sb_size = 128 + dsb->sb_extslots", - "sbi->xattr_prefix_start = le32toh(dsb->xattr_prefix_start)", - "erofs_validate_device_size", - "erofs_load_generation_seed", - "z_erofs_parse_cfgs", - ], - "superblock trust order", -) - -mountfs = extract_function(current["super.c"], "erofs_mountfs") -require_order( - mountfs, - [ - "erofs_read_superblock", - "erofs_scan_devices", - "erofs_init_packed_inode", - "erofs_init_metabox_inode", - "erofs_read_inode(sbi, sbi->root_nid", - "erofs_xattr_prefixes_init", - "mp->mnt_data = sbi", - ], - "FreeBSD mount publication", -) -if "fail:\n\terofs_sb_free(sbi);\n\treturn (error);" not in mountfs: - raise SystemExit("mount failure no longer funnels through erofs_sb_free") -sb_free = extract_function(current["super.c"], "erofs_sb_free") -require_order( - sb_free, - [ - "z_erofs_extent_cache_fini", - "erofs_xattr_prefixes_cleanup", - "erofs_drop_internal_inodes", - "erofs_free_dev_context", - "erofs_release_device_info(&sbi->dif0)", - "free(sbi, M_EROFS)", - ], - "mount cleanup", -) -release = extract_function(current["super.c"], "erofs_release_device_info") -require_order( - release, - ["g_topology_lock", "g_vfs_close", "g_topology_unlock", "vrele", "dev_rel"], - "GEOM release", -) -if any("return (-E" in current[name] for name in current): - raise SystemExit("negative errno entered the FreeBSD B21 write set") - -result = { - "baseline": baseline, - "checksum_before_protected_fields": True, - "dead_fragments_exception_removed": True, - "feature_helpers": True, - "geom_release_preserved": True, - "internal_h_unchanged": True, - "mount_cleanup_preserved": True, - "mount_publication_preserved": True, - "positive_errno_preserved": True, - "status": "PASS", -} -output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii") -print("B21 source: exact trust-order and feature-helper transforms present") -PY -then - : -else - pre15_dut_fail 'B21 source contract failed' -fi -fi - -if python3 - "$artifacts/B21-fixture-index.json" \ - "$artifacts/oracle-first.json" "$artifacts/B21-matrix.tsv" <<'PY' -import json -from pathlib import Path -import sys - -index = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -oracle = json.loads(Path(sys.argv[2]).read_text(encoding="ascii")) -if index["case_count"] != 13 or index["legal_count"] != 1 or index["damaged_count"] != 12: - raise SystemExit("B21 fixture cardinality changed") -if oracle["status"] != "PASS" or oracle["passed_count"] != 13: - raise SystemExit("B21 oracle matrix is incomplete") -lines = ["id\tauthenticated\terrno\treject\tsha256"] -for fixture, result in zip(index["cases"], oracle["results"], strict=True): - lines.append( - f"{fixture['id']}\t{str(fixture['authenticated']).lower()}\t" - f"{result['actual_errno']}\t{result['actual_reject']}\t{fixture['sha256']}" - ) -Path(sys.argv[3]).write_text("\n".join(lines) + "\n", encoding="ascii") -print( - f"B21 fixtures: {index['case_count']} " - f"({index['legal_count']} legal, {index['damaged_count']} damaged)" -) -PY -then - : -else - pre15_runner_fail 'B21 fixture matrix summary failed' -fi - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' \ - 'B21 super gate: host PASS' \ - 'TC172 QEMU runtime NOT_RUN in host mode' - exit 0 -fi - -for tool in awk clang file nm scp tar timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B21 QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -fixture_archive=$PRE15_CASE_TMP/B21-fixtures.tar.gz -module=$PRE15_CASE_TMP/B21-erofs-zstdio0.ko -if ! timeout -k 10 600 /bin/sh "$kld_builder" "$PRE15_DUT" \ - "$PRE15_FREEBSD_SRC" "$module" "$PRE15_CASE_TMP/kld-work" 0 \ - >"$artifacts/B21-kld-build.stdout" 2>"$artifacts/B21-kld-build.stderr"; then - pre15_dut_fail 'B21 cross-target zstdio0 KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/B21-kld-file.txt" -sha256sum "$module" >"$artifacts/B21-kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/B21-kld-nm-u.txt" -tar -C "$first" -czf "$fixture_archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -if ! pre15_scp "$fixture_archive" /root/B21-fixtures.tar.gz || \ - ! pre15_scp "$qemu_probe" /root/B21-qemu-probe.c || \ - ! pre15_scp "$module" /root/B21-erofs-zstdio0.ko; then - pre15_infra_blocked 'could not transfer B21 module, probe, or fixtures' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/B21-fixtures && mkdir /root/B21-fixtures && tar -xzf /root/B21-fixtures.tar.gz -C /root/B21-fixtures && cc -O2 -Wall -Wextra -Werror -std=c17 -o /root/B21-qemu-probe /root/B21-qemu-probe.c'; then - pre15_infra_blocked 'could not prepare B21 guest fixtures/probe' -fi -if pre15_guest_ssh_bounded kldstat -q -m erofs >/dev/null 2>&1; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' -fi -if ! pre15_guest_ssh_bounded kldload /root/B21-erofs-zstdio0.ko \ - >"$artifacts/B21-kldload.stdout" 2>"$artifacts/B21-kldload.stderr"; then - if grep -q 'module already loaded or in kernel' \ - "$artifacts/B21-kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - pre15_dut_fail 'B21 exact-source zstdio0 KLD failed to load' -fi -pre15_own_guest_kld erofs 'B21 exact-source KLD' -pre15_guest_ssh_bounded dmesg >"$artifacts/B21-dmesg-before.txt" -pre15_guest_ssh_bounded mkdir -p /mnt/pre15-b21 - -b21_unown() -{ - kind=$1 - value=$2 - tmp=$PRE15_CASE_TMP/ownership.$$ - awk -F '\t' -v kind="$kind" -v value="$value" \ - '!($1 == kind && $2 == value)' "$PRE15_OWNERSHIP_FILE" >"$tmp" - mv "$tmp" "$PRE15_OWNERSHIP_FILE" -} - -printf 'id\texpected_errno\tactual_errno\treject\n' \ - >"$artifacts/B21-qemu-matrix.tsv" -while IFS="$(printf '\t')" read -r id authenticated expected_errno reject fixture_hash; do - test "$id" != id || continue - b21_md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/B21-fixtures/$id.erofs") || \ - pre15_dut_fail "$id md attach failed" - case "$b21_md" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected B21 md unit: $b21_md" ;; - esac - pre15_own_guest_md "$b21_md" "$id md" - result=$(pre15_guest_ssh_bounded /root/B21-qemu-probe \ - "/dev/$b21_md" /mnt/pre15-b21) || \ - pre15_infra_blocked "$id guest mount probe failed" - actual_errno=$(printf '%s\n' "$result" | sed -n 's/.* errno=\([0-9][0-9]*\).*/\1/p') - test -n "$actual_errno" || pre15_runner_fail "$id mount probe did not report errno" - if test "$actual_errno" = 0; then - pre15_own_guest_mount /mnt/pre15-b21 "$id mount" - fi - printf '%s\t%s\t%s\t%s\n' "$id" "$expected_errno" \ - "$actual_errno" "$reject" >>"$artifacts/B21-qemu-matrix.tsv" - if test "$actual_errno" != "$expected_errno"; then - pre15_dut_fail "$id expected errno $expected_errno, received $actual_errno" - fi - if test "$actual_errno" = 0; then - pre15_guest_ssh_bounded find /mnt/pre15-b21 -mindepth 1 -maxdepth 1 \ - -print >"$artifacts/B21-legal-readdir.txt" - pre15_guest_ssh_bounded umount /mnt/pre15-b21 || \ - pre15_dut_fail "$id unmount failed" - b21_unown guest-mount /mnt/pre15-b21 - fi - pre15_guest_ssh_bounded mdconfig -d -u "$b21_md" || \ - pre15_dut_fail "$id md detach failed" - b21_unown guest-md "$b21_md" -done <"$artifacts/B21-matrix.tsv" - -pre15_guest_ssh_bounded dmesg >"$artifacts/B21-dmesg-after.txt" -diff -u "$artifacts/B21-dmesg-before.txt" "$artifacts/B21-dmesg-after.txt" \ - >"$artifacts/B21-dmesg.diff" || true -sed -n '/^+++ /d; /^+/s/^+//p' "$artifacts/B21-dmesg.diff" \ - >"$artifacts/B21-dmesg-added.txt" -if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free' \ - "$artifacts/B21-dmesg-added.txt"; then - pre15_dut_fail 'TC172 produced panic, WITNESS, or UAF evidence' -fi -printf '%s\n' \ - 'TC172-super-trust B21 QEMU PASS' \ - '13/13 legal/damaged images returned exact mount errno with checksum trust order preserved' diff --git a/tests/pre15/cases/B22-zmap-arithmetic.sh b/tests/pre15/cases/B22-zmap-arithmetic.sh deleted file mode 100755 index 9c311b3..0000000 --- a/tests/pre15/cases/B22-zmap-arithmetic.sh +++ /dev/null @@ -1,570 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=8a8761af91654a025fd991d14da906c0b612bd7d -fixture=$PRE15_DUT/tests/pre15/fixtures/B22-zmap-arithmetic.json -kld_builder=$PRE15_DUT/tests/pre15/fixtures/B22-build-kld.sh -zmap=$PRE15_DUT/src/zmap.c -linux_zmap=$PRE15_ROOT/src-linux/zmap.c -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in cc git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B22 host tool: $tool" -done - -pre15_record_fixture b22-arithmetic "$fixture" -pre15_record_fixture b22-kld-builder "$kld_builder" -pre15_record_fixture b22-case "$PRE15_DUT/tests/pre15/cases/B22-zmap-arithmetic.sh" -pre15_record_fixture b22-zmap "$zmap" -pre15_record_fixture b22-linux-zmap "$linux_zmap" -mkdir -p "$artifacts" -pre15_target_reached - -if python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" "$fixture" \ - "$artifacts" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -fixture_path = Path(sys.argv[4]) -artifacts = Path(sys.argv[5]) -src = dut / "src" -u32_max = (1 << 32) - 1 -u64_max = (1 << 64) - 1 -eintegrity = 97 - - -def committed(path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B22 baseline {path}: {completed.stderr}") - return completed.stdout - - -def function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if match is None: - raise SystemExit(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - if brace < 0: - raise SystemExit(f"missing function body: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def add(left: int, right: int) -> tuple[str, int | None]: - value = left + right - if value > u64_max: - return ("EINTEGRITY", None) - return ("PASS", value) - - -def multiply(left: int, right: int) -> tuple[str, int | None]: - value = left * right - if value > u64_max: - return ("EINTEGRITY", None) - return ("PASS", value) - - -def model(case: dict[str, object]) -> tuple[str, int | None]: - operation = case["operation"] - if operation == "compact_pblk": - return add(int(case["base"]), int(case["nblk"])) - if operation == "index_base": - status, end = add(int(case["inode_off"]), int(case["inode_isize"])) - if status != "PASS": - return (status, None) - status, end = add(int(end), int(case.get("xattr_isize", 0))) - if status != "PASS": - return (status, None) - status, end = add(int(end), 7) - if status != "PASS": - return (status, None) - end = int(end) & ~7 - return add(end, 8) - if operation == "index_advance": - status, delta = multiply(int(case["count"]), int(case["unit"])) - if status != "PASS": - return (status, None) - return add(int(case["position"]), int(delta)) - if operation == "lcluster_count": - bits = int(case["lclusterbits"]) - if bits >= 64: - return ("EINTEGRITY", None) - size = int(case["size"]) - count = size >> bits - if size & ((1 << bits) - 1): - return add(count, 1) - return ("PASS", count) - if operation == "lcluster_pos": - bits = int(case["lclusterbits"]) - if bits >= 64: - return ("EINTEGRITY", None) - status, base = multiply(int(case["lcn"]), 1 << bits) - if status != "PASS": - return (status, None) - return add(int(base), int(case["clusterofs"])) - if operation == "lcn_advance": - return add(int(case["lcn"]), int(case["delta"])) - if operation == "physical_end": - status, pend = add(int(case["pa"]), int(case["plen"])) - if status != "PASS": - return (status, None) - limit = (1 << 48) * int(case["block_size"]) - if limit <= u64_max and int(pend) > limit: - return ("EINTEGRITY", None) - return ("PASS", None) - if operation == "post_eof": - la = int(case["la"]) - size = int(case["size"]) - if la < size: - return ("EINTEGRITY", None) - length = la - size + 1 - return ("PASS", min(length, u64_max)) - if operation == "fragment_offset": - low = int(case["low"]) - high = int(case["high"]) - if low > u32_max or high > u32_max: - return ("EINTEGRITY", None) - return ("PASS", low | (high << 32)) - raise SystemExit(f"unknown B22 operation: {operation}") - - -fixture = json.loads(fixture_path.read_text(encoding="ascii")) -if fixture.get("schema") != 1 or fixture.get("batch") != "B22": - raise SystemExit("invalid B22 fixture identity") -if fixture.get("test") != "TC170-zmap-arithmetic": - raise SystemExit("invalid B22 test identity") -if fixture.get("candidates") != [ - "P15-047", - "P15-059", - "P15-071", - "P15-079", - "P15-084", -]: - raise SystemExit("B22 candidate set changed") -if fixture.get("errno") != { - "corruption": eintegrity, - "provider_io": "unchanged", - "sign": "positive", -}: - raise SystemExit("B22 errno contract changed") - -cases = fixture.get("cases") -if not isinstance(cases, list) or not cases: - raise SystemExit("B22 fixture has no cases") -names: set[str] = set() -markers: set[str] = set() -operations: set[str] = set() -decoded: list[dict[str, object]] = [] -for case in cases: - if not isinstance(case, dict): - raise SystemExit("B22 fixture case is not an object") - name = str(case.get("name", "")) - marker = str(case.get("target_marker", "")) - if not name or name in names or not marker.startswith("TC170:"): - raise SystemExit(f"invalid B22 case identity: {name!r}") - names.add(name) - markers.add(marker) - operations.add(str(case.get("operation", ""))) - status, value = model(case) - if status != case.get("status"): - raise SystemExit(f"B22 independent status mismatch: {name}") - if status == "PASS" and value is not None and value != case.get("expected"): - raise SystemExit(f"B22 independent value mismatch: {name}") - mutated = case.get("mutated_fields") - if status == "EINTEGRITY" and ( - not isinstance(mutated, list) or len(mutated) != 1 - ): - raise SystemExit(f"B22 negative is not single-field: {name}") - if status == "PASS" and mutated != []: - raise SystemExit(f"B22 positive declares a mutation: {name}") - decoded.append({"name": name, "status": status, "value": value}) - -required_markers = { - "TC170:compact-pblk-32bit-crossing", - "TC170:index-position", - "TC170:delta-lcn", - "TC170:physical-end-48bit", - "TC170:post-eof", - "TC170:fragment-high-bits", -} -if markers != required_markers: - raise SystemExit("B22 target-marker set is incomplete") -if operations != { - "compact_pblk", - "fragment_offset", - "index_advance", - "index_base", - "lcluster_count", - "lcluster_pos", - "lcn_advance", - "physical_end", - "post_eof", -}: - raise SystemExit("B22 operation set is incomplete") - -current = (src / "zmap.c").read_text(encoding="utf-8") -base = committed("src/zmap.c") -linux = (root / "src-linux/zmap.c").read_text(encoding="utf-8") -helpers = [ - "z_erofs_index_base", - "z_erofs_index_advance", - "z_erofs_lcluster_count", - "z_erofs_lcluster_pos", - "z_erofs_lcn_advance", - "z_erofs_compact_pblk", - "z_erofs_fragment_offset", - "z_erofs_post_eof_len", - "z_erofs_physical_end", -] -for helper in helpers: - if helper in base or current.count(helper) < 2: - raise SystemExit(f"B22 helper is missing, duplicated, or pre-existing: {helper}") - -old_freebsd = ( - "m->pblk = le32dec(in + packsize - sizeof(uint32_t)) + nblk;", - "map->m_llen = map->m_la + 1 - vi->size;", - "vi->z_fragmentoff |= map->m_pa << 32;", - "lcn += m->delta[1];", - "pos += lcn << amortizedshift;", - "(pend >> sbi->blkszbits) >= (1ULL << 48)", -) -if not all(anchor in base for anchor in old_freebsd): - raise SystemExit("B22 baseline arithmetic anchors changed") -if any(anchor in current for anchor in old_freebsd): - raise SystemExit("B22 left an unchecked baseline arithmetic anchor") -linux_anchors = ( - "m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;", - "map->m_llen = map->m_la + 1 - inode->i_size;", - "vi->z_fragmentoff |= map->m_pa << 32;", - "lcn += m->delta[1];", - "(pend >> sbi->blkszbits) >= BIT_ULL(48)", -) -if not all(anchor in linux for anchor in linux_anchors): - raise SystemExit("Linux zmap semantic anchor changed") -if function(current, "z_erofs_read_index") != function(base, "z_erofs_read_index"): - raise SystemExit("B22 changed the FreeBSD metadata/provider reader") -if current.count("erofs_read_metadata") != base.count("erofs_read_metadata"): - raise SystemExit("B22 changed metadata I/O call count") -if current.count("erofs_put_metabuf") != base.count("erofs_put_metabuf"): - raise SystemExit("B22 changed metadata release call count") -for public in ("z_erofs_fill_inode", "z_erofs_map_blocks"): - old_signature = function(base, public).split("{", 1)[0] - new_signature = function(current, public).split("{", 1)[0] - if old_signature != new_signature: - raise SystemExit(f"B22 changed public zmap ABI: {public}") -if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", current): - raise SystemExit("B22 introduced Linux negative errno") - -legacy_mismatches: set[str] = set() -for case in cases: - operation = case["operation"] - status = case["status"] - expected = case.get("expected") - if operation == "compact_pblk" and status == "PASS": - old = (int(case["base"]) + int(case["nblk"])) & u32_max - if old != expected: - legacy_mismatches.add("compact-pblk") - elif operation == "index_advance" and status == "EINTEGRITY": - old = (int(case["position"]) + int(case["count"]) * int(case["unit"])) & u64_max - if old <= u64_max: - legacy_mismatches.add("index-position") - elif operation == "lcluster_pos" and status == "EINTEGRITY": - old = ((int(case["lcn"]) << int(case["lclusterbits"])) + int(case["clusterofs"])) & u64_max - if old <= u64_max: - legacy_mismatches.add("delta-lcn") - elif operation == "physical_end" and case["name"] == "physical-last-48bit-block": - pend = int(case["pa"]) + int(case["plen"]) - if (pend >> 12) >= (1 << 48): - legacy_mismatches.add("physical-end") - elif operation == "post_eof" and case["name"] == "post-eof-saturates-empty-inode": - old = ((int(case["la"]) + 1) & u64_max) - int(case["size"]) - if old != expected: - legacy_mismatches.add("post-eof") - elif operation == "fragment_offset" and status == "EINTEGRITY": - old = int(case["low"]) | ((int(case["high"]) << 32) & u64_max) - if old <= u64_max: - legacy_mismatches.add("fragment-high") -if legacy_mismatches != { - "compact-pblk", - "delta-lcn", - "fragment-high", - "index-position", - "physical-end", - "post-eof", -}: - raise SystemExit("B22 fixtures do not distinguish every legacy bug") - -helper_source = "\n\n".join(function(current, name) for name in helpers) -c_lines = [ - "#include ", - "#include ", - "#define EINTEGRITY 97", - "#define rounddown2(x, y) ((x) & ~((y) - 1))", - "typedef uint64_t erofs_off_t;", - "typedef uint64_t erofs_blk_t;", - "struct z_erofs_map_header { uint8_t bytes[8]; };", - "struct erofs_inode { erofs_off_t inode_off; unsigned int inode_isize; unsigned int xattr_isize; };", - helper_source, - "int main(void)", - "{", - "\tuint64_t value;", - "\tint error, failures = 0;", -] - - -def u64(value: object) -> str: - return f"UINT64_C({int(value)})" - - -for case in cases: - name = str(case["name"]) - operation = case["operation"] - expected_error = 0 if case["status"] == "PASS" else eintegrity - c_lines.append(f"\t/* {name} */") - if operation == "compact_pblk": - call = f"z_erofs_compact_pblk(UINT32_C({int(case['base'])}), {int(case['nblk'])}U, &value)" - elif operation == "index_base": - c_lines.extend([ - "\t{", - "\t\tstruct erofs_inode vi = {", - f"\t\t\t.inode_off = {u64(case['inode_off'])},", - f"\t\t\t.inode_isize = {int(case['inode_isize'])}U,", - f"\t\t\t.xattr_isize = {int(case.get('xattr_isize', 0))}U,", - "\t\t};", - ]) - call = "z_erofs_index_base(&vi, &value)" - elif operation == "index_advance": - c_lines.append(f"\tvalue = {u64(case['position'])};") - call = f"z_erofs_index_advance(&value, {u64(case['count'])}, {u64(case['unit'])})" - elif operation == "lcluster_count": - call = f"z_erofs_lcluster_count({u64(case['size'])}, {int(case['lclusterbits'])}U, &value)" - elif operation == "lcluster_pos": - call = f"z_erofs_lcluster_pos({u64(case['lcn'])}, {int(case['lclusterbits'])}U, {u64(case['clusterofs'])}, &value)" - elif operation == "lcn_advance": - c_lines.append(f"\tvalue = {u64(case['lcn'])};") - call = f"z_erofs_lcn_advance(&value, {u64(case['delta'])})" - elif operation == "physical_end": - call = f"z_erofs_physical_end({u64(case['pa'])}, {u64(case['plen'])}, {u64(case['block_size'])})" - elif operation == "post_eof": - call = f"z_erofs_post_eof_len({u64(case['la'])}, {u64(case['size'])}, &value)" - elif operation == "fragment_offset": - call = f"z_erofs_fragment_offset({u64(case['low'])}, {u64(case['high'])}, &value)" - else: - raise SystemExit(f"cannot generate C for operation: {operation}") - c_lines.append(f"\terror = {call};") - c_lines.append(f"\tif (error != {expected_error}) {{") - c_lines.append(f"\t\tfprintf(stderr, \"{name}: error=%d\\n\", error);") - c_lines.append("\t\tfailures++;\n\t}") - if case["status"] == "PASS" and "expected" in case: - c_lines.append(f"\tif (value != {u64(case['expected'])}) {{") - c_lines.append(f"\t\tfprintf(stderr, \"{name}: value mismatch\\n\");") - c_lines.append("\t\tfailures++;\n\t}") - if operation == "index_base": - c_lines.append("\t}") -c_lines.extend([ - "\tif (failures != 0)", - "\t\treturn (1);", - f"\tprintf(\"B22 extracted arithmetic PASS cases={len(cases)}\\n\");", - "\treturn (0);", - "}", -]) -c_path = artifacts / "B22-zmap-arithmetic.c" -binary = artifacts / "B22-zmap-arithmetic" -c_path.write_text("\n".join(c_lines) + "\n", encoding="ascii") -compiled = subprocess.run( - ["cc", "-std=gnu11", "-Wall", "-Wextra", "-Werror", str(c_path), "-o", str(binary)], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -(artifacts / "B22-compile.stdout").write_text(compiled.stdout, encoding="utf-8") -(artifacts / "B22-compile.stderr").write_text(compiled.stderr, encoding="utf-8") -if compiled.returncode != 0: - raise SystemExit("B22 extracted arithmetic compilation failed") -executed = subprocess.run( - [str(binary)], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -(artifacts / "B22-extracted.stdout").write_text(executed.stdout, encoding="utf-8") -(artifacts / "B22-extracted.stderr").write_text(executed.stderr, encoding="utf-8") -if executed.returncode != 0: - raise SystemExit("B22 extracted arithmetic execution failed") - -result = { - "status": "PASS", - "batch": "B22", - "test": "TC170-zmap-arithmetic", - "baseline": baseline, - "case_count": len(cases), - "negative_count": sum(case["status"] == "EINTEGRITY" for case in cases), - "target_markers": sorted(markers), - "legacy_mismatches": sorted(legacy_mismatches), - "freebsd_positive_errno": True, - "linux_algorithm_locations_audited": True, - "provider_io_calls_unchanged": True, - "metadata_release_calls_unchanged": True, - "decompressor_abi_unchanged": True, - "ondisk_abi_unchanged": True, - "qemu_scope": "exact-source KLD load plus extracted helper replay when PRE15_MODE=qemu", - "full_feature_suite": "NOT_RUN", -} -(artifacts / "B22-decoded-cases.json").write_text( - json.dumps(decoded, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", -) -(artifacts / "B22-result.json").write_text( - json.dumps(result, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", -) -print( - f"B22 host PASS cases={len(cases)} negatives={result['negative_count']} " - f"markers={len(markers)}" -) -PY -then - : -else - pre15_dut_fail 'B22 source, fixture, or extracted arithmetic check failed' -fi - -sha256sum "$artifacts/B22-zmap-arithmetic.c" \ - "$artifacts/B22-zmap-arithmetic" \ - "$artifacts/B22-decoded-cases.json" "$artifacts/B22-result.json" \ - > "$artifacts/SHA256SUMS" - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' \ - 'TC170 host extracted zmap arithmetic and fixture oracle PASS' \ - 'QEMU exact-source KLD load and FreeBSD helper replay NOT_RUN in host mode' - exit 0 -fi - -for tool in awk clang file nm scp; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B22 QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -module=$PRE15_CASE_TMP/B22-erofs.ko -if ! /bin/sh "$kld_builder" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work" >"$artifacts/B22-kld-build.stdout" \ - 2>"$artifacts/B22-kld-build.stderr"; then - pre15_dut_fail 'B22 cross-target KLD build failed' -fi -pre15_record_module "$module" -file "$module" > "$artifacts/B22-kld-file.txt" -sha256sum "$module" > "$artifacts/B22-kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort > "$artifacts/B22-kld-nm-u.txt" - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -if ! pre15_scp "$module" /root/B22-erofs.ko || \ - ! pre15_scp "$artifacts/B22-zmap-arithmetic.c" \ - /root/B22-zmap-arithmetic.c; then - pre15_infra_blocked 'could not transfer B22 module or arithmetic source' -fi -if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then - pre15_infra_blocked 'FreeBSD guest already has an EROFS module loaded' -fi -if ! pre15_guest_ssh_bounded kldload /root/B22-erofs.ko \ - >"$artifacts/B22-kldload.stdout" 2>"$artifacts/B22-kldload.stderr"; then - if grep -q 'module already loaded or in kernel' \ - "$artifacts/B22-kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - pre15_dut_fail 'B22 exact-source KLD failed to load' -fi -pre15_own_guest_kld erofs 'B22 exact-source KLD' -if ! pre15_guest_ssh_bounded cc -std=gnu11 -Wall -Wextra -Werror \ - /root/B22-zmap-arithmetic.c -o /root/B22-zmap-arithmetic; then - pre15_infra_blocked 'could not compile B22 arithmetic helper in the guest' -fi -if ! pre15_guest_ssh_bounded /root/B22-zmap-arithmetic \ - >"$artifacts/B22-guest-arithmetic.stdout" \ - 2>"$artifacts/B22-guest-arithmetic.stderr"; then - pre15_dut_fail 'B22 FreeBSD arithmetic helper replay failed' -fi -pre15_guest_ssh_bounded kldstat -n erofs > "$artifacts/B22-kldstat.txt" -printf '%s\n' \ - 'TC170 QEMU PASS exact-source KLD load and FreeBSD arithmetic helper replay' \ - 'No full feature suite or shared GEOM provider was used' diff --git a/tests/pre15/cases/B23-explicit-table.sh b/tests/pre15/cases/B23-explicit-table.sh deleted file mode 100644 index 1e2072a..0000000 --- a/tests/pre15/cases/B23-explicit-table.sh +++ /dev/null @@ -1,412 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=083acc65f842c409c5a2e089e50060d0062b677c -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -spec=$fixture_dir/B23-explicit-spec.json -generator=$fixture_dir/B23-explicit-generate.py -oracle=$fixture_dir/B23-explicit-oracle.py -probe=$fixture_dir/B23-explicit-probe.c -kld_builder=$fixture_dir/B23-build-kld.sh -zmap=$PRE15_DUT/src/zmap.c -linux_zmap=$PRE15_ROOT/src-linux/zmap.c -artifacts=$PRE15_RUN_DIR/artifacts -first=$PRE15_CASE_TMP/first -second=$PRE15_CASE_TMP/second - -for tool in cc diff git mkfs.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B23 host tool: $tool" -done - -pre15_record_fixture b23-spec "$spec" -pre15_record_fixture b23-generator "$generator" -pre15_record_fixture b23-oracle "$oracle" -pre15_record_fixture b23-probe "$probe" -pre15_record_fixture b23-kld-builder "$kld_builder" -pre15_record_fixture b23-case "$PRE15_DUT/tests/pre15/cases/B23-explicit-table.sh" -pre15_record_fixture b23-zmap "$zmap" -pre15_record_fixture b23-linux-zmap "$linux_zmap" -mkdir -p "$artifacts" - -if ! python3 -B "$generator" --spec "$spec" --output "$first" \ - >"$artifacts/B23-generate-first.stdout" \ - 2>"$artifacts/B23-generate-first.stderr"; then - pre15_runner_fail 'B23 first fixture generation failed' -fi -if ! python3 -B "$generator" --spec "$spec" --output "$second" \ - >"$artifacts/B23-generate-second.stdout" \ - 2>"$artifacts/B23-generate-second.stderr"; then - pre15_runner_fail 'B23 repeat fixture generation failed' -fi -if ! diff -u "$first/SHA256SUMS" "$second/SHA256SUMS" \ - >"$artifacts/B23-repeat-sums.diff" || \ - ! diff -u "$first/fixture-manifest.json" "$second/fixture-manifest.json" \ - >"$artifacts/B23-repeat-manifest.diff"; then - pre15_runner_fail 'B23 fixture generation is not byte deterministic' -fi -if ! python3 -B "$oracle" --spec "$spec" --fixtures "$first" \ - --report "$artifacts/B23-oracle-first.json" \ - >"$artifacts/B23-oracle-first.stdout" \ - 2>"$artifacts/B23-oracle-first.stderr"; then - pre15_runner_fail 'B23 independent fixture oracle failed' -fi -if ! python3 -B "$oracle" --spec "$spec" --fixtures "$second" \ - --report "$artifacts/B23-oracle-second.json" \ - >"$artifacts/B23-oracle-second.stdout" \ - 2>"$artifacts/B23-oracle-second.stderr"; then - pre15_runner_fail 'B23 repeat fixture oracle failed' -fi -cp "$first/SHA256SUMS" "$artifacts/B23-fixture-SHA256SUMS" -cp "$first/fixture-manifest.json" "$artifacts/B23-fixture-manifest.json" -pre15_record_fixture b23-generated-sums "$artifacts/B23-fixture-SHA256SUMS" - -if ! cc -std=c11 -Wall -Wextra -Werror -c "$probe" \ - -o "$PRE15_CASE_TMP/B23-explicit-probe.o" \ - >"$artifacts/B23-probe-compile.stdout" \ - 2>"$artifacts/B23-probe-compile.stderr"; then - pre15_runner_fail 'B23 guest probe does not compile on the host' -fi - -if python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" "$spec" \ - "$artifacts/B23-source-audit.json" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -spec_path = Path(sys.argv[4]) -report_path = Path(sys.argv[5]) - - -def committed(path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B23 baseline {path}: {completed.stderr}") - return completed.stdout - - -def function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if match is None: - raise SystemExit(f"missing function: {name}") - start = source.rfind("\n", 0, source.rfind("\n", 0, match.start())) + 1 - brace = source.find("{", match.end()) - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -spec = json.loads(spec_path.read_text(encoding="ascii")) -current = (dut / "src/zmap.c").read_text(encoding="utf-8") -before = committed("src/zmap.c") -validator = function(current, "z_erofs_validate_extent_table") -old_validator = function(before, "z_erofs_validate_extent_table") -if validator == old_validator: - raise SystemExit("B23 validator did not change") -if function(current, "z_erofs_read_extent") != function(before, "z_erofs_read_extent"): - raise SystemExit("B23 changed the local record decoder") -for name in ("z_erofs_map_blocks_ext", "z_erofs_fill_inode"): - if function(current, name) != function(before, name): - raise SystemExit(f"B23 changed {name}") -if "#define Z_EROFS_EXTENT_VALIDATE_CHUNK_SIZE (64 * 1024)" not in current: - raise SystemExit("B23 fixed validation chunk bound is missing") -required = ( - "z_erofs_read_extent(sbi, vi, record_pos, recsz, &ext)", - "while (scan_pos < table_end)", - "Z_EROFS_EXTENT_VALIDATE_CHUNK_SIZE", - "erofs_read_metadata(sbi, vi->nid, scan_pos, chunk_len, &buf)", - "memcpy(&ext, (const char *)buf.data + offset, recsz)", - "z_erofs_extent_lstart(&ext, recsz)", - "erofs_put_metabuf(&buf)", - "return (index == vi->z_extents ? 0 : EINTEGRITY)", -) -if not all(marker in validator for marker in required): - raise SystemExit("B23 validator is missing a bounded-scan contract marker") -if validator.count("z_erofs_read_extent(") != 1: - raise SystemExit("B23 short tail probe count changed") -if validator.count("erofs_read_metadata(") != 1: - raise SystemExit("B23 long-table reader is not chunk-scoped") -if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", validator): - raise SystemExit("B23 introduced Linux negative errno") - -allowed = { - "repo-pre-15/src/zmap.c", - "repo-pre-15/tests/pre15/cases/B23-explicit-table.sh", - "repo-pre-15/tests/pre15/fixtures/B23-build-kld.sh", - "repo-pre-15/tests/pre15/fixtures/B23-explicit-generate.py", - "repo-pre-15/tests/pre15/fixtures/B23-explicit-oracle.py", - "repo-pre-15/tests/pre15/fixtures/B23-explicit-probe.c", - "repo-pre-15/tests/pre15/fixtures/B23-explicit-spec.json", -} -changed = set( - subprocess.run( - ["git", "-C", str(root), "diff", "--name-only", baseline, "--", "repo-pre-15"], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() -) -changed.update( - subprocess.run( - [ - "git", "-C", str(root), "ls-files", "--others", - "--exclude-standard", "--", "repo-pre-15", - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() -) -if changed != allowed: - raise SystemExit(f"B23 write set differs: {sorted(changed ^ allowed)}") -for path in ( - "src/erofs_fs.h", - "src/data.c", - "src/decompressor.c", - "src/decompressor_lz4.c", - "src/decompressor_lzma.c", - "src/decompressor_deflate.c", - "src/decompressor_zstd.c", - "src/internal.h", -): - if (dut / path).read_text(encoding="utf-8") != committed(path): - raise SystemExit(f"B23 changed an excluded provider/codec/ABI file: {path}") - -linux = (root / "src-linux/zmap.c").read_text(encoding="utf-8") -for marker in ( - "recsz <= offsetof(struct z_erofs_extent, pstart_hi)", - "le64_to_cpu(*(__le64 *)ext)", - "le32_to_cpu(ext->pstart_lo)", - "le32_to_cpu(ext->pstart_hi) << 32", - "le32_to_cpu(ext->lstart_hi) << 32", - "erofs_inode_in_metabox(inode)", -): - if marker not in linux: - raise SystemExit(f"Linux explicit-table anchor changed: {marker}") - -report = { - "baseline": baseline, - "batch": "B23", - "candidates": spec["candidates"], - "chunk_size": spec["chunk_size"], - "decompressor_files_unchanged": True, - "freebsd_positive_errno": True, - "linux_record_decode_audited": True, - "ondisk_header_unchanged": True, - "primary_metabox_reader_unchanged": True, - "status": "PASS", - "test": spec["test"], - "write_set": sorted(changed), -} -report_path.write_text( - json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", -) -print("B23 source/write-set/Linux-FreeBSD audit PASS") -PY -then - : -else - pre15_dut_fail 'B23 source, write-set, or cross-tree audit failed' -fi - -sha256sum "$artifacts/B23-fixture-SHA256SUMS" \ - "$artifacts/B23-fixture-manifest.json" \ - "$artifacts/B23-oracle-first.json" \ - "$artifacts/B23-oracle-second.json" \ - "$artifacts/B23-source-audit.json" \ - >"$artifacts/SHA256SUMS" -pre15_target_reached - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' \ - 'TC171 host real-fixture, independent oracle, and bounded source audit PASS' \ - 'QEMU exact-source mapped/backing runtime NOT_RUN in host mode' \ - 'Full feature suite NOT_RUN' - exit 0 -fi - -for tool in awk clang file nm scp tar; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B23 QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -fixture_archive=$PRE15_CASE_TMP/B23-fixtures.tar.gz -module=$PRE15_CASE_TMP/B23-erofs.ko -if ! /bin/sh "$kld_builder" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work" >"$artifacts/B23-kld-build.stdout" \ - 2>"$artifacts/B23-kld-build.stderr"; then - pre15_dut_fail 'B23 cross-target zstdio0 KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/B23-kld-file.txt" -sha256sum "$module" >"$artifacts/B23-kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/B23-kld-nm-u.txt" -tar -C "$first" -czf "$fixture_archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -if ! pre15_scp "$fixture_archive" /root/B23-fixtures.tar.gz || \ - ! pre15_scp "$probe" /root/B23-explicit-probe.c || \ - ! pre15_scp "$module" /root/B23-erofs.ko; then - pre15_infra_blocked 'could not transfer B23 module, probe, or fixtures' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/pre15-b23-fixtures && mkdir /root/pre15-b23-fixtures && tar -xzf /root/B23-fixtures.tar.gz -C /root/pre15-b23-fixtures'; then - pre15_infra_blocked 'could not prepare B23 guest fixtures' -fi -if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then - pre15_infra_blocked 'FreeBSD guest already has an EROFS module loaded' -fi -if ! pre15_guest_ssh_bounded kldload /root/B23-erofs.ko \ - >"$artifacts/B23-kldload.stdout" 2>"$artifacts/B23-kldload.stderr"; then - if grep -q 'module already loaded or in kernel' "$artifacts/B23-kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - pre15_dut_fail 'B23 exact-source KLD failed to load' -fi -pre15_own_guest_kld erofs 'B23 exact-source KLD' -if ! pre15_guest_ssh_bounded cc -std=c11 -Wall -Wextra -Werror \ - -o /root/B23-explicit-probe /root/B23-explicit-probe.c; then - pre15_infra_blocked 'could not compile the B23 guest probe' -fi - -python3 -B - "$first/fixture-manifest.json" >"$PRE15_CASE_TMP/B23-qemu-cases.tsv" <<'PY' -import json -from pathlib import Path -import sys - -manifest = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -for name, case in sorted(manifest["cases"].items()): - offsets = ",".join(str(value) for value in case["probe_offsets"]) - print( - name, - case["image"], - case["expected_errno"], - case["file_size"], - offsets, - case["target_marker"], - sep="\t", - ) -PY - -pre15_attach_md() -{ - pre15_md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode -f "$1") || \ - pre15_dut_fail "could not attach B23 provider: $1" - case "$pre15_md" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected mdconfig output: $pre15_md" ;; - esac - pre15_md=${pre15_md#md} - pre15_own_guest_md "$pre15_md" "$2" - printf '%s\n' "$pre15_md" -} - -while IFS="$(printf '\t')" read -r case_id image_name expected_errno \ - file_size offsets target_marker; do - mountpoint=/mnt/pre15-b23-$case_id - unit=$(pre15_attach_md "/root/pre15-b23-fixtures/$image_name" \ - "B23 $case_id primary") - pre15_guest_ssh_bounded mkdir -p "$mountpoint" - if ! pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/md$unit" \ - "$mountpoint"; then - pre15_dut_fail "B23 $case_id mount failed before $target_marker" - fi - pre15_own_guest_mount "$mountpoint" "B23 $case_id mount" - if test "$expected_errno" -eq 0; then - old_ifs=$IFS - IFS=, - set -- $offsets - IFS=$old_ifs - if ! pre15_guest_ssh_bounded /root/B23-explicit-probe pass \ - "$mountpoint/target.bin" "$file_size" "$@"; then - pre15_dut_fail "B23 mapped positive failed at $target_marker" - fi - else - if ! pre15_guest_ssh_bounded /root/B23-explicit-probe errno \ - "$mountpoint/target.bin" "$expected_errno"; then - pre15_dut_fail "B23 corruption result failed at $target_marker" - fi - fi -done <"$PRE15_CASE_TMP/B23-qemu-cases.tsv" - -pre15_guest_ssh_bounded dmesg >"$artifacts/B23-dmesg.txt" -if grep -Eq 'panic:|Fatal trap|lock order reversal|KDB: stack backtrace' \ - "$artifacts/B23-dmesg.txt"; then - pre15_dut_fail 'B23 runtime produced a kernel diagnostic' -fi -printf '%s\n' \ - 'TC171 QEMU explicit primary/metabox mapped and corruption runtime PASS' \ - 'All errno values are positive FreeBSD ABI values; full feature suite NOT_RUN' diff --git a/tests/pre15/cases/B24-zmap-order.sh b/tests/pre15/cases/B24-zmap-order.sh deleted file mode 100755 index 5c7c876..0000000 --- a/tests/pre15/cases/B24-zmap-order.sh +++ /dev/null @@ -1,337 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=fb79d17edba7c47fae6e60b2a92771a8f3a2fcdf -zmap=$PRE15_DUT/src/zmap.c -internal=$PRE15_DUT/src/internal.h -linux_zmap=$PRE15_ROOT/src-linux/zmap.c -oracle=$PRE15_DUT/tests/pre15/fixtures/B07-map-oracle.json -gate_input=$PRE15_DUT/tests/pre15/gates/P15-006-input.json -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B24 host tool: $tool" -done - -pre15_record_fixture b24-zmap "$zmap" -pre15_record_fixture b24-internal "$internal" -pre15_record_fixture b24-linux-zmap "$linux_zmap" -pre15_record_fixture b24-map-oracle "$oracle" -pre15_record_fixture b24-gate-input "$gate_input" -pre15_record_fixture b24-case \ - "$PRE15_DUT/tests/pre15/cases/B24-zmap-order.sh" - -mkdir -p "$artifacts" -pre15_target_reached - -if python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" "$oracle" \ - "$gate_input" "$artifacts/B24-zmap-order.json" <<'PY' -from __future__ import annotations - -from collections import Counter -import hashlib -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -oracle_path = Path(sys.argv[4]) -gate_input_path = Path(sys.argv[5]) -report_path = Path(sys.argv[6]) - - -def committed(path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B24 baseline {path}: {completed.stderr}") - return completed.stdout - - -def function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if match is None: - raise SystemExit(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - if brace < 0: - raise SystemExit(f"missing function body: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -function_order = [ - "z_erofs_index_base", - "z_erofs_index_advance", - "z_erofs_lcluster_count", - "z_erofs_lcluster_pos", - "z_erofs_lcn_advance", - "z_erofs_compact_pblk", - "z_erofs_fragment_offset", - "z_erofs_post_eof_len", - "z_erofs_physical_end", - "z_erofs_read_index", - "z_erofs_load_full_lcluster", - "decode_compactedbits", - "get_compacted_la_distance", - "z_erofs_load_compact_lcluster", - "z_erofs_load_lcluster_from_disk", - "z_erofs_extent_lookback", - "z_erofs_get_extent_compressedlen", - "z_erofs_get_extent_decompressedlen", - "z_erofs_extent_add", - "z_erofs_extent_roundup", - "z_erofs_extent_table_pos", - "z_erofs_extent_record_pos", - "z_erofs_read_extent", - "z_erofs_extent_lstart", - "z_erofs_validate_extent_table", - "z_erofs_map_blocks_fo", - "z_erofs_map_blocks_ext", - "z_erofs_fill_inode", - "z_erofs_map_sanity_check", - "z_erofs_map_blocks", -] -linux_algorithm_order = [ - "z_erofs_load_full_lcluster", - "decode_compactedbits", - "get_compacted_la_distance", - "z_erofs_load_compact_lcluster", - "z_erofs_load_lcluster_from_disk", - "z_erofs_extent_lookback", - "z_erofs_get_extent_compressedlen", - "z_erofs_get_extent_decompressedlen", - "z_erofs_map_blocks_fo", - "z_erofs_map_blocks_ext", - "z_erofs_fill_inode", - "z_erofs_map_sanity_check", -] - -current = (dut / "src/zmap.c").read_text(encoding="utf-8") -before = committed("src/zmap.c") -current_internal = (dut / "src/internal.h").read_text(encoding="utf-8") -before_internal = committed("src/internal.h") -linux = (root / "src-linux/zmap.c").read_text(encoding="utf-8") - -moved_start = before.index("static int\nz_erofs_extent_add(") -moved_end = before.index("static int\nz_erofs_map_blocks_ext(", moved_start) -moved = before[moved_start:moved_end] -without_moved = before[:moved_start] + before[moved_end:] -insert_at = without_moved.index("static int\nz_erofs_map_blocks_fo(") -expected = without_moved[:insert_at] + moved + without_moved[insert_at:] -if current != expected: - raise SystemExit("zmap.c differs from the exact B24 validator-group move") - -before_functions = {name: function(before, name) for name in function_order} -current_functions = {name: function(current, name) for name in function_order} -for name in function_order: - if current_functions[name] != before_functions[name]: - raise SystemExit(f"B24 changed function text instead of moving it: {name}") - -def definition_position(source: str, name: str) -> int: - match = re.search( - r"^(?:" + re.escape(name) + r"|(?:static\s+)?" - r"[A-Za-z_][A-Za-z0-9_ *]*\b" + re.escape(name) + r")\s*\(", - source, - re.MULTILINE, - ) - if match is None: - raise SystemExit(f"missing definition position: {name}") - return match.start() - - -positions = [definition_position(current, name) for name in function_order] -if positions != sorted(positions): - raise SystemExit("B24 zmap definition order does not match the reviewed order") -for source, label in ((current, "FreeBSD"), (linux, "Linux")): - positions = [definition_position(source, name) for name in linux_algorithm_order] - if positions != sorted(positions): - raise SystemExit(f"{label} common zmap algorithm order drifted") - -def direct_calls(functions: dict[str, str]) -> Counter[tuple[str, str]]: - calls: Counter[tuple[str, str]] = Counter() - for caller, body in functions.items(): - brace = body.find("{") - for callee in re.findall(r"\b([A-Za-z_][A-Za-z0-9_]*)\s*\(", body[brace + 1 :]): - if callee not in {"if", "for", "while", "switch", "return", "sizeof"}: - calls[(caller, callee)] += 1 - return calls - - -before_calls = direct_calls(before_functions) -current_calls = direct_calls(current_functions) -if current_calls != before_calls: - raise SystemExit("B24 changed the direct-call multiset") - -global_definitions = sorted( - name for name, body in current_functions.items() if not body.startswith("static ") -) -if global_definitions != ["z_erofs_fill_inode", "z_erofs_map_blocks"]: - raise SystemExit(f"unexpected zmap global definitions: {global_definitions}") -if current_internal != before_internal: - raise SystemExit("B24 changed internal.h despite no removable zmap global") -inode_source = (dut / "src/inode.c").read_text(encoding="utf-8") -data_source = (dut / "src/data.c").read_text(encoding="utf-8") -if inode_source.count("z_erofs_fill_inode(sbi, vi)") != 1: - raise SystemExit("eager z_erofs_fill_inode no longer has its inode consumer") -if data_source.count("z_erofs_map_blocks(sbi, vi, &next)") != 1: - raise SystemExit("compressed map backend no longer has its common adapter consumer") -for prototype in ( - "int z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi);", - "int z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n" - " struct erofs_map_blocks *map);", -): - if current_internal.count(prototype) != 1: - raise SystemExit(f"required FreeBSD zmap prototype drifted: {prototype!r}") -if "int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map," not in linux: - raise SystemExit("Linux zmap iterator naming anchor changed") - -for marker in ("erofs_read_metadata(", "erofs_put_metabuf(", "goto out;"): - if current.count(marker) != before.count(marker): - raise SystemExit(f"B24 changed provider I/O or cleanup marker count: {marker}") -if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", current): - raise SystemExit("B24 introduced Linux negative errno") - -oracle = json.loads(oracle_path.read_text(encoding="ascii")) -gate_input = json.loads(gate_input_path.read_text(encoding="ascii")) -records = oracle.get("records", []) -if oracle.get("map_case_count") != 80 or len(records) != 80: - raise SystemExit("frozen G02/B07 map tuple denominator changed") -if gate_input.get("expected_device_case_count") != 13: - raise SystemExit("frozen G02/B07 device denominator changed") -tuple_bytes = b"".join(bytes.fromhex(record["tuple_hex"]) for record in records) -tuple_digest = hashlib.sha256(tuple_bytes).hexdigest() -if tuple_digest != oracle.get("tuple_bytes_sha256"): - raise SystemExit("frozen G02/B07 tuple bytes no longer match their digest") -if len({record["id"] for record in records}) != 80: - raise SystemExit("frozen G02/B07 tuple IDs are not unique") - -changed = set( - subprocess.run( - ["git", "-C", str(root), "diff", "--name-only", baseline, "--", "repo-pre-15"], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() -) -changed.update( - subprocess.run( - [ - "git", "-C", str(root), "ls-files", "--others", "--exclude-standard", - "--", "repo-pre-15", - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() -) -allowed = { - "repo-pre-15/src/zmap.c", - "repo-pre-15/tests/pre15/cases/B24-zmap-order.sh", -} -if changed != allowed: - raise SystemExit(f"B24 write set differs: {sorted(changed ^ allowed)}") - -function_digest = hashlib.sha256( - "\n\n".join(current_functions[name] for name in function_order).encode("utf-8") -).hexdigest() -report = { - "baseline": baseline, - "batch": "B24", - "candidate": "P15-016", - "direct_call_edges": sum(current_calls.values()), - "freebsd_backend_name": "z_erofs_map_blocks", - "freebsd_positive_errno": True, - "function_count": len(function_order), - "function_text_sha256": function_digest, - "global_definitions": global_definitions, - "linux_iterator_name": "z_erofs_map_blocks_iter", - "map_cases": len(records), - "device_cases": gate_input["expected_device_case_count"], - "provider_io_unchanged": True, - "metadata_cleanup_unchanged": True, - "status": "PASS", - "tuple_bytes_sha256": tuple_digest, - "write_set": sorted(changed), -} -report_path.write_text( - json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", -) -print( - "B24 order/callgraph PASS " - f"functions={len(function_order)} calls={sum(current_calls.values())} globals=2" -) -print( - "B24 map tuples PASS " - f"maps={len(records)} devices={gate_input['expected_device_case_count']} " - f"sha256={tuple_digest}" -) -PY -then - : -else - pre15_dut_fail 'B24 order, callgraph, visibility, or tuple proof failed' -fi - -sha256sum "$artifacts/B24-zmap-order.json" >"$artifacts/SHA256SUMS" -printf '%s\n' \ - 'B24 host PASS exact validator move and Linux algorithm order' \ - 'B24 host PASS frozen G02 map tuples; QEMU and full feature suite NOT_RUN' diff --git a/tests/pre15/cases/B25-codec-errors.sh b/tests/pre15/cases/B25-codec-errors.sh deleted file mode 100755 index 90e5a91..0000000 --- a/tests/pre15/cases/B25-codec-errors.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=c750850264cbc3c52ef4407aaf0c8d02ec26e973 -fixture=$PRE15_DUT/tests/pre15/fixtures/B25-codec-errors.json -oracle=$PRE15_DUT/tests/pre15/fixtures/B25-codec-oracle.py -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in cc git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B25 host tool: $tool" -done - -pre15_record_fixture b25-case "$PRE15_DUT/tests/pre15/cases/B25-codec-errors.sh" -pre15_record_fixture b25-fixture "$fixture" -pre15_record_fixture b25-oracle "$oracle" -for source in compress.h decompressor.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c zdata.c; do - pre15_record_fixture "b25-$source" "$PRE15_DUT/src/$source" -done - -mkdir -p "$artifacts" -pre15_target_reached - -if python3 -B "$oracle" --root "$PRE15_ROOT" --dut "$PRE15_DUT" \ - --baseline "$baseline" --fixture "$fixture" --artifacts "$artifacts" -then - pre15_record_fixture b25-report "$artifacts/B25-codec-errors.json" -else - pre15_dut_fail 'B25 typed codec errno oracle failed' -fi diff --git a/tests/pre15/cases/B27-stream-tail.sh b/tests/pre15/cases/B27-stream-tail.sh deleted file mode 100755 index 319505e..0000000 --- a/tests/pre15/cases/B27-stream-tail.sh +++ /dev/null @@ -1,680 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -spec=$fixture_dir/B27-stream-tail.json -generator=$fixture_dir/B27-stream-fixtures.py -probe=$fixture_dir/B27-stream-probe.c -compact_probe=$fixture_dir/B27-compact-finalization.c -kld_builder=$fixture_dir/B27-build-kld.sh -gate_script=$PRE15_DUT/tests/pre15/gates/P15-083.sh -gate_input=$PRE15_DUT/tests/pre15/gates/P15-083-input.json -artifacts=$PRE15_RUN_DIR/artifacts -gate_clone=$PRE15_CASE_TMP/gate-clone -gate_output=$PRE15_CASE_TMP/gate-output -first=$PRE15_CASE_TMP/fixtures-first -second=$PRE15_CASE_TMP/fixtures-second - -for tool in cc cmp diff dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B27 host tool: $tool" -done - -pre15_record_fixture b27-case "$PRE15_DUT/tests/pre15/cases/B27-stream-tail.sh" -pre15_record_fixture b27-spec "$spec" -pre15_record_fixture b27-generator "$generator" -pre15_record_fixture b27-probe "$probe" -pre15_record_fixture b27-compact-finalization "$compact_probe" -pre15_record_fixture b27-kld-builder "$kld_builder" -pre15_record_fixture b27-gate "$gate_script" -pre15_record_fixture b27-gate-input "$gate_input" -for source in decompressor_deflate.c decompressor_lzma.c decompressor_zstd.c \ - decompressor.c zdata.c zmap.c compress.h; do - pre15_record_fixture "b27-$source" "$PRE15_DUT/src/$source" -done -mkdir -p "$artifacts" - -if ! git clone -q --shared --no-checkout "$PRE15_ROOT" "$gate_clone" \ - >"$artifacts/gate-clone.stdout" 2>"$artifacts/gate-clone.stderr" || \ - ! git -C "$gate_clone" checkout -q --detach \ - 338ba8daf81c1461b5d28ca9c5345686e4e0f2eb \ - >"$artifacts/gate-checkout.stdout" 2>"$artifacts/gate-checkout.stderr"; then - pre15_runner_fail 'could not materialize the frozen P15-083 gate commit' -fi -frozen_gate=$gate_clone/repo-pre-15/tests/pre15/gates/P15-083.sh -if ! (umask 022 && timeout -k 10 240 "$frozen_gate" \ - --base 68bbe94c44e35d53cec8ab55d007f40b01cf0502 \ - --output "$gate_output") >"$artifacts/gate.stdout" \ - 2>"$artifacts/gate.stderr"; then - pre15_runner_fail 'frozen P15-083 gate replay failed' -fi - -if ! (umask 022 && python3 -B "$generator" --spec "$spec" --output "$first") \ - >"$artifacts/generate-first.json" 2>"$artifacts/generate-first.stderr"; then - pre15_runner_fail 'B27 first real EROFS fixture generation failed' -fi -if ! (umask 022 && python3 -B "$generator" --spec "$spec" --output "$second") \ - >"$artifacts/generate-second.json" 2>"$artifacts/generate-second.stderr"; then - pre15_runner_fail 'B27 second real EROFS fixture generation failed' -fi -if ! diff -qr "$first" "$second" >"$artifacts/fixture-repeat.diff"; then - pre15_runner_fail 'B27 real EROFS fixtures are not byte reproducible' -fi - -cp "$gate_output/result.json" "$artifacts/P15-083-result.json" -cp "$gate_output/codec-results.json" "$artifacts/P15-083-codec-results.json" -cp "$first/fixture-index.json" "$artifacts/B27-fixture-index.json" - -if ! cc -std=c11 -Wall -Wextra -Werror "$compact_probe" \ - -o "$PRE15_CASE_TMP/B27-compact-finalization" \ - >"$artifacts/B27-compact-finalization-build.stdout" \ - 2>"$artifacts/B27-compact-finalization-build.stderr" || \ - ! "$PRE15_CASE_TMP/B27-compact-finalization" \ - >"$artifacts/B27-compact-finalization.stdout" \ - 2>"$artifacts/B27-compact-finalization.stderr"; then - pre15_dut_fail 'B27 compact HEAD finalization regression failed' -fi -if ! python3 -B - "$PRE15_DUT/src/zmap.c" \ - "$PRE15_DUT/src/decompressor_zstd.c" \ - "$artifacts/B27-compact-finalization-source.json" <<'PY' -import json -from pathlib import Path -import sys - -source = Path(sys.argv[1]).read_text(encoding="utf-8") -zstd = Path(sys.argv[2]).read_text(encoding="utf-8") -rejected = """\t\t\t\tif ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { -\t\t\t\t\tif (i == 0) { -\t\t\t\t\t\terofs_put_metabuf(&buf); -\t\t\t\t\t\treturn (EINTEGRITY); -\t\t\t\t\t} -""" -accepted = """\t\t\t\tif ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { -\t\t\t\t\t--i; -\t\t\t\t\tnblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; -\t\t\t\t\tcontinue; -""" -if rejected in source or source.count(accepted) != 1: - raise SystemExit("B27 compact HEAD finalization source mismatch") -if zstd.count("\t.supports_subextent = 0,\n") != 1: - raise SystemExit("B27 Zstd ordinary subextent fallback is disabled") -Path(sys.argv[3]).write_text(json.dumps({ - "cblkcnt_first_slot": "accepted", - "linux_completion_semantics": True, - "status": "PASS", - "zstd_ordinary_subextent": False, - "zstd_partial_reference_completion": True, -}, indent=2, sort_keys=True) + "\n", encoding="ascii") -PY -then - pre15_dut_fail 'B27 compact HEAD finalization source audit failed' -fi - -if test "${PRE15_QEMU_TARGET_ONLY:-0}" = 1; then - pre15_target_reached -else -if ! python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$spec" \ - "$gate_script" "$gate_input" "$gate_output/result.json" \ - "$gate_output/codec-results.json" "$first/fixture-index.json" \ - "$artifacts/B27-source-audit.json" "$PRE15_CASE_TMP/finish-harness.c" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -spec = json.loads(Path(sys.argv[3]).read_text(encoding="ascii")) -gate_script = Path(sys.argv[4]) -gate_input = Path(sys.argv[5]) -result = json.loads(Path(sys.argv[6]).read_text(encoding="ascii")) -codec_results = json.loads(Path(sys.argv[7]).read_text(encoding="ascii")) -fixture_index = json.loads(Path(sys.argv[8]).read_text(encoding="ascii")) -report_path = Path(sys.argv[9]) -harness_path = Path(sys.argv[10]) -baseline = spec["baseline"] -implementation = "0eda2fcfc94aba1e51d3f9ee9885260969240de1" -runtime_base = "445da30f913dc292666f340588236498ea793db3" - - -def fail(message: str) -> None: - raise SystemExit(message) - - -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def committed(commit: str, name: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{commit}:repo-pre-15/src/{name}"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if completed.returncode != 0: - fail(f"cannot read B27 source {commit}:{name}: {completed.stderr.strip()}") - return completed.stdout - - -def replace_once(source: str, old: str, new: str, label: str) -> str: - count = source.count(old) - if count != 1: - fail(f"{label}: expected one baseline occurrence, found {count}") - return source.replace(old, new, 1) - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if match is None: - fail(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - depth = 0 - for index in range(brace, len(source)): - if source[index] == "{": - depth += 1 - elif source[index] == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - fail(f"unterminated function: {name}") - - -if ( - spec.get("schema") != 1 - or spec.get("batch") != "B27" - or spec.get("candidate") != "P15-083" - or spec.get("test") != "TC176-stream-runtime" -): - fail("B27 fixture identity changed") -if sha256(gate_script) != spec["gate"]["script_sha256"]: - fail("tracked P15-083 gate script differs from G04 GO") -if sha256(gate_input) != spec["gate"]["input_sha256"]: - fail("tracked P15-083 gate input differs from G04 GO") -if not ( - result.get("status") == "GO" - and result.get("b27") == "AUTHORIZED" - and result.get("cleanup") == "PASS" - and result.get("typed_errno") == "PASS" - and result.get("codecs") == {"deflate": "GO", "lzma": "GO", "zstd": "GO"} -): - fail("frozen P15-083 gate did not reproduce B27 authorization") - -observed = {item["codec"]: item for item in codec_results} -if set(observed) != set(spec["codecs"]): - fail("G04 codec result set changed") -for codec, expected in spec["codecs"].items(): - item = observed[codec] - gate = expected["gate"] - extent = expected["extent"] - if any(item["extent"][key] != extent[key] for key in ( - "leading_zero_bytes", "logical_length", "logical_offset", - "physical_length", "physical_offset", "stream_bytes" - )): - fail(f"{codec} legal leading-padding extent changed") - checks = ( - item["full"]["consumed"] == gate["full_consumed"], - item["full"]["policy_errno"] == 0, - item["partial"]["consumed"] == gate["partial_consumed"], - item["partial"]["requested_bytes"] == extent["partial_size"], - item["partial"]["policy_errno"] == 0, - item["partial"]["corrupt_policy_errno"] == 0, - item["tail"]["consumed"] == gate["tail_consumed"], - item["tail"]["policy_errno"] == 97, - item["tail"]["fsck"]["exit"] == gate["tail_fsck_exit"], - item["truncated"]["consumed"] == gate["truncated_consumed"], - item["truncated"]["policy_errno"] == 97, - item["corruption"]["full_errno"] == 97, - item["corruption"]["starts_at_stream_byte"] == expected["corruption_start"], - item["corruption"]["starts_after_partial_consumed"] is True, - ) - if not all(checks): - fail(f"{codec} G04 completion/trailing/partial contract changed") - for phase in ("full", "partial", "tail", "truncated"): - if item[phase]["cleanup"] != 1 or item[phase]["guards"] != 1: - fail(f"{codec} {phase} oracle cleanup/guard failed") - -if fixture_index.get("fixture_count") != 12 or fixture_index.get("status") != "READY": - fail("B27 runtime fixture matrix is incomplete") -classes = {(item["codec"], item["class"]) for item in fixture_index["fixtures"]} -if classes != { - (codec, kind) - for codec in spec["codecs"] - for kind in ("valid", "tail", "truncated", "corrupt") -}: - fail("B27 runtime fixture class set changed") - -base = { - name: committed(baseline, name) - for name in ( - "compress.h", "decompressor.c", "decompressor_deflate.c", - "decompressor_lzma.c", "decompressor_zstd.c", "zdata.c" - ) -} -implemented = {name: committed(implementation, name) for name in base} -for name in ("compress.h", "decompressor.c", "zdata.c"): - if implemented[name] != base[name]: - fail(f"B27 changed excluded provider/GEOM/buffer-lifetime file: {name}") - -deflate_helper = '''static int -z_erofs_deflate_finish(const struct z_erofs_decompress_req *rq, int ret, - uInt avail_in) -{ - -\tif (rq->partial_decoding) -\t\treturn (0); -\tif (ret != Z_STREAM_END || avail_in != 0) -\t\treturn (EINTEGRITY); -\treturn (0); -} - -''' -expected = replace_once( - base["decompressor_deflate.c"], - "static int\nz_erofs_deflate_decompress", - deflate_helper + "static int\nz_erofs_deflate_decompress", - "Deflate completion helper", -) -expected = replace_once( - expected, - "\telse if (error == 0 && !rq->partial_decoding &&\n" - "\t (ret != Z_STREAM_END || strm.avail_in != 0))\n" - "\t\terror = EINTEGRITY;", - "\telse if (error == 0)\n" - "\t\terror = z_erofs_deflate_finish(rq, ret, strm.avail_in);", - "Deflate completion call", -) -if implemented["decompressor_deflate.c"] != expected: - fail("Deflate differs from the exact B27 transform") - -lzma_helper = '''static int -z_erofs_lzma_finish(const struct z_erofs_decompress_req *rq, enum xz_ret ret, - size_t input_pos) -{ - -\tif (rq->partial_decoding && -\t (ret == XZ_OK || ret == XZ_STREAM_END)) -\t\treturn (0); -\tif (!rq->partial_decoding && ret == XZ_STREAM_END && -\t input_pos == rq->inputsize) -\t\treturn (0); -\treturn (z_erofs_lzma_error(ret)); -} - -''' -expected = replace_once( - base["decompressor_lzma.c"], - "static int\nz_erofs_lzma_decompress", - lzma_helper + "static int\nz_erofs_lzma_decompress", - "LZMA completion helper", -) -expected = replace_once( - expected, - "\telse if (rq->partial_decoding &&\n" - "\t (ret == XZ_OK || ret == XZ_STREAM_END))\n" - "\t\terror = 0;\n" - "\telse if (!rq->partial_decoding && ret == XZ_STREAM_END &&\n" - "\t buffer.in_pos == rq->inputsize)\n" - "\t\terror = 0;\n" - "\telse\n" - "\t\terror = z_erofs_lzma_error(ret);", - "\telse\n" - "\t\terror = z_erofs_lzma_finish(rq, ret, buffer.in_pos);", - "LZMA completion call", -) -if implemented["decompressor_lzma.c"] != expected: - fail("LZMA differs from the exact B27 transform") - -zstd_helper = '''static int -z_erofs_zstd_finish(const struct z_erofs_decompress_req *rq, size_t ret, - size_t input_pos, size_t input_size) -{ - -\tif (rq->partial_decoding) -\t\treturn (0); -\tif (ret != 0 || input_pos != input_size) -\t\treturn (EINTEGRITY); -\treturn (0); -} - -''' -expected = replace_once( - base["decompressor_zstd.c"], - "static const ZSTD_customMem zstd_erofs_alloc = {\n" - "\t.customAlloc = zstd_alloc,\n" - "\t.customFree = zstd_free,\n" - "\t.opaque = M_EROFS,\n" - "};\n\n" - "static int\nz_erofs_zstd_decompress", - "static const ZSTD_customMem zstd_erofs_alloc = {\n" - "\t.customAlloc = zstd_alloc,\n" - "\t.customFree = zstd_free,\n" - "\t.opaque = M_EROFS,\n" - "};\n\n" + zstd_helper + "static int\nz_erofs_zstd_decompress", - "Zstd completion helper", -) -expected = replace_once( - expected, - "\telse if (error == 0 && !rq->partial_decoding &&\n" - "\t (ret != 0 || input.pos != input.size))\n" - "\t\terror = EINTEGRITY;", - "\telse if (error == 0)\n" - "\t\terror = z_erofs_zstd_finish(rq, ret, input.pos, input.size);", - "Zstd completion call", -) -if implemented["decompressor_zstd.c"] != expected: - fail("Zstd differs from the exact B27 transform") - -runtime_names = ( - "decompressor_deflate.c", "decompressor_lzma.c", - "decompressor_zstd.c", "zmap.c", -) -runtime_before = {name: committed(runtime_base, name) for name in runtime_names} -current = { - name: (dut / "src" / name).read_text(encoding="utf-8") - for name in runtime_names -} -if current["decompressor_deflate.c"] != runtime_before["decompressor_deflate.c"]: - fail("B27 remediation changed Deflate beyond the accepted implementation") -if current["decompressor_lzma.c"] != runtime_before["decompressor_lzma.c"]: - fail("B27 remediation changed LZMA beyond the accepted implementation") -if current["decompressor_zstd.c"] != runtime_before["decompressor_zstd.c"]: - fail("B27 remediation Zstd completion differs from the exact transform") -expected = replace_once( - runtime_before["zmap.c"], - "\t\t\t\t\tif (i == 0) {\n" - "\t\t\t\t\t\terofs_put_metabuf(&buf);\n" - "\t\t\t\t\t\treturn (EINTEGRITY);\n" - "\t\t\t\t\t}\n", - "", - "compact HEAD first-slot CBLKCNT completion", -) -if current["zmap.c"] != expected: - fail("B27 remediation compact HEAD completion differs from the exact transform") - -changed_source = set( - subprocess.run( - ["git", "-C", str(root), "diff", "--name-only", runtime_base, "--", "repo-pre-15/src"], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() -) -if changed_source != { - "repo-pre-15/src/zmap.c", -}: - fail(f"B27 source write set changed: {sorted(changed_source)}") - -all_changed = set( - subprocess.run( - ["git", "-C", str(root), "diff", "--name-only", runtime_base, "--", "repo-pre-15"], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() -) -all_changed.update( - subprocess.run( - ["git", "-C", str(root), "ls-files", "--others", "--exclude-standard", "--", "repo-pre-15"], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() -) -expected_write_set = { - "repo-pre-15/src/zmap.c", - "repo-pre-15/tests/pre15/cases/B27-stream-tail.sh", - "repo-pre-15/tests/pre15/fixtures/B27-compact-finalization.c", -} -if all_changed != expected_write_set: - fail(f"B27 exact write set differs: {sorted(all_changed ^ expected_write_set)}") - -joined = "\n".join(current.values()) -if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", joined): - fail("negative Linux errno entered B27") -if "ZSTD_getErrorCode" in current["decompressor_zstd.c"]: - fail("B27 expanded the optional Zstd provider ABI") - -linux = { - codec: (root / f"src-linux/decompressor_{codec}.c").read_text(encoding="utf-8") - for codec in spec["codecs"] -} -for codec, marker in { - "deflate": "if (zerr == Z_STREAM_END && !rq->outputsize)", - "lzma": "xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize", - "zstd": "zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);", -}.items(): - if marker not in linux[codec]: - fail(f"Linux {codec} stream-tail anchor changed") - -harness = f'''#include -#include - -#define EINTEGRITY 97 -#define ENOMEM 12 -#define EOPNOTSUPP 95 -#define Z_STREAM_END 1 -typedef unsigned int uInt; - -enum xz_ret {{ - XZ_OK, XZ_STREAM_END, XZ_UNSUPPORTED_CHECK, XZ_MEM_ERROR, - XZ_MEMLIMIT_ERROR, XZ_FORMAT_ERROR, XZ_OPTIONS_ERROR, - XZ_DATA_ERROR, XZ_BUF_ERROR -}}; - -struct z_erofs_decompress_req {{ - int partial_decoding; - size_t inputsize; -}}; - -{extract_function(current["decompressor_lzma.c"], "z_erofs_lzma_error")} - -{extract_function(current["decompressor_deflate.c"], "z_erofs_deflate_finish")} - -{extract_function(current["decompressor_lzma.c"], "z_erofs_lzma_finish")} - -{extract_function(current["decompressor_zstd.c"], "z_erofs_zstd_finish")} - -static int failures; - -static void -check(const char *name, int actual, int expected) -{{ - if (actual != expected) {{ - fprintf(stderr, "%s: actual=%d expected=%d\\n", name, actual, expected); - failures++; - }} -}} - -int -main(void) -{{ - struct z_erofs_decompress_req full = {{ 0, 548 }}; - struct z_erofs_decompress_req partial = {{ 1, 548 }}; - - check("deflate exact", z_erofs_deflate_finish(&full, Z_STREAM_END, 0), 0); - check("deflate tail", z_erofs_deflate_finish(&full, Z_STREAM_END, 8), EINTEGRITY); - check("deflate no end", z_erofs_deflate_finish(&full, 0, 0), EINTEGRITY); - check("deflate partial", z_erofs_deflate_finish(&partial, 0, 8), 0); - check("lzma exact", z_erofs_lzma_finish(&full, XZ_STREAM_END, 548), 0); - check("lzma tail", z_erofs_lzma_finish(&full, XZ_STREAM_END, 540), EINTEGRITY); - check("lzma no end", z_erofs_lzma_finish(&full, XZ_OK, 548), EINTEGRITY); - check("lzma partial", z_erofs_lzma_finish(&partial, XZ_OK, 352), 0); - check("lzma partial error", z_erofs_lzma_finish(&partial, XZ_DATA_ERROR, 352), EINTEGRITY); - check("zstd exact", z_erofs_zstd_finish(&full, 0, 548, 548), 0); - check("zstd tail", z_erofs_zstd_finish(&full, 0, 540, 548), EINTEGRITY); - check("zstd no end", z_erofs_zstd_finish(&full, 1, 548, 548), EINTEGRITY); - check("zstd partial", z_erofs_zstd_finish(&partial, 1, 352, 548), 0); - return failures != 0; -}} -''' -harness_path.write_text(harness, encoding="ascii") - -report = { - "baseline": baseline, - "batch": "B27", - "candidate": "P15-083", - "cleanup_and_guards": True, - "codecs": sorted(spec["codecs"]), - "fixture_count": fixture_index["fixture_count"], - "freebsd_positive_errno": True, - "gate": "GO", - "legal_leading_padding_preserved": True, - "linux_tail_semantics_audited": True, - "optional_zstd_abi_preserved": True, - "provider_geom_and_buffer_lifetime_unchanged": True, - "runtime_base": runtime_base, - "status": "PASS", - "test": "TC176-stream-runtime", - "write_set": sorted(all_changed), -} -report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") -print(json.dumps(report, indent=2, sort_keys=True)) -PY -then - pre15_runner_fail 'B27 gate/source/write-set audit failed' -fi - -if ! cc -std=c11 -Wall -Wextra -Werror "$PRE15_CASE_TMP/finish-harness.c" \ - -o "$PRE15_CASE_TMP/finish-harness" >"$artifacts/finish-harness-build.stdout" \ - 2>"$artifacts/finish-harness-build.stderr" || \ - ! "$PRE15_CASE_TMP/finish-harness" >"$artifacts/finish-harness.stdout" \ - 2>"$artifacts/finish-harness.stderr"; then - pre15_dut_fail 'B27 extracted completion policy harness failed' -fi -sha256sum "$artifacts/P15-083-result.json" \ - "$artifacts/P15-083-codec-results.json" \ - "$artifacts/B27-fixture-index.json" \ - "$artifacts/B27-source-audit.json" >"$artifacts/SHA256SUMS" -fi -pre15_target_reached - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' \ - 'TC176 B27 host real LZMA/Deflate/Zstd stream-tail subset PASS' \ - 'Legal padding, trailing garbage, truncation, partial decode, corruption, positive errno, and cleanup PASS' \ - 'QEMU runtime NOT_RUN in host mode' \ - 'Full feature suite NOT_RUN' - exit 0 -fi - -for tool in awk clang file nm scp tar; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B27 QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -fixture_archive=$PRE15_CASE_TMP/B27-fixtures.tar.gz -module=$PRE15_CASE_TMP/B27-erofs-zstdio1.ko -if ! /bin/sh "$kld_builder" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work" 1 >"$artifacts/B27-kld-build.stdout" \ - 2>"$artifacts/B27-kld-build.stderr"; then - pre15_dut_fail 'B27 cross-target zstdio1 KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/B27-kld-file.txt" -sha256sum "$module" >"$artifacts/B27-kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/B27-kld-nm-u.txt" -tar -C "$first" -czf "$fixture_archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -if ! pre15_scp "$fixture_archive" /root/B27-fixtures.tar.gz || \ - ! pre15_scp "$probe" /root/B27-stream-probe.c || \ - ! pre15_scp "$module" /root/B27-erofs-zstdio1.ko; then - pre15_infra_blocked 'could not transfer B27 module, probe, or fixtures' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/B27-fixtures && mkdir /root/B27-fixtures && tar -xzf /root/B27-fixtures.tar.gz -C /root/B27-fixtures && cc -O2 -Wall -Wextra -Werror -o /root/B27-stream-probe /root/B27-stream-probe.c'; then - pre15_infra_blocked 'could not prepare B27 guest fixtures/probe' -fi -if pre15_guest_ssh_bounded kldstat -q -m erofs >/dev/null 2>&1; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' -fi -if ! pre15_guest_ssh_bounded kldload /root/B27-erofs-zstdio1.ko \ - >"$artifacts/B27-kldload.stdout" 2>"$artifacts/B27-kldload.stderr"; then - if grep -q 'module already loaded or in kernel' \ - "$artifacts/B27-kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - if grep -Eq 'link_elf|symbol|not defined' "$artifacts/B27-kldload.stderr"; then - pre15_infra_blocked 'guest kernel lacks the planned ZSTDIO provider ABI' - fi - pre15_dut_fail 'B27 exact-source zstdio1 KLD failed to load' -fi -pre15_own_guest_kld erofs 'B27 exact-source KLD' - -attach_mount() -{ - image=$1 - label=$2 - B27_MD=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/B27-fixtures/images/$image") || \ - pre15_dut_fail "$label md attach failed" - case "$B27_MD" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected B27 md unit: $B27_MD" ;; - esac - pre15_own_guest_md "$B27_MD" "$label md" - B27_MOUNT=/mnt/pre15-b27-$label - pre15_guest_ssh_bounded mkdir -p "$B27_MOUNT" - pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$B27_MD" "$B27_MOUNT" || \ - pre15_dut_fail "$label mount failed" - pre15_own_guest_mount "$B27_MOUNT" "$label mount" -} - -reference=/root/B27-fixtures/source/payload.bin -for codec in deflate lzma zstd; do - attach_mount "$codec-valid.erofs" "$codec-valid" - pre15_guest_ssh_bounded cmp "$B27_MOUNT/payload.bin" "$reference" || \ - pre15_dut_fail "$codec legal leading-padding read mismatch" - for kind in tail truncated; do - attach_mount "$codec-$kind.erofs" "$codec-$kind" - pre15_guest_ssh_bounded /root/B27-stream-probe errno \ - "$B27_MOUNT/payload.bin" 97 || \ - pre15_dut_fail "$codec $kind did not return EINTEGRITY" - done - case "$codec" in - deflate) logical_offset=137204; partial_size=3587 ;; - lzma|zstd) logical_offset=0; partial_size=4096 ;; - esac - attach_mount "$codec-corrupt.erofs" "$codec-corrupt" - pre15_guest_ssh_bounded /root/B27-stream-probe range \ - "$B27_MOUNT/payload.bin" "$reference" "$logical_offset" \ - "$partial_size" || \ - pre15_dut_fail "$codec range-before-corruption partial decode failed" - pre15_guest_ssh_bounded /root/B27-stream-probe errno \ - "$B27_MOUNT/payload.bin" 97 || \ - pre15_dut_fail "$codec full corruption did not return EINTEGRITY" -done -pre15_guest_ssh_bounded dmesg >"$artifacts/B27-dmesg.txt" -printf '%s\n' \ - 'TC176 B27 QEMU real LZMA/Deflate/Zstd stream-tail subset PASS' \ - 'valid=3 trailing=3 truncated=3 partial-before-corruption=3 full-corruption=3 errno=97' \ - 'Full feature suite NOT_RUN' diff --git a/tests/pre15/cases/B28-partial.sh b/tests/pre15/cases/B28-partial.sh deleted file mode 100755 index 7dab0e2..0000000 --- a/tests/pre15/cases/B28-partial.sh +++ /dev/null @@ -1,667 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -spec=$fixture_dir/B28-partial.json -generator=$fixture_dir/B28-partial-fixtures.py -probe=$fixture_dir/B28-partial-probe.c -kld_builder=$fixture_dir/B28-build-kld.sh -gate_script=$PRE15_DUT/tests/pre15/gates/P15-086.sh -gate_input=$PRE15_DUT/tests/pre15/gates/P15-086-input.json -gate_result=$PRE15_ROOT/planning/pre15/evidence/20260815T154915Z-G04-G05-P15-086/gate-output/result.json -gate_capabilities=$PRE15_ROOT/planning/pre15/evidence/20260815T154915Z-G04-G05-P15-086/gate-output/authorized-capabilities.json -artifacts=$PRE15_RUN_DIR/artifacts -first=$PRE15_CASE_TMP/fixtures-first -second=$PRE15_CASE_TMP/fixtures-second -action=${1:-runtime} - -case "$action" in -runtime|k2) ;; -*) pre15_fail_usage "unknown B28 action: $action" ;; -esac - -if test "$action" = k2; then - for tool in awk clang cmp diff file nm sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B28 K2 tool: $tool" - done - pre15_record_fixture b28-case "$PRE15_DUT/tests/pre15/cases/B28-partial.sh" - pre15_record_fixture b28-kld-builder "$kld_builder" - for source in compress.h decompressor.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c zdata.c; do - pre15_record_fixture "b28-$source" "$PRE15_DUT/src/$source" - done - mkdir -p "$artifacts" - for config in 0 1; do - module=$PRE15_CASE_TMP/B28-zstdio$config.ko - if ! timeout -k 10 600 /bin/sh "$kld_builder" "$PRE15_DUT" \ - "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work-$config" "$config" \ - >"$artifacts/B28-build-zstdio$config.stdout" \ - 2>"$artifacts/B28-build-zstdio$config.stderr"; then - pre15_dut_fail "B28 zstdio$config cross-target KLD build failed" - fi - file "$module" >"$artifacts/B28-zstdio$config-file.txt" - sha256sum "$module" >"$artifacts/B28-zstdio$config-sha256.txt" - nm -g "$module" | LC_ALL=C sort \ - >"$artifacts/B28-zstdio$config-nm-global.txt" - nm -u "$module" | LC_ALL=C sort \ - >"$artifacts/B28-zstdio$config-nm-u.txt" - done - awk '$1 != "U" { print $NF }' "$artifacts/B28-zstdio0-nm-global.txt" \ - >"$PRE15_CASE_TMP/B28-zstdio0-global-names.txt" - awk '$1 != "U" { print $NF }' "$artifacts/B28-zstdio1-nm-global.txt" \ - >"$PRE15_CASE_TMP/B28-zstdio1-global-names.txt" - if ! diff -u "$PRE15_CASE_TMP/B28-zstdio0-global-names.txt" \ - "$PRE15_CASE_TMP/B28-zstdio1-global-names.txt" \ - >"$artifacts/B28-config-nm-global.diff"; then - pre15_dut_fail 'B28 K2 global symbols differ by ZSTDIO configuration' - fi - if ! grep -q ' z_erofs_decompress_supports_subextent$' \ - "$artifacts/B28-zstdio0-nm-global.txt"; then - pre15_dut_fail 'B28 K2 partial capability symbol is absent' - fi - pre15_target_reached - printf '%s\n' \ - 'B28 targeted FreeBSD 15 K2 PASS' \ - 'zstdio0/zstdio1 link, defined-global parity, optional Zstd ABI, and partial capability symbol PASS' \ - 'KLD and object files removed by owned case cleanup' - exit 0 -fi - -for tool in cc cmp diff file fsck.erofs git mkfs.erofs nm python3 sha256sum tar timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B28 host tool: $tool" -done - -pre15_record_fixture b28-case "$PRE15_DUT/tests/pre15/cases/B28-partial.sh" -pre15_record_fixture b28-spec "$spec" -pre15_record_fixture b28-generator "$generator" -pre15_record_fixture b28-probe "$probe" -pre15_record_fixture b28-kld-builder "$kld_builder" -pre15_record_fixture b28-gate "$gate_script" -pre15_record_fixture b28-gate-input "$gate_input" -pre15_record_fixture b28-gate-result "$gate_result" -pre15_record_fixture b28-gate-capabilities "$gate_capabilities" -for source in compress.h decompressor.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c zdata.c; do - pre15_record_fixture "b28-$source" "$PRE15_DUT/src/$source" -done -mkdir -p "$artifacts" - -if ! timeout -k 5 120 python3 -B "$generator" --spec "$spec" \ - --output "$first" >"$artifacts/generate-first.stdout" \ - 2>"$artifacts/generate-first.stderr"; then - pre15_runner_fail 'B28 first real EROFS fixture generation failed' -fi -if ! timeout -k 5 120 python3 -B "$generator" --spec "$spec" \ - --output "$second" >"$artifacts/generate-second.stdout" \ - 2>"$artifacts/generate-second.stderr"; then - pre15_runner_fail 'B28 second real EROFS fixture generation failed' -fi -if ! diff -ru "$first" "$second" >"$artifacts/fixture-repeat.diff"; then - pre15_runner_fail 'B28 real EROFS fixtures are not byte reproducible' -fi -cp "$first/fixture-index.json" "$artifacts/B28-fixture-index.json" - -if test "${PRE15_QEMU_TARGET_ONLY:-0}" = 1; then - pre15_target_reached -else -if ! python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$spec" "$gate_result" \ - "$gate_capabilities" "$artifacts/B28-source-audit.json" \ - "$PRE15_CASE_TMP/decision-harness.c" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -spec_path = Path(sys.argv[3]) -gate_result_path = Path(sys.argv[4]) -gate_capabilities_path = Path(sys.argv[5]) -report_path = Path(sys.argv[6]) -harness_path = Path(sys.argv[7]) -spec = json.loads(spec_path.read_text(encoding="ascii")) -baseline = spec["baseline"] -implementation = "7a0d7e4a4047ac6d6130a3be5cb677cb94979fe7" -runtime_base = "d09924362eb29819bd393e1e86602eb38c7608c6" - - -def fail(message: str) -> None: - raise SystemExit(message) - - -def sha256_path(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def committed(commit: str, name: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{commit}:repo-pre-15/src/{name}"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if completed.returncode != 0: - fail(f"cannot read B28 source {commit}:{name}: {completed.stderr.strip()}") - return completed.stdout - - -def function(source: str, name: str) -> str: - marker = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if marker is None: - fail(f"missing function: {name}") - name_line = source.rfind("\n", 0, marker.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", marker.end()) - if brace < 0: - fail(f"missing function body: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - fail(f"unterminated function: {name}") - - -if ( - spec.get("schema") != 1 - or spec.get("batch") != "B28" - or spec.get("candidate") != "P15-086" - or spec.get("test") != "TC176-stream-runtime" -): - fail("B28 fixture identity changed") -for name, commit in ( - ("baseline", baseline), - ("implementation", implementation), - ("runtime base", runtime_base), -): - if subprocess.check_output( - ["git", "-C", str(root), "rev-parse", commit], text=True - ).strip() != commit: - fail(f"B28 {name} identity changed") -gate = json.loads(gate_result_path.read_text(encoding="ascii")) -capabilities = json.loads(gate_capabilities_path.read_text(encoding="ascii")) -if ( - gate.get("status") != "GO" - or gate.get("g04") != "GO" - or gate.get("g05") != "GO" - or gate.get("b28") != "AUTHORIZED" - or gate.get("resolved_base") != "6bf5724619be70f805bbe7d1ba77dd70cdced69f" -): - fail("frozen P15-086 gate no longer authorizes B28") -if capabilities != { - "full_fallback": ["zstd"], - "partial": ["deflate", "lz4", "lzma"], - "persistent_state_added": False, - "schema": 1, -}: - fail("P15-086 per-codec authorization changed") -for name, expected in ( - ("gate script", spec["gate"]["script_sha256"]), - ("gate input", spec["gate"]["input_sha256"]), - ("gate result", spec["gate"]["result_sha256"]), -): - path = { - "gate script": dut / "tests/pre15/gates/P15-086.sh", - "gate input": dut / "tests/pre15/gates/P15-086-input.json", - "gate result": gate_result_path, - }[name] - if sha256_path(path) != expected: - fail(f"{name} hash changed") - -source_names = ( - "compress.h", - "decompressor.c", - "decompressor_lz4.c", - "decompressor_lzma.c", - "decompressor_deflate.c", - "decompressor_zstd.c", - "zdata.c", -) -baseline_sources = {name: committed(baseline, name) for name in source_names} -implemented_sources = {name: committed(implementation, name) for name in source_names} -sources = { - name: (dut / "src" / name).read_text(encoding="utf-8") - for name in source_names -} -if implemented_sources["decompressor_lz4.c"] != baseline_sources["decompressor_lz4.c"]: - fail("B28 changed the already-capable LZ4 backend implementation") -if "bool supports_subextent;" not in implemented_sources["compress.h"]: - fail("B28 descriptor capability is absent") -if implemented_sources["compress.h"].count("z_erofs_decompress_supports_subextent") != 1: - fail("B28 capability prototype count changed") -if "bool supports_subextent;" not in sources["compress.h"]: - fail("current tree lost the B28 descriptor capability") -if sources["compress.h"].count("z_erofs_decompress_supports_subextent") != 1: - fail("current B28 capability prototype count changed") - -def read_descriptor_decisions(source_set: dict[str, str]) -> dict[str, bool]: - decisions = {} - for codec in ("lzma", "deflate", "zstd"): - value = re.search( - rf"const struct z_erofs_decompressor z_erofs_{codec}_decomp = \{{.*?\.supports_subextent = ([01]),", - source_set[f"decompressor_{codec}.c"], - re.DOTALL, - ) - if value is None: - fail(f"missing {codec} descriptor capability") - decisions[codec] = value.group(1) == "1" - lz4 = re.search( - r"static const struct z_erofs_decompressor z_erofs_lz4_decomp = \{.*?\.supports_subextent = ([01]),", - source_set["decompressor.c"], - re.DOTALL, - ) - if lz4 is None: - fail("missing LZ4 descriptor capability") - decisions["lz4"] = lz4.group(1) == "1" - return decisions - - -implemented_decisions = read_descriptor_decisions(implemented_sources) -if implemented_decisions != { - "lz4": True, "lzma": True, "deflate": True, "zstd": False -}: - fail(f"B28 implementation decisions differ from G04/G05: {implemented_decisions}") -descriptor_decisions = read_descriptor_decisions(sources) -if descriptor_decisions != { - "lz4": True, "lzma": True, "deflate": True, "zstd": False -}: - fail(f"current partial descriptor decisions changed: {descriptor_decisions}") -expected_zstd = committed(runtime_base, "decompressor_zstd.c") -if function(sources["decompressor_zstd.c"], "z_erofs_zstd_finish") != \ - function(expected_zstd, "z_erofs_zstd_finish"): - fail("current Zstd completion policy differs from the accepted transform") - -unchanged_zdata = ( - "z_erofs_extent_cache_match", - "z_erofs_extent_cache_copy", - "z_erofs_extent_cache_publish", - "z_erofs_extent_cache_init", - "z_erofs_extent_cache_fini", - "z_erofs_extent_cache_eligible", - "z_erofs_read_data", - "z_erofs_read_uio", -) -baseline_zdata = baseline_sources["zdata.c"] -for name in unchanged_zdata: - if function(implemented_sources["zdata.c"], name) != function(baseline_zdata, name): - fail(f"B28 changed unrelated zdata function {name}") -implemented_decode_length = function(implemented_sources["zdata.c"], "z_erofs_decode_length") -implemented_read_extent = function(implemented_sources["zdata.c"], "z_erofs_read_extent") -implemented_do_read = function(implemented_sources["zdata.c"], "z_erofs_do_read") -decode_length = function(sources["zdata.c"], "z_erofs_decode_length") -decode_extent = function(sources["zdata.c"], "z_erofs_decode_extent") -do_read = function(sources["zdata.c"], "z_erofs_do_read") -if decode_length != implemented_decode_length: - fail("current tree changed the B28 decode-length transform") -if decode_extent[decode_extent.index("{") :] != implemented_read_extent[ - implemented_read_extent.index("{") : -]: - fail("current tree changed B28 provider or decoded-buffer ownership") -for anchor in ( - "z_erofs_decompress_supports_subextent(map)", - "*partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0;", - "*decoded_len = end;", - "*partial = true;", -): - if anchor not in implemented_decode_length: - fail(f"B28 decode-length anchor missing: {anchor}") -for anchor in ( - "erofs_read_metadata(", - "erofs_read_physical(", - "erofs_put_metabuf(&buf);", - "erofs_brelse(compressed);", - "free(decoded, M_EROFS);", - "*bufp = decoded;", -): - if implemented_read_extent.count(anchor) != function(baseline_zdata, "z_erofs_read_extent").count(anchor): - fail(f"B28 changed provider or cleanup count: {anchor}") -if not ( - implemented_do_read.index("z_erofs_extent_cache_copy") - < implemented_do_read.index("z_erofs_decode_length") - < implemented_do_read.index("z_erofs_read_extent") - < implemented_do_read.index("cache_eligible && !partial") - < implemented_do_read.index("z_erofs_extent_cache_publish") -): - fail("B28 cache/decode/publication order changed") -if "partial, &decoded" not in implemented_do_read: - fail("B28 does not pass the selected partial mode to extent decode") -if not ( - do_read.index("z_erofs_decode_length") - < do_read.index("cache_eligible = !partial") - < do_read.index("z_erofs_decode_extent") -): - fail("current tree no longer excludes partial decodes from cache admission") -if "partial, &decoded" not in do_read: - fail("current tree does not pass B28 partial mode to extent decode") - -changed = set( - subprocess.check_output( - [ - "git", - "-C", - str(root), - "diff-tree", - "--no-commit-id", - "--name-only", - "-r", - implementation, - ], - text=True, - ).splitlines() -) -expected_write_set = { - "repo-pre-15/src/compress.h", - "repo-pre-15/src/decompressor.c", - "repo-pre-15/src/decompressor_deflate.c", - "repo-pre-15/src/decompressor_lzma.c", - "repo-pre-15/src/decompressor_zstd.c", - "repo-pre-15/src/zdata.c", - "repo-pre-15/tests/pre15/cases/B28-partial.sh", - "repo-pre-15/tests/pre15/fixtures/B28-build-kld.sh", - "repo-pre-15/tests/pre15/fixtures/B28-partial-fixtures.py", - "repo-pre-15/tests/pre15/fixtures/B28-partial-probe.c", - "repo-pre-15/tests/pre15/fixtures/B28-partial.json", -} -if changed != expected_write_set: - fail(f"B28 exact write set differs: {sorted(changed ^ expected_write_set)}") -if any(re.search(r"return\s*\(\s*-E[A-Z0-9_]+", source) for source in sources.values()): - fail("negative Linux errno entered B28") - -harness = r'''#include -#include -#include -#include -#define EINTEGRITY 97 -#define EOVERFLOW 84 -#define EROFS_MAP_PARTIAL_REF 0x20 -typedef uint64_t erofs_off_t; -struct erofs_map_blocks { - uint64_t m_llen; - unsigned int m_flags; - unsigned int m_algorithmformat; -}; -static bool z_erofs_decompress_supports_subextent(const struct erofs_map_blocks *map) -{ - return map->m_algorithmformat <= 2; -} -''' + decode_length + r''' -static int failures; -static void check(const char *name, struct erofs_map_blocks map, uint64_t off, - size_t want, int expected_error, size_t expected_length, bool expected_partial) -{ - size_t length = 0; - bool partial = false; - int error = z_erofs_decode_length(&map, off, want, &length, &partial); - if (error != expected_error || (!error && - (length != expected_length || partial != expected_partial))) { - fprintf(stderr, "%s error=%d length=%zu partial=%d\n", name, - error, length, partial); - failures++; - } -} -int main(void) -{ - check("lz4 prefix", (struct erofs_map_blocks){1038906, 0, 0}, 0, - 4096, 0, 4096, true); - check("lzma middle", (struct erofs_map_blocks){151552, 0, 1}, 8192, - 4096, 0, 12288, true); - check("deflate cross", (struct erofs_map_blocks){22878, 0, 2}, 3584, - 4096, 0, 7680, true); - check("zstd prefix", (struct erofs_map_blocks){151552, 0, 3}, 0, - 4096, 0, 151552, false); - check("lzma tail", (struct erofs_map_blocks){151552, 0, 1}, 147456, - 4096, 0, 151552, false); - check("zstd partial ref", (struct erofs_map_blocks){151552, - EROFS_MAP_PARTIAL_REF, 3}, 0, 4096, 0, 4096, true); - check("shifted fallback", (struct erofs_map_blocks){65536, 0, 4}, 0, - 4096, 0, 65536, false); - check("bad range", (struct erofs_map_blocks){4096, 0, 0}, 4090, - 16, EINTEGRITY, 0, false); - return failures != 0; -} -''' -harness_path.write_text(harness, encoding="ascii") -report = { - "baseline": baseline, - "batch": "B28", - "cache_hit_before_decode": True, - "candidate": "P15-086", - "descriptor_decisions": descriptor_decisions, - "frozen_gate_full_fallback": ["zstd"], - "current_full_fallback": ["shifted", "interlaced", "unknown", "zstd"], - "current_partial": ["deflate", "lz4", "lzma"], - "gate": "GO", - "implementation": implementation, - "partial_cache_publication": False, - "partial_reference_preserved": True, - "persistent_state_added": False, - "positive_errno": True, - "provider_and_cleanup_unchanged": True, - "runtime_base": runtime_base, - "status": "PASS", - "test": "TC176-stream-runtime", - "write_set": sorted(changed), -} -report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") -print(json.dumps(report, indent=2, sort_keys=True)) -PY -then - pre15_runner_fail 'B28 gate/source/write-set audit failed' -fi - -if ! cc -std=c11 -Wall -Wextra -Werror "$PRE15_CASE_TMP/decision-harness.c" \ - -o "$PRE15_CASE_TMP/decision-harness" \ - >"$artifacts/decision-harness-build.stdout" \ - 2>"$artifacts/decision-harness-build.stderr" || \ - ! "$PRE15_CASE_TMP/decision-harness" \ - >"$artifacts/decision-harness.stdout" \ - 2>"$artifacts/decision-harness.stderr"; then - pre15_dut_fail 'B28 extracted decode-length harness failed' -fi -cp "$gate_result" "$artifacts/P15-086-result.json" -cp "$gate_capabilities" "$artifacts/P15-086-capabilities.json" -sha256sum "$artifacts/P15-086-result.json" \ - "$artifacts/P15-086-capabilities.json" \ - "$artifacts/B28-fixture-index.json" \ - "$artifacts/B28-source-audit.json" >"$artifacts/SHA256SUMS" -fi -pre15_target_reached - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' \ - 'TC176 B28 host partial-subextent subset PASS' \ - 'B28 LZ4/LZMA/Deflate partial policy and Zstd full fallback, cache exclusion, arithmetic, real fixtures, and cleanup PASS' \ - 'QEMU runtime NOT_RUN in host mode' \ - 'Full feature suite NOT_RUN' - exit 0 -fi - -for tool in awk clang scp; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B28 QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -fixture_archive=$PRE15_CASE_TMP/B28-fixtures.tar.gz -module=$PRE15_CASE_TMP/B28-erofs-zstdio1.ko -if ! /bin/sh "$kld_builder" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work" 1 >"$artifacts/B28-kld-build.stdout" \ - 2>"$artifacts/B28-kld-build.stderr"; then - pre15_dut_fail 'B28 cross-target zstdio1 KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/B28-kld-file.txt" -sha256sum "$module" >"$artifacts/B28-kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/B28-kld-nm-u.txt" -tar -C "$first" -czf "$fixture_archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -pre15_guest_ssh_bounded() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" ssh -n -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -p "$PRE15_QEMU_SSH_PORT" \ - "$PRE15_QEMU_SSH_USER@127.0.0.1" "$@" -} - -if ! pre15_scp "$fixture_archive" /root/B28-fixtures.tar.gz || \ - ! pre15_scp "$probe" /root/B28-partial-probe.c || \ - ! pre15_scp "$module" /root/B28-erofs-zstdio1.ko; then - pre15_infra_blocked 'could not transfer B28 module, probe, or fixtures' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/B28-fixtures && mkdir /root/B28-fixtures && tar -xzf /root/B28-fixtures.tar.gz -C /root/B28-fixtures && cc -O2 -Wall -Wextra -Werror -o /root/B28-partial-probe /root/B28-partial-probe.c'; then - pre15_infra_blocked 'could not prepare B28 guest fixtures/probe' -fi -if pre15_guest_ssh_bounded kldstat -q -m erofs >/dev/null 2>&1; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' -fi -if ! pre15_guest_ssh_bounded kldload /root/B28-erofs-zstdio1.ko \ - >"$artifacts/B28-kldload.stdout" 2>"$artifacts/B28-kldload.stderr"; then - if grep -q 'module already loaded or in kernel' \ - "$artifacts/B28-kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - if grep -Eq 'link_elf|symbol|not defined' "$artifacts/B28-kldload.stderr"; then - pre15_infra_blocked 'guest kernel lacks the planned ZSTDIO provider ABI' - fi - pre15_dut_fail 'B28 exact-source zstdio1 KLD failed to load' -fi -pre15_own_guest_kld B28-erofs-zstdio1.ko 'B28 exact-source KLD' - -attach_mount() -{ - image=$1 - label=$2 - B28_MD=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/B28-fixtures/images/$image") || \ - pre15_dut_fail "$label md attach failed" - case "$B28_MD" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected B28 md unit: $B28_MD" ;; - esac - pre15_own_guest_md "$B28_MD" "$label md" - B28_MOUNT=/mnt/pre15-b28-$label - pre15_guest_ssh_bounded mkdir -p "$B28_MOUNT" - pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$B28_MD" "$B28_MOUNT" || \ - pre15_dut_fail "$label mount failed" - pre15_own_guest_mount "$B28_MOUNT" "$label mount" -} - -valid_assertions=0 -corrupt_range_assertions=0 -full_corruption_assertions=0 -while IFS='|' read -r codec source logical_offset logical_length decision; do - reference=/root/B28-fixtures/sources/$source/payload.bin - attach_mount "$codec-valid.erofs" "$codec-valid" - for range in '0 4096' '3584 4096' '8192 4096'; do - set -- $range - offset=$((logical_offset + $1)) - pre15_guest_ssh_bounded /root/B28-partial-probe range \ - "$B28_MOUNT/payload.bin" "$reference" "$offset" "$2" || \ - pre15_dut_fail "$codec valid subextent mismatch at $offset" - valid_assertions=$((valid_assertions + 1)) - done - tail_offset=$((logical_offset + logical_length - 4096)) - pre15_guest_ssh_bounded /root/B28-partial-probe range \ - "$B28_MOUNT/payload.bin" "$reference" "$tail_offset" 4096 || \ - pre15_dut_fail "$codec valid tail fallback mismatch" - valid_assertions=$((valid_assertions + 1)) - - attach_mount "$codec-corrupt.erofs" "$codec-corrupt" - if test "$decision" = GO; then - pre15_guest_ssh_bounded /root/B28-partial-probe range \ - "$B28_MOUNT/payload.bin" "$reference" "$logical_offset" 4096 || \ - pre15_dut_fail "$codec range-before-corruption partial decode failed" - else - pre15_guest_ssh_bounded /root/B28-partial-probe range-errno \ - "$B28_MOUNT/payload.bin" "$logical_offset" 4096 97 || \ - pre15_dut_fail "$codec did not use exact full fallback" - fi - corrupt_range_assertions=$((corrupt_range_assertions + 1)) - pre15_guest_ssh_bounded /root/B28-partial-probe full-errno \ - "$B28_MOUNT/payload.bin" 97 || \ - pre15_dut_fail "$codec full corruption did not return EINTEGRITY" - full_corruption_assertions=$((full_corruption_assertions + 1)) -done <<'EOF' -deflate|stream|114326|22878|GO -lz4|lz4|0|1038906|GO -lzma|stream|0|151552|GO -zstd|stream|0|151552|FULL_FALLBACK -EOF -if test "$valid_assertions" -ne 16 || \ - test "$corrupt_range_assertions" -ne 4 || \ - test "$full_corruption_assertions" -ne 4; then - pre15_runner_fail "B28 assertion count mismatch: valid=$valid_assertions corrupt-range=$corrupt_range_assertions full-corruption=$full_corruption_assertions" -fi - -pre15_guest_ssh_bounded dmesg >"$artifacts/B28-dmesg.txt" -printf '%s\n' \ - 'TC176 B28 QEMU partial-subextent subset PASS' \ - 'valid-ranges=16 partial-before-corruption=4 full-corruption=4 errno=97' \ - 'Full feature suite NOT_RUN' diff --git a/tests/pre15/cases/B29-context-pool.sh b/tests/pre15/cases/B29-context-pool.sh deleted file mode 100644 index 9c3ab7a..0000000 --- a/tests/pre15/cases/B29-context-pool.sh +++ /dev/null @@ -1,1247 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -case_script=$PRE15_DUT/tests/pre15/cases/B29-context-pool.sh -kld_builder=$PRE15_DUT/tests/pre15/fixtures/B28-build-kld.sh -old_gate=$PRE15_DUT/tests/pre15/gates/P15-076.sh -old_input=$PRE15_DUT/tests/pre15/gates/P15-076-input.json -old_evidence=$PRE15_ROOT/planning/pre15/evidence/20260815T171051Z-G05-P15-076 -checkpoint=$PRE15_ROOT/planning/pre15/evidence/20260817T153426Z-B29/checkpoint.md -artifacts=$PRE15_RUN_DIR/artifacts -fixtures=$PRE15_CASE_TMP/fixtures -action=${1:-runtime} - -case "$action" in -runtime|k2) ;; -*) pre15_fail_usage "unknown B29 action: $action" ;; -esac - -for tool in awk cc cmp file fsck.erofs git mkfs.erofs nm python3 sha256sum \ - tar timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B29 tool: $tool" -done -for fixture in "$case_script" "$kld_builder" "$old_gate" "$old_input" \ - "$old_evidence/gate-output/benchmark-results.json" \ - "$old_evidence/gate-output/state-model-result.json" \ - "$old_evidence/VERDICT.md" "$checkpoint"; do - pre15_record_fixture "b29-$(basename "$fixture")" "$fixture" -done -for source in internal.h super.c decompressor_lzma.c decompressor_deflate.c \ - decompressor_zstd.c; do - pre15_record_fixture "b29-$source" "$PRE15_DUT/src/$source" -done -for source in decompressor_lzma.c decompressor_deflate.c \ - decompressor_zstd.c; do - pre15_record_fixture "b29-linux-$source" "$PRE15_ROOT/src-linux/$source" -done -mkdir -p "$artifacts" - -if test "$action" = k2; then - for config in 0 1; do - module=$PRE15_CASE_TMP/B29-zstdio$config.ko - if ! timeout -k 10 600 /bin/sh "$kld_builder" "$PRE15_DUT" \ - "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work-$config" "$config" \ - >"$artifacts/B29-build-zstdio$config.stdout" \ - 2>"$artifacts/B29-build-zstdio$config.stderr"; then - pre15_dut_fail "B29 zstdio$config exact-ABI KLD build failed" - fi - file "$module" >"$artifacts/B29-zstdio$config-file.txt" - sha256sum "$module" >"$artifacts/B29-zstdio$config-sha256.txt" - nm -g "$module" | LC_ALL=C sort \ - >"$artifacts/B29-zstdio$config-nm-global.txt" - nm -u "$module" | LC_ALL=C sort \ - >"$artifacts/B29-zstdio$config-nm-u.txt" - done - awk '$1 != "U" { print $NF }' "$artifacts/B29-zstdio0-nm-global.txt" \ - >"$PRE15_CASE_TMP/B29-zstdio0-global-names.txt" - awk '$1 != "U" { print $NF }' "$artifacts/B29-zstdio1-nm-global.txt" \ - >"$PRE15_CASE_TMP/B29-zstdio1-global-names.txt" - if ! diff -u "$PRE15_CASE_TMP/B29-zstdio0-global-names.txt" \ - "$PRE15_CASE_TMP/B29-zstdio1-global-names.txt" \ - >"$artifacts/B29-config-nm-global.diff"; then - pre15_dut_fail 'B29 defined globals differ by ZSTDIO configuration' - fi - awk '$NF ~ /^ZSTD_/ { print $NF }' "$artifacts/B29-zstdio1-nm-u.txt" \ - >"$PRE15_CASE_TMP/B29-zstd-symbols.txt" - printf '%s\n' ZSTD_DCtx_setParameter ZSTD_createDCtx_advanced \ - ZSTD_decompressStream ZSTD_freeDCtx ZSTD_isError \ - >"$PRE15_CASE_TMP/B29-zstd-expected.txt" - if ! cmp "$PRE15_CASE_TMP/B29-zstd-expected.txt" \ - "$PRE15_CASE_TMP/B29-zstd-symbols.txt"; then - pre15_dut_fail 'B29 changed the accepted five-symbol Zstd provider ABI' - fi - pre15_target_reached - printf '%s\n' \ - 'B29 FreeBSD 15 exact-ABI K2 PASS' \ - 'zstdio0/zstdio1 compile, link, global-symbol parity, and five-symbol optional Zstd ABI PASS' - exit 0 -fi - -if ! python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" \ - "$old_evidence" "$artifacts/B29-source-audit.json" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -freebsd = Path(sys.argv[3]) -evidence = Path(sys.argv[4]) -report_path = Path(sys.argv[5]) - - -def fail(message: str) -> None: - raise SystemExit(message) - - -current = { - name: (dut / "src" / name).read_text(encoding="utf-8") - for name in ( - "internal.h", - "super.c", - "decompressor_lzma.c", - "decompressor_deflate.c", - "decompressor_zstd.c", - ) -} -linux = { - name: (root / "src-linux" / name).read_text(encoding="utf-8") - for name in ( - "decompressor_lzma.c", - "decompressor_deflate.c", - "decompressor_zstd.c", - ) -} -benchmark = json.loads( - (evidence / "gate-output/benchmark-results.json").read_text(encoding="ascii") -) -old_state = json.loads( - (evidence / "gate-output/state-model-result.json").read_text(encoding="ascii") -) -old_verdict = (evidence / "VERDICT.md").read_text(encoding="utf-8") - -assertions = 0 - - -def require(condition: bool, message: str) -> None: - global assertions - assertions += 1 - if not condition: - fail(message) - - -for codec, record in benchmark.items(): - require(record["baseline_context_allocator_share_percent"] >= 10.0, - f"{codec} old allocator-share benefit regressed") - require(record["allocator_event_reduction_percent"] >= 25.0, - f"{codec} old lifecycle benefit regressed") -require(old_state["pool_exhaustion_fallback"] == "fresh baseline allocation", - "historical unbounded fallback record changed") -require("global-exhaustion scenario" in old_verdict, - "historical global-exhaustion rejection is no longer explicit") -require("maximum window log 16" in old_verdict and "8 MiB" in old_verdict, - "historical maximum-parameter gap is no longer explicit") - -super_source = current["super.c"] -hard_caps = { - "EROFS_STREAM_MOUNT_HARD_BUDGET": "32UL * 1024 * 1024", - "EROFS_STREAM_GLOBAL_HARD_BUDGET": "128UL * 1024 * 1024", - "EROFS_STREAM_ALLOCATION_HARD_MAX": "4UL * 1024 * 1024", - "EROFS_STREAM_MOUNT_HARD_CONTEXTS": "16U", - "EROFS_STREAM_GLOBAL_HARD_CONTEXTS": "64U", - "EROFS_STREAM_MOUNT_CACHED_PER_CODEC": "2U", - "EROFS_STREAM_GLOBAL_CACHED_PER_CODEC": "16U", -} -for name, value in hard_caps.items(): - require(f"#define {name} ({value})" in super_source, - f"missing production hard cap {name}") -for token in ( - 'TUNABLE_ULONG("vfs.erofs.stream_pool.mount_budget"', - 'TUNABLE_ULONG("vfs.erofs.stream_pool.global_budget"', - 'TUNABLE_INT("vfs.erofs.stream_pool.global_contexts"', - "MIN(erofs_stream_mount_budget,", - "MIN(erofs_stream_global_budget,", - "uma_zone_set_max(erofs_stream_zone,", - "erofs_stream_reserve_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE)", - "++erofs_stream_fallbacks", - "++erofs_stream_exhaustions", - "EVENTHANDLER_REGISTER(vm_lowmem", - "while (!erofs_stream_pool_empty_locked(&sbi->stream_pool))", - "KASSERT(erofs_stream_global_contexts == 0", -): - require(token in super_source, f"missing bounded pool token: {token}") -require(super_source.find("erofs_stream_reserve_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE)") < - super_source.find("uma_zalloc(erofs_stream_zone"), - "wrapper allocation occurs before hard-budget reservation") -require("fresh baseline allocation" not in super_source, - "production retained the historical unbounded fallback") - -internal = current["internal.h"] -for field in ("resident_bytes", "contexts", "cached", "borrowed", "idle_count", - "closing", "stream_pool_initialized"): - require(re.search(rf"\b{field}\b", internal) is not None, - f"pool state omits {field}") - -lzma = current["decompressor_lzma.c"] -deflate = current["decompressor_deflate.c"] -zstd = current["decompressor_zstd.c"] -case_source = (dut / "tests/pre15/cases/B29-context-pool.sh").read_text( - encoding="utf-8" -) -require("xz_dec_microlzma_alloc(XZ_SINGLE" in lzma, - "FreeBSD LZMA no longer uses the bounded XZ_SINGLE mode") -require("ctx->state_bytes = sizeof(*ctx->state);" in lzma, - "LZMA fixed state is not charged exactly") -require("dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE" in lzma, - "LZMA 8 MiB format hard limit is absent") -require("M_WAITOK" not in lzma, - "LZMA backend still has an unbounded waiting allocation") -require("z_erofs_stream_ctx_alloc(pool, bytes)" in deflate, - "Deflate backend allocations bypass hard accounting") -require("inflateReset2(strm" in deflate, - "Deflate reusable context is not reset") -require("z_erofs_stream_ctx_alloc(opaque, size)" in zstd, - "Zstd backend allocations bypass hard accounting") -require("zstd->windowlog > 10" in zstd and - "rq->sbi->zstd_windowlog + 10 > 20" in zstd, - "Zstd maximum window parameter is not enforced") -require("ZSTD_DCtx_reset" not in zstd, - "B29 added a new Zstd provider ABI symbol") -require("error == 0 && !rq->partial_decoding" in zstd, - "partial Zstd state is incorrectly cached without reset") -require('"$B29_BAD_MOUNT" 45' in case_source, - "B29 runtime uses the Linux EOPNOTSUPP value instead of FreeBSD 45") -require("run_exhaustion_phase lzma byte" in case_source and - "run_exhaustion_phase zstd byte" in case_source and - "run_exhaustion_phase lzma count" in case_source, - "B29 runtime omits a production LZMA/Zstd byte or global count gate") -for source in (deflate, zstd): - require("M_WAITOK" not in source, - "codec backend retained an unbounded waiting allocation") - -xz_source = (freebsd / "sys/contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c").read_text( - encoding="utf-8" -) -xz_private = (freebsd / "sys/contrib/xz-embedded/linux/lib/xz/xz_private.h").read_text( - encoding="utf-8" -) -require("if (DEC_IS_MULTI(mode))" in xz_source and - "s->s.dict.buf = vmalloc(dict_size);" in xz_source, - "FreeBSD MicroLZMA allocation branch changed") -require("#\tdefine DEC_IS_MULTI(mode) ((mode) != XZ_SINGLE)" in xz_private, - "FreeBSD XZ_SINGLE dictionary-allocation proof changed") - -for codec, tokens in { - "lzma": ("get an available lzma context", "push back LZMA stream context"), - "deflate": ("get an available DEFLATE context", "push back DEFLATE stream context"), - "zstd": ("get an available ZSTD context", "push back ZSTD stream context"), -}.items(): - for token in tokens: - require(token in linux[f"decompressor_{codec}.c"], - f"Linux reusable-stream intent changed for {codec}") - -status = subprocess.run( - ["git", "-C", str(root), "status", "--porcelain"], - check=True, text=True, stdout=subprocess.PIPE, -).stdout.splitlines() -allowed_implementation = { - "repo-pre-15/src/decompressor_lzma.c", - "repo-pre-15/src/decompressor_deflate.c", - "repo-pre-15/src/decompressor_zstd.c", - "repo-pre-15/src/internal.h", - "repo-pre-15/src/super.c", - "repo-pre-15/tests/pre15/cases/B29-context-pool.sh", -} -for line in status: - path = line[3:] - require(path in allowed_implementation or path.startswith("planning/pre15/evidence/20260817T153426Z-B29/") or - path in { - "planning/pre15/execution-status.md", - "planning/pre15/final-review/closure.tsv", - "planning/pre15/final-review/completion-audit.md", - "planning/pre15/final-review/completion-audit.tsv", - }, f"B29 dirty path escapes allowed scope: {path}") - -report = { - "assertions": assertions, - "batch": "B29", - "benefit_gate_retained": True, - "freebsd_difference": { - "allocator": "UMA wrapper plus exact mount/global byte charging", - "lzma": "XZ_SINGLE fixed state; no dictionary buffer allocation", - "locking": "one FreeBSD mutex, no Linux waitqueue/per-CPU API copy", - }, - "global_exhaustion_required": True, - "hard_caps": hard_caps, - "linux_reusable_stream_intent": True, - "status": "PASS", -} -report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") -print(json.dumps(report, indent=2, sort_keys=True)) -PY -then - pre15_dut_fail 'B29 benefit/source/hard-cap audit failed' -fi - -cat >"$PRE15_CASE_TMP/B29-pool-oracle.c" <<'EOF' -#include -#include -#include -#include -#include -#include -#include -#include - -#define WRAPPER_BYTES 256U -#define MOUNT_CACHED 2U -#define GLOBAL_CACHED 16U - -struct budget { - pthread_mutex_t lock; - pthread_cond_t cv; - size_t bytes; - size_t byte_limit; - unsigned contexts; - unsigned context_limit; - unsigned cached; - unsigned borrowed; - unsigned exhaustions; -}; - -struct mount_pool { - struct budget *global; - size_t bytes; - size_t byte_limit; - unsigned contexts; - unsigned context_limit; - unsigned cached; - unsigned borrowed; - unsigned idle; - bool closing; -}; - -struct context { - struct mount_pool *mount; - bool cached; -}; - -static unsigned assertions; - -static void -check(bool condition, const char *message) -{ - ++assertions; - if (!condition) { - fprintf(stderr, "B29 oracle failure: %s\n", message); - exit(1); - } -} - -static void -budget_init(struct budget *global, size_t bytes, unsigned contexts) -{ - memset(global, 0, sizeof(*global)); - pthread_mutex_init(&global->lock, NULL); - pthread_cond_init(&global->cv, NULL); - global->byte_limit = bytes; - global->context_limit = contexts; -} - -static void -budget_fini(struct budget *global) -{ - pthread_cond_destroy(&global->cv); - pthread_mutex_destroy(&global->lock); -} - -static void -mount_init(struct mount_pool *mount, struct budget *global, size_t bytes, - unsigned contexts) -{ - memset(mount, 0, sizeof(*mount)); - mount->global = global; - mount->byte_limit = bytes; - mount->context_limit = contexts; -} - -static int -acquire(struct mount_pool *mount, bool inject_failure, struct context **out) -{ - struct budget *global = mount->global; - struct context *ctx; - bool cached; - - *out = NULL; - pthread_mutex_lock(&global->lock); - if (mount->closing) { - pthread_mutex_unlock(&global->lock); - return ENXIO; - } - if (mount->idle != 0) { - --mount->idle; - ++mount->borrowed; - ++global->borrowed; - pthread_mutex_unlock(&global->lock); - ctx = calloc(1, sizeof(*ctx)); - check(ctx != NULL, "idle wrapper allocation"); - ctx->mount = mount; - ctx->cached = true; - *out = ctx; - return 0; - } - cached = mount->cached < MOUNT_CACHED && global->cached < GLOBAL_CACHED; - if (mount->contexts >= mount->context_limit || - global->contexts >= global->context_limit || - mount->bytes > mount->byte_limit - WRAPPER_BYTES || - global->bytes > global->byte_limit - WRAPPER_BYTES) { - ++global->exhaustions; - pthread_mutex_unlock(&global->lock); - return ENOMEM; - } - mount->bytes += WRAPPER_BYTES; - global->bytes += WRAPPER_BYTES; - ++mount->contexts; - ++global->contexts; - ++mount->borrowed; - ++global->borrowed; - if (cached) { - ++mount->cached; - ++global->cached; - } - pthread_mutex_unlock(&global->lock); - if (inject_failure) { - pthread_mutex_lock(&global->lock); - mount->bytes -= WRAPPER_BYTES; - global->bytes -= WRAPPER_BYTES; - --mount->contexts; - --global->contexts; - --mount->borrowed; - --global->borrowed; - if (cached) { - --mount->cached; - --global->cached; - } - pthread_cond_broadcast(&global->cv); - pthread_mutex_unlock(&global->lock); - return ENOMEM; - } - ctx = calloc(1, sizeof(*ctx)); - check(ctx != NULL, "new wrapper allocation"); - ctx->mount = mount; - ctx->cached = cached; - *out = ctx; - return 0; -} - -static void -release(struct context *ctx, bool reusable) -{ - struct mount_pool *mount = ctx->mount; - struct budget *global = mount->global; - - pthread_mutex_lock(&global->lock); - if (reusable && ctx->cached && !mount->closing) { - --mount->borrowed; - --global->borrowed; - ++mount->idle; - pthread_cond_broadcast(&global->cv); - pthread_mutex_unlock(&global->lock); - free(ctx); - return; - } - --mount->borrowed; - --global->borrowed; - --mount->contexts; - --global->contexts; - mount->bytes -= WRAPPER_BYTES; - global->bytes -= WRAPPER_BYTES; - if (ctx->cached) { - --mount->cached; - --global->cached; - } - pthread_cond_broadcast(&global->cv); - pthread_mutex_unlock(&global->lock); - free(ctx); -} - -static void -drain(struct mount_pool *mount) -{ - struct budget *global = mount->global; - - pthread_mutex_lock(&global->lock); - mount->closing = true; - while (mount->idle != 0) { - --mount->idle; - --mount->contexts; - --global->contexts; - --mount->cached; - --global->cached; - mount->bytes -= WRAPPER_BYTES; - global->bytes -= WRAPPER_BYTES; - } - while (mount->contexts != 0) - pthread_cond_wait(&global->cv, &global->lock); - pthread_mutex_unlock(&global->lock); -} - -struct acquire_result { - struct mount_pool *mount; - int error; -}; - -static void * -acquire_worker(void *opaque) -{ - struct acquire_result *result = opaque; - struct context *ctx; - - result->error = acquire(result->mount, false, &ctx); - if (result->error == 0) - release(ctx, false); - return NULL; -} - -struct delayed_release { - struct context *ctx; -}; - -static void * -release_worker(void *opaque) -{ - struct delayed_release *delayed = opaque; - struct timespec delay = { .tv_nsec = 50000000 }; - - nanosleep(&delay, NULL); - release(delayed->ctx, true); - return NULL; -} - -int -main(void) -{ - struct acquire_result results[2]; - struct delayed_release delayed; - struct mount_pool a, b, c; - struct budget global; - struct context *one, *two, *three; - pthread_t threads[2]; - unsigned before; - - budget_init(&global, 4096, 8); - mount_init(&a, &global, 4096, 8); - check(acquire(&a, false, &one) == 0, "first cached acquire"); - release(one, true); - check(a.idle == 1 && a.cached == 1 && global.contexts == 1, - "first release did not cache"); - check(acquire(&a, false, &one) == 0, "idle reuse acquire"); - check(a.idle == 0 && a.borrowed == 1, "idle reuse counters"); - release(one, true); - drain(&a); - check(global.contexts == 0 && global.bytes == 0, "reuse drain leaked"); - budget_fini(&global); - - budget_init(&global, 4096, 8); - mount_init(&a, &global, 4096, 8); - check(acquire(&a, false, &one) == 0, "fallback first acquire"); - check(acquire(&a, false, &two) == 0, "fallback second acquire"); - check(acquire(&a, false, &three) == 0, "bounded fallback acquire"); - check(one->cached && two->cached && !three->cached, - "cache saturation did not select bounded temporary context"); - release(one, true); - release(two, true); - release(three, true); - check(a.contexts == 2 && a.idle == 2 && global.contexts == 2, - "temporary fallback was retained or cached contexts were lost"); - drain(&a); - check(global.contexts == 0 && global.bytes == 0, "fallback drain leaked"); - budget_fini(&global); - - budget_init(&global, 4096, 1); - mount_init(&a, &global, 4096, 8); - mount_init(&b, &global, 4096, 8); - mount_init(&c, &global, 4096, 8); - check(acquire(&a, false, &one) == 0, "global owner acquire"); - release(one, true); - before = global.exhaustions; - results[0].mount = &b; - results[1].mount = &c; - check(pthread_create(&threads[0], NULL, acquire_worker, &results[0]) == 0, - "first exhaustion worker create"); - check(pthread_create(&threads[1], NULL, acquire_worker, &results[1]) == 0, - "second exhaustion worker create"); - check(pthread_join(threads[0], NULL) == 0, "first exhaustion worker join"); - check(pthread_join(threads[1], NULL) == 0, "second exhaustion worker join"); - check(results[0].error == ENOMEM && results[1].error == ENOMEM, - "concurrent global exhaustion errno is unstable"); - check(global.exhaustions == before + 2 && global.contexts == 1 && - global.bytes == WRAPPER_BYTES, - "global exhaustion changed retained ownership"); - drain(&a); - check(acquire(&b, false, &one) == 0, "recovery acquire failed"); - release(one, true); - drain(&b); - drain(&c); - check(global.contexts == 0 && global.bytes == 0 && global.borrowed == 0, - "global recovery leaked resources"); - budget_fini(&global); - - budget_init(&global, 511, 8); - mount_init(&a, &global, 511, 8); - mount_init(&b, &global, 511, 8); - check(acquire(&a, false, &one) == 0, "byte owner acquire"); - check(acquire(&b, false, &two) == ENOMEM, - "global byte exhaustion did not return ENOMEM"); - release(one, false); - check(acquire(&b, false, &two) == 0, "byte recovery acquire failed"); - release(two, false); - drain(&a); - drain(&b); - check(global.bytes == 0 && global.contexts == 0, "byte recovery leaked"); - budget_fini(&global); - - budget_init(&global, 4096, 8); - mount_init(&a, &global, 4096, 8); - check(acquire(&a, true, &one) == ENOMEM, - "construction failure did not return ENOMEM"); - check(global.bytes == 0 && global.contexts == 0 && global.borrowed == 0, - "construction failure did not roll back reservations"); - drain(&a); - budget_fini(&global); - - budget_init(&global, 4096, 8); - mount_init(&a, &global, 4096, 8); - check(acquire(&a, false, &one) == 0, "drain borrower acquire"); - delayed.ctx = one; - check(pthread_create(&threads[0], NULL, release_worker, &delayed) == 0, - "drain release worker create"); - drain(&a); - check(pthread_join(threads[0], NULL) == 0, "drain release worker join"); - check(global.bytes == 0 && global.contexts == 0 && global.borrowed == 0, - "unmount drain did not wait for borrower cleanup"); - budget_fini(&global); - - printf("B29 pool oracle PASS assertions=%u global-count global-bytes failure rollback recovery drain\n", - assertions); - return 0; -} -EOF - -if ! cc -std=c11 -Wall -Wextra -Werror -pthread \ - "$PRE15_CASE_TMP/B29-pool-oracle.c" -o "$PRE15_CASE_TMP/B29-pool-oracle" \ - >"$artifacts/B29-pool-oracle-build.stdout" \ - 2>"$artifacts/B29-pool-oracle-build.stderr" || \ - ! "$PRE15_CASE_TMP/B29-pool-oracle" \ - >"$artifacts/B29-pool-oracle.stdout" \ - 2>"$artifacts/B29-pool-oracle.stderr"; then - pre15_dut_fail 'B29 concurrent pool accounting oracle failed' -fi - -if ! python3 -B - "$fixtures" "$artifacts/B29-fixture-index.json" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import shutil -import struct -import subprocess -import sys - - -output = Path(sys.argv[1]) -index_path = Path(sys.argv[2]) -source = output / "source" -images = output / "images" -output.mkdir(parents=True) -source.mkdir() -images.mkdir() - - -def run(argv: list[str]) -> None: - completed = subprocess.run( - argv, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, timeout=120, - ) - if completed.returncode != 0: - raise SystemExit(f"command failed ({completed.returncode}): {' '.join(argv)}\n{completed.stdout}") - - -block = bytes((index * 29 + 17) & 0xFF for index in range(4096)) -for index in range(3): - payload = (block[index * 257 :] + block[: index * 257]) * 64 - (source / f"payload-{index}.bin").write_bytes(payload) - -specs = { - "lzma": "-zlzma,level=6,dictsize=65536", - "lzma-max": "-zlzma,level=6,dictsize=8388608", - "deflate": "-zdeflate,level=1,dictsize=32768", - "zstd": "-zzstd,level=3,dictsize=65536", - "zstd-max": "-zzstd,level=3,dictsize=1048576", -} -records = [] -for name, option in specs.items(): - image = images / f"{name}.erofs" - run([ - "mkfs.erofs", "-T0", "--all-time", "--force-uid=0", "--force-gid=0", - "-C65536", "--max-extent-bytes=1048576", option, str(image), str(source), - ]) - run(["fsck.erofs", "--extract=" + str(output / (name + "-extract")), str(image)]) - for index in range(3): - actual = output / (name + "-extract") / f"payload-{index}.bin" - if actual.read_bytes() != (source / f"payload-{index}.bin").read_bytes(): - raise SystemExit(f"{name} extraction mismatch for payload-{index}") - records.append({"name": name, "sha256": hashlib.sha256(image.read_bytes()).hexdigest()}) - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - polynomial = 0x82F63B78 - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (polynomial if value & 1 else 0) - return value & 0xFFFFFFFF - - -def mutate(source_image: Path, target: int, mutate) -> Path: - data = bytearray(source_image.read_bytes()) - super_offset = 1024 - block_size = 1 << data[super_offset + 12] - sb_size = 128 + data[super_offset + 13] * 16 - algorithms = struct.unpack_from(" super_offset: - checksum_size -= super_offset - checksum = crc32c( - data[super_offset + 8 : super_offset + checksum_size], - seed=0x5045B54A, - ) - struct.pack_into(""$artifacts/SHA256SUMS" - -if test "${PRE15_QEMU_TARGET_ONLY:-0}" = 1; then - pre15_target_reached -elif test "${PRE15_MODE:-host}" != qemu; then - pre15_target_reached - printf '%s\n' \ - 'B29 focused host/static gate PASS' \ - "$(cat "$artifacts/B29-pool-oracle.stdout")" \ - 'Old G05 benefit retained; production hard caps, bounded fallback, true global count/byte exhaustion model, rollback, drain, and recovery PASS' \ - 'QEMU runtime NOT_RUN in host mode; full feature suite NOT_RUN' - exit 0 -fi - -for tool in scp ssh; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B29 QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -module=$PRE15_CASE_TMP/B29-erofs-zstdio1.ko -fixture_archive=$PRE15_CASE_TMP/B29-fixtures.tar.gz -if ! /bin/sh "$kld_builder" "$PRE15_DUT" "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work-qemu" 1 >"$artifacts/B29-kld-build.stdout" \ - 2>"$artifacts/B29-kld-build.stderr"; then - pre15_dut_fail 'B29 exact-ABI zstdio1 QEMU KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/B29-kld-file.txt" -sha256sum "$module" >"$artifacts/B29-kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/B29-kld-nm-u.txt" -tar -C "$fixtures" -czf "$fixture_archive" images source - -cat >"$PRE15_CASE_TMP/B29-read-probe.c" <<'EOF' -#include -#include -#include -#include -#include -#include - -static int -read_all(const char *path, unsigned char **datap, size_t *sizep, int *errorp) -{ - unsigned char *data; - size_t capacity, size; - ssize_t count; - int fd; - - fd = open(path, O_RDONLY); - if (fd < 0) { - *errorp = errno; - return (-1); - } - capacity = 1024 * 1024; - data = malloc(capacity); - if (data == NULL) - return (-1); - size = 0; - while ((count = read(fd, data + size, capacity - size)) > 0) { - size += (size_t)count; - if (size == capacity) - break; - } - if (count < 0) { - *errorp = errno; - close(fd); - free(data); - return (-1); - } - close(fd); - *datap = data; - *sizep = size; - *errorp = 0; - return (0); -} - -int -main(int argc, char **argv) -{ - unsigned char *actual, *expected; - size_t actual_size, expected_size; - int actual_error, expected_error; - - if (argc == 4 && strcmp(argv[1], "pass") == 0) { - if (read_all(argv[2], &actual, &actual_size, &actual_error) != 0 || - read_all(argv[3], &expected, &expected_size, &expected_error) != 0) - return (1); - if (actual_size != expected_size || - memcmp(actual, expected, actual_size) != 0) - return (1); - free(actual); - free(expected); - return (0); - } - if (argc == 4 && strcmp(argv[1], "errno") == 0) { - expected_error = atoi(argv[3]); - if (read_all(argv[2], &actual, &actual_size, &actual_error) == 0) { - free(actual); - return (1); - } - return (actual_error == expected_error ? 0 : 1); - } - fprintf(stderr, "usage: B29-read-probe pass ACTUAL EXPECTED | errno PATH ERRNO\n"); - return (2); -} -EOF - -cat >"$PRE15_CASE_TMP/B29-mount-probe.c" <<'EOF' -#include -#include - -#include -#include -#include -#include - -static void -add_iovec(struct iovec **iov, int *iovlen, const char *name, const char *value) -{ - *iov = realloc(*iov, sizeof(**iov) * (size_t)(*iovlen + 2)); - (*iov)[*iovlen].iov_base = __DECONST(char *, name); - (*iov)[*iovlen].iov_len = strlen(name) + 1; - ++*iovlen; - (*iov)[*iovlen].iov_base = __DECONST(char *, value); - (*iov)[*iovlen].iov_len = strlen(value) + 1; - ++*iovlen; -} - -int -main(int argc, char **argv) -{ - struct iovec *iov; - int error, expected, iovlen; - - if (argc != 4) - return (2); - expected = atoi(argv[3]); - iov = NULL; - iovlen = 0; - add_iovec(&iov, &iovlen, "fstype", "erofs"); - add_iovec(&iov, &iovlen, "fspath", argv[2]); - add_iovec(&iov, &iovlen, "from", argv[1]); - if (nmount(iov, (unsigned)iovlen, MNT_RDONLY) == 0) { - unmount(argv[2], 0); - free(iov); - return (1); - } - error = errno; - free(iov); - return (error == expected ? 0 : 1); -} -EOF - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -pre15_guest_ssh_bounded() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" ssh -n -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -p "$PRE15_QEMU_SSH_PORT" \ - "$PRE15_QEMU_SSH_USER@127.0.0.1" "$@" -} - -if ! pre15_scp "$fixture_archive" /root/B29-fixtures.tar.gz || \ - ! pre15_scp "$module" /root/B29-erofs-zstdio1.ko || \ - ! pre15_scp "$PRE15_CASE_TMP/B29-read-probe.c" /root/B29-read-probe.c || \ - ! pre15_scp "$PRE15_CASE_TMP/B29-mount-probe.c" /root/B29-mount-probe.c; then - pre15_infra_blocked 'could not transfer B29 module, fixtures, or probes' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/B29-fixtures && mkdir /root/B29-fixtures && tar -xzf /root/B29-fixtures.tar.gz -C /root/B29-fixtures && cc -O2 -Wall -Wextra -Werror -o /root/B29-read-probe /root/B29-read-probe.c && cc -O2 -Wall -Wextra -Werror -o /root/B29-mount-probe /root/B29-mount-probe.c'; then - pre15_infra_blocked 'could not prepare B29 guest fixtures/probes' -fi -if pre15_guest_ssh_bounded kldstat -q -m erofs >/dev/null 2>&1; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' -fi -pre15_guest_ssh_bounded \ - 'kenv -u vfs.erofs.stream_pool.global_budget >/dev/null 2>&1 || true; kenv -u vfs.erofs.stream_pool.global_contexts >/dev/null 2>&1 || true; kenv -u vfs.erofs.stream_pool.global_cached_per_codec >/dev/null 2>&1 || true' -if ! pre15_guest_ssh_bounded kldload /root/B29-erofs-zstdio1.ko \ - >"$artifacts/B29-kldload.stdout" 2>"$artifacts/B29-kldload.stderr"; then - if grep -Eq 'link_elf|symbol|not defined' "$artifacts/B29-kldload.stderr"; then - pre15_infra_blocked 'guest lacks the exact B29 KLD provider ABI' - fi - pre15_dut_fail 'B29 exact-ABI KLD failed to load' -fi -pre15_own_guest_kld B29-erofs-zstdio1.ko 'B29 exact-source KLD' - -attach_mount() -{ - image=$1 - label=$2 - B29_MD=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/B29-fixtures/images/$image") || \ - pre15_dut_fail "$label md attach failed" - case "$B29_MD" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected B29 md unit: $B29_MD" ;; - esac - pre15_own_guest_md "$B29_MD" "$label md" - B29_MOUNT=/mnt/pre15-b29-$label - pre15_guest_ssh_bounded mkdir -p "$B29_MOUNT" - pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$B29_MD" \ - "$B29_MOUNT" || pre15_dut_fail "$label mount failed" - pre15_own_guest_mount "$B29_MOUNT" "$label mount" - B29_MOUNTS="$B29_MOUNTS $B29_MOUNT" -} - -drop_guest_ownership() -{ - kind=$1 - value=$2 - temporary=$PRE15_CASE_TMP/B29-ownership.$$ - awk -v kind="$kind" -v value="$value" \ - 'BEGIN { FS = "\\t" } !($1 == kind && $2 == value)' \ - "$PRE15_OWNERSHIP_FILE" >"$temporary" - mv "$temporary" "$PRE15_OWNERSHIP_FILE" -} - -unmount_owned() -{ - mountpoint=$1 - label=$2 - pre15_guest_ssh_bounded umount "$mountpoint" || \ - pre15_dut_fail "$label unmount failed" - drop_guest_ownership guest-mount "$mountpoint" -} - -unload_owned_kld() -{ - label=$1 - if ! pre15_guest_ssh_bounded kldunload B29-erofs-zstdio1.ko; then - pre15_dut_fail "$label module unload did not drain cleanly" - fi - drop_guest_ownership guest-kld B29-erofs-zstdio1.ko -} - -clear_pool_tunables() -{ - pre15_guest_ssh_bounded \ - 'kenv -u vfs.erofs.stream_pool.global_budget >/dev/null 2>&1 || true; kenv -u vfs.erofs.stream_pool.global_contexts >/dev/null 2>&1 || true; kenv -u vfs.erofs.stream_pool.global_cached_per_codec >/dev/null 2>&1 || true' -} - -run_exhaustion_phase() -{ - codec=$1 - limit_kind=$2 - limit_value=$3 - phase=$4 - clear_pool_tunables - case "$limit_kind" in - byte) - pre15_guest_ssh_bounded kenv \ - "vfs.erofs.stream_pool.global_budget=$limit_value" - expected_budget=$limit_value - expected_contexts=64 - ;; - count) - pre15_guest_ssh_bounded kenv \ - "vfs.erofs.stream_pool.global_contexts=$limit_value" - expected_budget=134217728 - expected_contexts=$limit_value - ;; - *) pre15_runner_fail "unknown B29 exhaustion limit: $limit_kind" ;; - esac - pre15_guest_ssh_bounded kenv \ - vfs.erofs.stream_pool.global_cached_per_codec=1 - if ! pre15_guest_ssh_bounded kldload /root/B29-erofs-zstdio1.ko \ - >"$artifacts/B29-kldload-$phase.stdout" \ - 2>"$artifacts/B29-kldload-$phase.stderr"; then - pre15_dut_fail "B29 $phase constrained-limit KLD reload failed" - fi - pre15_own_guest_kld B29-erofs-zstdio1.ko "$phase exact-source KLD" - pre15_guest_ssh_bounded \ - 'sysctl -n vfs.erofs.stream_pool.global_budget; sysctl -n vfs.erofs.stream_pool.global_contexts_limit' \ - >"$artifacts/B29-$phase-effective-limits.txt" - set -- $(cat "$artifacts/B29-$phase-effective-limits.txt") - if test "$1" -ne "$expected_budget" || test "$2" -ne "$expected_contexts"; then - pre15_dut_fail "B29 $phase production tunable did not reach its clamped limit" - fi - runtime_assertions=$((runtime_assertions + 2)) - - B29_MOUNTS= - attach_mount "$codec.erofs" "$phase-owner" - B29_OWNER_MOUNT=$B29_MOUNT - attach_mount "$codec.erofs" "$phase-b" - B29_B_MOUNT=$B29_MOUNT - attach_mount "$codec.erofs" "$phase-c" - B29_C_MOUNT=$B29_MOUNT - pre15_guest_ssh_bounded /root/B29-read-probe pass \ - "$B29_OWNER_MOUNT/payload-0.bin" \ - /root/B29-fixtures/source/payload-0.bin || \ - pre15_dut_fail "B29 $phase global exhaustion owner read failed" - runtime_assertions=$((runtime_assertions + 1)) - - pre15_guest_ssh_bounded \ - 'sysctl -n vfs.erofs.stream_pool.contexts; sysctl -n vfs.erofs.stream_pool.idle; sysctl -n vfs.erofs.stream_pool.borrowed; sysctl -n vfs.erofs.stream_pool.resident_bytes; sysctl -n vfs.erofs.stream_pool.exhaustions' \ - >"$artifacts/B29-$phase-before-exhaustion.txt" - set -- $(cat "$artifacts/B29-$phase-before-exhaustion.txt") - before_contexts=$1 - before_idle=$2 - before_borrowed=$3 - before_resident=$4 - before_exhaustions=$5 - if test "$before_contexts" -ne 1 || test "$before_idle" -ne 1 || \ - test "$before_borrowed" -ne 0 || test "$before_resident" -le 256; then - pre15_dut_fail "B29 $phase owner did not retain one real idle context" - fi - if test "$limit_kind" = byte && \ - test "$before_resident" -ne "$limit_value"; then - pre15_dut_fail "B29 $phase did not consume the exact production byte budget" - fi - runtime_assertions=$((runtime_assertions + 4)) - - if ! pre15_guest_ssh_bounded \ - "/root/B29-read-probe errno '$B29_B_MOUNT/payload-1.bin' 12 & p1=\$!; /root/B29-read-probe errno '$B29_C_MOUNT/payload-2.bin' 12 & p2=\$!; wait \$p1; r1=\$?; wait \$p2; r2=\$?; test \$r1 -eq 0 -a \$r2 -eq 0"; then - pre15_dut_fail "B29 $phase concurrent exhaustion errno was not stable ENOMEM" - fi - runtime_assertions=$((runtime_assertions + 2)) - pre15_guest_ssh_bounded \ - 'sysctl -n vfs.erofs.stream_pool.contexts; sysctl -n vfs.erofs.stream_pool.idle; sysctl -n vfs.erofs.stream_pool.borrowed; sysctl -n vfs.erofs.stream_pool.resident_bytes; sysctl -n vfs.erofs.stream_pool.exhaustions' \ - >"$artifacts/B29-$phase-after-exhaustion.txt" - set -- $(cat "$artifacts/B29-$phase-after-exhaustion.txt") - if test "$1" -ne 1 || test "$2" -ne 1 || test "$3" -ne 0 || \ - test "$4" -ne "$before_resident" || \ - test "$5" -lt $((before_exhaustions + 2)); then - pre15_dut_fail "B29 $phase exhaustion changed ownership or missed accounting" - fi - runtime_assertions=$((runtime_assertions + 5)) - - unmount_owned "$B29_OWNER_MOUNT" "B29 $phase owner" - pre15_guest_ssh_bounded \ - 'sysctl -n vfs.erofs.stream_pool.contexts; sysctl -n vfs.erofs.stream_pool.resident_bytes' \ - >"$artifacts/B29-$phase-after-owner-unmount.txt" - set -- $(cat "$artifacts/B29-$phase-after-owner-unmount.txt") - if test "$1" -ne 0 || test "$2" -ne 0; then - pre15_dut_fail "B29 $phase owner unmount did not release global resources" - fi - runtime_assertions=$((runtime_assertions + 2)) - pre15_guest_ssh_bounded /root/B29-read-probe pass \ - "$B29_B_MOUNT/payload-1.bin" \ - /root/B29-fixtures/source/payload-1.bin || \ - pre15_dut_fail "B29 $phase read did not recover after resource release" - runtime_assertions=$((runtime_assertions + 1)) - - unmount_owned "$B29_B_MOUNT" "B29 $phase recovery B" - unmount_owned "$B29_C_MOUNT" "B29 $phase recovery C" - pre15_guest_ssh_bounded \ - 'sysctl -n vfs.erofs.stream_pool.contexts; sysctl -n vfs.erofs.stream_pool.resident_bytes; sysctl -n vfs.erofs.stream_pool.borrowed; sysctl -n vfs.erofs.stream_pool.idle' \ - >"$artifacts/B29-$phase-final.txt" - set -- $(cat "$artifacts/B29-$phase-final.txt") - if test "$1" -ne 0 || test "$2" -ne 0 || test "$3" -ne 0 || \ - test "$4" -ne 0; then - pre15_dut_fail "B29 $phase final pool state is not fully drained" - fi - runtime_assertions=$((runtime_assertions + 4)) - unload_owned_kld "B29 $phase final" - runtime_assertions=$((runtime_assertions + 1)) -} - -runtime_assertions=0 -B29_LZMA_CONTEXT_BYTES= -B29_ZSTD_CONTEXT_BYTES= -B29_MOUNTS= -for codec in lzma deflate zstd; do - attach_mount "$codec.erofs" "$codec" - for index in 0 1 2; do - pre15_guest_ssh_bounded /root/B29-read-probe pass \ - "$B29_MOUNT/payload-$index.bin" \ - "/root/B29-fixtures/source/payload-$index.bin" || \ - pre15_dut_fail "$codec ordinary payload-$index read failed" - runtime_assertions=$((runtime_assertions + 1)) - done - case "$codec" in - lzma|zstd) - pre15_guest_ssh_bounded \ - 'sysctl -n vfs.erofs.stream_pool.contexts; sysctl -n vfs.erofs.stream_pool.idle; sysctl -n vfs.erofs.stream_pool.resident_bytes' \ - >"$artifacts/B29-$codec-context-bytes.txt" - set -- $(cat "$artifacts/B29-$codec-context-bytes.txt") - if test "$1" -ne 1 || test "$2" -ne 1 || test "$3" -le 256; then - pre15_dut_fail "B29 $codec did not retain one charged production context" - fi - case "$codec" in - lzma) B29_LZMA_CONTEXT_BYTES=$3 ;; - zstd) B29_ZSTD_CONTEXT_BYTES=$3 ;; - esac - runtime_assertions=$((runtime_assertions + 3)) - ;; - esac - unmount_owned "$B29_MOUNT" "B29 $codec ordinary" -done -for codec in lzma-max zstd-max; do - attach_mount "$codec.erofs" "$codec" - pre15_guest_ssh_bounded /root/B29-read-probe pass \ - "$B29_MOUNT/payload-0.bin" \ - /root/B29-fixtures/source/payload-0.bin || \ - pre15_dut_fail "$codec maximum-parameter workload failed" - runtime_assertions=$((runtime_assertions + 1)) - unmount_owned "$B29_MOUNT" "B29 $codec maximum-parameter" -done - -for codec in lzma zstd; do - B29_BAD_MD=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/B29-fixtures/images/$codec-max-over.erofs") || \ - pre15_dut_fail "$codec over-limit md attach failed" - pre15_own_guest_md "$B29_BAD_MD" "$codec over-limit md" - B29_BAD_MOUNT=/mnt/pre15-b29-$codec-over - pre15_guest_ssh_bounded mkdir -p "$B29_BAD_MOUNT" - pre15_guest_ssh_bounded /root/B29-mount-probe "/dev/$B29_BAD_MD" \ - "$B29_BAD_MOUNT" 45 || \ - pre15_dut_fail "$codec over-limit config did not return EOPNOTSUPP" - runtime_assertions=$((runtime_assertions + 1)) -done - -pre15_guest_ssh_bounded \ - 'sysctl -n vfs.erofs.stream_pool.creations; sysctl -n vfs.erofs.stream_pool.reuses' \ - >"$artifacts/B29-pool-legal-metrics.txt" -set -- $(cat "$artifacts/B29-pool-legal-metrics.txt") -if test "$1" -lt 5 || test "$2" -lt 6; then - pre15_dut_fail 'B29 legal workloads did not create and reuse real contexts' -fi -runtime_assertions=$((runtime_assertions + 2)) -pre15_guest_ssh_bounded \ - 'sysctl -n vfs.erofs.stream_pool.contexts; sysctl -n vfs.erofs.stream_pool.resident_bytes' \ - >"$artifacts/B29-pool-after-legal-unmount.txt" -set -- $(cat "$artifacts/B29-pool-after-legal-unmount.txt") -if test "$1" -ne 0 || test "$2" -ne 0; then - pre15_dut_fail 'B29 legal workload unmount did not drain all contexts' -fi -runtime_assertions=$((runtime_assertions + 2)) -unload_owned_kld 'B29 first' -runtime_assertions=$((runtime_assertions + 1)) -test -n "$B29_LZMA_CONTEXT_BYTES" || \ - pre15_runner_fail 'B29 LZMA production context size was not recorded' -test -n "$B29_ZSTD_CONTEXT_BYTES" || \ - pre15_runner_fail 'B29 Zstd production context size was not recorded' -run_exhaustion_phase lzma byte "$B29_LZMA_CONTEXT_BYTES" lzma-byte -run_exhaustion_phase zstd byte "$B29_ZSTD_CONTEXT_BYTES" zstd-byte -run_exhaustion_phase lzma count 1 global-count -clear_pool_tunables - -if test "$runtime_assertions" -ne 90; then - pre15_runner_fail "B29 QEMU assertion count mismatch: $runtime_assertions" -fi -pre15_guest_ssh_bounded dmesg >"$artifacts/B29-dmesg.txt" -pre15_target_reached -printf '%s\n' \ - 'B29 FreeBSD 15 QEMU context-pool PASS' \ - 'ordinary=9 charged-codec-state=6 maximum-parameter=2 over-limit=2 lifecycle=5 constrained-phases=66 total=90' \ - 'LZMA-byte, Zstd-byte, and global-count cross-mount exhaustion returned ENOMEM=12; each owner drain reached zero and each retry succeeded; full feature suite NOT_RUN' diff --git a/tests/pre15/cases/B30-codec-structure.sh b/tests/pre15/cases/B30-codec-structure.sh deleted file mode 100755 index b8f5ec6..0000000 --- a/tests/pre15/cases/B30-codec-structure.sh +++ /dev/null @@ -1,448 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=1b4b8abc249d9bbbb75e19b5259c5f46f91909ef -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B30 host tool: $tool" -done - -for source in internal.h compress.h decompressor.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c zdata.c zmap.c; do - pre15_record_fixture "b30-$source" "$PRE15_DUT/src/$source" -done -pre15_record_fixture b30-linux-compress "$PRE15_ROOT/src-linux/compress.h" -pre15_record_fixture b30-linux-decompressor "$PRE15_ROOT/src-linux/decompressor.c" -pre15_record_fixture b30-case \ - "$PRE15_DUT/tests/pre15/cases/B30-codec-structure.sh" - -mkdir -p "$artifacts" -pre15_target_reached - -if python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" \ - "$artifacts/B30-codec-structure.json" <<'PY' -from __future__ import annotations - -from collections import Counter -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -report_path = Path(sys.argv[4]) -src = dut / "src" - - -def committed(path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B30 baseline {path}: {completed.stderr}") - return completed.stdout - - -def replace_once(source: str, old: str, new: str, label: str) -> str: - count = source.count(old) - if count != 1: - raise SystemExit(f"B30 baseline anchor count for {label}: {count}") - return source.replace(old, new, 1) - - -expected_changed = { - "repo-pre-15/src/compress.h", - "repo-pre-15/src/decompressor.c", - "repo-pre-15/src/decompressor_deflate.c", - "repo-pre-15/src/decompressor_lzma.c", - "repo-pre-15/src/decompressor_zstd.c", - "repo-pre-15/src/internal.h", - "repo-pre-15/src/zdata.c", - "repo-pre-15/tests/pre15/cases/B30-codec-structure.sh", -} -changed = subprocess.run( - ["git", "-C", str(root), "diff", "--name-only", baseline, "--", "repo-pre-15"], - check=True, - text=True, - stdout=subprocess.PIPE, -).stdout.splitlines() -changed.extend( - subprocess.run( - [ - "git", - "-C", - str(root), - "ls-files", - "--others", - "--exclude-standard", - "--", - "repo-pre-15", - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() -) -if set(changed) != expected_changed or len(changed) != len(expected_changed): - raise SystemExit(f"B30 repo-pre-15 write set mismatch: {changed!r}") - -current = { - path: (src / path).read_text(encoding="utf-8") - for path in ( - "internal.h", - "compress.h", - "decompressor.c", - "decompressor_lz4.c", - "decompressor_lzma.c", - "decompressor_deflate.c", - "decompressor_zstd.c", - "zdata.c", - "zmap.c", - ) -} -before = { - path: committed(f"src/{path}") - for path in current -} - -expected = before["internal.h"] -expected = replace_once( - expected, - "struct erofs_sb_lz4_info {\n" - "\tuint16_t max_distance_pages;\n" - "\tuint16_t max_pclusterblks;\n" - "};\n\n", - "", - "LZ4 state type", -) -expected = replace_once( - expected, - "\tstruct erofs_sb_lz4_info lz4;\n", - "", - "LZ4 state member", -) -if current["internal.h"] != expected: - raise SystemExit("internal.h differs from exact B30 dead-state removal") - -expected = before["compress.h"] -expected = replace_once( - expected, - "struct z_erofs_decompressor {\n" - "\tconst char *name;\n" - "\tbool supports_subextent;\n" - "\t/* Callbacks return zero or a positive FreeBSD errno. */\n" - "\tint (*config)(struct erofs_sb_info *, const struct erofs_super_block *,\n" - "\t const void *, size_t);\n" - "\tint (*decompress)(const struct z_erofs_decompress_req *);\n" - "};", - "struct z_erofs_decompressor {\n" - "\t/* Callbacks return zero or a positive FreeBSD errno. */\n" - "\tint (*config)(struct erofs_sb_info *, const struct erofs_super_block *,\n" - "\t const void *, size_t);\n" - "\tint (*decompress)(const struct z_erofs_decompress_req *);\n" - "\tbool supports_subextent;\n" - "\tconst char *name;\n" - "};", - "descriptor fields", -) -if current["compress.h"] != expected: - raise SystemExit("compress.h differs from exact B30 descriptor alignment") - -expected = before["decompressor.c"] -for old, new, label in ( - ("\t\tdistance = le16toh(lz4->max_distance);\n", "", "unused cfg distance"), - ("\t\tmax_pclusterblks = 1;\n\t\tsbi->available_compr_algs", "\t\tsbi->available_compr_algs", "legacy dead local store"), - ( - "\tsbi->lz4.max_pclusterblks = max_pclusterblks;\n" - "\tsbi->lz4.max_distance_pages = distance != 0 ?\n" - "\t howmany(distance, PAGE_SIZE) + 1 :\n" - "\t howmany(UINT16_MAX, PAGE_SIZE) + 1;\n", - "", - "runtime LZ4 stores", - ), - ( - "static const struct z_erofs_decompressor z_erofs_shifted_decomp = {\n" - "\t.name = \"shifted\",\n" - "\t.decompress = z_erofs_transform_plain,\n" - "};", - "static const struct z_erofs_decompressor z_erofs_shifted_decomp = {\n" - "\t.decompress = z_erofs_transform_plain,\n" - "\t.name = \"shifted\",\n" - "};", - "shifted descriptor", - ), - ( - "static const struct z_erofs_decompressor z_erofs_interlaced_decomp = {\n" - "\t.name = \"interlaced\",\n" - "\t.decompress = z_erofs_transform_plain,\n" - "};", - "static const struct z_erofs_decompressor z_erofs_interlaced_decomp = {\n" - "\t.decompress = z_erofs_transform_plain,\n" - "\t.name = \"interlaced\",\n" - "};", - "interlaced descriptor", - ), - ( - "static const struct z_erofs_decompressor z_erofs_lz4_decomp = {\n" - "\t.name = \"lz4\",\n" - "\t.supports_subextent = 1,\n" - "\t.config = z_erofs_load_lz4_config,\n" - "\t.decompress = z_erofs_lz4_decompress,\n" - "};", - "static const struct z_erofs_decompressor z_erofs_lz4_decomp = {\n" - "\t.config = z_erofs_load_lz4_config,\n" - "\t.decompress = z_erofs_lz4_decompress,\n" - "\t.supports_subextent = 1,\n" - "\t.name = \"lz4\",\n" - "};", - "LZ4 descriptor", - ), - ( - "static const struct z_erofs_decompressor * const z_erofs_decomp[] = {\n" - "\t[Z_EROFS_COMPRESSION_LZ4] = &z_erofs_lz4_decomp,\n" - "\t[Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp,\n" - "\t[Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp,\n" - "\t[Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp,\n" - "\t[Z_EROFS_COMPRESSION_SHIFTED] = &z_erofs_shifted_decomp,\n" - "\t[Z_EROFS_COMPRESSION_INTERLACED] = &z_erofs_interlaced_decomp,\n" - "};", - "static const struct z_erofs_decompressor * const z_erofs_decomp[] = {\n" - "\t[Z_EROFS_COMPRESSION_SHIFTED] = &z_erofs_shifted_decomp,\n" - "\t[Z_EROFS_COMPRESSION_INTERLACED] = &z_erofs_interlaced_decomp,\n" - "\t[Z_EROFS_COMPRESSION_LZ4] = &z_erofs_lz4_decomp,\n" - "\t[Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp,\n" - "\t[Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp,\n" - "\t[Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp,\n" - "};", - "decompressor table", - ), -): - expected = replace_once(expected, old, new, label) -if current["decompressor.c"] != expected: - raise SystemExit("decompressor.c differs from exact B30 structural edits") - -for path, name, capability in ( - ("decompressor_lzma.c", "lzma", "1"), - ("decompressor_deflate.c", "deflate", "1"), - ("decompressor_zstd.c", "zstd", "0"), -): - expected = before[path] - expected = replace_once( - expected, - f"const struct z_erofs_decompressor z_erofs_{name}_decomp = {{\n" - f"\t.name = \"{name}\",\n" - f"\t.supports_subextent = {capability},\n" - f"\t.config = z_erofs_load_{name}_config,\n" - f"\t.decompress = z_erofs_{name}_decompress,\n" - "};", - f"const struct z_erofs_decompressor z_erofs_{name}_decomp = {{\n" - f"\t.config = z_erofs_load_{name}_config,\n" - f"\t.decompress = z_erofs_{name}_decompress,\n" - f"\t.supports_subextent = {capability},\n" - f"\t.name = \"{name}\",\n" - "};", - f"{name} descriptor", - ) - if current[path] != expected: - raise SystemExit(f"{path} differs from exact B30 descriptor reorder") - -if current["decompressor_lz4.c"] != before["decompressor_lz4.c"]: - raise SystemExit("B30 changed the independent FreeBSD LZ4 backend") -expected = before["zdata.c"] -if expected.count("z_erofs_read_extent") != 2: - raise SystemExit("B30 zdata baseline rename denominator changed") -expected = expected.replace("z_erofs_read_extent", "z_erofs_decode_extent") -if current["zdata.c"] != expected: - raise SystemExit("zdata.c differs from the exact two-token decode rename") -if current["zmap.c"] != before["zmap.c"]: - raise SystemExit("B30 changed the zmap record reader") - -descriptor_markers = [ - "int (*config)(", - "int (*decompress)(", - "bool supports_subextent;", - "const char *name;", -] -positions = [current["compress.h"].index(marker) for marker in descriptor_markers] -if positions != sorted(positions): - raise SystemExit("FreeBSD descriptor field order drifted") - - -def initializer_fields(source: str, object_name: str) -> list[tuple[str, str]]: - match = re.search( - rf"(?:static )?const struct z_erofs_decompressor {object_name} = \{{\n" - rf"(?P.*?)\n\}};", - source, - re.DOTALL, - ) - if match is None: - raise SystemExit(f"missing descriptor initializer: {object_name}") - return re.findall(r"^\s*\.([a-z_]+)\s*=\s*([^,]+),$", match.group("body"), re.MULTILINE) - - -descriptors = { - "shifted": initializer_fields(current["decompressor.c"], "z_erofs_shifted_decomp"), - "interlaced": initializer_fields(current["decompressor.c"], "z_erofs_interlaced_decomp"), - "lz4": initializer_fields(current["decompressor.c"], "z_erofs_lz4_decomp"), - "lzma": initializer_fields(current["decompressor_lzma.c"], "z_erofs_lzma_decomp"), - "deflate": initializer_fields(current["decompressor_deflate.c"], "z_erofs_deflate_decomp"), - "zstd": initializer_fields(current["decompressor_zstd.c"], "z_erofs_zstd_decomp"), -} -expected_descriptors = { - "shifted": [("decompress", "z_erofs_transform_plain"), ("name", '"shifted"')], - "interlaced": [("decompress", "z_erofs_transform_plain"), ("name", '"interlaced"')], - "lz4": [ - ("config", "z_erofs_load_lz4_config"), - ("decompress", "z_erofs_lz4_decompress"), - ("supports_subextent", "1"), - ("name", '"lz4"'), - ], - "lzma": [ - ("config", "z_erofs_load_lzma_config"), - ("decompress", "z_erofs_lzma_decompress"), - ("supports_subextent", "1"), - ("name", '"lzma"'), - ], - "deflate": [ - ("config", "z_erofs_load_deflate_config"), - ("decompress", "z_erofs_deflate_decompress"), - ("supports_subextent", "1"), - ("name", '"deflate"'), - ], - "zstd": [ - ("config", "z_erofs_load_zstd_config"), - ("decompress", "z_erofs_zstd_decompress"), - ("supports_subextent", "0"), - ("name", '"zstd"'), - ], -} -if descriptors != expected_descriptors: - raise SystemExit(f"backend descriptor initializer mismatch: {descriptors!r}") - -table_match = re.search( - r"static const struct z_erofs_decompressor \* const z_erofs_decomp\[\] = \{\n" - r"(?P.*?)\n\};", - current["decompressor.c"], - re.DOTALL, -) -if table_match is None: - raise SystemExit("missing FreeBSD decompressor table") -table_order = re.findall(r"\[(Z_EROFS_COMPRESSION_[A-Z0-9_]+)\]", table_match.group("body")) -expected_order = [ - "Z_EROFS_COMPRESSION_SHIFTED", - "Z_EROFS_COMPRESSION_INTERLACED", - "Z_EROFS_COMPRESSION_LZ4", - "Z_EROFS_COMPRESSION_LZMA", - "Z_EROFS_COMPRESSION_DEFLATE", - "Z_EROFS_COMPRESSION_ZSTD", -] -if table_order != expected_order: - raise SystemExit(f"FreeBSD decompressor table order mismatch: {table_order!r}") -linux_decompressor = (root / "src-linux/decompressor.c").read_text(encoding="utf-8") -linux_table = linux_decompressor[linux_decompressor.index("z_erofs_decomp[] = {") :] -linux_order = re.findall(r"\[(Z_EROFS_COMPRESSION_[A-Z0-9_]+)\]", linux_table)[:6] -if linux_order != expected_order: - raise SystemExit(f"Linux decompressor table anchor drifted: {linux_order!r}") - -callbacks = [] -for path in sorted(src.glob("decompressor*.c")): - for line in path.read_text(encoding="utf-8").splitlines(): - match = re.match(r"\s*\.(config|decompress)\s*=\s*([^,]+),", line) - if match: - callbacks.append((path.name, match.group(1), match.group(2))) -expected_callbacks = [ - ("decompressor.c", "decompress", "z_erofs_transform_plain"), - ("decompressor.c", "decompress", "z_erofs_transform_plain"), - ("decompressor.c", "config", "z_erofs_load_lz4_config"), - ("decompressor.c", "decompress", "z_erofs_lz4_decompress"), - ("decompressor_deflate.c", "config", "z_erofs_load_deflate_config"), - ("decompressor_deflate.c", "decompress", "z_erofs_deflate_decompress"), - ("decompressor_lzma.c", "config", "z_erofs_load_lzma_config"), - ("decompressor_lzma.c", "decompress", "z_erofs_lzma_decompress"), - ("decompressor_zstd.c", "config", "z_erofs_load_zstd_config"), - ("decompressor_zstd.c", "decompress", "z_erofs_zstd_decompress"), -] -if Counter(callbacks) != Counter(expected_callbacks) or len(callbacks) != 10: - raise SystemExit(f"G01 decompressor callback multiset changed: {callbacks!r}") - -all_source = "\n".join(current.values()) -for removed in ("erofs_sb_lz4_info", "max_distance_pages", "sbi->lz4"): - if removed in all_source: - raise SystemExit(f"removed LZ4 runtime state remains: {removed}") -config = current["decompressor.c"][: current["decompressor.c"].index("static int\nz_erofs_transform_plain")] -for marker in ( - "uint32_t max_pclusterblks;", - "max_pclusterblks = le16toh(lz4->max_pclusterblks);", - "if (max_pclusterblks == 0)", - "max_pclusterblks = 1;", - "max_pclusterblks >\n\t\t (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits)", - "return (EOPNOTSUPP);", - "distance = le16toh(dsb->u1.lz4_max_distance);", - "if (distance == 0 && !erofs_sb_has_lz4_0padding(sbi))", -): - if marker not in config: - raise SystemExit(f"LZ4 config validation marker missing: {marker!r}") - -if current["zdata.c"].count("z_erofs_decode_extent") != 2: - raise SystemExit("zdata decode helper rename denominator changed") -if "z_erofs_read_extent" in current["zdata.c"]: - raise SystemExit("old zdata decode helper name remains") -if current["zmap.c"].count("z_erofs_read_extent") != 4: - raise SystemExit("zmap record reader definition/calls changed") -if "z_erofs_decode_extent" in current["zmap.c"]: - raise SystemExit("zmap record reader was incorrectly renamed") -if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", all_source): - raise SystemExit("B30 introduced Linux negative errno") - -for path, load_name, decode_name, descriptor_name in ( - ("decompressor_lzma.c", "z_erofs_load_lzma_config", "z_erofs_lzma_decompress", "z_erofs_lzma_decomp"), - ("decompressor_deflate.c", "z_erofs_load_deflate_config", "z_erofs_deflate_decompress", "z_erofs_deflate_decomp"), - ("decompressor_zstd.c", "z_erofs_load_zstd_config", "z_erofs_zstd_decompress", "z_erofs_zstd_decomp"), -): - positions = [current[path].index(name) for name in (load_name, decode_name, descriptor_name)] - if positions != sorted(positions): - raise SystemExit(f"backend definition order drifted: {path}") - -report = { - "status": "PASS", - "baseline": baseline, - "changed_paths": sorted(changed), - "descriptor_fields": ["config", "decompress", "supports_subextent", "name"], - "descriptor_table_order": table_order, - "callback_field_count": len(callbacks), - "callback_target_multiset": sorted(callbacks), - "removed_runtime_state": ["erofs_sb_lz4_info", "max_distance_pages", "max_pclusterblks store"], - "retained_config_validation": ["max_pclusterblks local", "format cap", "legacy distance branch"], - "rename": {"zdata_decode": 2, "zmap_record_reader": 4}, - "preserved": ["B25 typed errno", "B27 trailing policy", "B28 partial fallback ownership"], -} -with report_path.open("x", encoding="ascii") as stream: - json.dump(report, stream, ensure_ascii=True, indent=2, sort_keys=True) - stream.write("\n") -print("B30-descriptors PASS fields=4 backends=6 callbacks=10") -print("B30-write-only PASS dead-state=2 config-local=1 rename=2/4") -PY -then - : -else - pre15_dut_fail 'B30 descriptor, dead-state, or rename oracle failed' -fi - -sha256sum "$artifacts/B30-codec-structure.json" > "$artifacts/SHA256SUMS" -printf 'B30 PASS codec structure aligned without behavior changes\n' diff --git a/tests/pre15/cases/B31-visibility.sh b/tests/pre15/cases/B31-visibility.sh deleted file mode 100755 index 08f9710..0000000 --- a/tests/pre15/cases/B31-visibility.sh +++ /dev/null @@ -1,350 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=643b83c9cea6a99398fa3dd053f2e8f65e108ca0 -artifacts=$PRE15_RUN_DIR/artifacts -action=${1:-callgraph} -shift || : - -case "$action" in -callgraph|nm-allowlist) ;; -*) pre15_fail_usage "unknown B31 action: $action" ;; -esac - -for tool in git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B31 host tool: $tool" -done - -pre15_record_fixture b31-data "$PRE15_DUT/src/data.c" -pre15_record_fixture b31-dir "$PRE15_DUT/src/dir.c" -pre15_record_fixture b31-inode "$PRE15_DUT/src/inode.c" -pre15_record_fixture b31-internal "$PRE15_DUT/src/internal.h" -pre15_record_fixture b31-case \ - "$PRE15_DUT/tests/pre15/cases/B31-visibility.sh" -mkdir -p "$artifacts" - -if ! git -C "$PRE15_ROOT" diff --quiet "$baseline" -- \ - repo-pre-15/src/data.c repo-pre-15/src/dir.c \ - repo-pre-15/src/inode.c repo-pre-15/src/internal.h; then - pre15_target_reached - pre15_dut_fail 'B31 production files differ from the B30 source baseline' -fi - -if test "$action" = callgraph; then - test "$#" -eq 0 || pre15_fail_usage 'B31 callgraph takes no arguments' - for source in zdata.c super.c namei.c erofs_fs.h; do - pre15_record_fixture "b31-$source" "$PRE15_DUT/src/$source" - done - pre15_record_fixture b31-linux-data "$PRE15_ROOT/src-linux/data.c" - pre15_record_fixture b31-linux-internal "$PRE15_ROOT/src-linux/internal.h" - pre15_record_fixture b31-linux-fileio "$PRE15_ROOT/src-linux/fileio.c" - pre15_record_fixture b31-linux-fscache "$PRE15_ROOT/src-linux/fscache.c" - pre15_record_fixture b31-linux-inode "$PRE15_ROOT/src-linux/inode.c" - pre15_record_fixture b31-linux-xattr "$PRE15_ROOT/src-linux/xattr.c" - pre15_record_fixture b31-linux-zmap "$PRE15_ROOT/src-linux/zmap.c" - pre15_record_fixture b31-linux-zdata "$PRE15_ROOT/src-linux/zdata.c" - - pre15_target_reached - if ! python3 -B - "$PRE15_ROOT" "$PRE15_DUT" \ - "$artifacts/B31-callgraph.json" <<'PY' -from __future__ import annotations - -from collections import Counter -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -report_path = Path(sys.argv[3]) -src = dut / "src" - -symbols = { - "erofs_map_dev": "data.c", - "erofs_map_blocks": "data.c", - "erofs_dirent_namelen": "dir.c", - "erofs_iloc": "inode.c", -} -expected_occurrences = { - "erofs_map_dev": {"data.c": 2}, - "erofs_map_blocks": {"data.c": 3, "super.c": 1, "zdata.c": 1}, - "erofs_dirent_namelen": {"dir.c": 3, "namei.c": 1}, - "erofs_iloc": {"inode.c": 3}, -} -expected_cross_tu = { - "erofs_map_dev": [], - "erofs_map_blocks": ["super.c", "zdata.c"], - "erofs_dirent_namelen": ["namei.c"], - "erofs_iloc": [], -} -introduced_by = { - ("erofs_map_blocks", "super.c"): "55c609db", - ("erofs_map_blocks", "zdata.c"): "55c609db", - ("erofs_dirent_namelen", "namei.c"): "c7508502", -} - - -def strip_noncode(source: str) -> str: - output = list(source) - state = "code" - index = 0 - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - output[index] = output[index + 1] = " " - state = "block" - index += 2 - continue - if char == "/" and following == "/": - output[index] = output[index + 1] = " " - state = "line" - index += 2 - continue - if char == '"': - output[index] = " " - state = "string" - elif char == "'": - output[index] = " " - state = "character" - elif state == "block": - if char == "*" and following == "/": - output[index] = output[index + 1] = " " - state = "code" - index += 2 - continue - if char != "\n": - output[index] = " " - elif state == "line": - if char == "\n": - state = "code" - else: - output[index] = " " - else: - if char == "\\" and following: - output[index] = output[index + 1] = " " - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - if char != "\n": - output[index] = " " - index += 1 - if state == "block": - raise SystemExit("unterminated block comment") - return "".join(output) - - -sources = { - path.name: strip_noncode(path.read_text(encoding="utf-8")) - for path in sorted(src.glob("*.c")) -} -locations: dict[str, list[dict[str, object]]] = {} -cross_tu: dict[str, list[str]] = {} -history: dict[str, str] = {} - -for symbol, owner in symbols.items(): - pattern = re.compile(r"\b" + re.escape(symbol) + r"\s*\(") - sites = [] - counts = Counter() - for filename, source in sources.items(): - for match in pattern.finditer(source): - line = source.count("\n", 0, match.start()) + 1 - sites.append({"file": filename, "line": line}) - counts[filename] += 1 - actual_counts = dict(sorted(counts.items())) - if actual_counts != expected_occurrences[symbol]: - raise SystemExit( - f"{symbol} call-token inventory changed: {actual_counts!r}" - ) - consumers = sorted(filename for filename in counts if filename != owner) - if consumers != expected_cross_tu[symbol]: - raise SystemExit(f"{symbol} cross-TU consumers changed: {consumers!r}") - locations[symbol] = sites - cross_tu[symbol] = consumers - - for filename in consumers: - for site in sites: - if site["file"] != filename: - continue - blamed = subprocess.run( - [ - "git", - "-C", - str(root), - "blame", - "--line-porcelain", - f"-L{site['line']},{site['line']}", - "--", - f"repo-pre-15/src/{filename}", - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines()[0].split()[0] - expected = introduced_by[(symbol, filename)] - if not blamed.startswith(expected): - raise SystemExit( - f"{symbol} {filename} introducer changed: {blamed}" - ) - history[f"{symbol}:{filename}"] = blamed - -internal = (src / "internal.h").read_text(encoding="utf-8") -for symbol in symbols: - prototype_count = len( - re.findall(r"\b" + re.escape(symbol) + r"\s*\(", internal) - ) - if prototype_count != 1: - raise SystemExit(f"{symbol} prototype count changed: {prototype_count}") - -all_source = "\n".join( - path.read_text(encoding="utf-8") for path in sorted(src.glob("*.[ch]")) -) -mtime_helper_invocations = internal.count( - "EROFS_FEATURE_FUNCS(mtime, compat, COMPAT_MTIME)" -) -mtime_explicit_references = len(re.findall(r"\berofs_sb_has_mtime\b", all_source)) -if mtime_helper_invocations != 1 or mtime_explicit_references != 0: - raise SystemExit( - "mtime helper inventory changed: " - f"generator={mtime_helper_invocations} explicit={mtime_explicit_references}" - ) - -erofs_fs = (src / "erofs_fs.h").read_text(encoding="utf-8") -inode = (src / "inode.c").read_text(encoding="utf-8") -super_source = (src / "super.c").read_text(encoding="utf-8") -timestamp_anchors = { - "ondisk_constant": "#define EROFS_FEATURE_COMPAT_MTIME" in erofs_fs, - "compact_checked_add": "__builtin_add_overflow(sbi->epoch," in inode, - "extended_timestamp": "(int64_t)le64toh(die->i_mtime)" in inode, - "epoch_decode": "sbi->epoch = (int64_t)le64toh(dsb->epoch);" in super_source, - "fixed_nsec_decode": "sbi->fixed_nsec = le32toh(dsb->fixed_nsec);" in super_source, -} -if not all(timestamp_anchors.values()): - raise SystemExit(f"timestamp path changed: {timestamp_anchors!r}") - -linux_src = root / "src-linux" -linux_text = { - path.name: strip_noncode(path.read_text(encoding="utf-8")) - for path in ( - linux_src / "data.c", - linux_src / "fileio.c", - linux_src / "fscache.c", - linux_src / "inode.c", - linux_src / "xattr.c", - linux_src / "zmap.c", - linux_src / "zdata.c", - linux_src / "internal.h", - ) -} -linux_internal = linux_text["internal.h"] -if not re.search(r"\bint\s+erofs_map_dev\s*\(", linux_internal): - raise SystemExit("Linux erofs_map_dev external prototype is absent") -if not re.search(r"\bint\s+erofs_map_blocks\s*\(", linux_internal): - raise SystemExit("Linux erofs_map_blocks external prototype is absent") -if not re.search( - r"static\s+inline\s+erofs_off_t\s+erofs_iloc\s*\(", linux_internal -): - raise SystemExit("Linux erofs_iloc is no longer static inline") -if re.search(r"\berofs_dirent_namelen\s*\(", "\n".join(linux_text.values())): - raise SystemExit("unexpected Linux erofs_dirent_namelen symbol") - -linux_consumers = {} -for symbol in ("erofs_map_dev", "erofs_map_blocks", "erofs_iloc"): - pattern = re.compile(r"\b" + re.escape(symbol) + r"\s*\(") - linux_consumers[symbol] = sorted( - filename for filename, source in linux_text.items() if pattern.search(source) - ) - -report = { - "decision": "STOP-NO-SOURCE", - "freebsd_call_tokens": locations, - "freebsd_cross_tu_consumers": cross_tu, - "introduced_by": history, - "linux_linkage": { - "erofs_map_dev": "external", - "erofs_map_blocks": "external", - "erofs_dirent_namelen": "absent", - "erofs_iloc": "static inline", - }, - "linux_consumer_files_in_probe": linux_consumers, - "mtime_helper": { - "generator_invocations": mtime_helper_invocations, - "explicit_references": mtime_explicit_references, - }, - "timestamp_anchors": timestamp_anchors, -} -report_path.write_text( - json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii" -) -PY - then - pre15_dut_fail 'B31 cross-TU visibility audit failed' - fi - printf '%s\n' \ - 'B31 callgraph audit reached mandatory STOP' \ - 'erofs_map_blocks: cross-TU consumers super.c,zdata.c introduced by B07c' \ - 'erofs_dirent_namelen: cross-TU consumer namei.c introduced by B10' \ - 'erofs_map_dev and erofs_iloc remain same-TU only; atomic B31 source edit is blocked' \ - 'mtime helper remains generated with zero explicit references; ondisk constant and timestamp paths remain' - pre15_gate_stop \ - 'B31 blocked: B07c/B10 introduced cross-TU consumers for listed symbols' -fi - -test "$#" -eq 2 || \ - pre15_fail_usage 'B31 nm-allowlist requires zstdio0 and zstdio1 modules' -for tool in awk diff nm; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B31 nm tool: $tool" -done - -module0=$1 -module1=$2 -pre15_record_fixture b31-zstdio0-module "$module0" -pre15_record_fixture b31-zstdio1-module "$module1" -pre15_target_reached - -for config in 0 1; do - eval module=\$module$config - nm -g --defined-only "$module" | awk '{ print $NF }' | LC_ALL=C sort \ - >"$artifacts/B31-zstdio$config-global-names.txt" - nm -u "$module" | awk '{ print $NF }' | LC_ALL=C sort \ - >"$artifacts/B31-zstdio$config-undefined-names.txt" - for symbol in erofs_map_dev erofs_map_blocks erofs_dirent_namelen erofs_iloc; do - if ! awk -v symbol="$symbol" '$0 == symbol { found = 1 } END { exit !found }' \ - "$artifacts/B31-zstdio$config-global-names.txt"; then - pre15_dut_fail "B31 zstdio$config lost required global $symbol" - fi - done - if awk '$0 == "erofs_sb_has_mtime" { found = 1 } END { exit !found }' \ - "$artifacts/B31-zstdio$config-global-names.txt" || \ - awk '$0 == "erofs_sb_has_mtime" { found = 1 } END { exit !found }' \ - "$artifacts/B31-zstdio$config-undefined-names.txt"; then - pre15_dut_fail "B31 zstdio$config unexpectedly emits erofs_sb_has_mtime" - fi -done - -if ! diff -u "$artifacts/B31-zstdio0-global-names.txt" \ - "$artifacts/B31-zstdio1-global-names.txt" \ - >"$artifacts/B31-config-global.diff"; then - pre15_dut_fail 'B31 defined globals differ by ZSTDIO configuration' -fi - -printf '%s\n' \ - 'B31 nm allowlist PASS' \ - 'global delta from B30 source baseline: zero' \ - 'four listed symbols remain global in both configurations after mandatory STOP' \ - 'erofs_sb_has_mtime is not emitted or undefined in either configuration' diff --git a/tests/pre15/cases/B31r-visibility.sh b/tests/pre15/cases/B31r-visibility.sh deleted file mode 100755 index 2b606e7..0000000 --- a/tests/pre15/cases/B31r-visibility.sh +++ /dev/null @@ -1,334 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -artifacts=$PRE15_RUN_DIR/artifacts -action=${1:-callgraph} -shift || : - -case "$action" in -callgraph|nm-allowlist) ;; -*) pre15_fail_usage "unknown B31r action: $action" ;; -esac - -for tool in git nm python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B31r host tool: $tool" -done - -for source in data.c dir.c inode.c internal.h namei.c super.c zdata.c zmap.c; do - pre15_record_fixture "b31r-$source" "$PRE15_DUT/src/$source" -done -pre15_record_fixture b31r-case \ - "$PRE15_DUT/tests/pre15/cases/B31r-visibility.sh" -mkdir -p "$artifacts" - -if test "$action" = callgraph; then - test "$#" -eq 0 || pre15_fail_usage 'B31r callgraph takes no arguments' - pre15_target_reached - if ! python3 -B - "$PRE15_ROOT" "$PRE15_DUT" \ - "$artifacts/B31r-callgraph.json" <<'PY' -from __future__ import annotations - -from collections import Counter -import hashlib -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -report_path = Path(sys.argv[3]) -src = dut / "src" - -symbols = { - "erofs_map_dev": "data.c", - "erofs_map_blocks": "data.c", - "erofs_dirent_namelen": "dir.c", - "erofs_iloc": "inode.c", -} -expected_occurrences = { - "erofs_map_dev": {"data.c": 2}, - "erofs_map_blocks": {"data.c": 3, "super.c": 1, "zdata.c": 1}, - "erofs_dirent_namelen": {"dir.c": 3, "namei.c": 1}, - "erofs_iloc": {"inode.c": 3}, -} -expected_cross_tu = { - "erofs_map_dev": [], - "erofs_map_blocks": ["super.c", "zdata.c"], - "erofs_dirent_namelen": ["namei.c"], - "erofs_iloc": [], -} -introduced_by = { - ("erofs_map_blocks", "super.c"): "55c609db", - ("erofs_map_blocks", "zdata.c"): "55c609db", - ("erofs_dirent_namelen", "namei.c"): "c7508502", -} - - -def strip_noncode(source: str) -> str: - output = list(source) - state = "code" - index = 0 - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - output[index] = output[index + 1] = " " - state = "block" - index += 2 - continue - if char == "/" and following == "/": - output[index] = output[index + 1] = " " - state = "line" - index += 2 - continue - if char == '"': - output[index] = " " - state = "string" - elif char == "'": - output[index] = " " - state = "character" - elif state == "block": - if char == "*" and following == "/": - output[index] = output[index + 1] = " " - state = "code" - index += 2 - continue - if char != "\n": - output[index] = " " - elif state == "line": - if char == "\n": - state = "code" - else: - output[index] = " " - else: - if char == "\\" and following: - output[index] = output[index + 1] = " " - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - if char != "\n": - output[index] = " " - index += 1 - if state == "block": - raise SystemExit("unterminated block comment") - return "".join(output) - - -sources = { - path.name: strip_noncode(path.read_text(encoding="utf-8")) - for path in sorted(src.glob("*.c")) -} -locations: dict[str, list[dict[str, object]]] = {} -cross_tu: dict[str, list[str]] = {} -history: dict[str, str] = {} - -for symbol, owner in symbols.items(): - pattern = re.compile(r"\b" + re.escape(symbol) + r"\s*\(") - sites = [] - counts = Counter() - for filename, source in sources.items(): - for match in pattern.finditer(source): - line = source.count("\n", 0, match.start()) + 1 - sites.append({"file": filename, "line": line}) - counts[filename] += 1 - actual_counts = dict(sorted(counts.items())) - if actual_counts != expected_occurrences[symbol]: - raise SystemExit( - f"{symbol} call-token inventory changed: {actual_counts!r}" - ) - consumers = sorted(filename for filename in counts if filename != owner) - if consumers != expected_cross_tu[symbol]: - raise SystemExit(f"{symbol} cross-TU consumers changed: {consumers!r}") - locations[symbol] = sites - cross_tu[symbol] = consumers - - for filename in consumers: - for site in sites: - if site["file"] != filename: - continue - blamed = subprocess.run( - [ - "git", - "-C", - str(root), - "blame", - "--line-porcelain", - f"-L{site['line']},{site['line']}", - "--", - f"repo-pre-15/src/{filename}", - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines()[0].split()[0] - expected = introduced_by[(symbol, filename)] - if not blamed.startswith(expected): - raise SystemExit( - f"{symbol} {filename} introducer changed: {blamed}" - ) - history[f"{symbol}:{filename}"] = blamed - -internal = (src / "internal.h").read_text(encoding="utf-8") -for symbol in symbols: - prototype_count = len( - re.findall(r"\b" + re.escape(symbol) + r"\s*\(", internal) - ) - expected = 1 if symbol in {"erofs_map_blocks", "erofs_dirent_namelen"} else 0 - if prototype_count != expected: - raise SystemExit( - f"{symbol} prototype count changed: {prototype_count}, expected {expected}" - ) - -data = (src / "data.c").read_text(encoding="utf-8") -inode = (src / "inode.c").read_text(encoding="utf-8") -if not re.search(r"\bstatic\s+int\s+erofs_map_dev\s*\(", data): - raise SystemExit("erofs_map_dev is not static in data.c") -if not re.search(r"\bstatic\s+erofs_off_t\s+erofs_iloc\s*\(", inode): - raise SystemExit("erofs_iloc is not static in inode.c") - -all_source = "\n".join( - path.read_text(encoding="utf-8") for path in sorted(src.glob("*.[ch]")) -) -mtime_helper_invocations = all_source.count( - "EROFS_FEATURE_FUNCS(mtime, compat, COMPAT_MTIME)" -) -mtime_explicit_references = len(re.findall(r"\berofs_sb_has_mtime\b", all_source)) -if mtime_helper_invocations != 0 or mtime_explicit_references != 0: - raise SystemExit( - "mtime helper inventory changed: " - f"generator={mtime_helper_invocations} explicit={mtime_explicit_references}" - ) - -erofs_fs = (src / "erofs_fs.h").read_text(encoding="utf-8") -super_source = (src / "super.c").read_text(encoding="utf-8") -timestamp_anchors = { - "ondisk_constant": "#define EROFS_FEATURE_COMPAT_MTIME" in erofs_fs, - "compact_checked_add": "__builtin_add_overflow(sbi->epoch," in inode, - "extended_timestamp": "(int64_t)le64toh(die->i_mtime)" in inode, - "epoch_decode": "sbi->epoch = (int64_t)le64toh(dsb->epoch);" in super_source, - "fixed_nsec_decode": "sbi->fixed_nsec = le32toh(dsb->fixed_nsec);" in super_source, -} -if not all(timestamp_anchors.values()): - raise SystemExit(f"timestamp path changed: {timestamp_anchors!r}") - -source_hashes = {} -for path in sorted(src.glob("*.[ch]")): - source_hashes[path.relative_to(dut).as_posix()] = hashlib.sha256( - path.read_bytes() - ).hexdigest() - -report = { - "decision": "PASS", - "freebsd_call_tokens": locations, - "freebsd_cross_tu_consumers": cross_tu, - "introduced_by": history, - "linkage": { - "erofs_map_dev": "static", - "erofs_map_blocks": "external", - "erofs_dirent_namelen": "external", - "erofs_iloc": "static", - }, - "internal_prototypes": { - symbol: len(re.findall(r"\b" + re.escape(symbol) + r"\s*\(", internal)) - for symbol in symbols - }, - "mtime_helper": { - "generator_invocations": mtime_helper_invocations, - "explicit_references": mtime_explicit_references, - }, - "timestamp_anchors": timestamp_anchors, - "source_hashes": source_hashes, -} -report_path.write_text( - json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii" -) -(report_path.parent / "B31r-source-sha256.txt").write_text( - "".join(f"{digest} {path}\n" for path, digest in source_hashes.items()), - encoding="ascii", -) -PY - then - pre15_dut_fail 'B31r independent callgraph audit failed' - fi - printf '%s\n' \ - 'B31r independent cross-TU callgraph PASS' \ - 'erofs_map_dev and erofs_iloc are same-TU and static' \ - 'erofs_map_blocks and erofs_dirent_namelen retain external linkage' \ - 'erofs_sb_has_mtime helper is absent; timestamp anchors remain' \ - 'source hashes and exact consumers recorded in B31r-callgraph.json' - exit 0 - fi - -test "$#" -eq 2 || \ - pre15_fail_usage 'B31r nm-allowlist requires zstdio0 and zstdio1 modules' -for module in "$@"; do - pre15_record_fixture "b31r-module-$module" "$module" -done -pre15_target_reached - -for config in 0 1; do - case "$config" in - 0) module=$1 ;; - 1) module=$2 ;; - esac - nm -an "$module" >"$artifacts/B31r-zstdio$config-nm-an.txt" - awk '$NF ~ /^(erofs_map_dev|erofs_map_blocks|erofs_dirent_namelen|erofs_iloc|erofs_sb_has_mtime)$/' \ - "$artifacts/B31r-zstdio$config-nm-an.txt" \ - >"$artifacts/B31r-zstdio$config-target-bindings.txt" - nm -g --defined-only "$module" | awk '{ print $NF }' | LC_ALL=C sort \ - >"$artifacts/B31r-zstdio$config-global-names.txt" - nm -u "$module" | awk '{ print $NF }' | LC_ALL=C sort \ - >"$artifacts/B31r-zstdio$config-undefined-names.txt" - for symbol in erofs_map_blocks erofs_dirent_namelen; do - if ! awk -v symbol="$symbol" '$0 == symbol { found = 1 } END { exit !found }' \ - "$artifacts/B31r-zstdio$config-global-names.txt"; then - pre15_dut_fail "B31r zstdio$config lost required global $symbol" - fi - done - for symbol in erofs_map_dev erofs_iloc erofs_sb_has_mtime; do - if awk -v symbol="$symbol" '$0 == symbol { found = 1 } END { exit !found }' \ - "$artifacts/B31r-zstdio$config-global-names.txt" || \ - awk -v symbol="$symbol" '$0 == symbol { found = 1 } END { exit !found }' \ - "$artifacts/B31r-zstdio$config-undefined-names.txt"; then - pre15_dut_fail "B31r zstdio$config unexpectedly exposes $symbol" - fi - done - for symbol in erofs_map_dev erofs_iloc; do - if ! awk -v symbol="$symbol" \ - '$NF == symbol && $(NF - 1) !~ /^[a-z]$/ { bad = 1 } END { exit bad }' \ - "$artifacts/B31r-zstdio$config-nm-an.txt"; then - pre15_dut_fail "B31r zstdio$config has non-local binding for $symbol" - fi - done - if awk '$NF == "erofs_sb_has_mtime" { found = 1 } END { exit !found }' \ - "$artifacts/B31r-zstdio$config-nm-an.txt"; then - pre15_dut_fail "B31r zstdio$config emits erofs_sb_has_mtime" - fi -done - -if ! diff -u "$artifacts/B31r-zstdio0-global-names.txt" \ - "$artifacts/B31r-zstdio1-global-names.txt" \ - >"$artifacts/B31r-config-global.diff"; then - pre15_dut_fail 'B31r defined globals differ by ZSTDIO configuration' -fi - -printf '%s\n' \ - 'B31r nm allowlist PASS' \ - 'cross-TU symbols remain global in both configurations' \ - 'same-TU symbols are absent or locally bound and never global/undefined' \ - 'erofs_sb_has_mtime is absent from all symbol bindings' \ - 'zstdio0/zstdio1 defined-global sets match' diff --git a/tests/pre15/cases/B32-cache-state.sh b/tests/pre15/cases/B32-cache-state.sh deleted file mode 100755 index 6a0b0ca..0000000 --- a/tests/pre15/cases/B32-cache-state.sh +++ /dev/null @@ -1,601 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -spec=$fixture_dir/B32-cache-state.json -oracle=$fixture_dir/B32-cache-oracle.c -generator=$fixture_dir/B32-cache-generate.py -probe=$fixture_dir/B32-cache-probe.c -qemu_runner=$fixture_dir/B32-qemu-run.sh -kld_builder=$fixture_dir/B28-build-kld.sh -artifacts=$PRE15_RUN_DIR/artifacts -scenario=${PRE15_B32_SCENARIO:-TC168-cache-inflight} - -case "$scenario" in -TC168-cache-inflight|TC184-vnode-races) ;; -*) pre15_fail_usage "unknown B32 scenario: $scenario" ;; -esac - -for tool in cc git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B32 host tool: $tool" -done -for fixture in "$spec" "$oracle" "$generator" "$probe" "$qemu_runner" \ - "$kld_builder"; do - pre15_record_fixture "b32-$(basename "$fixture")" "$fixture" -done -for source in internal.h decompressor.c zdata.c zmap.c; do - pre15_record_fixture "b32-$source" "$PRE15_DUT/src/$source" -done -mkdir -p "$artifacts" - -if test "${PRE15_QEMU_TARGET_ONLY:-0}" = 1; then - pre15_target_reached -else -if ! python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$spec" \ - "$artifacts/B32-source-audit.json" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -spec = json.loads(Path(sys.argv[3]).read_text(encoding="ascii")) -report_path = Path(sys.argv[4]) -baseline = spec["baseline"] -implementation = "c6a502184da6e973b1f0fa2ff8c0290c041e7092" -current_owners = { - "internal.h": "aa20623132b6821c6ad4651b263fc0ac0f7a14c8", - "decompressor.c": implementation, - "zdata.c": "bc56f830918b76029871b60cca2e53992de70a2e", - "zmap.c": "d09924362eb29819bd393e1e86602eb38c7608c6", -} - - -def fail(message: str) -> None: - raise SystemExit(message) - - -def committed(commit: str, path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{commit}:repo-pre-15/{path}"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if completed.returncode != 0: - fail(f"cannot read B32 source {commit}:{path}: {completed.stderr.strip()}") - return completed.stdout - - -def function(source: str, name: str) -> str: - match = re.search(rf"\n(?:static\s+)?[^\n]+\n{name}\([^{{]+\n\{{", source) - if match is None: - fail(f"cannot locate function {name}") - start = match.start() + 1 - brace = source.find("{", match.start()) - depth = 0 - for index in range(brace, len(source)): - if source[index] == "{": - depth += 1 - elif source[index] == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - fail(f"unterminated function {name}") - - -expected_changed = { - "repo-pre-15/src/decompressor.c", - "repo-pre-15/src/internal.h", - "repo-pre-15/src/zdata.c", - "repo-pre-15/src/zmap.c", - "repo-pre-15/tests/pre15/cases/B32-cache-state.sh", - "repo-pre-15/tests/pre15/fixtures/B32-cache-generate.py", - "repo-pre-15/tests/pre15/fixtures/B32-cache-oracle.c", - "repo-pre-15/tests/pre15/fixtures/B32-cache-probe.c", - "repo-pre-15/tests/pre15/fixtures/B32-qemu-run.sh", - "repo-pre-15/tests/pre15/fixtures/B32-cache-state.json", -} -changed = subprocess.run( - [ - "git", - "-C", - str(root), - "diff-tree", - "--no-commit-id", - "--name-only", - "-r", - implementation, - ], - check=True, - stdout=subprocess.PIPE, - text=True, -).stdout.splitlines() -if set(changed) != expected_changed or len(changed) != len(expected_changed): - fail(f"B32 implementation write set mismatch: {changed!r}") - -for name, commit in (("baseline", baseline), ("implementation", implementation)): - if subprocess.check_output( - ["git", "-C", str(root), "rev-parse", commit], text=True - ).strip() != commit: - fail(f"B32 {name} identity changed") - -sources = { - name: (dut / "src" / name).read_text(encoding="utf-8") - for name in ("internal.h", "decompressor.c", "zdata.c", "zmap.c") -} -implemented_sources = { - name: committed(implementation, f"src/{name}") for name in sources -} -for name, owner in current_owners.items(): - if sources[name] != committed(owner, f"src/{name}"): - fail(f"current B32 source differs from accepted owner {owner}: {name}") -required_key_tokens = { - "nid": "cache->nid == vi->nid", - "decoded_size": "cache->decoded_size == decoded_size", - "m_pa": "cache->map.m_pa == map->m_pa", - "m_la": "cache->map.m_la == map->m_la", - "m_plen": "cache->map.m_plen == map->m_plen", - "m_llen": "cache->map.m_llen == map->m_llen", - "m_deviceid": "cache->map.m_deviceid == map->m_deviceid", - "m_algorithmformat": "cache->map.m_algorithmformat == map->m_algorithmformat", - "m_flags": "cache->map.m_flags == map->m_flags", -} -if list(required_key_tokens) != spec["key_fields"]: - fail("B32 key fixture field order changed") - - -def audit_b32_sources(label: str, source_set: dict[str, str]) -> None: - internal = source_set["internal.h"] - zdata = source_set["zdata.c"] - decompressor = source_set["decompressor.c"] - zmap = source_set["zmap.c"] - map_match = re.search( - r"struct erofs_map_blocks \{(?P.*?)\n\};", internal, re.S - ) - cache_match = re.search( - r"struct erofs_zextent_cache \{(?P.*?)\n\};", internal, re.S - ) - if map_match is None or cache_match is None: - fail(f"{label} canonical map/cache structures are absent") - if map_match.start() > cache_match.start(): - fail(f"{label} canonical map is not defined before embedded cache use") - map_body = map_match.group("body") - cache_body = cache_match.group("body") - if "uint8_t m_algorithmformat;" not in map_body: - fail(f"{label} m_algorithmformat is not an unsigned narrow type") - if "struct erofs_map_blocks map;" not in cache_body: - fail(f"{label} cache does not embed the canonical map") - for duplicate in required_key_tokens: - if duplicate in ("nid", "decoded_size"): - continue - if re.search( - rf"\b{duplicate}\b", - cache_body.replace("struct erofs_map_blocks map;", ""), - ): - fail(f"{label} cache duplicates canonical map field {duplicate}") - for field in ("decoded_size", "waiters", "error", "state", "cv", "closing"): - if not re.search(rf"\b{field}\b", cache_body): - fail(f"{label} cache state is missing {field}") - - match_body = function(zdata, "z_erofs_extent_cache_match") - for field, token in required_key_tokens.items(): - if match_body.count(token) != 1: - fail(f"{label} cache key does not compare {field} exactly once") - for token in ( - "cv_wait(&cache->cv, &sbi->z_extent_cache_lock)", - "cv_broadcast(&cache->cv)", - "cache->state = EROFS_ZCACHE_INFLIGHT", - "cache->state = EROFS_ZCACHE_READY", - "cache->state = EROFS_ZCACHE_FAILED", - "cache->waiters != 0", - "cache->error = error", - "cache->closing = true", - ): - if token not in zdata: - fail(f"{label} cache state-machine token is absent: {token}") - if "cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0" not in zdata: - fail(f"{label} eviction does not protect registered waiter generations") - if "z_erofs_extent_cache_eligible(sbi, vi, &map, decoded_len)" not in zdata: - fail(f"{label} cache admission is not based on decoded size") - if "Z_EROFS_CACHE_BYPASS" not in zdata or "free(decoded, M_EROFS);" not in zdata: - fail(f"{label} no-cache fallback is absent") - if ( - "uint8_t algorithm;" not in decompressor - or "(unsigned char)map->m_algorithmformat" in decompressor - ): - fail(f"{label} decompressor algorithm boundary is not unsigned narrow") - for token in ( - "(uint8_t)(map->m_plen >>", - "(uint8_t)(fmt - 1)", - "(uint8_t)(h->h_algorithmtype & 15)", - "(uint8_t)(h->h_algorithmtype >> 4)", - ): - if token not in zmap: - fail(f"{label} explicit on-disk conversion is absent: {token}") - for forbidden in ("bitlock", "wait_on_bit", "workqueue", "work_struct", "folio"): - if forbidden in "\n".join(source_set.values()): - fail(f"{label} introduced Linux-only primitive: {forbidden}") - - -audit_b32_sources("implementation", implemented_sources) -audit_b32_sources("current", sources) - -baseline_zdata = committed(baseline, "src/zdata.c") -for preserved in ("z_erofs_decode_length", "z_erofs_decode_extent"): - if function(implemented_sources["zdata.c"], preserved) != function( - baseline_zdata, preserved - ): - fail(f"B28 {preserved} changed outside B32 cache state") -for untouched in ( - "compress.h", - "decompressor_lz4.c", - "decompressor_lzma.c", - "decompressor_deflate.c", - "decompressor_zstd.c", -): - if committed(implementation, f"src/{untouched}") != committed( - baseline, f"src/{untouched}" - ): - fail(f"B32 changed B25/B27/B28 source outside its write set: {untouched}") - current = (dut / "src" / untouched).read_text(encoding="utf-8") - owner = current_owners["zmap.c"] if untouched == "decompressor_zstd.c" else baseline - if current != committed(owner, f"src/{untouched}"): - fail(f"current preserved codec source differs from accepted owner: {untouched}") - -report = { - "algorithm_type": "uint8_t", - "baseline": baseline, - "batch": "B32", - "cache_map_embedded": True, - "current_owners": current_owners, - "decoded_size_admission": True, - "implementation": implementation, - "key_fields": spec["key_fields"], - "lock": "z_extent_cache_lock", - "no_cache_fallback": True, - "preserved": ["B25-positive-errno", "B27-trailing", "B28-partial-fallback"], - "states": spec["states"], - "write_set": sorted(changed), -} -report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") -print(json.dumps(report, indent=2, sort_keys=True)) -PY -then - pre15_runner_fail 'B32 source, lock, key, or preservation audit failed' -fi - -if ! cc -std=c11 -Wall -Wextra -Werror -pthread "$oracle" \ - -o "$PRE15_CASE_TMP/B32-cache-oracle" \ - >"$artifacts/oracle-build.stdout" 2>"$artifacts/oracle-build.stderr"; then - pre15_runner_fail 'B32 host concurrency oracle did not compile' -fi -if ! timeout -k 5 60 "$PRE15_CASE_TMP/B32-cache-oracle" \ - >"$artifacts/oracle.stdout" 2>"$artifacts/oracle.stderr"; then - pre15_dut_fail 'B32 host concurrency oracle failed' -fi -sha256sum "$artifacts/B32-source-audit.json" "$artifacts/oracle.stdout" \ - >"$artifacts/SHA256SUMS" -fi -pre15_target_reached - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' \ - 'TC168-cache-inflight host PASS' \ - 'Canonical key, one owner, waiter bytes/positive errno, retry, eviction, fallback, and shutdown PASS' \ - 'TC126/TC127/TC129 host oracle NOT_RUN (not necessary)' \ - 'QEMU and full feature suite NOT_RUN in host mode' - exit 0 -fi - -for tool in clang cmp diff file mkfs.erofs nm scp tar timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B32 QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -first=$PRE15_CASE_TMP/fixtures-first -second=$PRE15_CASE_TMP/fixtures-second -if ! (umask 022 && timeout -k 5 180 python3 -B "$generator" --spec "$spec" \ - --output "$first") >"$artifacts/generate-first.json" \ - 2>"$artifacts/generate-first.stderr" || \ - ! (umask 022 && timeout -k 5 180 python3 -B "$generator" --spec "$spec" \ - --output "$second") >"$artifacts/generate-second.json" \ - 2>"$artifacts/generate-second.stderr"; then - pre15_runner_fail 'B32 fixture generation failed' -fi -if ! diff -qr "$first" "$second" >"$artifacts/fixture-repeat.diff"; then - pre15_runner_fail 'B32 fixtures are not byte reproducible' -fi -cp "$first/fixture-index.json" "$artifacts/B32-fixture-index.json" - -module=$PRE15_CASE_TMP/B32-erofs-zstdio0.ko -if ! timeout -k 10 600 /bin/sh "$kld_builder" "$PRE15_DUT" \ - "$PRE15_FREEBSD_SRC" "$module" "$PRE15_CASE_TMP/kld-work" 0 \ - >"$artifacts/kld-build.stdout" 2>"$artifacts/kld-build.stderr"; then - pre15_dut_fail 'B32 cross-target zstdio0 KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/kld-file.txt" -sha256sum "$module" >"$artifacts/kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/kld-nm-u.txt" - -fixture_archive=$PRE15_CASE_TMP/B32-fixtures.tar.gz -tar -C "$first" -czf "$fixture_archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -pre15_guest_ssh_bounded() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" ssh -n -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -p "$PRE15_QEMU_SSH_PORT" \ - "$PRE15_QEMU_SSH_USER@127.0.0.1" "$@" -} - -if ! pre15_scp "$fixture_archive" /root/B32-fixtures.tar.gz || \ - ! pre15_scp "$probe" /root/B32-cache-probe.c || \ - ! pre15_scp "$module" /root/B32-erofs-zstdio0.ko; then - pre15_infra_blocked 'could not transfer B32 module, probe, or fixtures' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/B32-fixtures && mkdir /root/B32-fixtures && tar -xzf /root/B32-fixtures.tar.gz -C /root/B32-fixtures && cc -O2 -Wall -Wextra -Werror -pthread -o /root/B32-cache-probe /root/B32-cache-probe.c'; then - pre15_infra_blocked 'could not prepare B32 guest fixtures/probe' -fi -if pre15_guest_ssh_bounded kldstat -q -m erofs >/dev/null 2>&1; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' -fi -pre15_guest_ssh_bounded kldstat >"$artifacts/guest-kldstat-before-load.txt" -pre15_guest_ssh_bounded dmesg >"$artifacts/guest-dmesg-before-load.txt" -cache_tunable=vfs.erofs.decoded_cache.minimum_decode_work -cache_tunable_before=$(pre15_guest_ssh_bounded kenv -q "$cache_tunable" || true) -printf 'name=%s before=%s test-value=0\n' "$cache_tunable" \ - "${cache_tunable_before:-unset}" >"$artifacts/cache-tunable.txt" -if ! pre15_guest_ssh_bounded kenv "$cache_tunable=0"; then - pre15_infra_blocked 'could not force B32 decoded-cache admission' -fi -if ! pre15_guest_ssh_bounded kldload /root/B32-erofs-zstdio0.ko \ - >"$artifacts/kldload.stdout" 2>"$artifacts/kldload.stderr"; then - if test -n "$cache_tunable_before"; then - pre15_guest_ssh_bounded kenv "$cache_tunable=$cache_tunable_before" || true - else - pre15_guest_ssh_bounded kenv -u "$cache_tunable" || true - fi - if grep -q 'module already loaded or in kernel' \ - "$artifacts/kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - pre15_dut_fail 'B32 exact-source zstdio0 KLD failed to load' -fi -if test -n "$cache_tunable_before"; then - pre15_guest_ssh_bounded kenv "$cache_tunable=$cache_tunable_before" || \ - pre15_infra_blocked 'could not restore B32 cache tunable' -else - pre15_guest_ssh_bounded kenv -u "$cache_tunable" || \ - pre15_infra_blocked 'could not remove B32 cache tunable override' -fi -cache_tunable_after=$(pre15_guest_ssh_bounded kenv -q "$cache_tunable" || true) -printf 'after=%s\n' "${cache_tunable_after:-unset}" \ - >>"$artifacts/cache-tunable.txt" -if test "$cache_tunable_after" != "$cache_tunable_before"; then - pre15_runner_fail 'B32 cache tunable was not restored exactly' -fi -if ! pre15_guest_ssh_bounded kldstat -q -m erofs; then - pre15_dut_fail 'B32 kldload returned success without erofs.1 ownership' -fi -pre15_guest_ssh_bounded kldstat >"$artifacts/guest-kldstat-after-load.txt" -pre15_guest_ssh_bounded dmesg >"$artifacts/guest-dmesg-before.txt" -diff -u "$artifacts/guest-dmesg-before-load.txt" \ - "$artifacts/guest-dmesg-before.txt" >"$artifacts/guest-dmesg-load.diff" || true -sed -n '/^+++ /d; /^+/s/^+//p' "$artifacts/guest-dmesg-load.diff" \ - >"$artifacts/guest-dmesg-load-added.txt" -if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free|fatal trap|pager fault|linker.*(error|undefined)|undefined symbol' \ - "$artifacts/guest-dmesg-load-added.txt"; then - pre15_dut_fail 'B32 exact-source KLD load produced kernel or linker errors' -fi -pre15_own_guest_kld B32-erofs-zstdio0.ko 'B32 exact-source KLD' - -b32_unown() -{ - kind=$1 - value=$2 - tmp=$PRE15_CASE_TMP/ownership.$$ - awk -F ' ' -v kind="$kind" -v value="$value" \ - '!($1 == kind && $2 == value)' "$PRE15_OWNERSHIP_FILE" >"$tmp" - mv "$tmp" "$PRE15_OWNERSHIP_FILE" -} - -b32_attach() -{ - image=$1 - label=$2 - B32_MD=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/B32-fixtures/images/$image") || \ - pre15_dut_fail "$label md attach failed" - case "$B32_MD" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected B32 md unit: $B32_MD" ;; - esac - pre15_own_guest_md "$B32_MD" "$label md" - B32_MOUNT=/mnt/pre15-b32-$label - pre15_guest_ssh_bounded mkdir -p "$B32_MOUNT" - pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$B32_MD" "$B32_MOUNT" || \ - pre15_dut_fail "$label mount failed" - pre15_own_guest_mount "$B32_MOUNT" "$label mount" -} - -b32_detach() -{ - if pre15_guest_ssh_bounded mount | grep -F " on $B32_MOUNT " >/dev/null; then - pre15_guest_ssh_bounded umount "$B32_MOUNT" || \ - pre15_dut_fail "$B32_MOUNT unmount failed" - fi - b32_unown guest-mount "$B32_MOUNT" - pre15_guest_ssh_bounded mdconfig -d -u "$B32_MD" || \ - pre15_dut_fail "$B32_MD detach failed" - b32_unown guest-md "$B32_MD" -} - -b32_start_lifecycle() -{ - label=$1 - B32_CONTROL=/tmp/B32-$label-control - B32_LIFECYCLE_PID=$(pre15_guest_ssh_bounded \ - "rm -rf '$B32_CONTROL'; mkdir '$B32_CONTROL'; nohup /root/B32-cache-probe lifecycle '$B32_MOUNT/payload.bin' '$B32_CONTROL' >/tmp/B32-$label.stdout 2>/tmp/B32-$label.stderr /dev/null; do i=\$((i + 1)); test \$i -lt 600 || exit 1; sleep 0.1; done" || \ - pre15_infra_blocked 'B32 lifecycle process did not exit' -} - -reference=/root/B32-fixtures/source/payload.bin -b32_attach lzma-valid.erofs valid -if ! pre15_guest_ssh_bounded /root/B32-cache-probe concurrent \ - "$B32_MOUNT/payload.bin" "$reference" 0 64 8 0 65536 \ - >"$artifacts/valid-concurrent.stdout" \ - 2>"$artifacts/valid-concurrent.stderr"; then - pre15_dut_fail 'TC168 valid same-key concurrent reads diverged' -fi -if ! grep -qx 'concurrent PASS workers=64 loops=8 assertions=512 errno=0 offset=0 length=65536' \ - "$artifacts/valid-concurrent.stdout"; then - pre15_runner_fail 'TC168 valid assertion count was not exactly 512' -fi -if test "$scenario" = TC184-vnode-races; then - if ! pre15_guest_ssh_bounded /root/B32-cache-probe vnode-race \ - "$B32_MOUNT/payload.bin" "$reference" 64 50 \ - >"$artifacts/vnode-race.stdout" \ - 2>"$artifacts/vnode-race.stderr"; then - pre15_dut_fail 'TC184 lookup/open/read/reclaim loop diverged' - fi - if ! grep -qx 'vnode-race PASS workers=64 loops=50 assertions=3200' \ - "$artifacts/vnode-race.stdout"; then - pre15_runner_fail 'TC184 vnode assertion count was not exactly 3200' - fi -fi -b32_detach - -if test "$scenario" = TC168-cache-inflight; then - b32_attach lzma-truncated.erofs truncated - if ! pre15_guest_ssh_bounded /root/B32-cache-probe concurrent \ - "$B32_MOUNT/payload.bin" "$reference" 97 1 1 0 65536 \ - >"$artifacts/failure-single.stdout" \ - 2>"$artifacts/failure-single.stderr"; then - pre15_dut_fail 'TC168 single corrupt read did not return EINTEGRITY' - fi - if ! grep -qx 'concurrent PASS workers=1 loops=1 assertions=1 errno=97 offset=0 length=65536' \ - "$artifacts/failure-single.stdout"; then - pre15_runner_fail 'TC168 single corrupt assertion count was not 1' - fi - if ! pre15_guest_ssh_bounded /root/B32-cache-probe concurrent \ - "$B32_MOUNT/payload.bin" "$reference" 97 64 2 0 65536 \ - >"$artifacts/failure-concurrent.stdout" \ - 2>"$artifacts/failure-concurrent.stderr"; then - pre15_dut_fail 'TC168 failure waiters did not receive EINTEGRITY' - fi - if ! grep -qx 'concurrent PASS workers=64 loops=2 assertions=128 errno=97 offset=0 length=65536' \ - "$artifacts/failure-concurrent.stdout"; then - pre15_runner_fail 'TC168 failure assertion count was not exactly 128' - fi - b32_detach -else - b32_attach lzma-valid.erofs normal-busy - b32_start_lifecycle normal-busy - if pre15_guest_ssh_bounded umount "$B32_MOUNT" \ - >"$artifacts/normal-unmount.stdout" 2>"$artifacts/normal-unmount.stderr"; then - pre15_dut_fail 'TC184 normal unmount with held vnode unexpectedly succeeded' - fi - if ! grep -qi 'busy' "$artifacts/normal-unmount.stderr"; then - pre15_dut_fail 'TC184 normal unmount did not report EBUSY' - fi - b32_stop_lifecycle - b32_detach - - b32_attach lzma-valid.erofs forced-held - b32_start_lifecycle forced-held - pre15_guest_ssh_bounded umount -f "$B32_MOUNT" || \ - pre15_dut_fail 'TC184 held-vnode forced unmount command failed' - b32_unown guest-mount "$B32_MOUNT" - pre15_guest_ssh_bounded touch "$B32_CONTROL/read" - pre15_guest_ssh_bounded \ - "i=0; while test ! -f '$B32_CONTROL/result'; do i=\$((i + 1)); test \$i -lt 600 || exit 1; sleep 0.1; done" || \ - pre15_infra_blocked 'TC184 held-vnode result timed out' - pre15_guest_ssh_bounded cat "$B32_CONTROL/result" \ - >"$artifacts/forced-held-result.txt" - if ! grep -qx -- '-1 6' "$artifacts/forced-held-result.txt"; then - pre15_dut_fail 'TC184 revoked held vnode did not return exact ENXIO' - fi - b32_detach - - b32_attach lzma-valid.erofs forced-closed - b32_start_lifecycle forced-closed - pre15_guest_ssh_bounded touch "$B32_CONTROL/close" - pre15_guest_ssh_bounded \ - "i=0; while test ! -f '$B32_CONTROL/closed'; do i=\$((i + 1)); test \$i -lt 600 || exit 1; sleep 0.1; done" || \ - pre15_infra_blocked 'TC184 closed-descriptor marker timed out' - pre15_guest_ssh_bounded umount -f "$B32_MOUNT" || \ - pre15_dut_fail 'TC184 closed-descriptor forced unmount command failed' - b32_unown guest-mount "$B32_MOUNT" - pre15_guest_ssh_bounded touch "$B32_CONTROL/read" - pre15_guest_ssh_bounded \ - "i=0; while test ! -f '$B32_CONTROL/result'; do i=\$((i + 1)); test \$i -lt 600 || exit 1; sleep 0.1; done" || \ - pre15_infra_blocked 'TC184 closed-descriptor result timed out' - pre15_guest_ssh_bounded cat "$B32_CONTROL/result" \ - >"$artifacts/forced-closed-result.txt" - if ! grep -qx -- '-1 9' "$artifacts/forced-closed-result.txt"; then - pre15_dut_fail 'TC184 closed descriptor did not return exact EBADF' - fi - b32_detach -fi - -pre15_guest_ssh_bounded dmesg >"$artifacts/guest-dmesg-after.txt" -diff -u "$artifacts/guest-dmesg-before.txt" \ - "$artifacts/guest-dmesg-after.txt" >"$artifacts/guest-dmesg.diff" || true -sed -n '/^+++ /d; /^+/s/^+//p' "$artifacts/guest-dmesg.diff" \ - >"$artifacts/guest-dmesg-added.txt" -if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free|fatal trap|pager fault|linker.*(error|undefined)|undefined symbol' \ - "$artifacts/guest-dmesg-added.txt"; then - pre15_dut_fail "$scenario produced kernel, linker, WITNESS, or UAF evidence" -fi -printf '%s PASS\n' "$scenario" -printf '%s\n' \ - 'QEMU used an owned overlay and dynamic forwarded port' \ - 'KLD erofs.1 ownership and cache tunable restoration PASS' \ - 'TC130 and full feature suite NOT_RUN' diff --git a/tests/pre15/cases/B33-cache-policy.sh b/tests/pre15/cases/B33-cache-policy.sh deleted file mode 100755 index a1c8c44..0000000 --- a/tests/pre15/cases/B33-cache-policy.sh +++ /dev/null @@ -1,641 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=93d0c64e321f50cf68026c3f1c0c95293bf17302 -case_script=$PRE15_DUT/tests/pre15/cases/B33-cache-policy.sh -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -gate_dir=$PRE15_DUT/tests/pre15/gates -oracle=$fixture_dir/B33-cache-oracle.c -qemu_spec=$fixture_dir/B33-cache-state.json -qemu_generator=$fixture_dir/B33-cache-generate.py -qemu_runner=$fixture_dir/B33-qemu-run.sh -b32_probe=$fixture_dir/B32-cache-probe.c -kld_builder=$fixture_dir/B28-build-kld.sh -gate_input=$gate_dir/P15-038-input.json -gate_result=$PRE15_ROOT/planning/pre15/evidence/20260816T021736Z-G05-P15-038/result.json -gate_benefit=$PRE15_ROOT/planning/pre15/evidence/20260816T021736Z-G05-P15-038/benefit.json -gate_state=$PRE15_ROOT/planning/pre15/evidence/20260816T021736Z-G05-P15-038/state-model.json -gate_hashes=$PRE15_ROOT/planning/pre15/evidence/20260816T021736Z-G05-P15-038/SHA256SUMS -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in cc git python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B33 host tool: $tool" -done -for fixture in "$case_script" "$oracle" "$qemu_spec" "$qemu_generator" "$qemu_runner" \ - "$b32_probe" "$kld_builder" "$gate_input" "$gate_result" \ - "$gate_benefit" "$gate_state" "$gate_hashes"; do - pre15_record_fixture "b33-$(basename "$fixture")" "$fixture" -done -for source in internal.h zdata.c; do - pre15_record_fixture "b33-$source" "$PRE15_DUT/src/$source" -done -mkdir -p "$artifacts" - -if test "${PRE15_QEMU_TARGET_ONLY:-0}" = 1; then - pre15_target_reached -else -if ! python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" "$gate_input" \ - "$gate_result" "$gate_benefit" "$gate_state" \ - "$artifacts/B33-source-audit.json" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -implementation = "bc56f830918b76029871b60cca2e53992de70a2e" -current_owners = { - "internal.h": "aa20623132b6821c6ad4651b263fc0ac0f7a14c8", - "zdata.c": implementation, -} -gate_input = json.loads(Path(sys.argv[4]).read_text(encoding="ascii")) -gate_result = json.loads(Path(sys.argv[5]).read_text(encoding="ascii")) -gate_benefit = json.loads(Path(sys.argv[6]).read_text(encoding="ascii")) -gate_state = json.loads(Path(sys.argv[7]).read_text(encoding="ascii")) -report_path = Path(sys.argv[8]) - - -def fail(message: str) -> None: - raise SystemExit(message) - - -def committed(commit: str, relative: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{commit}:{relative}"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=30, - ) - if completed.returncode != 0: - fail(f"cannot read B33 source {commit}:{relative}: {completed.stderr.strip()}") - return completed.stdout - - -def function(source: str, name: str) -> str: - match = re.search(rf"\n(?:static\s+)?[^\n]+\n{name}\([^{{]+\n\{{", source) - if match is None: - fail(f"cannot locate function {name}") - start = match.start() + 1 - brace = source.find("{", match.start()) - depth = 0 - for index in range(brace, len(source)): - if source[index] == "{": - depth += 1 - elif source[index] == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - fail(f"unterminated function {name}") - - -expected_changed = { - "repo-pre-15/src/internal.h", - "repo-pre-15/src/zdata.c", - "repo-pre-15/tests/pre15/cases/B33-cache-policy.sh", - "repo-pre-15/tests/pre15/fixtures/B33-cache-generate.py", - "repo-pre-15/tests/pre15/fixtures/B33-cache-oracle.c", - "repo-pre-15/tests/pre15/fixtures/B33-cache-state.json", - "repo-pre-15/tests/pre15/fixtures/B33-qemu-run.sh", -} -changed = subprocess.run( - [ - "git", - "-C", - str(root), - "diff-tree", - "--no-commit-id", - "--name-only", - "-r", - implementation, - ], - check=True, - stdout=subprocess.PIPE, - text=True, - timeout=30, -).stdout.splitlines() -if set(changed) != expected_changed or len(changed) != len(expected_changed): - fail(f"B33 implementation write set mismatch: {changed!r}") - -for name, commit in (("baseline", baseline), ("implementation", implementation)): - if subprocess.check_output( - ["git", "-C", str(root), "rev-parse", commit], text=True - ).strip() != commit: - fail(f"B33 {name} identity changed") - -internal = (dut / "src/internal.h").read_text(encoding="utf-8") -zdata = (dut / "src/zdata.c").read_text(encoding="utf-8") -implemented_internal = committed(implementation, "repo-pre-15/src/internal.h") -implemented_zdata = committed(implementation, "repo-pre-15/src/zdata.c") -for name, source in (("internal.h", internal), ("zdata.c", zdata)): - owner = current_owners[name] - if source != committed(owner, f"repo-pre-15/src/{name}"): - fail(f"current B33 source differs from accepted owner {owner}: {name}") - -baseline_zdata = committed(baseline, "repo-pre-15/src/zdata.c") -for preserved in ("z_erofs_decode_length", "z_erofs_decode_extent"): - if function(implemented_zdata, preserved) != function(baseline_zdata, preserved): - fail(f"B25/B27/B28 read/decode path changed in B33: {preserved}") - -do_read = function(implemented_zdata, "z_erofs_do_read") -baseline_do_read = function(baseline_zdata, "z_erofs_do_read") -bypass_delta = ( - "\t\t\t} else\n" - "\t\t\t\tz_erofs_extent_cache_record_bypass(sbi, &map);" -) -if do_read.count(bypass_delta) != 1 or do_read.replace(bypass_delta, "\t\t\t}") != baseline_do_read: - fail("B33 z_erofs_do_read differs beyond policy-rejection accounting") - -required_b32 = ( - "cache->nid == vi->nid", - "cache->decoded_size == decoded_size", - "cache->map.m_pa == map->m_pa", - "cache->map.m_la == map->m_la", - "cache->map.m_plen == map->m_plen", - "cache->map.m_llen == map->m_llen", - "cache->map.m_deviceid == map->m_deviceid", - "cache->map.m_algorithmformat == map->m_algorithmformat", - "cache->map.m_flags == map->m_flags", - "cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0", - "cv_wait(&cache->cv, &sbi->z_extent_cache_lock)", - "cache->error = error", - "cache->closing = true", - "Z_EROFS_CACHE_BYPASS", - "free(decoded, M_EROFS);", -) -missing = [token for token in required_b32 if token not in zdata] -if missing: - fail(f"B32 key/inflight/failure/fallback token missing: {missing!r}") - -policy = function(zdata, "z_erofs_extent_cache_eligible") -for named_codec in ( - "Z_EROFS_COMPRESSION_LZ4", - "Z_EROFS_COMPRESSION_LZMA", - "Z_EROFS_COMPRESSION_DEFLATE", - "Z_EROFS_COMPRESSION_ZSTD", -): - if named_codec in policy: - fail(f"B33 admission still names a codec: {named_codec}") -for token in ( - "map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX", - "len <= cache->budget_bytes", - "decode_work >= cache->minimum_decode_work", - "z_erofs_cache_enabled != 0", -): - if token not in policy: - fail(f"codec-neutral work/size policy token is absent: {token}") -if "map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA" in zdata: - fail("LZMA-only admission branch remains") -record_bypass = function(zdata, "z_erofs_extent_cache_record_bypass") -if "++cache->metrics[map->m_algorithmformat].bypasses" not in record_bypass or ( - "z_erofs_extent_cache_record_bypass(sbi, &map)" not in do_read -): - fail("B33 policy rejection is absent from four-codec accounting") - -budget = gate_input["budget"] -expected_macros = { - "EROFS_ZCACHE_MOUNT_HARD_BUDGET": budget["per_mount_bytes"] // 1024, - "EROFS_ZCACHE_GLOBAL_HARD_BUDGET": budget["global_bytes"] // 1024, - "EROFS_ZCACHE_MIN_DECODE_WORK": budget["minimum_decode_work_bytes"] // 1024, -} -for name, kib in expected_macros.items(): - if f"#define {name} ({kib}UL * 1024)" not in zdata: - fail(f"B33 hard policy differs from G05: {name}") -for token in ( - 'TUNABLE_INT("vfs.erofs.decoded_cache.enabled"', - 'TUNABLE_ULONG("vfs.erofs.decoded_cache.mount_budget"', - 'TUNABLE_ULONG("vfs.erofs.decoded_cache.global_budget"', - 'TUNABLE_ULONG("vfs.erofs.decoded_cache.minimum_decode_work"', - "MIN(z_erofs_cache_mount_budget,", - "MIN(z_erofs_cache_global_budget,", - "EVENTHANDLER_REGISTER(vm_lowmem", - "EVENTHANDLER_DEREGISTER(vm_lowmem", - "LIST_FOREACH(sbi, &z_erofs_cache_mounts, z_extent_cache_link)", - "mtx_trylock(&sbi->z_extent_cache_lock)", -): - if token not in zdata: - fail(f"B33 budget/config/reclaim token is absent: {token}") - -claim = function(zdata, "z_erofs_extent_cache_claim") -if claim.find("z_erofs_extent_cache_reserve(decoded_size)") > claim.find( - "cache->state = EROFS_ZCACHE_INFLIGHT" -): - fail("B33 does not reserve before inflight decode") -drop_position = claim.find("z_erofs_extent_cache_drop_locked(cache, true, false)") -empty_position = claim.find("cache->state = EROFS_ZCACHE_EMPTY", drop_position) -reserve_position = claim.find("z_erofs_extent_cache_reserve(decoded_size)") -if drop_position < 0 or not (drop_position < empty_position < reserve_position): - fail("B33 replacement reservation failure can leave READY without data") -complete = function(zdata, "z_erofs_extent_cache_complete") -if complete.find("z_erofs_extent_cache_release(cache->charged_bytes)") > complete.find( - "cache->state = EROFS_ZCACHE_FAILED" -): - fail("B33 failed decode publishes before releasing reservation") - -for field in ( - "hits", - "misses", - "bypasses", - "evictions", - "reclaims", - "resident_bytes", - "charged_bytes", - "budget_bytes", - "minimum_decode_work", - "metrics[Z_EROFS_COMPRESSION_MAX]", -): - if field not in internal: - fail(f"B33 four-codec accounting field is absent: {field}") -for forbidden in ("shrinker", "workqueue", "work_struct", "wait_on_bit", "folio"): - if forbidden in internal + zdata: - fail(f"Linux-only cache primitive introduced: {forbidden}") - -if gate_result.get("status") != "GO" or gate_result.get("b33") != "AUTHORIZED": - fail("P15-038 G05 result is not GO") -if gate_state.get("status") != "PASS" or gate_state["budget"]["global_remaining"] != 0: - fail("P15-038 state model is not closed") -if gate_benefit.get("codecs_meeting_threshold", 0) < 2 or any( - len(summary["current_ns"]) != 5 or len(summary["candidate_ns"]) != 5 - for summary in gate_benefit.get("summaries", []) -): - fail("P15-038 fixed five-sample benefit is incomplete") - -report = { - "baseline": baseline, - "batch": "B33", - "b32_contract_preserved": True, - "codec_neutral": True, - "current_owners": current_owners, - "gate_codecs_meeting_threshold": gate_benefit["codecs_meeting_threshold"], - "global_hard_budget": budget["global_bytes"], - "implementation": implementation, - "lock_order": ["global-list", "mount-cache", "global-budget"], - "mount_hard_budget": budget["per_mount_bytes"], - "no_cache_fallback": True, - "reclaim": "FreeBSD-vm_lowmem", - "write_set": sorted(changed), -} -report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") -print(json.dumps(report, indent=2, sort_keys=True)) -PY -then - pre15_runner_fail 'B33 source, gate, write-set, or preservation audit failed' -fi - -if ! (cd "$PRE15_ROOT/planning/pre15/evidence/20260816T021736Z-G05-P15-038" && \ - timeout -k 5 30 sha256sum -c SHA256SUMS) \ - >"$artifacts/gate-manifest.stdout" 2>"$artifacts/gate-manifest.stderr"; then - pre15_runner_fail 'P15-038 gate evidence manifest failed' -fi -if ! cc -std=c11 -O2 -Wall -Wextra -Werror "$oracle" \ - -o "$PRE15_CASE_TMP/B33-cache-oracle" \ - >"$artifacts/oracle-build.stdout" 2>"$artifacts/oracle-build.stderr"; then - pre15_runner_fail 'B33 cache budget oracle did not compile' -fi -if ! timeout -k 5 60 "$PRE15_CASE_TMP/B33-cache-oracle" \ - >"$artifacts/oracle.json" 2>"$artifacts/oracle.stderr"; then - pre15_dut_fail 'B33 cache budget/accounting oracle failed' -fi -if ! python3 -B - "$artifacts/oracle.json" <<'PY' -import json -from pathlib import Path -import sys - -result = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -if result.get("status") != "PASS": - raise SystemExit("B33 oracle status is not PASS") -if result.get("global_peak") != result.get("global_limit") or result.get("global_remaining") != 0: - raise SystemExit("B33 oracle global budget did not close") -metrics = result.get("metrics", []) -if len(metrics) != 4 or [item.get("codec") for item in metrics] != list(range(4)): - raise SystemExit("B33 four-codec metric set is incomplete") -for item in metrics: - if item.get("hits", 0) < 1 or item.get("misses", 0) < 1 or item.get("evictions", 0) < 1: - raise SystemExit(f"B33 codec metric is incomplete: {item!r}") - if item.get("resident_bytes") != 0: - raise SystemExit(f"B33 codec resident bytes leaked: {item!r}") -PY -then - pre15_dut_fail 'B33 four-codec accounting result is incomplete' -fi - -sha256sum "$artifacts/B33-source-audit.json" "$artifacts/oracle.json" \ - "$artifacts/gate-manifest.stdout" >"$artifacts/SHA256SUMS" -fi -pre15_target_reached - -if test "${PRE15_MODE:-host}" != qemu; then - printf '%s\n' \ - 'B33-cache-budget host PASS' \ - 'Four-codec admission/accounting, mount/global hard budgets, exhaustion fallback, reclaim, failure retry, disable, and eviction hash PASS' \ - 'P15-038 host codec-cost gate replay manifest PASS; no guest vnode performance claimed' \ - 'TC084/TC105/TC129 NOT_RUN (not necessary)' \ - 'QEMU TC168, TC130, and full feature suite NOT_RUN in host mode' - exit 0 -fi - -test "${PRE15_B32_SCENARIO:-}" = TC168-cache-inflight || \ - pre15_runner_fail 'B33 QEMU mode requires TC168-cache-inflight' -for tool in clang cmp diff file mkfs.erofs nm scp tar timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B33 QEMU tool: $tool" -done -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -first=$PRE15_CASE_TMP/fixtures-first -second=$PRE15_CASE_TMP/fixtures-second -if ! (umask 022 && timeout -k 5 180 python3 -B "$qemu_generator" \ - --spec "$qemu_spec" --output "$first") \ - >"$artifacts/generate-first.json" 2>"$artifacts/generate-first.stderr" || \ - ! (umask 022 && timeout -k 5 180 python3 -B "$qemu_generator" \ - --spec "$qemu_spec" --output "$second") \ - >"$artifacts/generate-second.json" 2>"$artifacts/generate-second.stderr"; then - pre15_runner_fail 'B33 fixture generation failed' -fi -if ! diff -qr "$first" "$second" >"$artifacts/fixture-repeat.diff"; then - pre15_runner_fail 'B33 fixtures are not byte reproducible' -fi -cp "$first/fixture-index.json" "$artifacts/B33-fixture-index.json" - -module=$PRE15_CASE_TMP/B33-erofs-zstdio0.ko -if ! timeout -k 10 600 /bin/sh "$kld_builder" "$PRE15_DUT" \ - "$PRE15_FREEBSD_SRC" "$module" "$PRE15_CASE_TMP/kld-work" 0 \ - >"$artifacts/kld-build.stdout" 2>"$artifacts/kld-build.stderr"; then - pre15_dut_fail 'B33 cross-target zstdio0 KLD build failed' -fi -pre15_record_module "$module" -file "$module" >"$artifacts/kld-file.txt" -sha256sum "$module" >"$artifacts/kld-sha256.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/kld-nm-u.txt" - -fixture_archive=$PRE15_CASE_TMP/B33-fixtures.tar.gz -tar -C "$first" -czf "$fixture_archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -pre15_guest_ssh_bounded() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" ssh -n -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -p "$PRE15_QEMU_SSH_PORT" \ - "$PRE15_QEMU_SSH_USER@127.0.0.1" "$@" -} - -if ! pre15_scp "$fixture_archive" /root/B33-fixtures.tar.gz || \ - ! pre15_scp "$b32_probe" /root/B33-cache-probe.c || \ - ! pre15_scp "$module" /root/B33-erofs-zstdio0.ko; then - pre15_infra_blocked 'could not transfer B33 module, probe, or fixtures' -fi -if ! pre15_guest_ssh_bounded \ - 'rm -rf /root/B33-fixtures && mkdir /root/B33-fixtures && tar -xzf /root/B33-fixtures.tar.gz -C /root/B33-fixtures && cc -O2 -Wall -Wextra -Werror -pthread -o /root/B33-cache-probe /root/B33-cache-probe.c'; then - pre15_infra_blocked 'could not prepare B33 guest fixtures/probe' -fi -if pre15_guest_ssh_bounded kldstat -q -m erofs >/dev/null 2>&1; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' -fi -module_sha256=$(sha256sum "$module" | awk '{print $1}') -guest_module_sha256=$(pre15_guest_ssh_bounded sha256 -q /root/B33-erofs-zstdio0.ko) || \ - pre15_infra_blocked 'could not hash transferred B33 KLD' -printf 'host=%s guest=%s\n' "$module_sha256" "$guest_module_sha256" \ - >"$artifacts/guest-module-sha256.txt" -test "$guest_module_sha256" = "$module_sha256" || \ - pre15_runner_fail 'transferred B33 KLD hash differs from host module' -pre15_guest_ssh_bounded kldstat >"$artifacts/guest-kldstat-before-load.txt" -pre15_guest_ssh_bounded dmesg >"$artifacts/guest-dmesg-before-load.txt" -cache_tunable=vfs.erofs.decoded_cache.minimum_decode_work -cache_tunable_before=$(pre15_guest_ssh_bounded kenv -q "$cache_tunable" || true) -printf 'name=%s before=%s test-value=0\n' "$cache_tunable" \ - "${cache_tunable_before:-unset}" >"$artifacts/cache-tunable.txt" -if ! pre15_guest_ssh_bounded kenv "$cache_tunable=0" \ - >"$artifacts/cache-tunable-set.stdout" \ - 2>"$artifacts/cache-tunable-set.stderr"; then - pre15_infra_blocked 'could not force B33 decoded-cache admission' -fi -if ! pre15_guest_ssh_bounded kldload /root/B33-erofs-zstdio0.ko \ - >"$artifacts/kldload.stdout" 2>"$artifacts/kldload.stderr"; then - if test -n "$cache_tunable_before"; then - pre15_guest_ssh_bounded kenv "$cache_tunable=$cache_tunable_before" || true - else - pre15_guest_ssh_bounded kenv -u "$cache_tunable" || true - fi - if grep -q 'module already loaded or in kernel' \ - "$artifacts/kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - pre15_dut_fail 'B33 exact-source zstdio0 KLD failed to load' -fi -if test -n "$cache_tunable_before"; then - pre15_guest_ssh_bounded kenv "$cache_tunable=$cache_tunable_before" \ - >"$artifacts/cache-tunable-restore.stdout" \ - 2>"$artifacts/cache-tunable-restore.stderr" || \ - pre15_infra_blocked 'could not restore B33 cache tunable' -else - pre15_guest_ssh_bounded kenv -u "$cache_tunable" \ - >"$artifacts/cache-tunable-restore.stdout" \ - 2>"$artifacts/cache-tunable-restore.stderr" || \ - pre15_infra_blocked 'could not remove B33 cache tunable override' -fi -cache_tunable_after=$(pre15_guest_ssh_bounded kenv -q "$cache_tunable" || true) -printf 'after=%s\n' "${cache_tunable_after:-unset}" \ - >>"$artifacts/cache-tunable.txt" -if test "$cache_tunable_after" != "$cache_tunable_before"; then - pre15_runner_fail 'B33 cache tunable was not restored exactly' -fi -if ! pre15_guest_ssh_bounded kldstat -q -m erofs; then - pre15_dut_fail 'B33 kldload returned success without erofs.1 ownership' -fi -if ! pre15_guest_ssh_bounded kldstat -q -n B33-erofs-zstdio0.ko; then - pre15_dut_fail 'B33 kldload returned success without exact KLD file ownership' -fi -pre15_guest_ssh_bounded kldstat >"$artifacts/guest-kldstat-after-load.txt" -pre15_guest_ssh_bounded dmesg >"$artifacts/guest-dmesg-before.txt" -diff -u "$artifacts/guest-dmesg-before-load.txt" \ - "$artifacts/guest-dmesg-before.txt" >"$artifacts/guest-dmesg-load.diff" || true -sed -n '/^+++ /d; /^+/s/^+//p' "$artifacts/guest-dmesg-load.diff" \ - >"$artifacts/guest-dmesg-load-added.txt" -if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free|fatal trap|pager fault|linker.*(error|undefined)|undefined symbol' \ - "$artifacts/guest-dmesg-load-added.txt"; then - pre15_dut_fail 'B33 exact-source KLD load produced kernel or linker errors' -fi -pre15_own_guest_kld B33-erofs-zstdio0.ko 'B33 exact-source KLD' -pre15_guest_ssh_bounded kldstat >"$artifacts/guest-kldstat-before-trace.txt" -if ! pre15_guest_ssh_bounded command -v dtrace \ - >"$artifacts/guest-dtrace-command.txt" 2>"$artifacts/guest-dtrace-command.stderr"; then - pre15_infra_blocked 'guest dtrace tool is unavailable for B33 call proof' -fi -set +e -pre15_guest_ssh_bounded kldload dtraceall \ - >"$artifacts/guest-dtrace-kldload.stdout" \ - 2>"$artifacts/guest-dtrace-kldload.stderr" -dtrace_load_rc=$? -set -e -pre15_guest_ssh_bounded kldstat >"$artifacts/guest-kldstat-after-trace.txt" -awk 'NR == FNR { if (FNR > 1) seen[$5] = 1; next } - FNR > 1 && !seen[$5] { print $1 "\t" $5 }' \ - "$artifacts/guest-kldstat-before-trace.txt" \ - "$artifacts/guest-kldstat-after-trace.txt" \ - >"$artifacts/guest-trace-klds-all.tsv" -awk '$2 == "dtraceall.ko"' "$artifacts/guest-trace-klds-all.tsv" \ - >"$artifacts/guest-trace-klds.tsv" -while IFS="$(printf '\t')" read -r trace_kld_id trace_kld_name; do - test -n "$trace_kld_id" || continue - pre15_own_guest_kld "$trace_kld_name" 'B33 FBT call-proof dependency' -done <"$artifacts/guest-trace-klds.tsv" -test "$dtrace_load_rc" -eq 0 || \ - pre15_infra_blocked 'guest dtraceall KLD failed to load for B33 call proof' -if ! pre15_guest_ssh_bounded \ - "dtrace -l -n 'fbt::z_erofs_extent_cache_init:entry' -n 'fbt::z_erofs_do_read:entry'" \ - >"$artifacts/guest-dtrace-probe-list.txt" \ - 2>"$artifacts/guest-dtrace-probe-list.stderr"; then - pre15_infra_blocked 'B33 cache init/read FBT probes are unavailable' -fi -grep -q 'z_erofs_extent_cache_init.*entry' \ - "$artifacts/guest-dtrace-probe-list.txt" || \ - pre15_runner_fail 'B33 cache init FBT probe was not listed exactly' -grep -q 'z_erofs_do_read.*entry' \ - "$artifacts/guest-dtrace-probe-list.txt" || \ - pre15_runner_fail 'B33 compressed-read FBT probe was not listed exactly' - -b33_unown() -{ - kind=$1 - value=$2 - tmp=$PRE15_CASE_TMP/ownership.$$ - awk -F ' ' -v kind="$kind" -v value="$value" \ - '!($1 == kind && $2 == value)' "$PRE15_OWNERSHIP_FILE" >"$tmp" - mv "$tmp" "$PRE15_OWNERSHIP_FILE" -} - -b33_attach() -{ - image=$1 - label=$2 - B33_MD=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/B33-fixtures/images/$image") || \ - pre15_dut_fail "$label md attach failed" - case "$B33_MD" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected B33 md unit: $B33_MD" ;; - esac - pre15_own_guest_md "$B33_MD" "$label md" - B33_MOUNT=/mnt/pre15-b33-$label - pre15_guest_ssh_bounded mkdir -p "$B33_MOUNT" - if test "${B33_TRACE_MOUNT:-0}" = 1; then - mount_trace=/tmp/pre15-b33-cache-init.trace - pre15_guest_ssh_bounded \ - "dtrace -q -o '$mount_trace' -n 'fbt::z_erofs_extent_cache_init:entry { @calls = count(); } END { printa(@calls); }' -c 'mount -t erofs -o ro /dev/$B33_MD $B33_MOUNT'" \ - >"$artifacts/valid-mount.stdout" \ - 2>"$artifacts/valid-mount.stderr" || \ - pre15_dut_fail "$label traced mount failed" - pre15_guest_ssh_bounded cat "$mount_trace" \ - >"$artifacts/guest-cache-init-fbt.txt" - if ! awk '$1 ~ /^[0-9]+$/ && $1 > 0 { found = 1 } - END { exit !found }' "$artifacts/guest-cache-init-fbt.txt"; then - pre15_dut_fail 'B33 guest mount did not call decoded-cache init' - fi - else - pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$B33_MD" "$B33_MOUNT" || \ - pre15_dut_fail "$label mount failed" - fi - pre15_own_guest_mount "$B33_MOUNT" "$label mount" -} - -b33_detach() -{ - if pre15_guest_ssh_bounded mount | grep -F " on $B33_MOUNT " >/dev/null; then - pre15_guest_ssh_bounded umount "$B33_MOUNT" || \ - pre15_dut_fail "$B33_MOUNT unmount failed" - fi - b33_unown guest-mount "$B33_MOUNT" - pre15_guest_ssh_bounded mdconfig -d -u "$B33_MD" || \ - pre15_dut_fail "$B33_MD detach failed" - b33_unown guest-md "$B33_MD" -} - -reference=/root/B33-fixtures/source/payload.bin -B33_TRACE_MOUNT=1 -b33_attach lz4-valid.erofs valid -unset B33_TRACE_MOUNT -trace_file=/tmp/pre15-b33-do-read.trace -if ! pre15_guest_ssh_bounded \ - "dtrace -q -o '$trace_file' -n 'fbt::z_erofs_do_read:entry { @calls = count(); } END { printa(@calls); }' -c '/root/B33-cache-probe concurrent $B33_MOUNT/payload.bin $reference 0 64 4 0 65536'" \ - >"$artifacts/valid-concurrent.stdout" \ - 2>"$artifacts/valid-concurrent.stderr"; then - pre15_dut_fail 'TC168 valid full-extent concurrent reads diverged' -fi -if ! grep -qx 'concurrent PASS workers=64 loops=4 assertions=256 errno=0 offset=0 length=65536' \ - "$artifacts/valid-concurrent.stdout"; then - pre15_runner_fail 'TC168 valid assertion count was not exactly 256' -fi -pre15_guest_ssh_bounded cat "$trace_file" \ - >"$artifacts/guest-do-read-fbt.txt" -if ! awk '$1 ~ /^[0-9]+$/ && $1 > 0 { found = 1 } - END { exit !found }' "$artifacts/guest-do-read-fbt.txt"; then - pre15_dut_fail 'B33 guest reads did not call the target read implementation' -fi -b33_detach - -b33_attach lz4-truncated.erofs truncated -if ! pre15_guest_ssh_bounded /root/B33-cache-probe concurrent \ - "$B33_MOUNT/payload.bin" "$reference" 97 1 1 0 65536 \ - >"$artifacts/failure-single.stdout" \ - 2>"$artifacts/failure-single.stderr"; then - pre15_dut_fail 'TC168 single corrupt read did not return EINTEGRITY' -fi -if ! grep -qx 'concurrent PASS workers=1 loops=1 assertions=1 errno=97 offset=0 length=65536' \ - "$artifacts/failure-single.stdout"; then - pre15_runner_fail 'TC168 single corrupt assertion count was not 1' -fi -if ! pre15_guest_ssh_bounded /root/B33-cache-probe concurrent \ - "$B33_MOUNT/payload.bin" "$reference" 97 64 2 0 65536 \ - >"$artifacts/failure-concurrent.stdout" \ - 2>"$artifacts/failure-concurrent.stderr"; then - pre15_dut_fail 'TC168 failure waiters did not receive EINTEGRITY' -fi -if ! grep -qx 'concurrent PASS workers=64 loops=2 assertions=128 errno=97 offset=0 length=65536' \ - "$artifacts/failure-concurrent.stdout"; then - pre15_runner_fail 'TC168 failure assertion count was not exactly 128' -fi -b33_detach - -pre15_guest_ssh_bounded dmesg >"$artifacts/guest-dmesg-after.txt" -diff -u "$artifacts/guest-dmesg-before.txt" \ - "$artifacts/guest-dmesg-after.txt" >"$artifacts/guest-dmesg.diff" || true -sed -n '/^+++ /d; /^+/s/^+//p' "$artifacts/guest-dmesg.diff" \ - >"$artifacts/guest-dmesg-added.txt" -if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free|fatal trap|pager fault|linker.*(error|undefined)|undefined symbol' \ - "$artifacts/guest-dmesg-added.txt"; then - pre15_dut_fail 'TC168 produced panic, WITNESS, or UAF evidence' -fi -printf '%s\n' \ - 'TC168-cache-inflight B33 QEMU PASS' \ - 'Exact KLD ownership, FBT-proven cache init/read calls, full-extent success/failure waiters, unmount drain, and cleanup exercised' \ - 'TC130 and full feature suite NOT_RUN' diff --git a/tests/pre15/cases/B34-build-groups.sh b/tests/pre15/cases/B34-build-groups.sh deleted file mode 100755 index c082a43..0000000 --- a/tests/pre15/cases/B34-build-groups.sh +++ /dev/null @@ -1,412 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -baseline=5cb420221ab0a4da7838c68db55280999474038b -makefile=$PRE15_DUT/src/Makefile -linux_makefile=$PRE15_ROOT/src-linux/Makefile -case_file=$PRE15_DUT/tests/pre15/cases/B34-build-groups.sh -artifacts=$PRE15_RUN_DIR/artifacts - -for tool in git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing B34 host tool: $tool" -done - -pre15_record_fixture b34-makefile "$makefile" -pre15_record_fixture b34-linux-makefile "$linux_makefile" -pre15_record_fixture b34-case "$case_file" - -mkdir -p "$artifacts" -pre15_target_reached - -if python3 -B - "$PRE15_ROOT" "$PRE15_DUT" "$baseline" \ - "$artifacts/B34-object-list.json" <<'PY' -from __future__ import annotations - -from collections import Counter -from dataclasses import dataclass -from fnmatch import fnmatchcase -import json -from pathlib import Path -import re -import subprocess -import sys - - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -baseline = sys.argv[3] -report_path = Path(sys.argv[4]) -makefile_path = dut / "src/Makefile" - - -def committed(path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B34 baseline {path}: {completed.stderr}") - return completed.stdout - - -def changed_paths() -> list[str]: - changed = subprocess.run( - [ - "git", - "-C", - str(root), - "diff", - "--name-only", - baseline, - "--", - "repo-pre-15", - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() - changed.extend( - subprocess.run( - [ - "git", - "-C", - str(root), - "ls-files", - "--others", - "--exclude-standard", - "--", - "repo-pre-15", - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() - ) - return changed - - -expected_changed = [ - "repo-pre-15/src/Makefile", - "repo-pre-15/tests/pre15/cases/B34-build-groups.sh", -] -changed = changed_paths() -if sorted(changed) != expected_changed or len(changed) != len(expected_changed): - raise SystemExit(f"B34 repo-pre-15 write set mismatch: {changed!r}") - - -@dataclass -class Evaluation: - variables: dict[str, str] - includes: list[str] - src_operations: list[tuple[str, list[str]]] - - -class MakeError(Exception): - pass - - -def logical_lines(source: str) -> list[tuple[int, str]]: - result: list[tuple[int, str]] = [] - parts: list[str] = [] - start_line = 0 - for line_number, raw_line in enumerate(source.splitlines(), 1): - content = raw_line.split("#", 1)[0].rstrip() - if not parts and not content.strip(): - continue - if not parts: - start_line = line_number - continued = content.endswith("\\") - if continued: - content = content[:-1].rstrip() - parts.append(content.strip()) - if not continued: - result.append((start_line, " ".join(part for part in parts if part))) - parts = [] - if parts: - raise SystemExit(f"unterminated Makefile continuation at line {start_line}") - return result - - -def expand(value: str, variables: dict[str, str]) -> str: - pattern = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_.]*)\}") - previous = None - while value != previous: - previous = value - value = pattern.sub(lambda match: variables.get(match.group(1), ""), value) - return value - - -def condition_value(expression: str, variables: dict[str, str], line: int) -> bool: - terms = [term.strip() for term in expression.split("&&")] - values: list[bool] = [] - for term in terms: - empty_match = re.fullmatch( - r"empty\(([A-Za-z_][A-Za-z0-9_.]*):M([^()]*)\)", term - ) - if empty_match: - variable, pattern = empty_match.groups() - words = variables.get(variable, "").split() - values.append(not any(fnmatchcase(word, pattern) for word in words)) - continue - compare_match = re.fullmatch( - r'\$\{([A-Za-z_][A-Za-z0-9_.]*)\}\s*(==|!=)\s*"?([^" ]+)"?', - term, - ) - if compare_match: - variable, operator, expected = compare_match.groups() - equal = variables.get(variable, "") == expected - values.append(equal if operator == "==" else not equal) - continue - raise SystemExit(f"unsupported Makefile condition at line {line}: {term}") - return all(values) - - -def evaluate(source: str, initial: dict[str, str]) -> Evaluation: - variables = dict(initial) - includes: list[str] = [] - src_operations: list[tuple[str, list[str]]] = [] - active_stack = [True] - for line_number, line in logical_lines(source): - if line.startswith(".if "): - parent_active = active_stack[-1] - active_stack.append( - parent_active - and condition_value(line.removeprefix(".if "), variables, line_number) - ) - continue - if line == ".endif": - if len(active_stack) == 1: - raise SystemExit(f"unmatched .endif at line {line_number}") - active_stack.pop() - continue - if not active_stack[-1]: - continue - if line.startswith(".error "): - raise MakeError(line.removeprefix(".error ")) - if line.startswith(".include "): - includes.append(line.removeprefix(".include ").strip()) - continue - if line.startswith("."): - raise SystemExit(f"unsupported Makefile directive at line {line_number}: {line}") - assignment = re.fullmatch( - r"([A-Za-z_][A-Za-z0-9_.]*)\s*(\?=|\+=|=)\s*(.*)", line - ) - if assignment is None: - raise SystemExit(f"unsupported Makefile statement at line {line_number}: {line}") - variable, operator, raw_value = assignment.groups() - value = expand(raw_value, variables) - if operator == "?=" and variable in variables: - continue - if operator == "+=": - old_value = variables.get(variable, "") - variables[variable] = " ".join(part for part in (old_value, value) if part) - else: - variables[variable] = value - if variable == "SRCS": - src_operations.append((operator, value.split())) - if len(active_stack) != 1: - raise SystemExit("unterminated Makefile condition") - return Evaluation(variables, includes, src_operations) - - -def evaluate_error(source: str, initial: dict[str, str]) -> str: - try: - evaluate(source, initial) - except MakeError as error: - return str(error) - raise SystemExit(f"Makefile unexpectedly accepted variables: {initial!r}") - - -def multiset_list(items: Counter[str]) -> list[str]: - return sorted(items.elements()) - - -def object_multiset(sources: list[str]) -> Counter[str]: - objects: Counter[str] = Counter() - for source in sources: - if source.endswith((".c", ".cc", ".cpp", ".S", ".s")): - objects[str(Path(source).with_suffix(".o"))] += 1 - return objects - - -before = committed("src/Makefile") -current = makefile_path.read_text(encoding="utf-8") -linux = (root / "src-linux/Makefile").read_text(encoding="utf-8") - -expected_groups = [ - ( - "=", - [ - "super.c", - "inode.c", - "data.c", - "namei.c", - "dir.c", - "erofs_vnops.c", - "vnode_if.h", - ], - ), - ("+=", ["xattr.c"]), - ("+=", ["decompressor.c", "zmap.c", "zdata.c"]), - ( - "+=", - [ - "decompressor_lz4.c", - "decompressor_lzma.c", - "decompressor_deflate.c", - "decompressor_zstd.c", - ], - ), -] - -scenario_reports: dict[str, object] = {} -for config in ("0", "1"): - initial = { - "MACHINE_ARCH": "amd64", - "SYSDIR": "/freebsd/sys", - "WITH_ZSTDIO": config, - } - before_eval = evaluate(before, initial) - current_eval = evaluate(current, initial) - before_sources = before_eval.variables.get("SRCS", "").split() - current_sources = current_eval.variables.get("SRCS", "").split() - before_source_multiset = Counter(before_sources) - current_source_multiset = Counter(current_sources) - before_objects = object_multiset(before_sources) - current_objects = object_multiset(current_sources) - before_metadata = Counter(source for source in before_sources if not source.endswith(".c")) - current_metadata = Counter(source for source in current_sources if not source.endswith(".c")) - - if current_eval.src_operations != expected_groups: - raise SystemExit( - f"B34 semantic SRCS groups differ for zstdio{config}: " - f"{current_eval.src_operations!r}" - ) - if current_source_multiset != before_source_multiset: - raise SystemExit(f"B34 SRCS multiset changed for zstdio{config}") - if current_objects != before_objects: - raise SystemExit(f"B34 object multiset changed for zstdio{config}") - if current_metadata != before_metadata or current_metadata != Counter({"vnode_if.h": 1}): - raise SystemExit(f"B34 vnode_if handling changed for zstdio{config}") - if current_eval.includes != before_eval.includes or current_eval.includes != [""]: - raise SystemExit(f"B34 bsd.kmod.mk ownership changed for zstdio{config}") - if ( - current_eval.variables.get("KMOD") != before_eval.variables.get("KMOD") - or current_eval.variables.get("KMOD") != "erofs" - ): - raise SystemExit(f"B34 KMOD ownership changed for zstdio{config}") - cflags_name = "CFLAGS.decompressor_zstd.c" - if current_eval.variables.get(cflags_name) != before_eval.variables.get(cflags_name): - raise SystemExit(f"B34 Zstd flags changed for zstdio{config}") - expected_flags = "-I/freebsd/sys/contrib/zstd/lib/freebsd" - if config == "1": - expected_flags += " -DZSTDIO" - if current_eval.variables.get(cflags_name) != expected_flags: - raise SystemExit(f"B34 ZSTDIO expansion changed for zstdio{config}") - - scenario_reports[f"zstdio{config}"] = { - "sources": multiset_list(current_source_multiset), - "objects": multiset_list(current_objects), - "metadata": multiset_list(current_metadata), - "zstd_cflags": current_eval.variables[cflags_name].split(), - "includes": current_eval.includes, - } - -default_initial = {"MACHINE_ARCH": "amd64", "SYSDIR": "/freebsd/sys"} -before_default = evaluate(before, default_initial) -current_default = evaluate(current, default_initial) -if before_default.variables.get("WITH_ZSTDIO") != "0": - raise SystemExit("B34 baseline default is not WITH_ZSTDIO=0") -if current_default.variables.get("WITH_ZSTDIO") != "0": - raise SystemExit("B34 changed the WITH_ZSTDIO default") -before_default_variables = dict(before_default.variables) -current_default_variables = dict(current_default.variables) -del before_default_variables["SRCS"] -del current_default_variables["SRCS"] -if current_default_variables != before_default_variables: - raise SystemExit("B34 changed a default-config Makefile variable expansion") -if current_default.includes != before_default.includes: - raise SystemExit("B34 changed a default-config Makefile include") - -invalid_message = "WITH_ZSTDIO must be 0 or 1" -for source, label in ((before, "baseline"), (current, "current")): - message = evaluate_error( - source, - {"MACHINE_ARCH": "amd64", "SYSDIR": "/freebsd/sys", "WITH_ZSTDIO": "2"}, - ) - if message != invalid_message: - raise SystemExit(f"B34 {label} invalid-value contract drifted: {message!r}") - -arch_message = "erofs supports only MACHINE_ARCH=amd64" -for config in ("0", "1"): - initial = { - "MACHINE_ARCH": "arm64", - "SYSDIR": "/freebsd/sys", - "WITH_ZSTDIO": config, - } - before_message = evaluate_error(before, initial) - current_message = evaluate_error(current, initial) - if before_message != arch_message or current_message != arch_message: - raise SystemExit(f"B34 amd64 gate changed for zstdio{config}") - -for forbidden in ("CONFIG_", "obj-", "erofs-objs", "stub"): - if forbidden in current: - raise SystemExit(f"B34 introduced forbidden Linux/config stub surface: {forbidden}") - -linux_categories = { - "core": "erofs-objs :=" in linux, - "xattr": "CONFIG_EROFS_FS_XATTR" in linux, - "compression": "CONFIG_EROFS_FS_ZIP" in linux, - "algorithms": all( - token in linux - for token in ( - "CONFIG_EROFS_FS_ZIP_LZMA", - "CONFIG_EROFS_FS_ZIP_DEFLATE", - "CONFIG_EROFS_FS_ZIP_ZSTD", - ) - ), -} -if not all(linux_categories.values()): - raise SystemExit(f"B34 Linux semantic grouping reference drifted: {linux_categories!r}") - -report = { - "baseline": baseline, - "write_set": changed, - "groups": [ - {"operator": operator, "sources": sources} - for operator, sources in expected_groups - ], - "configurations": scenario_reports, - "default_with_zstdio": current_default.variables["WITH_ZSTDIO"], - "invalid_with_zstdio_error": invalid_message, - "non_amd64_error": arch_message, - "linux_reference_categories": linux_categories, -} -report_path.write_text( - json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii" -) - -for config in ("zstdio0", "zstdio1"): - details = scenario_reports[config] - print(f"B34 {config} SRCS multiset: {' '.join(details['sources'])}") - print(f"B34 {config} object multiset: {' '.join(details['objects'])}") - print(f"B34 {config} Zstd flags: {' '.join(details['zstd_cflags'])}") -print("B34 default/invalid/amd64 gates: PASS") -print("B34 bsd.kmod.mk and vnode_if ownership: PASS") -print("B34 object-list equivalence: PASS") -PY -then - : -else - pre15_dut_fail 'B34 object-list or conditional expansion audit failed' -fi diff --git a/tests/pre15/cases/SMOKE-LZ4.sh b/tests/pre15/cases/SMOKE-LZ4.sh deleted file mode 100755 index f9715d6..0000000 --- a/tests/pre15/cases/SMOKE-LZ4.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -PRE15_SMOKE_CODEC=lz4 -export PRE15_SMOKE_CODEC -exec /bin/sh "$PRE15_DUT/tests/pre15/cases/SMOKE-common.sh" diff --git a/tests/pre15/cases/SMOKE-LZMA.sh b/tests/pre15/cases/SMOKE-LZMA.sh deleted file mode 100755 index 0501cd3..0000000 --- a/tests/pre15/cases/SMOKE-LZMA.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -PRE15_SMOKE_CODEC=lzma -export PRE15_SMOKE_CODEC -exec /bin/sh "$PRE15_DUT/tests/pre15/cases/SMOKE-common.sh" diff --git a/tests/pre15/cases/SMOKE-PLAIN.sh b/tests/pre15/cases/SMOKE-PLAIN.sh deleted file mode 100755 index eaa4cf7..0000000 --- a/tests/pre15/cases/SMOKE-PLAIN.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -PRE15_SMOKE_CODEC=plain -export PRE15_SMOKE_CODEC -exec /bin/sh "$PRE15_DUT/tests/pre15/cases/SMOKE-common.sh" diff --git a/tests/pre15/cases/SMOKE-ZSTD.sh b/tests/pre15/cases/SMOKE-ZSTD.sh deleted file mode 100755 index 3b3be49..0000000 --- a/tests/pre15/cases/SMOKE-ZSTD.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -PRE15_SMOKE_CODEC=zstd -export PRE15_SMOKE_CODEC -exec /bin/sh "$PRE15_DUT/tests/pre15/cases/SMOKE-common.sh" diff --git a/tests/pre15/cases/SMOKE-common.sh b/tests/pre15/cases/SMOKE-common.sh deleted file mode 100755 index bd96cf2..0000000 --- a/tests/pre15/cases/SMOKE-common.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -: "${PRE15_SMOKE_CODEC:?PRE15_SMOKE_CODEC is required}" -. "$PRE15_LIB_DIR/runner.sh" - -fixture_dir=$PRE15_DUT/tests/pre15/fixtures -generator=$fixture_dir/SMOKE-generate.py -kldsym_probe=$fixture_dir/SMOKE-kldsym.c -kld_builder=$fixture_dir/B28-build-kld.sh -artifacts=$PRE15_RUN_DIR/artifacts -source_first=$PRE15_CASE_TMP/source-first -source_second=$PRE15_CASE_TMP/source-second -image=$PRE15_CASE_TMP/SMOKE-$PRE15_SMOKE_CODEC.erofs -module=$PRE15_CASE_TMP/SMOKE-$PRE15_SMOKE_CODEC-erofs.ko - -case "$PRE15_SMOKE_CODEC" in -plain) - config=0 - uuid=00000000-0000-4000-8000-000000000100 - mkfs_codec= - case_timeout=180 - ;; -lz4) - config=0 - uuid=00000000-0000-4000-8000-000000000104 - mkfs_codec='-zlz4 -C4096' - case_timeout=240 - ;; -lzma) - config=0 - uuid=00000000-0000-4000-8000-0000000001a0 - mkfs_codec='-zlzma,level=6,dictsize=65536 -C4096' - case_timeout=300 - ;; -zstd) - config=1 - uuid=00000000-0000-4000-8000-00000000025d - mkfs_codec='-zzstd,level=3,dictsize=65536 -C4096' - case_timeout=300 - ;; -*) pre15_runner_fail "unknown smoke codec: $PRE15_SMOKE_CODEC" ;; -esac - -for tool in clang cmp diff file mkfs.erofs nm python3 scp sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing smoke tool: $tool" -done -for fixture in "$generator" "$kldsym_probe" "$kld_builder" \ - "$PRE15_DUT/tests/pre15/cases/SMOKE-common.sh"; do - pre15_record_fixture "smoke-$(basename "$fixture")" "$fixture" -done -mkdir -p "$artifacts" -if ! python3 -B "$generator" --output "$source_first" \ - >"$artifacts/source-first.json" 2>"$artifacts/source-first.stderr" || \ - ! python3 -B "$generator" --output "$source_second" \ - >"$artifacts/source-second.json" 2>"$artifacts/source-second.stderr"; then - pre15_runner_fail 'smoke fixture generation failed' -fi -if ! diff -qr "$source_first" "$source_second" \ - >"$artifacts/source-repeat.diff" || \ - ! cmp "$artifacts/source-first.json" "$artifacts/source-second.json"; then - pre15_runner_fail 'smoke source fixture is not byte reproducible' -fi -if ! timeout -k 5 120 sh -c \ - "mkfs.erofs -d0 -T0 --all-time --all-root --workers=1 --sort=path -x-1 -U$uuid $mkfs_codec '$image' '$source_first'" \ - >"$artifacts/mkfs.stdout" 2>"$artifacts/mkfs.stderr"; then - pre15_runner_fail "$PRE15_SMOKE_CODEC smoke image generation failed" -fi -if ! timeout -k 10 600 /bin/sh "$kld_builder" "$PRE15_DUT" \ - "$PRE15_FREEBSD_SRC" "$module" "$PRE15_CASE_TMP/kld-work" "$config" \ - >"$artifacts/kld-build.stdout" 2>"$artifacts/kld-build.stderr"; then - pre15_dut_fail "$PRE15_SMOKE_CODEC smoke cross-target KLD build failed" -fi -pre15_record_module "$module" -sha256sum "$image" >"$artifacts/image-sha256.txt" -sha256sum "$module" >"$artifacts/kld-sha256.txt" -file "$module" >"$artifacts/kld-file.txt" -nm -u "$module" | LC_ALL=C sort >"$artifacts/kld-nm-u.txt" -pre15_target_reached - -test "${PRE15_MODE:-}" = qemu || \ - pre15_runner_fail 'smoke cases require QEMU mode' -: "${PRE15_QEMU_CONTROL_PATH:?QEMU control path is required}" -: "${PRE15_QEMU_SSH_KEY:?QEMU SSH key is required}" -: "${PRE15_QEMU_SSH_PORT:?QEMU SSH port is required}" -: "${PRE15_QEMU_SSH_USER:?QEMU SSH user is required}" - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -remote_root=/root/pre15-smoke-$PRE15_SMOKE_CODEC -pre15_guest_ssh_bounded rm -rf "$remote_root" -pre15_guest_ssh_bounded mkdir -p "$remote_root" -if ! pre15_scp "$image" "$remote_root/image.erofs" || \ - ! pre15_scp "$module" "$remote_root/erofs.ko" || \ - ! pre15_scp "$kldsym_probe" "$remote_root/SMOKE-kldsym.c"; then - pre15_infra_blocked "$PRE15_SMOKE_CODEC smoke transfer failed" -fi -if test "$PRE15_SMOKE_CODEC" = zstd; then - if ! pre15_guest_ssh_bounded \ - "cc -O2 -Wall -Wextra -Werror -std=c17 '$remote_root/SMOKE-kldsym.c' -o '$remote_root/SMOKE-kldsym'"; then - pre15_infra_blocked 'could not compile guest ZSTDIO capability probe' - fi - if ! pre15_guest_ssh_bounded "$remote_root/SMOKE-kldsym" \ - ZSTD_DCtx_setParameter ZSTD_createDCtx_advanced \ - ZSTD_decompressStream ZSTD_freeDCtx ZSTD_isError \ - >"$artifacts/zstdio-capability.txt" 2>"$artifacts/zstdio-capability.stderr"; then - pre15_infra_blocked 'running guest kernel lacks required ZSTDIO symbols' - fi -fi -if pre15_guest_ssh_bounded kldstat -q -m erofs >/dev/null 2>&1; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' -fi -if ! pre15_guest_ssh_bounded kldload "$remote_root/erofs.ko" \ - >"$artifacts/kldload.stdout" 2>"$artifacts/kldload.stderr"; then - if grep -q 'module already loaded or in kernel' \ - "$artifacts/kldload.stderr"; then - pre15_infra_blocked 'guest kernel already owns the erofs.1 interface' - fi - if test "$PRE15_SMOKE_CODEC" = zstd && \ - grep -Eqi 'link_elf|symbol|not defined' "$artifacts/kldload.stderr"; then - pre15_infra_blocked 'zstdio1 EROFS KLD cannot load on this guest kernel' - fi - pre15_dut_fail "$PRE15_SMOKE_CODEC smoke KLD failed to load" -fi -pre15_own_guest_kld erofs "$PRE15_SMOKE_CODEC smoke KLD" -smoke_md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "$remote_root/image.erofs") || \ - pre15_dut_fail "$PRE15_SMOKE_CODEC smoke md attach failed" -case "$smoke_md" in -md[0-9]*) ;; -*) pre15_runner_fail "unexpected smoke md unit: $smoke_md" ;; -esac -pre15_own_guest_md "$smoke_md" "$PRE15_SMOKE_CODEC smoke md" -smoke_mount=/mnt/pre15-smoke-$PRE15_SMOKE_CODEC -pre15_guest_ssh_bounded mkdir -p "$smoke_mount" -if ! pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$smoke_md" "$smoke_mount"; then - pre15_dut_fail "$PRE15_SMOKE_CODEC smoke readonly mount failed" -fi -pre15_own_guest_mount "$smoke_mount" "$PRE15_SMOKE_CODEC smoke mount" - -python3 -B - "$artifacts/source-first.json" "$artifacts/expected-files.tsv" <<'PY' -import json -from pathlib import Path -import sys - -manifest = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -lines = [f"{item['path']}\t{item['size']}\t{item['sha256']}" for item in manifest["files"]] -Path(sys.argv[2]).write_text("\n".join(lines) + "\n", encoding="ascii") -PY -while IFS="$(printf '\t')" read -r path expected_size expected_hash; do - actual_size=$(pre15_guest_ssh_bounded stat -f %z "$smoke_mount/$path") || \ - pre15_dut_fail "$PRE15_SMOKE_CODEC smoke stat failed: $path" - actual_hash=$(pre15_guest_ssh_bounded sha256 -q "$smoke_mount/$path") || \ - pre15_dut_fail "$PRE15_SMOKE_CODEC smoke hash failed: $path" - printf '%s\t%s\t%s\n' "$path" "$actual_size" "$actual_hash" \ - >>"$artifacts/guest-files.tsv" - test "$actual_size" = "$expected_size" && test "$actual_hash" = "$expected_hash" || \ - pre15_dut_fail "$PRE15_SMOKE_CODEC smoke file mismatch: $path" -done <"$artifacts/expected-files.tsv" -pre15_guest_ssh_bounded \ - "find '$smoke_mount' -mindepth 1 -maxdepth 1 -exec basename '{}' ';' | sort" \ - >"$artifacts/guest-root-entries.txt" -printf '%s\n' empty nested payload.bin >"$artifacts/expected-root-entries.txt" -cmp "$artifacts/expected-root-entries.txt" "$artifacts/guest-root-entries.txt" || \ - pre15_dut_fail "$PRE15_SMOKE_CODEC smoke root readdir set mismatch" - -if test "$PRE15_SMOKE_CODEC" = lz4; then - dd if="$source_first/payload.bin" bs=1 skip=4093 count=131071 2>/dev/null | \ - sha256sum | awk '{print $1}' >"$artifacts/expected-range-sha256.txt" - pre15_guest_ssh_bounded \ - "dd if='$smoke_mount/payload.bin' bs=1 skip=4093 count=131071 2>/dev/null | sha256 -q" \ - >"$artifacts/guest-range-sha256.txt" - cmp "$artifacts/expected-range-sha256.txt" "$artifacts/guest-range-sha256.txt" || \ - pre15_dut_fail 'LZ4 smoke cross-pcluster partial range mismatch' -fi - -pre15_guest_ssh_bounded dmesg >"$artifacts/guest-dmesg.txt" -if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free' \ - "$artifacts/guest-dmesg.txt"; then - pre15_dut_fail "$PRE15_SMOKE_CODEC smoke observed kernel failure evidence" -fi -printf '%s\n' \ - "SMOKE-$PRE15_SMOKE_CODEC PASS mode=zstdio$config deadline=$case_timeout" \ - 'readonly mount, exact file hashes/sizes, readdir set, and owned cleanup exercised' diff --git a/tests/pre15/cases/TC162-xattr-legacy.sh b/tests/pre15/cases/TC162-xattr-legacy.sh deleted file mode 100755 index 9a8e796..0000000 --- a/tests/pre15/cases/TC162-xattr-legacy.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh -set -eu - -: "${PRE15_DUT:?PRE15_DUT is required}" -: "${PRE15_ROOT:?PRE15_ROOT is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -. "$PRE15_LIB_DIR/runner.sh" - -spec=$PRE15_DUT/tests/pre15/fixtures/B01-xattr-legacy.json -helper=$PRE15_DUT/tests/pre15/fixtures/g3.py -output=$PRE15_CASE_TMP/xattr-legacy -pre15_record_fixture xattr-legacy-contract "$spec" -pre15_record_fixture xattr-legacy-generator "$helper" - -if python3 -B "$helper" make-xattr-legacy \ - --spec "$spec" --output "$output" \ - --freebsd-root "$PRE15_DUT/src" --linux-root "$PRE15_ROOT/src-linux"; then - : -else - pre15_runner_fail 'legacy xattr fixture generation or independent parsing failed' -fi - -mkdir -p "$PRE15_RUN_DIR/artifacts" -cp "$output/fixture-manifest.json" \ - "$PRE15_RUN_DIR/artifacts/TC162-fixture-manifest.json" -pre15_record_fixture xattr-legacy-manifest \ - "$PRE15_RUN_DIR/artifacts/TC162-fixture-manifest.json" -python3 - "$PRE15_RUN_DIR/artifacts/TC162-fixture-manifest.json" <<'PY' -import json -from pathlib import Path -import sys - -manifest = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -positive = { - "legacy-primary": "primary-legacy", - "explicit-plain": "primary", - "packed-carrier": "packed", - "metabox-carrier": "metabox", -} -for name, carrier in positive.items(): - item = manifest["results"][name] - if item["carrier"] != carrier or not item["checksum_valid"]: - raise SystemExit(f"{name}: carrier/checksum oracle failed") - if item["base_index"] != 1 or item["infix"] != "repo.pre15.legacy.": - raise SystemExit(f"{name}: prefix value differs") -for name in ("legacy-length-invalid", "legacy-offset-outside"): - item = manifest["results"][name] - if item["expected_errno"] != "EINTEGRITY" or not item["checksum_valid"]: - raise SystemExit(f"{name}: exact negative oracle failed") -print("TC162: legacy, explicit plain, packed, and metabox carriers read one value") -print("TC162: single-field negatives return exactly EINTEGRITY") -PY - -pre15_target_reached -if python3 - "$PRE15_DUT/src/xattr.c" "$PRE15_ROOT/src-linux/xattr.c" <<'PY' -from pathlib import Path -import sys - -freebsd = Path(sys.argv[1]).read_text(encoding="utf-8") -linux = Path(sys.argv[2]).read_text(encoding="utf-8") -freebsd_markers = ( - "prefix_en = NULL;", - "EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX", - "else if (em->packed_inode != NULL)", - "else if (em->packed_nid != 0)", - "erofs_xattr_read_metadata(em, prefix_en", -) -linux_markers = ( - "bool plain = erofs_sb_has_plain_xattr_pfx(sbi);", - "else if (sbi->packed_inode)", - "else\n\t\t\tplain = true;", - "if (plain)\n\t\t(void)erofs_init_metabuf(&buf, sb, false);", -) -if not all(marker in freebsd for marker in freebsd_markers): - raise SystemExit("FreeBSD legacy primary fallback changed") -if not all(marker in linux for marker in linux_markers): - raise SystemExit("Linux legacy primary fallback changed") -print("TC162: FreeBSD and Linux retain the no-carrier primary fallback") -PY -then - : -else - pre15_dut_fail 'legacy xattr primary fallback branch no longer matches the frozen contract' -fi diff --git a/tests/pre15/fixtures/B01-g3-archives.json b/tests/pre15/fixtures/B01-g3-archives.json deleted file mode 100644 index 5bb7d42..0000000 --- a/tests/pre15/fixtures/B01-g3-archives.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "schema_version": 1, - "archives": [ - { - "id": "metadata-vfs", - "lines": 277, - "path": "tests/results/manual/2026-08-08T2337Z-metadata-vfs/prepare-fixtures.sh", - "sha256": "93618e0aa5528177a398b1923befc01397db70db24ceda0c32defe84728004fd" - }, - { - "id": "final-review", - "lines": 271, - "path": "tests/results/manual/2026-08-09T0124Z-final-review/prepare-fixtures.sh", - "sha256": "e1ee03c5d845f6d3fb0107dce5952458b4f64d1112c7dafd2129aa01f12d978d" - } - ] -} diff --git a/tests/pre15/fixtures/B01-runner-controls.json b/tests/pre15/fixtures/B01-runner-controls.json deleted file mode 100644 index 46ca1d6..0000000 --- a/tests/pre15/fixtures/B01-runner-controls.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema_version": 1, - "controls": { - "cleanup-failure": ["RUNNER_FAIL", "runner", "FAIL", "0"], - "dut-mismatch": ["DUT_FAIL", "dut", "PASS", "0"], - "guest-command-failure": ["RUNNER_FAIL", "runner", "PASS", "0"], - "known-good": ["PASS", "none", "PASS", "0"], - "owned-cleanup": ["PASS", "none", "PASS", "0"], - "qemu-early-exit": ["INFRA_BLOCKED", "infrastructure", "PASS", "0"], - "ssh-failure": ["INFRA_BLOCKED", "infrastructure", "PASS", "0"], - "timeout": ["RUNNER_FAIL", "runner", "PASS", "1"] - } -} diff --git a/tests/pre15/fixtures/B01-xattr-legacy.json b/tests/pre15/fixtures/B01-xattr-legacy.json deleted file mode 100644 index 610df85..0000000 --- a/tests/pre15/fixtures/B01-xattr-legacy.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "schema_version": 1, - "base_index": 1, - "infix": "repo.pre15.legacy.", - "prefix_start": 2048, - "positive": [ - { - "name": "legacy-primary", - "plain": false, - "carrier": "primary-legacy" - }, - { - "name": "explicit-plain", - "plain": true, - "carrier": "primary" - }, - { - "name": "packed-carrier", - "plain": false, - "carrier": "packed" - }, - { - "name": "metabox-carrier", - "plain": false, - "carrier": "metabox" - } - ], - "negative": [ - { - "name": "legacy-length-invalid", - "field": "record_length", - "value": 257, - "expected_errno": "EINTEGRITY" - }, - { - "name": "legacy-offset-outside", - "field": "prefix_start", - "value": 8192, - "expected_errno": "EINTEGRITY" - } - ] -} diff --git a/tests/pre15/fixtures/B06-ownership.json b/tests/pre15/fixtures/B06-ownership.json deleted file mode 100644 index a857703..0000000 --- a/tests/pre15/fixtures/B06-ownership.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "schema": 1, - "map_blocks_sha256": "ae883c958dfad3a3df9c6950af0e24723c64523908b901d400c2a3c5eeaf515e", - "freebsd_backend_function_sha256": { - "data.c:erofs_map_dev": "de77c2b1f5f12a6a7a9bc0414c42bf1ce1745a60331b0abed8c8df87f5b7dcd7", - "data.c:erofs_bread_device": "c274cd0274ecf79ceea214e123c9d70f3e1f41c6a56449470be554a291ffa808", - "data.c:erofs_bread": "74ae2446f1ce8629642402ff63b83d3258d768fdc20dc2196643e3e1bd0382a3", - "data.c:erofs_read_physical": "aa33e6fcf99ebf46c1eda54da4308ca160a4948f5afbb14038630ce7ce964e09", - "data.c:erofs_brelse": "1d50316f726878cf3390e92eaf6c27072ca8fccc3efceb14fde11f796886e1c5", - "super.c:erofs_release_device_info": "6504bb87bb1c0f5e8bbaba70ab093053cfef2850c6f36eae588603277068b185", - "super.c:erofs_init_device": "b1cb3b21a2b2b581fc4924e361f4b763bd97a6a909c2c081cfa0861dc837b0c1" - }, - "consumers": ["compressed", "inode", "map", "plain", "super", "xattr"], - "direct_metadata_functions": [ - "data.c:erofs_map_blocks_chunk", - "data.c:erofs_read_data", - "data.c:erofs_read_uio", - "decompressor.c:z_erofs_read_cfg", - "inode.c:erofs_read_inode", - "super.c:erofs_load_generation_seed", - "super.c:erofs_read_superblock", - "super.c:erofs_scan_devices", - "super.c:erofs_superblock_csum_verify", - "xattr.c:erofs_xattr_read_backing", - "zdata.c:z_erofs_read_extent", - "zmap.c:z_erofs_fill_inode", - "zmap.c:z_erofs_map_blocks_ext", - "zmap.c:z_erofs_read_extent", - "zmap.c:z_erofs_read_index" - ], - "helper_paths": [ - "helper\tprimary-success\t1\t1", - "helper\tmetabox-success\t1\t1", - "helper\tprovider-error\t0\t0", - "helper\toffset-overflow\t0\t0", - "helper\tnull-release\t0\t0" - ], - "function_metrics": { - "data.c:erofs_map_blocks_chunk": [1, 1, 0, 0], - "data.c:erofs_read_data": [1, 1, 1, 0], - "data.c:erofs_read_metadata": [0, 0, 0, 1], - "data.c:erofs_read_uio": [1, 1, 1, 0], - "decompressor.c:z_erofs_parse_cfgs": [1, 1, 0, 0], - "decompressor.c:z_erofs_read_cfg": [1, 1, 0, 0], - "inode.c:erofs_read_inode": [1, 18, 0, 0], - "super.c:erofs_load_generation_seed": [1, 1, 0, 0], - "super.c:erofs_read_superblock": [1, 1, 0, 0], - "super.c:erofs_scan_devices": [1, 2, 0, 0], - "super.c:erofs_superblock_csum_verify": [1, 1, 0, 0], - "xattr.c:erofs_getxattr": [1, 1, 0, 0], - "xattr.c:erofs_inode_has_noacl": [1, 1, 0, 0], - "xattr.c:erofs_listxattr": [1, 1, 0, 0], - "xattr.c:erofs_xattr_iter_shared": [1, 1, 0, 0], - "xattr.c:erofs_xattr_load_body": [1, 1, 0, 0], - "xattr.c:erofs_xattr_load_shared_entry": [1, 1, 0, 0], - "xattr.c:erofs_xattr_prefixes_init": [1, 2, 0, 0], - "xattr.c:erofs_xattr_read_backing": [0, 0, 0, 1], - "xattr.c:erofs_xattr_read_metadata": [1, 1, 0, 0], - "zdata.c:z_erofs_do_read": [0, 0, 1, 0], - "zdata.c:z_erofs_read_extent": [1, 1, 1, 0], - "zmap.c:z_erofs_fill_inode": [1, 6, 0, 0], - "zmap.c:z_erofs_load_compact_lcluster": [1, 7, 0, 0], - "zmap.c:z_erofs_load_full_lcluster": [1, 2, 0, 0], - "zmap.c:z_erofs_map_blocks_ext": [1, 1, 0, 0], - "zmap.c:z_erofs_read_extent": [1, 1, 0, 0], - "zmap.c:z_erofs_read_index": [0, 0, 0, 0] - }, - "metric_order": [ - "object_local_count", - "put_count", - "legacy_release_count", - "raw_bread_count" - ] -} diff --git a/tests/pre15/fixtures/B06-tuples.json b/tests/pre15/fixtures/B06-tuples.json deleted file mode 100644 index 2a01d07..0000000 --- a/tests/pre15/fixtures/B06-tuples.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "schema": 1, - "base": "edf098905a34de764185e72fc7e92d7b8f4e7285", - "fields": [ - "m_la", - "m_pa", - "m_llen", - "m_plen", - "m_deviceid", - "m_flags", - "errno", - "acquire_count", - "release_count" - ], - "records": [ - {"id": "plain-block-edge", "m_la": 4095, "m_pa": 135167, "m_llen": 1, "m_plen": 1, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 0, "release_count": 0}, - {"id": "inline-tail", "m_la": 4500, "m_pa": 8692, "m_llen": 500, "m_plen": 500, "m_deviceid": 0, "m_flags": 3, "errno": 0, "acquire_count": 0, "release_count": 0}, - {"id": "plain-hole", "m_la": 512, "m_pa": 0, "m_llen": 3584, "m_plen": 3584, "m_deviceid": 0, "m_flags": 0, "errno": 0, "acquire_count": 0, "release_count": 0}, - {"id": "chunk-index", "m_la": 9000, "m_pa": 316200, "m_llen": 7384, "m_plen": 7384, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1}, - {"id": "chunk-48bit-device2", "m_la": 12345, "m_pa": 17592186069049, "m_llen": 4039, "m_plen": 4039, "m_deviceid": 2, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1}, - {"id": "chunk-index-out-of-provider", "m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 97, "acquire_count": 0, "release_count": 0}, - {"id": "chunk-shift-overflow-after-acquire", "m_la": 0, "m_pa": 0, "m_llen": 4096, "m_plen": 4096, "m_deviceid": 0, "m_flags": 0, "errno": 84, "acquire_count": 1, "release_count": 1}, - {"id": "chunk-index-short-read", "m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 5, "acquire_count": 0, "release_count": 0}, - {"id": "explicit-compressed", "m_la": 0, "m_pa": 74565, "m_llen": 4096, "m_plen": 2048, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1}, - {"id": "explicit-partial-reference", "m_la": 0, "m_pa": 262144, "m_llen": 4096, "m_plen": 1024, "m_deviceid": 0, "m_flags": 9, "errno": 0, "acquire_count": 1, "release_count": 1}, - {"id": "explicit-fragment-tail", "m_la": 4096, "m_pa": 0, "m_llen": 2048, "m_plen": 8192, "m_deviceid": 0, "m_flags": 16, "errno": 0, "acquire_count": 1, "release_count": 1}, - {"id": "compressed-post-eof", "m_la": 4096, "m_pa": 0, "m_llen": 18, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 0, "acquire_count": 0, "release_count": 0}, - {"id": "compressed-index-short-read", "m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 5, "acquire_count": 0, "release_count": 0}, - {"id": "compressed-sanity-error-after-acquire", "m_la": 0, "m_pa": 74565, "m_llen": 0, "m_plen": 1024, "m_deviceid": 0, "m_flags": 1, "errno": 45, "acquire_count": 1, "release_count": 1} - ] -} diff --git a/tests/pre15/fixtures/B07-map-oracle.json b/tests/pre15/fixtures/B07-map-oracle.json deleted file mode 100644 index 8a1b353..0000000 --- a/tests/pre15/fixtures/B07-map-oracle.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "baseline": "6673f51152a5195a8a8903aa801f820abce7936e", - "coverage": [ - "bounds", - "chunk", - "compressed", - "fragment", - "hole", - "inline", - "invalid", - "multidevice", - "overflow", - "partial-reference", - "plain", - "post-EOF" - ], - "map_case_count": 80, - "model_sha256": "effcf0b6cc6a6e916eae89a14ad52d7b96273a453556647b6eabf2926fcb4abb", - "records": [ - {"id": "plain-start", "tuple_hex": "0000000000000000000002000000000000100000000000000010000000000000000000000100000000000000000000000000000000000000"}, - {"id": "plain-block-last", "tuple_hex": "ff0f000000000000ff0f02000000000001000000000000000100000000000000000000000100000000000000000000000000000000000000"}, - {"id": "plain-second-start", "tuple_hex": "0010000000000000001002000000000000100000000000000010000000000000000000000100000000000000000000000000000000000000"}, - {"id": "plain-final-byte", "tuple_hex": "ff2f000000000000ff2f02000000000001000000000000000100000000000000000000000100000000000000000000000000000000000000"}, - {"id": "plain-eof", "tuple_hex": "0030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {"id": "plain-post-eof", "tuple_hex": "1130000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {"id": "plain-empty", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {"id": "plain-hole-start", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000000000000000000"}, - {"id": "plain-hole-edge", "tuple_hex": "ff0f000000000000000000000000000001000000000000000100000000000000000000000000000000000000000000000000000000000000"}, - {"id": "plain-shift-overflow", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000610000000000000000000000"}, - {"id": "plain-add-overflow", "tuple_hex": "0010000000000000000000000000000000100000000000000010000000000000000000000000000000000000610000000000000000000000"}, - {"id": "plain-invalid-layout", "tuple_hex": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000002d0000000000000000000000"}, - {"id": "inline-head-start", "tuple_hex": "0000000000000000008002000000000000100000000000000010000000000000000000000100000000000000000000000000000000000000"}, - {"id": "inline-head-last", "tuple_hex": "ff0f000000000000ff8f02000000000001000000000000000100000000000000000000000100000000000000000000000000000000000000"}, - {"id": "inline-tail-start", "tuple_hex": "0010000000000000602000000000000088030000000000008803000000000000000000000300000000000000000000000000000000000000"}, - {"id": "inline-tail-middle", "tuple_hex": "9411000000000000f421000000000000f401000000000000f401000000000000000000000300000000000000000000000000000000000000"}, - {"id": "inline-final-byte", "tuple_hex": "8713000000000000e72300000000000001000000000000000100000000000000000000000300000000000000000000000000000000000000"}, - {"id": "inline-eof", "tuple_hex": "8813000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {"id": "inline-head-hole", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000000000000000000"}, - {"id": "inline-one-block", "tuple_hex": "0000000000000000602000000000000000100000000000000010000000000000000000000300000000000000000000000000000000000000"}, - {"id": "inline-one-byte-tail-head", "tuple_hex": "ff0f000000000000ff8f02000000000001000000000000000100000000000000000000000100000000000000000000000000000000000000"}, - {"id": "inline-one-byte-tail", "tuple_hex": "0010000000000000602000000000000001000000000000000100000000000000000000000300000000000000000000000000000000000000"}, - {"id": "inline-inode-add-overflow", "tuple_hex": "0010000000000000200000000000000088030000000000008803000000000000000000000000000000000000610000000000000000000000"}, - {"id": "inline-xattr-add-overflow", "tuple_hex": "0010000000000000310000000000000088030000000000008803000000000000000000000000000000000000610000000000000000000000"}, - {"id": "inline-offset-add-overflow", "tuple_hex": "7017000000000000c70300000000000090080000000000009008000000000000000000000000000000000000610000000000000000000000"}, - {"id": "chunk-raw32-start", "tuple_hex": "000000000000000000d004000000000000100000000000000010000000000000000000000100000000000000000000000100000001000000"}, - {"id": "chunk-raw32-last", "tuple_hex": "ff0f000000000000ffdf04000000000001000000000000000100000000000000000000000100000000000000000000000100000001000000"}, - {"id": "chunk-raw32-next", "tuple_hex": "001000000000000000e004000000000000100000000000000010000000000000000000000100000000000000000000000100000001000000"}, - {"id": "chunk-raw32-final", "tuple_hex": "0f270000000000000ff704000000000001000000000000000100000000000000000000000100000000000000000000000100000001000000"}, - {"id": "chunk-raw32-hole", "tuple_hex": "00020000000000000000000000000000000e000000000000000e000000000000000000000000000000000000000000000100000001000000"}, - {"id": "chunk-index32-mask", "tuple_hex": "000000000000000000d004000000000000100000000000000010000000000000030000000100000000000000000000000100000001000000"}, - {"id": "chunk-index32-middle", "tuple_hex": "282300000000000028d3040000000000d81c000000000000d81c000000000000000000000100000000000000000000000100000001000000"}, - {"id": "chunk-index32-hole", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000100000001000000"}, - {"id": "chunk-index48-start", "tuple_hex": "0000000000000000005000000010000000100000000000000010000000000000020000000100000000000000000000000100000001000000"}, - {"id": "chunk-index48-middle", "tuple_hex": "39300000000000003960000000100000c70f000000000000c70f000000000000020000000100000000000000000000000100000001000000"}, - {"id": "chunk-index48-hole", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000100000001000000"}, - {"id": "chunk-device-mask-zero", "tuple_hex": "0000000000000000001005000000000000100000000000000010000000000000000000000100000000000000000000000100000001000000"}, - {"id": "chunk-eof", "tuple_hex": "0010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {"id": "chunk-inode-add-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"}, - {"id": "chunk-xattr-add-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"}, - {"id": "chunk-align-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"}, - {"id": "chunk-index-multiply-overflow", "tuple_hex": "feffffffffffffff000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"}, - {"id": "chunk-image-size-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"}, - {"id": "chunk-primary-bounds", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"}, - {"id": "chunk-primary-tail-bounds", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"}, - {"id": "chunk-metabox-missing", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"}, - {"id": "chunk-metabox-bounds", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"}, - {"id": "chunk-reader-short", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000"}, - {"id": "chunk-shift-overflow", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000540000000100000001000000"}, - {"id": "chunk-offset-overflow", "tuple_hex": "0010000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000100000001000000"}, - {"id": "extent4-start", "tuple_hex": "0000000000000000000001000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"}, - {"id": "extent4-first-last", "tuple_hex": "0000000000000000000001000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"}, - {"id": "extent4-second-start", "tuple_hex": "0010000000000000000401000000000000100000000000000008000000000000000000000100000000000000000000000300000003000000"}, - {"id": "extent4-second-middle", "tuple_hex": "0010000000000000000401000000000000100000000000000008000000000000000000000100000000000000000000000300000003000000"}, - {"id": "extent8-start", "tuple_hex": "0000000000000000000002000000000000100000000000000004000000000000000000000100000000000000000000000100000001000000"}, - {"id": "extent8-second-start", "tuple_hex": "0010000000000000000003000000000000100000000000000008000000000000000000000100000000000000000000000100000001000000"}, - {"id": "extent8-hole", "tuple_hex": "0010000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000100000001000000"}, - {"id": "extent16-start", "tuple_hex": "0000000000000000000004000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"}, - {"id": "extent16-first-last", "tuple_hex": "0000000000000000000004000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"}, - {"id": "extent16-second-start", "tuple_hex": "0010000000000000000005000000000000100000000000000008000000000000000000000100000000000000000000000200000002000000"}, - {"id": "extent16-second-middle", "tuple_hex": "0010000000000000000005000000000000100000000000000008000000000000000000000100000000000000000000000200000002000000"}, - {"id": "extent16-third-start", "tuple_hex": "0020000000000000000006000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"}, - {"id": "extent16-physical-48bit", "tuple_hex": "0000000000000000090000000001000000100000000000000004000000000000000000000100000000000000000000000100000001000000"}, - {"id": "extent32-logical-64bit", "tuple_hex": "0000000001000000000008000000000000100000000000000008000000000000000000000100000000000000000000000100000001000000"}, - {"id": "extent-partial-reference", "tuple_hex": "0000000000000000000009000000000000100000000000000004000000000000000000000900000000000000000000000100000001000000"}, - {"id": "extent-fragment-tail", "tuple_hex": "0010000000000000000000000000000000080000000000000020000000000000000000001000000000000000000000000100000001000000"}, - {"id": "extent-interlaced-shifted", "tuple_hex": "000000000000000000000a000000000000100000000000000010000000000000000000000100000005000000000000000100000001000000"}, - {"id": "extent-algorithm-one", "tuple_hex": "000000000000000000000b000000000000100000000000000004000000000000000000000100000001000000000000000100000001000000"}, - {"id": "extent-algorithm-unavailable", "tuple_hex": "000000000000000000000b000000000000000000000000000004000000000000000000000100000001000000610000000100000001000000"}, - {"id": "extent-algorithm-runtime-invalid", "tuple_hex": "000000000000000000000c0000000000000000000000000000040000000000000000000001000000060000002d0000000100000001000000"}, - {"id": "extent-plen-too-large", "tuple_hex": "000000000000000000000d0000000000000000000000000001001000000000000000000001000000040000002d0000000100000001000000"}, - {"id": "extent-full-short-logical", "tuple_hex": "000000000000000000000e000000000000000000000000000020000000000000000000000100000000000000610000000100000001000000"}, - {"id": "extent-pa-add-overflow", "tuple_hex": "000000000000000000feffffffffffff00000000000000000004000000000000000000000100000000000000610000000100000001000000"}, - {"id": "extent-physical-48bit-limit", "tuple_hex": "0000000000000000000000000000001000000000000000000004000000000000000000000100000000000000610000000100000001000000"}, - {"id": "extent-reader-short", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000"}, - {"id": "extent-table-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"}, - {"id": "extent-record-multiply-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"}, - {"id": "extent-fragment-bounds", "tuple_hex": "0010000000000000000000000000000000000000000000000020000000000000000000001000000000000000610000000100000001000000"}, - {"id": "compressed-eof", "tuple_hex": "0010000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {"id": "compressed-post-eof", "tuple_hex": "0010000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000000000000000000000"} - ], - "schema": 1, - "subcategories": [ - "chunk-bounds", - "chunk-holes", - "chunk-index32", - "chunk-index48", - "chunk-overflow", - "chunk-raw32", - "cleanup-after-acquire", - "cleanup-before-acquire", - "compressed-bounds", - "compressed-flags", - "compressed-invalid", - "compressed-overflow", - "device-mask", - "device-resolution", - "extent-16", - "extent-32", - "extent-4", - "extent-8", - "inline-boundaries", - "inline-overflow", - "plain-boundaries", - "plain-overflow" - ], - "tuple_byte_layout": " argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--root", required=True, type=Path) - parser.add_argument("--dut", required=True, type=Path) - parser.add_argument("--baseline", required=True) - parser.add_argument("--spec", required=True, type=Path) - parser.add_argument("--report", required=True, type=Path) - return parser.parse_args() - - -def committed(root: Path, baseline: str, path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read B10 baseline {path}: {completed.stderr}") - return completed.stdout - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - if brace < 0: - raise SystemExit(f"missing function body: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def require_order(source: str, markers: list[str], label: str) -> None: - position = -1 - for marker in markers: - position = source.find(marker, position + 1) - if position < 0: - raise SystemExit(f"{label} is missing ordered marker: {marker}") - - -def function_position(source: str, name: str) -> int: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - raise SystemExit(f"missing function position: {name}") - return match.start() - - -def source_checks(root: Path, dut: Path, baseline: str) -> dict[str, object]: - src = dut / "src" - current = { - name: (src / name).read_text(encoding="utf-8") - for name in ("dir.c", "namei.c", "internal.h", "erofs_vnops.c") - } - base = { - name: committed(root, baseline, f"repo-pre-15/src/{name}") - for name in current - } - if current["dir.c"] == base["dir.c"] or current["namei.c"] == base["namei.c"]: - raise SystemExit("B10 did not change both declared directory sources") - if current["internal.h"] != base["internal.h"]: - raise SystemExit("B10 added directory-wide state outside its write set") - if current["erofs_vnops.c"] != base["erofs_vnops.c"]: - raise SystemExit("B10 changed the FreeBSD VOP adapter") - - completed = subprocess.run( - [ - "git", - "-C", - str(root), - "diff", - "--name-only", - baseline, - "--", - "repo-pre-15/src", - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ) - changed_sources = set(completed.stdout.splitlines()) - expected_sources = { - "repo-pre-15/src/dir.c", - "repo-pre-15/src/namei.c", - } - if changed_sources != expected_sources: - raise SystemExit(f"unexpected B10 source write set: {sorted(changed_sources)}") - - dir_source = current["dir.c"] - namei_source = current["namei.c"] - validator = extract_function(dir_source, "erofs_validate_dirblock") - fill = extract_function(dir_source, "erofs_fill_dentries") - readdir = extract_function(dir_source, "erofs_readdir_block") - previous = extract_function(dir_source, "erofs_previous_dirname") - read_block = extract_function(namei_source, "erofs_read_dirblock") - neighbors = extract_function(namei_source, "erofs_validate_dirblock_neighbors") - find_block = extract_function(namei_source, "erofs_find_target_block") - namei = extract_function(namei_source, "erofs_namei") - lookup = extract_function(namei_source, "erofs_lookup") - - require_order( - validator, - ["erofs_dirent_namelen", "erofs_dirname_order", "EINTEGRITY"], - "block lexical validator", - ) - require_order( - fill, - ["erofs_dirent_name", "erofs_nid_is_valid", "bzero", "d_fileno", "erofs_uiodir"], - "readdir publication boundary", - ) - require_order( - readdir, - [ - "GENERIC_MINDIRSIZ", - "(size_t)INT_MAX", - "SIZE_MAX / sizeof(*cookiebuf)", - "(int)cookie_count", - ], - "cookie clamp", - ) - if "uio->uio_resid / 8" in readdir or "MAX(1" in readdir: - raise SystemExit("legacy unchecked cookie sizing remains") - require_order( - readdir, - [ - "erofs_validate_dirblock", - "erofs_previous_dirname", - "erofs_dirname_order", - "erofs_fill_dentries", - ], - "linear readdir order validation", - ) - if previous.count("erofs_read_data") != 1 or previous.count("erofs_brelse") != 1: - raise SystemExit("readdir predecessor ownership is not one acquire/release") - require_order(read_block, ["erofs_read_data", "erofs_validate_dirblock"], "lookup block read") - for marker in ("block - 1", "block + 1", "erofs_dirblock_order"): - if marker not in neighbors: - raise SystemExit(f"bounded neighbor validation is missing: {marker}") - if find_block.count("while (head <= back)") != 1 or re.search(r"\bfor\s*\(", find_block): - raise SystemExit("lookup is no longer a single binary block search") - require_order( - find_block, - ["erofs_read_dirblock", "erofs_validate_dirblock_neighbors", "erofs_dirnamecmp"], - "touched-block lookup validation", - ) - require_order(namei, ["erofs_nid_is_valid", "*nid = found_nid", "*d_type"], "lookup NID publication") - require_order( - lookup, - ["erofs_namei", "(cnp->cn_flags & ISDOTDOT) == 0", "vn_vget_ino", "erofs_vget"], - "FreeBSD self-NID lock boundary", - ) - if lookup.count("vn_vget_ino") != 1 or lookup.count("erofs_vget") != 1: - raise SystemExit("FreeBSD dotdot/ordinary vnode acquisition changed shape") - for marker in ("vref(dvp)", "cache_enter(dvp, NULL, cnp)", "cache_enter(dvp, vp, cnp)"): - if marker not in lookup: - raise SystemExit(f"FreeBSD lookup behavior is missing: {marker}") - if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", dir_source + namei_source): - raise SystemExit("negative Linux errno entered B10") - - dir_order = [ - "erofs_validate_dirblock", - "erofs_fill_dentries", - "erofs_readdir_block", - ] - if [function_position(dir_source, name) for name in dir_order] != sorted( - function_position(dir_source, name) for name in dir_order - ): - raise SystemExit("dir core order does not retain validator-before-Linux-core shape") - namei_order = [ - "find_target_dirent", - "erofs_find_target_block", - "erofs_namei", - "erofs_lookup", - ] - if [function_position(namei_source, name) for name in namei_order] != sorted( - function_position(namei_source, name) for name in namei_order - ): - raise SystemExit("namei core order diverges from the Linux reference shape") - - return { - "changed_sources": sorted(changed_sources), - "dir_core_order": dir_order, - "namei_core_order": namei_order, - "full_directory_state_added": False, - "positive_errno": True, - "vfs_cache_lookup_preserved": "vfs_cache_lookup" in current["erofs_vnops.c"], - } - - -def validate_block(names: list[str]) -> None: - if not names or any(not name for name in names): - raise ValueError(EINTEGRITY) - encoded = [name.encode("ascii") for name in names] - if any(left >= right for left, right in zip(encoded, encoded[1:])): - raise ValueError(EINTEGRITY) - - -def validate_boundary(left: list[str], right: list[str]) -> None: - if left[-1].encode("ascii") >= right[0].encode("ascii"): - raise ValueError(EINTEGRITY) - - -def lookup_model(blocks: list[list[str]], target: str) -> tuple[str, int, list[int]]: - head = 0 - back = len(blocks) - 1 - candidate = None - reads = 0 - touched = [] - target_bytes = target.encode("ascii") - try: - while head <= back: - mid = head + (back - head) // 2 - touched.append(mid) - validate_block(blocks[mid]) - reads += 1 - if mid > 0: - validate_block(blocks[mid - 1]) - validate_boundary(blocks[mid - 1], blocks[mid]) - reads += 1 - if mid + 1 < len(blocks): - validate_block(blocks[mid + 1]) - validate_boundary(blocks[mid], blocks[mid + 1]) - reads += 1 - first = blocks[mid][0].encode("ascii") - if target_bytes < first: - back = mid - 1 - continue - candidate = mid - if target_bytes == first: - return PASS, reads, touched - head = mid + 1 - if candidate is None: - return "ENOENT", reads, touched - names = [name.encode("ascii") for name in blocks[candidate]] - left = 1 - right = len(names) - 1 - while left <= right: - mid = left + (right - left) // 2 - if names[mid] == target_bytes: - return PASS, reads, touched - if names[mid] < target_bytes: - left = mid + 1 - else: - right = mid - 1 - return "ENOENT", reads, touched - except ValueError: - return EINTEGRITY, reads, touched - - -def readdir_model(blocks: list[list[str]]) -> tuple[str, list[str]]: - output = [] - previous = None - try: - for block in blocks: - validate_block(block) - if previous is not None: - validate_boundary(previous, block) - output.extend(block) - previous = block - except ValueError: - return EINTEGRITY, output - return PASS, output - - -def run_order_cases(spec: dict[str, object]) -> list[dict[str, object]]: - results = [] - for case in spec["order_cases"]: - lookup_status, reads, touched = lookup_model(case["blocks"], case["target"]) - readdir_status, raw = readdir_model(case["blocks"]) - if lookup_status != case["expected_lookup"]: - raise SystemExit(f"lookup order oracle mismatch: {case['id']}") - if readdir_status != case["expected_readdir"]: - raise SystemExit(f"readdir order oracle mismatch: {case['id']}") - if "expected_raw" in case and raw != case["expected_raw"]: - raise SystemExit(f"raw directory order changed: {case['id']}") - results.append( - { - "id": case["id"], - "lookup": lookup_status, - "readdir": readdir_status, - "lookup_reads": reads, - "lookup_touched": touched, - "raw_entries": raw if readdir_status == PASS else None, - } - ) - return results - - -def nid_is_valid(config: dict[str, object], case: dict[str, object]) -> bool: - uint64_max = (1 << 64) - 1 - metabox_bit = config["metabox_bit"] - nid = case["nid"] - low = nid & (metabox_bit - 1) - if low > (uint64_max >> 5): - return False - offset = low << 5 - in_metabox = bool(nid & metabox_bit) - if in_metabox: - if not case["metabox_feature"] or case["metabox_size"] is None: - return False - return offset <= case["metabox_size"] and config["compact_inode_size"] <= case["metabox_size"] - offset - primary = config["primary"] - if primary["blocks"] > (uint64_max >> primary["blkszbits"]): - return False - metadata = primary["meta_blkaddr"] << primary["blkszbits"] - if offset > uint64_max - metadata: - return False - offset += metadata - image_size = primary["blocks"] << primary["blkszbits"] - compact = config["compact_inode_size"] - return ( - offset <= image_size - and compact <= image_size - offset - and offset <= primary["mediasize"] - and compact <= primary["mediasize"] - offset - ) - - -def run_nid_cases(config: dict[str, object]) -> list[dict[str, object]]: - results = [] - for case in config["cases"]: - valid = nid_is_valid(config, case) - lookup = EINTEGRITY if not valid else PASS - if valid and case["relation"] == "ordinary" and case["nid"] == case["current_nid"]: - lookup = EINTEGRITY - readdir = PASS if valid else EINTEGRITY - if valid != case["expected_valid"] or lookup != case["expected_lookup"] or readdir != case["expected_readdir"]: - raise SystemExit(f"NID oracle mismatch: {case['id']}") - results.append({"id": case["id"], "valid": valid, "lookup": lookup, "readdir": readdir}) - return results - - -def run_cookie_cases(config: dict[str, object]) -> list[dict[str, int]]: - results = [] - allocation_limit = config["size_max"] // 8 - for case in config["cases"]: - count = 0 - if case["resid"] > 0: - count = case["resid"] // config["min_dirent_size"] - count = min(count, config["int_max"], allocation_limit) - if count != case["expected"]: - raise SystemExit(f"cookie clamp oracle mismatch: {case['id']}") - results.append({"id": case["id"], "resid": case["resid"], "cookies": count}) - return results - - -def complexity_result(config: dict[str, int]) -> dict[str, object]: - blocks = config["directory_bytes"] // config["block_size"] - if blocks != config["expected_blocks"]: - raise SystemExit("TC153 sparse directory block denominator changed") - max_iterations = blocks.bit_length() - if max_iterations != config["max_binary_iterations"]: - raise SystemExit("TC153 binary iteration bound mismatch") - scenarios = [] - for target in (0, blocks // 2, blocks - 1): - head = 0 - back = blocks - 1 - iterations = 0 - reads = 0 - while head <= back: - mid = head + (back - head) // 2 - iterations += 1 - reads += 1 + int(mid > 0) + int(mid + 1 < blocks) - if mid == target: - break - if mid < target: - head = mid + 1 - else: - back = mid - 1 - if iterations > config["max_binary_iterations"] or reads > config["max_reads_with_neighbors"]: - raise SystemExit("TC153 lookup exceeded the logarithmic neighbor-read bound") - scenarios.append({"target_block": target, "iterations": iterations, "reads": reads}) - return { - "directory_bytes": config["directory_bytes"], - "blocks": blocks, - "bound": "O(log n) selected blocks plus two neighbors per selection", - "scenarios": scenarios, - } - - -def main() -> None: - args = parse_args() - spec = json.loads(args.spec.read_text(encoding="ascii")) - if spec.get("schema") != 1 or spec.get("batch") != "B10": - raise SystemExit("invalid B10 fixture identity") - if spec.get("candidates") != ["P15-012", "P15-054", "P15-055", "P15-078", "P15-080"]: - raise SystemExit("B10 candidate list changed") - - report = { - "schema": 1, - "batch": "B10", - "source": source_checks(args.root, args.dut, args.baseline), - "order": run_order_cases(spec), - "nid": run_nid_cases(spec["nid"]), - "cookie": run_cookie_cases(spec["cookie"]), - "complexity": complexity_result(spec["complexity"]), - } - args.report.write_text( - json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", - ) - print(f"TC163-directory-order PASS {len(report['order'])} cases") - print(f"TC173-readdir-boundaries PASS {len(report['nid']) + len(report['cookie'])} cases") - print("TC153-large-directory-block-index PASS O(log n)+bounded-neighbors") - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B10-directory-spec.json b/tests/pre15/fixtures/B10-directory-spec.json deleted file mode 100644 index da07ac4..0000000 --- a/tests/pre15/fixtures/B10-directory-spec.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "batch": "B10", - "candidates": [ - "P15-012", - "P15-054", - "P15-055", - "P15-078", - "P15-080" - ], - "complexity": { - "block_size": 4096, - "directory_bytes": 8796093022208, - "expected_blocks": 2147483648, - "max_binary_iterations": 32, - "max_reads_with_neighbors": 96 - }, - "cookie": { - "int_max": 2147483647, - "min_dirent_size": 24, - "size_max": 18446744073709551615, - "cases": [ - {"id": "negative-resid", "resid": -1, "expected": 0}, - {"id": "zero-resid", "resid": 0, "expected": 0}, - {"id": "one-byte", "resid": 1, "expected": 0}, - {"id": "below-minimum", "resid": 23, "expected": 0}, - {"id": "one-dirent", "resid": 24, "expected": 1}, - {"id": "below-two-dirents", "resid": 47, "expected": 1}, - {"id": "two-dirents", "resid": 48, "expected": 2}, - {"id": "normal-buffer", "resid": 4096, "expected": 170}, - {"id": "int-clamp", "resid": 51539607552, "expected": 2147483647} - ] - }, - "nid": { - "compact_inode_size": 32, - "metabox_bit": 9223372036854775808, - "primary": { - "blkszbits": 12, - "blocks": 8, - "mediasize": 32768, - "meta_blkaddr": 1 - }, - "cases": [ - {"id": "primary-first", "nid": 0, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"}, - {"id": "primary-last", "nid": 895, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"}, - {"id": "primary-past-end", "nid": 896, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": false, "expected_lookup": "EINTEGRITY", "expected_readdir": "EINTEGRITY"}, - {"id": "primary-shift-overflow", "nid": 9223372036854775807, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": false, "expected_lookup": "EINTEGRITY", "expected_readdir": "EINTEGRITY"}, - {"id": "metabox-valid", "nid": 9223372036854775811, "current_nid": 1, "relation": "ordinary", "metabox_feature": true, "metabox_size": 128, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"}, - {"id": "metabox-no-feature", "nid": 9223372036854775808, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": false, "expected_lookup": "EINTEGRITY", "expected_readdir": "EINTEGRITY"}, - {"id": "metabox-past-end", "nid": 9223372036854775812, "current_nid": 1, "relation": "ordinary", "metabox_feature": true, "metabox_size": 128, "expected_valid": false, "expected_lookup": "EINTEGRITY", "expected_readdir": "EINTEGRITY"}, - {"id": "ordinary-self", "nid": 42, "current_nid": 42, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "EINTEGRITY", "expected_readdir": "PASS"}, - {"id": "dot-self", "nid": 42, "current_nid": 42, "relation": "dot", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"}, - {"id": "dotdot-self", "nid": 42, "current_nid": 42, "relation": "dotdot", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"} - ] - }, - "order_cases": [ - { - "id": "valid-raw-order", - "blocks": [[".", "..", "alpha", "alphabet"], ["beta", "delta"], ["omega"]], - "target": "delta", - "expected_lookup": "PASS", - "expected_readdir": "PASS", - "expected_raw": [".", "..", "alpha", "alphabet", "beta", "delta", "omega"] - }, - { - "id": "long-common-prefix", - "blocks": [["prefix-0000000000000000", "prefix-0000000000000001"], ["prefix-0000000000000010", "prefix-0000000000000011"]], - "target": "prefix-0000000000000011", - "expected_lookup": "PASS", - "expected_readdir": "PASS", - "expected_raw": ["prefix-0000000000000000", "prefix-0000000000000001", "prefix-0000000000000010", "prefix-0000000000000011"] - }, - { - "id": "within-block-reverse", - "blocks": [["alpha", "charlie", "bravo"]], - "target": "bravo", - "expected_lookup": "EINTEGRITY", - "expected_readdir": "EINTEGRITY" - }, - { - "id": "within-block-duplicate", - "blocks": [["alpha", "beta", "beta"]], - "target": "beta", - "expected_lookup": "EINTEGRITY", - "expected_readdir": "EINTEGRITY" - }, - { - "id": "cross-block-reverse", - "blocks": [["alpha", "delta"], ["charlie", "omega"]], - "target": "delta", - "expected_lookup": "EINTEGRITY", - "expected_readdir": "EINTEGRITY" - }, - { - "id": "cross-block-duplicate", - "blocks": [["alpha", "delta"], ["delta", "omega"]], - "target": "delta", - "expected_lookup": "EINTEGRITY", - "expected_readdir": "EINTEGRITY" - }, - { - "id": "remote-corruption-not-prescanned", - "blocks": [["a", "b"], ["c", "d"], ["e", "f"], ["g", "h"], ["i", "j"], ["k", "l"], ["z", "zz"], ["m", "n"]], - "target": "a", - "expected_lookup": "PASS", - "expected_readdir": "EINTEGRITY", - "remote_corruption_not_claimed": true - } - ], - "schema": 1 -} diff --git a/tests/pre15/fixtures/B11-dtype-oracle.py b/tests/pre15/fixtures/B11-dtype-oracle.py deleted file mode 100755 index 788e846..0000000 --- a/tests/pre15/fixtures/B11-dtype-oracle.py +++ /dev/null @@ -1,396 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import hashlib -import json -from pathlib import Path -import re -import subprocess -import tempfile - - -VTYPE_NUMBER = { - "VNON": 0, - "VREG": 1, - "VDIR": 2, - "VBLK": 3, - "VCHR": 4, - "VLNK": 5, - "VSOCK": 6, - "VFIFO": 7, -} - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--root", type=Path, required=True) - parser.add_argument("--dut", type=Path, required=True) - parser.add_argument("--spec", type=Path, required=True) - parser.add_argument("--gate-output", type=Path, required=True) - parser.add_argument("--artifacts", type=Path, required=True) - return parser.parse_args() - - -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def committed(root: Path, commit: str, path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{commit}:{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read {path} at {commit}: {completed.stderr}") - return completed.stdout - - -def replace_once(source: str, old: str, new: str, label: str) -> str: - count = source.count(old) - if count != 1: - raise SystemExit(f"{label}: expected one transform site, found {count}") - return source.replace(old, new, 1) - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if match is None: - raise SystemExit(f"function not found: {name}") - start = source.rfind("\n\n", 0, match.start()) + 2 - brace = source.find("{", match.end()) - if brace < 0: - raise SystemExit(f"function body not found: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] + "\n" - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def run_helper(helper: str, records: list[dict], artifacts: Path) -> list[dict]: - harness = f'''#include -#include -#include -#include - -#define EROFS_FT_UNKNOWN 0 -#define EROFS_FT_REG_FILE 1 -#define EROFS_FT_DIR 2 -#define EROFS_FT_CHRDEV 3 -#define EROFS_FT_BLKDEV 4 -#define EROFS_FT_FIFO 5 -#define EROFS_FT_SOCK 6 -#define EROFS_FT_SYMLINK 7 -#define __enum_uint8(name) enum name - -enum vtype {{ - VNON, - VREG, - VDIR, - VBLK, - VCHR, - VLNK, - VSOCK, - VFIFO, -}}; - -{helper} -int -main(int argc, char **argv) -{{ - unsigned long file_type, vtype; - - if (argc != 3) - return (2); - file_type = strtoul(argv[1], NULL, 0); - vtype = strtoul(argv[2], NULL, 0); - if (file_type > UINT8_MAX || vtype > VFIFO) - return (2); - printf("%d\\n", erofs_dirent_type_matches((uint8_t)file_type, - (enum vtype)vtype) ? 0 : 97); - return (0); -}} -''' - source = artifacts / "B11-helper-harness.c" - source.write_text(harness, encoding="ascii") - with tempfile.TemporaryDirectory(prefix="b11-helper-") as temporary: - binary = Path(temporary) / "helper" - completed = subprocess.run( - [ - "cc", - "-std=c11", - "-Wall", - "-Wextra", - "-Werror", - str(source), - "-o", - str(binary), - ], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - if completed.returncode != 0: - raise SystemExit(f"B11 helper harness did not compile:\n{completed.stdout}") - results = [] - for record in records: - completed = subprocess.run( - [ - str(binary), - str(record["file_type"]), - str(VTYPE_NUMBER[record["vtype"]]), - ], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - if completed.returncode != 0: - raise SystemExit(f"B11 helper failed for {record['id']}") - actual = int(completed.stdout.strip()) - expected = record["expected_errno"] - if actual != expected: - raise SystemExit( - f"B11 helper result differs for {record['id']}: {actual} != {expected}" - ) - results.append( - { - "actual_errno": actual, - "expected_errno": expected, - "id": record["id"], - } - ) - return results - - -def main() -> None: - args = parse_args() - spec = json.loads(args.spec.read_text(encoding="ascii")) - if spec.get("schema") != 1 or spec.get("batch") != "B11": - raise SystemExit("invalid B11 dtype spec") - args.artifacts.mkdir(parents=True, exist_ok=True) - gate_decision = subprocess.check_output( - ["git", "-C", str(args.root), "rev-parse", spec["gate_decision"]], - text=True, - ).strip() - if gate_decision != spec["gate_decision"]: - raise SystemExit("B11 gate decision identity changed") - - gate_script = args.dut / "tests/pre15/gates/P15-081.sh" - gate_input = args.dut / "tests/pre15/gates/P15-081-input.json" - if sha256(gate_script) != spec["gate_script_sha256"]: - raise SystemExit("B11 gate script identity changed") - if sha256(gate_input) != spec["gate_input_sha256"]: - raise SystemExit("B11 gate input identity changed") - for name, expected in spec["gate_evidence_sha256"].items(): - path = args.gate_output / name - if sha256(path) != expected: - raise SystemExit(f"B11 gate replay identity changed: {name}") - - gate_result = json.loads( - (args.gate_output / "result.json").read_text(encoding="ascii") - ) - gate_oracle = json.loads( - (args.gate_output / "oracle.json").read_text(encoding="ascii") - ) - if gate_result["status"] != "GO" or gate_result["b11"] != "AUTHORIZED": - raise SystemExit("P15-081 gate no longer authorizes B11") - for key, expected in spec["expected"].items(): - actual = ( - gate_result["fixture_set_sha256"] - if key == "fixture_set_sha256" - else gate_result[key] - ) - if actual != expected: - raise SystemExit(f"B11 gate count changed for {key}: {actual} != {expected}") - - baseline = spec["baseline"] - paths = ("inode.c", "internal.h", "namei.c", "dir.c", "erofs_vnops.c") - before = { - name: committed(args.root, baseline, f"repo-pre-15/src/{name}") - for name in paths - } - current = { - name: (args.dut / "src" / name).read_text(encoding="utf-8") - for name in paths - } - - helper = '''bool -erofs_dirent_type_matches(uint8_t file_type, __enum_uint8(vtype) vtype) -{ -\tswitch (file_type) { -\tcase EROFS_FT_REG_FILE: -\t\treturn (vtype == VREG); -\tcase EROFS_FT_DIR: -\t\treturn (vtype == VDIR); -\tcase EROFS_FT_CHRDEV: -\t\treturn (vtype == VCHR); -\tcase EROFS_FT_BLKDEV: -\t\treturn (vtype == VBLK); -\tcase EROFS_FT_FIFO: -\t\treturn (vtype == VFIFO); -\tcase EROFS_FT_SOCK: -\t\treturn (vtype == VSOCK); -\tcase EROFS_FT_SYMLINK: -\t\treturn (vtype == VLNK); -\tdefault: -\t\treturn (true); -\t} -} - -''' - inode_marker = "/*\n * Read and decode a disk inode." - expected_inode = replace_once( - before["inode.c"], inode_marker, helper + inode_marker, "inode helper" - ) - expected_internal = replace_once( - before["internal.h"], - "int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,\n", - "bool erofs_dirent_type_matches(uint8_t file_type,\n" - " __enum_uint8(vtype) vtype);\n" - "int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,\n", - "internal prototype", - ) - lookup_marker = "\tif (error != 0)\n\t\treturn (error);\n\t*ap->a_vpp = vp;\n" - lookup_replacement = ( - "\tif (error != 0)\n" - "\t\treturn (error);\n" - "\tif ((cnp->cn_flags & ISDOTDOT) == 0 &&\n" - "\t !erofs_dirent_type_matches(dtype, VTOE(vp)->vtype)) {\n" - "\t\tvput(vp);\n" - "\t\treturn (EINTEGRITY);\n" - "\t}\n" - "\t*ap->a_vpp = vp;\n" - ) - expected_namei = replace_once( - before["namei.c"], lookup_marker, lookup_replacement, "lookup validator" - ) - expected = { - "inode.c": expected_inode, - "internal.h": expected_internal, - "namei.c": expected_namei, - "dir.c": before["dir.c"], - "erofs_vnops.c": before["erofs_vnops.c"], - } - for name in paths: - if current[name] != expected[name]: - raise SystemExit(f"B11 source differs from the exact transform: {name}") - - lookup = extract_function(current["namei.c"], "erofs_lookup") - validator = lookup.index("erofs_dirent_type_matches") - if lookup.index("error = erofs_vget(") > validator: - raise SystemExit("B11 validator moved before the ordinary vnode lookup") - if lookup.index("error = vn_vget_ino(") > validator: - raise SystemExit("B11 validator moved before the dotdot vnode lookup") - if validator > lookup.index("*ap->a_vpp = vp"): - raise SystemExit("B11 validator moved after vnode publication") - if validator > lookup.index("cache_enter(dvp, vp, cnp)"): - raise SystemExit("B11 validator moved after namecache publication") - validator_start = lookup.rfind( - "if ((cnp->cn_flags & ISDOTDOT) == 0", 0, validator - ) - if validator_start < 0: - raise SystemExit("B11 validator condition is absent") - validator_block = lookup[validator_start : lookup.index("*ap->a_vpp = vp")] - if "(cnp->cn_flags & ISDOTDOT) == 0" not in validator_block: - raise SystemExit("B11 dotdot bypass is absent") - if "vput(vp);\n\t\treturn (EINTEGRITY);" not in validator_block: - raise SystemExit("B11 mismatch cleanup or positive errno changed") - for token in ("erofs_vget(", "vn_vget_ino(", "vfs_hash_get(", "VOP_LOCK(", "vn_lock("): - if token in validator_block: - raise SystemExit(f"B11 validator introduced a lock/vget operation: {token}") - for token in ("erofs_vget(", "vn_vget_ino(", "vfs_hash_get("): - if token in current["dir.c"]: - raise SystemExit(f"B11 readdir path introduced vnode loading: {token}") - - actual_helper = extract_function(current["inode.c"], "erofs_dirent_type_matches") - records = [] - for record in gate_oracle["known_match_records"]: - records.append( - { - "expected_errno": record["expected_errno"], - "file_type": record["file_type"], - "id": record["case"], - "vtype": record["vtype"], - } - ) - for record in gate_oracle["case_records"]: - records.append( - { - "expected_errno": record["expected_errno"], - "file_type": record["file_type"], - "id": record["id"], - "vtype": record["vtype"], - } - ) - helper_results = run_helper(actual_helper, records, args.artifacts) - - report = { - "batch": "B11", - "candidate": "P15-081", - "dir_source_unchanged": True, - "dotdot_bypasses_validator": True, - "gate_status": gate_result["status"], - "helper_case_count": len(helper_results), - "helper_results": helper_results, - "known_mismatch_errno": "+EINTEGRITY", - "namecache_publication_after_validator": True, - "readdir_vget_count": 0, - "source_transform_exact": True, - "status": "PASS", - "unknown_and_reserved_tolerated": True, - "vnode_source_unchanged": True, - } - (args.artifacts / "B11-dtype-report.json").write_text( - json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - print( - "B11 dtype: PASS " - f"helper_cases={len(helper_results)} known_mismatch=+EINTEGRITY " - "unknown=accepted readdir_vgets=0 dotdot=bypass" - ) - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B11-dtype-spec.json b/tests/pre15/fixtures/B11-dtype-spec.json deleted file mode 100644 index b4719a2..0000000 --- a/tests/pre15/fixtures/B11-dtype-spec.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "baseline": "e2e3fb86b6fffcb01d6fd29c17dd95628ad070de", - "batch": "B11", - "candidate": "P15-081", - "expected": { - "fixture_count": 14, - "fixture_set_sha256": "b562a7e42e139b16f3ce399d585aa7c373a66aa478a084bfed67b2b2aa9f2bd3", - "forward_compatible_case_count": 5, - "known_match_count": 8, - "known_mismatch_count": 8, - "normal_entry_count": 13, - "prototype_case_count": 21 - }, - "gate_decision": "e2cb014fede2a8eb35387ff84f613f59a555defc", - "gate_evidence_sha256": { - "SHA256SUMS": "36b486617920b04ca87e88016e2fc0f519cda3a5f99dc675c9a89549615a02e3", - "candidate.patch": "427097da9304f5bedb770be8cab5f589706d316fb18a152ca16cb9d0663713fd", - "fixture-index.json": "1a46b8ebfec16109c5d193001dc650a1b56000045e5d9e1e9b145fe7c5df9a16", - "lock-ledger.json": "e8c2e6ba797fbcc83d832af6583d3f74f344750f6897111e7fe501130e577c13", - "normal-entry.tsv": "b2841448068f70633ed0c02d901177a639cfed150f0e1c58977cc6c7b1e7313e", - "oracle.json": "932e558d3ead05572f38c89635bb10cae45393e42028031f81dfc7a56f2093f6", - "prototype.c": "1a295d18830c61a9708d465c3b1e415efb3ff8276100e4e32139bfa715195d8b", - "result.json": "4ec8e053c60ae75ae2fc4580a3d820a0452cf519f6b85376953135e18a7ec0ec" - }, - "gate_input_sha256": "a887fad56927545adb48462b770100d1e64b22b32554ed8ffad14c72b8febfc3", - "gate_script_sha256": "6215a3005214ba8d25dacd63e039320a0c63308d2334b5faff99a3927cad7d2e", - "schema": 1 -} diff --git a/tests/pre15/fixtures/B12-build-kld.sh b/tests/pre15/fixtures/B12-build-kld.sh deleted file mode 100755 index 50c3a29..0000000 --- a/tests/pre15/fixtures/B12-build-kld.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -set -eu - -dut=${1:?DUT path is required} -freebsd_src=${2:?FreeBSD source path is required} -output=${3:?output module path is required} -work=${4:?work directory is required} -src=$dut/src -sys=$freebsd_src/sys -target=x86_64-unknown-freebsd15.0 - -test -d "$sys" || { - printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2 - exit 21 -} -for tool in awk clang grep nm; do - command -v "$tool" >/dev/null 2>&1 || { - printf '%s\n' "missing B12 KLD tool: $tool" >&2 - exit 21 - } -done -test ! -e "$work" || { - printf '%s\n' "B12 KLD work path exists: $work" >&2 - exit 20 -} -mkdir -p "$work" "$(dirname -- "$output")" -cd "$work" -ln -s "$sys/amd64/include" machine -ln -s "$sys/x86/include" x86 -ln -s "$sys/i386/include" i386 -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h -: > opt_global.h - -cflags="-O2 -pipe -fno-common -fno-strict-aliasing \ --D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \ --I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \ --mno-red-zone -mno-mmx -mno-sse -msoft-float \ --fno-asynchronous-unwind-tables -ffreestanding -fwrapv \ --fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \ --Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \ --Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \ --Wno-unknown-pragmas -Wno-address-of-packed-member \ --Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \ --D__printf__=__freebsd_kprintf__ --target=$target" - -for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \ - decompressor.c zmap.c zdata.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do - printf 'CC %s\n' "$file" - clang $cflags -I"$sys/contrib/zstd/lib/freebsd" \ - -c "$src/$file" -o "${file%.c}.o" -done - -clang --target="$target" -r -nostdlib ./*.o -o "$output" -nm -u "$output" | LC_ALL=C sort > B12-nm-u.txt -nm -g "$output" | LC_ALL=C sort > B12-nm-global.txt -if ! awk '$NF == "bcmp" { found = 1 } END { exit found }' B12-nm-u.txt; then - printf '%s\n' 'B12 KLD contains an unresolved bcmp reference' >&2 - exit 1 -fi -if grep -q ' ZSTD_' B12-nm-u.txt; then - printf '%s\n' 'B12 KLD contains an unexpected Zstd symbol' >&2 - exit 1 -fi diff --git a/tests/pre15/fixtures/B12-qemu-accept.sh b/tests/pre15/fixtures/B12-qemu-accept.sh deleted file mode 100755 index 57aac14..0000000 --- a/tests/pre15/fixtures/B12-qemu-accept.sh +++ /dev/null @@ -1,228 +0,0 @@ -#!/bin/sh -set -eu - -dut=${1:?DUT path is required} -freebsd_src=${2:?FreeBSD source path is required} -base=${3:?base image is required} -askpass=${4:?askpass path is required} -output=${5:?output path is required} -protected_pid=26318 -protected_port=9222 -builder=$dut/tests/pre15/fixtures/B12-build-kld.sh - -test ! -e "$output" || { - printf '%s\n' "B12 QEMU output exists: $output" >&2 - exit 20 -} -mkdir -p "$output" -work=$(mktemp -d /tmp/pre15-b12-qemu.XXXXXX) -overlay=$work/guest-overlay.qcow2 -module=$work/B12-erofs-zstdio0.ko -qemu_pid= -port= -guest_ready=0 -module_loaded=0 - -protected_start=$(awk '{print $22}' /proc/$protected_pid/stat) -protected_cmd=$(sha256sum /proc/$protected_pid/cmdline | awk '{print $1}') -stat -c 'path=%n size=%s inode=%i mode=%f mtime=%Y ctime=%Z' "$base" \ - >"$output/base-metadata-before.txt" - -ssh_options() -{ - printf '%s\n' \ - -q \ - -o BatchMode=no \ - -o PubkeyAuthentication=no \ - -o PreferredAuthentications=keyboard-interactive,password \ - -o NumberOfPasswordPrompts=1 \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=5 \ - -p "$port" -} - -guest() -{ - timeout -k 5 "${B12_GUEST_TIMEOUT:-60}" env DISPLAY=:0 \ - SSH_ASKPASS="$askpass" SSH_ASKPASS_REQUIRE=force \ - ssh $(ssh_options) root@127.0.0.1 "$@" -} - -copy_to_guest() -{ - timeout -k 5 "${B12_COPY_TIMEOUT:-120}" env DISPLAY=:0 \ - SSH_ASKPASS="$askpass" SSH_ASKPASS_REQUIRE=force \ - scp -O -q \ - -o BatchMode=no \ - -o PubkeyAuthentication=no \ - -o PreferredAuthentications=keyboard-interactive,password \ - -o NumberOfPasswordPrompts=1 \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=5 \ - -P "$port" "$1" root@127.0.0.1:"$2" -} - -port_free() -{ - python3 - "$1" <<'PY' -import socket -import sys - -with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.settimeout(0.2) - raise SystemExit(0 if sock.connect_ex(("127.0.0.1", int(sys.argv[1]))) != 0 else 1) -PY -} - -cleanup() -{ - set +e - if test "$guest_ready" = 1 && test "$module_loaded" = 1; then - guest kldunload /root/B12-erofs-zstdio0.ko \ - >>"$output/cleanup.log" 2>&1 - module_loaded=0 - fi - if test -n "$qemu_pid" && kill -0 "$qemu_pid" 2>/dev/null; then - kill -TERM "$qemu_pid" 2>/dev/null - wait_count=0 - while kill -0 "$qemu_pid" 2>/dev/null && test "$wait_count" -lt 20; do - sleep 1 - wait_count=$((wait_count + 1)) - done - if kill -0 "$qemu_pid" 2>/dev/null; then - kill -KILL "$qemu_pid" 2>/dev/null - fi - wait "$qemu_pid" 2>/dev/null - fi - rm -rf "$work" -} -trap cleanup EXIT HUP INT TERM - -for tool in awk clang file nm python3 qemu-img qemu-system-x86_64 scp \ - sha256sum ssh stat timeout; do - command -v "$tool" >/dev/null 2>&1 || { - printf '%s\n' "missing B12 QEMU tool: $tool" >&2 - exit 21 - } -done -test -d "$freebsd_src/sys" -test -f "$base" -test -x "$askpass" -test -r /proc/$protected_pid/stat - -timeout -k 10 600 /bin/sh "$builder" "$dut" "$freebsd_src" "$module" \ - "$work/kld-work" >"$output/build.stdout" 2>"$output/build.stderr" -file "$module" >"$output/module.file" -sha256sum "$module" >"$output/module.sha256" -nm -g "$module" | LC_ALL=C sort >"$output/module.nm-global" -nm -u "$module" | LC_ALL=C sort >"$output/module.nm-u" - -qemu-img create -q -f qcow2 -F qcow2 -b "$base" "$overlay" -port=$(python3 - "$protected_port" <<'PY' -import socket -import sys - -protected = int(sys.argv[1]) -while True: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - if port != protected: - print(port) - break -PY -) -printf '%s\n' "$port" >"$output/owned-port.txt" -qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 \ - -m 3072 -smp 2 \ - -drive "file=$overlay,if=virtio,format=qcow2,cache=none" \ - -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$port-:22" \ - -device virtio-net-pci,netdev=net0 -display none \ - -serial "file:$output/serial.log" -monitor none \ - -D "$output/qemu.log" >"$output/qemu-process.log" 2>&1 & -qemu_pid=$! -printf '%s\n' "$qemu_pid" >"$output/owned-pid.txt" -printf '%s\n' \ - "qemu-system-x86_64 -m 3072 -smp 2 -drive owned-overlay -port $port" \ - >"$output/qemu-argv-summary.txt" - -boot_wait=0 -while test "$boot_wait" -lt "${B12_BOOT_TIMEOUT:-300}"; do - if ! kill -0 "$qemu_pid" 2>/dev/null; then - printf '%s\n' 'owned QEMU exited before SSH readiness' >&2 - exit 21 - fi - if B12_GUEST_TIMEOUT=10 guest true >/dev/null 2>&1; then - guest_ready=1 - break - fi - sleep 2 - boot_wait=$((boot_wait + 2)) -done -test "$guest_ready" = 1 || { - printf '%s\n' 'owned QEMU SSH boot deadline expired' >&2 - exit 21 -} - -guest uname -a >"$output/guest-uname.txt" -guest sysctl -n kern.osreldate >"$output/guest-osreldate.txt" -test "$(tr -d '\r\n' < "$output/guest-osreldate.txt")" = 1500068 || { - printf '%s\n' 'guest OSREL is not exact FreeBSD 15 ABI 1500068' >&2 - exit 10 -} -if guest kldstat -q -m erofs >/dev/null 2>&1; then - printf '%s\n' 'guest already has an EROFS module loaded' >&2 - exit 21 -fi -copy_to_guest "$module" /root/B12-erofs-zstdio0.ko -guest sha256 -q /root/B12-erofs-zstdio0.ko >"$output/guest-module.sha256" -test "$(awk '{print $1}' "$output/module.sha256")" = \ - "$(tr -d '\r\n' < "$output/guest-module.sha256")" -guest dmesg >"$output/dmesg-before.txt" -guest kldload /root/B12-erofs-zstdio0.ko >"$output/kldload.stdout" \ - 2>"$output/kldload.stderr" -module_loaded=1 -guest kldstat -n B12-erofs-zstdio0.ko >"$output/kldstat-loaded.txt" -guest kldunload /root/B12-erofs-zstdio0.ko >"$output/kldunload.stdout" \ - 2>"$output/kldunload.stderr" -module_loaded=0 -guest dmesg >"$output/dmesg-after.txt" -diff -u "$output/dmesg-before.txt" "$output/dmesg-after.txt" \ - >"$output/dmesg.diff" || true -sed -n '/^+++ /d; /^+/s/^+//p' "$output/dmesg.diff" \ - >"$output/dmesg-added.txt" -if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free|link_elf_obj: symbol .* undefined' \ - "$output/dmesg-added.txt"; then - printf '%s\n' 'B12 exact-ABI load produced a kernel diagnostic' >&2 - exit 10 -fi - -cleanup -trap - EXIT HUP INT TERM -stat -c 'path=%n size=%s inode=%i mode=%f mtime=%Y ctime=%Z' "$base" \ - >"$output/base-metadata-after.txt" -cmp "$output/base-metadata-before.txt" "$output/base-metadata-after.txt" -test -r /proc/$protected_pid/stat -test "$(awk '{print $22}' /proc/$protected_pid/stat)" = "$protected_start" -test "$(sha256sum /proc/$protected_pid/cmdline | awk '{print $1}')" = "$protected_cmd" -port_free "$port" -cat >"$output/result.json" <&2; exit 21; } -for tool in awk clang grep nm; do - command -v "$tool" >/dev/null 2>&1 || { - printf '%s\n' "missing B14 KLD tool: $tool" >&2 - exit 21 - } -done -test ! -e "$work" || { printf '%s\n' "B14 KLD work path exists: $work" >&2; exit 20; } -mkdir "$work" -cd "$work" -ln -s "$sys/amd64/include" machine -ln -s "$sys/x86/include" x86 -ln -s "$sys/i386/include" i386 -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h -: > opt_global.h - -cflags="-O2 -pipe -fno-common -fno-strict-aliasing \ --D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \ --I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \ --mno-red-zone -mno-mmx -mno-sse -msoft-float \ --fno-asynchronous-unwind-tables -ffreestanding -fwrapv \ --fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \ --Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \ --Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \ --Wno-unknown-pragmas -Wno-address-of-packed-member \ --Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \ --D__printf__=__freebsd_kprintf__ --target=$target" - -for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \ - decompressor.c zmap.c zdata.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do - extra= - if test "$file" = decompressor_zstd.c; then - extra="-I$sys/contrib/zstd/lib/freebsd" - fi - clang $cflags $extra -c "$src/$file" -o "${file%.c}.o" -done - -clang --target="$target" -r -nostdlib ./*.o -o "$output" -if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then - : -else - printf '%s\n' 'B14 KLD contains an unresolved bcmp reference' >&2 - exit 1 -fi -if nm -u "$output" | grep -q ' ZSTD_'; then - printf '%s\n' 'B14 KLD contains an unexpected Zstd symbol' >&2 - exit 1 -fi diff --git a/tests/pre15/fixtures/B14-chunk-fixtures.py b/tests/pre15/fixtures/B14-chunk-fixtures.py deleted file mode 100755 index a89c05c..0000000 --- a/tests/pre15/fixtures/B14-chunk-fixtures.py +++ /dev/null @@ -1,661 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import hashlib -import json -import math -from pathlib import Path -import re -import stat -import struct -import subprocess -import sys -from typing import Any - - -SCRIPT_DIR = Path(__file__).resolve().parent -TESTS_DIR = SCRIPT_DIR.parent.parent -sys.path.insert(0, str(TESTS_DIR)) - -from erofs_fixture import ErofsImage, FEATURE_INCOMPAT_48BIT, SUPER - - -BLOCK_SIZE = 4096 -CHUNK_FORMAT_INDEXES = 0x0020 -CHUNK_FORMAT_48BIT = 0x0040 -CHUNK_FORMAT_ALL = 0x007F -FEATURE_INCOMPAT_CHUNKED_FILE = 0x00000004 -EROFS_I_DOT_OMITTED_BIT = 4 -EROFS_I_NLINK_1_BIT = 5 -EROFS_FT_REG_FILE = 1 -EROFS_FT_DIR = 2 -EROFS_FT_CHRDEV = 3 -EROFS_FT_SYMLINK = 7 -UUID = "67360174-0014-0000-0000-000000000174" -TARGET = b"target.txt" -TARGET_CONTENT = b"B14 target content\n" -ENTRY_CONTENT = b"B14 directory entry\n" - - -class FixtureError(RuntimeError): - pass - - -def run(command: list[str], *, capture: bool = False) -> str: - result = subprocess.run( - command, - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - if result.returncode != 0: - raise FixtureError( - f"command failed ({result.returncode}): {' '.join(command)}\n{result.stdout}" - ) - return result.stdout if capture else "" - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def align_up(value: int, alignment: int) -> int: - return (value + alignment - 1) & ~(alignment - 1) - - -def source_bytes(label: str, size: int) -> bytes: - chunks = [] - for index in range(math.ceil(size / 32)): - chunks.append(hashlib.sha256(f"B14:{label}:{index}".encode("ascii")).digest()) - return b"".join(chunks)[:size] - - -def make_source(root: Path) -> None: - real_dir = root / "real-dir" - real_dir.mkdir(parents=True) - (real_dir / "entry.txt").write_bytes(ENTRY_CONTENT) - (root / "target.txt").write_bytes(TARGET_CONTENT) - (root / "chunk-dir").write_bytes(source_bytes("chunk-dir", 8192)) - link_data = TARGET + source_bytes("chunk-link", 8192 - len(TARGET)) - (root / "chunk-link").write_bytes(link_data) - (root / "chunk-char").write_bytes(source_bytes("chunk-char", 8192)) - - -def make_image(mkfs: str, primary: Path, blob: Path, source: Path) -> None: - run( - [ - mkfs, - "-T0", - "--all-time", - "--all-root", - "--workers=1", - f"-U{UUID}", - "--chunksize=4096", - f"--blobdev={blob}", - str(primary), - str(source), - ] - ) - - -def root_entry(image: ErofsImage, path: str): - _, entry = image.resolve_root_entry(path) - return entry - - -def inode_data(image: ErofsImage, inode) -> bytes: - if inode.layout == 0: - offset = inode.start_block << image.block_bits - elif inode.layout == 2: - offset = inode.offset + inode.inode_size + inode.xattr_size - else: - raise FixtureError(f"unsupported source data layout {inode.layout}") - end = offset + inode.size - if end > len(image.data): - raise FixtureError("source inode data exceeds primary image") - return bytes(image.data[offset:end]) - - -def chunk_info(image: ErofsImage, path: str) -> dict[str, int]: - entry = root_entry(image, path) - inode = image.inode(entry.nid) - if inode.layout != 4: - raise FixtureError(f"{path}: expected chunk layout, got {inode.layout}") - chunk_format, reserved = struct.unpack_from(" None: - entry = root_entry(image, path) - offset = entry.offset + 10 - if image.data[offset] != EROFS_FT_REG_FILE: - raise FixtureError(f"{path}: root entry is not a regular file") - image.data[offset] = file_type - - -def patch_mode(image: ErofsImage, path: str, mode_type: int) -> None: - info = chunk_info(image, path) - if stat.S_IFMT(info["mode"]) != stat.S_IFREG: - raise FixtureError(f"{path}: carrier inode is not regular") - image.put_u16(info["inode_offset"] + 4, mode_type | (info["mode"] & 0o7777)) - - -def patch_size(image: ErofsImage, path: str, size: int) -> None: - info = chunk_info(image, path) - if info["inode_size"] == 32: - image.put_u32(info["inode_offset"] + 8, size) - else: - image.put_u64(info["inode_offset"] + 8, size) - - -def patch_directory(image: ErofsImage, blob: bytearray) -> None: - real_entry = root_entry(image, "/real-dir") - real_inode = image.inode(real_entry.nid) - directory_data = bytearray(inode_data(image, real_inode)) - info = chunk_info(image, "/chunk-dir") - if info["device_id"] != 1 or info["high"] != 0: - raise FixtureError("chunk directory carrier is not on external slot 1") - blob_offset = info["low"] << image.block_bits - if blob_offset + image.block_size > len(blob): - raise FixtureError("chunk directory carrier exceeds blob") - first_name_offset = struct.unpack_from(" None: - patch_mode(image, "/chunk-link", stat.S_IFLNK) - patch_size(image, "/chunk-link", len(TARGET)) - patch_root_type(image, "/chunk-link", EROFS_FT_SYMLINK) - - -def patch_character(image: ErofsImage) -> None: - patch_mode(image, "/chunk-char", stat.S_IFCHR) - patch_size(image, "/chunk-char", 0) - patch_root_type(image, "/chunk-char", EROFS_FT_CHRDEV) - - -def enable_48bit(image: ErofsImage) -> None: - if image.feature_incompat & FEATURE_INCOMPAT_48BIT: - raise FixtureError("48-bit feature already enabled") - root_nid = image.root_nid - image.put_u32(SUPER + 80, image.feature_incompat | FEATURE_INCOMPAT_48BIT) - image.put_u16(SUPER + 14, 0) - image.put_u64(SUPER + 112, root_nid) - for path in ("/chunk-dir", "/chunk-link", "/chunk-char"): - info = chunk_info(image, path) - if info["entry_size"] != 8 or info["high"] != 0: - raise FixtureError(f"{path}: cannot promote chunk index to 48-bit") - image.put_u16(info["inode_offset"] + 16, info["format"] | CHUNK_FORMAT_48BIT) - - -def save_image(image: ErofsImage, path: Path) -> None: - if image.feature_compat & 1: - image.update_checksum() - image.validate_superblock() - image.save(path) - - -def patch_index_low(image: ErofsImage, path: str, value: int) -> None: - info = chunk_info(image, path) - if info["entry_size"] != 8: - raise FixtureError("bad-index fixture requires an indexed chunk") - image.put_u32(info["index_base"] + 4, value) - - -def patch_index48(image: ErofsImage, path: str, high: int, low: int) -> None: - info = chunk_info(image, path) - if info["entry_size"] != 8 or not info["format"] & CHUNK_FORMAT_48BIT: - raise FixtureError("bad-index48 fixture requires a 48-bit chunk index") - image.put_u16(info["index_base"], high) - image.put_u32(info["index_base"] + 4, low) - - -def load_spec(path: Path) -> dict[str, Any]: - spec = json.loads(path.read_text(encoding="ascii")) - if spec.get("schema") != 1 or spec.get("batch") != "B14": - raise FixtureError("invalid B14 fixture schema") - if spec.get("candidate") != "P15-056" or spec.get("test") != "TC174-chunk-nonregular": - raise FixtureError("invalid B14 fixture identity") - cases = spec.get("cases") - if not isinstance(cases, list) or len(cases) != 10: - raise FixtureError("B14 fixture case denominator changed") - identifiers = [case.get("id") for case in cases] - if len(set(identifiers)) != len(identifiers) or not all(isinstance(value, str) for value in identifiers): - raise FixtureError("invalid or duplicate B14 case ID") - return spec - - -def write_sums(output: Path, names: list[str]) -> dict[str, str]: - hashes = {name: sha256(output / name) for name in sorted(names)} - with (output / "SHA256SUMS").open("w", encoding="ascii") as stream: - for name, digest in hashes.items(): - stream.write(f"{digest} {name}\n") - return hashes - - -def generate(args: argparse.Namespace) -> None: - output = args.output - if output.exists(): - raise FixtureError(f"output already exists: {output}") - spec = load_spec(args.spec) - output.mkdir(parents=True) - source = output / "source" - source.mkdir() - make_source(source) - raw_primary = output / "raw.erofs" - blob_path = output / "B14-slot1.blob" - blob_path.write_bytes(b"") - make_image(args.mkfs, raw_primary, blob_path, source) - - image = ErofsImage.load(raw_primary) - image.validate_superblock() - if not image.feature_incompat & FEATURE_INCOMPAT_CHUNKED_FILE: - raise FixtureError("mkfs image lacks CHUNKED_FILE") - blob = bytearray(blob_path.read_bytes()) - for path in ("/chunk-dir", "/chunk-link", "/chunk-char"): - info = chunk_info(image, path) - if info["entry_size"] != 8 or info["device_id"] != 1 or info["high"] != 0: - raise FixtureError(f"{path}: unexpected mkfs chunk carrier") - - patch_directory(image, blob) - patch_symlink(image) - patch_character(image) - blob_path.write_bytes(blob) - good32 = image.clone() - save_image(good32, output / "B14-good32.erofs") - - good48 = image.clone() - enable_48bit(good48) - save_image(good48, output / "B14-good48.erofs") - - blob_blocks = len(blob) // BLOCK_SIZE - bad_index = good32.clone() - patch_index_low(bad_index, "/chunk-link", blob_blocks + 1) - save_image(bad_index, output / "B14-bad-index.erofs") - - bad_index48 = good48.clone() - patch_index48(bad_index48, "/chunk-link", 0xFFFF, 0xFFFFFFFE) - save_image(bad_index48, output / "B14-bad-index48.erofs") - - bad_reserved = good32.clone() - directory = chunk_info(bad_reserved, "/chunk-dir") - bad_reserved.put_u16(directory["inode_offset"] + 18, 1) - save_image(bad_reserved, output / "B14-bad-reserved.erofs") - - bad_format = good32.clone() - link = chunk_info(bad_format, "/chunk-link") - bad_format.put_u16(link["inode_offset"] + 16, link["format"] | (CHUNK_FORMAT_ALL + 1)) - save_image(bad_format, output / "B14-bad-format.erofs") - - bad_no_index = good48.clone() - link48 = chunk_info(bad_no_index, "/chunk-link") - bad_no_index.put_u16( - link48["inode_offset"] + 16, - (link48["format"] | CHUNK_FORMAT_48BIT) & ~CHUNK_FORMAT_INDEXES, - ) - save_image(bad_no_index, output / "B14-bad-48-no-index.erofs") - raw_primary.unlink() - - names = [ - "B14-slot1.blob", - "B14-good32.erofs", - "B14-good48.erofs", - "B14-bad-index.erofs", - "B14-bad-index48.erofs", - "B14-bad-reserved.erofs", - "B14-bad-format.erofs", - "B14-bad-48-no-index.erofs", - ] - hashes = write_sums(output, names) - manifest = { - "schema": 1, - "batch": "B14", - "candidate": "P15-056", - "test": "TC174-chunk-nonregular", - "mkfs_version": run([args.mkfs, "-V"], capture=True).splitlines()[0], - "hashes": hashes, - "blob_blocks": blob_blocks, - "target": TARGET.decode("ascii"), - "qualification": { - "good32": "fsck.erofs plus independent parser", - "good48": "independent parser; erofs-utils 1.8.6 lacks 48-bit support", - }, - "cases": [case["id"] for case in spec["cases"]], - "good32": { - path: chunk_info(good32, path) - for path in ("/chunk-dir", "/chunk-link", "/chunk-char") - }, - "good48": { - path: chunk_info(good48, path) - for path in ("/chunk-dir", "/chunk-link", "/chunk-char") - }, - } - (output / "manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - - -def verify(args: argparse.Namespace) -> None: - spec = load_spec(args.spec) - output = args.output - manifest = json.loads((output / "manifest.json").read_text(encoding="ascii")) - if manifest.get("cases") != [case["id"] for case in spec["cases"]]: - raise FixtureError("manifest case list differs from B14 spec") - for name, expected in manifest.get("hashes", {}).items(): - if sha256(output / name) != expected: - raise FixtureError(f"fixture hash mismatch: {name}") - - good32 = ErofsImage.load(output / "B14-good32.erofs") - good48 = ErofsImage.load(output / "B14-good48.erofs") - blob = (output / "B14-slot1.blob").read_bytes() - for image, width in ((good32, 32), (good48, 48)): - image.validate_superblock() - directory = chunk_info(image, "/chunk-dir") - link = chunk_info(image, "/chunk-link") - character = chunk_info(image, "/chunk-char") - for info in (directory, link, character): - if info["device_id"] != 1 or info["entry_size"] != 8: - raise FixtureError(f"{width}-bit external chunk contract changed") - has_48bit = bool(info["format"] & CHUNK_FORMAT_48BIT) - if has_48bit != (width == 48) or info["high"] != 0: - raise FixtureError(f"{width}-bit chunk format contract changed") - if stat.S_IFMT(directory["mode"]) != stat.S_IFDIR: - raise FixtureError("chunk directory mode changed") - if stat.S_IFMT(link["mode"]) != stat.S_IFLNK or link["size"] != len(TARGET): - raise FixtureError("chunk symlink mode or size changed") - if stat.S_IFMT(character["mode"]) != stat.S_IFCHR or character["size"] != 0: - raise FixtureError("chunk character mode or size changed") - link_offset = link["low"] << image.block_bits - if blob[link_offset : link_offset + len(TARGET)] != TARGET: - raise FixtureError("chunk symlink payload changed") - directory_offset = directory["low"] << image.block_bits - directory_data = blob[directory_offset : directory_offset + directory["size"]] - first_name_offset = struct.unpack_from(" str: - match = re.search( - r"^(?:[A-Za-z_][^\n;{}]*\s+)?" + re.escape(name) + r"\s*\(", - source, - re.MULTILINE, - ) - if not match: - raise FixtureError(f"missing function: {name}") - start = source.rfind("\n\n", 0, match.start()) + 2 - brace = source.find("{", match.end()) - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or (state == "character" and char == "'"): - state = "code" - index += 1 - raise FixtureError(f"unterminated function: {name}") - - -def committed(root: Path, baseline: str, path: str) -> str: - result = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if result.returncode != 0: - raise FixtureError(f"cannot read B14 baseline {path}: {result.stderr}") - return result.stdout - - -def audit(args: argparse.Namespace) -> None: - load_spec(args.spec) - root = args.root - dut = args.dut - inode = (dut / "src/inode.c").read_text(encoding="utf-8") - data = (dut / "src/data.c").read_text(encoding="utf-8") - vnops = (dut / "src/erofs_vnops.c").read_text(encoding="utf-8") - linux_inode = (root / "src-linux/inode.c").read_text(encoding="utf-8") - base_inode = committed(root, args.baseline, "src/inode.c") - base_data = committed(root, args.baseline, "src/data.c") - old_gate = "\t\tif (!erofs_sb_has_chunked_file(sbi) || vi->vtype != VREG) {" - new_gate = "\t\tif (!erofs_sb_has_chunked_file(sbi)) {" - if base_inode.count(old_gate) != 1: - raise FixtureError("B14 baseline type gate changed") - expected_inode = base_inode.replace(old_gate, new_gate, 1) - if inode != expected_inode: - raise FixtureError("inode.c differs from the single declared B14 transform") - if data != base_data: - raise FixtureError("data.c changed although the chunk mapper is already type-neutral") - - read_inode = extract_function(inode, "erofs_read_inode") - chunk_mapper = extract_function(data, "erofs_map_blocks_chunk") - map_device = extract_function(data, "erofs_map_dev") - open_vnode = extract_function(vnops, "erofs_open") - read_vnode = extract_function(vnops, "erofs_read") - readdir_vnode = extract_function(vnops, "erofs_readdir") - readlink_vnode = extract_function(vnops, "erofs_readlink") - linux_read_inode = extract_function(linux_inode, "erofs_read_inode") - - required_inode = ( - "!erofs_sb_has_chunked_file(sbi)", - "le16toh(chunk_info.reserved) != 0", - "vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL", - "EROFS_CHUNK_FORMAT_48BIT", - "EROFS_CHUNK_FORMAT_INDEXES", - "vi->chunkbits >= 64", - "case VREG:", - "case VDIR:", - "case VLNK:", - ) - for marker in required_inode: - if marker not in read_inode: - raise FixtureError(f"B14 removed inode invariant: {marker}") - if "vi->vtype != VREG" in read_inode: - raise FixtureError("nonregular chunk type gate remains") - if read_inode.count("return (EINTEGRITY);") < 8 or "return (EOPNOTSUPP);" not in read_inode: - raise FixtureError("FreeBSD inode errno contract changed") - - required_mapper = ( - "vi->chunkbits", - "vi->chunkformat", - "vi->inode_off", - "vi->inode_isize", - "vi->xattr_isize", - "map->m_deviceid = raw_device_id & sbi->device_id_mask", - "map->m_flags |= EROFS_MAP_MAPPED", - ) - for marker in required_mapper: - if marker not in chunk_mapper: - raise FixtureError(f"chunk mapper contract changed: {marker}") - if "vtype" in chunk_mapper or "VREG" in chunk_mapper or "VLNK" in chunk_mapper or "VDIR" in chunk_mapper: - raise FixtureError("chunk mapper acquired an OS vnode type dependency") - if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", chunk_mapper + map_device): - raise FixtureError("negative Linux errno entered the FreeBSD map path") - for marker in ("ENODEV", "dif->devvp", "dif->cp", "erofs_check_device_range"): - if marker not in map_device: - raise FixtureError(f"FreeBSD device/GEOM boundary changed: {marker}") - - if "VN_ISDEV(vp)" not in open_vnode or "return (EOPNOTSUPP);" not in open_vnode: - raise FixtureError("special vnode open safety changed") - if "case VREG:" not in read_vnode or "case VDIR:" not in read_vnode: - raise FixtureError("FreeBSD VOP_READ type boundary changed") - if "v_type != VDIR" not in readdir_vnode or "v_type != VLNK" not in readlink_vnode: - raise FixtureError("FreeBSD readdir/readlink type boundary changed") - - for marker in ("case S_IFDIR:", "case S_IFREG:", "case S_IFLNK:"): - if marker not in linux_read_inode: - raise FixtureError(f"Linux common data-layout type marker changed: {marker}") - linux_chunk = linux_read_inode[linux_read_inode.index("if (vi->datalayout == EROFS_INODE_CHUNK_BASED)") :] - if "vi->chunkformat" not in linux_chunk or "S_ISREG" in linux_chunk.split("inode_set_atime", 1)[0]: - raise FixtureError("Linux chunk summary is no longer type-neutral") - - report = { - "schema": 1, - "batch": "B14", - "candidate": "P15-056", - "baseline": args.baseline, - "source_transform_count": 1, - "data_c_unchanged": True, - "linux_common_types": ["S_IFDIR", "S_IFREG", "S_IFLNK"], - "freebsd_vnode_types": ["VDIR", "VREG", "VLNK"], - "freebsd_boundaries": [ - "positive errno", - "explicit GEOM device slots", - "special vnode open EOPNOTSUPP", - "typed readdir/readlink VOPs", - ], - } - args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest="command", required=True) - generate_parser = subparsers.add_parser("generate") - generate_parser.add_argument("--output", type=Path, required=True) - generate_parser.add_argument("--spec", type=Path, required=True) - generate_parser.add_argument("--mkfs", default="mkfs.erofs") - verify_parser = subparsers.add_parser("verify") - verify_parser.add_argument("--output", type=Path, required=True) - verify_parser.add_argument("--spec", type=Path, required=True) - verify_parser.add_argument("--fsck", default="fsck.erofs") - audit_parser = subparsers.add_parser("audit") - audit_parser.add_argument("--root", type=Path, required=True) - audit_parser.add_argument("--dut", type=Path, required=True) - audit_parser.add_argument("--baseline", required=True) - audit_parser.add_argument("--spec", type=Path, required=True) - audit_parser.add_argument("--report", type=Path, required=True) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - if args.command == "generate": - generate(args) - elif args.command == "verify": - verify(args) - else: - audit(args) - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B14-chunk-probe.c b/tests/pre15/fixtures/B14-chunk-probe.c deleted file mode 100644 index 9261a94..0000000 --- a/tests/pre15/fixtures/B14-chunk-probe.c +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include - -static int -fail(const char *message) -{ - fprintf(stderr, "%s\n", message); - return (1); -} - -int -main(int argc, char **argv) -{ - char target[256]; - ssize_t length; - int descriptor; - - if (argc < 3) - return (fail("usage: B14-chunk-probe OP PATH [TARGET]")); - if (strcmp(argv[1], "readlink-pass") == 0) { - if (argc != 4) - return (fail("readlink-pass requires an expected target")); - length = readlink(argv[2], target, sizeof(target)); - if (length < 0) - return (fail("readlink-pass unexpectedly failed")); - if ((size_t)length != strlen(argv[3]) || - memcmp(target, argv[3], (size_t)length) != 0) - return (fail("readlink-pass returned the wrong target")); - return (0); - } - if (strcmp(argv[1], "readlink-integrity") == 0) { - errno = 0; - if (readlink(argv[2], target, sizeof(target)) >= 0 || - errno != EINTEGRITY) - return (fail("readlink did not return EINTEGRITY")); - return (0); - } - if (strcmp(argv[1], "open-unsupported") == 0) { - errno = 0; - descriptor = open(argv[2], O_RDONLY | O_NONBLOCK); - if (descriptor >= 0) { - close(descriptor); - return (fail("special vnode unexpectedly opened")); - } - if (errno != EOPNOTSUPP) - return (fail("special vnode did not return EOPNOTSUPP")); - return (0); - } - if (strcmp(argv[1], "stat-integrity") == 0 || - strcmp(argv[1], "stat-unsupported") == 0) { - struct stat status; - int expected; - - expected = strcmp(argv[1], "stat-integrity") == 0 ? - EINTEGRITY : EOPNOTSUPP; - errno = 0; - if (lstat(argv[2], &status) == 0 || errno != expected) - return (fail("stat did not return the expected errno")); - return (0); - } - return (fail("unknown B14 probe operation")); -} diff --git a/tests/pre15/fixtures/B14-chunk-spec.json b/tests/pre15/fixtures/B14-chunk-spec.json deleted file mode 100644 index f6fd0af..0000000 --- a/tests/pre15/fixtures/B14-chunk-spec.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "schema": 1, - "batch": "B14", - "candidate": "P15-056", - "test": "TC174-chunk-nonregular", - "cases": [ - { - "id": "directory-indexed32-external", - "fixture": "B14-good32.erofs", - "path": "/chunk-dir", - "vnode": "VDIR", - "operation": "lookup-readdir", - "device_id": 1, - "expected": "PASS" - }, - { - "id": "symlink-indexed32-external", - "fixture": "B14-good32.erofs", - "path": "/chunk-link", - "vnode": "VLNK", - "operation": "follow-readlink", - "device_id": 1, - "expected": "PASS" - }, - { - "id": "character-vnode-not-operational", - "fixture": "B14-good32.erofs", - "path": "/chunk-char", - "vnode": "VCHR", - "operation": "stat-open", - "device_id": 1, - "expected": "STAT_PASS_OPEN_EOPNOTSUPP" - }, - { - "id": "directory-indexed48-zero-high", - "fixture": "B14-good48.erofs", - "path": "/chunk-dir", - "vnode": "VDIR", - "operation": "lookup-readdir", - "device_id": 1, - "expected": "PASS" - }, - { - "id": "symlink-indexed48-zero-high", - "fixture": "B14-good48.erofs", - "path": "/chunk-link", - "vnode": "VLNK", - "operation": "follow-readlink", - "device_id": 1, - "expected": "PASS" - }, - { - "id": "symlink-index-outside-device", - "fixture": "B14-bad-index.erofs", - "path": "/chunk-link", - "vnode": "VLNK", - "operation": "readlink", - "device_id": 1, - "expected": "EINTEGRITY" - }, - { - "id": "symlink-index48-outside-device", - "fixture": "B14-bad-index48.erofs", - "path": "/chunk-link", - "vnode": "VLNK", - "operation": "readlink", - "device_id": 1, - "expected": "EINTEGRITY" - }, - { - "id": "directory-reserved-nonzero", - "fixture": "B14-bad-reserved.erofs", - "path": "/chunk-dir", - "vnode": "VDIR", - "operation": "stat", - "device_id": 1, - "expected": "EINTEGRITY" - }, - { - "id": "symlink-unsupported-format", - "fixture": "B14-bad-format.erofs", - "path": "/chunk-link", - "vnode": "VLNK", - "operation": "stat", - "device_id": 1, - "expected": "EOPNOTSUPP" - }, - { - "id": "symlink-48bit-without-indexes", - "fixture": "B14-bad-48-no-index.erofs", - "path": "/chunk-link", - "vnode": "VLNK", - "operation": "stat", - "device_id": 0, - "expected": "EINTEGRITY" - } - ] -} diff --git a/tests/pre15/fixtures/B16-build-kld.sh b/tests/pre15/fixtures/B16-build-kld.sh deleted file mode 100755 index 7ea9a15..0000000 --- a/tests/pre15/fixtures/B16-build-kld.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/sh -set -eu - -dut=${1:?DUT path is required} -freebsd_src=${2:?FreeBSD source path is required} -output=${3:?output module path is required} -work=${4:?work directory is required} -src=$dut/src -sys=$freebsd_src/sys -target=x86_64-unknown-freebsd15.0 - -test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; } -for tool in awk clang grep nm; do - command -v "$tool" >/dev/null 2>&1 || { - printf '%s\n' "missing B16 KLD tool: $tool" >&2 - exit 21 - } -done -test ! -e "$work" || { printf '%s\n' "B16 KLD work path exists: $work" >&2; exit 20; } -mkdir "$work" -cd "$work" -ln -s "$sys/amd64/include" machine -ln -s "$sys/x86/include" x86 -ln -s "$sys/i386/include" i386 -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h -: > opt_global.h - -cflags="-O2 -pipe -fno-common -fno-strict-aliasing \ --D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \ --I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \ --mno-red-zone -mno-mmx -mno-sse -msoft-float \ --fno-asynchronous-unwind-tables -ffreestanding -fwrapv \ --fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \ --Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \ --Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \ --Wno-unknown-pragmas -Wno-address-of-packed-member \ --Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \ --D__printf__=__freebsd_kprintf__ --target=$target" - -for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \ - decompressor.c zmap.c zdata.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do - extra= - if test "$file" = decompressor_zstd.c; then - extra="-I$sys/contrib/zstd/lib/freebsd" - fi - clang $cflags $extra -c "$src/$file" -o "${file%.c}.o" -done - -clang --target="$target" -r -nostdlib ./*.o -o "$output" -if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then - : -else - printf '%s\n' 'B16 KLD contains an unresolved bcmp reference' >&2 - exit 1 -fi -if nm -u "$output" | grep -q ' ZSTD_'; then - printf '%s\n' 'B16 KLD contains an unexpected Zstd symbol' >&2 - exit 1 -fi diff --git a/tests/pre15/fixtures/B16-symlink-probe.c b/tests/pre15/fixtures/B16-symlink-probe.c deleted file mode 100644 index 301fc9a..0000000 --- a/tests/pre15/fixtures/B16-symlink-probe.c +++ /dev/null @@ -1,158 +0,0 @@ -#define _POSIX_C_SOURCE 200809L - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#define PROBE_BUFFER_SIZE 2048 - -static const char pattern[] = "p15-019-safe-target/"; - -static int -fail(const char *message) -{ - fprintf(stderr, "%s\n", message); - return (1); -} - -static int -parse_positive(const char *text, long minimum, long maximum, long *value) -{ - char *end; - long parsed; - - errno = 0; - parsed = strtol(text, &end, 10); - if (errno != 0 || *text == '\0' || *end != '\0' || parsed < minimum || - parsed > maximum) - return (-1); - *value = parsed; - return (0); -} - -static void -fill_expected(char *buffer, size_t length) -{ - size_t index; - - for (index = 0; index < length; index++) - buffer[index] = pattern[index % (sizeof(pattern) - 1)]; -} - -static int -readlink_pass(const char *path, size_t expected_length) -{ - char actual[PROBE_BUFFER_SIZE]; - char expected[PROBE_BUFFER_SIZE]; - ssize_t length; - - if (expected_length + 1 > sizeof(actual)) - return (fail("expected length exceeds probe buffer")); - memset(actual, 0xa5, sizeof(actual)); - fill_expected(expected, expected_length); - errno = 0; - length = readlink(path, actual, sizeof(actual)); - if (length < 0) - return (fail("readlink-pass returned an error")); - if ((size_t)length != expected_length) - return (fail("readlink-pass returned the wrong length")); - if (memcmp(actual, expected, expected_length) != 0) - return (fail("readlink-pass returned the wrong target")); - if ((unsigned char)actual[expected_length] != 0xa5) - return (fail("readlink-pass appended or copied an extra byte")); - return (0); -} - -static int -readlink_errno(const char *path, int expected_errno) -{ - char actual[PROBE_BUFFER_SIZE]; - ssize_t length; - size_t index; - - memset(actual, 0xa5, sizeof(actual)); - errno = 0; - length = readlink(path, actual, sizeof(actual)); - if (length != -1 || errno != expected_errno) - return (fail("readlink-errno returned the wrong result")); - for (index = 0; index < sizeof(actual); index++) { - if ((unsigned char)actual[index] != 0xa5) - return (fail("failed readlink modified the caller buffer")); - } - return (0); -} - -static int -follow_errno(const char *path, int expected_errno) -{ - struct stat status; - - errno = 0; - if (stat(path, &status) != -1 || errno != expected_errno) - return (fail("follow-errno returned the wrong result")); - return (0); -} - -static int -concurrent_readlink(const char *path, size_t length, long workers, long loops) -{ - long worker; - int status; - pid_t child; - - for (worker = 0; worker < workers; worker++) { - child = fork(); - if (child < 0) - return (fail("fork failed")); - if (child == 0) { - for (long iteration = 0; iteration < loops; iteration++) { - if (readlink_pass(path, length) != 0) - _exit(1); - } - _exit(0); - } - } - for (worker = 0; worker < workers; worker++) { - if (wait(&status) < 0 || !WIFEXITED(status) || - WEXITSTATUS(status) != 0) - return (fail("concurrent readlink worker failed")); - } - return (0); -} - -int -main(int argc, char **argv) -{ - long first, second, third; - - if (argc == 4 && strcmp(argv[1], "readlink-pass") == 0) { - if (parse_positive(argv[3], 1, PROBE_BUFFER_SIZE - 1, &first) != 0) - return (fail("invalid readlink-pass length")); - return (readlink_pass(argv[2], (size_t)first)); - } - if (argc == 4 && strcmp(argv[1], "readlink-errno") == 0) { - if (parse_positive(argv[3], 1, 255, &first) != 0) - return (fail("invalid expected errno")); - return (readlink_errno(argv[2], (int)first)); - } - if (argc == 4 && strcmp(argv[1], "follow-errno") == 0) { - if (parse_positive(argv[3], 1, 255, &first) != 0) - return (fail("invalid expected errno")); - return (follow_errno(argv[2], (int)first)); - } - if (argc == 6 && strcmp(argv[1], "concurrent") == 0) { - if (parse_positive(argv[3], 1, PROBE_BUFFER_SIZE - 1, &first) != 0 || - parse_positive(argv[4], 1, 64, &second) != 0 || - parse_positive(argv[5], 1, 1000, &third) != 0) - return (fail("invalid concurrent arguments")); - return (concurrent_readlink(argv[2], (size_t)first, second, third)); - } - return (fail("usage: B16-symlink-probe MODE PATH ARG...")); -} diff --git a/tests/pre15/fixtures/B16-symlink-spec.json b/tests/pre15/fixtures/B16-symlink-spec.json deleted file mode 100644 index 9405ea1..0000000 --- a/tests/pre15/fixtures/B16-symlink-spec.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "batch": "B16", - "candidate": "P15-019", - "cases": [ - "normal-short", - "normal-max", - "empty", - "nul-first", - "nul-middle", - "nul-last", - "too-long" - ], - "errno": { - "EINTEGRITY": 97, - "ENAMETOOLONG": 63 - }, - "fixture_set_sha256": "5e99e5ad9112508991e78ae6e65928973d6b0da78285a059894bed06010cf373", - "gate": { - "base": "d645feb720c7022d2138d2a62eb72c022eb75351", - "commit": "aea34aba38a298d493d0a584f3985cf3589f358e", - "input_sha256": "291bf12838981ef650e16bbf7381eaa9b2921de957e97dec46e6a573750a19b8", - "script_sha256": "afa20cbd28ca9dfc8064596a312248756ed7cde54398efb5ef713818cc04ea54" - }, - "layouts": { - "chunk": { - "layout": 4, - "short_length": 31 - }, - "compressed": { - "layout": 3, - "short_length": 64 - }, - "fragment": { - "layout": 1, - "short_length": 31 - }, - "inline": { - "layout": 2, - "short_length": 31 - }, - "plain": { - "layout": 0, - "short_length": 31 - } - }, - "maxpathlen": 1024, - "qemu_cases": [ - "normal-short", - "normal-max", - "empty", - "nul-middle", - "too-long" - ], - "schema": 1, - "test": "TC166-symlink-layouts" -} diff --git a/tests/pre15/fixtures/B17-xattr-generate.py b/tests/pre15/fixtures/B17-xattr-generate.py deleted file mode 100755 index 2ebd29d..0000000 --- a/tests/pre15/fixtures/B17-xattr-generate.py +++ /dev/null @@ -1,529 +0,0 @@ -#!/usr/bin/env python3 -"""Build deterministic B17 EROFS xattr images from one audited seed.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import shutil -import struct -import subprocess - - -SUPER_OFFSET = 1024 -FEATURE_COMPAT_XATTR_FILTER = 0x00000004 -ACL_FILTER_BITS = (1 << 21) | (1 << 30) -CRC32C_POLY = 0x82F63B78 - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def write_json(path: Path, value: object) -> None: - path.write_text( - json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -class Image: - def __init__(self, data: bytes | bytearray): - self.data = bytearray(data) - if len(self.data) < SUPER_OFFSET + 144: - raise ValueError("seed is shorter than the EROFS superblock") - if self.u32(SUPER_OFFSET) != 0xE0F5E1E2: - raise ValueError("seed has the wrong EROFS magic") - self.block_bits = self.data[SUPER_OFFSET + 12] - self.block_size = 1 << self.block_bits - self.meta_blkaddr = self.u32(SUPER_OFFSET + 40) - self.xattr_blkaddr = self.u32(SUPER_OFFSET + 44) - self.root_nid = self.u16(SUPER_OFFSET + 14) - self.packed_nid = self.u64(SUPER_OFFSET + 96) - - @classmethod - def load(cls, path: Path) -> "Image": - return cls(path.read_bytes()) - - def u16(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - struct.pack_into(" None: - struct.pack_into(" dict[str, int]: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - inode_format = self.u16(offset) - inode_size = 64 if inode_format & 1 else 32 - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) - size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8) - return { - "nid": nid, - "offset": offset, - "inode_size": inode_size, - "xattr_count": xattr_count, - "xattr_size": xattr_size, - "layout": (inode_format >> 1) & 7, - "size": size, - "start_block": self.u32(offset + 16), - } - - def inline_data_offset(self, inode: dict[str, int]) -> int: - if inode["layout"] != 2: - raise ValueError("fixture seed expected flat-inline metadata") - return inode["offset"] + inode["inode_size"] + inode["xattr_size"] - - def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]: - start = self.inline_data_offset(inode) - data = self.data[start : start + inode["size"]] - first_name = struct.unpack_from(" len(data): - raise ValueError("seed root directory is malformed") - count = first_name // 12 - entries = [] - for index in range(count): - entry = index * 12 - nid = struct.unpack_from(" dict[str, int]: - inode = self.inode(self.root_nid) - for component in path.strip("/").encode("ascii").split(b"/"): - if not component: - continue - nid = next( - (candidate for name, candidate in self.directory_entries(inode) - if name == component), - None, - ) - if nid is None: - raise ValueError(f"seed path is absent: {path}") - inode = self.inode(nid) - return inode - - def body_offset(self, inode: dict[str, int]) -> int: - return inode["offset"] + inode["inode_size"] - - def first_inline_entry(self, path: str) -> tuple[dict[str, int], int]: - inode = self.resolve(path) - body = self.body_offset(inode) - shared_count = self.data[body + 4] - entry = body + 12 + shared_count * 4 - if entry + 4 > body + inode["xattr_size"]: - raise ValueError(f"seed path has no inline xattr entry: {path}") - return inode, entry - - def first_shared_entry(self, path: str) -> tuple[dict[str, int], int, int]: - inode = self.resolve(path) - body = self.body_offset(inode) - if self.data[body + 4] == 0: - raise ValueError(f"seed path has no shared xattr id: {path}") - shared_id = self.u32(body + 12) - entry = (self.xattr_blkaddr << self.block_bits) + shared_id * 4 - return inode, body, entry - - def prefix_record_offset(self) -> int: - if self.packed_nid == 0: - raise ValueError("seed has no packed prefix carrier") - packed = self.inode(self.packed_nid) - logical = self.u32(SUPER_OFFSET + 92) << 2 - if logical + 2 > packed["size"]: - raise ValueError("seed prefix header is outside packed data") - return self.inline_data_offset(packed) + logical - - def update_checksum(self) -> None: - checksum = SUPER_OFFSET + 4 - self.put_u32(checksum, 0) - end = SUPER_OFFSET + self.block_size - SUPER_OFFSET - if end > len(self.data): - raise ValueError("seed does not contain the checksummed block") - self.put_u32(checksum, crc32c(self.data[SUPER_OFFSET:end])) - - -def load_spec(path: Path) -> dict: - value = json.loads(path.read_text(encoding="ascii")) - if value.get("schema") != 1 or value.get("batch") != "B17": - raise SystemExit("invalid B17 fixture spec identity") - cases = value.get("cases") - if not isinstance(cases, list) or not cases: - raise SystemExit("B17 fixture spec has no cases") - identifiers = [case.get("id") for case in cases] - if any(not isinstance(item, str) for item in identifiers): - raise SystemExit("B17 fixture case has an invalid id") - if len(identifiers) != len(set(identifiers)): - raise SystemExit("B17 fixture case ids are not unique") - return value - - -def create_source(root: Path) -> None: - root.mkdir(parents=True) - root.chmod(0o755) - files = { - "acl.bin": b"acl\n", - "inline.bin": b"inline\n", - "prefix.bin": b"prefix\n", - "shared-0.bin": b"shared-0\n", - "shared-1.bin": b"shared-1\n", - "shared-2.bin": b"shared-2\n", - "shared-3.bin": b"shared-3\n", - } - for name, payload in files.items(): - path = root / name - path.write_bytes(payload) - path.chmod(0o644) - os.setxattr(root / "acl.bin", b"user.aclx", bytes([0xA5]) * 24) - os.setxattr(root / "inline.bin", b"user.alpha", b"value\0binary\xff") - os.setxattr( - root / "prefix.bin", b"user.company.branch.leaf", b"prefix-value" - ) - for index in range(4): - os.setxattr( - root / f"shared-{index}.bin", b"user.shared", b"shared-value" - ) - for path in sorted(root.iterdir()): - os.utime(path, (0, 0), follow_symlinks=False) - os.utime(root, (0, 0), follow_symlinks=False) - - -def build_seed(spec: dict, output: Path, work: Path) -> dict[str, object]: - mkfs = shutil.which("mkfs.erofs") - if mkfs is None: - raise SystemExit("mkfs.erofs is required to rebuild the B17 seed") - version_run = subprocess.run( - [mkfs, "-V"], check=False, text=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) - version = version_run.stdout.splitlines()[0] if version_run.stdout else "" - if version != spec["seed"]["mkfs_version"]: - raise SystemExit( - f"mkfs.erofs version differs: expected {spec['seed']['mkfs_version']!r}, " - f"got {version!r}" - ) - source = work / "source" - create_source(source) - command = [ - mkfs, - "-d0", - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "--sort=path", - f"-U{spec['seed']['uuid']}", - "-x2", - "-Exattr-name-filter,force-inode-extended", - "--xattr-prefix=user.company.", - str(output), - str(source), - ] - completed = subprocess.run( - command, check=False, text=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) - if completed.returncode != 0: - raise SystemExit( - f"mkfs.erofs failed with {completed.returncode}:\n{completed.stdout}" - ) - image = Image.load(output) - packed = image.inode(image.packed_nid) - image.put_u16(packed["offset"] + 4, 0o100644) - image.update_checksum() - output.write_bytes(image.data) - return {"version": version, "command": command, "stdout": completed.stdout} - - -def configure_acl(image: Image, direct: list[dict[str, object]]) -> None: - inode, entry = image.first_inline_entry("/acl.bin") - if image.data[entry] != 4 or image.data[entry + 1] != 1: - raise ValueError("seed ACL staging entry shape changed") - if image.u16(entry + 2) != 24: - raise ValueError("seed ACL staging value shape changed") - acl = b"".join( - [ - struct.pack(" list[dict[str, object]]: - direct: list[dict[str, object]] = [] - if name == "none": - return direct - if name == "acl-empty-suffix": - configure_acl(image, direct) - elif name in {"filter-reserved-one", "filter-reserved-255"}: - value = 1 if name.endswith("one") else 255 - offset = SUPER_OFFSET + 104 - image.data[offset] = value - direct.append({"field": "super.xattr_filter_reserved", "offset": offset, "bytes": 1}) - elif name in {"filter-feature-off", "filter-feature-off-reserved-one"}: - offset = SUPER_OFFSET + 8 - image.put_u32(offset, image.u32(offset) & ~FEATURE_COMPAT_XATTR_FILTER) - direct.append({"field": "super.feature_compat", "offset": offset, "bytes": 4}) - if name.endswith("reserved-one"): - reserved = SUPER_OFFSET + 104 - image.data[reserved] = 1 - direct.append({"field": "super.xattr_filter_reserved", "offset": reserved, "bytes": 1}) - elif name == "unknown-filter-acl-present": - configure_acl(image, direct) - inode = image.resolve("/acl.bin") - body = image.body_offset(inode) - image.put_u32(body, image.u32(body) | ACL_FILTER_BITS) - image.data[SUPER_OFFSET + 104] = 1 - direct.extend( - [ - {"field": "ibody.h_name_filter", "offset": body, "bytes": 4}, - {"field": "super.xattr_filter_reserved", "offset": SUPER_OFFSET + 104, "bytes": 1}, - ] - ) - elif name in {"inline-name-nul", "short-name-index", "inline-value-size"}: - _, entry = image.first_inline_entry("/inline.bin") - if name == "inline-name-nul": - offset = entry + 4 + image.data[entry] // 2 - image.data[offset] = 0 - direct.append({"field": "inline.e_name", "offset": offset, "bytes": 1}) - elif name == "short-name-index": - image.data[entry + 1] = 7 - direct.append({"field": "inline.e_name_index", "offset": entry + 1, "bytes": 1}) - else: - image.put_u16(entry + 2, 0xFFFF) - direct.append({"field": "inline.e_value_size", "offset": entry + 2, "bytes": 2}) - elif name in {"shared-name-nul", "shared-value-size"}: - _, _, entry = image.first_shared_entry("/shared-0.bin") - if name == "shared-name-nul": - offset = entry + 4 + image.data[entry] // 2 - image.data[offset] = 0 - direct.append({"field": "shared.e_name", "offset": offset, "bytes": 1}) - else: - image.put_u16(entry + 2, 0xFFFF) - direct.append({"field": "shared.e_value_size", "offset": entry + 2, "bytes": 2}) - elif name == "shared-id-offset": - _, body, _ = image.first_shared_entry("/shared-0.bin") - image.put_u32(body + 12, 0xFFFFFFFF) - direct.append({"field": "ibody.h_shared_xattrs[0]", "offset": body + 12, "bytes": 4}) - elif name in {"long-prefix-id",}: - _, entry = image.first_inline_entry("/prefix.bin") - image.data[entry + 1] = 0xFF - direct.append({"field": "inline.e_name_index", "offset": entry + 1, "bytes": 1}) - elif name in {"prefix-infix-nul", "prefix-base-index", "prefix-record-truncated"}: - prefix = image.prefix_record_offset() - if name == "prefix-infix-nul": - offset = prefix + 4 - image.data[offset] = 0 - direct.append({"field": "prefix.infix", "offset": offset, "bytes": 1}) - elif name == "prefix-base-index": - image.data[prefix + 2] = 0 - direct.append({"field": "prefix.base_index", "offset": prefix + 2, "bytes": 1}) - else: - image.put_u16(prefix, 255) - direct.append({"field": "prefix.record_length", "offset": prefix, "bytes": 2}) - elif name == "prefix-offset-bounds": - offset = SUPER_OFFSET + 92 - image.put_u32(offset, 0x100) - direct.append({"field": "super.xattr_prefix_start", "offset": offset, "bytes": 4}) - elif name in {"header-shared-count", "header-only", "ibody-bounds", "inline-entry-truncated"}: - inode = image.resolve("/inline.bin") - body = image.body_offset(inode) - if name == "header-shared-count": - image.data[body + 4] = (inode["xattr_size"] - 12) // 4 + 1 - direct.append({"field": "ibody.h_shared_count", "offset": body + 4, "bytes": 1}) - else: - count = {"header-only": 1, "ibody-bounds": 0xFFFF, "inline-entry-truncated": 3}[name] - image.put_u16(inode["offset"] + 2, count) - direct.append({"field": "inode.i_xattr_icount", "offset": inode["offset"] + 2, "bytes": 2}) - else: - raise ValueError(f"unknown B17 mutation: {name}") - return direct - - -def validate_seed_shape(image: Image) -> None: - feature = image.u32(SUPER_OFFSET + 8) - if not feature & FEATURE_COMPAT_XATTR_FILTER: - raise ValueError("seed does not declare the xattr filter feature") - if image.data[SUPER_OFFSET + 104] != 0: - raise ValueError("seed xattr filter reserved byte is not zero") - _, inline = image.first_inline_entry("/inline.bin") - if bytes(image.data[inline + 4 : inline + 4 + image.data[inline]]) != b"alpha": - raise ValueError("seed inline xattr changed") - shared_inode, shared_body, shared = image.first_shared_entry("/shared-0.bin") - if image.data[shared_body + 4] != 1 or shared_inode["xattr_size"] < 16: - raise ValueError("seed shared xattr header changed") - if bytes(image.data[shared + 4 : shared + 4 + image.data[shared]]) != b"shared": - raise ValueError("seed shared xattr entry changed") - _, long_entry = image.first_inline_entry("/prefix.bin") - if not image.data[long_entry + 1] & 0x80: - raise ValueError("seed long-prefix entry changed") - prefix = image.prefix_record_offset() - length = image.u16(prefix) - if image.data[prefix + 2] != 1 or bytes(image.data[prefix + 3 : prefix + 2 + length]) != b"company.": - raise ValueError("seed long-prefix record changed") - _, acl = image.first_inline_entry("/acl.bin") - if image.data[acl] != 4 or image.u16(acl + 2) != 24: - raise ValueError("seed ACL staging entry changed") - - -def command_rebuild(spec_path: Path, seed_path: Path, output: Path, work: Path) -> int: - spec = load_spec(spec_path) - if output.exists() or work.exists(): - raise SystemExit("rebuild output and work paths must not exist") - work.mkdir(parents=True) - details = build_seed(spec, output, work) - actual = sha256(output) - expected = spec["seed"]["sha256"] - tracked = sha256(seed_path) - if actual != expected or tracked != expected: - raise SystemExit( - f"B17 seed reproducibility failed: generated={actual} tracked={tracked} expected={expected}" - ) - validate_seed_shape(Image.load(output)) - print(json.dumps({"status": "PASS", "seed_sha256": actual, **details}, sort_keys=True)) - return 0 - - -def command_create_seed(spec_path: Path, output: Path, work: Path) -> int: - spec = load_spec(spec_path) - if output.exists() or work.exists(): - raise SystemExit("seed output and work paths must not exist") - work.mkdir(parents=True) - details = build_seed(spec, output, work) - validate_seed_shape(Image.load(output)) - actual = sha256(output) - print(json.dumps({"status": "PASS", "seed_sha256": actual, **details}, sort_keys=True)) - return 0 - - -def command_generate(spec_path: Path, seed_path: Path, output: Path) -> int: - spec = load_spec(spec_path) - if output.exists(): - raise SystemExit(f"refusing to overwrite fixture output: {output}") - expected_seed = spec["seed"]["sha256"] - actual_seed = sha256(seed_path) - if actual_seed != expected_seed: - raise SystemExit( - f"B17 seed hash differs: expected {expected_seed}, got {actual_seed}" - ) - seed = Image.load(seed_path) - validate_seed_shape(seed) - output.mkdir(parents=True) - records = [] - aggregate = hashlib.sha256() - for case in spec["cases"]: - image = Image(seed.data) - before = bytes(image.data) - direct = apply_mutation(image, case["mutation"]) - image.update_checksum() - destination = output / f"{case['id']}.erofs" - destination.write_bytes(image.data) - changed = [index for index, pair in enumerate(zip(before, image.data)) if pair[0] != pair[1]] - record = { - "id": case["id"], - "class": case["class"], - "mutation": case["mutation"], - "path": destination.name, - "bytes": destination.stat().st_size, - "sha256": sha256(destination), - "direct_mutations": direct, - "changed_byte_count_with_checksum": len(changed), - } - records.append(record) - aggregate.update(case["id"].encode("ascii")) - aggregate.update(b"\0") - aggregate.update(record["sha256"].encode("ascii")) - aggregate.update(b"\n") - manifest = { - "schema": 1, - "batch": "B17", - "seed_sha256": actual_seed, - "fixture_count": len(records), - "legal_count": sum(record["class"] == "legal" for record in records), - "damaged_count": sum(record["class"] == "damaged" for record in records), - "fixture_set_sha256": aggregate.hexdigest(), - "fixtures": records, - } - write_json(output / "fixture-index.json", manifest) - print( - json.dumps( - { - "status": "PASS", - "fixture_count": manifest["fixture_count"], - "fixture_set_sha256": manifest["fixture_set_sha256"], - }, - sort_keys=True, - ) - ) - return 0 - - -def main() -> int: - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest="command", required=True) - create = subparsers.add_parser("create-seed") - create.add_argument("--spec", type=Path, required=True) - create.add_argument("--output", type=Path, required=True) - create.add_argument("--work", type=Path, required=True) - rebuild = subparsers.add_parser("rebuild-seed") - rebuild.add_argument("--spec", type=Path, required=True) - rebuild.add_argument("--seed", type=Path, required=True) - rebuild.add_argument("--output", type=Path, required=True) - rebuild.add_argument("--work", type=Path, required=True) - generate = subparsers.add_parser("generate") - generate.add_argument("--spec", type=Path, required=True) - generate.add_argument("--seed", type=Path, required=True) - generate.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - if args.command == "create-seed": - return command_create_seed(args.spec, args.output, args.work) - if args.command == "rebuild-seed": - return command_rebuild(args.spec, args.seed, args.output, args.work) - return command_generate(args.spec, args.seed, args.output) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/pre15/fixtures/B17-xattr-oracle.py b/tests/pre15/fixtures/B17-xattr-oracle.py deleted file mode 100755 index fb9f866..0000000 --- a/tests/pre15/fixtures/B17-xattr-oracle.py +++ /dev/null @@ -1,482 +0,0 @@ -#!/usr/bin/env python3 -"""Independently parse B17 images and compare frozen FreeBSD expectations.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -from pathlib import Path -import shutil -import struct -import subprocess - - -SUPER_OFFSET = 1024 -FEATURE_COMPAT_XATTR_FILTER = 0x00000004 -FEATURE_COMPAT_PLAIN_XATTR_PFX = 0x00000010 -FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040 -ACL_FILTER_BITS = (1 << 21) | (1 << 30) -CRC32C_POLY = 0x82F63B78 - - -class Reject(Exception): - def __init__(self, errno_name: str, point: str): - super().__init__(f"{errno_name} at {point}") - self.errno_name = errno_name - self.point = point - - -def crc32c(data: bytes, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -class Reader: - def __init__(self, path: Path): - self.path = path - self.data = path.read_bytes() - if len(self.data) < SUPER_OFFSET + 144: - raise Reject("EINTEGRITY", "super.bounds") - if self.u32(SUPER_OFFSET) != 0xE0F5E1E2: - raise Reject("EINTEGRITY", "super.magic") - self.block_bits = self.data[SUPER_OFFSET + 12] - if self.block_bits < 9 or self.block_bits > 16: - raise Reject("EINTEGRITY", "super.block-size") - self.block_size = 1 << self.block_bits - self.blocks = self.u32(SUPER_OFFSET + 36) - self.limit = self.blocks << self.block_bits - if self.limit > len(self.data): - raise Reject("EINTEGRITY", "super.image-bounds") - self.feature_compat = self.u32(SUPER_OFFSET + 8) - self.feature_incompat = self.u32(SUPER_OFFSET + 80) - self.filter_reserved = self.data[SUPER_OFFSET + 104] - self.meta_blkaddr = self.u32(SUPER_OFFSET + 40) - self.xattr_blkaddr = self.u32(SUPER_OFFSET + 44) - self.root_nid = self.u16(SUPER_OFFSET + 14) - self.packed_nid = self.u64(SUPER_OFFSET + 96) - self.prefix_count = self.data[SUPER_OFFSET + 91] - self.prefix_start = self.u32(SUPER_OFFSET + 92) - self.verify_checksum() - - def u16(self, offset: int) -> int: - if offset < 0 or offset + 2 > len(self.data): - raise Reject("EINTEGRITY", "raw.u16-bounds") - return struct.unpack_from(" int: - if offset < 0 or offset + 4 > len(self.data): - raise Reject("EINTEGRITY", "raw.u32-bounds") - return struct.unpack_from(" int: - if offset < 0 or offset + 8 > len(self.data): - raise Reject("EINTEGRITY", "raw.u64-bounds") - return struct.unpack_from(" None: - if not self.feature_compat & 1: - return - end = self.block_size - if end > len(self.data): - raise Reject("EINTEGRITY", "super.checksum-bounds") - expected = self.u32(SUPER_OFFSET + 4) - block = bytearray(self.data[SUPER_OFFSET:end]) - block[4:8] = bytes(4) - if crc32c(bytes(block)) != expected: - raise Reject("EINTEGRITY", "super.checksum") - - def inode(self, nid: int) -> dict[str, int]: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - if offset + 32 > self.limit: - raise Reject("EINTEGRITY", "inode.bounds") - inode_format = self.u16(offset) - inode_size = 64 if inode_format & 1 else 32 - if offset + inode_size > self.limit: - raise Reject("EINTEGRITY", "inode.bounds") - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) - size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8) - return { - "nid": nid, - "offset": offset, - "inode_size": inode_size, - "xattr_count": xattr_count, - "xattr_size": xattr_size, - "layout": (inode_format >> 1) & 7, - "size": size, - "start_block": self.u32(offset + 16), - } - - def inode_range(self, inode: dict[str, int], logical: int, length: int, point: str) -> bytes: - if logical > inode["size"] or length > inode["size"] - logical: - raise Reject("EINTEGRITY", point) - if inode["layout"] == 2: - physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical - elif inode["layout"] == 0: - physical = (inode["start_block"] << self.block_bits) + logical - else: - raise Reject("EOPNOTSUPP", "inode.layout") - if physical > self.limit or length > self.limit - physical: - raise Reject("EINTEGRITY", point) - return self.data[physical : physical + length] - - def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]: - data = self.inode_range(inode, 0, inode["size"], "directory.bounds") - if len(data) < 12: - raise Reject("EINTEGRITY", "directory.header") - first_name = struct.unpack_from(" len(data): - raise Reject("EINTEGRITY", "directory.name-offset") - count = first_name // 12 - entries = [] - for index in range(count): - entry = index * 12 - nid = struct.unpack_from(" name_end or name_end > len(data): - raise Reject("EINTEGRITY", "directory.name-bounds") - name = data[name_start:name_end].split(b"\0", 1)[0] - entries.append((name, nid)) - return entries - - def resolve(self, path: str) -> dict[str, int]: - inode = self.inode(self.root_nid) - for component in path.strip("/").encode("ascii").split(b"/"): - if not component: - continue - nid = next( - (candidate for name, candidate in self.directory_entries(inode) - if name == component), - None, - ) - if nid is None: - raise Reject("ENOATTR", "path.lookup") - inode = self.inode(nid) - return inode - - def load_prefixes(self) -> list[tuple[int, bytes]]: - if not self.feature_incompat & FEATURE_INCOMPAT_XATTR_PREFIXES or self.prefix_count == 0: - return [] - if self.feature_compat & FEATURE_COMPAT_PLAIN_XATTR_PFX: - raise Reject("EOPNOTSUPP", "prefix.plain-unexpected") - if self.packed_nid == 0: - raise Reject("EINTEGRITY", "prefix.carrier") - packed = self.inode(self.packed_nid) - logical = self.prefix_start << 2 - prefixes = [] - for _ in range(self.prefix_count): - while logical % 4: - logical += 1 - header = self.inode_range(packed, logical, 2, "prefix.header-bounds") - length = struct.unpack(" 256: - raise Reject("EINTEGRITY", "prefix.length") - payload = self.inode_range( - packed, logical + 2, length, "prefix.payload-bounds" - ) - base_index = payload[0] - infix = payload[1:] - if b"\0" in infix: - raise Reject("EINTEGRITY", "prefix.infix-nul") - prefixes.append((base_index, infix)) - logical += 2 + length - return prefixes - - def load_body(self, inode: dict[str, int]) -> tuple[int, int, list[int]]: - size = inode["xattr_size"] - if size == 0: - return 0, 0, [] - body = inode["offset"] + inode["inode_size"] - if body > self.limit or size > self.limit - body: - raise Reject("EINTEGRITY", "ibody.bounds") - if size < 12: - raise Reject("EINTEGRITY", "ibody.header-bounds") - if size == 12: - raise Reject("EOPNOTSUPP", "ibody.header-only") - shared_count = self.data[body + 4] - header_size = 12 + shared_count * 4 - if header_size > size: - raise Reject("EINTEGRITY", "ibody.shared-count") - shared = [self.u32(body + 12 + index * 4) for index in range(shared_count)] - return body, header_size, shared - - def entry(self, offset: int, limit: int, kind: str) -> dict[str, object]: - if offset > limit or 4 > limit - offset: - raise Reject("EINTEGRITY", f"{kind}.entry-header") - name_length = self.data[offset] - name_index = self.data[offset + 1] - value_length = self.u16(offset + 2) - name_end = offset + 4 + name_length - if name_end > limit: - raise Reject("EINTEGRITY", f"{kind}.name-bounds") - value_end = name_end + value_length - if value_end > limit: - raise Reject("EINTEGRITY", f"{kind}.value-bounds") - aligned_end = (value_end + 3) & ~3 - if aligned_end > limit: - raise Reject("EINTEGRITY", f"{kind}.padding-bounds") - name = self.data[offset + 4 : name_end] - if b"\0" in name: - raise Reject("EINTEGRITY", f"{kind}.name-nul") - return { - "offset": offset, - "next": aligned_end, - "name_index": name_index, - "name": name, - "value": self.data[name_end:value_end], - } - - def entries(self, inode: dict[str, int]) -> tuple[list[dict[str, object]], int]: - body, header_size, shared_ids = self.load_body(inode) - if body == 0: - return [], 0 - body_end = body + inode["xattr_size"] - cursor = body + header_size - entries = [] - while cursor < body_end: - item = self.entry(cursor, body_end, "inline") - entries.append(item) - cursor = int(item["next"]) - for shared_id in shared_ids: - offset = (self.xattr_blkaddr << self.block_bits) + shared_id * 4 - if offset > self.limit or 4 > self.limit - offset: - raise Reject("EINTEGRITY", "shared.offset-bounds") - entries.append(self.entry(offset, self.limit, "shared")) - return entries, len(shared_ids) - - def resolved_name( - self, entry: dict[str, object], prefixes: list[tuple[int, bytes]] - ) -> tuple[str, bytes]: - index = int(entry["name_index"]) - infix = b"" - from_prefix = False - if index & 0x80: - prefix_id = index & 0x7F - if prefix_id >= len(prefixes): - raise Reject("ENOATTR", "name.long-prefix-id") - index, infix = prefixes[prefix_id] - from_prefix = True - mapping = { - 1: ("user", b""), - 2: ("system", b"posix_acl_access"), - 3: ("system", b"posix_acl_default"), - 4: ("system", b"trusted."), - 6: ("system", b"security."), - } - if index not in mapping: - point = "name.prefix-base-index" if from_prefix else "name.short-index" - raise Reject("ENOATTR", point) - namespace, fixed = mapping[index] - return namespace, fixed + infix + bytes(entry["name"]) - - def getxattr(self, inode: dict[str, int], namespace: str, name: bytes, - prefixes: list[tuple[int, bytes]]) -> tuple[bytes, int]: - entries, shared_count = self.entries(inode) - deferred: Reject | None = None - for entry in entries: - try: - actual_namespace, actual_name = self.resolved_name(entry, prefixes) - except Reject as error: - deferred = error - continue - if actual_namespace == namespace and actual_name == name: - return bytes(entry["value"]), shared_count - if deferred is not None: - raise deferred - raise Reject("ENOATTR", "name.not-found") - - def listxattr(self, inode: dict[str, int], namespace: str, - prefixes: list[tuple[int, bytes]]) -> tuple[list[bytes], int]: - entries, shared_count = self.entries(inode) - names = [] - deferred: Reject | None = None - for entry in entries: - try: - actual_namespace, actual_name = self.resolved_name(entry, prefixes) - except Reject as error: - deferred = error - continue - if actual_namespace == namespace: - names.append(actual_name) - if deferred is not None and not names: - raise deferred - return names, shared_count - - -def parse_acl(value: bytes) -> list[list[int]]: - if len(value) < 4 or (len(value) - 4) % 8: - raise Reject("EINTEGRITY", "acl.value-size") - if struct.unpack_from(" dict[str, object]: - reader = Reader(image_path) - prefixes = reader.load_prefixes() - if "expected_feature_filter" in case: - actual_feature = bool(reader.feature_compat & FEATURE_COMPAT_XATTR_FILTER) - if actual_feature != case["expected_feature_filter"]: - raise AssertionError("raw xattr filter feature differs") - if reader.filter_reserved != case["expected_filter_reserved"]: - raise AssertionError("raw xattr filter reserved byte differs") - usable = actual_feature and reader.filter_reserved == 0 - if usable != case["expected_filter_usable"]: - raise AssertionError("xattr filter use-site gate differs") - operation = case["operation"] - result: dict[str, object] = {} - if operation == "mount": - return result - inode = reader.resolve(case["target"]) - if "expected_acl_filter_negative" in case: - body, _, _ = reader.load_body(inode) - actual_negative = ( - reader.u32(body) & ACL_FILTER_BITS - ) == ACL_FILTER_BITS - if actual_negative != case["expected_acl_filter_negative"]: - raise AssertionError("ACL name-filter declaration differs") - if operation == "get": - value, shared_count = reader.getxattr( - inode, case["namespace"], case["name"].encode("ascii"), prefixes - ) - result["value_hex"] = value.hex() - result["shared_count"] = shared_count - if "expected_value_hex" in case and value.hex() != case["expected_value_hex"]: - raise AssertionError("xattr value differs") - if "expected_shared_count" in case and shared_count != case["expected_shared_count"]: - raise AssertionError("xattr shared count differs") - elif operation == "list": - names, shared_count = reader.listxattr(inode, case["namespace"], prefixes) - result["names"] = [name.decode("ascii") for name in names] - result["shared_count"] = shared_count - elif operation == "acl": - value, _ = reader.getxattr( - inode, "system", b"posix_acl_access", prefixes - ) - acl = parse_acl(value) - result["acl"] = acl - if acl != case["expected_acl"]: - raise AssertionError("ACL empty-suffix value differs") - else: - raise AssertionError(f"unknown operation: {operation}") - return result - - -def fsck_accept(path: Path) -> dict[str, object]: - fsck = shutil.which("fsck.erofs") - if fsck is None: - raise SystemExit("fsck.erofs is required for legal B17 fixture baseline") - completed = subprocess.run( - [fsck, "-d0", "--xattrs", str(path)], check=False, text=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) - return {"exit": completed.returncode, "stdout": completed.stdout} - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--spec", type=Path, required=True) - parser.add_argument("--fixtures", type=Path, required=True) - parser.add_argument("--report", type=Path, required=True) - args = parser.parse_args() - spec = json.loads(args.spec.read_text(encoding="ascii")) - index = json.loads( - (args.fixtures / "fixture-index.json").read_text(encoding="ascii") - ) - indexed = {item["id"]: item for item in index["fixtures"]} - errnos = spec["freebsd_errno"] - results = [] - failures = [] - for case in spec["cases"]: - image_path = args.fixtures / indexed[case["id"]]["path"] - actual_errno = 0 - actual_errno_name = None - actual_reject = "accepted" - details: dict[str, object] = {} - try: - details = execute(case, image_path) - except Reject as error: - actual_errno_name = error.errno_name - actual_errno = errnos[error.errno_name] - actual_reject = error.point - except Exception as error: - failures.append({"id": case["id"], "reason": repr(error)}) - results.append({"id": case["id"], "status": "FAIL", "exception": repr(error)}) - continue - expected_name = case.get("expected_errno_name") - matched = ( - actual_errno == case["expected_errno"] - and actual_errno_name == expected_name - and actual_reject == case["expected_reject"] - ) - fsck = None - if case.get("fsck_accept"): - fsck = fsck_accept(image_path) - matched = matched and fsck["exit"] == 0 - result = { - "id": case["id"], - "class": case["class"], - "status": "PASS" if matched else "FAIL", - "expected_errno": case["expected_errno"], - "actual_errno": actual_errno, - "actual_errno_name": actual_errno_name, - "expected_reject": case["expected_reject"], - "actual_reject": actual_reject, - "details": details, - } - if fsck is not None: - result["fsck_exit"] = fsck["exit"] - result["fsck_stdout_sha256"] = hashlib.sha256( - fsck["stdout"].encode("utf-8") - ).hexdigest() - results.append(result) - if not matched: - failures.append(result) - report = { - "schema": 1, - "batch": "B17", - "status": "PASS" if not failures else "FAIL", - "fixture_count": len(results), - "legal_passed": sum( - item["status"] == "PASS" and item["class"] == "legal" - for item in results - ), - "damaged_passed": sum( - item["status"] == "PASS" and item["class"] == "damaged" - for item in results - ), - "failures": failures, - "results": results, - } - args.report.write_text( - json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - print( - json.dumps( - { - "status": report["status"], - "fixtures": report["fixture_count"], - "legal_passed": report["legal_passed"], - "damaged_passed": report["damaged_passed"], - }, - sort_keys=True, - ) - ) - return 0 if not failures else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/pre15/fixtures/B17-xattr-seed.erofs b/tests/pre15/fixtures/B17-xattr-seed.erofs deleted file mode 100644 index c352a2d85f7f7e9cfaf9b0150c49f86fe05394fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4096 zcmeH`&rZTX5XPrK`6GDNqZf`e0`XvCVl?qRT(E@3QV2!@H@x})Jb=-w58%6)D0kz@ zcrdZfc4utUE+m}UIh5bd?0&QR?E+xJIA9#uRtJ{vUvJmT#cK%-s6ibn8Y3)CPNnjD z>C$DKh?ZT%V=Wb}8r<~7*u7HU#zgqHZamdS{5S)6p;U;mYCpMn*+12Ze8}PHR1A3D$`_+H>j8janaH8xeU(|_Xj&+h{+>TDaxlW(& zGN~hZ1O3)I{c>8E#vACjV||GHuabuJpYI~wMyFo^t2-QwL@;yG)6Lj8U>q -#include -#include -#include -#include -#include -#include -#include -#include - -#define B19A_WORKERS 64 -#define B19A_BODY_SIZE 256 -#define B19A_ENTRY_LIMIT 65536 -#define B19A_MOUNT_BUDGET 1048576 -#define B19A_EINTEGRITY 97 - -enum cache_state { - CACHE_EMPTY, - CACHE_INFLIGHT, - CACHE_READY, - CACHE_FAILED, -}; - -enum cache_lookup { - CACHE_BYPASS, - CACHE_HIT, - CACHE_OWNER, - CACHE_ERROR, -}; - -struct mount_budget { - pthread_mutex_t lock; - size_t resident; -}; - -struct cache { - pthread_mutex_t lock; - pthread_cond_t cv; - struct mount_budget *budget; - unsigned char *body; - size_t body_size; - size_t charged; - unsigned int waiters; - int error; - enum cache_state state; - bool closing; -}; - -struct wave { - struct cache *cache; - pthread_barrier_t barrier; - unsigned char payload[B19A_BODY_SIZE]; - atomic_uint loads; - atomic_uint bypasses; - int expected_error; - int results[B19A_WORKERS]; -}; - -struct worker_arg { - struct wave *wave; - unsigned int index; -}; - -struct close_arg { - struct cache *cache; - atomic_bool done; -}; - -static void -fail(const char *message) -{ - fprintf(stderr, "B19a cache model failure: %s\n", message); - exit(1); -} - -static void -check_pthread(int error, const char *operation) -{ - if (error != 0) { - errno = error; - perror(operation); - exit(1); - } -} - -static void -budget_init(struct mount_budget *budget) -{ - memset(budget, 0, sizeof(*budget)); - check_pthread(pthread_mutex_init(&budget->lock, NULL), - "pthread_mutex_init budget"); -} - -static bool -budget_reserve(struct mount_budget *budget, size_t amount) -{ - bool reserved; - - check_pthread(pthread_mutex_lock(&budget->lock), - "pthread_mutex_lock budget"); - reserved = amount <= B19A_MOUNT_BUDGET - budget->resident; - if (reserved) - budget->resident += amount; - check_pthread(pthread_mutex_unlock(&budget->lock), - "pthread_mutex_unlock budget"); - return (reserved); -} - -static void -budget_release(struct mount_budget *budget, size_t amount) -{ - check_pthread(pthread_mutex_lock(&budget->lock), - "pthread_mutex_lock budget"); - if (amount > budget->resident) - fail("mount budget underflow"); - budget->resident -= amount; - check_pthread(pthread_mutex_unlock(&budget->lock), - "pthread_mutex_unlock budget"); -} - -static size_t -budget_resident(struct mount_budget *budget) -{ - size_t resident; - - check_pthread(pthread_mutex_lock(&budget->lock), - "pthread_mutex_lock budget"); - resident = budget->resident; - check_pthread(pthread_mutex_unlock(&budget->lock), - "pthread_mutex_unlock budget"); - return (resident); -} - -static void -budget_destroy(struct mount_budget *budget) -{ - if (budget_resident(budget) != 0) - fail("mount budget remained charged at destroy"); - check_pthread(pthread_mutex_destroy(&budget->lock), - "pthread_mutex_destroy budget"); -} - -static void -cache_init(struct cache *cache, struct mount_budget *budget) -{ - memset(cache, 0, sizeof(*cache)); - cache->budget = budget; - check_pthread(pthread_mutex_init(&cache->lock, NULL), - "pthread_mutex_init cache"); - check_pthread(pthread_cond_init(&cache->cv, NULL), - "pthread_cond_init cache"); -} - -static enum cache_lookup -cache_claim(struct cache *cache, size_t body_size, unsigned char *output, - int *errorp) -{ - enum cache_lookup result; - - *errorp = 0; - check_pthread(pthread_mutex_lock(&cache->lock), - "pthread_mutex_lock cache"); - if (cache->closing) { - *errorp = ENXIO; - goto bypass; - } - if (cache->state == CACHE_READY) { - if (cache->body == NULL || cache->body_size != body_size) - fail("READY cache body is inconsistent"); - memcpy(output, cache->body, body_size); - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock cache"); - return (CACHE_HIT); - } - if (cache->state == CACHE_INFLIGHT) { - ++cache->waiters; - do { - check_pthread(pthread_cond_wait(&cache->cv, &cache->lock), - "pthread_cond_wait cache"); - } while (cache->state == CACHE_INFLIGHT); - if (cache->state == CACHE_READY) { - if (cache->body == NULL || cache->body_size != body_size) - fail("published cache body is inconsistent"); - memcpy(output, cache->body, body_size); - result = CACHE_HIT; - } else { - if (cache->state != CACHE_FAILED || cache->error <= 0) - fail("waiter observed an untyped failure"); - *errorp = cache->error; - result = CACHE_ERROR; - } - --cache->waiters; - if (cache->state == CACHE_FAILED && cache->waiters == 0) { - cache->error = 0; - cache->state = CACHE_EMPTY; - } - if (cache->waiters == 0) - check_pthread(pthread_cond_broadcast(&cache->cv), - "pthread_cond_broadcast cache"); - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock cache"); - return (result); - } - if (cache->state == CACHE_FAILED) { - if (cache->waiters != 0) - goto bypass; - cache->error = 0; - cache->state = CACHE_EMPTY; - } - if (body_size > B19A_ENTRY_LIMIT || - !budget_reserve(cache->budget, body_size)) - goto bypass; - cache->body_size = body_size; - cache->charged = body_size; - cache->state = CACHE_INFLIGHT; - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock cache"); - return (CACHE_OWNER); - -bypass: - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock cache"); - return (CACHE_BYPASS); -} - -static void -cache_complete(struct cache *cache, unsigned char *body, size_t body_size, - int error) -{ - if ((error == 0) != (body != NULL)) - fail("owner completion is not typed"); - check_pthread(pthread_mutex_lock(&cache->lock), - "pthread_mutex_lock cache"); - if (cache->state != CACHE_INFLIGHT || cache->body_size != body_size) - fail("cache owner lost its inflight body"); - if (error == 0) { - cache->body = body; - cache->error = 0; - cache->state = CACHE_READY; - } else { - if (error < 1) - fail("owner published a non-positive errno"); - budget_release(cache->budget, cache->charged); - cache->charged = 0; - cache->body_size = 0; - cache->error = error; - cache->state = CACHE_FAILED; - if (cache->waiters == 0) { - cache->error = 0; - cache->state = CACHE_EMPTY; - } - } - check_pthread(pthread_cond_broadcast(&cache->cv), - "pthread_cond_broadcast cache"); - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock cache"); -} - -static unsigned char * -body_copy(const unsigned char *payload, size_t body_size) -{ - unsigned char *copy; - - copy = malloc(body_size); - if (copy == NULL) - fail("body allocation failed"); - memcpy(copy, payload, body_size); - return (copy); -} - -static void -wait_for_waiters(struct cache *cache, unsigned int expected) -{ - struct timespec delay = { .tv_sec = 0, .tv_nsec = 1000000 }; - unsigned int attempt, waiters; - - for (attempt = 0; attempt < 5000; ++attempt) { - check_pthread(pthread_mutex_lock(&cache->lock), - "pthread_mutex_lock cache"); - waiters = cache->waiters; - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock cache"); - if (waiters == expected) - return; - nanosleep(&delay, NULL); - } - fail("workers did not register as same-vnode waiters"); -} - -static void * -wave_worker(void *opaque) -{ - struct worker_arg *arg; - struct wave *wave; - unsigned char output[B19A_BODY_SIZE]; - enum cache_lookup lookup; - int barrier_error, error; - - arg = opaque; - wave = arg->wave; - barrier_error = pthread_barrier_wait(&wave->barrier); - if (barrier_error != 0 && barrier_error != PTHREAD_BARRIER_SERIAL_THREAD) - check_pthread(barrier_error, "pthread_barrier_wait"); - lookup = cache_claim(wave->cache, sizeof(output), output, &error); - if (lookup == CACHE_OWNER) { - atomic_fetch_add_explicit(&wave->loads, 1, memory_order_relaxed); - wait_for_waiters(wave->cache, B19A_WORKERS - 1); - if (wave->expected_error != 0) { - cache_complete(wave->cache, NULL, sizeof(output), - wave->expected_error); - wave->results[arg->index] = wave->expected_error; - } else { - memcpy(output, wave->payload, sizeof(output)); - cache_complete(wave->cache, - body_copy(wave->payload, sizeof(wave->payload)), - sizeof(output), 0); - wave->results[arg->index] = - memcmp(output, wave->payload, sizeof(output)) == 0 ? 0 : EIO; - } - } else if (lookup == CACHE_HIT) { - wave->results[arg->index] = - memcmp(output, wave->payload, sizeof(output)) == 0 ? 0 : EIO; - } else if (lookup == CACHE_ERROR) { - wave->results[arg->index] = error; - } else { - atomic_fetch_add_explicit(&wave->bypasses, 1, memory_order_relaxed); - wave->results[arg->index] = error != 0 ? error : EBUSY; - } - return (NULL); -} - -static void -run_wave(struct cache *cache, int expected_error) -{ - struct wave wave; - struct worker_arg args[B19A_WORKERS]; - pthread_t threads[B19A_WORKERS]; - unsigned int index; - - memset(&wave, 0, sizeof(wave)); - wave.cache = cache; - wave.expected_error = expected_error; - for (index = 0; index < sizeof(wave.payload); ++index) - wave.payload[index] = (unsigned char)(index ^ 0x5a); - check_pthread(pthread_barrier_init(&wave.barrier, NULL, B19A_WORKERS), - "pthread_barrier_init"); - for (index = 0; index < B19A_WORKERS; ++index) { - args[index].wave = &wave; - args[index].index = index; - check_pthread(pthread_create(&threads[index], NULL, wave_worker, - &args[index]), "pthread_create"); - } - for (index = 0; index < B19A_WORKERS; ++index) - check_pthread(pthread_join(threads[index], NULL), "pthread_join"); - check_pthread(pthread_barrier_destroy(&wave.barrier), - "pthread_barrier_destroy"); - if (atomic_load_explicit(&wave.loads, memory_order_relaxed) != 1) - fail("same-vnode wave did not have exactly one owner"); - if (atomic_load_explicit(&wave.bypasses, memory_order_relaxed) != 0) - fail("same-vnode wave bypassed the cache"); - for (index = 0; index < B19A_WORKERS; ++index) { - if (wave.results[index] != expected_error) - fail("same-vnode waiter observed a different result"); - } -} - -static void -cache_invalidate(struct cache *cache) -{ - unsigned char *body; - size_t charged; - - check_pthread(pthread_mutex_lock(&cache->lock), - "pthread_mutex_lock cache"); - while (cache->state == CACHE_INFLIGHT || cache->waiters != 0) - check_pthread(pthread_cond_wait(&cache->cv, &cache->lock), - "pthread_cond_wait cache"); - body = cache->body; - charged = cache->charged; - cache->body = NULL; - cache->body_size = 0; - cache->charged = 0; - cache->error = 0; - cache->state = CACHE_EMPTY; - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock cache"); - free(body); - if (charged != 0) - budget_release(cache->budget, charged); -} - -static void -cache_close(struct cache *cache) -{ - check_pthread(pthread_mutex_lock(&cache->lock), - "pthread_mutex_lock cache"); - cache->closing = true; - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock cache"); - cache_invalidate(cache); - check_pthread(pthread_cond_destroy(&cache->cv), - "pthread_cond_destroy cache"); - check_pthread(pthread_mutex_destroy(&cache->lock), - "pthread_mutex_destroy cache"); -} - -static void * -close_worker(void *opaque) -{ - struct close_arg *arg; - - arg = opaque; - cache_close(arg->cache); - atomic_store_explicit(&arg->done, true, memory_order_release); - return (NULL); -} - -static void -test_hits_and_failure(struct mount_budget *budget) -{ - struct cache cache; - unsigned char output[B19A_BODY_SIZE]; - enum cache_lookup lookup; - unsigned int index; - int error; - - cache_init(&cache, budget); - run_wave(&cache, 0); - if (budget_resident(budget) != B19A_BODY_SIZE) - fail("success wave did not charge one body"); - for (index = 0; index < 100; ++index) { - lookup = cache_claim(&cache, sizeof(output), output, &error); - if (lookup != CACHE_HIT || error != 0) - fail("ready cache did not hit"); - } - cache_invalidate(&cache); - if (budget_resident(budget) != 0 || cache.state != CACHE_EMPTY) - fail("cache invalidation did not release its body"); - run_wave(&cache, B19A_EINTEGRITY); - if (budget_resident(budget) != 0 || cache.state != CACHE_EMPTY) - fail("failed publication leaked body state"); - run_wave(&cache, 0); - cache_close(&cache); - if (budget_resident(budget) != 0) - fail("cache close did not release the retry body"); -} - -static void -test_limits(struct mount_budget *budget) -{ - struct cache caches[17]; - unsigned char *body, output[B19A_ENTRY_LIMIT + 1]; - enum cache_lookup lookup; - unsigned int index; - int error; - - cache_init(&caches[0], budget); - lookup = cache_claim(&caches[0], B19A_ENTRY_LIMIT + 1, output, &error); - if (lookup != CACHE_BYPASS || budget_resident(budget) != 0) - fail("oversized body did not bypass without charge"); - cache_close(&caches[0]); - - for (index = 0; index < 17; ++index) - cache_init(&caches[index], budget); - for (index = 0; index < 16; ++index) { - lookup = cache_claim(&caches[index], B19A_ENTRY_LIMIT, output, &error); - if (lookup != CACHE_OWNER) - fail("mount budget rejected an in-budget body"); - body = calloc(1, B19A_ENTRY_LIMIT); - if (body == NULL) - fail("budget body allocation failed"); - cache_complete(&caches[index], body, B19A_ENTRY_LIMIT, 0); - } - if (budget_resident(budget) != B19A_MOUNT_BUDGET) - fail("mount budget did not reach its exact hard limit"); - lookup = cache_claim(&caches[16], B19A_ENTRY_LIMIT, output, &error); - if (lookup != CACHE_BYPASS || budget_resident(budget) != B19A_MOUNT_BUDGET) - fail("mount budget exhaustion did not bypass"); - for (index = 0; index < 17; ++index) - cache_close(&caches[index]); - if (budget_resident(budget) != 0) - fail("mount budget did not drain after reclaim"); -} - -static void -test_close_inflight(struct mount_budget *budget) -{ - struct cache cache; - struct close_arg close_arg; - pthread_t closer; - unsigned char output[B19A_BODY_SIZE]; - enum cache_lookup lookup; - struct timespec delay = { .tv_sec = 0, .tv_nsec = 10000000 }; - int error; - - cache_init(&cache, budget); - lookup = cache_claim(&cache, sizeof(output), output, &error); - if (lookup != CACHE_OWNER) - fail("close test did not establish an inflight owner"); - memset(&close_arg, 0, sizeof(close_arg)); - close_arg.cache = &cache; - check_pthread(pthread_create(&closer, NULL, close_worker, &close_arg), - "pthread_create close"); - nanosleep(&delay, NULL); - if (atomic_load_explicit(&close_arg.done, memory_order_acquire)) - fail("cache close did not wait for inflight owner"); - cache_complete(&cache, body_copy(output, sizeof(output)), sizeof(output), 0); - check_pthread(pthread_join(closer, NULL), "pthread_join close"); - if (!atomic_load_explicit(&close_arg.done, memory_order_acquire) || - budget_resident(budget) != 0) - fail("cache close did not drain inflight completion"); -} - -int -main(void) -{ - struct mount_budget budget; - - budget_init(&budget); - test_hits_and_failure(&budget); - test_limits(&budget); - test_close_inflight(&budget); - budget_destroy(&budget); - printf("{\"body_limit\":%u,\"failure_publication\":\"PASS\"," - "\"inflight_close\":\"PASS\",\"mount_budget\":%u," - "\"owner_waiter\":\"PASS\",\"reclaim\":\"PASS\"," - "\"resident_after\":0,\"status\":\"PASS\"}\n", - B19A_ENTRY_LIMIT, B19A_MOUNT_BUDGET); - return (0); -} diff --git a/tests/pre15/fixtures/B19a-xattr-generate.py b/tests/pre15/fixtures/B19a-xattr-generate.py deleted file mode 100755 index e0464ed..0000000 --- a/tests/pre15/fixtures/B19a-xattr-generate.py +++ /dev/null @@ -1,418 +0,0 @@ -#!/usr/bin/env python3 -"""Generate deterministic real EROFS fixtures for B19a.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import shutil -import struct -import subprocess - - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -CRC32C_POLY = 0x82F63B78 - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -def deterministic_value(label: str, length: int) -> bytes: - seed = (label + "|").encode("ascii") - return (seed * (length // len(seed) + 1))[:length] - - -def load_spec(path: Path) -> dict: - spec = json.loads(path.read_text(encoding="ascii")) - if ( - spec.get("schema") != 1 - or spec.get("batch") != "B19a" - or spec.get("candidate") != "P15-022" - ): - raise SystemExit("invalid B19a fixture spec identity") - return spec - - -class Image: - def __init__(self, data: bytes | bytearray): - self.data = bytearray(data) - if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC: - raise ValueError("invalid EROFS image") - self.block_bits = self.data[SUPER + 12] - if self.block_bits < 9 or self.block_bits > 16: - raise ValueError("invalid EROFS block size") - self.block_size = 1 << self.block_bits - self.blocks = self.u32(SUPER + 36) - self.limit = self.blocks << self.block_bits - self.meta_blkaddr = self.u32(SUPER + 40) - self.xattr_blkaddr = self.u32(SUPER + 44) - self.root_nid = self.u16(SUPER + 14) - if self.limit > len(self.data): - raise ValueError("EROFS image is truncated") - - @classmethod - def load(cls, path: Path) -> "Image": - return cls(path.read_bytes()) - - def u16(self, offset: int) -> int: - if offset < 0 or offset + 2 > len(self.data): - raise ValueError("u16 read is out of bounds") - return struct.unpack_from(" int: - if offset < 0 or offset + 4 > len(self.data): - raise ValueError("u32 read is out of bounds") - return struct.unpack_from(" dict[str, int]: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - if offset > self.limit or 64 > self.limit - offset: - raise ValueError("inode is out of bounds") - inode_format = self.u16(offset) - inode_size = 64 if inode_format & 1 else 32 - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) - size = struct.unpack_from( - "> 1) & 7, - "size": size, - "start_block": self.u32(offset + 16), - } - - def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes: - if logical > inode["size"] or length > inode["size"] - logical: - raise ValueError("inode data range is out of bounds") - if inode["layout"] == 0: - physical = (inode["start_block"] << self.block_bits) + logical - elif inode["layout"] == 2: - tail_start = ((inode["size"] + self.block_size - 1) // self.block_size - 1) * self.block_size - if logical < tail_start: - if logical + length > tail_start: - raise ValueError("inode data read crosses inline tail") - physical = (inode["start_block"] << self.block_bits) + logical - else: - physical = ( - inode["offset"] - + inode["inode_size"] - + inode["xattr_size"] - + logical - - tail_start - ) - else: - raise ValueError("unsupported directory layout") - if physical > self.limit or length > self.limit - physical: - raise ValueError("inode data is outside the image") - return bytes(self.data[physical : physical + length]) - - @staticmethod - def parse_dirblock(data: bytes) -> list[tuple[bytes, int]]: - if len(data) < 12: - raise ValueError("directory block is short") - first_name = struct.unpack_from("= len(data) or first_name % 12: - raise ValueError("directory first name offset is invalid") - count = first_name // 12 - entries = [] - previous_offset = 0 - previous_name: bytes | None = None - for index in range(count): - entry = index * 12 - nid = struct.unpack_from("= len(data) - or (index == 0 and start != first_name) - or (index != 0 and start <= previous_offset) - or end <= start - or end > len(data) - ): - raise ValueError("directory name bounds are invalid") - span = data[start:end] - if index + 1 < count: - if b"\0" in span: - raise ValueError("non-trailing directory name contains NUL") - name = span - else: - name = span.split(b"\0", 1)[0] - if not name or len(name) > 255 or b"/" in name: - raise ValueError("directory name is invalid") - if previous_name is not None and previous_name >= name: - raise ValueError("directory names are not strictly ordered") - entries.append((name, nid)) - previous_offset = start - previous_name = name - return entries - - def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]: - result = [] - previous_name: bytes | None = None - logical = 0 - while logical < inode["size"]: - length = min(self.block_size, inode["size"] - logical) - block = self.parse_dirblock(self.inode_data(inode, logical, length)) - if previous_name is not None and previous_name >= block[0][0]: - raise ValueError("directory block boundary is not ordered") - result.extend(block) - previous_name = block[-1][0] - logical += length - return result - - def resolve(self, path: str) -> dict[str, int]: - inode = self.inode(self.root_nid) - for component in path.strip("/").encode("ascii").split(b"/"): - if not component: - continue - matches = [nid for name, nid in self.directory_entries(inode) if name == component] - if len(matches) != 1: - raise ValueError(f"fixture path is absent or ambiguous: {path}") - inode = self.inode(matches[0]) - return inode - - def body_offset(self, inode: dict[str, int]) -> int: - return inode["offset"] + inode["inode_size"] - - def body_shape(self, inode: dict[str, int]) -> tuple[int, list[int], list[int]]: - body = self.body_offset(inode) - size = inode["xattr_size"] - if size <= 12 or body > self.limit or size > self.limit - body: - raise ValueError("target xattr body is invalid") - shared_count = self.data[body + 4] - header_size = 12 + shared_count * 4 - if header_size > size: - raise ValueError("target shared count is invalid") - shared_ids = [self.u32(body + 12 + index * 4) for index in range(shared_count)] - inline_entries = [] - cursor = body + header_size - end = body + size - while cursor < end: - if cursor + 4 > end: - raise ValueError("target inline xattr header is truncated") - name_length = self.data[cursor] - value_length = self.u16(cursor + 2) - total = (4 + name_length + value_length + 3) & ~3 - if total > end - cursor: - raise ValueError("target inline xattr entry is truncated") - inline_entries.append(cursor) - cursor += total - return header_size, shared_ids, inline_entries - - def update_checksum(self) -> None: - if self.u32(SUPER + 8) & 1: - struct.pack_into(" None: - path.write_bytes(self.data) - - -def set_epoch(path: Path) -> None: - for entry in sorted(path.rglob("*"), reverse=True): - os.utime(entry, (0, 0), follow_symlinks=False) - os.utime(path, (0, 0), follow_symlinks=False) - - -def create_source(path: Path, spec: dict) -> None: - path.mkdir(parents=True) - path.chmod(0o755) - shared_value = deterministic_value( - "shared-value-p15-022", spec["fixture"]["shared_value_bytes"] - ) - inline_value = deterministic_value( - "inline-value-p15-022", spec["fixture"]["inline_value_bytes"] - ) - files = [] - for index in range(spec["fixture"]["peer_count"]): - prefix = f"peer-{index:03d}-" - suffix = "x" * (spec["fixture"]["peer_name_bytes"] - len(prefix) - 4) - files.append(path / f"{prefix}{suffix}.bin") - target = path / "target.bin" - files.append(target) - for index, entry in enumerate(files): - entry.write_bytes(f"P15-022 file {index:03d}\n".encode("ascii")) - entry.chmod(0o644) - os.setxattr(entry, b"user.shared", shared_value) - if entry == target: - os.setxattr(entry, b"user.inline", inline_value) - else: - os.setxattr( - entry, - b"user.peer", - deterministic_value(f"peer-{index:03d}", 31), - ) - - budget = path / "budget" - budget.mkdir() - budget.chmod(0o755) - for index in range(spec["fixture"]["budget_file_count"]): - entry = budget / f"budget-{index:03d}.bin" - entry.write_bytes(b"B19a budget\n") - entry.chmod(0o644) - value = bytearray( - deterministic_value( - f"budget-value-{index:03d}", - spec["fixture"]["budget_value_bytes"], - ) - ) - struct.pack_into(" list[str]: - mkfs = shutil.which("mkfs.erofs") - if mkfs is None: - raise SystemExit("mkfs.erofs is required") - command = [ - mkfs, - "-d0", - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "--sort=path", - f"-U{spec['fixture']['uuid']}", - "-x2", - "-Eforce-inode-extended", - str(output), - str(source), - ] - completed = subprocess.run( - command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - if completed.returncode != 0: - raise SystemExit( - "mkfs.erofs failed: " + completed.stdout.decode("utf-8", "replace") - ) - return command - - -def mutate(valid: Path, output: Path, mutation: str, spec: dict) -> None: - image = Image.load(valid) - target = image.resolve(spec["fixture"]["target"]) - body = image.body_offset(target) - _, shared_ids, inline_entries = image.body_shape(target) - if mutation == "corrupt-shared-count": - image.data[body + 4] = 255 - elif mutation == "corrupt-shared-id": - if not shared_ids: - raise SystemExit("valid fixture has no shared xattr ID") - struct.pack_into(" dict: - if output.exists() or work.exists(): - raise SystemExit("B19a output and work paths must be absent") - output.mkdir(parents=True) - source = work / "source" - create_source(source, spec) - valid = output / "valid.erofs" - command = build_image(source, valid, spec) - image = Image.load(valid) - root = image.inode(image.root_nid) - root_entries = image.directory_entries(root) - target = image.resolve(spec["fixture"]["target"]) - _, shared_ids, inline_entries = image.body_shape(target) - if root["size"] <= image.block_size: - raise SystemExit("B19a root directory did not span multiple blocks") - if not shared_ids or not inline_entries: - raise SystemExit("B19a target lacks both shared and inline xattrs") - target_positions = [ - index for index, (name, _) in enumerate(root_entries) if name == b"target.bin" - ] - if target_positions != [len(root_entries) - 1]: - raise SystemExit("B19a target is not uniquely sorted after all peers") - for mutation in ( - "corrupt-shared-count", - "corrupt-shared-id", - "corrupt-inline-name", - ): - mutate(valid, output / f"{mutation}.erofs", mutation, spec) - fsck = shutil.which("fsck.erofs") - if fsck is None: - raise SystemExit("fsck.erofs is required") - completed = subprocess.run( - [fsck, str(valid)], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - if completed.returncode != 0: - raise SystemExit( - "fsck.erofs rejected B19a valid fixture: " - + completed.stdout.decode("utf-8", "replace") - ) - hashes = {path.name: sha256(path) for path in sorted(output.glob("*.erofs"))} - frozen = spec.get("fixture_sha256", {}) - if frozen and hashes != frozen: - raise SystemExit(f"B19a fixture hashes differ: {hashes}") - report = { - "block_size": image.block_size, - "command": [*command[:-2], "OUTPUT/valid.erofs", "WORK/source"], - "fixture_sha256": hashes, - "inline_entries": len(inline_entries), - "root_directory_blocks": (root["size"] + image.block_size - 1) // image.block_size, - "root_entry_count": len(root_entries), - "shared_count": len(shared_ids), - "status": "PASS", - "target_entry_index": target_positions[0], - } - (output / "manifest.json").write_text( - json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - return report - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--spec", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--work", type=Path, required=True) - args = parser.parse_args() - spec = load_spec(args.spec) - try: - report = generate(args.output, args.work, spec) - finally: - shutil.rmtree(args.work, ignore_errors=True) - print(json.dumps(report, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/pre15/fixtures/B19a-xattr-oracle.py b/tests/pre15/fixtures/B19a-xattr-oracle.py deleted file mode 100755 index 763ae8b..0000000 --- a/tests/pre15/fixtures/B19a-xattr-oracle.py +++ /dev/null @@ -1,853 +0,0 @@ -#!/usr/bin/env python3 -"""Independent multi-block directory and xattr oracle for B19a.""" - -from __future__ import annotations - -import argparse -from concurrent.futures import ThreadPoolExecutor -import hashlib -import json -import os -from pathlib import Path -import statistics -import struct -import threading -import time - - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -CRC32C_POLY = 0x82F63B78 - - -class Reject(RuntimeError): - def __init__(self, errno_name: str, point: str): - super().__init__(f"{errno_name} at {point}") - self.errno_name = errno_name - self.point = point - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def crc32c(data: bytes, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -def load_spec(path: Path) -> dict: - spec = json.loads(path.read_text(encoding="ascii")) - if ( - spec.get("schema") != 1 - or spec.get("batch") != "B19a" - or spec.get("candidate") != "P15-022" - ): - raise SystemExit("invalid B19a oracle spec identity") - return spec - - -class ReadStats: - def __init__(self) -> None: - self.lock = threading.Lock() - self.calls = 0 - self.bytes = 0 - self.block_reads = 0 - - def add(self, offset: int, length: int, block_size: int) -> None: - with self.lock: - self.calls += 1 - self.bytes += length - if length: - self.block_reads += ( - (offset + length - 1) // block_size - offset // block_size + 1 - ) - - def reset(self) -> None: - with self.lock: - self.calls = 0 - self.bytes = 0 - self.block_reads = 0 - - def snapshot(self) -> dict[str, int]: - with self.lock: - return { - "bytes": self.bytes, - "calls": self.calls, - "provider_block_reads": self.block_reads, - } - - -class Reader: - def __init__(self, path: Path): - self.path = path - self.fd = os.open(path, os.O_RDONLY) - self.size = os.fstat(self.fd).st_size - self.stats = ReadStats() - header = self.read(SUPER, 144, "super") - if struct.unpack_from(" 16: - raise Reject("EINTEGRITY", "super.block-size") - self.block_size = 1 << self.block_bits - self.root_nid = struct.unpack_from(" self.size: - raise Reject("EINTEGRITY", "super.bounds") - if self.feature_compat & 1: - expected = struct.unpack_from(" None: - os.close(self.fd) - - def __enter__(self) -> "Reader": - return self - - def __exit__(self, *_: object) -> None: - self.close() - - def read(self, offset: int, length: int, point: str) -> bytes: - limit = self.limit if hasattr(self, "limit") else self.size - if offset < 0 or length < 0 or offset > limit or length > limit - offset: - raise Reject("EINTEGRITY", f"{point}.bounds") - data = os.pread(self.fd, length, offset) - if len(data) != length: - raise Reject("EINTEGRITY", f"{point}.short") - block_size = self.block_size if hasattr(self, "block_size") else 4096 - self.stats.add(offset, length, block_size) - return data - - def inode(self, nid: int) -> dict[str, int]: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - raw = self.read(offset, 64, "inode") - inode_format, xattr_count = struct.unpack_from("> 1) & 7, - "size": size, - "start_block": struct.unpack_from(" bytes: - if logical > inode["size"] or length > inode["size"] - logical: - raise Reject("EINTEGRITY", "inode.data-range") - if inode["layout"] == 0: - physical = (inode["start_block"] << self.block_bits) + logical - elif inode["layout"] == 2: - tail_start = ((inode["size"] + self.block_size - 1) // self.block_size - 1) * self.block_size - if logical < tail_start: - if logical + length > tail_start: - raise Reject("EINTEGRITY", "inode.inline-crossing") - physical = (inode["start_block"] << self.block_bits) + logical - else: - physical = ( - inode["offset"] - + inode["inode_size"] - + inode["xattr_size"] - + logical - - tail_start - ) - else: - raise Reject("EOPNOTSUPP", "inode.layout") - return self.read(physical, length, "inode.data") - - @staticmethod - def parse_dirblock(data: bytes, block_index: int) -> list[dict[str, object]]: - if len(data) < 12: - raise Reject("EINTEGRITY", "directory.header") - first_name = struct.unpack_from("= len(data) or first_name % 12: - raise Reject("EINTEGRITY", "directory.name-offset") - count = first_name // 12 - entries = [] - previous_offset = 0 - previous_name: bytes | None = None - for index in range(count): - offset = index * 12 - nid, start, file_type = struct.unpack_from("= len(data) - or (index == 0 and start != first_name) - or (index != 0 and start <= previous_offset) - or end <= start - or end > len(data) - ): - raise Reject("EINTEGRITY", "directory.name-bounds") - span = data[start:end] - if index + 1 < count: - if b"\0" in span: - raise Reject("EINTEGRITY", "directory.name-nul") - name = span - else: - name = span.split(b"\0", 1)[0] - if not name or len(name) > 255 or b"/" in name: - raise Reject("EINTEGRITY", "directory.name") - if previous_name is not None and previous_name >= name: - raise Reject("EINTEGRITY", "directory.order") - entries.append( - { - "block": block_index, - "entry": index, - "file_type": file_type, - "name": name, - "nid": nid, - } - ) - previous_offset = start - previous_name = name - return entries - - @staticmethod - def validate_boundary( - left: list[dict[str, object]], right: list[dict[str, object]] - ) -> None: - if left[-1]["name"] >= right[0]["name"]: - raise Reject("EINTEGRITY", "directory.boundary-order") - - @staticmethod - def resolve_entries(entries: list[dict[str, object]], name: bytes) -> int: - matches = [int(entry["nid"]) for entry in entries if entry["name"] == name] - if not matches: - raise Reject("ENOENT", "path.missing") - if len(matches) != 1: - raise Reject("EINTEGRITY", "path.duplicate") - return matches[0] - - def directory_entries(self, inode: dict[str, int]) -> list[dict[str, object]]: - entries = [] - previous_name: bytes | None = None - logical = 0 - block_index = 0 - while logical < inode["size"]: - length = min(self.block_size, inode["size"] - logical) - block_entries = self.parse_dirblock( - self.inode_data(inode, logical, length), block_index - ) - if entries: - self.validate_boundary(entries, block_entries) - entries.extend(block_entries) - previous_name = block_entries[-1]["name"] - logical += length - block_index += 1 - return entries - - def resolve(self, path: str) -> dict[str, int]: - inode = self.inode(self.root_nid) - for component in path.strip("/").encode("ascii").split(b"/"): - if not component: - continue - inode = self.inode( - self.resolve_entries(self.directory_entries(inode), component) - ) - return inode - - def body_offset(self, inode: dict[str, int]) -> int: - return inode["offset"] + inode["inode_size"] - - @staticmethod - def parse_entry(raw: bytes, offset: int, limit: int, point: str) -> tuple[dict, int]: - if offset > limit or 4 > limit - offset: - raise Reject("EINTEGRITY", f"{point}.header") - name_length, name_index, value_length = struct.unpack_from(" limit - offset: - raise Reject("EINTEGRITY", f"{point}.bounds") - name_start = offset + 4 - name_end = name_start + name_length - value_end = name_end + value_length - name = raw[name_start:name_end] - if b"\0" in name: - raise Reject("EINTEGRITY", f"{point}.name-nul") - return ( - { - "index": name_index, - "name": name, - "value": raw[name_end:value_end], - }, - total, - ) - - def shared_entry(self, shared_id: int) -> dict: - offset = (self.xattr_blkaddr << self.block_bits) + shared_id * 4 - header = self.read(offset, 4, "shared.header") - name_length, _, value_length = struct.unpack(" dict: - size = inode["xattr_size"] - if size < 12: - raise Reject("EINTEGRITY", "ibody.header") - if size == 12: - raise Reject("EOPNOTSUPP", "ibody.header-only") - raw = self.read(self.body_offset(inode), size, "ibody") - shared_count = raw[4] - header_size = 12 + shared_count * 4 - if header_size > size: - raise Reject("EINTEGRITY", "ibody.shared-count") - shared_ids = [ - struct.unpack_from(" bool: - with self.lock: - if amount > self.limit - self.resident: - return False - self.resident += amount - return True - - def release(self, amount: int) -> None: - with self.lock: - if amount > self.resident: - raise RuntimeError("cache budget underflow") - self.resident -= amount - - -class BodyCache: - EMPTY = "EMPTY" - INFLIGHT = "INFLIGHT" - READY = "READY" - FAILED = "FAILED" - - def __init__(self, spec: dict, budget: MountBudget): - self.limit = spec["model"]["entry_body_limit_bytes"] - self.budget = budget - self.cv = threading.Condition() - self.state = self.EMPTY - self.body: dict | None = None - self.error: Reject | None = None - self.waiters = 0 - self.loads = 0 - self.charged = 0 - self.closing = False - self.wait_for_waiters = 0 - - def acquire(self, reader: Reader, inode: dict[str, int]) -> dict: - owner = False - reserved = False - with self.cv: - if self.closing: - raise Reject("ENXIO", "cache.closing") - if self.state == self.READY: - if self.body is None: - raise RuntimeError("READY cache has no body") - return self.body - if self.state == self.INFLIGHT: - self.waiters += 1 - try: - while self.state == self.INFLIGHT: - self.cv.wait() - if self.state == self.READY: - if self.body is None: - raise RuntimeError("published cache has no body") - return self.body - if self.state != self.FAILED or self.error is None: - raise RuntimeError("cache waiter has no typed result") - raise Reject(self.error.errno_name, self.error.point) - finally: - self.waiters -= 1 - if self.state == self.FAILED and self.waiters == 0: - self.error = None - self.state = self.EMPTY - if self.waiters == 0: - self.cv.notify_all() - if self.state == self.FAILED: - if self.waiters: - return reader.load_body(inode, validate_shared=True) - self.error = None - self.state = self.EMPTY - if inode["xattr_size"] <= self.limit: - reserved = self.budget.reserve(inode["xattr_size"]) - if not reserved: - return reader.load_body(inode, validate_shared=True) - self.state = self.INFLIGHT - self.charged = inode["xattr_size"] - owner = True - if not owner: - raise RuntimeError("cache claim lost owner state") - try: - if self.wait_for_waiters: - deadline = time.monotonic() + 5 - while True: - with self.cv: - if self.waiters >= self.wait_for_waiters: - break - if time.monotonic() >= deadline: - raise RuntimeError("cache waiters did not reach owner barrier") - time.sleep(0.001) - body = reader.load_body(inode, validate_shared=True) - with self.cv: - self.loads += 1 - self.body = body - self.error = None - self.state = self.READY - self.cv.notify_all() - return body - except Reject as error: - with self.cv: - self.loads += 1 - self.budget.release(self.charged) - self.charged = 0 - self.error = error - self.state = self.FAILED - if self.waiters == 0: - self.error = None - self.state = self.EMPTY - self.cv.notify_all() - raise - - def invalidate(self) -> None: - with self.cv: - while self.state == self.INFLIGHT or self.waiters: - self.cv.wait() - if self.state == self.READY: - self.body = None - self.budget.release(self.charged) - self.charged = 0 - self.error = None - self.state = self.EMPTY - - def close(self) -> None: - with self.cv: - self.closing = True - while self.state == self.INFLIGHT or self.waiters: - self.cv.wait() - if self.state == self.READY: - self.body = None - self.budget.release(self.charged) - self.charged = 0 - self.error = None - self.state = self.EMPTY - - -def body_for_operation( - reader: Reader, inode: dict[str, int], variant: str, cache: BodyCache | None -) -> dict: - if variant == "baseline": - return reader.load_body(inode, validate_shared=False) - if variant != "candidate" or cache is None: - raise RuntimeError("invalid B19a oracle variant") - return cache.acquire(reader, inode) - - -def entries_for_body(reader: Reader, body: dict) -> list[dict]: - entries = list(body["inline"]) - entries.extend(reader.shared_entry(shared_id) for shared_id in body["shared_ids"]) - return entries - - -def getxattr( - reader: Reader, - inode: dict[str, int], - name: bytes, - variant: str, - cache: BodyCache | None, -) -> bytes: - body = body_for_operation(reader, inode, variant, cache) - for entry in body["inline"]: - if entry["index"] == 1 and entry["name"] == name: - return bytes(entry["value"]) - for shared_id in body["shared_ids"]: - entry = reader.shared_entry(shared_id) - if entry["index"] == 1 and entry["name"] == name: - return bytes(entry["value"]) - raise Reject("ENOATTR", "xattr.not-found") - - -def listxattr( - reader: Reader, - inode: dict[str, int], - variant: str, - cache: BodyCache | None, -) -> list[bytes]: - return [ - bytes(entry["name"]) - for entry in entries_for_body( - reader, body_for_operation(reader, inode, variant, cache) - ) - if entry["index"] == 1 - ] - - -def expected_value(label: str, length: int) -> bytes: - seed = (label + "|").encode("ascii") - return (seed * (length // len(seed) + 1))[:length] - - -def encode_dirblock(names: list[bytes], size: int = 128) -> bytes: - first_name = len(names) * 12 - payload = bytearray(size) - cursor = first_name - for index, name in enumerate(names): - if cursor + len(name) > size: - raise ValueError("synthetic directory block is too small") - struct.pack_into(" dict[str, str]: - try: - function() - except Reject as error: - if error.errno_name != errno_name or error.point != point: - raise SystemExit( - f"oracle control expected {errno_name} at {point}, got {error}" - ) - return {"errno": error.errno_name, "point": error.point} - raise SystemExit(f"oracle control unexpectedly passed: {errno_name} at {point}") - - -def run_selftest(output: Path) -> dict: - controls = {} - first = Reader.parse_dirblock( - encode_dirblock([b"alpha", b"boundary-last"], 96), - 0, - ) - second = Reader.parse_dirblock( - encode_dirblock([b"target", b"zeta"], 80), 1 - ) - Reader.validate_boundary(first, second) - combined = first + second - target_nid = Reader.resolve_entries(combined, b"target") - matches = [entry for entry in combined if entry["name"] == b"target"] - if ( - len(matches) != 1 - or matches[0]["block"] != 1 - or int(matches[0]["nid"]) != target_nid - ): - raise SystemExit("target was not resolved after a block boundary") - controls["target-after-boundary"] = "PASS" - controls["short-final-block"] = "PASS" - - controls["duplicate-within-block"] = expect_reject( - lambda: Reader.parse_dirblock(encode_dirblock([b"dup", b"dup"]), 0), - "EINTEGRITY", - "directory.order", - ) - left = Reader.parse_dirblock(encode_dirblock([b"alpha", b"dup"]), 0) - right = Reader.parse_dirblock(encode_dirblock([b"dup", b"zeta"]), 1) - controls["duplicate-across-blocks"] = expect_reject( - lambda: Reader.validate_boundary(left, right), - "EINTEGRITY", - "directory.boundary-order", - ) - controls["missing-target"] = expect_reject( - lambda: Reader.resolve_entries(combined, b"missing"), - "ENOENT", - "path.missing", - ) - duplicate_entries = combined + [{**combined[-1], "nid": 999, "name": b"target"}] - controls["duplicate-resolution"] = expect_reject( - lambda: Reader.resolve_entries(duplicate_entries, b"target"), - "EINTEGRITY", - "path.duplicate", - ) - backwards = Reader.parse_dirblock(encode_dirblock([b"aardvark", b"beta"]), 1) - controls["backwards-boundary"] = expect_reject( - lambda: Reader.validate_boundary(first, backwards), - "EINTEGRITY", - "directory.boundary-order", - ) - malformed = bytearray(encode_dirblock([b"alpha", b"bravo"])) - struct.pack_into(" dict[str, object]: - budget = MountBudget(spec["model"]["cache_budget_bytes"]) - cache = BodyCache(spec, budget) if variant == "candidate" else None - try: - with Reader(image) as reader: - inode = reader.resolve(spec["fixture"]["target"]) - reader.stats.reset() - if operation == "get-inline": - value = getxattr(reader, inode, b"inline", variant, cache) - return {"status": "PASS", "value_sha256": hashlib.sha256(value).hexdigest()} - if operation == "get-shared": - value = getxattr(reader, inode, b"shared", variant, cache) - return {"status": "PASS", "value_sha256": hashlib.sha256(value).hexdigest()} - if operation == "list-user": - names = listxattr(reader, inode, variant, cache) - return {"names": [name.decode("ascii") for name in names], "status": "PASS"} - raise RuntimeError(f"unknown operation: {operation}") - except Reject as error: - return {"errno": spec["errno"].get(error.errno_name, -1), "point": error.point, "status": error.errno_name} - finally: - if cache is not None: - cache.close() - if budget.resident != 0: - raise RuntimeError("operation cache budget leaked") - - -def run_correctness(fixtures: Path, spec: dict, output: Path) -> dict: - valid = fixtures / "valid.erofs" - with Reader(valid) as reader: - root = reader.inode(reader.root_nid) - root_entries = reader.directory_entries(root) - target_entries = [entry for entry in root_entries if entry["name"] == b"target.bin"] - if len(target_entries) != 1 or int(target_entries[0]["block"]) == 0: - raise SystemExit("real target did not resolve after the first block") - if [entry["name"] for entry in root_entries] != sorted( - entry["name"] for entry in root_entries - ): - raise SystemExit("real directory order is unstable") - boundary = [] - for left, right in zip(root_entries, root_entries[1:]): - if left["block"] != right["block"]: - boundary.append( - { - "left": bytes(left["name"]).decode("ascii"), - "right": bytes(right["name"]).decode("ascii"), - } - ) - inode = reader.resolve(spec["fixture"]["target"]) - body = reader.load_body(inode, validate_shared=True) - if not body["inline"] or not body["shared_ids"]: - raise SystemExit("real target lacks inline/shared xattr coverage") - directory = { - "block_count": (root["size"] + reader.block_size - 1) // reader.block_size, - "boundaries": boundary, - "entry_count": len(root_entries), - "target_block": target_entries[0]["block"], - "target_entry": target_entries[0]["entry"], - } - - expected_inline = expected_value( - "inline-value-p15-022", spec["fixture"]["inline_value_bytes"] - ) - expected_shared = expected_value( - "shared-value-p15-022", spec["fixture"]["shared_value_bytes"] - ) - cases = [] - for variant in ("baseline", "candidate"): - inline = operation_result(valid, spec, variant, "get-inline") - shared = operation_result(valid, spec, variant, "get-shared") - listed = operation_result(valid, spec, variant, "list-user") - if inline.get("value_sha256") != hashlib.sha256(expected_inline).hexdigest(): - raise SystemExit(f"{variant} inline value mismatch") - if shared.get("value_sha256") != hashlib.sha256(expected_shared).hexdigest(): - raise SystemExit(f"{variant} shared value mismatch") - if sorted(listed.get("names", [])) != ["inline", "shared"]: - raise SystemExit(f"{variant} list result mismatch: {listed}") - cases.extend( - [ - {"id": "valid-inline", "variant": variant, "result": inline}, - {"id": "valid-shared", "variant": variant, "result": shared}, - {"id": "valid-list", "variant": variant, "result": listed}, - ] - ) - - damaged = [ - ("corrupt-shared-count.erofs", "get-inline", "EINTEGRITY"), - ("corrupt-shared-id.erofs", "get-shared", "EINTEGRITY"), - ("corrupt-inline-name.erofs", "list-user", "EINTEGRITY"), - ] - for filename, operation, expected in damaged: - for variant in ("baseline", "candidate"): - result = operation_result(fixtures / filename, spec, variant, operation) - if result["status"] != expected or int(result.get("errno", -1)) < 0: - raise SystemExit(f"{filename} {variant} mismatch: {result}") - cases.append({"id": filename, "variant": variant, "result": result}) - - budget = MountBudget(spec["model"]["cache_budget_bytes"]) - cache = BodyCache(spec, budget) - cache.wait_for_waiters = spec["concurrency"]["workers"] - 1 - with Reader(valid) as reader: - inode = reader.resolve(spec["fixture"]["target"]) - barrier = threading.Barrier(spec["concurrency"]["workers"]) - - def worker(_: int) -> str: - barrier.wait() - value = getxattr(reader, inode, b"inline", "candidate", cache) - return hashlib.sha256(value).hexdigest() - - with ThreadPoolExecutor(max_workers=spec["concurrency"]["workers"]) as executor: - hashes = list(executor.map(worker, range(spec["concurrency"]["workers"]))) - if len(set(hashes)) != 1 or cache.loads != 1 or cache.state != BodyCache.READY: - raise SystemExit("candidate owner/waiter publication mismatch") - first_charge = budget.resident - cache.invalidate() - if budget.resident != 0 or cache.state != BodyCache.EMPTY: - raise SystemExit("candidate invalidation did not release cache") - cache.wait_for_waiters = 0 - getxattr(reader, inode, b"inline", "candidate", cache) - if cache.loads != 2 or budget.resident != first_charge: - raise SystemExit("candidate re-establishment mismatch") - cache.close() - if budget.resident != 0: - raise SystemExit("candidate reclaim/close leaked budget") - - report = { - "cases": cases, - "concurrency": { - "body_loads": 1, - "completed": spec["concurrency"]["workers"], - "workers": spec["concurrency"]["workers"], - }, - "directory": directory, - "fixture_sha256": { - path.name: sha256(path) for path in sorted(fixtures.glob("*.erofs")) - }, - "lifecycle": { - "establish": "PASS", - "hit": "PASS", - "invalidate": "PASS", - "reclaim": "PASS", - "resident_after": budget.resident, - }, - "status": "PASS", - } - output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") - return report - - -def run_sample( - image: Path, spec: dict, variant: str, sample: int, output: Path -) -> dict: - budget = MountBudget(spec["model"]["cache_budget_bytes"]) - cache = BodyCache(spec, budget) if variant == "candidate" else None - operations = spec["benchmark"]["operations"] - with Reader(image) as reader: - inode = reader.resolve(spec["fixture"]["target"]) - - def one_loop() -> None: - for operation in operations: - if operation == "get-inline": - getxattr(reader, inode, b"inline", variant, cache) - elif operation == "get-shared": - getxattr(reader, inode, b"shared", variant, cache) - elif operation == "list-user": - listxattr(reader, inode, variant, cache) - else: - raise RuntimeError(f"unknown benchmark operation: {operation}") - - for _ in range(spec["benchmark"]["warmup_loops"]): - one_loop() - reader.stats.reset() - started = time.monotonic_ns() - for _ in range(spec["benchmark"]["loops_per_sample"]): - one_loop() - elapsed = time.monotonic_ns() - started - reads = reader.stats.snapshot() - loads = cache.loads if cache is not None else 0 - resident_before_close = budget.resident - if cache is not None: - cache.close() - if budget.resident != 0: - raise SystemExit("benchmark cache budget leaked") - report = { - "cache_body_loads": loads, - "cache_resident_before_close": resident_before_close, - "elapsed_ns": elapsed, - "fixture_sha256": sha256(image), - "host": os.uname().sysname + " " + os.uname().release, - "loops": spec["benchmark"]["loops_per_sample"], - "operations": operations, - "reads": reads, - "sample": sample, - "status": "PASS", - "variant": variant, - "warmup_loops": spec["benchmark"]["warmup_loops"], - } - output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") - print(json.dumps(report, sort_keys=True)) - return report - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--spec", type=Path, required=True) - subparsers = parser.add_subparsers(dest="command", required=True) - selftest = subparsers.add_parser("selftest") - selftest.add_argument("--output", type=Path, required=True) - correctness = subparsers.add_parser("correctness") - correctness.add_argument("--fixtures", type=Path, required=True) - correctness.add_argument("--output", type=Path, required=True) - sample = subparsers.add_parser("sample") - sample.add_argument("--image", type=Path, required=True) - sample.add_argument("--variant", choices=("baseline", "candidate"), required=True) - sample.add_argument("--sample", type=int, required=True) - sample.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - spec = load_spec(args.spec) - if args.command == "selftest": - report = run_selftest(args.output) - print(json.dumps(report, sort_keys=True)) - elif args.command == "correctness": - report = run_correctness(args.fixtures, spec, args.output) - print(json.dumps(report, sort_keys=True)) - else: - run_sample(args.image, spec, args.variant, args.sample, args.output) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/pre15/fixtures/B19a-xattr-probe.c b/tests/pre15/fixtures/B19a-xattr-probe.c deleted file mode 100644 index eafab5f..0000000 --- a/tests/pre15/fixtures/B19a-xattr-probe.c +++ /dev/null @@ -1,189 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -static int -fail(const char *message) -{ - fprintf(stderr, "%s\n", message); - return (1); -} - -static int -get_repeat(const char *path, const char *name, size_t expected_size) -{ - uint8_t first[1024], second[1024]; - ssize_t first_size, second_size; - - if (expected_size > sizeof(first)) - return (fail("xattr size exceeds probe buffer")); - memset(first, 0xa5, sizeof(first)); - memset(second, 0x5a, sizeof(second)); - errno = 0; - first_size = extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, - first, sizeof(first)); - if (first_size != (ssize_t)expected_size || errno != 0) - return (fail("first xattr read returned the wrong size or errno")); - errno = 0; - second_size = extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, - second, sizeof(second)); - if (second_size != first_size || errno != 0 || - memcmp(first, second, expected_size) != 0) - return (fail("repeated xattr read changed its result")); - for (size_t offset = expected_size; offset < sizeof(first); offset++) { - if (first[offset] != 0xa5 || second[offset] != 0x5a) - return (fail("xattr read wrote beyond its value")); - } - return (0); -} - -static int -get_errno(const char *path, const char *name, int expected_errno) -{ - uint8_t value[32]; - ssize_t size; - - memset(value, 0xa5, sizeof(value)); - errno = 0; - size = extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, value, - sizeof(value)); - if (size != -1 || errno != expected_errno) { - fprintf(stderr, "xattr errno mismatch: size=%zd errno=%d expected=%d\n", - size, errno, expected_errno); - return (fail("xattr returned the wrong positive errno")); - } - for (size_t offset = 0; offset < sizeof(value); offset++) { - if (value[offset] != 0xa5) - return (fail("failed xattr lookup modified the output")); - } - return (0); -} - -static int -list_user(const char *path) -{ - uint8_t *buffer; - ssize_t size; - size_t offset; - unsigned int seen; - - errno = 0; - size = extattr_list_file(path, EXTATTR_NAMESPACE_USER, NULL, 0); - if (size <= 0 || errno != 0) - return (fail("xattr list sizing failed")); - buffer = malloc((size_t)size); - if (buffer == NULL) - return (fail("xattr list allocation failed")); - if (extattr_list_file(path, EXTATTR_NAMESPACE_USER, buffer, - (size_t)size) != size) { - free(buffer); - return (fail("xattr list transfer failed")); - } - seen = 0; - for (offset = 0; offset < (size_t)size;) { - const char *name; - uint8_t name_size; - - name_size = buffer[offset++]; - if (name_size == 0 || name_size > (size_t)size - offset) { - free(buffer); - return (fail("xattr list record is malformed")); - } - name = (const char *)(buffer + offset); - if (name_size == sizeof("inline") - 1 && - memcmp(name, "inline", name_size) == 0) { - if ((seen & 1U) != 0) { - free(buffer); - return (fail("xattr list duplicated inline")); - } - seen |= 1U; - } else if (name_size == sizeof("shared") - 1 && - memcmp(name, "shared", name_size) == 0) { - if ((seen & 2U) != 0) { - free(buffer); - return (fail("xattr list duplicated shared")); - } - seen |= 2U; - } else { - free(buffer); - return (fail("xattr list returned an unexpected name")); - } - offset += name_size; - } - free(buffer); - return (seen == 3U ? 0 : fail("xattr list omitted a name")); -} - -static int -valid(const char *path) -{ - if (get_repeat(path, "inline", 127) != 0 || - get_repeat(path, "shared", 511) != 0 || - list_user(path) != 0 || - get_errno(path, "missing", ENOATTR) != 0) - return (1); - return (0); -} - -static int -corrupt(const char *path) -{ - return (get_errno(path, "inline", EINTEGRITY)); -} - -static int -concurrent(const char *path, long workers, long loops) -{ - int status; - pid_t child; - - for (long worker = 0; worker < workers; worker++) { - child = fork(); - if (child < 0) - return (fail("xattr worker fork failed")); - if (child == 0) { - for (long iteration = 0; iteration < loops; iteration++) { - if (valid(path) != 0) - _exit(1); - } - _exit(0); - } - } - for (long worker = 0; worker < workers; worker++) { - if (wait(&status) < 0 || !WIFEXITED(status) || - WEXITSTATUS(status) != 0) - return (fail("xattr worker failed")); - } - return (0); -} - -int -main(int argc, char **argv) -{ - char *end; - long workers, loops; - - if (argc == 3 && strcmp(argv[1], "valid") == 0) - return (valid(argv[2])); - if (argc == 3 && strcmp(argv[1], "corrupt") == 0) - return (corrupt(argv[2])); - if (argc == 5 && strcmp(argv[1], "concurrent") == 0) { - errno = 0; - workers = strtol(argv[3], &end, 10); - if (errno != 0 || end[0] != '\0' || workers < 1 || workers > 64) - return (fail("invalid worker count")); - errno = 0; - loops = strtol(argv[4], &end, 10); - if (errno != 0 || end[0] != '\0' || loops < 1 || loops > 100) - return (fail("invalid loop count")); - return (concurrent(argv[2], workers, loops)); - } - return (fail("usage: B19a-xattr-probe valid|corrupt|concurrent ...")); -} diff --git a/tests/pre15/fixtures/B19a-xattr-spec.json b/tests/pre15/fixtures/B19a-xattr-spec.json deleted file mode 100644 index 2a55aef..0000000 --- a/tests/pre15/fixtures/B19a-xattr-spec.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "batch": "B19a", - "benchmark": { - "loops_per_sample": 200, - "operations": [ - "get-inline", - "get-shared", - "list-user" - ], - "sample_timeout_seconds": 30, - "samples": 5, - "warmup_loops": 20 - }, - "candidate": "P15-022", - "concurrency": { - "loops_per_worker": 10, - "workers": 64 - }, - "errno": { - "EINTEGRITY": 97, - "ENOATTR": 87, - "ENOENT": 2, - "ENOMEM": 12, - "ENXIO": 6 - }, - "fixture": { - "budget_file_count": 400, - "budget_value_bytes": 3000, - "inline_name": "inline", - "inline_value_bytes": 127, - "peer_count": 64, - "peer_name_bytes": 72, - "shared_name": "shared", - "shared_value_bytes": 511, - "target": "/target.bin", - "uuid": "00000000-0000-0000-0000-000000000022" - }, - "fixture_sha256": { - "corrupt-inline-name.erofs": "6b41a7ea8beb359c8ddf36bc1e700915fd594666a9d2a814df847fbae9f854f5", - "corrupt-shared-count.erofs": "ce3f29a56201e9b0f3bfe7720fa9cd8293f7eee2169c02f8c58c2ccf7e984cd9", - "corrupt-shared-id.erofs": "09ed7ea43f304d9ed9e38a29e0b7b66b6ef063a1ae3906a3b86a455af19985af", - "valid.erofs": "1970bf1625077be62f1dff08bd5303010e1da5c8a820da1317bfdd645812d5c6" - }, - "model": { - "cache_budget_bytes": 1048576, - "entry_body_limit_bytes": 65536 - }, - "schema": 1, - "thresholds": { - "minimum_provider_metadata_read_reduction_percent": 25.0 - } -} diff --git a/tests/pre15/fixtures/B19b-build-kld.sh b/tests/pre15/fixtures/B19b-build-kld.sh deleted file mode 100755 index 0b19072..0000000 --- a/tests/pre15/fixtures/B19b-build-kld.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -set -eu - -dut=${1:?DUT path is required} -freebsd_src=${2:?FreeBSD source path is required} -output=${3:?output module path is required} -work=${4:?work directory is required} -src=$dut/src -sys=$freebsd_src/sys -target=x86_64-unknown-freebsd15.0 - -test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; } -for tool in awk clang grep nm; do - command -v "$tool" >/dev/null 2>&1 || { - printf '%s\n' "missing B19b KLD tool: $tool" >&2 - exit 21 - } -done -test ! -e "$work" || { printf '%s\n' "B19b KLD work path exists: $work" >&2; exit 20; } -mkdir "$work" -cd "$work" -ln -s "$sys/amd64/include" machine -ln -s "$sys/x86/include" x86 -ln -s "$sys/i386/include" i386 -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h -: > opt_global.h - -cflags="-O2 -pipe -fno-common -fno-strict-aliasing \ --D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \ --I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \ --mno-red-zone -mno-mmx -mno-sse -msoft-float \ --fno-asynchronous-unwind-tables -ffreestanding -fwrapv \ --fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \ --Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \ --Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \ --Wno-unknown-pragmas -Wno-address-of-packed-member \ --Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \ --D__printf__=__freebsd_kprintf__ --target=$target" - -for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \ - decompressor.c zmap.c zdata.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do - extra= - if test "$file" = decompressor_zstd.c; then - extra="-I$sys/contrib/zstd/lib/freebsd" - fi - clang $cflags $extra -c "$src/$file" -o "${file%.c}.o" -done - -clang --target="$target" -r -nostdlib ./*.o -o "$output" -if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then - : -else - printf '%s\n' 'B19b KLD contains an unresolved bcmp reference' >&2 - exit 1 -fi -if nm -u "$output" | grep -qi 'xxh'; then - printf '%s\n' 'B19b KLD contains an external xxh symbol' >&2 - exit 1 -fi -if nm -u "$output" | grep -q ' ZSTD_'; then - printf '%s\n' 'B19b KLD contains an unexpected Zstd symbol' >&2 - exit 1 -fi diff --git a/tests/pre15/fixtures/B19b-xattr-generate.py b/tests/pre15/fixtures/B19b-xattr-generate.py deleted file mode 100755 index b2129f2..0000000 --- a/tests/pre15/fixtures/B19b-xattr-generate.py +++ /dev/null @@ -1,253 +0,0 @@ -#!/usr/bin/env python3 -"""Generate deterministic real EROFS Bloom fixtures for B19b.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import shutil -import struct -import subprocess - - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -CRC32C_POLY = 0x82F63B78 - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def write_json(path: Path, value: object) -> None: - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -def load_spec(path: Path) -> dict: - spec = json.loads(path.read_text(encoding="ascii")) - if spec.get("schema") != 1 or spec.get("batch") != "B19b" or spec.get("candidate") != "P15-021": - raise SystemExit("invalid B19b fixture spec identity") - return spec - - -class Image: - def __init__(self, data: bytes | bytearray): - self.data = bytearray(data) - if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC: - raise ValueError("invalid EROFS image") - self.block_bits = self.data[SUPER + 12] - self.block_size = 1 << self.block_bits - self.blocks = self.u32(SUPER + 36) - self.limit = self.blocks << self.block_bits - self.meta_blkaddr = self.u32(SUPER + 40) - self.root_nid = self.u16(SUPER + 14) - if self.limit > len(self.data): - raise ValueError("EROFS image is truncated") - - @classmethod - def load(cls, path: Path) -> "Image": - return cls(path.read_bytes()) - - def u16(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - struct.pack_into(" dict[str, int]: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - inode_format = self.u16(offset) - inode_size = 64 if inode_format & 1 else 32 - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) - size = struct.unpack_from("> 1) & 7, - "size": size, - "start_block": self.u32(offset + 16), - } - - def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes: - if inode["layout"] == 2: - physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical - elif inode["layout"] == 0: - physical = (inode["start_block"] << self.block_bits) + logical - else: - raise ValueError("fixture directory has an unsupported layout") - if physical > self.limit or length > self.limit - physical: - raise ValueError("fixture directory is out of bounds") - return bytes(self.data[physical:physical + length]) - - def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]: - result = [] - logical = 0 - while logical < inode["size"]: - length = min(self.block_size, inode["size"] - logical) - data = self.inode_data(inode, logical, length) - if len(data) < 12: - raise ValueError("fixture directory block is short") - first_name = struct.unpack_from(" len(data): - raise ValueError("fixture directory name offset is invalid") - count = first_name // 12 - for index in range(count): - entry = index * 12 - nid = struct.unpack_from(" end or end > len(data): - raise ValueError("fixture directory name is out of bounds") - result.append((data[start:end].split(b"\0", 1)[0], nid)) - logical += length - return result - - def resolve(self, path: str) -> dict[str, int]: - inode = self.inode(self.root_nid) - for component in path.strip("/").encode().split(b"/"): - matches = [nid for name, nid in self.directory_entries(inode) if name == component] - if len(matches) != 1: - raise ValueError(f"fixture path is absent or ambiguous: {path}") - inode = self.inode(matches[0]) - return inode - - def update_checksum(self) -> None: - compat = self.u32(SUPER + 8) - if compat & 1: - self.put_u32(SUPER + 4, 0) - self.put_u32(SUPER + 4, crc32c(self.data[SUPER:self.block_size])) - - def write(self, path: Path) -> None: - path.write_bytes(self.data) - - -def create_source(path: Path, spec: dict) -> None: - path.mkdir(parents=True) - path.chmod(0o755) - files = [path / f"peer-{index:03d}.bin" for index in range(spec["fixture"]["peer_count"])] - files.append(path / "target.bin") - for entry in files: - entry.write_bytes(b"P15-021\n") - entry.chmod(0o644) - for index in range(spec["fixture"]["attribute_count"]): - name = f"user.attr{index:02d}".encode() - prefix = f"value-{index:02d}-".encode() - length = spec["fixture"]["attribute_value_bytes"] - value = (prefix * (length // len(prefix) + 1))[:length] - os.setxattr(entry, name, value) - os.utime(entry, (0, 0), follow_symlinks=False) - os.utime(path, (0, 0), follow_symlinks=False) - - -def build_valid(output: Path, source: Path, spec: dict) -> list[str]: - mkfs = shutil.which("mkfs.erofs") - if mkfs is None: - raise SystemExit("mkfs.erofs is required") - command = [ - mkfs, - "-d0", - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "--sort=path", - f"-U{spec['fixture']['uuid']}", - "-x2", - "-Exattr-name-filter,force-inode-extended", - str(output), - str(source), - ] - completed = subprocess.run(command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if completed.returncode != 0: - raise SystemExit(f"mkfs.erofs failed: {completed.stdout.decode('utf-8', 'replace')}") - return command - - -def mutate(valid: Path, output: Path, mutation: str, spec: dict) -> None: - image = Image.load(valid) - target = image.resolve(spec["fixture"]["target"]) - body = target["offset"] + target["inode_size"] - if mutation == "unknown-filter": - image.data[SUPER + 104] = 1 - elif mutation == "feature-off": - image.put_u32(SUPER + 8, image.u32(SUPER + 8) & ~spec["format"]["feature_compat"]) - elif mutation == "corrupt-shared-count": - image.data[body + 4] = 255 - elif mutation == "corrupt-shared-id": - if image.data[body + 4] == 0: - raise SystemExit("valid fixture has no shared IDs") - image.put_u32(body + 12, 0xFFFFFFFF) - else: - raise SystemExit(f"unknown mutation: {mutation}") - image.update_checksum() - image.write(output) - - -def generate(output: Path, work: Path, spec: dict) -> None: - if output.exists(): - raise SystemExit(f"refusing existing output: {output}") - if work.exists(): - raise SystemExit(f"refusing existing work directory: {work}") - output.mkdir(parents=True) - work.mkdir(parents=True) - source = work / "source" - create_source(source, spec) - valid = output / "valid.erofs" - command = build_valid(valid, source, spec) - for mutation in ("unknown-filter", "feature-off", "corrupt-shared-count", "corrupt-shared-id"): - mutate(valid, output / f"{mutation}.erofs", mutation, spec) - hashes = {path.name: sha256(path) for path in sorted(output.glob("*.erofs"))} - if hashes != spec["fixture_sha256"]: - raise SystemExit(f"B19b generated fixture hashes differ: {hashes}") - fsck = shutil.which("fsck.erofs") - if fsck is None: - raise SystemExit("fsck.erofs is required") - for name in ("valid.erofs", "unknown-filter.erofs", "feature-off.erofs"): - completed = subprocess.run([fsck, str(output / name)], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if completed.returncode != 0: - raise SystemExit(f"fsck.erofs rejected legal fixture {name}") - sums = "".join(f"{digest} {name}\n" for name, digest in sorted(hashes.items())) - (output / "SHA256SUMS").write_text(sums, encoding="ascii") - write_json(output / "manifest.json", { - "command": [*command[:-2], "OUTPUT/valid.erofs", "WORK/source"], - "fixture_sha256": hashes, - "schema": 1, - "status": "PASS", - }) - print(json.dumps({"fixture_sha256": hashes, "status": "PASS"}, sort_keys=True)) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("generate", choices=("generate",)) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--work", type=Path, required=True) - parser.add_argument("--spec", type=Path, required=True) - args = parser.parse_args() - generate(args.output, args.work, load_spec(args.spec)) - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B19b-xattr-oracle.py b/tests/pre15/fixtures/B19b-xattr-oracle.py deleted file mode 100755 index 42a751a..0000000 --- a/tests/pre15/fixtures/B19b-xattr-oracle.py +++ /dev/null @@ -1,426 +0,0 @@ -#!/usr/bin/env python3 -"""Independently verify B19b fixtures, Bloom semantics, and source scope.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -from pathlib import Path -import re -import statistics -import struct -import subprocess - - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 - - -class Reject(RuntimeError): - def __init__(self, errno_name: str, point: str): - super().__init__(f"{errno_name} at {point}") - self.errno_name = errno_name - self.point = point - - -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def write_json(path: Path, value: object) -> None: - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def rotl32(value: int, count: int) -> int: - return ((value << count) | (value >> (32 - count))) & 0xFFFFFFFF - - -def xxh32(data: bytes, seed: int) -> int: - prime1 = 2654435761 - prime2 = 2246822519 - prime3 = 3266489917 - prime4 = 668265263 - prime5 = 374761393 - cursor = 0 - - def round32(accumulator: int, lane: int) -> int: - accumulator = (accumulator + lane * prime2) & 0xFFFFFFFF - return (rotl32(accumulator, 13) * prime1) & 0xFFFFFFFF - - if len(data) >= 16: - accumulator1 = (seed + prime1 + prime2) & 0xFFFFFFFF - accumulator2 = (seed + prime2) & 0xFFFFFFFF - accumulator3 = seed & 0xFFFFFFFF - accumulator4 = (seed - prime1) & 0xFFFFFFFF - limit = len(data) - 16 - while cursor <= limit: - accumulator1 = round32(accumulator1, int.from_bytes(data[cursor:cursor + 4], "little")) - accumulator2 = round32(accumulator2, int.from_bytes(data[cursor + 4:cursor + 8], "little")) - accumulator3 = round32(accumulator3, int.from_bytes(data[cursor + 8:cursor + 12], "little")) - accumulator4 = round32(accumulator4, int.from_bytes(data[cursor + 12:cursor + 16], "little")) - cursor += 16 - value = ( - rotl32(accumulator1, 1) - + rotl32(accumulator2, 7) - + rotl32(accumulator3, 12) - + rotl32(accumulator4, 18) - ) & 0xFFFFFFFF - else: - value = (seed + prime5) & 0xFFFFFFFF - value = (value + len(data)) & 0xFFFFFFFF - while cursor + 4 <= len(data): - value = (value + int.from_bytes(data[cursor:cursor + 4], "little") * prime3) & 0xFFFFFFFF - value = (rotl32(value, 17) * prime4) & 0xFFFFFFFF - cursor += 4 - while cursor < len(data): - value = (value + data[cursor] * prime5) & 0xFFFFFFFF - value = (rotl32(value, 11) * prime1) & 0xFFFFFFFF - cursor += 1 - value ^= value >> 15 - value = (value * prime2) & 0xFFFFFFFF - value ^= value >> 13 - value = (value * prime3) & 0xFFFFFFFF - value ^= value >> 16 - return value & 0xFFFFFFFF - - -class Reader: - def __init__(self, path: Path, spec: dict): - self.data = path.read_bytes() - self.spec = spec - self.calls = 0 - self.bytes = 0 - self.blocks_read: set[int] = set() - header = self.read(SUPER, 144, "super") - if struct.unpack_from(" len(self.data): - raise Reject("EINTEGRITY", "super.bounds") - - def read(self, offset: int, length: int, point: str) -> bytes: - limit = self.limit if hasattr(self, "limit") else len(self.data) - if offset < 0 or length < 0 or offset > limit or length > limit - offset: - raise Reject("EINTEGRITY", f"{point}.bounds") - self.calls += 1 - self.bytes += length - if length: - self.blocks_read.update(range(offset // 4096, (offset + length - 1) // 4096 + 1)) - return self.data[offset:offset + length] - - def reset_reads(self) -> None: - self.calls = 0 - self.bytes = 0 - self.blocks_read.clear() - - def inode(self, nid: int) -> dict[str, int]: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - raw = self.read(offset, 64, "inode") - inode_format, xattr_count = struct.unpack_from("> 1) & 7, - "size": size, - "start_block": struct.unpack_from(" bytes: - if inode["layout"] == 2: - physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical - elif inode["layout"] == 0: - physical = (inode["start_block"] << self.block_bits) + logical - else: - raise Reject("EINTEGRITY", "directory.layout") - return self.read(physical, length, "directory.data") - - def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]: - entries = [] - logical = 0 - while logical < inode["size"]: - length = min(self.block_size, inode["size"] - logical) - data = self.inode_data(inode, logical, length) - if len(data) < 12: - raise Reject("EINTEGRITY", "directory.header") - first_name = struct.unpack_from(" len(data): - raise Reject("EINTEGRITY", "directory.name-offset") - count = first_name // 12 - for index in range(count): - offset = index * 12 - nid = struct.unpack_from(" end or end > len(data): - raise Reject("EINTEGRITY", "directory.name-bounds") - entries.append((data[start:end].split(b"\0", 1)[0], nid)) - logical += length - return entries - - def resolve(self, path: str) -> dict[str, int]: - inode = self.inode(self.root_nid) - for component in path.strip("/").encode().split(b"/"): - matches = [nid for name, nid in self.directory_entries(inode) if name == component] - if len(matches) != 1: - raise Reject("ENOATTR", "path.lookup") - inode = self.inode(matches[0]) - return inode - - def stats(self, scanned: bool) -> dict[str, int | bool]: - return { - "bytes": self.bytes, - "calls": self.calls, - "provider_blocks": len(self.blocks_read), - "scanned": scanned, - } - - def lookup(self, inode: dict[str, int], name: bytes, candidate: bool) -> tuple[str, bytes | None, dict]: - self.reset_reads() - body_offset = inode["offset"] + inode["inode_size"] - usable = self.feature_compat & self.spec["format"]["feature_compat"] and self.filter_reserved == 0 - if candidate and usable: - if inode["xattr_size"] < 12: - raise Reject("EINTEGRITY", "ibody.header") - if inode["xattr_size"] == 12: - raise Reject("EOPNOTSUPP", "ibody.header-only") - header = self.read(body_offset, 12, "ibody.header") - shared_count = header[4] - if 12 + shared_count * 4 > inode["xattr_size"]: - raise Reject("EINTEGRITY", "ibody.shared-count") - name_filter = struct.unpack_from(" len(body): - raise Reject("EINTEGRITY", "ibody.shared-count") - cursor = header_size - while cursor < len(body): - if cursor + 4 > len(body): - raise Reject("EINTEGRITY", "inline.header") - name_length, name_index, value_length = struct.unpack_from(" len(body) - cursor: - raise Reject("EINTEGRITY", "inline.bounds") - actual = body[cursor + 4:cursor + 4 + name_length] - if b"\0" in actual: - raise Reject("EINTEGRITY", "inline.name-nul") - if name_index == 1 and actual == name: - start = cursor + 4 + name_length - return "PASS", body[start:start + value_length], self.stats(True) - cursor += total - for index in range(shared_count): - shared_id = struct.unpack_from(" dict: - try: - reader = Reader(path, spec) - inode = reader.resolve(spec["fixture"]["target"]) - status, value, reads = reader.lookup(inode, name, candidate) - return {"errno": 0 if status == "PASS" else spec["errno"][status], "reads": reads, "status": status, "value_hex": None if value is None else value.hex()} - except Reject as error: - return {"errno": spec["errno"].get(error.errno_name, -1), "point": error.point, "status": error.errno_name, "value_hex": None} - - -def function(source: str, name: str) -> str: - match = re.search(rf"\n{name}\([^;]*?\n\{{", source, re.DOTALL) - if match is None: - raise SystemExit(f"missing source function: {name}") - start = match.start() + 1 - brace = source.index("{", match.start()) - depth = 0 - for index in range(brace, len(source)): - if source[index] == "{": - depth += 1 - elif source[index] == "}": - depth -= 1 - if depth == 0: - return source[start:index + 1] - raise SystemExit(f"unterminated source function: {name}") - - -def audit_source(root: Path, dut: Path, spec: dict) -> dict: - header = (dut / "src/erofs_fs.h").read_text(encoding="utf-8") - internal = (dut / "src/internal.h").read_text(encoding="utf-8") - xattr = (dut / "src/xattr.c").read_text(encoding="utf-8") - linux_header = (root / "src-linux/erofs_fs.h").read_text(encoding="utf-8") - linux_xattr = (root / "src-linux/xattr.c").read_text(encoding="utf-8") - for marker in ( - "#define EROFS_XATTR_FILTER_BITS", - "#define EROFS_XATTR_FILTER_DEFAULT", - "#define EROFS_XATTR_FILTER_SEED", - ): - if marker not in header or marker not in linux_header: - raise SystemExit(f"Bloom format marker missing: {marker}") - if "erofs_sb_has_xattr_filter_v1" not in internal or "xattr_filter_reserved == 0" not in internal: - raise SystemExit("FreeBSD unknown-filter fallback helper is incomplete") - hash_body = function(xattr, "erofs_xxh32") - filter_name = function(xattr, "erofs_xattr_filter_name") - filter_negative = function(xattr, "erofs_xattr_filter_negative") - getxattr = function(xattr, "erofs_getxattr") - if "static uint32_t\nerofs_xxh32" not in xattr: - raise SystemExit("xxh32 is not file-local and namespaced") - for marker in ("le32dec(cursor)", "UINT32_C(2654435761)", "hash ^= hash >> 16"): - if marker not in hash_body: - raise SystemExit(f"xxh32 source marker missing: {marker}") - for marker in ( - "EXTATTR_NAMESPACE_USER", - "EXTATTR_NAMESPACE_SYSTEM", - "EROFS_XATTR_INDEX_USER", - "EROFS_XATTR_INDEX_POSIX_ACL_ACCESS", - "EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT", - "EROFS_XATTR_INDEX_TRUSTED", - "EROFS_XATTR_INDEX_SECURITY", - ): - if marker not in filter_name: - raise SystemExit(f"FreeBSD namespace mapping marker missing: {marker}") - for marker in ( - "erofs_sb_has_xattr_filter_v1(sbi)", - "header_size > vi->xattr_isize", - "erofs_xxh32(filter_name, filter_name_len", - "erofs_put_metabuf(&buf)", - ): - if marker not in filter_negative: - raise SystemExit(f"fast-negative integrity marker missing: {marker}") - if any(marker in filter_negative.lower() for marker in ("malloc", "mtx_", "cv_", "cache")): - raise SystemExit("B19b introduced cache/allocation/locking into the fast-negative") - if getxattr.index("erofs_xattr_filter_negative") > getxattr.index("erofs_xattr_load_body"): - raise SystemExit("fast-negative runs after the full xattr body read") - if "if (filter_negative)\n\t\treturn (ENOATTR);" not in getxattr: - raise SystemExit("getxattr does not restrict the shortcut to proven negatives") - if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", header + internal + xattr): - raise SystemExit("Linux negative errno entered the FreeBSD implementation") - for marker in ( - "xxh32(name, strlen(name)", - "EROFS_XATTR_FILTER_SEED + index", - "vi->xattr_name_filter & (1U << hashbit)", - "!sbi->xattr_filter_reserved", - ): - if marker not in linux_xattr: - raise SystemExit(f"Linux comparison anchor changed: {marker}") - changed = subprocess.run( - ["git", "-C", str(root), "diff", "--name-only", spec["gate"]["commit"], "--", "repo-pre-15/src"], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.splitlines() - expected = ["repo-pre-15/src/erofs_fs.h", "repo-pre-15/src/internal.h", "repo-pre-15/src/xattr.c"] - if sorted(changed) != expected: - raise SystemExit(f"B19b source write set differs: {changed}") - return { - "freebsd_errno": "positive", - "freebsd_namespace": "extattr user/system mapping retained", - "hash_symbol": "file-local erofs_xxh32", - "linux_semantics": "xxh32(name suffix, seed + index), inverse 32-bit filter", - "source_write_set": expected, - "status": "PASS", - } - - -def verify(fixtures: Path, spec: dict, root: Path, dut: Path) -> dict: - hashes = {path.name: sha256(path) for path in sorted(fixtures.glob("*.erofs"))} - if hashes != spec["fixture_sha256"]: - raise SystemExit(f"fixture hashes differ: {hashes}") - cases = [ - ("hit", "valid.erofs", b"attr00", "PASS", True), - ("miss", "valid.erofs", spec["fixture"]["miss"].encode(), "ENOATTR", False), - ("false-positive", "valid.erofs", spec["fixture"]["collision"].encode(), "ENOATTR", True), - ("unknown-filter", "unknown-filter.erofs", spec["fixture"]["miss"].encode(), "ENOATTR", True), - ("feature-off", "feature-off.erofs", spec["fixture"]["miss"].encode(), "ENOATTR", True), - ("corrupt-shared-count", "corrupt-shared-count.erofs", spec["fixture"]["miss"].encode(), "EINTEGRITY", None), - ("corrupt-shared-id", "corrupt-shared-id.erofs", b"attr00", "EINTEGRITY", None), - ] - results = [] - for identifier, image, name, expected_status, expected_scan in cases: - baseline = lookup(fixtures / image, name, False, spec) - candidate = lookup(fixtures / image, name, True, spec) - if baseline["status"] != expected_status or candidate["status"] != expected_status: - raise SystemExit(f"{identifier} status differs: {baseline}, {candidate}") - if baseline["errno"] < 0 or candidate["errno"] < 0: - raise SystemExit(f"{identifier} returned negative errno") - if expected_scan is not None and candidate["reads"]["scanned"] != expected_scan: - raise SystemExit(f"{identifier} scan decision differs") - results.append({"baseline": baseline, "candidate": candidate, "id": identifier, "name": name.decode()}) - baseline_calls = [] - candidate_calls = [] - baseline_blocks = [] - candidate_blocks = [] - for _ in range(spec["thresholds"]["samples"]): - baseline = lookup(fixtures / "valid.erofs", spec["fixture"]["miss"].encode(), False, spec)["reads"] - candidate = lookup(fixtures / "valid.erofs", spec["fixture"]["miss"].encode(), True, spec)["reads"] - baseline_calls.append(baseline["calls"]) - candidate_calls.append(candidate["calls"]) - baseline_blocks.append(baseline["provider_blocks"]) - candidate_blocks.append(candidate["provider_blocks"]) - call_reduction = 100 * (statistics.median(baseline_calls) - statistics.median(candidate_calls)) / statistics.median(baseline_calls) - block_reduction = 100 * (statistics.median(baseline_blocks) - statistics.median(candidate_blocks)) / statistics.median(baseline_blocks) - threshold = spec["thresholds"]["minimum_provider_metadata_read_reduction_percent"] - if call_reduction < threshold or block_reduction < threshold: - raise SystemExit("B19b source model no longer meets the measured benefit gate") - return { - "benchmark": { - "baseline_calls": baseline_calls, - "baseline_provider_blocks": baseline_blocks, - "call_reduction_percent": call_reduction, - "candidate_calls": candidate_calls, - "candidate_provider_blocks": candidate_blocks, - "provider_block_reduction_percent": block_reduction, - "threshold_percent": threshold, - }, - "cases": results, - "fixture_sha256": hashes, - "source": audit_source(root, dut, spec), - "status": "PASS", - "test": spec["test"], - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--dut", type=Path, required=True) - parser.add_argument("--fixtures", type=Path, required=True) - parser.add_argument("--report", type=Path, required=True) - parser.add_argument("--root", type=Path, required=True) - parser.add_argument("--spec", type=Path, required=True) - args = parser.parse_args() - spec = json.loads(args.spec.read_text(encoding="ascii")) - if spec.get("schema") != 1 or spec.get("batch") != "B19b": - raise SystemExit("invalid B19b oracle spec") - report = verify(args.fixtures, spec, args.root, args.dut) - write_json(args.report, report) - print(json.dumps({"case_count": len(report["cases"]), "status": "PASS", "test": report["test"]}, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B19b-xattr-probe.c b/tests/pre15/fixtures/B19b-xattr-probe.c deleted file mode 100644 index 7e9b94b..0000000 --- a/tests/pre15/fixtures/B19b-xattr-probe.c +++ /dev/null @@ -1,207 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#define VALUE_SIZE 96 - -static int -fail(const char *message) -{ - fprintf(stderr, "%s\n", message); - return (1); -} - -static int -parse_long(const char *text, long minimum, long maximum, long *valuep) -{ - char *end; - long value; - - errno = 0; - value = strtol(text, &end, 10); - if (errno != 0 || text[0] == '\0' || end[0] != '\0' || - value < minimum || value > maximum) - return (-1); - *valuep = value; - return (0); -} - -static int -namespace_id(const char *name) -{ - if (strcmp(name, "user") == 0) - return (EXTATTR_NAMESPACE_USER); - if (strcmp(name, "system") == 0) - return (EXTATTR_NAMESPACE_SYSTEM); - return (-1); -} - -static void -expected_value(unsigned int index, uint8_t *value) -{ - char prefix[16]; - size_t length; - - (void)snprintf(prefix, sizeof(prefix), "value-%02u-", index); - length = strlen(prefix); - for (size_t offset = 0; offset < VALUE_SIZE; offset++) - value[offset] = prefix[offset % length]; -} - -static int -get_hit(const char *path, const char *name, unsigned int index) -{ - uint8_t actual[VALUE_SIZE + 16], expected[VALUE_SIZE]; - ssize_t length; - - memset(actual, 0xa5, sizeof(actual)); - expected_value(index, expected); - errno = 0; - length = extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, actual, - sizeof(actual)); - if (length != VALUE_SIZE || errno != 0) - return (fail("extattr hit returned the wrong length or errno")); - if (memcmp(actual, expected, sizeof(expected)) != 0) - return (fail("extattr hit returned the wrong value")); - for (size_t offset = VALUE_SIZE; offset < sizeof(actual); offset++) { - if (actual[offset] != 0xa5) - return (fail("extattr hit wrote beyond the value")); - } - return (0); -} - -static int -get_errno(const char *path, int attrnamespace, const char *name, - int expected_errno) -{ - uint8_t actual[VALUE_SIZE + 16]; - ssize_t length; - - memset(actual, 0xa5, sizeof(actual)); - errno = 0; - length = extattr_get_file(path, attrnamespace, name, actual, - sizeof(actual)); - if (length != -1 || errno != expected_errno) - return (fail("extattr failure returned the wrong positive errno")); - for (size_t offset = 0; offset < sizeof(actual); offset++) { - if (actual[offset] != 0xa5) - return (fail("failed extattr lookup modified the output")); - } - return (0); -} - -static int -list_user(const char *path) -{ - uint8_t *buffer; - ssize_t length; - size_t offset; - unsigned int seen; - - errno = 0; - length = extattr_list_file(path, EXTATTR_NAMESPACE_USER, NULL, 0); - if (length <= 0 || errno != 0) - return (fail("extattr list sizing failed")); - buffer = malloc((size_t)length); - if (buffer == NULL) - return (fail("extattr list allocation failed")); - if (extattr_list_file(path, EXTATTR_NAMESPACE_USER, buffer, - (size_t)length) != length) { - free(buffer); - return (fail("extattr list transfer failed")); - } - seen = 0; - for (offset = 0; offset < (size_t)length;) { - char expected[16]; - uint8_t name_length; - unsigned int index; - - name_length = buffer[offset++]; - if (name_length == 0 || name_length > (size_t)length - offset) { - free(buffer); - return (fail("extattr list record is malformed")); - } - for (index = 0; index < 8; index++) { - (void)snprintf(expected, sizeof(expected), "attr%02u", index); - if (strlen(expected) == name_length && - memcmp(buffer + offset, expected, name_length) == 0) - break; - } - if (index == 8 || (seen & (1U << index)) != 0) { - free(buffer); - return (fail("extattr list returned an unknown or duplicate name")); - } - seen |= 1U << index; - offset += name_length; - } - free(buffer); - if (seen != 0xff) - return (fail("extattr list omitted a user attribute")); - return (0); -} - -static int -concurrent(const char *path, const char *hit, const char *miss, - const char *collision, long workers, long loops) -{ - long worker; - int status; - pid_t child; - - for (worker = 0; worker < workers; worker++) { - child = fork(); - if (child < 0) - return (fail("fork failed")); - if (child == 0) { - for (long iteration = 0; iteration < loops; iteration++) { - if (get_hit(path, hit, 0) != 0 || - get_errno(path, EXTATTR_NAMESPACE_USER, miss, ENOATTR) != 0 || - get_errno(path, EXTATTR_NAMESPACE_USER, collision, ENOATTR) != 0) - _exit(1); - } - _exit(0); - } - } - for (worker = 0; worker < workers; worker++) { - if (wait(&status) < 0 || !WIFEXITED(status) || - WEXITSTATUS(status) != 0) - return (fail("concurrent extattr worker failed")); - } - return (0); -} - -int -main(int argc, char **argv) -{ - long first, second; - int attrnamespace; - - if (argc == 5 && strcmp(argv[1], "hit") == 0) { - if (parse_long(argv[4], 0, 7, &first) != 0) - return (fail("invalid hit index")); - return (get_hit(argv[2], argv[3], (unsigned int)first)); - } - if (argc == 6 && strcmp(argv[1], "errno") == 0) { - attrnamespace = namespace_id(argv[3]); - if (attrnamespace < 0 || parse_long(argv[5], 1, 255, &first) != 0) - return (fail("invalid errno arguments")); - return (get_errno(argv[2], attrnamespace, argv[4], (int)first)); - } - if (argc == 3 && strcmp(argv[1], "list") == 0) - return (list_user(argv[2])); - if (argc == 8 && strcmp(argv[1], "concurrent") == 0) { - if (parse_long(argv[6], 1, 64, &first) != 0 || - parse_long(argv[7], 1, 1000, &second) != 0) - return (fail("invalid concurrency arguments")); - return (concurrent(argv[2], argv[3], argv[4], argv[5], first, - second)); - } - return (fail("usage: B19b-xattr-probe MODE PATH ARG...")); -} diff --git a/tests/pre15/fixtures/B19b-xattr-spec.json b/tests/pre15/fixtures/B19b-xattr-spec.json deleted file mode 100644 index 73aa7cc..0000000 --- a/tests/pre15/fixtures/B19b-xattr-spec.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "batch": "B19b", - "candidate": "P15-021", - "concurrency": { - "loops_per_worker": 10, - "workers": 64 - }, - "errno": { - "EINTEGRITY": 97, - "ENOATTR": 87 - }, - "fixture": { - "attribute_count": 8, - "attribute_value_bytes": 96, - "collision": "absent-00000", - "miss": "absent-00001", - "peer_count": 64, - "shared_count": 8, - "target": "/target.bin", - "uuid": "00000000-0000-0000-0000-000000000021" - }, - "fixture_sha256": { - "corrupt-shared-count.erofs": "c1f061f1a7f60c4e347b70cb17f7dac01b9c4e98904aa9da038cc12dad26f15d", - "corrupt-shared-id.erofs": "1ea7b408d8b725352970d14c7b73146055db26bc15ac5fb9ca8a9be05bb22314", - "feature-off.erofs": "2c802394381dbab9d30e50486975e5c1f40688f8b3a21ffeac26edc8ab09ce85", - "unknown-filter.erofs": "4333632363689cb16a715c4204d8fd9d2a5cf5cc2ace77828d03edcb391e4fe5", - "valid.erofs": "b06b7c4c30adb3684d11505edf815dc0a395c03f6a95120b3150361008a182cb" - }, - "format": { - "bits": 32, - "feature_compat": 4, - "seed": 633069711 - }, - "gate": { - "commit": "675ed9b650b3bc157b38bcc165a0cb215a2aa989", - "decision_base": "666e52f710363df07f7c93919eb835d41092d011", - "input_sha256": "2c39d55e5d9db278beb638a40188b60d343617be6c1e7c30a658ae92fe12d9bd", - "script_sha256": "376200a950ec09ad8df29f1e76b0595533911e6e253201aa31b351bdef009947" - }, - "schema": 1, - "test": "TC169-xattr-filter", - "thresholds": { - "minimum_provider_metadata_read_reduction_percent": 25.0, - "samples": 5 - } -} diff --git a/tests/pre15/fixtures/B21-manual-run.sh b/tests/pre15/fixtures/B21-manual-run.sh deleted file mode 100755 index 7c5495e..0000000 --- a/tests/pre15/fixtures/B21-manual-run.sh +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env bash -set -eu - -evidence=${1:?evidence directory is required} -root=/work/erofs-freebsd-pre -dut=$root/repo-pre-15 -base=/work/build/vm-freebsd-build.qcow2.bp -askpass=/work/build/.repo22-ssh-askpass -module=${B21_MODULE:-$root/planning/pre15/evidence/20260816T-B21-abi15-build/B21-erofs-zstdio0.ko} -source_tree=${B21_FREEBSD_SRC:-/work/dev-freebsd-releng} -owned_pid= -port= -loaded=0 -mounted=0 -md_units= - -test ! -e "$evidence" -mkdir -p "$evidence" -: >"$evidence/cleanup.log" - -ssh_args=(env DISPLAY=:0 SSH_ASKPASS="$askpass" SSH_ASKPASS_REQUIRE=force ssh - -n - -o BatchMode=no -o PubkeyAuthentication=no - -o PreferredAuthentications=keyboard-interactive,password - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -o LogLevel=ERROR -o ConnectTimeout=5) -scp_args=(env DISPLAY=:0 SSH_ASKPASS="$askpass" SSH_ASKPASS_REQUIRE=force scp - -O -q -o BatchMode=no -o PubkeyAuthentication=no - -o PreferredAuthentications=keyboard-interactive,password - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -o LogLevel=ERROR -o ConnectTimeout=5) - -guest() -{ - timeout -k 5 60 "${ssh_args[@]}" -p "$port" root@127.0.0.1 "$@" -} - -capture() -{ - local name=$1 - shift - set +e - guest "$@" >"$evidence/$name.stdout" 2>"$evidence/$name.stderr" - local rc=$? - set -e - printf '%s\n' "$rc" >"$evidence/$name.rc" -} - -scp_to() -{ - local source_path=$1 - local destination_path=$2 - timeout -k 5 60 "${scp_args[@]}" -P "$port" "$source_path" \ - "root@127.0.0.1:$destination_path" -} - -cleanup() -{ - set +e - local cleanup_rc=0 - if test "$mounted" = 1; then guest umount /mnt/pre15-b21 >>"$evidence/cleanup.log" 2>&1 || cleanup_rc=1; fi - for md in $md_units; do guest mdconfig -d -u "$md" >>"$evidence/cleanup.log" 2>&1 || cleanup_rc=1; done - if test "$loaded" = 1; then guest kldunload /root/B21-erofs-zstdio0.ko >>"$evidence/cleanup.log" 2>&1 || cleanup_rc=1; fi - if test -n "$owned_pid" && kill -0 "$owned_pid" 2>/dev/null; then - test "$owned_pid" != 26318 || exit 99 - kill "$owned_pid" 2>/dev/null - wait "$owned_pid" 2>/dev/null - fi - test ! -e "$evidence/owned-overlay.qcow2" || unlink "$evidence/owned-overlay.qcow2" - python3 - "$port" <<'PY' -import socket -import sys -if not sys.argv[1]: - raise SystemExit(0) -with socket.socket() as sock: - sock.settimeout(0.2) - raise SystemExit(0 if sock.connect_ex(("127.0.0.1", int(sys.argv[1]))) else 1) -PY - test $? = 0 || cleanup_rc=1 - stat -c 'path=%n size=%s inode=%i mode=%f mtime=%Y ctime=%Z' "$base" \ - >"$evidence/base-metadata-after.txt" - cmp -s "$evidence/base-metadata-before.txt" "$evidence/base-metadata-after.txt" || cleanup_rc=1 - if test "$cleanup_rc" = 0 && test ! -e "$evidence/owned-overlay.qcow2"; then - printf '%s\n' PASS >"$evidence/cleanup.status" - else - printf '%s\n' FAIL >"$evidence/cleanup.status" - fi -} -trap cleanup EXIT HUP INT TERM - -test -x "$askpass" -test -f "$module" -test -f "$base" -stat -c 'path=%n size=%s inode=%i mode=%f mtime=%Y ctime=%Z' "$base" \ - >"$evidence/base-metadata-before.txt" -git -C "$dut" rev-parse HEAD >"$evidence/dut-head.txt" -{ - git -C "$source_tree" log -1 --format='%H %cs %s' - sed -n '76,77p' "$source_tree/sys/sys/param.h" - sed -n '53,54p' "$source_tree/sys/conf/newvers.sh" -} >"$evidence/source-identity.txt" -printf '%s\n' 'SSH options precede root@127.0.0.1 and the remote command' \ - >"$evidence/transport-contract.txt" - -python3 -B "$dut/tests/pre15/fixtures/B21-super-generate.py" \ - --spec "$dut/tests/pre15/fixtures/B21-super-spec.json" \ - --output "$evidence/first" --work "$evidence/first-work" \ - >"$evidence/generate-first.stdout" 2>"$evidence/generate-first.stderr" -python3 -B "$dut/tests/pre15/fixtures/B21-super-generate.py" \ - --spec "$dut/tests/pre15/fixtures/B21-super-spec.json" \ - --output "$evidence/second" --work "$evidence/second-work" \ - >"$evidence/generate-second.stdout" 2>"$evidence/generate-second.stderr" -cmp "$evidence/first/fixture-index.json" "$evidence/second/fixture-index.json" -python3 -B "$dut/tests/pre15/fixtures/B21-super-oracle.py" \ - --fixtures "$evidence/first" --report "$evidence/oracle.json" \ - >"$evidence/oracle.stdout" 2>"$evidence/oracle.stderr" - -python3 - "$evidence/first/fixture-index.json" "$evidence/oracle.json" \ - "$evidence/matrix.tsv" <<'PY' -import json -import sys -from pathlib import Path - -index = json.loads(Path(sys.argv[1]).read_text()) -oracle = json.loads(Path(sys.argv[2]).read_text()) -assert index["case_count"] == 13 and oracle["passed_count"] == 13 -rows = ["id\texpected_errno\treject"] -for fixture, result in zip(index["cases"], oracle["results"], strict=True): - rows.append( - f"{fixture['id']}\t{result['actual_errno']}\t" - f"{fixture['expected_reject']}" - ) -Path(sys.argv[3]).write_text("\n".join(rows) + "\n") -PY -tar -C "$evidence/first" -czf "$evidence/fixtures.tar.gz" . - -qemu-img create -f qcow2 -F qcow2 -b "$base" \ - "$evidence/owned-overlay.qcow2" >"$evidence/qemu-img.stdout" \ - 2>"$evidence/qemu-img.stderr" -port=$(python3 - <<'PY' -import socket -with socket.socket() as sock: - sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - if port == 9222: - raise SystemExit("protected port selected") - print(port) -PY -) -printf '%s\n' "$port" >"$evidence/owned-port.txt" -qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 -m 6144 -smp 4 \ - -drive "file=$evidence/owned-overlay.qcow2,if=virtio,format=qcow2" \ - -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$port-:22" \ - -device virtio-net-pci,netdev=net0 -display none \ - -serial "file:$evidence/serial.log" -monitor none \ - >"$evidence/qemu.stdout" 2>"$evidence/qemu.stderr" & -owned_pid=$! -printf '%s\n' "$owned_pid" >"$evidence/owned-pid.txt" -test "$owned_pid" != 26318 - -ready=0 -for attempt in $(seq 1 180); do - if ! kill -0 "$owned_pid" 2>/dev/null; then exit 21; fi - set +e - guest true >"$evidence/readiness.stdout" 2>"$evidence/readiness.stderr" - readiness_rc=$? - set -e - printf '%s\n' "$readiness_rc" >"$evidence/readiness.rc" - if test "$readiness_rc" = 0; then ready=1; break; fi - sleep 1 -done -printf 'attempts=%s ready=%s\n' "$attempt" "$ready" >"$evidence/readiness.txt" -test "$ready" = 1 - -capture guest-uname uname -a -capture guest-freebsd-version freebsd-version -kru -capture guest-osreldate sysctl -n kern.osreldate -capture guest-kldstat-before kldstat -capture guest-kldstat-module-before kldstat -q -m erofs -capture guest-dmesg-before dmesg -test "$(cat "$evidence/guest-kldstat-module-before.rc")" = 1 - -scp_to "$module" /root/B21-erofs-zstdio0.ko \ - >"$evidence/upload-module.stdout" 2>"$evidence/upload-module.stderr" -scp_to "$dut/tests/pre15/fixtures/B21-qemu-probe.c" /root/B21-qemu-probe.c \ - >"$evidence/upload-probe.stdout" 2>"$evidence/upload-probe.stderr" -scp_to "$evidence/fixtures.tar.gz" /root/B21-fixtures.tar.gz \ - >"$evidence/upload-fixtures.stdout" 2>"$evidence/upload-fixtures.stderr" -capture guest-module-file file /root/B21-erofs-zstdio0.ko -capture guest-module-sha256 sha256 -q /root/B21-erofs-zstdio0.ko -capture guest-prepare sh -c \ - 'rm -rf /root/B21-fixtures /mnt/pre15-b21; mkdir /root/B21-fixtures /mnt/pre15-b21; tar -xzf /root/B21-fixtures.tar.gz -C /root/B21-fixtures; cc -O2 -Wall -Wextra -Werror -std=c17 -o /root/B21-qemu-probe /root/B21-qemu-probe.c' -test "$(cat "$evidence/guest-prepare.rc")" = 0 - -capture B21-kldload kldload /root/B21-erofs-zstdio0.ko -test "$(cat "$evidence/B21-kldload.rc")" = 0 -loaded=1 -capture guest-kldstat-after-load kldstat -capture guest-kldstat-module-after-load kldstat -q -m erofs -capture guest-dmesg-after-load dmesg -test "$(cat "$evidence/guest-kldstat-module-after-load.rc")" = 0 - -printf 'id\texpected_errno\tactual_errno\treject\n' \ - >"$evidence/qemu-matrix.tsv" -while IFS="$(printf '\t')" read -r id expected_errno reject; do - test "$id" != id || continue - set +e - md=$(guest mdconfig -a -t vnode -f "/root/B21-fixtures/$id.erofs" \ - 2>"$evidence/$id-md.stderr") - md_rc=$? - set -e - printf '%s\n' "$md_rc" >"$evidence/$id-md.rc" - test "$md_rc" = 0 - md_units="$md_units $md" - set +e - probe_result=$(guest /root/B21-qemu-probe "/dev/$md" /mnt/pre15-b21 \ - 2>"$evidence/$id-probe.stderr") - probe_rc=$? - set -e - printf '%s\n' "$probe_rc" >"$evidence/$id-probe.rc" - printf '%s\n' "$probe_result" >"$evidence/$id-probe.stdout" - actual_errno=$(printf '%s\n' "$probe_result" | \ - sed -n 's/.*errno=\([0-9][0-9]*\).*/\1/p') - test -n "$actual_errno" - printf '%s\t%s\t%s\t%s\n' "$id" "$expected_errno" "$actual_errno" \ - "$reject" >>"$evidence/qemu-matrix.tsv" - test "$actual_errno" = "$expected_errno" - if test "$actual_errno" = 0; then - mounted=1 - guest find /mnt/pre15-b21 -mindepth 1 -maxdepth 1 -print \ - >"$evidence/$id-readdir.txt" - guest umount /mnt/pre15-b21 - mounted=0 - fi - guest mdconfig -d -u "$md" - md_units=$(printf '%s\n' "$md_units" | sed "s/ $md//") -done <"$evidence/matrix.tsv" - -capture guest-dmesg-after dmesg -capture guest-kldstat-after kldstat -capture guest-kldstat-module-after kldstat -q -m erofs -diff -u "$evidence/guest-dmesg-before.stdout" \ - "$evidence/guest-dmesg-after.stdout" >"$evidence/dmesg.diff" || true -if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free' \ - "$evidence/dmesg.diff"; then - exit 10 -fi -guest kldunload /root/B21-erofs-zstdio0.ko >"$evidence/guest-kldunload.stdout" \ - 2>"$evidence/guest-kldunload.stderr" -loaded=0 -capture guest-kldstat-module-clean kldstat -q -m erofs -capture guest-mount-clean mount -p -capture guest-md-clean mdconfig -l -test "$(cat "$evidence/guest-kldstat-module-clean.rc")" = 1 -test ! -s "$evidence/guest-md-clean.stdout" -test -z "$(awk '$3 == "erofs" { print }' "$evidence/guest-mount-clean.stdout")" -printf '%s\n' reached >"$evidence/target.marker" -printf '%s\n' 'B21_PASS TC172-super-trust 13/13' >"$evidence/RESULT.txt" diff --git a/tests/pre15/fixtures/B21-qemu-probe.c b/tests/pre15/fixtures/B21-qemu-probe.c deleted file mode 100644 index 5e5abb2..0000000 --- a/tests/pre15/fixtures/B21-qemu-probe.c +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -int -main(int argc, char **argv) -{ - struct iovec options[6]; - int result; - - if (argc != 3) { - fprintf(stderr, "usage: B21-qemu-probe DEVICE MOUNTPOINT\n"); - return (2); - } - options[0].iov_base = "fstype"; - options[0].iov_len = sizeof("fstype"); - options[1].iov_base = "erofs"; - options[1].iov_len = sizeof("erofs"); - options[2].iov_base = "fspath"; - options[2].iov_len = sizeof("fspath"); - options[3].iov_base = argv[2]; - options[3].iov_len = strlen(argv[2]) + 1; - options[4].iov_base = "from"; - options[4].iov_len = sizeof("from"); - options[5].iov_base = argv[1]; - options[5].iov_len = strlen(argv[1]) + 1; - errno = 0; - result = nmount(options, sizeof(options) / sizeof(options[0]), MNT_RDONLY); - printf("rc=%d errno=%d\n", result, result == -1 ? errno : 0); - return (0); -} diff --git a/tests/pre15/fixtures/B21-super-generate.py b/tests/pre15/fixtures/B21-super-generate.py deleted file mode 100644 index 5261717..0000000 --- a/tests/pre15/fixtures/B21-super-generate.py +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/env python3 -"""Generate deterministic B21 superblock trust-order fixtures.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import shutil -import struct -import subprocess - - -SUPER = 1024 -FEATURE_COMPAT_SB_CHKSUM = 0x00000001 -FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040 -UNKNOWN_INCOMPAT = 0x80000000 -CRC32C_POLY = 0x82F63B78 - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def write_json(path: Path, value: object) -> None: - path.write_text( - json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - - -class Image: - def __init__(self, data: bytes | bytearray): - self.data = bytearray(data) - if len(self.data) < SUPER + 144: - raise ValueError("image is shorter than the extended superblock") - if self.u32(SUPER) != 0xE0F5E1E2: - raise ValueError("image has the wrong EROFS magic") - if not self.u32(SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM: - raise ValueError("image does not declare a superblock checksum") - - @classmethod - def load(cls, path: Path) -> "Image": - return cls(path.read_bytes()) - - def clone(self) -> "Image": - return Image(self.data) - - def u16(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - struct.pack_into(" None: - struct.pack_into(" int: - return self.data[SUPER + 12] - - @property - def block_size(self) -> int: - return 1 << self.block_bits - - @property - def checksum_end(self) -> int: - span = self.block_size - if span > SUPER: - span -= SUPER - return SUPER + span - - def checksum_valid(self) -> bool: - expected = self.u32(SUPER + 4) - canonical = bytearray(self.data[SUPER : self.checksum_end]) - struct.pack_into(" None: - self.put_u32(SUPER + 4, 0) - checksum = crc32c(self.data[SUPER : self.checksum_end]) - self.put_u32(SUPER + 4, checksum) - if not self.checksum_valid(): - raise AssertionError("updated B21 checksum does not verify") - - def inode_offset(self, nid: int) -> int: - return (self.u32(SUPER + 40) << self.block_bits) + (nid << 5) - - def fix_packed_inode_mode(self) -> None: - packed_nid = self.u64(SUPER + 96) - if packed_nid == 0: - raise ValueError("B21 seed did not materialize a packed prefix carrier") - self.put_u16(self.inode_offset(packed_nid) + 4, 0o100644) - - -def load_spec(path: Path) -> dict: - spec = json.loads(path.read_text(encoding="ascii")) - if spec.get("schema") != 1 or spec.get("batch") != "B21": - raise ValueError("invalid B21 fixture spec identity") - if not isinstance(spec.get("cases"), list) or not spec["cases"]: - raise ValueError("B21 fixture spec has no cases") - return spec - - -def create_source(root: Path) -> None: - root.mkdir(parents=True) - control = root / "control.txt" - prefix = root / "prefix.bin" - control.write_bytes(b"B21 superblock control\n") - prefix.write_bytes(b"B21 prefix control\n") - control.chmod(0o644) - prefix.chmod(0o644) - os.setxattr(prefix, b"user.company.branch.leaf", b"B21-prefix-value") - for path in (control, prefix, root): - os.utime(path, (0, 0), follow_symlinks=False) - - -def build_seed(spec: dict, output: Path, work: Path) -> dict[str, object]: - mkfs = shutil.which("mkfs.erofs") - if mkfs is None: - raise SystemExit("mkfs.erofs is required for B21 fixtures") - version_run = subprocess.run( - [mkfs, "-V"], check=False, text=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) - version = version_run.stdout.splitlines()[0] if version_run.stdout else "" - if version != spec["seed"]["mkfs_version"]: - raise SystemExit( - f"mkfs.erofs version differs: expected {spec['seed']['mkfs_version']!r}, " - f"got {version!r}" - ) - source = work / "source" - create_source(source) - command = [ - mkfs, - "-d0", - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "--sort=path", - f"-U{spec['seed']['uuid']}", - "-x2", - "-Eforce-inode-extended", - "--xattr-prefix=user.company.", - str(output), - str(source), - ] - completed = subprocess.run( - command, check=False, text=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) - if completed.returncode != 0: - raise SystemExit( - f"mkfs.erofs failed with {completed.returncode}:\n{completed.stdout}" - ) - image = Image.load(output) - image.fix_packed_inode_mode() - image.update_checksum() - output.write_bytes(image.data) - if not image.u32(SUPER + 80) & FEATURE_INCOMPAT_XATTR_PREFIXES: - raise ValueError("B21 seed does not declare xattr prefixes") - return {"version": version} - - -def mutate(image: Image, mutation: str) -> list[dict[str, int | str]]: - changes: list[dict[str, int | str]] = [] - - def record(field: str, offset: int, size: int) -> None: - changes.append({"field": field, "offset": offset, "size": size}) - - if mutation == "none": - return changes - if mutation in {"unknown-feature", "feature-prefix"}: - offset = SUPER + 80 - image.put_u32(offset, image.u32(offset) | UNKNOWN_INCOMPAT) - record("feature_incompat", offset, 4) - if mutation == "blocks": - offset = SUPER + 36 - image.put_u32(offset, 0xFFFFFFFF) - record("blocks_lo", offset, 4) - elif mutation == "extslots": - offset = SUPER + 13 - image.data[offset] = 0xFF - record("sb_extslots", offset, 1) - elif mutation == "dirblk": - offset = SUPER + 90 - image.data[offset] = 1 - record("dirblkbits", offset, 1) - elif mutation in {"prefix-offset", "feature-prefix"}: - offset = SUPER + 92 - image.put_u32(offset, 0xFFFFFFFF) - record("xattr_prefix_start", offset, 4) - elif mutation != "unknown-feature": - raise ValueError(f"unknown B21 mutation: {mutation}") - return changes - - -def generate(spec_path: Path, output: Path, work: Path) -> int: - spec = load_spec(spec_path) - if output.exists() or work.exists(): - raise SystemExit("B21 output and work paths must not already exist") - output.mkdir(parents=True) - work.mkdir(parents=True) - seed_path = work / "seed.erofs" - build = build_seed(spec, seed_path, work / "seed-work") - seed = Image.load(seed_path) - seed_hash = sha256(seed_path) - cases = [] - for item in spec["cases"]: - image = seed.clone() - changes = mutate(image, item["mutation"]) - if item["authenticated"] and item["mutation"] != "none": - image.update_checksum() - path = output / f"{item['id']}.erofs" - path.write_bytes(image.data) - checksum_valid = image.checksum_valid() - if checksum_valid != item["authenticated"]: - raise AssertionError(f"checksum state mismatch for {item['id']}") - cases.append( - { - **item, - "changes": changes, - "checksum_valid": checksum_valid, - "filename": path.name, - "sha256": sha256(path), - "size": path.stat().st_size, - } - ) - index = { - "batch": "B21", - "build": build, - "case_count": len(cases), - "cases": cases, - "damaged_count": sum(item["class"] == "damaged" for item in cases), - "legal_count": sum(item["class"] == "legal" for item in cases), - "schema": 1, - "seed_sha256": seed_hash, - } - write_json(output / "fixture-index.json", index) - print(json.dumps({"status": "PASS", **index}, sort_keys=True)) - return 0 - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--spec", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--work", type=Path, required=True) - args = parser.parse_args() - return generate(args.spec, args.output, args.work) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/pre15/fixtures/B21-super-oracle.py b/tests/pre15/fixtures/B21-super-oracle.py deleted file mode 100644 index 0109d2f..0000000 --- a/tests/pre15/fixtures/B21-super-oracle.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 -"""Independent B21 superblock ordering oracle.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -import struct - - -SUPER = 1024 -PAGE_SHIFT = 12 -FEATURE_COMPAT_SB_CHKSUM = 0x00000001 -FEATURE_INCOMPAT_ALL = 0x000001FF -FEATURE_INCOMPAT_48BIT = 0x00000080 -FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040 -CRC32C_POLY = 0x82F63B78 - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -def u16(data: bytes, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" bool: - block_bits = data[SUPER + 12] - if not 9 <= block_bits <= PAGE_SHIFT: - return False - span = 1 << block_bits - if span > SUPER: - span -= SUPER - end = SUPER + span - expected = u32(data, SUPER + 4) - return expected == crc32c(data[SUPER + 8 : end], 0x5045B54A) - - -def classify(data: bytes) -> tuple[int, str]: - if u32(data, SUPER) != 0xE0F5E1E2: - return 22, "magic" - block_bits = data[SUPER + 12] - if not 9 <= block_bits <= PAGE_SHIFT: - return 22, "block-size" - feature_compat = u32(data, SUPER + 8) - if feature_compat & FEATURE_COMPAT_SB_CHKSUM and not checksum_valid(data): - return 97, "checksum" - if data[SUPER + 90] != 0: - return 45, "dirblkbits" - feature_incompat = u32(data, SUPER + 80) - if feature_incompat & ~FEATURE_INCOMPAT_ALL: - if u32(data, SUPER + 92) == 0xFFFFFFFF: - return 45, "feature-before-prefix" - return 45, "feature-incompat" - if 128 + data[SUPER + 13] * 16 > (1 << PAGE_SHIFT) - SUPER: - return 22, "super-extension" - blocks = u32(data, SUPER + 36) - root_nid_8b = u64(data, SUPER + 112) - if feature_incompat & FEATURE_INCOMPAT_48BIT and root_nid_8b != 0: - blocks |= u16(data, SUPER + 14) << 32 - if blocks == 0: - return 97, "blocks-zero" - if blocks << block_bits > len(data): - return 6, "provider-size" - if ( - feature_incompat & FEATURE_INCOMPAT_XATTR_PREFIXES - and data[SUPER + 91] != 0 - and u32(data, SUPER + 92) == 0xFFFFFFFF - ): - return 97, "prefix-offset" - return 0, "accepted" - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--fixtures", type=Path, required=True) - parser.add_argument("--report", type=Path, required=True) - args = parser.parse_args() - index = json.loads( - (args.fixtures / "fixture-index.json").read_text(encoding="ascii") - ) - results = [] - for item in index["cases"]: - path = args.fixtures / item["filename"] - actual_errno, actual_reject = classify(path.read_bytes()) - passed = ( - actual_errno == item["expected_errno"] - and actual_reject == item["expected_reject"] - ) - results.append( - { - "actual_errno": actual_errno, - "actual_reject": actual_reject, - "expected_errno": item["expected_errno"], - "expected_reject": item["expected_reject"], - "id": item["id"], - "passed": passed, - } - ) - report = { - "batch": "B21", - "case_count": len(results), - "passed_count": sum(item["passed"] for item in results), - "results": results, - "schema": 1, - "status": "PASS" if all(item["passed"] for item in results) else "FAIL", - } - args.report.write_text( - json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - for item in results: - print( - f"{'PASS' if item['passed'] else 'FAIL'} {item['id']} " - f"errno={item['actual_errno']} reject={item['actual_reject']}" - ) - return 0 if report["status"] == "PASS" else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/pre15/fixtures/B21-super-spec.json b/tests/pre15/fixtures/B21-super-spec.json deleted file mode 100644 index e0464d0..0000000 --- a/tests/pre15/fixtures/B21-super-spec.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "batch": "B21", - "candidates": [ - "P15-048", - "P15-049", - "P15-072" - ], - "cases": [ - { - "authenticated": true, - "class": "legal", - "expected_errno": 0, - "expected_reject": "accepted", - "id": "legal-control", - "mutation": "none" - }, - { - "authenticated": false, - "class": "damaged", - "expected_errno": 97, - "expected_reject": "checksum", - "id": "unknown-feature-unauthenticated", - "mutation": "unknown-feature" - }, - { - "authenticated": true, - "class": "damaged", - "expected_errno": 45, - "expected_reject": "feature-incompat", - "id": "unknown-feature-authenticated", - "mutation": "unknown-feature" - }, - { - "authenticated": false, - "class": "damaged", - "expected_errno": 97, - "expected_reject": "checksum", - "id": "blocks-unauthenticated", - "mutation": "blocks" - }, - { - "authenticated": true, - "class": "damaged", - "expected_errno": 6, - "expected_reject": "provider-size", - "id": "blocks-authenticated", - "mutation": "blocks" - }, - { - "authenticated": false, - "class": "damaged", - "expected_errno": 97, - "expected_reject": "checksum", - "id": "extslots-unauthenticated", - "mutation": "extslots" - }, - { - "authenticated": true, - "class": "damaged", - "expected_errno": 22, - "expected_reject": "super-extension", - "id": "extslots-authenticated", - "mutation": "extslots" - }, - { - "authenticated": false, - "class": "damaged", - "expected_errno": 97, - "expected_reject": "checksum", - "id": "dirblk-unauthenticated", - "mutation": "dirblk" - }, - { - "authenticated": true, - "class": "damaged", - "expected_errno": 45, - "expected_reject": "dirblkbits", - "id": "dirblk-authenticated", - "mutation": "dirblk" - }, - { - "authenticated": false, - "class": "damaged", - "expected_errno": 97, - "expected_reject": "checksum", - "id": "prefix-offset-unauthenticated", - "mutation": "prefix-offset" - }, - { - "authenticated": true, - "class": "damaged", - "expected_errno": 97, - "expected_reject": "prefix-offset", - "id": "prefix-offset-authenticated", - "mutation": "prefix-offset" - }, - { - "authenticated": false, - "class": "damaged", - "expected_errno": 97, - "expected_reject": "checksum", - "id": "feature-prefix-unauthenticated", - "mutation": "feature-prefix" - }, - { - "authenticated": true, - "class": "damaged", - "expected_errno": 45, - "expected_reject": "feature-before-prefix", - "id": "feature-prefix-authenticated", - "mutation": "feature-prefix" - } - ], - "freebsd_errno": { - "EINVAL": 22, - "EINTEGRITY": 97, - "ENXIO": 6, - "EOPNOTSUPP": 45 - }, - "schema": 1, - "seed": { - "mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6", - "uuid": "00000000-0000-0000-0000-000000000321" - } -} diff --git a/tests/pre15/fixtures/B22-build-kld.sh b/tests/pre15/fixtures/B22-build-kld.sh deleted file mode 100755 index e88c0c7..0000000 --- a/tests/pre15/fixtures/B22-build-kld.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/sh -set -eu - -dut=${1:?DUT path is required} -freebsd_src=${2:?FreeBSD source path is required} -output=${3:?output module path is required} -work=${4:?work directory is required} -src=$dut/src -sys=$freebsd_src/sys -target=x86_64-unknown-freebsd15.0 - -test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; } -for tool in awk clang grep nm; do - command -v "$tool" >/dev/null 2>&1 || { - printf '%s\n' "missing B22 KLD tool: $tool" >&2 - exit 21 - } -done -test ! -e "$work" || { printf '%s\n' "B22 KLD work path exists: $work" >&2; exit 20; } -mkdir "$work" -cd "$work" -ln -s "$sys/amd64/include" machine -ln -s "$sys/x86/include" x86 -ln -s "$sys/i386/include" i386 -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h -: > opt_global.h - -cflags="-O2 -pipe -fno-common -fno-strict-aliasing \ --D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \ --I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \ --mno-red-zone -mno-mmx -mno-sse -msoft-float \ --fno-asynchronous-unwind-tables -ffreestanding -fwrapv \ --fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \ --Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \ --Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \ --Wno-unknown-pragmas -Wno-address-of-packed-member \ --Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \ --D__printf__=__freebsd_kprintf__ --target=$target" - -for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \ - decompressor.c zmap.c zdata.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do - extra= - if test "$file" = decompressor_zstd.c; then - extra="-I$sys/contrib/zstd/lib/freebsd" - fi - clang $cflags $extra -c "$src/$file" -o "${file%.c}.o" -done - -clang --target="$target" -r -nostdlib ./*.o -o "$output" -if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then - : -else - printf '%s\n' 'B22 KLD contains an unresolved bcmp reference' >&2 - exit 1 -fi -if nm -u "$output" | grep -q ' ZSTD_'; then - printf '%s\n' 'B22 KLD contains an unexpected Zstd symbol' >&2 - exit 1 -fi diff --git a/tests/pre15/fixtures/B22-zmap-arithmetic.json b/tests/pre15/fixtures/B22-zmap-arithmetic.json deleted file mode 100644 index e5001cd..0000000 --- a/tests/pre15/fixtures/B22-zmap-arithmetic.json +++ /dev/null @@ -1,320 +0,0 @@ -{ - "batch": "B22", - "candidates": [ - "P15-047", - "P15-059", - "P15-071", - "P15-079", - "P15-084" - ], - "cases": [ - { - "base": 4294967295, - "expected": 4294967296, - "mutated_fields": [], - "name": "compact-pblk-crosses-u32", - "nblk": 1, - "operation": "compact_pblk", - "status": "PASS", - "target_marker": "TC170:compact-pblk-32bit-crossing" - }, - { - "base": 4294967295, - "expected": 4294967310, - "mutated_fields": [], - "name": "compact-pblk-widened-run", - "nblk": 15, - "operation": "compact_pblk", - "status": "PASS", - "target_marker": "TC170:compact-pblk-32bit-crossing" - }, - { - "expected": 4168, - "inode_isize": 64, - "inode_off": 4096, - "mutated_fields": [], - "name": "index-base-aligned", - "operation": "index_base", - "status": "PASS", - "target_marker": "TC170:index-position" - }, - { - "expected": 4144, - "inode_isize": 32, - "inode_off": 4097, - "mutated_fields": [], - "name": "index-base-roundup", - "operation": "index_base", - "status": "PASS", - "target_marker": "TC170:index-position", - "xattr_isize": 4 - }, - { - "inode_isize": 32, - "inode_off": 18446744073709551584, - "mutated_fields": [ - "inode_off" - ], - "name": "index-base-add-overflow", - "operation": "index_base", - "status": "EINTEGRITY", - "target_marker": "TC170:index-position" - }, - { - "inode_isize": 0, - "inode_off": 18446744073709551609, - "mutated_fields": [ - "inode_off" - ], - "name": "index-base-roundup-overflow", - "operation": "index_base", - "status": "EINTEGRITY", - "target_marker": "TC170:index-position" - }, - { - "count": 5, - "expected": 4136, - "mutated_fields": [], - "name": "index-advance-normal", - "operation": "index_advance", - "position": 4096, - "status": "PASS", - "target_marker": "TC170:index-position", - "unit": 8 - }, - { - "count": 1, - "expected": 18446744073709551615, - "mutated_fields": [], - "name": "index-advance-last-byte", - "operation": "index_advance", - "position": 18446744073709551608, - "status": "PASS", - "target_marker": "TC170:index-position", - "unit": 7 - }, - { - "count": 1, - "mutated_fields": [ - "unit" - ], - "name": "index-advance-add-overflow", - "operation": "index_advance", - "position": 18446744073709551608, - "status": "EINTEGRITY", - "target_marker": "TC170:index-position", - "unit": 8 - }, - { - "count": 2305843009213693952, - "mutated_fields": [ - "count" - ], - "name": "index-advance-multiply-overflow", - "operation": "index_advance", - "position": 0, - "status": "EINTEGRITY", - "target_marker": "TC170:index-position", - "unit": 8 - }, - { - "expected": 3, - "lclusterbits": 12, - "mutated_fields": [], - "name": "lcluster-count-roundup", - "operation": "lcluster_count", - "size": 8193, - "status": "PASS", - "target_marker": "TC170:delta-lcn" - }, - { - "expected": 4503599627370496, - "lclusterbits": 12, - "mutated_fields": [], - "name": "lcluster-count-u64-max", - "operation": "lcluster_count", - "size": 18446744073709551615, - "status": "PASS", - "target_marker": "TC170:delta-lcn" - }, - { - "lclusterbits": 64, - "mutated_fields": [ - "lclusterbits" - ], - "name": "lcluster-count-invalid-shift", - "operation": "lcluster_count", - "size": 4096, - "status": "EINTEGRITY", - "target_marker": "TC170:delta-lcn" - }, - { - "clusterofs": 4095, - "expected": 18446744073709551615, - "lclusterbits": 12, - "lcn": 4503599627370495, - "mutated_fields": [], - "name": "lcluster-last-byte", - "operation": "lcluster_pos", - "status": "PASS", - "target_marker": "TC170:delta-lcn" - }, - { - "clusterofs": 0, - "lclusterbits": 12, - "lcn": 4503599627370496, - "mutated_fields": [ - "lcn" - ], - "name": "lcluster-shift-overflow", - "operation": "lcluster_pos", - "status": "EINTEGRITY", - "target_marker": "TC170:delta-lcn" - }, - { - "clusterofs": 4096, - "lclusterbits": 12, - "lcn": 4503599627370495, - "mutated_fields": [ - "clusterofs" - ], - "name": "lcluster-add-overflow", - "operation": "lcluster_pos", - "status": "EINTEGRITY", - "target_marker": "TC170:delta-lcn" - }, - { - "delta": 7, - "expected": 18446744073709551615, - "lcn": 18446744073709551608, - "mutated_fields": [], - "name": "lcn-delta-last-value", - "operation": "lcn_advance", - "status": "PASS", - "target_marker": "TC170:delta-lcn" - }, - { - "delta": 1, - "lcn": 18446744073709551615, - "mutated_fields": [ - "delta" - ], - "name": "lcn-delta-overflow", - "operation": "lcn_advance", - "status": "EINTEGRITY", - "target_marker": "TC170:delta-lcn" - }, - { - "block_size": 4096, - "mutated_fields": [], - "name": "physical-last-48bit-block", - "operation": "physical_end", - "pa": 1152921504606842880, - "plen": 4096, - "status": "PASS", - "target_marker": "TC170:physical-end-48bit" - }, - { - "block_size": 4096, - "mutated_fields": [ - "pa" - ], - "name": "physical-one-byte-beyond-48bit", - "operation": "physical_end", - "pa": 1152921504606842881, - "plen": 4096, - "status": "EINTEGRITY", - "target_marker": "TC170:physical-end-48bit" - }, - { - "block_size": 4096, - "mutated_fields": [ - "plen" - ], - "name": "physical-u64-add-overflow", - "operation": "physical_end", - "pa": 18446744073709550592, - "plen": 1024, - "status": "EINTEGRITY", - "target_marker": "TC170:physical-end-48bit" - }, - { - "block_size": 65536, - "mutated_fields": [], - "name": "physical-64k-representable-end", - "operation": "physical_end", - "pa": 18446744073709547520, - "plen": 4095, - "status": "PASS", - "target_marker": "TC170:physical-end-48bit" - }, - { - "expected": 1, - "la": 4096, - "mutated_fields": [], - "name": "post-eof-at-eof", - "operation": "post_eof", - "size": 4096, - "status": "PASS", - "target_marker": "TC170:post-eof" - }, - { - "expected": 4097, - "la": 8192, - "mutated_fields": [], - "name": "post-eof-bounded", - "operation": "post_eof", - "size": 4096, - "status": "PASS", - "target_marker": "TC170:post-eof" - }, - { - "expected": 18446744073709551615, - "la": 18446744073709551615, - "mutated_fields": [], - "name": "post-eof-last-offset", - "operation": "post_eof", - "size": 1, - "status": "PASS", - "target_marker": "TC170:post-eof" - }, - { - "expected": 18446744073709551615, - "la": 18446744073709551615, - "mutated_fields": [], - "name": "post-eof-saturates-empty-inode", - "operation": "post_eof", - "size": 0, - "status": "PASS", - "target_marker": "TC170:post-eof" - }, - { - "expected": 18446744073709551615, - "high": 4294967295, - "low": 4294967295, - "mutated_fields": [], - "name": "fragment-offset-max-representable", - "operation": "fragment_offset", - "status": "PASS", - "target_marker": "TC170:fragment-high-bits" - }, - { - "high": 4294967296, - "low": 4294967295, - "mutated_fields": [ - "high" - ], - "name": "fragment-offset-high-unrepresentable", - "operation": "fragment_offset", - "status": "EINTEGRITY", - "target_marker": "TC170:fragment-high-bits" - } - ], - "errno": { - "corruption": 97, - "provider_io": "unchanged", - "sign": "positive" - }, - "schema": 1, - "test": "TC170-zmap-arithmetic" -} diff --git a/tests/pre15/fixtures/B23-build-kld.sh b/tests/pre15/fixtures/B23-build-kld.sh deleted file mode 100644 index 82a12ae..0000000 --- a/tests/pre15/fixtures/B23-build-kld.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/sh -set -eu - -dut=${1:?DUT path is required} -freebsd_src=${2:?FreeBSD source path is required} -output=${3:?output module path is required} -work=${4:?work directory is required} -src=$dut/src -sys=$freebsd_src/sys -target=x86_64-unknown-freebsd15.0 - -test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; } -for tool in awk clang grep nm; do - command -v "$tool" >/dev/null 2>&1 || { - printf '%s\n' "missing B23 KLD tool: $tool" >&2 - exit 21 - } -done -test ! -e "$work" || { printf '%s\n' "B23 KLD work path exists: $work" >&2; exit 20; } -mkdir "$work" -cd "$work" -ln -s "$sys/amd64/include" machine -ln -s "$sys/x86/include" x86 -ln -s "$sys/i386/include" i386 -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h -: > opt_global.h - -cflags="-O2 -pipe -fno-common -fno-strict-aliasing \ --D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \ --I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \ --mno-red-zone -mno-mmx -mno-sse -msoft-float \ --fno-asynchronous-unwind-tables -ffreestanding -fwrapv \ --fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \ --Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \ --Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \ --Wno-unknown-pragmas -Wno-address-of-packed-member \ --Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \ --D__printf__=__freebsd_kprintf__ --target=$target" - -for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \ - decompressor.c zmap.c zdata.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do - extra= - if test "$file" = decompressor_zstd.c; then - extra="-I$sys/contrib/zstd/lib/freebsd" - fi - clang $cflags $extra -c "$src/$file" -o "${file%.c}.o" -done - -clang --target="$target" -r -nostdlib ./*.o -o "$output" -if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then - : -else - printf '%s\n' 'B23 KLD contains an unresolved bcmp reference' >&2 - exit 1 -fi -if nm -u "$output" | grep -q ' ZSTD_'; then - printf '%s\n' 'B23 zstdio0 KLD contains an unexpected Zstd symbol' >&2 - exit 1 -fi diff --git a/tests/pre15/fixtures/B23-explicit-generate.py b/tests/pre15/fixtures/B23-explicit-generate.py deleted file mode 100644 index bc50fae..0000000 --- a/tests/pre15/fixtures/B23-explicit-generate.py +++ /dev/null @@ -1,538 +0,0 @@ -#!/usr/bin/env python3 -"""Generate deterministic EROFS explicit-extent fixtures for Pre15 B23.""" - -from __future__ import annotations - -import argparse -from dataclasses import dataclass -import hashlib -import json -import os -from pathlib import Path -import shutil -import stat -import struct -import subprocess - - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -FEATURE_COMPAT_SB_CHKSUM = 0x00000001 -FEATURE_INCOMPAT_METABOX = 0x00000100 -METABOX_NID_BIT = 1 << 63 -LAYOUT_FLAT_PLAIN = 0 -LAYOUT_COMPRESSED_FULL = 1 -Z_EROFS_ADVISE_EXTENTS = 0x0001 -BLOCK_SIZE = 4096 -CARRIER_BYTES = 512 * 1024 -CRC32C_POLYNOMIAL = 0x82F63B78 -FIXED_UUID = "23232323-1515-4234-8123-000000000023" - - -class FixtureError(RuntimeError): - pass - - -@dataclass(frozen=True) -class Inode: - nid: int - offset: int - inode_size: int - layout: int - size: int - start_block: int - - -@dataclass(frozen=True) -class DirectoryEntry: - name: bytes - nid: int - offset: int - - -def align(value: int, alignment: int) -> int: - return (value + alignment - 1) & -alignment - - -def sha256_bytes(data: bytes | bytearray) -> str: - return hashlib.sha256(data).hexdigest() - - -def crc32c(data: bytes | bytearray) -> int: - checksum = 0xFFFFFFFF - for byte in data: - checksum ^= byte - for _ in range(8): - checksum = (checksum >> 1) ^ ( - CRC32C_POLYNOMIAL if checksum & 1 else 0 - ) - return checksum & 0xFFFFFFFF - - -class Image: - def __init__(self, data: bytes | bytearray) -> None: - self.data = bytearray(data) - self.validate_super() - - @classmethod - def load(cls, path: Path) -> "Image": - return cls(path.read_bytes()) - - def clone(self) -> "Image": - return Image(self.data) - - def u16(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - self.data[offset] = value - - def put_u16(self, offset: int, value: int) -> None: - struct.pack_into(" None: - struct.pack_into(" None: - struct.pack_into(" int: - return self.data[SUPER + 12] - - @property - def block_size(self) -> int: - return 1 << self.block_bits - - @property - def meta_offset(self) -> int: - return self.u32(SUPER + 40) << self.block_bits - - @property - def root_nid(self) -> int: - return self.u16(SUPER + 14) - - @property - def blocks(self) -> int: - return self.u32(SUPER + 36) - - @property - def declared_size(self) -> int: - return self.blocks << self.block_bits - - def validate_super(self) -> None: - if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC: - raise FixtureError("invalid EROFS superblock") - if self.block_size != BLOCK_SIZE: - raise FixtureError("B23 requires a 4096-byte EROFS block") - if self.declared_size > len(self.data): - raise FixtureError("declared EROFS image exceeds provider bytes") - if self.u32(SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM: - window = bytearray(self.data[SUPER : self.block_size]) - expected = struct.unpack_from(" None: - if not self.u32(SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM: - raise FixtureError("B23 seed lacks the checksum feature") - self.put_u32(SUPER + 4, 0) - self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.block_size])) - - def inode(self, nid: int) -> Inode: - offset = self.meta_offset + (nid << 5) - if offset + 32 > self.declared_size: - raise FixtureError(f"inode {nid} exceeds the primary metadata") - inode_format = self.u16(offset) - inode_size = 64 if inode_format & 1 else 32 - size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8) - return Inode( - nid=nid, - offset=offset, - inode_size=inode_size, - layout=(inode_format >> 1) & 7, - size=size, - start_block=self.u32(offset + 16), - ) - - def inode_data(self, inode: Inode) -> tuple[bytes, int]: - if inode.layout == LAYOUT_FLAT_PLAIN: - start = inode.start_block << self.block_bits - end = start + inode.size - if end > self.declared_size: - raise FixtureError("plain inode data exceeds the image") - return bytes(self.data[start:end]), start - if inode.layout != 2: - raise FixtureError(f"unsupported host inode layout {inode.layout}") - start = inode.offset + inode.inode_size - end = start + inode.size - if end > self.declared_size: - raise FixtureError("inline inode data exceeds the image") - return bytes(self.data[start:end]), start - - def directory_entries(self, inode: Inode) -> list[DirectoryEntry]: - data, data_base = self.inode_data(inode) - entries: list[DirectoryEntry] = [] - for block_start in range(0, len(data), self.block_size): - block = data[block_start : block_start + self.block_size] - if len(block) < 12: - raise FixtureError("short directory block") - first_name = struct.unpack_from(" len(block): - raise FixtureError("invalid directory entry table") - count = first_name // 12 - for index in range(count): - item = index * 12 - name_start = struct.unpack_from(" tuple[DirectoryEntry, Inode]: - encoded = name.encode("ascii") - for entry in self.directory_entries(self.inode(self.root_nid)): - if entry.name == encoded: - return entry, self.inode(entry.nid) - raise FixtureError(f"root entry not found: {name}") - - def ensure_size(self, end: int) -> None: - rounded = align(end, self.block_size) - if len(self.data) < rounded: - self.data.extend(bytes(rounded - len(self.data))) - self.put_u32(SUPER + 36, rounded >> self.block_bits) - - -def normalize_times(root: Path) -> None: - for path in sorted(root.rglob("*"), reverse=True): - os.utime(path, (0, 0), follow_symlinks=False) - os.utime(root, (0, 0), follow_symlinks=False) - - -def payload_block() -> bytes: - return bytes((index * 37 + 11) & 0xFF for index in range(BLOCK_SIZE)) - - -def make_seed(output: Path, expected_version: str) -> Image: - source = output / ".source" - source.mkdir() - (source / "target.bin").write_bytes(b"placeholder\n") - (source / "payload.bin").write_bytes(payload_block()) - (source / "carrier.bin").write_bytes(bytes(CARRIER_BYTES)) - normalize_times(source) - version_output = subprocess.run( - ["mkfs.erofs", "-V"], check=True, text=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ).stdout.strip() - version = version_output.splitlines()[0] if version_output else "" - if version != expected_version: - raise FixtureError(f"mkfs version changed: {version!r}") - seed = output / ".seed.erofs" - command = [ - "mkfs.erofs", "-d0", "-x-1", "-T0", "--all-time", "--all-root", - "--workers=1", "--sort=path", "-U", FIXED_UUID, - "-E", "force-inode-extended", str(seed), str(source), - ] - subprocess.run(command, check=True) - shutil.rmtree(source) - image = Image.load(seed) - _, payload = image.resolve_root("payload.bin") - _, carrier = image.resolve_root("carrier.bin") - if payload.layout != LAYOUT_FLAT_PLAIN or payload.size != BLOCK_SIZE: - raise FixtureError("payload seed is not one plain block") - if carrier.layout != LAYOUT_FLAT_PLAIN or carrier.size != CARRIER_BYTES: - raise FixtureError("carrier seed is not the expected plain file") - return image - - -def write_inode(image: Image, offset: int, size: int) -> None: - image.data[offset : offset + 64] = bytes(64) - image.put_u16(offset, 1 | (LAYOUT_COMPRESSED_FULL << 1)) - image.put_u16(offset + 4, stat.S_IFREG | 0o444) - image.put_u64(offset + 8, size) - image.put_u32(offset + 16, 1) - image.put_u32(offset + 44, 1) - - -def logical_starts(case: dict[str, object]) -> tuple[list[int], int, list[int]]: - count = int(case["count"]) - mutation = case.get("mutation") - if case.get("logical_mode") == "high32": - starts = [0, 1 << 32, (1 << 32) + BLOCK_SIZE] - return starts, (1 << 32) + 2 * BLOCK_SIZE, starts[1:] - if mutation == "descending-lstart": - starts = [0, 2 * BLOCK_SIZE, 3 * BLOCK_SIZE] - return starts, 4 * BLOCK_SIZE, [] - starts = [index * BLOCK_SIZE for index in range(count)] - size = count * BLOCK_SIZE - probes = sorted({0, ((count // 2) * BLOCK_SIZE), size - BLOCK_SIZE}) - return starts, size, probes - - -def extent_span(record_size: int, count: int) -> int: - return 72 + (8 if record_size == 4 else 0) + record_size * count - - -def write_extent_table( - image: Image, - storage_base: int, - inode_offset: int, - record_size: int, - starts: list[int], - file_size: int, - payload_offset: int, -) -> dict[str, int]: - write_inode(image, storage_base + inode_offset, file_size) - header = align(inode_offset + 64, 8) - table = align(header + 8, record_size) - advise = Z_EROFS_ADVISE_EXTENTS | ({4: 0, 8: 1, 16: 2, 32: 3}[record_size] << 1) - struct.pack_into( - "> 32, - ) - records = table - if record_size == 4: - image.put_u64(storage_base + table, payload_offset) - records += 8 - for index, lstart in enumerate(starts): - offset = storage_base + records + index * record_size - image.data[offset : offset + record_size] = bytes(record_size) - image.put_u32(offset, BLOCK_SIZE) - if record_size >= 8: - image.put_u32(offset + 4, payload_offset & 0xFFFFFFFF) - if record_size >= 16: - image.put_u32(offset + 8, payload_offset >> 32) - image.put_u32(offset + 12, lstart & 0xFFFFFFFF) - if record_size == 32: - image.put_u32(offset + 16, lstart >> 32) - return { - "header": header, - "table": table, - "records": records, - "end": records + record_size * len(starts), - } - - -def enable_metabox(image: Image) -> None: - if image.meta_offset != 0: - raise FixtureError("B23 metabox seed expects metadata block zero") - metadata = bytes(image.data[: image.block_size]) - relocated = align(len(image.data), image.block_size) - image.ensure_size(relocated + image.block_size) - image.data[relocated : relocated + image.block_size] = metadata - image.put_u32(SUPER + 40, relocated >> image.block_bits) - image.put_u8(SUPER + 13, 1) - image.put_u32( - SUPER + 80, image.u32(SUPER + 80) | FEATURE_INCOMPAT_METABOX - ) - - -def save_case( - output: Path, - seed: Image, - case: dict[str, object], - payload_offset: int, -) -> dict[str, object]: - image = seed.clone() - backing = str(case["backing"]) - record_size = int(case["record_size"]) - starts, file_size, probes = logical_starts(case) - stored_count = len(starts) - mutation = case.get("mutation") - if backing == "primary": - inode_absolute = align(len(image.data), image.block_size) + 4000 - inode_offset = inode_absolute - storage_base = 0 - stored_end = inode_absolute + extent_span(record_size, stored_count) - image.ensure_size(stored_end) - target_entry, _ = image.resolve_root("target.bin") - nid_delta = inode_absolute - image.meta_offset - if nid_delta < 0 or nid_delta % 32: - raise FixtureError("primary synthetic inode is not NID aligned") - target_nid = nid_delta >> 5 - image.put_u64(target_entry.offset, target_nid) - carrier_inode_offset = None - carrier_size = None - elif backing == "metabox": - enable_metabox(image) - target_entry, _ = image.resolve_root("target.bin") - _, carrier = image.resolve_root("carrier.bin") - storage_base = carrier.start_block << image.block_bits - span = extent_span(record_size, stored_count) - inode_offset = ((CARRIER_BYTES - span) // 32) * 32 - carrier_size = inode_offset + span - if inode_offset < 4096 or carrier_size > CARRIER_BYTES: - raise FixtureError("metabox explicit table does not fit the carrier") - target_nid = METABOX_NID_BIT | (inode_offset >> 5) - image.put_u64(target_entry.offset, target_nid) - carrier_inode_offset = carrier.offset - image.put_u64(carrier.offset + 8, carrier_size) - image.put_u64(SUPER + 128, carrier.nid) - else: - raise FixtureError(f"unknown backing: {backing}") - - layout = write_extent_table( - image, storage_base, inode_offset, record_size, starts, file_size, - payload_offset, - ) - image.update_checksum() - baseline = image.clone() - mutation_record = None - if mutation == "missing-tail": - image.put_u32(storage_base + layout["header"], 2) - mutation_record = { - "field": "map_header.h_extents_lo", - "offset": storage_base + layout["header"], - "size": 4, - } - elif mutation == "huge-count": - image.put_u16(storage_base + layout["header"] + 6, 256) - mutation_record = { - "field": "map_header.h_extents_hi", - "offset": storage_base + layout["header"] + 6, - "size": 2, - } - elif mutation == "descending-lstart": - changed = storage_base + layout["records"] + 2 * record_size + 12 - image.put_u32(changed, BLOCK_SIZE) - mutation_record = { - "field": "extent[2].lstart_lo", - "offset": changed, - "size": 4, - } - starts[2] = BLOCK_SIZE - elif mutation == "tail-minus-one": - if carrier_inode_offset is None or carrier_size is None: - raise FixtureError("tail-minus-one requires metabox backing") - image.put_u64(carrier_inode_offset + 8, carrier_size - 1) - mutation_record = { - "field": "metabox_carrier.i_size", - "offset": carrier_inode_offset + 8, - "size": 8, - } - elif mutation is not None: - raise FixtureError(f"unknown mutation: {mutation}") - - name = str(case["name"]) - image_path = output / f"{name}.erofs" - image_path.write_bytes(image.data) - baseline_name = None - if mutation_record is not None: - baseline_name = f"{name}-baseline.erofs" - (output / baseline_name).write_bytes(baseline.data) - - declared_count = stored_count - if mutation == "missing-tail": - declared_count = 2 - elif mutation == "huge-count": - declared_count = 1 + (256 << 32) - table_bytes = record_size * declared_count - scan_calls = 1 if record_size <= 8 else ( - (table_bytes + 65535) // 65536 - if case["outcome"] == "PASS" else 1 - ) - return { - "backing": backing, - "baseline": baseline_name, - "carrier_inode_offset": carrier_inode_offset, - "carrier_size": carrier_size, - "declared_count": declared_count, - "expected_errno": 0 if case["outcome"] == "PASS" else 97, - "expected_scan_calls": scan_calls, - "file_size": file_size, - "header_image_offset": storage_base + layout["header"], - "header_logical_offset": layout["header"], - "image": image_path.name, - "image_sha256": sha256_bytes(image.data), - "inode_image_offset": storage_base + inode_offset, - "inode_logical_offset": inode_offset, - "lstarts": starts, - "mutation": mutation_record, - "payload_offset": payload_offset, - "probe_offsets": probes, - "record_image_offset": storage_base + layout["records"], - "record_logical_offset": layout["records"], - "record_size": record_size, - "stored_count": stored_count, - "table_end_logical": layout["end"], - "target_dirent_offset": target_entry.offset, - "target_marker": case["target_marker"], - "target_nid": target_nid, - } - - -def generate(spec_path: Path, output: Path) -> None: - spec = json.loads(spec_path.read_text(encoding="ascii")) - if spec.get("schema") != 1 or spec.get("batch") != "B23": - raise FixtureError("invalid B23 spec identity") - if output.exists() and any(output.iterdir()): - raise FixtureError(f"output is not empty: {output}") - output.mkdir(parents=True, exist_ok=True) - seed = make_seed(output, str(spec["mkfs_version"])) - _, payload = seed.resolve_root("payload.bin") - payload_offset = payload.start_block << seed.block_bits - if seed.data[payload_offset : payload_offset + BLOCK_SIZE] != payload_block(): - raise FixtureError("plain payload bytes changed") - - cases: dict[str, object] = {} - for case in spec["cases"]: - cases[str(case["name"])] = save_case( - output, seed, case, payload_offset - ) - (output / ".seed.erofs").unlink() - manifest = { - "batch": "B23", - "cases": cases, - "chunk_size": spec["chunk_size"], - "mkfs_version": spec["mkfs_version"], - "payload_sha256": sha256_bytes(payload_block()), - "schema": 1, - "test": spec["test"], - } - (output / "fixture-manifest.json").write_text( - json.dumps(manifest, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", - ) - with (output / "SHA256SUMS").open("w", encoding="ascii") as sums: - for path in sorted(output.glob("*.erofs")): - sums.write(f"{sha256_bytes(path.read_bytes())} {path.name}\n") - print( - f"B23 fixtures generated cases={len(cases)} " - f"images={len(list(output.glob('*.erofs')))}" - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--spec", required=True, type=Path) - parser.add_argument("--output", required=True, type=Path) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - generate(args.spec, args.output) - - -if __name__ == "__main__": - try: - main() - except (FixtureError, OSError, subprocess.CalledProcessError) as error: - raise SystemExit(f"B23 fixture generation failed: {error}") from error diff --git a/tests/pre15/fixtures/B23-explicit-oracle.py b/tests/pre15/fixtures/B23-explicit-oracle.py deleted file mode 100644 index b70d63b..0000000 --- a/tests/pre15/fixtures/B23-explicit-oracle.py +++ /dev/null @@ -1,316 +0,0 @@ -#!/usr/bin/env python3 -"""Independently verify B23 explicit-extent images and expectations.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import math -from pathlib import Path -import struct - - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -FEATURE_COMPAT_SB_CHKSUM = 0x00000001 -METABOX_NID_BIT = 1 << 63 -BLOCK_SIZE = 4096 -CRC32C_POLYNOMIAL = 0x82F63B78 - - -class OracleError(RuntimeError): - pass - - -def sha256(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def crc32c(data: bytes | bytearray) -> int: - checksum = 0xFFFFFFFF - for byte in data: - checksum ^= byte - for _ in range(8): - checksum = (checksum >> 1) ^ ( - CRC32C_POLYNOMIAL if checksum & 1 else 0 - ) - return checksum & 0xFFFFFFFF - - -def u16(data: bytes, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" tuple[int, int]: - if len(data) < SUPER + 144 or u32(data, SUPER) != MAGIC: - raise OracleError("invalid EROFS superblock") - block_bits = data[SUPER + 12] - if block_bits != 12: - raise OracleError("B23 image block size changed") - declared_size = u32(data, SUPER + 36) << block_bits - if declared_size > len(data): - raise OracleError("declared image exceeds provider") - if u32(data, SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM: - window = bytearray(data[SUPER : 1 << block_bits]) - expected = u32(window, 4) - struct.pack_into(" bytes: - return bytes((index * 37 + 11) & 0xFF for index in range(BLOCK_SIZE)) - - -def changed_offsets(left: bytes, right: bytes) -> list[int]: - if len(left) != len(right): - raise OracleError("negative and baseline provider sizes differ") - return [index for index, pair in enumerate(zip(left, right)) if pair[0] != pair[1]] - - -def verify_mutation(directory: Path, image: bytes, case: dict[str, object]) -> None: - mutation = case.get("mutation") - baseline_name = case.get("baseline") - if mutation is None: - if baseline_name is not None: - raise OracleError("positive case unexpectedly names a baseline") - return - if not isinstance(mutation, dict) or not isinstance(baseline_name, str): - raise OracleError("negative case lacks mutation evidence") - baseline = (directory / baseline_name).read_bytes() - differences = changed_offsets(image, baseline) - start = int(mutation["offset"]) - end = start + int(mutation["size"]) - if not differences or any(offset < start or offset >= end for offset in differences): - raise OracleError(f"mutation is not confined to {mutation['field']}") - - -def record_values( - image: bytes, case: dict[str, object] -) -> tuple[list[int], list[int], list[int]]: - record_size = int(case["record_size"]) - record_offset = int(case["record_image_offset"]) - stored_count = int(case["stored_count"]) - payload_offset = int(case["payload_offset"]) - plens: list[int] = [] - pstarts: list[int] = [] - lstarts: list[int] = [] - physical = u64(image, int(case["header_image_offset"]) + 8) if record_size == 4 else 0 - for index in range(stored_count): - offset = record_offset + index * record_size - if offset + record_size > len(image): - raise OracleError("stored record exceeds provider bytes") - plen = u32(image, offset) - if record_size == 4: - pstart = physical - physical += plen & ((2 * 1024 * 1024) - 1) - lstart = index * BLOCK_SIZE - else: - pstart = u32(image, offset + 4) - if record_size >= 16: - pstart |= u32(image, offset + 8) << 32 - lstart = u32(image, offset + 12) - if record_size == 32: - lstart |= u32(image, offset + 16) << 32 - else: - lstart = index * BLOCK_SIZE - plens.append(plen) - pstarts.append(pstart) - lstarts.append(lstart) - if any(plen != BLOCK_SIZE for plen in plens): - raise OracleError("mapped shifted extent length changed") - if any(pstart != payload_offset for pstart in pstarts): - raise OracleError("mapped shifted extent physical offset changed") - return plens, pstarts, lstarts - - -def verify_case( - directory: Path, - spec_case: dict[str, object], - case: dict[str, object], - chunk_size: int, -) -> dict[str, object]: - image_path = directory / str(case["image"]) - image = image_path.read_bytes() - block_size, declared_size = verify_super(image) - if sha256(image) != case["image_sha256"]: - raise OracleError(f"image hash mismatch: {image_path.name}") - if image[int(case["payload_offset"]) : int(case["payload_offset"]) + block_size] != expected_payload(): - raise OracleError(f"payload block mismatch: {image_path.name}") - verify_mutation(directory, image, case) - - if u64(image, int(case["target_dirent_offset"])) != int(case["target_nid"]): - raise OracleError("target dirent NID changed") - if case["backing"] == "metabox": - if not int(case["target_nid"]) & METABOX_NID_BIT: - raise OracleError("metabox target lacks bit-63 NID") - elif int(case["target_nid"]) & METABOX_NID_BIT: - raise OracleError("primary target unexpectedly has bit-63 NID") - - inode = int(case["inode_image_offset"]) - inode_format = u16(image, inode) - if inode_format != 3 or u16(image, inode + 4) != 0o100444: - raise OracleError("synthetic inode format or mode changed") - if u64(image, inode + 8) != int(case["file_size"]): - raise OracleError("synthetic inode size changed") - header = int(case["header_image_offset"]) - advise = u16(image, header + 4) - observed_record_size = 4 << ((advise >> 1) & 3) - if not advise & 1 or observed_record_size != int(case["record_size"]): - raise OracleError("explicit extent advise changed") - declared_count = u32(image, header) | (u16(image, header + 6) << 32) - if int(case["record_size"]) >= 16 and declared_count != int(case["declared_count"]): - raise OracleError("explicit extent count changed") - - _, _, observed_lstarts = record_values(image, case) - if observed_lstarts != case["lstarts"]: - raise OracleError("little-endian lstart decode differs from the manifest") - outcome = str(spec_case["outcome"]) - violations = [ - index for index in range(1, len(observed_lstarts)) - if observed_lstarts[index] <= observed_lstarts[index - 1] - ] - if spec_case.get("mutation") == "descending-lstart": - if violations != [2]: - raise OracleError("ordering negative lacks its single violation") - elif violations: - raise OracleError("non-ordering case has unordered logical starts") - - if outcome == "PASS": - if int(case["expected_errno"]) != 0: - raise OracleError("positive case has an errno") - if int(case["record_size"]) <= 8: - expected_calls = 1 - else: - expected_calls = math.ceil( - int(case["stored_count"]) * int(case["record_size"]) / chunk_size - ) - if int(case["expected_scan_calls"]) != expected_calls: - raise OracleError("bounded scan call count changed") - if int(case["stored_count"]) >= 4096 and expected_calls >= int(case["stored_count"]): - raise OracleError("large table regressed to per-record I/O") - elif int(case["expected_errno"]) != 97: - raise OracleError("corruption case does not use positive EINTEGRITY") - - mutation = spec_case.get("mutation") - if mutation == "missing-tail": - requested_end = int(case["record_logical_offset"]) + 2 * int(case["record_size"]) - if requested_end <= declared_size: - raise OracleError("primary missing-tail request remains provider-backed") - elif mutation == "tail-minus-one": - if int(case["carrier_size"]) != int(case["table_end_logical"]): - raise OracleError("metabox baseline does not end at the final record") - if u64(image, int(case["carrier_inode_offset"]) + 8) + 1 != int(case["table_end_logical"]): - raise OracleError("metabox tail is not exactly one byte short") - elif mutation == "huge-count": - if int(case["declared_count"]) <= (1 << 32): - raise OracleError("huge-count fixture does not exercise 48-bit count decode") - - return { - "backing": case["backing"], - "expected_scan_calls": case["expected_scan_calls"], - "name": spec_case["name"], - "outcome": outcome, - "record_size": case["record_size"], - "stored_count": case["stored_count"], - "target_marker": case["target_marker"], - } - - -def verify(spec_path: Path, directory: Path, report_path: Path) -> None: - spec = json.loads(spec_path.read_text(encoding="ascii")) - manifest = json.loads((directory / "fixture-manifest.json").read_text(encoding="ascii")) - if ( - spec.get("schema") != 1 - or spec.get("batch") != "B23" - or spec.get("test") != "TC171-explicit-extents" - or spec.get("candidates") != ["P15-060", "P15-085"] - ): - raise OracleError("B23 spec identity changed") - if ( - manifest.get("schema") != 1 - or manifest.get("batch") != "B23" - or manifest.get("test") != spec["test"] - or manifest.get("chunk_size") != spec["chunk_size"] - ): - raise OracleError("B23 manifest identity changed") - if spec.get("errno") != { - "corruption": 97, - "provider_io": "unchanged", - "sign": "positive", - }: - raise OracleError("B23 errno contract changed") - - cases = manifest.get("cases") - if not isinstance(cases, dict) or set(cases) != { - str(case["name"]) for case in spec["cases"] - }: - raise OracleError("B23 case set changed") - decoded = [] - markers = set() - for spec_case in spec["cases"]: - name = str(spec_case["name"]) - case = cases[name] - if case["target_marker"] != spec_case["target_marker"]: - raise OracleError("target marker changed") - markers.add(str(case["target_marker"])) - decoded.append( - verify_case(directory, spec_case, case, int(spec["chunk_size"])) - ) - if len(markers) != len(spec["cases"]): - raise OracleError("target markers are not unique") - if {int(case["record_size"]) for case in decoded} != {4, 8, 16, 32}: - raise OracleError("record-size coverage is incomplete") - if {str(case["backing"]) for case in decoded} != {"primary", "metabox"}: - raise OracleError("backing coverage is incomplete") - - report = { - "batch": "B23", - "case_count": len(decoded), - "cases": decoded, - "chunk_size": spec["chunk_size"], - "large_scan_calls": { - str(case["name"]): case["expected_scan_calls"] - for case in decoded if int(case["stored_count"]) >= 4096 - }, - "status": "PASS", - "test": spec["test"], - } - report_path.write_text( - json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", - ) - print( - f"B23 independent oracle PASS cases={len(decoded)} " - f"markers={len(markers)}" - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--spec", required=True, type=Path) - parser.add_argument("--fixtures", required=True, type=Path) - parser.add_argument("--report", required=True, type=Path) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - verify(args.spec, args.fixtures, args.report) - - -if __name__ == "__main__": - try: - main() - except (OracleError, OSError, ValueError, KeyError, struct.error) as error: - raise SystemExit(f"B23 oracle failed: {error}") from error diff --git a/tests/pre15/fixtures/B23-explicit-probe.c b/tests/pre15/fixtures/B23-explicit-probe.c deleted file mode 100644 index 1b62de6..0000000 --- a/tests/pre15/fixtures/B23-explicit-probe.c +++ /dev/null @@ -1,122 +0,0 @@ -#define _POSIX_C_SOURCE 200809L - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define B23_BLOCK_SIZE 4096 - -static int -parse_u64(const char *text, uint64_t *value) -{ - char *end; - uintmax_t parsed; - - errno = 0; - parsed = strtoumax(text, &end, 10); - if (errno != 0 || end == text || *end != '\0' || parsed > INT64_MAX) - return (-1); - *value = (uint64_t)parsed; - return (0); -} - -static int -check_payload(const unsigned char *data) -{ - size_t index; - - for (index = 0; index < B23_BLOCK_SIZE; index++) { - if (data[index] != (unsigned char)((index * 37 + 11) & 0xff)) - return (-1); - } - return (0); -} - -static int -check_pass(int argc, char **argv) -{ - unsigned char data[B23_BLOCK_SIZE]; - struct stat status; - uint64_t expected_size, offset; - ssize_t count; - int descriptor, index; - - if (argc < 5 || parse_u64(argv[3], &expected_size) != 0) - return (2); - if (stat(argv[2], &status) != 0) { - perror("stat positive explicit target"); - return (1); - } - if ((uint64_t)status.st_size != expected_size) { - fprintf(stderr, "size mismatch: got=%jd expected=%" PRIu64 "\n", - (intmax_t)status.st_size, expected_size); - return (1); - } - descriptor = open(argv[2], O_RDONLY); - if (descriptor < 0) { - perror("open positive explicit target"); - return (1); - } - for (index = 4; index < argc; index++) { - if (parse_u64(argv[index], &offset) != 0 || - offset > expected_size || B23_BLOCK_SIZE > expected_size - offset) { - fprintf(stderr, "invalid probe offset: %s\n", argv[index]); - close(descriptor); - return (2); - } - count = pread(descriptor, data, sizeof(data), (off_t)offset); - if (count != (ssize_t)sizeof(data) || check_payload(data) != 0) { - fprintf(stderr, "mapped payload mismatch at offset=%" PRIu64 - " count=%zd errno=%d\n", offset, count, errno); - close(descriptor); - return (1); - } - } - if (close(descriptor) != 0) { - perror("close positive explicit target"); - return (1); - } - return (0); -} - -static int -check_errno(int argc, char **argv) -{ - struct stat status; - uint64_t expected; - - if (argc != 4 || parse_u64(argv[3], &expected) != 0 || expected > INT_MAX) - return (2); - errno = 0; - if (stat(argv[2], &status) == 0) { - fprintf(stderr, "corrupt explicit target unexpectedly published\n"); - return (1); - } - if (errno != (int)expected) { - fprintf(stderr, "errno mismatch: got=%d expected=%" PRIu64 "\n", - errno, expected); - return (1); - } - return (0); -} - -int -main(int argc, char **argv) -{ - if (argc >= 2 && strcmp(argv[1], "pass") == 0) - return (check_pass(argc, argv)); - if (argc >= 2 && strcmp(argv[1], "errno") == 0) - return (check_errno(argc, argv)); - fprintf(stderr, "usage: %s pass PATH SIZE OFFSET... | errno PATH ERRNO\n", - argv[0]); - return (2); -} diff --git a/tests/pre15/fixtures/B23-explicit-spec.json b/tests/pre15/fixtures/B23-explicit-spec.json deleted file mode 100644 index 2457e29..0000000 --- a/tests/pre15/fixtures/B23-explicit-spec.json +++ /dev/null @@ -1,152 +0,0 @@ -{ - "batch": "B23", - "candidates": [ - "P15-060", - "P15-085" - ], - "cases": [ - { - "backing": "primary", - "count": 1, - "name": "primary-r4-mapped", - "outcome": "PASS", - "record_size": 4, - "target_marker": "TC171:primary-r4-tail-backed" - }, - { - "backing": "primary", - "count": 1, - "name": "primary-r8-mapped", - "outcome": "PASS", - "record_size": 8, - "target_marker": "TC171:primary-r8-tail-backed" - }, - { - "backing": "primary", - "count": 3, - "name": "primary-r16-mapped", - "outcome": "PASS", - "record_size": 16, - "target_marker": "TC171:primary-r16-mapped" - }, - { - "backing": "primary", - "count": 3, - "logical_mode": "high32", - "name": "primary-r32-lstart-hi", - "outcome": "PASS", - "record_size": 32, - "target_marker": "TC171:primary-r32-lstart-hi" - }, - { - "backing": "primary", - "count": 8192, - "name": "primary-r16-large", - "outcome": "PASS", - "record_size": 16, - "target_marker": "TC171:primary-r16-bounded-scan" - }, - { - "backing": "primary", - "count": 4097, - "name": "primary-r32-large", - "outcome": "PASS", - "record_size": 32, - "target_marker": "TC171:primary-r32-bounded-scan" - }, - { - "backing": "primary", - "count": 1, - "mutation": "missing-tail", - "name": "primary-r16-missing-tail", - "outcome": "EINTEGRITY", - "record_size": 16, - "target_marker": "TC171:primary-missing-tail" - }, - { - "backing": "primary", - "count": 1, - "mutation": "huge-count", - "name": "primary-r32-huge-count", - "outcome": "EINTEGRITY", - "record_size": 32, - "target_marker": "TC171:primary-huge-count" - }, - { - "backing": "primary", - "count": 3, - "mutation": "descending-lstart", - "name": "primary-r16-order", - "outcome": "EINTEGRITY", - "record_size": 16, - "target_marker": "TC171:primary-order" - }, - { - "backing": "metabox", - "count": 1, - "name": "metabox-r4-mapped", - "outcome": "PASS", - "record_size": 4, - "target_marker": "TC171:metabox-r4-tail-backed" - }, - { - "backing": "metabox", - "count": 1, - "name": "metabox-r8-mapped", - "outcome": "PASS", - "record_size": 8, - "target_marker": "TC171:metabox-r8-tail-backed" - }, - { - "backing": "metabox", - "count": 3, - "name": "metabox-r16-mapped", - "outcome": "PASS", - "record_size": 16, - "target_marker": "TC171:metabox-r16-mapped" - }, - { - "backing": "metabox", - "count": 3, - "name": "metabox-r32-mapped", - "outcome": "PASS", - "record_size": 32, - "target_marker": "TC171:metabox-r32-mapped" - }, - { - "backing": "metabox", - "count": 1, - "mutation": "tail-minus-one", - "name": "metabox-r4-tail-minus-one", - "outcome": "EINTEGRITY", - "record_size": 4, - "target_marker": "TC171:metabox-r4-tail-minus-one" - }, - { - "backing": "metabox", - "count": 1, - "mutation": "tail-minus-one", - "name": "metabox-r8-tail-minus-one", - "outcome": "EINTEGRITY", - "record_size": 8, - "target_marker": "TC171:metabox-r8-tail-minus-one" - }, - { - "backing": "metabox", - "count": 4097, - "name": "metabox-r16-large", - "outcome": "PASS", - "record_size": 16, - "target_marker": "TC171:metabox-bounded-scan" - } - ], - "chunk_size": 65536, - "errno": { - "corruption": 97, - "provider_io": "unchanged", - "sign": "positive" - }, - "mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6", - "schema": 1, - "test": "TC171-explicit-extents" -} diff --git a/tests/pre15/fixtures/B25-codec-errors.json b/tests/pre15/fixtures/B25-codec-errors.json deleted file mode 100644 index 7890e16..0000000 --- a/tests/pre15/fixtures/B25-codec-errors.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "candidate": "P15-023", - "config": [ - { - "class": "malformed", - "errno": "EINTEGRITY" - }, - { - "class": "unsupported", - "errno": "EOPNOTSUPP" - } - ], - "dispatch": [ - { - "backend_errno": "EINTEGRITY", - "expected": "EINTEGRITY" - }, - { - "backend_errno": "ENOMEM", - "expected": "ENOMEM" - }, - { - "backend_errno": "EIO", - "expected": "EIO" - } - ], - "schema": 1, - "xz": [ - { - "expected": "ENOMEM", - "status": "XZ_MEM_ERROR" - }, - { - "expected": "EOPNOTSUPP", - "status": "XZ_MEMLIMIT_ERROR" - }, - { - "expected": "EOPNOTSUPP", - "status": "XZ_OPTIONS_ERROR" - }, - { - "expected": "EOPNOTSUPP", - "status": "XZ_UNSUPPORTED_CHECK" - }, - { - "expected": "EINTEGRITY", - "status": "XZ_FORMAT_ERROR" - }, - { - "expected": "EINTEGRITY", - "status": "XZ_DATA_ERROR" - }, - { - "expected": "EINTEGRITY", - "status": "XZ_BUF_ERROR" - } - ], - "zlib": [ - { - "expected": "ENOMEM", - "status": "Z_MEM_ERROR" - }, - { - "expected": "EOPNOTSUPP", - "status": "Z_VERSION_ERROR" - }, - { - "expected": "EINTEGRITY", - "status": "Z_NEED_DICT" - }, - { - "expected": "EINTEGRITY", - "status": "Z_DATA_ERROR" - }, - { - "expected": "EINTEGRITY", - "status": "Z_BUF_ERROR" - }, - { - "expected": "EIO", - "status": "Z_STREAM_ERROR" - }, - { - "expected": "EIO", - "status": "Z_ERRNO" - } - ], - "zstd": [ - { - "expected": "ENOMEM", - "operation": "create_context" - }, - { - "expected": "EOPNOTSUPP", - "operation": "set_window_limit" - }, - { - "expected": "EINTEGRITY", - "operation": "decompress_stream" - }, - { - "expected": "EIO", - "operation": "free_context" - } - ] -} diff --git a/tests/pre15/fixtures/B25-codec-oracle.py b/tests/pre15/fixtures/B25-codec-oracle.py deleted file mode 100755 index 82146f9..0000000 --- a/tests/pre15/fixtures/B25-codec-oracle.py +++ /dev/null @@ -1,434 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -from pathlib import Path -import re -import subprocess - - -SOURCE_NAMES = ( - "compress.h", - "decompressor.c", - "decompressor_lz4.c", - "decompressor_lzma.c", - "decompressor_deflate.c", - "decompressor_zstd.c", - "zdata.c", -) - - -def fail(message: str) -> None: - raise SystemExit(message) - - -def replace_once(source: str, old: str, new: str, label: str) -> str: - count = source.count(old) - if count != 1: - fail(f"{label}: expected one source occurrence, found {count}") - return source.replace(old, new, 1) - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if not match: - fail(f"missing function: {name}") - name_line = source.rfind("\n", 0, match.start()) + 1 - start = source.rfind("\n", 0, name_line - 1) + 1 - brace = source.find("{", match.end()) - if brace < 0: - fail(f"missing function body: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - fail(f"unterminated function: {name}") - - -def committed(root: Path, baseline: str, name: str) -> str: - completed = subprocess.run( - ["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/src/{name}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - fail(f"cannot read B25 baseline {name}: {completed.stderr.strip()}") - return completed.stdout - - -def require_once(source: str, needle: str, label: str) -> None: - count = source.count(needle) - if count != 1: - fail(f"{label}: expected one occurrence, found {count}") - - -def verify_exact_small_transforms(current: dict[str, str], base: dict[str, str]) -> None: - expected = replace_once( - base["compress.h"], - "\tconst char *name;\n", - "\tconst char *name;\n" - "\t/* Callbacks return zero or a positive FreeBSD errno. */\n", - "callback errno contract", - ) - if current["compress.h"] != expected: - fail("compress.h differs from the declared callback contract annotation") - - expected = base["decompressor.c"] - expected = replace_once( - expected, - "\t\tif (size < sizeof(*lz4))\n\t\t\treturn (EINVAL);", - "\t\tif (size < sizeof(*lz4))\n\t\t\treturn (EINTEGRITY);", - "short LZ4 config", - ) - expected = replace_once( - expected, - "\t\t (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits))\n" - "\t\t\treturn (EINVAL);", - "\t\t (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits))\n" - "\t\t\treturn (EOPNOTSUPP);", - "unsupported LZ4 pcluster size", - ) - if expected.count("\t\treturn (EOVERFLOW);\n") != 2: - fail("baseline config arithmetic errno count changed") - expected = expected.replace( - "\t\treturn (EOVERFLOW);\n", "\t\treturn (EINTEGRITY);\n" - ) - expected = replace_once(expected, "\tint ret;\n", "", "folding temporary") - expected = replace_once( - expected, - "\tret = decompressor->decompress(&rq);\n" - "\treturn (ret == 0 ? 0 : EIO);", - "\treturn (decompressor->decompress(&rq));", - "dispatch preservation", - ) - if current["decompressor.c"] != expected: - fail("decompressor.c differs from the declared B25 dispatch/config transforms") - - expected = base["decompressor_lz4.c"] - if expected.count("return (-1);") != 12 or expected.count(": -1);") != 1: - fail("baseline LZ4 private-status count changed") - expected = expected.replace("return (-1);", "return (EINTEGRITY);") - expected = expected.replace(": -1);", ": EINTEGRITY);") - if current["decompressor_lz4.c"] != expected: - fail("LZ4 differs from the exact private-status conversion") - - if current["zdata.c"] != base["zdata.c"]: - fail("B25 changed the already-correct zdata I/O or cleanup path") - - -def verify_source_contracts( - source: dict[str, str], fixture: dict[str, object] -) -> dict[str, object]: - joined = "\n".join(source[name] for name in SOURCE_NAMES) - if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", joined): - fail("negative Linux errno entered the FreeBSD codec path") - for name in SOURCE_NAMES: - if "return (-1)" in source[name] or re.search(r":\s*-1\s*\)", source[name]): - fail(f"private -1 codec status remains in {name}") - - dispatch = extract_function(source["decompressor.c"], "z_erofs_decompress") - if dispatch.count("return (decompressor->decompress(&rq));") != 2: - fail("typed backend dispatch does not have two direct callback exits") - if "ret == 0 ? 0 : EIO" in dispatch: - fail("dispatcher still folds typed backend errors") - require_once(dispatch, "return (EOPNOTSUPP);", "unsupported algorithm") - if dispatch.count("return (EINTEGRITY);") != 2: - fail("dispatch-local corruption checks changed") - for case in fixture["dispatch"]: - if case.get("backend_errno") != case.get("expected"): - fail("dispatch fixture does not require exact errno preservation") - - config_expectations = { - "z_erofs_load_lz4_config": ("decompressor.c", "EINTEGRITY", "EOPNOTSUPP"), - "z_erofs_load_lzma_config": ( - "decompressor_lzma.c", - "EINTEGRITY", - "EOPNOTSUPP", - ), - "z_erofs_load_deflate_config": ( - "decompressor_deflate.c", - "EINTEGRITY", - "EOPNOTSUPP", - ), - "z_erofs_load_zstd_config": ( - "decompressor_zstd.c", - "EINTEGRITY", - "EOPNOTSUPP", - ), - } - for function, (name, malformed, unsupported) in config_expectations.items(): - body = extract_function(source[name], function) - if f"return ({malformed});" not in body: - fail(f"{function} lacks malformed-config errno") - if f"return ({unsupported});" not in body: - fail(f"{function} lacks unsupported-config errno") - if fixture["config"] != [ - {"class": "malformed", "errno": "EINTEGRITY"}, - {"class": "unsupported", "errno": "EOPNOTSUPP"}, - ]: - fail("config fixture does not encode the B25 typed classes") - - lzma = extract_function(source["decompressor_lzma.c"], "z_erofs_lzma_decompress") - require_once(lzma, "return (EOVERFLOW);", "LZMA ABI overflow") - require_once(lzma, "return (ENOMEM);", "LZMA allocation failure") - if not ( - lzma.index("xz_dec_microlzma_run") - < lzma.index("xz_dec_microlzma_end") - < lzma.index("return (error);") - ): - fail("LZMA state is not released before status publication") - - deflate = extract_function( - source["decompressor_deflate.c"], "z_erofs_deflate_decompress" - ) - require_once(deflate, "inflateEnd(&strm)", "Deflate cleanup") - require_once( - deflate, - "if (error == 0 && endret != Z_OK)", - "Deflate primary-error preservation", - ) - if deflate.index("inflateEnd(&strm)") > deflate.index("return (error);"): - fail("Deflate returns before releasing initialized state") - - zstd_source = source["decompressor_zstd.c"] - enabled = extract_function(zstd_source, "z_erofs_zstd_decompress") - require_once(enabled, "return (ENOMEM);", "Zstd allocation failure") - require_once( - enabled, - "if (error == 0 && ZSTD_isError(ret))", - "Zstd primary-error preservation", - ) - require_once(enabled, "error = EOPNOTSUPP;", "Zstd parameter mapping") - if enabled.count("error = EINTEGRITY;") != 4: - fail("Zstd stream corruption/completion mappings changed") - require_once(enabled, "error = EIO;", "Zstd release mapping") - if "ZSTD_getErrorCode" in zstd_source: - fail("B25 added a Zstd provider symbol outside the existing ABI") - if zstd_source.count("return (EOPNOTSUPP);") < 3: - fail("Zstd config/runtime disabled paths are not typed unsupported") - expected_zstd = { - "create_context": "ENOMEM", - "set_window_limit": "EOPNOTSUPP", - "decompress_stream": "EINTEGRITY", - "free_context": "EIO", - } - actual_zstd = { - case.get("operation"): case.get("expected") for case in fixture["zstd"] - } - if actual_zstd != expected_zstd or len(fixture["zstd"]) != len(expected_zstd): - fail("Zstd phase fixture does not match the source contract") - - read_extent = extract_function(source["zdata.c"], "z_erofs_read_extent") - decode_at = read_extent.index("error = z_erofs_decompress") - meta_release_at = read_extent.index("erofs_put_metabuf(&buf)", decode_at) - physical_release_at = read_extent.index("erofs_brelse(compressed)", decode_at) - error_at = read_extent.index("if (error != 0)", decode_at) - free_at = read_extent.index("free(decoded, M_EROFS)", error_at) - if not ( - decode_at < meta_release_at < error_at < free_at - and decode_at < physical_release_at < error_at < free_at - ): - fail("zdata input/output cleanup ordering changed") - - return { - "callbacks": 2, - "config_loaders": len(config_expectations), - "dispatch_direct": True, - "dispatch_fixture_cases": len(fixture["dispatch"]), - "negative_errno": False, - "zdata_unchanged": True, - } - - -def compile_status_harness( - source: dict[str, str], fixture: dict[str, object], artifacts: Path -) -> dict[str, int]: - xz_helper = extract_function(source["decompressor_lzma.c"], "z_erofs_lzma_error") - zlib_helper = extract_function( - source["decompressor_deflate.c"], "z_erofs_deflate_error" - ) - checks: list[str] = [] - for group, function in ( - ("xz", "z_erofs_lzma_error"), - ("zlib", "z_erofs_deflate_error"), - ): - for case in fixture[group]: - checks.append( - f'\tcheck("{group}:{case["status"]}", ' - f'{function}({case["status"]}), {case["expected"]});' - ) - program = f'''#include -#include - -#define EIO 5 -#define ENOMEM 12 -#define EOVERFLOW 75 -#define EOPNOTSUPP 95 -#define EINTEGRITY 97 - -enum xz_ret {{ -\tXZ_OK, -\tXZ_STREAM_END, -\tXZ_UNSUPPORTED_CHECK, -\tXZ_MEM_ERROR, -\tXZ_MEMLIMIT_ERROR, -\tXZ_FORMAT_ERROR, -\tXZ_OPTIONS_ERROR, -\tXZ_DATA_ERROR, -\tXZ_BUF_ERROR -}}; - -#define Z_OK 0 -#define Z_STREAM_END 1 -#define Z_NEED_DICT 2 -#define Z_ERRNO (-1) -#define Z_STREAM_ERROR (-2) -#define Z_DATA_ERROR (-3) -#define Z_MEM_ERROR (-4) -#define Z_BUF_ERROR (-5) -#define Z_VERSION_ERROR (-6) - -{xz_helper} - -{zlib_helper} - -static int failures; - -static void -check(const char *name, int actual, int expected) -{{ -\tif (actual != expected) {{ -\t\tfprintf(stderr, "%s: %d != %d\\n", name, actual, expected); -\t\t++failures; -\t}} -}} - -int -main(void) -{{ -{chr(10).join(checks)} -\tif (failures != 0) -\t\treturn (1); -\tprintf("status-map cases={len(checks)}\\n"); -\treturn (0); -}} -''' - harness = artifacts / "B25-status-map.c" - binary = artifacts / "B25-status-map" - harness.write_text(program, encoding="ascii") - compile_run = subprocess.run( - ["cc", "-std=c11", "-Wall", "-Wextra", "-Werror", str(harness), "-o", str(binary)], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - (artifacts / "B25-status-map-build.stdout").write_text( - compile_run.stdout, encoding="utf-8" - ) - (artifacts / "B25-status-map-build.stderr").write_text( - compile_run.stderr, encoding="utf-8" - ) - if compile_run.returncode != 0: - fail("B25 status-map harness did not compile") - execute = subprocess.run( - [str(binary)], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - (artifacts / "B25-status-map.stdout").write_text(execute.stdout, encoding="utf-8") - (artifacts / "B25-status-map.stderr").write_text(execute.stderr, encoding="utf-8") - if execute.returncode != 0: - fail("B25 status-map harness failed") - binary.unlink() - return { - "xz": len(fixture["xz"]), - "zlib": len(fixture["zlib"]), - "zstd": len(fixture["zstd"]), - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--artifacts", required=True, type=Path) - parser.add_argument("--baseline", required=True) - parser.add_argument("--dut", required=True, type=Path) - parser.add_argument("--fixture", required=True, type=Path) - parser.add_argument("--root", required=True, type=Path) - args = parser.parse_args() - - fixture = json.loads(args.fixture.read_text(encoding="ascii")) - if fixture.get("schema") != 1 or fixture.get("candidate") != "P15-023": - fail("invalid B25 fixture identity") - for key in ("config", "dispatch", "xz", "zlib", "zstd"): - if not isinstance(fixture.get(key), list) or not fixture[key]: - fail(f"empty B25 fixture group: {key}") - - source_dir = args.dut / "src" - current = { - name: (source_dir / name).read_text(encoding="utf-8") for name in SOURCE_NAMES - } - base = {name: committed(args.root, args.baseline, name) for name in SOURCE_NAMES} - verify_exact_small_transforms(current, base) - contracts = verify_source_contracts(current, fixture) - mappings = compile_status_harness(current, fixture, args.artifacts) - - report = { - "candidate": "P15-023", - "contracts": contracts, - "mapping_cases": mappings, - "status": "PASS", - } - (args.artifacts / "B25-codec-errors.json").write_text( - json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", - ) - print( - "B25 codec errno contract: " - f"xz={mappings['xz']} zlib={mappings['zlib']} zstd={mappings['zstd']}" - ) - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B27-build-kld.sh b/tests/pre15/fixtures/B27-build-kld.sh deleted file mode 100755 index f26ca8a..0000000 --- a/tests/pre15/fixtures/B27-build-kld.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh -set -eu - -dut=${1:?DUT path is required} -freebsd_src=${2:?FreeBSD source path is required} -output=${3:?output module path is required} -work=${4:?work directory is required} -zstdio=${5:?WITH_ZSTDIO value is required} -src=$dut/src -sys=$freebsd_src/sys -target=x86_64-unknown-freebsd15.0 - -case "$zstdio" in -0|1) ;; -*) printf '%s\n' 'WITH_ZSTDIO must be 0 or 1' >&2; exit 20 ;; -esac -test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; } -for tool in awk clang cmp grep nm; do - command -v "$tool" >/dev/null 2>&1 || { - printf '%s\n' "missing B27 KLD tool: $tool" >&2 - exit 21 - } -done -test ! -e "$work" || { printf '%s\n' "B27 KLD work path exists: $work" >&2; exit 20; } -mkdir "$work" -cd "$work" -ln -s "$sys/amd64/include" machine -ln -s "$sys/x86/include" x86 -ln -s "$sys/i386/include" i386 -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h -: > opt_global.h - -cflags="-O2 -pipe -fno-common -fno-strict-aliasing \ --D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \ --I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \ --mno-red-zone -mno-mmx -mno-sse -msoft-float \ --fno-asynchronous-unwind-tables -ffreestanding -fwrapv \ --fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \ --Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \ --Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \ --Wno-unknown-pragmas -Wno-address-of-packed-member \ --Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \ --D__printf__=__freebsd_kprintf__ --target=$target" - -for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \ - decompressor.c zmap.c zdata.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do - extra= - if test "$file" = decompressor_zstd.c; then - extra="-I$sys/contrib/zstd/lib/freebsd" - test "$zstdio" = 0 || extra="$extra -DZSTDIO" - fi - printf 'CC WITH_ZSTDIO=%s %s\n' "$zstdio" "$file" - clang $cflags $extra -c "$src/$file" -o "${file%.c}.o" -done - -clang --target="$target" -r -nostdlib ./*.o -o "$output" -nm -u "$output" | LC_ALL=C sort > B27-nm-u.txt -if ! nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then - printf '%s\n' 'B27 KLD contains an unresolved bcmp reference' >&2 - exit 1 -fi -if test "$zstdio" = 0; then - if grep -q ' ZSTD_' B27-nm-u.txt; then - printf '%s\n' 'B27 zstdio0 KLD contains an unexpected Zstd symbol' >&2 - exit 1 - fi -else - awk '$NF ~ /^ZSTD_/ { print $NF }' B27-nm-u.txt > B27-zstd-symbols.txt - printf '%s\n' ZSTD_DCtx_setParameter ZSTD_createDCtx_advanced \ - ZSTD_decompressStream ZSTD_freeDCtx ZSTD_isError > B27-zstd-expected.txt - if ! cmp B27-zstd-expected.txt B27-zstd-symbols.txt; then - printf '%s\n' 'B27 zstdio1 KLD Zstd ABI changed' >&2 - exit 1 - fi -fi diff --git a/tests/pre15/fixtures/B27-compact-finalization.c b/tests/pre15/fixtures/B27-compact-finalization.c deleted file mode 100644 index 30e5ec8..0000000 --- a/tests/pre15/fixtures/B27-compact-finalization.c +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include - -#define Z_EROFS_LI_D0_CBLKCNT 0x800U -#define Z_EROFS_LCLUSTER_TYPE_NONHEAD 2U - -static unsigned int -compact_head_nblocks(const uint16_t *values, const uint8_t *types, int index) -{ - unsigned int blocks, value; - - blocks = 0; - while (index > 0) { - --index; - value = values[index]; - if (types[index] == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { - if ((value & Z_EROFS_LI_D0_CBLKCNT) != 0) { - --index; - blocks += value & ~Z_EROFS_LI_D0_CBLKCNT; - continue; - } - if (value <= 1) - return (UINT32_MAX); - index -= value - 2; - continue; - } - ++blocks; - } - return (blocks); -} - -int -main(void) -{ - uint16_t values[16] = { 0 }; - uint8_t types[16] = { 0 }; - unsigned int blocks; - - values[0] = Z_EROFS_LI_D0_CBLKCNT | 1U; - types[0] = Z_EROFS_LCLUSTER_TYPE_NONHEAD; - values[4] = 5U; - types[4] = Z_EROFS_LCLUSTER_TYPE_NONHEAD; - blocks = compact_head_nblocks(values, types, 5); - if (blocks != 1U) { - fprintf(stderr, "compact HEAD finalization blocks=%u expected=1\n", - blocks); - return (1); - } - return (0); -} diff --git a/tests/pre15/fixtures/B27-stream-fixtures.py b/tests/pre15/fixtures/B27-stream-fixtures.py deleted file mode 100755 index fc63f5b..0000000 --- a/tests/pre15/fixtures/B27-stream-fixtures.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import shutil -import subprocess - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def load_spec(path: Path) -> dict[str, object]: - spec = json.loads(path.read_text(encoding="ascii")) - if ( - spec.get("schema") != 1 - or spec.get("batch") != "B27" - or spec.get("candidate") != "P15-083" - or spec.get("test") != "TC176-stream-runtime" - ): - raise SystemExit("invalid B27 stream fixture spec") - if set(spec.get("codecs", {})) != {"lzma", "deflate", "zstd"}: - raise SystemExit("B27 codec set changed") - return spec - - -def make_source(root: Path, spec: dict[str, object]) -> bytes: - root.mkdir(parents=True) - source_spec = spec["source"] - content = b"".join( - f"P15-083-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode("ascii") - for index in range(source_spec["line_count"]) - ) - if len(content) != source_spec["size"] or sha256_bytes(content) != source_spec["sha256"]: - raise SystemExit("B27 deterministic payload identity changed") - payload = root / "payload.bin" - payload.write_bytes(content) - os.utime(payload, (0, 0)) - os.utime(root, (0, 0)) - return content - - -def run(argv: list[str], cwd: Path) -> None: - completed = subprocess.run( - argv, - cwd=cwd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=120, - ) - if completed.returncode != 0: - raise SystemExit( - f"command failed ({completed.returncode}): {' '.join(argv)}\n{completed.stdout}" - ) - - -def build_codec( - codec: str, - codec_spec: dict[str, object], - spec: dict[str, object], - source_dir: Path, - images: Path, -) -> list[dict[str, object]]: - valid = images / f"{codec}-valid.erofs" - run(["mkfs.erofs", *codec_spec["mkfs_args"], str(valid), str(source_dir)], images) - valid_data = valid.read_bytes() - if ( - len(valid_data) != codec_spec["expected_image_size"] - or sha256_bytes(valid_data) != codec_spec["expected_image_sha256"] - ): - raise SystemExit(f"{codec} valid image is not the G04 reproducible fixture") - - extent = codec_spec["extent"] - start = extent["physical_offset"] - end = start + extent["physical_length"] - block = valid_data[start:end] - leading = next((index for index, value in enumerate(block) if value), len(block)) - stream = block[leading:] - if leading != extent["leading_zero_bytes"] or len(stream) != extent["stream_bytes"]: - raise SystemExit(f"{codec} pcluster padding/stream boundary changed") - - tail = bytes.fromhex(spec["tail_bytes_hex"]) - if not tail or any(value == 0 for value in tail) or leading <= len(tail): - raise SystemExit("B27 trailing mutation is not nonzero or lacks pcluster room") - variants: dict[str, bytearray] = {} - tail_data = bytearray(valid_data) - tail_data[start + leading - len(tail) : end] = stream + tail - variants["tail"] = tail_data - - truncated_data = bytearray(valid_data) - truncated_data[start + leading : end] = b"\0" + stream[:-1] - variants["truncated"] = truncated_data - - corruption_start = codec_spec["corruption_start"] - if not (codec_spec["gate"]["partial_consumed"] < corruption_start < len(stream)): - raise SystemExit(f"{codec} corruption is not after partial oracle consumption") - corrupt_data = bytearray(valid_data) - corrupt_data[start + leading : end] = ( - stream[:corruption_start] + b"\0" * (len(stream) - corruption_start) - ) - if corrupt_data == valid_data: - raise SystemExit(f"{codec} corruption mutation changed no bytes") - variants["corrupt"] = corrupt_data - - records = [ - { - "class": "valid", - "codec": codec, - "expected_errno": 0, - "path": valid.name, - "sha256": sha256_path(valid), - } - ] - for kind, data in variants.items(): - path = images / f"{codec}-{kind}.erofs" - path.write_bytes(data) - records.append( - { - "class": kind, - "codec": codec, - "expected_errno": 0 if kind == "corrupt-partial" else 97, - "path": path.name, - "sha256": sha256_path(path), - } - ) - return records - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--spec", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - spec = load_spec(args.spec) - if args.output.exists(): - raise SystemExit(f"refusing existing B27 output: {args.output}") - if shutil.which("mkfs.erofs") is None: - raise SystemExit("mkfs.erofs is required") - args.output.mkdir(parents=True) - source_dir = args.output / "source" - images = args.output / "images" - images.mkdir() - make_source(source_dir, spec) - records = [] - for codec in sorted(spec["codecs"]): - records.extend( - build_codec(codec, spec["codecs"][codec], spec, source_dir, images) - ) - index = { - "batch": "B27", - "candidate": "P15-083", - "fixture_count": len(records), - "fixtures": records, - "schema": 1, - "source_sha256": spec["source"]["sha256"], - "status": "READY", - "test": "TC176-stream-runtime", - } - (args.output / "fixture-index.json").write_text( - json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - print(json.dumps(index, indent=2, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B27-stream-probe.c b/tests/pre15/fixtures/B27-stream-probe.c deleted file mode 100644 index cebbe68..0000000 --- a/tests/pre15/fixtures/B27-stream-probe.c +++ /dev/null @@ -1,122 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -static int -parse_offset(const char *text, off_t *value) -{ - char *end; - long long parsed; - - errno = 0; - parsed = strtoll(text, &end, 10); - if (errno != 0 || end == text || *end != '\0' || parsed < 0) - return (-1); - *value = (off_t)parsed; - return (0); -} - -static int -parse_size(const char *text, size_t *value) -{ - char *end; - unsigned long parsed; - - errno = 0; - parsed = strtoul(text, &end, 10); - if (errno != 0 || end == text || *end != '\0' || parsed == 0 || - parsed > 1024 * 1024) - return (-1); - *value = (size_t)parsed; - return (0); -} - -static int -check_range(const char *path, const char *reference, off_t offset, size_t size) -{ - unsigned char *actual, *expected; - ssize_t actual_count, expected_count; - int actual_fd, expected_fd, result; - - actual = malloc(size); - expected = malloc(size); - if (actual == NULL || expected == NULL) - return (1); - actual_fd = open(path, O_RDONLY); - expected_fd = open(reference, O_RDONLY); - if (actual_fd < 0 || expected_fd < 0) { - perror("open B27 range input"); - return (1); - } - actual_count = pread(actual_fd, actual, size, offset); - expected_count = pread(expected_fd, expected, size, offset); - result = actual_count == (ssize_t)size && expected_count == (ssize_t)size && - memcmp(actual, expected, size) == 0 ? 0 : 1; - if (result != 0) - fprintf(stderr, "range mismatch offset=%jd size=%zu actual=%zd expected=%zd errno=%d\n", - (intmax_t)offset, size, actual_count, expected_count, errno); - close(actual_fd); - close(expected_fd); - free(actual); - free(expected); - return (result); -} - -static int -check_errno(const char *path, int expected_errno) -{ - unsigned char buffer[4096]; - ssize_t count; - int descriptor; - - descriptor = open(path, O_RDONLY); - if (descriptor < 0) { - if (errno == expected_errno) - return (0); - perror("open B27 damaged payload"); - return (1); - } - for (;;) { - errno = 0; - count = read(descriptor, buffer, sizeof(buffer)); - if (count < 0) { - int actual_errno = errno; - - close(descriptor); - if (actual_errno == expected_errno) - return (0); - fprintf(stderr, "read errno=%d expected=%d\n", actual_errno, - expected_errno); - return (1); - } - if (count == 0) { - close(descriptor); - fprintf(stderr, "damaged stream reached clean EOF\n"); - return (1); - } - } -} - -int -main(int argc, char **argv) -{ - off_t offset; - size_t size; - - if (argc == 6 && strcmp(argv[1], "range") == 0 && - parse_offset(argv[4], &offset) == 0 && - parse_size(argv[5], &size) == 0) - return (check_range(argv[2], argv[3], offset, size)); - if (argc == 4 && strcmp(argv[1], "errno") == 0) - return (check_errno(argv[2], atoi(argv[3]))); - fprintf(stderr, "usage: %s range PATH REFERENCE OFFSET SIZE | errno PATH ERRNO\n", - argv[0]); - return (2); -} diff --git a/tests/pre15/fixtures/B27-stream-tail.json b/tests/pre15/fixtures/B27-stream-tail.json deleted file mode 100644 index 7ca9533..0000000 --- a/tests/pre15/fixtures/B27-stream-tail.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "baseline": "029f3bf821d7bb5fabcf43d942ea6e9e40159ba6", - "batch": "B27", - "candidate": "P15-083", - "codecs": { - "deflate": { - "corruption_start": 2553, - "expected_image_sha256": "39455150c3e999bb7ae6c36402c15a408a5679726b6d099e7d121e009ef43af6", - "expected_image_size": 32768, - "extent": { - "leading_zero_bytes": 1479, - "logical_length": 14348, - "logical_offset": 137204, - "partial_size": 3587, - "physical_length": 4096, - "physical_offset": 28672, - "stream_bytes": 2617 - }, - "gate": { - "full_consumed": 2617, - "partial_consumed": 715, - "tail_consumed": 2617, - "tail_fsck_exit": 0, - "truncated_consumed": 2616 - }, - "mkfs_args": [ - "-T0", - "-U20000000-0000-4000-8000-000000000083", - "--all-root", - "-x-1", - "--workers=1", - "-zdeflate,level=1,dictsize=32768", - "-C4096" - ] - }, - "lzma": { - "corruption_start": 476, - "expected_image_sha256": "c26cf15844fe45a21a746bacbad2ef551c683bb9b2538de7a7eced37d8eadff9", - "expected_image_size": 8192, - "extent": { - "leading_zero_bytes": 3556, - "logical_length": 151552, - "logical_offset": 0, - "partial_size": 4096, - "physical_length": 4096, - "physical_offset": 4096, - "stream_bytes": 540 - }, - "gate": { - "full_consumed": 540, - "partial_consumed": 352, - "tail_consumed": 540, - "tail_fsck_exit": 1, - "truncated_consumed": 539 - }, - "mkfs_args": [ - "-T0", - "-U10000000-0000-4000-8000-000000000083", - "--all-root", - "-x-1", - "--workers=1", - "-zlzma,level=6,dictsize=65536", - "-C4096" - ] - }, - "zstd": { - "corruption_start": 2940, - "expected_image_sha256": "b905803f0e08500cc3c6cfe07a95fe027859165acae19ca8865856af256f32ca", - "expected_image_size": 8192, - "extent": { - "leading_zero_bytes": 1092, - "logical_length": 151552, - "logical_offset": 0, - "partial_size": 4096, - "physical_length": 4096, - "physical_offset": 4096, - "stream_bytes": 3004 - }, - "gate": { - "full_consumed": 3004, - "partial_consumed": 1457, - "tail_consumed": 3004, - "tail_fsck_exit": 1, - "truncated_consumed": 3003 - }, - "mkfs_args": [ - "-T0", - "-U30000000-0000-4000-8000-000000000083", - "--all-root", - "-x-1", - "--workers=1", - "-zzstd,level=3,dictsize=65536", - "-C4096" - ] - } - }, - "errno": { - "integrity": 97, - "success": 0 - }, - "gate": { - "commit": "338ba8daf81c1461b5d28ca9c5345686e4e0f2eb", - "input_sha256": "b3c71b4ae03c6f9511246813952aab3178a1451433ece96f04562ec7cc6c1d1e", - "required_base": "68bbe94c44e35d53cec8ab55d007f40b01cf0502", - "script_sha256": "6847541266b0668ad12442ffb54b67a772a29ed8ebe6fba0e1e25214db16b2c2" - }, - "schema": 1, - "source": { - "line_count": 4096, - "sha256": "f9ebc3ccae455cd9a01afae784c9bbbc28c45936880f444fdf2c8b64dff44bfc", - "size": 151552 - }, - "tail_bytes_hex": "a55ac33c96696996", - "test": "TC176-stream-runtime" -} diff --git a/tests/pre15/fixtures/B28-build-kld.sh b/tests/pre15/fixtures/B28-build-kld.sh deleted file mode 100755 index a42d3db..0000000 --- a/tests/pre15/fixtures/B28-build-kld.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/sh -set -eu - -dut=${1:?DUT path is required} -freebsd_src=${2:?FreeBSD source path is required} -output=${3:?output module path is required} -work=${4:?work directory is required} -zstdio=${5:?WITH_ZSTDIO value is required} -src=$dut/src -sys=$freebsd_src/sys -target=x86_64-unknown-freebsd15.0 - -case "$zstdio" in -0|1) ;; -*) printf '%s\n' 'WITH_ZSTDIO must be 0 or 1' >&2; exit 20 ;; -esac -test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; } -for tool in awk clang cmp grep nm; do - command -v "$tool" >/dev/null 2>&1 || { - printf '%s\n' "missing B28 KLD tool: $tool" >&2 - exit 21 - } -done -test ! -e "$work" || { printf '%s\n' "B28 KLD work path exists: $work" >&2; exit 20; } -mkdir "$work" -cd "$work" -ln -s "$sys/amd64/include" machine -ln -s "$sys/x86/include" x86 -ln -s "$sys/i386/include" i386 -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q -awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h -: > opt_global.h - -cflags="-O2 -pipe -fno-common -fno-strict-aliasing \ --D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \ --I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \ --mno-red-zone -mno-mmx -mno-sse -msoft-float \ --fno-asynchronous-unwind-tables -ffreestanding -fwrapv \ --fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \ --Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \ --Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \ --Wno-unknown-pragmas -Wno-address-of-packed-member \ --Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \ --D__printf__=__freebsd_kprintf__ --target=$target" - -for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \ - decompressor.c zmap.c zdata.c decompressor_lz4.c \ - decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do - extra= - if test "$file" = decompressor_zstd.c; then - extra="-I$sys/contrib/zstd/lib/freebsd" - test "$zstdio" = 0 || extra="$extra -DZSTDIO" - fi - printf 'CC WITH_ZSTDIO=%s %s\n' "$zstdio" "$file" - clang $cflags $extra -c "$src/$file" -o "${file%.c}.o" -done - -clang --target="$target" -r -nostdlib ./*.o -o "$output" -nm -u "$output" | LC_ALL=C sort > B28-nm-u.txt -nm -g "$output" | LC_ALL=C sort > B28-nm-global.txt -if ! awk '$NF == "z_erofs_decompress_supports_subextent" { found = 1 } END { exit !found }' \ - B28-nm-global.txt; then - printf '%s\n' 'B28 partial capability symbol is absent' >&2 - exit 1 -fi -if ! nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then - printf '%s\n' 'B28 KLD contains an unresolved bcmp reference' >&2 - exit 1 -fi -if test "$zstdio" = 0; then - if grep -q ' ZSTD_' B28-nm-u.txt; then - printf '%s\n' 'B28 zstdio0 KLD contains an unexpected Zstd symbol' >&2 - exit 1 - fi -else - awk '$NF ~ /^ZSTD_/ { print $NF }' B28-nm-u.txt > B28-zstd-symbols.txt - printf '%s\n' ZSTD_DCtx_setParameter ZSTD_createDCtx_advanced \ - ZSTD_decompressStream ZSTD_freeDCtx ZSTD_isError > B28-zstd-expected.txt - if ! cmp B28-zstd-expected.txt B28-zstd-symbols.txt; then - printf '%s\n' 'B28 zstdio1 KLD Zstd ABI changed' >&2 - exit 1 - fi -fi diff --git a/tests/pre15/fixtures/B28-partial-fixtures.py b/tests/pre15/fixtures/B28-partial-fixtures.py deleted file mode 100755 index e44a1f3..0000000 --- a/tests/pre15/fixtures/B28-partial-fixtures.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import shutil -import subprocess -import tempfile - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def load_spec(path: Path) -> dict[str, object]: - spec = json.loads(path.read_text(encoding="ascii")) - if ( - spec.get("schema") != 1 - or spec.get("batch") != "B28" - or spec.get("candidate") != "P15-086" - or spec.get("test") != "TC176-stream-runtime" - ): - raise SystemExit("invalid B28 partial fixture spec") - if set(spec.get("codecs", {})) != {"lz4", "lzma", "deflate", "zstd"}: - raise SystemExit("B28 codec set changed") - return spec - - -def make_sources(root: Path, spec: dict[str, object]) -> dict[str, bytes]: - sources: dict[str, bytes] = {} - lz4_spec = spec["sources"]["lz4"] - sources["lz4"] = b"".join( - bytes([segment["byte"]]) * segment["length"] - for segment in lz4_spec["segments"] - ) - stream_spec = spec["sources"]["stream"] - sources["stream"] = b"".join( - f"P15-086-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode( - "ascii" - ) - for index in range(stream_spec["line_count"]) - ) - for name, content in sources.items(): - expected = spec["sources"][name] - if len(content) != expected["size"] or sha256_bytes(content) != expected["sha256"]: - raise SystemExit(f"B28 {name} source identity changed") - directory = root / name - directory.mkdir(parents=True) - payload = directory / "payload.bin" - payload.write_bytes(content) - os.utime(payload, (0, 0)) - os.utime(directory, (0, 0)) - return sources - - -def run(argv: list[str], cwd: Path, expected: set[int] = {0}) -> subprocess.CompletedProcess[str]: - completed = subprocess.run( - argv, - cwd=cwd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=120, - ) - if completed.returncode not in expected: - raise SystemExit( - f"command failed ({completed.returncode}): {' '.join(argv)}\n{completed.stdout}" - ) - return completed - - -def verify_fsck(image: Path, expected: bytes, label: str) -> None: - with tempfile.TemporaryDirectory(prefix=f"b28-{label}-fsck-") as temporary: - destination = Path(temporary) - run(["fsck.erofs", f"--extract={destination}", str(image)], image.parent) - if (destination / "payload.bin").read_bytes() != expected: - raise SystemExit(f"B28 {label} fsck extraction mismatch") - - -def build_codec( - codec: str, - codec_spec: dict[str, object], - sources: dict[str, bytes], - source_root: Path, - images: Path, -) -> list[dict[str, object]]: - valid = images / f"{codec}-valid.erofs" - run( - [ - "mkfs.erofs", - *codec_spec["mkfs_args"], - str(valid), - str(source_root / codec_spec["source"]), - ], - images, - ) - valid_data = valid.read_bytes() - actual_image_sha256 = sha256_bytes(valid_data) - source = sources[codec_spec["source"]] - verify_fsck(valid, source, f"{codec}-valid") - - extent = codec_spec["extent"] - start = extent["physical_offset"] - end = start + extent["physical_length"] - block = valid_data[start:end] - leading = next((index for index, value in enumerate(block) if value), len(block)) - stream = block[leading:] - if ( - leading != extent["leading_zero_bytes"] - or len(stream) != extent["stream_bytes"] - or not (extent["prefix_consumed"] < codec_spec["corruption_start"] < len(stream)) - ): - raise SystemExit(f"{codec} pcluster or corruption boundary changed") - - corrupted_data = bytearray(valid_data) - corruption = start + leading + codec_spec["corruption_start"] - corrupted_data[corruption:end] = b"\0" * (end - corruption) - if corrupted_data == valid_data: - raise SystemExit(f"{codec} corruption mutation changed no bytes") - corrupted = images / f"{codec}-corrupt.erofs" - corrupted.write_bytes(corrupted_data) - with tempfile.TemporaryDirectory(prefix=f"b28-{codec}-corrupt-") as temporary: - failed = run( - ["fsck.erofs", f"--extract={temporary}", str(corrupted)], - images, - set(range(1, 256)), - ) - - return [ - { - "class": "valid", - "codec": codec, - "expected_errno": 0, - "gate_image_sha256": codec_spec["gate_image_sha256"], - "path": valid.name, - "sha256": actual_image_sha256, - }, - { - "class": "corrupt", - "codec": codec, - "expected_full_errno": 97, - "expected_prefix_errno": 0 if codec_spec["decision"] == "GO" else 97, - "fsck_exit": failed.returncode, - "path": corrupted.name, - "sha256": sha256_path(corrupted), - }, - ] - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--spec", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - spec = load_spec(args.spec) - if args.output.exists(): - raise SystemExit(f"refusing existing B28 output: {args.output}") - for tool in ("mkfs.erofs", "fsck.erofs"): - if shutil.which(tool) is None: - raise SystemExit(f"{tool} is required") - - args.output.mkdir(parents=True) - source_root = args.output / "sources" - images = args.output / "images" - images.mkdir() - sources = make_sources(source_root, spec) - records = [] - for codec in sorted(spec["codecs"]): - records.extend( - build_codec(codec, spec["codecs"][codec], sources, source_root, images) - ) - (args.output / "spec.json").write_bytes(args.spec.read_bytes()) - index = { - "batch": "B28", - "candidate": "P15-086", - "fixture_count": len(records), - "fixtures": records, - "schema": 1, - "source_sha256": { - name: sha256_bytes(content) for name, content in sorted(sources.items()) - }, - "status": "READY", - "test": "TC176-stream-runtime", - } - (args.output / "fixture-index.json").write_text( - json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - print(json.dumps(index, indent=2, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B28-partial-probe.c b/tests/pre15/fixtures/B28-partial-probe.c deleted file mode 100644 index 868adf8..0000000 --- a/tests/pre15/fixtures/B28-partial-probe.c +++ /dev/null @@ -1,175 +0,0 @@ -#define _POSIX_C_SOURCE 200809L - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -static int -parse_offset(const char *text, off_t *value) -{ - char *end; - long long parsed; - - errno = 0; - parsed = strtoll(text, &end, 10); - if (errno != 0 || end == text || *end != '\0' || parsed < 0) - return (-1); - *value = (off_t)parsed; - return (0); -} - -static int -parse_size(const char *text, size_t *value) -{ - char *end; - unsigned long parsed; - - errno = 0; - parsed = strtoul(text, &end, 10); - if (errno != 0 || end == text || *end != '\0' || parsed == 0 || - parsed > 1024 * 1024) - return (-1); - *value = (size_t)parsed; - return (0); -} - -static int -open_pair(const char *path, const char *reference, int *actual_fd, - int *expected_fd) -{ - - *actual_fd = open(path, O_RDONLY); - *expected_fd = open(reference, O_RDONLY); - if (*actual_fd >= 0 && *expected_fd >= 0) - return (0); - perror("open B28 range input"); - if (*actual_fd >= 0) - close(*actual_fd); - if (*expected_fd >= 0) - close(*expected_fd); - return (1); -} - -static int -check_range(const char *path, const char *reference, off_t offset, size_t size) -{ - unsigned char *actual, *expected; - ssize_t actual_count, expected_count; - int actual_fd, expected_fd, result; - - actual = malloc(size); - expected = malloc(size); - if (actual == NULL || expected == NULL) { - free(actual); - free(expected); - return (1); - } - if (open_pair(path, reference, &actual_fd, &expected_fd) != 0) { - free(actual); - free(expected); - return (1); - } - actual_count = pread(actual_fd, actual, size, offset); - expected_count = pread(expected_fd, expected, size, offset); - result = actual_count == (ssize_t)size && expected_count == (ssize_t)size && - memcmp(actual, expected, size) == 0 ? 0 : 1; - if (result != 0) - fprintf(stderr, "range mismatch offset=%jd size=%zu actual=%zd expected=%zd errno=%d\n", - (intmax_t)offset, size, actual_count, expected_count, errno); - close(actual_fd); - close(expected_fd); - free(actual); - free(expected); - return (result); -} - -static int -check_range_errno(const char *path, off_t offset, size_t size, - int expected_errno) -{ - unsigned char *buffer; - ssize_t count; - int descriptor, result; - - buffer = malloc(size); - if (buffer == NULL) - return (1); - descriptor = open(path, O_RDONLY); - if (descriptor < 0) { - perror("open B28 damaged payload"); - free(buffer); - return (1); - } - errno = 0; - count = pread(descriptor, buffer, size, offset); - result = count < 0 && errno == expected_errno ? 0 : 1; - if (result != 0) - fprintf(stderr, "range errno mismatch offset=%jd size=%zu count=%zd errno=%d expected=%d\n", - (intmax_t)offset, size, count, errno, expected_errno); - close(descriptor); - free(buffer); - return (result); -} - -static int -check_full_errno(const char *path, int expected_errno) -{ - unsigned char buffer[4096]; - ssize_t count; - int descriptor; - - descriptor = open(path, O_RDONLY); - if (descriptor < 0) { - if (errno == expected_errno) - return (0); - perror("open B28 damaged payload"); - return (1); - } - for (;;) { - errno = 0; - count = read(descriptor, buffer, sizeof(buffer)); - if (count < 0) { - int actual_errno = errno; - - close(descriptor); - if (actual_errno == expected_errno) - return (0); - fprintf(stderr, "full errno=%d expected=%d\n", actual_errno, - expected_errno); - return (1); - } - if (count == 0) { - close(descriptor); - fprintf(stderr, "damaged extent reached clean EOF\n"); - return (1); - } - } -} - -int -main(int argc, char **argv) -{ - off_t offset; - size_t size; - - if (argc == 6 && strcmp(argv[1], "range") == 0 && - parse_offset(argv[4], &offset) == 0 && - parse_size(argv[5], &size) == 0) - return (check_range(argv[2], argv[3], offset, size)); - if (argc == 6 && strcmp(argv[1], "range-errno") == 0 && - parse_offset(argv[3], &offset) == 0 && - parse_size(argv[4], &size) == 0) - return (check_range_errno(argv[2], offset, size, atoi(argv[5]))); - if (argc == 4 && strcmp(argv[1], "full-errno") == 0) - return (check_full_errno(argv[2], atoi(argv[3]))); - fprintf(stderr, "usage: %s range PATH REFERENCE OFFSET SIZE | range-errno PATH OFFSET SIZE ERRNO | full-errno PATH ERRNO\n", - argv[0]); - return (2); -} diff --git a/tests/pre15/fixtures/B28-partial.json b/tests/pre15/fixtures/B28-partial.json deleted file mode 100644 index 26cec97..0000000 --- a/tests/pre15/fixtures/B28-partial.json +++ /dev/null @@ -1,168 +0,0 @@ -{ - "baseline": "2c61589e542ae96db8a100d71f42a8dd58ced172", - "batch": "B28", - "candidate": "P15-086", - "codecs": { - "deflate": { - "corruption_start": 4031, - "decision": "GO", - "gate_image_sha256": "5f98ff39be30b1396be8164e54a8ae80b144d181d5295a640b2e7814406d7a2d", - "extent": { - "leading_zero_bytes": 1, - "logical_length": 22878, - "logical_offset": 114326, - "physical_length": 4096, - "physical_offset": 24576, - "prefix_consumed": 805, - "stream_bytes": 4095 - }, - "mkfs_args": [ - "-T0", - "-U86100000-0000-4000-8000-000000000086", - "--all-root", - "-x-1", - "--workers=1", - "-zdeflate,level=1,dictsize=32768", - "-C4096" - ], - "source": "stream" - }, - "lz4": { - "corruption_start": 4032, - "decision": "GO", - "gate_image_sha256": "853649c78b159161421a6e833e0c516216382f3e8d135fc80aacd509ddb27b2c", - "extent": { - "leading_zero_bytes": 0, - "logical_length": 1038906, - "logical_offset": 0, - "physical_length": 4096, - "physical_offset": 4096, - "prefix_consumed": 1032, - "stream_bytes": 4096 - }, - "mkfs_args": [ - "-T0", - "-U86000000-0000-4000-8000-000000000086", - "--all-root", - "-x-1", - "-zlz4", - "-C4096", - "-Elegacy-compress" - ], - "source": "lz4" - }, - "lzma": { - "corruption_start": 476, - "decision": "GO", - "gate_image_sha256": "887d4a9baf629533c8d512177a84ca256a20135157d29a201408534f63230889", - "extent": { - "leading_zero_bytes": 3556, - "logical_length": 151552, - "logical_offset": 0, - "physical_length": 4096, - "physical_offset": 4096, - "prefix_consumed": 352, - "stream_bytes": 540 - }, - "mkfs_args": [ - "-T0", - "-U86200000-0000-4000-8000-000000000086", - "--all-root", - "-x-1", - "--workers=1", - "-zlzma,level=6,dictsize=65536", - "-C4096" - ], - "source": "stream" - }, - "zstd": { - "corruption_start": 2940, - "decision": "FULL_FALLBACK", - "gate_image_sha256": "93f6beb46e77187ea8b0fc1ad3a5a1cb565d546d9ce40d51278f26d67187d5fa", - "extent": { - "leading_zero_bytes": 1092, - "logical_length": 151552, - "logical_offset": 0, - "physical_length": 4096, - "physical_offset": 4096, - "prefix_consumed": 1457, - "stream_bytes": 3004 - }, - "mkfs_args": [ - "-T0", - "-U86300000-0000-4000-8000-000000000086", - "--all-root", - "-x-1", - "--workers=1", - "-zzstd,level=3,dictsize=65536", - "-C4096" - ], - "source": "stream" - } - }, - "errno": { - "integrity": 97, - "overflow": 84 - }, - "gate": { - "decision_commit": "07a42b667f3b8754f84beb1d8dcccb4378c1309c", - "evidence_commit": "2c61589e542ae96db8a100d71f42a8dd58ced172", - "evidence_root": "planning/pre15/evidence/20260815T154915Z-G04-G05-P15-086", - "input_sha256": "62f7f505aef846fa85085039b2f47e252e11602e6ef1bf41f56841cdc9089a2a", - "result_sha256": "33ef86b73447cd8c8099dc33901b9c4fff6c8ecedda3837d25cac9463a369d4f", - "script_sha256": "591e407f20a6c8ab5d506329b3bd39ff7bd7f78b66ac86b9db2c45a498b31184" - }, - "ranges": [ - { - "length": 4096, - "name": "prefix", - "offset": 0 - }, - { - "length": 4096, - "name": "cross-page", - "offset": 3584 - }, - { - "length": 4096, - "name": "middle", - "offset": 8192 - }, - { - "length": 4096, - "name": "tail", - "offset": -4096 - } - ], - "schema": 1, - "sources": { - "lz4": { - "segments": [ - { - "byte": 69, - "length": 262144 - }, - { - "byte": 82, - "length": 262144 - }, - { - "byte": 79, - "length": 262144 - }, - { - "byte": 70, - "length": 262144 - } - ], - "sha256": "b4802e36692c8124aa80813711cf03d5cec47729ecd25acf5cc4fd53c484ea80", - "size": 1048576 - }, - "stream": { - "line_count": 4096, - "sha256": "3c6a1cd405abc8b67b0100dbceebd0f918836563c7a5f705b9cfc178b35fca28", - "size": 151552 - } - }, - "test": "TC176-stream-runtime" -} diff --git a/tests/pre15/fixtures/B32-cache-generate.py b/tests/pre15/fixtures/B32-cache-generate.py deleted file mode 100755 index 99ad122..0000000 --- a/tests/pre15/fixtures/B32-cache-generate.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import shutil -import subprocess - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def run(argv: list[str], cwd: Path) -> None: - completed = subprocess.run( - argv, - cwd=cwd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=120, - ) - if completed.returncode != 0: - raise SystemExit( - f"command failed ({completed.returncode}): {' '.join(argv)}\n" - f"{completed.stdout}" - ) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--spec", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - spec = json.loads(args.spec.read_text(encoding="ascii")) - if ( - spec.get("schema") != 1 - or spec.get("batch") != "B32" - or spec.get("test") != "TC168-cache-inflight" - ): - raise SystemExit("invalid B32 cache fixture spec") - if args.output.exists(): - raise SystemExit(f"refusing existing B32 output: {args.output}") - if shutil.which("mkfs.erofs") is None: - raise SystemExit("mkfs.erofs is required") - - source = (b"Pre15-B32-cache-state:0123456789abcdef\n" * 2000)[ - : spec["source"]["size"] - ] - if sha256_bytes(source) != spec["source"]["sha256"]: - raise SystemExit("B32 deterministic source identity changed") - - source_dir = args.output / "source" - images = args.output / "images" - source_dir.mkdir(parents=True) - images.mkdir() - payload = source_dir / "payload.bin" - payload.write_bytes(source) - os.utime(payload, (0, 0)) - os.utime(source_dir, (0, 0)) - - valid = images / "lzma-valid.erofs" - run(["mkfs.erofs", *spec["mkfs_args"], str(valid), str(source_dir)], images) - valid_data = valid.read_bytes() - if len(valid_data) != 8192: - raise SystemExit(f"unexpected B32 image size: {len(valid_data)}") - pcluster_start = 4096 - pcluster_end = 8192 - pcluster = valid_data[pcluster_start:pcluster_end] - leading = next( - (index for index, value in enumerate(pcluster) if value), len(pcluster) - ) - stream = pcluster[leading:] - if leading == len(pcluster) or len(stream) < 2: - raise SystemExit("B32 LZMA stream boundary is empty") - - truncated_data = bytearray(valid_data) - truncated_data[pcluster_start + leading : pcluster_end] = b"\0" + stream[:-1] - if truncated_data == valid_data: - raise SystemExit("B32 truncated mutation changed no bytes") - truncated = images / "lzma-truncated.erofs" - truncated.write_bytes(truncated_data) - - records = [] - for image_class, path, expected_errno in ( - ("valid", valid, 0), - ("truncated", truncated, 97), - ): - records.append( - { - "class": image_class, - "expected_errno": expected_errno, - "path": path.name, - "sha256": sha256_path(path), - } - ) - index = { - "batch": "B32", - "decoded_size": len(source), - "fixture_count": len(records), - "fixtures": records, - "leading_zero_bytes": leading, - "physical_length": len(pcluster), - "physical_offset": pcluster_start, - "schema": 1, - "source_sha256": sha256_path(payload), - "status": "READY", - "test": "TC168-cache-inflight", - } - (args.output / "fixture-index.json").write_text( - json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - print(json.dumps(index, indent=2, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B32-cache-oracle.c b/tests/pre15/fixtures/B32-cache-oracle.c deleted file mode 100644 index 62b93ea..0000000 --- a/tests/pre15/fixtures/B32-cache-oracle.c +++ /dev/null @@ -1,541 +0,0 @@ -#define _POSIX_C_SOURCE 200809L - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define B32_WORKERS 64 -#define B32_DATA_SIZE 32 - -enum cache_state { - CACHE_EMPTY, - CACHE_INFLIGHT, - CACHE_READY, - CACHE_FAILED, -}; - -enum cache_lookup { - CACHE_BYPASS, - CACHE_HIT, - CACHE_OWNER, - CACHE_ERROR, -}; - -struct map_blocks { - uint64_t m_pa; - uint64_t m_la; - uint64_t m_plen; - uint64_t m_llen; - uint16_t m_deviceid; - uint8_t m_algorithmformat; - unsigned int m_flags; -}; - -struct cache_key { - struct map_blocks map; - uint64_t nid; - size_t decoded_size; -}; - -struct cache { - pthread_mutex_t lock; - pthread_cond_t cv; - struct cache_key key; - unsigned char *data; - unsigned int waiters; - int error; - enum cache_state state; - bool closing; -}; - -struct wave { - struct cache *cache; - struct cache_key key; - pthread_barrier_t barrier; - unsigned char payload[B32_DATA_SIZE]; - atomic_uint decodes; - atomic_uint bypasses; - int expected_error; - int results[B32_WORKERS]; -}; - -struct worker_arg { - struct wave *wave; - unsigned int index; -}; - -struct fini_arg { - struct cache *cache; - atomic_bool done; -}; - -static void -fail(const char *message) -{ - fprintf(stderr, "B32 oracle failure: %s\n", message); - exit(1); -} - -static void -check_pthread(int error, const char *operation) -{ - if (error != 0) { - errno = error; - perror(operation); - exit(1); - } -} - -static bool -key_equal(const struct cache_key *left, const struct cache_key *right) -{ - return (left->nid == right->nid && - left->decoded_size == right->decoded_size && - left->map.m_pa == right->map.m_pa && - left->map.m_la == right->map.m_la && - left->map.m_plen == right->map.m_plen && - left->map.m_llen == right->map.m_llen && - left->map.m_deviceid == right->map.m_deviceid && - left->map.m_algorithmformat == right->map.m_algorithmformat && - left->map.m_flags == right->map.m_flags); -} - -static void -cache_init(struct cache *cache) -{ - memset(cache, 0, sizeof(*cache)); - check_pthread(pthread_mutex_init(&cache->lock, NULL), "pthread_mutex_init"); - check_pthread(pthread_cond_init(&cache->cv, NULL), "pthread_cond_init"); -} - -static enum cache_lookup -cache_claim(struct cache *cache, const struct cache_key *key, - unsigned char *output, int *errorp) -{ - unsigned char *old; - enum cache_lookup result; - - *errorp = 0; - old = NULL; - check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock"); - if (cache->closing) - goto bypass; - if (cache->state != CACHE_EMPTY && key_equal(&cache->key, key)) { - switch (cache->state) { - case CACHE_READY: - if (cache->data == NULL) - fail("READY state has no data"); - memcpy(output, cache->data, key->decoded_size); - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock"); - return (CACHE_HIT); - case CACHE_INFLIGHT: - ++cache->waiters; - do { - check_pthread(pthread_cond_wait(&cache->cv, &cache->lock), - "pthread_cond_wait"); - } while (cache->state == CACHE_INFLIGHT); - if (!key_equal(&cache->key, key)) - fail("inflight key changed before waiter consumed it"); - if (cache->state == CACHE_READY) { - if (cache->data == NULL) - fail("published success has no data"); - memcpy(output, cache->data, key->decoded_size); - result = CACHE_HIT; - } else { - if (cache->state != CACHE_FAILED || cache->error <= 0) - fail("published failure has no positive errno"); - *errorp = cache->error; - result = CACHE_ERROR; - } - --cache->waiters; - if (cache->state == CACHE_FAILED && cache->waiters == 0) { - cache->error = 0; - cache->state = CACHE_EMPTY; - } - if (cache->waiters == 0) - check_pthread(pthread_cond_broadcast(&cache->cv), - "pthread_cond_broadcast"); - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock"); - return (result); - case CACHE_FAILED: - if (cache->waiters != 0) - goto bypass; - cache->error = 0; - cache->state = CACHE_EMPTY; - break; - case CACHE_EMPTY: - break; - } - } - if (cache->state == CACHE_INFLIGHT || cache->waiters != 0) - goto bypass; - old = cache->data; - cache->data = NULL; - cache->key = *key; - cache->error = 0; - cache->state = CACHE_INFLIGHT; - check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock"); - free(old); - return (CACHE_OWNER); - -bypass: - check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock"); - return (CACHE_BYPASS); -} - -static void -cache_complete(struct cache *cache, const struct cache_key *key, - unsigned char *data, int error) -{ - if ((error == 0) != (data != NULL)) - fail("owner completion is not typed"); - check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock"); - if (cache->state != CACHE_INFLIGHT || !key_equal(&cache->key, key)) - fail("owner lost its inflight key"); - if (error == 0) { - cache->data = data; - cache->error = 0; - cache->state = CACHE_READY; - } else { - if (error < 1) - fail("owner published a non-positive errno"); - cache->error = error; - cache->state = CACHE_FAILED; - if (cache->waiters == 0) { - cache->error = 0; - cache->state = CACHE_EMPTY; - } - } - check_pthread(pthread_cond_broadcast(&cache->cv), - "pthread_cond_broadcast"); - check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock"); -} - -static unsigned char * -copy_payload(const unsigned char *payload, size_t size) -{ - unsigned char *copy; - - copy = malloc(size); - if (copy == NULL) - fail("malloc failed"); - memcpy(copy, payload, size); - return (copy); -} - -static void -wait_for_waiters(struct cache *cache, unsigned int expected) -{ - struct timespec delay = { .tv_sec = 0, .tv_nsec = 1000000 }; - unsigned int attempt, waiters; - - for (attempt = 0; attempt < 5000; ++attempt) { - check_pthread(pthread_mutex_lock(&cache->lock), - "pthread_mutex_lock"); - waiters = cache->waiters; - check_pthread(pthread_mutex_unlock(&cache->lock), - "pthread_mutex_unlock"); - if (waiters == expected) - return; - nanosleep(&delay, NULL); - } - fail("workers did not register as same-key waiters"); -} - -static void * -wave_worker(void *opaque) -{ - struct worker_arg *arg; - struct wave *wave; - unsigned char output[B32_DATA_SIZE]; - enum cache_lookup lookup; - int barrier_result, error; - - arg = opaque; - wave = arg->wave; - barrier_result = pthread_barrier_wait(&wave->barrier); - if (barrier_result != 0 && barrier_result != PTHREAD_BARRIER_SERIAL_THREAD) - check_pthread(barrier_result, "pthread_barrier_wait"); - lookup = cache_claim(wave->cache, &wave->key, output, &error); - if (lookup == CACHE_OWNER) { - atomic_fetch_add_explicit(&wave->decodes, 1, memory_order_relaxed); - wait_for_waiters(wave->cache, B32_WORKERS - 1); - if (wave->expected_error != 0) { - cache_complete(wave->cache, &wave->key, NULL, - wave->expected_error); - wave->results[arg->index] = wave->expected_error; - } else { - memcpy(output, wave->payload, sizeof(output)); - cache_complete(wave->cache, &wave->key, - copy_payload(wave->payload, sizeof(wave->payload)), 0); - wave->results[arg->index] = - memcmp(output, wave->payload, sizeof(output)) == 0 ? 0 : EIO; - } - } else if (lookup == CACHE_HIT) { - wave->results[arg->index] = - memcmp(output, wave->payload, sizeof(output)) == 0 ? 0 : EIO; - } else if (lookup == CACHE_ERROR) { - wave->results[arg->index] = error; - } else { - atomic_fetch_add_explicit(&wave->bypasses, 1, memory_order_relaxed); - wave->results[arg->index] = EBUSY; - } - return (NULL); -} - -static void -run_wave(struct cache *cache, const struct cache_key *key, int expected_error) -{ - struct wave wave; - struct worker_arg args[B32_WORKERS]; - pthread_t threads[B32_WORKERS]; - unsigned int index; - - memset(&wave, 0, sizeof(wave)); - wave.cache = cache; - wave.key = *key; - wave.expected_error = expected_error; - for (index = 0; index < sizeof(wave.payload); ++index) - wave.payload[index] = (unsigned char)(0xa0U + index); - check_pthread(pthread_barrier_init(&wave.barrier, NULL, B32_WORKERS), - "pthread_barrier_init"); - for (index = 0; index < B32_WORKERS; ++index) { - args[index].wave = &wave; - args[index].index = index; - check_pthread(pthread_create(&threads[index], NULL, wave_worker, - &args[index]), "pthread_create"); - } - for (index = 0; index < B32_WORKERS; ++index) - check_pthread(pthread_join(threads[index], NULL), "pthread_join"); - check_pthread(pthread_barrier_destroy(&wave.barrier), - "pthread_barrier_destroy"); - if (atomic_load_explicit(&wave.decodes, memory_order_relaxed) != 1) - fail("same-key wave had more than one owner decode"); - if (atomic_load_explicit(&wave.bypasses, memory_order_relaxed) != 0) - fail("same-key wave bypassed its owner generation"); - for (index = 0; index < B32_WORKERS; ++index) { - if (wave.results[index] != expected_error) - fail("same-key waiter received different bytes or errno"); - } -} - -static struct cache_key -base_key(void) -{ - return ((struct cache_key) { - .map = { - .m_pa = 0x1000, - .m_la = 0x2000, - .m_plen = 4096, - .m_llen = B32_DATA_SIZE, - .m_deviceid = 3, - .m_algorithmformat = 1, - .m_flags = 1, - }, - .nid = 42, - .decoded_size = B32_DATA_SIZE, - }); -} - -static void -seed_ready(struct cache *cache, const struct cache_key *key) -{ - unsigned char output[B32_DATA_SIZE]; - unsigned char payload[B32_DATA_SIZE]; - enum cache_lookup lookup; - int error; - - memset(payload, 0x5a, sizeof(payload)); - lookup = cache_claim(cache, key, output, &error); - if (lookup != CACHE_OWNER) - fail("seed miss did not become owner"); - cache_complete(cache, key, copy_payload(payload, sizeof(payload)), 0); -} - -static void -expect_distinct_key(struct cache *cache, const struct cache_key *base, - const struct cache_key *changed) -{ - unsigned char output[B32_DATA_SIZE]; - unsigned char payload[B32_DATA_SIZE]; - enum cache_lookup lookup; - int error; - - lookup = cache_claim(cache, changed, output, &error); - if (lookup != CACHE_OWNER) - fail("changed key incorrectly hit or bypassed"); - memset(payload, 0x33, sizeof(payload)); - cache_complete(cache, changed, copy_payload(payload, sizeof(payload)), 0); - lookup = cache_claim(cache, base, output, &error); - if (lookup != CACHE_OWNER) - fail("key reuse did not start a fresh owner generation"); - memset(payload, 0x5a, sizeof(payload)); - cache_complete(cache, base, copy_payload(payload, sizeof(payload)), 0); -} - -static void -test_exact_key(struct cache *cache, const struct cache_key *base) -{ - struct cache_key changed; - - seed_ready(cache, base); - changed = *base; - ++changed.nid; - expect_distinct_key(cache, base, &changed); - changed = *base; - --changed.decoded_size; - expect_distinct_key(cache, base, &changed); - changed = *base; - ++changed.map.m_pa; - expect_distinct_key(cache, base, &changed); - changed = *base; - ++changed.map.m_la; - expect_distinct_key(cache, base, &changed); - changed = *base; - ++changed.map.m_plen; - expect_distinct_key(cache, base, &changed); - changed = *base; - ++changed.map.m_llen; - expect_distinct_key(cache, base, &changed); - changed = *base; - ++changed.map.m_deviceid; - expect_distinct_key(cache, base, &changed); - changed = *base; - ++changed.map.m_algorithmformat; - expect_distinct_key(cache, base, &changed); - changed = *base; - changed.map.m_flags ^= 2U; - expect_distinct_key(cache, base, &changed); -} - -static void -test_fallback_and_waiter_generation(struct cache *cache, - const struct cache_key *base) -{ - struct cache_key changed; - unsigned char output[B32_DATA_SIZE]; - unsigned char payload[B32_DATA_SIZE]; - enum cache_lookup lookup; - int error; - - changed = *base; - ++changed.nid; - lookup = cache_claim(cache, base, output, &error); - if (lookup != CACHE_HIT) - fail("ready base key was not retained"); - lookup = cache_claim(cache, &changed, output, &error); - if (lookup != CACHE_OWNER) - fail("ready eviction did not create a new owner"); - lookup = cache_claim(cache, base, output, &error); - if (lookup != CACHE_BYPASS) - fail("different key did not use no-cache inflight fallback"); - memset(payload, 0x7c, sizeof(payload)); - cache_complete(cache, &changed, copy_payload(payload, sizeof(payload)), 0); - - check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock"); - cache->waiters = 1; - check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock"); - lookup = cache_claim(cache, base, output, &error); - if (lookup != CACHE_BYPASS) - fail("published generation was evicted before waiter consumption"); - check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock"); - cache->waiters = 0; - check_pthread(pthread_cond_broadcast(&cache->cv), - "pthread_cond_broadcast"); - check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock"); -} - -static void * -fini_worker(void *opaque) -{ - struct fini_arg *arg; - struct cache *cache; - unsigned char *data; - - arg = opaque; - cache = arg->cache; - check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock"); - cache->closing = true; - while (cache->state == CACHE_INFLIGHT || cache->waiters != 0) - check_pthread(pthread_cond_wait(&cache->cv, &cache->lock), - "pthread_cond_wait"); - data = cache->data; - cache->data = NULL; - cache->error = 0; - cache->state = CACHE_EMPTY; - check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock"); - free(data); - atomic_store_explicit(&arg->done, true, memory_order_release); - return (NULL); -} - -static void -test_shutdown(struct cache *cache, const struct cache_key *key) -{ - struct fini_arg arg; - struct timespec delay = { .tv_sec = 0, .tv_nsec = 20000000 }; - unsigned char output[B32_DATA_SIZE]; - unsigned char payload[B32_DATA_SIZE]; - pthread_t thread; - enum cache_lookup lookup; - int error; - - lookup = cache_claim(cache, key, output, &error); - if (lookup != CACHE_OWNER) - fail("shutdown setup did not create inflight owner"); - arg.cache = cache; - atomic_init(&arg.done, false); - check_pthread(pthread_create(&thread, NULL, fini_worker, &arg), - "pthread_create"); - nanosleep(&delay, NULL); - if (atomic_load_explicit(&arg.done, memory_order_acquire)) - fail("shutdown did not wait for inflight owner"); - memset(payload, 0x91, sizeof(payload)); - cache_complete(cache, key, copy_payload(payload, sizeof(payload)), 0); - check_pthread(pthread_join(thread, NULL), "pthread_join"); - if (!atomic_load_explicit(&arg.done, memory_order_acquire)) - fail("shutdown did not finish after owner publication"); - check_pthread(pthread_cond_destroy(&cache->cv), "pthread_cond_destroy"); - check_pthread(pthread_mutex_destroy(&cache->lock), "pthread_mutex_destroy"); -} - -int -main(void) -{ - struct cache cache; - struct cache_key key, failure_key, retry_key; - unsigned char output[B32_DATA_SIZE]; - unsigned char payload[B32_DATA_SIZE]; - enum cache_lookup lookup; - int error; - - cache_init(&cache); - key = base_key(); - run_wave(&cache, &key, 0); - failure_key = key; - ++failure_key.nid; - run_wave(&cache, &failure_key, EIO); - lookup = cache_claim(&cache, &failure_key, output, &error); - if (lookup != CACHE_OWNER) - fail("published failure did not become retryable"); - memset(payload, 0x44, sizeof(payload)); - cache_complete(&cache, &failure_key, - copy_payload(payload, sizeof(payload)), 0); - - retry_key = key; - retry_key.nid += 2; - test_exact_key(&cache, &retry_key); - test_fallback_and_waiter_generation(&cache, &retry_key); - retry_key.nid += 3; - test_shutdown(&cache, &retry_key); - printf("TC168-cache-inflight host oracle PASS: one owner, exact key, typed failure, retry, fallback, eviction, shutdown\n"); - return (0); -} diff --git a/tests/pre15/fixtures/B32-cache-probe.c b/tests/pre15/fixtures/B32-cache-probe.c deleted file mode 100644 index 5ad9183..0000000 --- a/tests/pre15/fixtures/B32-cache-probe.c +++ /dev/null @@ -1,405 +0,0 @@ -#define _POSIX_C_SOURCE 200809L - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct concurrent_ctx { - const unsigned char *expected; - size_t length; - off_t offset; - long loops; - int actual_fd; - int expected_errno; - pthread_barrier_t barrier; - pthread_mutex_t lock; - int failure; - ssize_t mismatch_count; - int mismatch_errno; -}; - -struct vnode_ctx { - const char *actual_path; - const unsigned char *expected; - size_t expected_size; - long loops; - pthread_barrier_t barrier; - pthread_mutex_t lock; - int failure; -}; - -static int -fail(const char *message) -{ - fprintf(stderr, "B32 probe failure: %s\n", message); - return (1); -} - -static long -parse_long(const char *text, long minimum, long maximum) -{ - char *end; - long value; - - errno = 0; - value = strtol(text, &end, 10); - if (errno != 0 || *text == '\0' || *end != '\0' || value < minimum || - value > maximum) - return (-1); - return (value); -} - -static unsigned char * -read_expected(const char *path, off_t offset, size_t length) -{ - unsigned char *buffer; - ssize_t count; - int descriptor; - - descriptor = open(path, O_RDONLY); - if (descriptor < 0) - return (NULL); - buffer = malloc(length); - if (buffer == NULL) { - close(descriptor); - return (NULL); - } - count = pread(descriptor, buffer, length, offset); - close(descriptor); - if (count != (ssize_t)length) { - free(buffer); - return (NULL); - } - return (buffer); -} - -static void -record_failure(pthread_mutex_t *lock, int *failure, int error) -{ - pthread_mutex_lock(lock); - if (*failure == 0) - *failure = error != 0 ? error : EIO; - pthread_mutex_unlock(lock); -} - -static void -record_concurrent_failure(struct concurrent_ctx *ctx, ssize_t count, - int saved_errno) -{ - - pthread_mutex_lock(&ctx->lock); - if (ctx->failure == 0) { - ctx->failure = saved_errno != 0 ? saved_errno : EIO; - ctx->mismatch_count = count; - ctx->mismatch_errno = saved_errno; - } - pthread_mutex_unlock(&ctx->lock); -} - -static void * -concurrent_worker(void *opaque) -{ - struct concurrent_ctx *ctx; - unsigned char *buffer; - ssize_t count; - long loop; - int barrier_result, saved_errno; - - ctx = opaque; - buffer = malloc(ctx->length); - if (buffer == NULL) { - record_failure(&ctx->lock, &ctx->failure, ENOMEM); - return (NULL); - } - barrier_result = pthread_barrier_wait(&ctx->barrier); - if (barrier_result != 0 && barrier_result != PTHREAD_BARRIER_SERIAL_THREAD) { - record_failure(&ctx->lock, &ctx->failure, barrier_result); - free(buffer); - return (NULL); - } - for (loop = 0; loop < ctx->loops; ++loop) { - errno = 0; - count = pread(ctx->actual_fd, buffer, ctx->length, ctx->offset); - saved_errno = errno; - if (ctx->expected_errno != 0) { - if (count != -1 || saved_errno != ctx->expected_errno) { - record_concurrent_failure(ctx, count, saved_errno); - break; - } - } else if (count != (ssize_t)ctx->length || - memcmp(buffer, ctx->expected, ctx->length) != 0) { - record_concurrent_failure(ctx, count, saved_errno); - break; - } - } - free(buffer); - return (NULL); -} - -static int -run_concurrent(const char *actual_path, const char *expected_path, - long expected_errno, long workers, long loops, off_t offset, size_t length) -{ - struct concurrent_ctx ctx; - pthread_t *threads; - unsigned char *expected; - long index; - - memset(&ctx, 0, sizeof(ctx)); - expected = expected_errno == 0 ? - read_expected(expected_path, offset, length) : calloc(1, length); - if (expected == NULL) - return (fail("could not prepare expected bytes")); - ctx.expected = expected; - ctx.length = length; - ctx.offset = offset; - ctx.loops = loops; - ctx.expected_errno = (int)expected_errno; - ctx.actual_fd = open(actual_path, O_RDONLY); - if (ctx.actual_fd < 0) { - free(expected); - return (fail("could not open actual file")); - } - if (pthread_barrier_init(&ctx.barrier, NULL, (unsigned int)workers) != 0 || - pthread_mutex_init(&ctx.lock, NULL) != 0) { - close(ctx.actual_fd); - free(expected); - return (fail("could not initialize concurrent controls")); - } - threads = calloc((size_t)workers, sizeof(*threads)); - if (threads == NULL) - return (fail("could not allocate worker handles")); - for (index = 0; index < workers; ++index) { - if (pthread_create(&threads[index], NULL, concurrent_worker, &ctx) != 0) - return (fail("could not create concurrent worker")); - } - for (index = 0; index < workers; ++index) { - if (pthread_join(threads[index], NULL) != 0) - return (fail("could not join concurrent worker")); - } - free(threads); - pthread_barrier_destroy(&ctx.barrier); - pthread_mutex_destroy(&ctx.lock); - close(ctx.actual_fd); - free(expected); - if (ctx.failure != 0) { - fprintf(stderr, - "concurrent mismatch count=%zd errno=%d expected_errno=%d\n", - ctx.mismatch_count, ctx.mismatch_errno, ctx.expected_errno); - errno = ctx.failure; - perror("concurrent read"); - return (1); - } - printf("concurrent PASS workers=%ld loops=%ld assertions=%ld errno=%ld " - "offset=%ld length=%ld\n", workers, loops, workers * loops, - expected_errno, (long)offset, (long)length); - return (0); -} - -static void * -vnode_worker(void *opaque) -{ - struct vnode_ctx *ctx; - unsigned char buffer[4096]; - struct stat status; - ssize_t count; - long loop; - int barrier_result, descriptor, saved_errno; - - ctx = opaque; - barrier_result = pthread_barrier_wait(&ctx->barrier); - if (barrier_result != 0 && barrier_result != PTHREAD_BARRIER_SERIAL_THREAD) { - record_failure(&ctx->lock, &ctx->failure, barrier_result); - return (NULL); - } - for (loop = 0; loop < ctx->loops; ++loop) { - descriptor = open(ctx->actual_path, O_RDONLY); - if (descriptor < 0) { - record_failure(&ctx->lock, &ctx->failure, errno); - break; - } - if (fstat(descriptor, &status) != 0 || - (size_t)status.st_size != ctx->expected_size) { - saved_errno = errno; - close(descriptor); - record_failure(&ctx->lock, &ctx->failure, - saved_errno != 0 ? saved_errno : EIO); - break; - } - count = pread(descriptor, buffer, sizeof(buffer), 0); - saved_errno = errno; - close(descriptor); - if (count != (ssize_t)sizeof(buffer) || - memcmp(buffer, ctx->expected, sizeof(buffer)) != 0) { - record_failure(&ctx->lock, &ctx->failure, - saved_errno != 0 ? saved_errno : EIO); - break; - } - } - return (NULL); -} - -static int -run_vnode_race(const char *actual_path, const char *expected_path, long workers, - long loops) -{ - struct vnode_ctx ctx; - struct stat status; - pthread_t *threads; - long index; - - memset(&ctx, 0, sizeof(ctx)); - if (stat(expected_path, &status) != 0 || status.st_size < 4096) - return (fail("could not stat vnode-race reference")); - ctx.actual_path = actual_path; - ctx.expected_size = (size_t)status.st_size; - ctx.expected = read_expected(expected_path, 0, 4096); - ctx.loops = loops; - if (ctx.expected == NULL) - return (fail("could not read vnode-race reference")); - if (pthread_barrier_init(&ctx.barrier, NULL, (unsigned int)workers) != 0 || - pthread_mutex_init(&ctx.lock, NULL) != 0) - return (fail("could not initialize vnode-race controls")); - threads = calloc((size_t)workers, sizeof(*threads)); - if (threads == NULL) - return (fail("could not allocate vnode-race handles")); - for (index = 0; index < workers; ++index) { - if (pthread_create(&threads[index], NULL, vnode_worker, &ctx) != 0) - return (fail("could not create vnode-race worker")); - } - for (index = 0; index < workers; ++index) { - if (pthread_join(threads[index], NULL) != 0) - return (fail("could not join vnode-race worker")); - } - free(threads); - pthread_barrier_destroy(&ctx.barrier); - pthread_mutex_destroy(&ctx.lock); - free((void *)ctx.expected); - if (ctx.failure != 0) { - errno = ctx.failure; - perror("vnode race"); - return (1); - } - printf("vnode-race PASS workers=%ld loops=%ld assertions=%ld\n", workers, - loops, workers * loops); - return (0); -} - -static bool -marker_exists(const char *directory, const char *name) -{ - char path[1024]; - - if (snprintf(path, sizeof(path), "%s/%s", directory, name) >= - (int)sizeof(path)) - return (false); - return (access(path, F_OK) == 0); -} - -static int -write_marker(const char *directory, const char *name, const char *content) -{ - char path[1024]; - ssize_t length; - int descriptor; - - if (snprintf(path, sizeof(path), "%s/%s", directory, name) >= - (int)sizeof(path)) - return (-1); - descriptor = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (descriptor < 0) - return (-1); - length = (ssize_t)strlen(content); - if (write(descriptor, content, (size_t)length) != length) { - close(descriptor); - return (-1); - } - return (close(descriptor)); -} - -static int -run_lifecycle(const char *path, const char *directory) -{ - struct timespec delay = { .tv_sec = 0, .tv_nsec = 10000000 }; - char result[128]; - ssize_t count; - long attempt; - int descriptor, saved_errno; - bool closed; - - descriptor = open(path, O_RDONLY); - if (descriptor < 0) - return (fail("lifecycle open failed")); - if (write_marker(directory, "ready", "ready\n") != 0) - return (fail("lifecycle ready marker failed")); - closed = false; - for (attempt = 0; attempt < 18000; ++attempt) { - if (!closed && marker_exists(directory, "close")) { - if (close(descriptor) != 0) - return (fail("lifecycle close failed")); - closed = true; - if (write_marker(directory, "closed", "closed\n") != 0) - return (fail("lifecycle closed marker failed")); - } - if (marker_exists(directory, "read")) { - errno = 0; - count = pread(descriptor, result, sizeof(result), 0); - saved_errno = errno; - if (snprintf(result, sizeof(result), "%zd %d\n", count, - saved_errno) >= (int)sizeof(result) || - write_marker(directory, "result", result) != 0) - return (fail("lifecycle result marker failed")); - if (!closed) - close(descriptor); - return (0); - } - if (marker_exists(directory, "exit")) { - if (!closed) - close(descriptor); - return (0); - } - nanosleep(&delay, NULL); - } - if (!closed) - close(descriptor); - return (fail("lifecycle control timed out")); -} - -int -main(int argc, char **argv) -{ - long expected_errno, workers, loops, offset, length; - - if (argc == 9 && strcmp(argv[1], "concurrent") == 0) { - expected_errno = parse_long(argv[4], 0, 255); - workers = parse_long(argv[5], 1, 256); - loops = parse_long(argv[6], 1, 10000); - offset = parse_long(argv[7], 0, INT64_MAX); - length = parse_long(argv[8], 1, 1048576); - if (expected_errno < 0 || workers < 0 || loops < 0 || offset < 0 || - length < 0) - return (fail("invalid concurrent arguments")); - return (run_concurrent(argv[2], argv[3], expected_errno, workers, - loops, (off_t)offset, (size_t)length)); - } - if (argc == 6 && strcmp(argv[1], "vnode-race") == 0) { - workers = parse_long(argv[4], 1, 256); - loops = parse_long(argv[5], 1, 10000); - if (workers < 0 || loops < 0) - return (fail("invalid vnode-race arguments")); - return (run_vnode_race(argv[2], argv[3], workers, loops)); - } - if (argc == 4 && strcmp(argv[1], "lifecycle") == 0) - return (run_lifecycle(argv[2], argv[3])); - return (fail("usage: concurrent ACTUAL EXPECTED ERRNO WORKERS LOOPS OFFSET LENGTH | vnode-race ACTUAL EXPECTED WORKERS LOOPS | lifecycle PATH CONTROL_DIR")); -} diff --git a/tests/pre15/fixtures/B32-cache-state.json b/tests/pre15/fixtures/B32-cache-state.json deleted file mode 100644 index 4c35656..0000000 --- a/tests/pre15/fixtures/B32-cache-state.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "baseline": "3000e04b230601dd39b6747032f7ad5dcced0ca4", - "batch": "B32", - "candidate": [ - "P15-075", - "P15-090" - ], - "key_fields": [ - "nid", - "decoded_size", - "m_pa", - "m_la", - "m_plen", - "m_llen", - "m_deviceid", - "m_algorithmformat", - "m_flags" - ], - "mkfs_args": [ - "-T0", - "-U32000000-0000-4000-8000-000000000075", - "--all-root", - "-x-1", - "--workers=1", - "-zlzma,level=6,dictsize=65536", - "-C4096" - ], - "schema": 1, - "source": { - "sha256": "082153d3c28ac1e3588deaabd9607a7b274d093ab9b9900060888ffc806934bb", - "size": 65536 - }, - "states": [ - "EMPTY", - "INFLIGHT", - "READY", - "FAILED" - ], - "test": "TC168-cache-inflight" -} diff --git a/tests/pre15/fixtures/B32-qemu-run.sh b/tests/pre15/fixtures/B32-qemu-run.sh deleted file mode 100755 index 4b8b8c1..0000000 --- a/tests/pre15/fixtures/B32-qemu-run.sh +++ /dev/null @@ -1,306 +0,0 @@ -#!/bin/sh -set -eu - -script=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/B32-qemu-run.sh - -if test "${1:-}" != --worker; then - test "$#" -eq 1 || { - printf '%s\n' 'usage: B32-qemu-run.sh TC168-cache-inflight|TC184-vnode-races' >&2 - exit 20 - } - case "$1" in - TC168-cache-inflight|TC184-vnode-races) ;; - *) printf 'unknown B32 QEMU scenario: %s\n' "$1" >&2; exit 20 ;; - esac - total=${PRE15_QEMU_TIMEOUT:-1200} - case "$total" in - ''|*[!0-9]*|0) printf '%s\n' 'invalid PRE15_QEMU_TIMEOUT' >&2; exit 20 ;; - esac - exec timeout -k 30 "$total" "$script" --worker "$1" -fi - -shift -test "$#" -eq 1 || exit 20 -scenario=$1 -: "${PRE15_EVIDENCE_ROOT:?PRE15_EVIDENCE_ROOT is required}" -: "${PRE15_QEMU_BASE_IMAGE:?PRE15_QEMU_BASE_IMAGE is required}" -: "${PRE15_QEMU_SSH_KEY:?PRE15_QEMU_SSH_KEY is required}" - -dut=$(CDPATH= cd -- "$(dirname -- "$script")/../../.." && pwd -P) -root=$(CDPATH= cd -- "$dut/.." && pwd -P) -lib=$dut/tests/pre15/lib -case_script=$dut/tests/pre15/cases/B32-cache-state.sh -freebsd_src=${PRE15_FREEBSD_SRC:-${FREEBSD_SRC:-/work/build/freebsd-src}} -user=${PRE15_QEMU_SSH_USER:-root} -boot_timeout=${PRE15_QEMU_BOOT_TIMEOUT:-180} -guest_timeout=${PRE15_GUEST_COMMAND_TIMEOUT:-60} -case_timeout=${PRE15_B32_QEMU_CASE_TIMEOUT:-900} -run_id=$(date -u '+%Y%m%dT%H%M%SZ')-qemu-B32-$scenario-$$ -run=$PRE15_EVIDENCE_ROOT/$run_id -case_run=$run/case -case_tmp=$run/case-tmp -overlay=$run/guest-overlay.qcow2 -control=$run/ssh-control -serial=$run/serial.log -qemu_log=$run/qemu.log -ownership=$case_run/ownership.tsv -cleanup_log=$case_run/cleanup.log -target_marker=$case_run/target.marker -reason_file=$case_run/reason.txt -fixture_hashes=$case_run/fixture-hashes.tsv -module_hash=$case_run/module-hash.tsv -qemu_pid= -port= -ssh_ready=0 -cleanup_done=0 -cleanup_status=PASS - -mkdir -p "$case_run" "$case_tmp" -: >"$ownership" -: >"$fixture_hashes" -: >"$module_hash" -: >"$reason_file" - -cleanup() -{ - test "$cleanup_done" -eq 0 || return - cleanup_done=1 - if test "$ssh_ready" -eq 1; then - PRE15_QEMU_SSH_PORT=$port - PRE15_QEMU_SSH_USER=$user - PRE15_QEMU_SSH_KEY=$PRE15_QEMU_SSH_KEY - PRE15_QEMU_CONTROL_PATH=$control - PRE15_GUEST_COMMAND_TIMEOUT=$guest_timeout - PRE15_OWNERSHIP_FILE=$ownership - PRE15_CLEANUP_LOG=$cleanup_log - PRE15_RUN_DIR=$case_run - export PRE15_QEMU_SSH_PORT PRE15_QEMU_SSH_USER PRE15_QEMU_SSH_KEY - export PRE15_QEMU_CONTROL_PATH PRE15_GUEST_COMMAND_TIMEOUT - export PRE15_OWNERSHIP_FILE PRE15_CLEANUP_LOG PRE15_RUN_DIR - . "$lib/runner.sh" - pre15_cleanup_owned || cleanup_status=FAIL - test "${PRE15_CLEANUP_STATUS_RESULT:-FAIL}" = PASS || cleanup_status=FAIL - else - printf '%s\n' 'PASS no guest resources reached before SSH readiness' \ - >"$cleanup_log" - fi - if test -S "$control"; then - timeout -k 5 15 ssh -S "$control" -O exit -p "${port:-22}" \ - "$user@127.0.0.1" >>"$run/host-cleanup.log" 2>&1 || true - fi - if test -n "$qemu_pid" && test "$qemu_pid" != 26318 && \ - kill -0 "$qemu_pid" 2>/dev/null; then - kill -TERM "$qemu_pid" 2>/dev/null || cleanup_status=FAIL - wait_count=0 - while kill -0 "$qemu_pid" 2>/dev/null && test "$wait_count" -lt 100; do - sleep 0.1 - wait_count=$((wait_count + 1)) - done - if kill -0 "$qemu_pid" 2>/dev/null; then - kill -KILL "$qemu_pid" 2>/dev/null || cleanup_status=FAIL - sleep 0.2 - fi - fi - if test -n "$qemu_pid" && kill -0 "$qemu_pid" 2>/dev/null; then - cleanup_status=FAIL - printf 'FAIL owned QEMU PID survived: %s\n' "$qemu_pid" \ - >>"$run/host-cleanup.log" - else - printf 'PASS owned QEMU PID stopped: %s\n' "${qemu_pid:-not-started}" \ - >>"$run/host-cleanup.log" - fi - if test -f "$overlay"; then - rm -f -- "$overlay" || cleanup_status=FAIL - fi - test ! -e "$overlay" || cleanup_status=FAIL - printf 'overlay_removed=%s\n' "$(test ! -e "$overlay" && printf yes || printf no)" \ - >>"$run/host-cleanup.log" - rm -rf -- "$case_tmp" - stat -c 'path=%n size=%s mode=%a inode=%i mtime=%Y ctime=%Z' \ - "$PRE15_QEMU_BASE_IMAGE" >"$run/base-identity-after.txt" || \ - cleanup_status=FAIL - if ! cmp -s "$run/base-identity-before.txt" "$run/base-identity-after.txt"; then - cleanup_status=FAIL - printf '%s\n' 'FAIL protected base metadata changed' \ - >>"$run/host-cleanup.log" - else - printf '%s\n' 'PASS protected base metadata unchanged' \ - >>"$run/host-cleanup.log" - fi - if test -n "$port"; then - if timeout 10s python3 -B - "$port" <<'PY' -import socket -import sys - -with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.settimeout(0.2) - raise SystemExit(0 if sock.connect_ex(("127.0.0.1", int(sys.argv[1]))) != 0 else 1) -PY - then - printf 'PASS dynamic port free: %s\n' "$port" \ - >>"$run/host-cleanup.log" - else - cleanup_status=FAIL - printf 'FAIL dynamic port remains open: %s\n' "$port" \ - >>"$run/host-cleanup.log" - fi - fi - printf '%s\n' "$cleanup_status" >"$run/cleanup.status" -} - -finish() -{ - rc=$? - trap - EXIT HUP INT TERM - cleanup - if test "$cleanup_status" != PASS; then - printf '%s\n' 'RUNNER_FAIL owned cleanup or base metadata check failed' \ - >>"$run/RESULT.txt" - rc=20 - fi - exit "$rc" -} - -trap finish EXIT HUP INT TERM - -for tool in cmp qemu-img qemu-system-x86_64 ssh stat timeout; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'INFRA_BLOCKED missing QEMU tool: %s\n' "$tool" >"$run/RESULT.txt" - exit 21 - } -done -test -f "$PRE15_QEMU_BASE_IMAGE" || { - printf '%s\n' 'INFRA_BLOCKED protected base image is absent' >"$run/RESULT.txt" - exit 21 -} -test -f "$PRE15_QEMU_SSH_KEY" || { - printf '%s\n' 'INFRA_BLOCKED SSH key is absent' >"$run/RESULT.txt" - exit 21 -} - -stat -c 'path=%n size=%s mode=%a inode=%i mtime=%Y ctime=%Z' \ - "$PRE15_QEMU_BASE_IMAGE" >"$run/base-identity-before.txt" -timeout -k 5 30 qemu-img info --output=json "$PRE15_QEMU_BASE_IMAGE" \ - >"$run/base-qemu-img-info.json" -timeout -k 5 30 qemu-img create -f qcow2 -F qcow2 \ - -b "$PRE15_QEMU_BASE_IMAGE" "$overlay" >"$run/qemu-img-create.stdout" \ - 2>"$run/qemu-img-create.stderr" - -port=$(timeout 10s python3 -B - <<'PY' -import socket - -while True: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - if port != 9222: - print(port) - break -PY -) -python3 -B - "$run/qemu-argv.json" "$overlay" "$port" "$serial" <<'PY' -import json -from pathlib import Path -import sys - -argv = [ - "qemu-system-x86_64", "-accel", "tcg,thread=multi", "-cpu", "qemu64", - "-m", "6144", "-smp", "4", - "-drive", f"file={sys.argv[2]},if=virtio,format=qcow2", - "-netdev", f"user,id=net0,hostfwd=tcp:127.0.0.1:{sys.argv[3]}-:22", - "-device", "virtio-net-pci,netdev=net0", "-display", "none", - "-serial", f"file:{sys.argv[4]}", "-monitor", "none", -] -Path(sys.argv[1]).write_text(json.dumps(argv, separators=(",", ":")) + "\n", encoding="ascii") -PY - -qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 -m 6144 -smp 4 \ - -drive "file=$overlay,if=virtio,format=qcow2" \ - -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$port-:22" \ - -device virtio-net-pci,netdev=net0 -display none -serial "file:$serial" \ - -monitor none >"$run/qemu.stdout" 2>"$qemu_log" & -qemu_pid=$! -printf '%s\n' "$qemu_pid" >"$run/qemu.pid" -test "$qemu_pid" != 26318 || { - printf '%s\n' 'RUNNER_FAIL QEMU PID collided with protected PID' >"$run/RESULT.txt" - exit 20 -} - -boot_marker=$run/ssh-ready -boot_start=$(date -u '+%s') -set +e -timeout -s KILL "$boot_timeout" env B32_QEMU_PID="$qemu_pid" \ - B32_QEMU_PORT="$port" B32_QEMU_USER="$user" \ - B32_QEMU_KEY="$PRE15_QEMU_SSH_KEY" B32_QEMU_MARKER="$boot_marker" \ - /bin/sh -c ' - while kill -0 "$B32_QEMU_PID" 2>/dev/null; do - if timeout -s KILL 8 ssh -o BatchMode=yes \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=5 -i "$B32_QEMU_KEY" \ - -p "$B32_QEMU_PORT" "$B32_QEMU_USER@127.0.0.1" true \ - >/dev/null 2>&1; then - : >"$B32_QEMU_MARKER" - exit 0 - fi - sleep 1 - done - exit 21 -' -boot_rc=$? -set -e -boot_end=$(date -u '+%s') -printf 'start_epoch=%s\nend_epoch=%s\nelapsed_seconds=%s\ndeadline_seconds=%s\nexit=%s\n' \ - "$boot_start" "$boot_end" "$((boot_end - boot_start))" "$boot_timeout" \ - "$boot_rc" >"$run/boot-timing.txt" -if test -f "$boot_marker"; then - ssh_ready=1 -elif ! kill -0 "$qemu_pid" 2>/dev/null; then - printf '%s\n' 'INFRA_BLOCKED QEMU exited before guest SSH' >"$run/RESULT.txt" - exit 21 -else - printf 'INFRA_BLOCKED guest SSH not ready within absolute %ss boot deadline\n' \ - "$boot_timeout" >"$run/RESULT.txt" - exit 21 -fi - -if ! timeout -k 5 30 ssh -MNf -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o ControlMaster=yes \ - -o "ControlPath=$control" -i "$PRE15_QEMU_SSH_KEY" -p "$port" \ - "$user@127.0.0.1"; then - printf '%s\n' 'INFRA_BLOCKED SSH ControlMaster failed' >"$run/RESULT.txt" - exit 21 -fi - -export PRE15_MODE=qemu -export PRE15_DUT=$dut PRE15_ROOT=$root PRE15_LIB_DIR=$lib -export PRE15_CASE_TMP=$case_tmp PRE15_RUN_DIR=$case_run -export PRE15_FREEBSD_SRC=$freebsd_src PRE15_B32_SCENARIO=$scenario -export PRE15_QEMU_CONTROL_PATH=$control PRE15_QEMU_SSH_PORT=$port -export PRE15_QEMU_SSH_USER=$user PRE15_QEMU_SSH_KEY -export PRE15_GUEST_COMMAND_TIMEOUT=$guest_timeout -export PRE15_OWNERSHIP_FILE=$ownership PRE15_CLEANUP_LOG=$cleanup_log -export PRE15_TARGET_MARKER_FILE=$target_marker PRE15_REASON_FILE=$reason_file -export PRE15_FIXTURE_HASHES=$fixture_hashes PRE15_MODULE_HASH=$module_hash - -set +e -timeout -k 10 "$case_timeout" /bin/sh "$case_script" \ - >"$case_run/stdout.log" 2>"$case_run/stderr.log" -case_rc=$? -set -e -case "$case_rc" in -0) - if test -f "$target_marker"; then - printf '%s PASS target=reached\n' "$scenario" >"$run/RESULT.txt" - else - printf '%s\n' 'RUNNER_FAIL case exited zero without target marker' \ - >"$run/RESULT.txt" - case_rc=20 - fi - ;; -10) printf '%s DUT_FAIL\n' "$scenario" >"$run/RESULT.txt" ;; -20) printf '%s RUNNER_FAIL\n' "$scenario" >"$run/RESULT.txt" ;; -21) printf '%s INFRA_BLOCKED\n' "$scenario" >"$run/RESULT.txt" ;; -124|137) printf '%s RUNNER_FAIL case-timeout=%ss\n' "$scenario" "$case_timeout" >"$run/RESULT.txt"; case_rc=20 ;; -*) printf '%s RUNNER_FAIL exit=%s\n' "$scenario" "$case_rc" >"$run/RESULT.txt"; case_rc=20 ;; -esac -exit "$case_rc" diff --git a/tests/pre15/fixtures/B33-cache-generate.py b/tests/pre15/fixtures/B33-cache-generate.py deleted file mode 100755 index 38b7147..0000000 --- a/tests/pre15/fixtures/B33-cache-generate.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import re -import shutil -import subprocess - - -EXTENT_RE = re.compile( - r"^\s*0:\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:\s*" - r"(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$", - re.MULTILINE, -) - - -def sha256_path(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def run(argv: list[str], cwd: Path) -> str: - completed = subprocess.run( - argv, - cwd=cwd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=120, - ) - if completed.returncode != 0: - raise SystemExit( - f"command failed ({completed.returncode}): {' '.join(argv)}\n" - f"{completed.stdout}" - ) - return completed.stdout - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--spec", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - spec = json.loads(args.spec.read_text(encoding="ascii")) - if ( - spec.get("schema") != 1 - or spec.get("batch") != "B33" - or spec.get("test") != "TC168-cache-inflight" - ): - raise SystemExit("invalid B33 cache fixture spec") - if args.output.exists(): - raise SystemExit(f"refusing existing B33 output: {args.output}") - for tool in ("dump.erofs", "mkfs.erofs"): - if shutil.which(tool) is None: - raise SystemExit(f"{tool} is required") - - pattern = b"Pre15-B33-cache-policy:0123456789abcdef\n" - expected = spec["source"] - source = (pattern * ((expected["size"] + len(pattern) - 1) // len(pattern)))[ - : expected["size"] - ] - if hashlib.sha256(source).hexdigest() != expected["sha256"]: - raise SystemExit("B33 deterministic source identity changed") - - source_dir = args.output / "source" - images = args.output / "images" - source_dir.mkdir(parents=True) - images.mkdir() - payload = source_dir / "payload.bin" - payload.write_bytes(source) - os.utime(payload, (0, 0)) - os.utime(source_dir, (0, 0)) - - valid = images / "lz4-valid.erofs" - run(["mkfs.erofs", *spec["mkfs_args"], str(valid), str(source_dir)], images) - extent_text = run( - ["dump.erofs", "-e", "--path=/payload.bin", str(valid)], images - ) - match = EXTENT_RE.search(extent_text) - if match is None: - raise SystemExit("B33 single extent could not be parsed") - logical, logical_end, logical_length, physical, physical_end, physical_length = ( - map(int, match.groups()) - ) - if ( - logical != 0 - or logical_end != expected["size"] - or logical_length != expected["size"] - or physical_end - physical != physical_length - or physical_length != 4096 - ): - raise SystemExit("B33 fixture is not one 64 KiB logical pcluster") - - valid_data = valid.read_bytes() - if physical + physical_length > len(valid_data): - raise SystemExit("B33 physical extent exceeds image") - pcluster = valid_data[physical : physical + physical_length] - leading = next( - (index for index, value in enumerate(pcluster) if value), len(pcluster) - ) - stream = pcluster[leading:] - if leading == len(pcluster) or len(stream) < 2: - raise SystemExit("B33 LZ4 stream boundary is empty") - truncated_data = bytearray(valid_data) - truncated_data[physical + leading : physical + physical_length] = ( - b"\0" + stream[:-1] - ) - if truncated_data == valid_data: - raise SystemExit("B33 truncated mutation changed no bytes") - truncated = images / "lz4-truncated.erofs" - truncated.write_bytes(truncated_data) - - records = [] - for image_class, path, expected_errno in ( - ("valid", valid, 0), - ("truncated", truncated, 97), - ): - records.append( - { - "class": image_class, - "expected_errno": expected_errno, - "path": path.name, - "sha256": sha256_path(path), - } - ) - index = { - "batch": "B33", - "decoded_size": len(source), - "fixture_count": len(records), - "fixtures": records, - "leading_zero_bytes": leading, - "physical_length": physical_length, - "physical_offset": physical, - "schema": 1, - "source_sha256": sha256_path(payload), - "status": "READY", - "test": "TC168-cache-inflight", - } - (args.output / "fixture-index.json").write_text( - json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - print(json.dumps(index, indent=2, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/tests/pre15/fixtures/B33-cache-oracle.c b/tests/pre15/fixtures/B33-cache-oracle.c deleted file mode 100644 index 3cb0bd3..0000000 --- a/tests/pre15/fixtures/B33-cache-oracle.c +++ /dev/null @@ -1,418 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#define CODEC_COUNT 4 -#define MOUNT_BUDGET (256U * 1024) -#define GLOBAL_BUDGET (512U * 1024) -#define MINIMUM_WORK (128U * 1024) - -enum cache_state { - CACHE_EMPTY, - CACHE_INFLIGHT, - CACHE_READY, -}; - -enum cache_lookup { - CACHE_BYPASS, - CACHE_HIT, - CACHE_OWNER, -}; - -struct metric { - uint64_t hits; - uint64_t misses; - uint64_t bypasses; - uint64_t evictions; - uint64_t reclaims; - size_t resident_bytes; -}; - -struct budget { - size_t limit; - size_t used; - size_t peak; -}; - -struct key { - uint64_t identity; - uint8_t codec; -}; - -struct cache { - struct budget *global; - struct metric metrics[CODEC_COUNT]; - struct key key; - unsigned char *data; - size_t charged_bytes; - size_t budget_bytes; - size_t peak_bytes; - enum cache_state state; - bool enabled; -}; - -static void -fail(const char *message) -{ - fprintf(stderr, "B33 cache oracle failure: %s\n", message); - exit(1); -} - -static uint64_t -hash_bytes(const unsigned char *data, size_t size) -{ - uint64_t hash; - size_t index; - - hash = UINT64_C(1469598103934665603); - for (index = 0; index < size; ++index) { - hash ^= data[index]; - hash *= UINT64_C(1099511628211); - } - return (hash); -} - -static void -fill(unsigned char *data, size_t size, const struct key *key) -{ - size_t index; - - for (index = 0; index < size; ++index) - data[index] = (unsigned char)(key->identity + key->codec * 31 + - index * 17); -} - -static bool -key_equal(const struct key *left, const struct key *right) -{ - - return (left->identity == right->identity && left->codec == right->codec); -} - -static bool -reserve(struct budget *budget, size_t bytes) -{ - - if (bytes > budget->limit || budget->used > budget->limit - bytes) - return (false); - budget->used += bytes; - if (budget->used > budget->peak) - budget->peak = budget->used; - return (true); -} - -static void -release(struct budget *budget, size_t bytes) -{ - - if (bytes > budget->used) - fail("global budget underflow"); - budget->used -= bytes; -} - -static void -cache_init(struct cache *cache, struct budget *global, bool enabled) -{ - - memset(cache, 0, sizeof(*cache)); - cache->global = global; - cache->budget_bytes = MOUNT_BUDGET; - cache->enabled = enabled; -} - -static bool -policy_admit(const struct cache *cache, uint8_t codec, size_t decoded_size, - size_t compressed_size) -{ - - return (cache->enabled && codec < CODEC_COUNT && - decoded_size <= cache->budget_bytes && - compressed_size + decoded_size >= MINIMUM_WORK); -} - -static void -drop(struct cache *cache, bool evicted, bool reclaimed) -{ - struct metric *metric; - - if (cache->charged_bytes == 0) - return; - if (cache->key.codec >= CODEC_COUNT) - fail("charged unknown codec"); - metric = &cache->metrics[cache->key.codec]; - if (cache->state == CACHE_READY) { - if (metric->resident_bytes != cache->charged_bytes) - fail("codec resident accounting mismatch"); - metric->resident_bytes = 0; - if (evicted) - ++metric->evictions; - if (reclaimed) - ++metric->reclaims; - } - release(cache->global, cache->charged_bytes); - cache->charged_bytes = 0; - free(cache->data); - cache->data = NULL; -} - -static enum cache_lookup -claim(struct cache *cache, const struct key *key, size_t decoded_size, - size_t compressed_size, unsigned char *output) -{ - struct metric *metric; - - if (key->codec >= CODEC_COUNT) - fail("unknown requested codec"); - metric = &cache->metrics[key->codec]; - if (!policy_admit(cache, key->codec, decoded_size, compressed_size)) { - ++metric->bypasses; - return (CACHE_BYPASS); - } - if (cache->state == CACHE_READY && key_equal(&cache->key, key)) { - memcpy(output, cache->data, decoded_size); - ++metric->hits; - return (CACHE_HIT); - } - if (cache->state == CACHE_INFLIGHT) { - ++metric->bypasses; - return (CACHE_BYPASS); - } - if (cache->state == CACHE_READY) - drop(cache, true, false); - cache->state = CACHE_EMPTY; - if (!reserve(cache->global, decoded_size)) { - ++metric->bypasses; - return (CACHE_BYPASS); - } - cache->charged_bytes = decoded_size; - if (decoded_size > cache->peak_bytes) - cache->peak_bytes = decoded_size; - cache->key = *key; - cache->state = CACHE_INFLIGHT; - ++metric->misses; - return (CACHE_OWNER); -} - -static void -complete(struct cache *cache, const struct key *key, unsigned char *data, - int error) -{ - struct metric *metric; - - if (cache->state != CACHE_INFLIGHT || !key_equal(&cache->key, key)) - fail("owner lost key"); - metric = &cache->metrics[key->codec]; - if (error != 0) { - if (data != NULL || error < 1) - fail("failure publication is untyped"); - release(cache->global, cache->charged_bytes); - cache->charged_bytes = 0; - cache->state = CACHE_EMPTY; - return; - } - if (data == NULL || metric->resident_bytes != 0) - fail("success publication is untyped"); - cache->data = data; - metric->resident_bytes = cache->charged_bytes; - cache->state = CACHE_READY; -} - -static bool -reclaim(struct cache *cache) -{ - - if (cache->state == CACHE_INFLIGHT) - return (false); - if (cache->state == CACHE_READY) { - drop(cache, true, true); - cache->state = CACHE_EMPTY; - } - return (true); -} - -static void -cache_fini(struct cache *cache) -{ - - if (cache->state == CACHE_INFLIGHT) - fail("unmount did not drain owner"); - drop(cache, false, false); - cache->state = CACHE_EMPTY; -} - -static unsigned char * -make_data(const struct key *key, size_t size) -{ - unsigned char *data; - - data = malloc(size); - if (data == NULL) - fail("decoded allocation"); - fill(data, size, key); - return (data); -} - -static void -test_codec_accounting(struct budget *global, struct metric output[CODEC_COUNT], - uint64_t *eviction_hash) -{ - struct cache cache; - struct key key; - unsigned char *expected, *readback; - uint64_t before, after; - unsigned int codec; - - cache_init(&cache, global, true); - expected = malloc(MOUNT_BUDGET); - readback = malloc(MOUNT_BUDGET); - if (expected == NULL || readback == NULL) - fail("accounting buffers"); - before = 0; - for (codec = 0; codec < CODEC_COUNT; ++codec) { - key = (struct key){ .identity = 100 + codec, .codec = codec }; - fill(expected, MOUNT_BUDGET, &key); - if (claim(&cache, &key, MOUNT_BUDGET, 4096, readback) != CACHE_OWNER) - fail("codec-neutral request was not admitted"); - complete(&cache, &key, make_data(&key, MOUNT_BUDGET), 0); - if (claim(&cache, &key, MOUNT_BUDGET, 4096, readback) != CACHE_HIT || - memcmp(readback, expected, MOUNT_BUDGET) != 0) - fail("codec cache hit changed bytes"); - if (codec == 0) - before = hash_bytes(readback, MOUNT_BUDGET); - } - key = (struct key){ .identity = 100, .codec = 0 }; - if (claim(&cache, &key, MOUNT_BUDGET, 4096, readback) != CACHE_OWNER) - fail("evicted codec key was not reusable"); - complete(&cache, &key, make_data(&key, MOUNT_BUDGET), 0); - if (claim(&cache, &key, MOUNT_BUDGET, 4096, readback) != CACHE_HIT) - fail("reused codec key did not hit"); - after = hash_bytes(readback, MOUNT_BUDGET); - if (before != after) - fail("eviction changed decoded hash"); - if (!reclaim(&cache) || global->used != 0) - fail("ready reclaim did not release bytes"); - if (cache.peak_bytes != MOUNT_BUDGET) - fail("mount peak did not equal hard budget"); - memcpy(output, cache.metrics, sizeof(cache.metrics)); - *eviction_hash = after; - free(readback); - free(expected); -} - -static uint64_t -test_global_exhaustion(struct budget *global) -{ - struct cache first, second, third; - struct key first_key = { .identity = 201, .codec = 0 }; - struct key second_key = { .identity = 202, .codec = 2 }; - struct key third_key = { .identity = 203, .codec = 3 }; - unsigned char *scratch; - uint64_t fallback_hash; - - cache_init(&first, global, true); - cache_init(&second, global, true); - cache_init(&third, global, true); - scratch = malloc(MOUNT_BUDGET); - if (scratch == NULL) - fail("global fallback allocation"); - if (claim(&first, &first_key, MOUNT_BUDGET, 4096, scratch) != CACHE_OWNER || - claim(&second, &second_key, MOUNT_BUDGET, 4096, scratch) != CACHE_OWNER) - fail("two mounts did not fill global budget"); - complete(&first, &first_key, make_data(&first_key, MOUNT_BUDGET), 0); - complete(&second, &second_key, make_data(&second_key, MOUNT_BUDGET), 0); - if (global->used != GLOBAL_BUDGET || global->peak != GLOBAL_BUDGET) - fail("global hard budget was not reached exactly"); - if (claim(&third, &third_key, MOUNT_BUDGET, 4096, scratch) != CACHE_BYPASS) - fail("global exhaustion did not bypass"); - fill(scratch, MOUNT_BUDGET, &third_key); - fallback_hash = hash_bytes(scratch, MOUNT_BUDGET); - if (!reclaim(&first) || - claim(&third, &third_key, MOUNT_BUDGET, 4096, scratch) != CACHE_OWNER) - fail("reclaimed global bytes were not reusable"); - complete(&third, &third_key, make_data(&third_key, MOUNT_BUDGET), 0); - cache_fini(&second); - cache_fini(&third); - if (global->used != 0) - fail("unmount did not release global bytes"); - free(scratch); - return (fallback_hash); -} - -static void -test_failure_and_policy(struct budget *global) -{ - struct cache cache, disabled; - struct key key = { .identity = 301, .codec = 1 }; - unsigned char output[MOUNT_BUDGET]; - unsigned int codec; - - cache_init(&cache, global, true); - if (claim(&cache, &key, MOUNT_BUDGET, 4096, output) != CACHE_OWNER) - fail("failure setup was not admitted"); - if (reclaim(&cache)) - fail("inflight reclaim did not report busy"); - complete(&cache, &key, NULL, EIO); - if (global->used != 0 || - claim(&cache, &key, MOUNT_BUDGET, 4096, output) != CACHE_OWNER) - fail("failure did not release budget for retry"); - complete(&cache, &key, make_data(&key, MOUNT_BUDGET), 0); - cache_fini(&cache); - - cache_init(&disabled, global, false); - for (codec = 0; codec < CODEC_COUNT; ++codec) { - key.codec = codec; - if (claim(&disabled, &key, MOUNT_BUDGET, 4096, output) != CACHE_BYPASS) - fail("disabled codec policy did not bypass"); - disabled.enabled = true; - if (claim(&disabled, &key, 64U * 1024, 1, output) != CACHE_BYPASS) - fail("low-work policy did not bypass"); - disabled.enabled = false; - } - disabled.enabled = true; - if (claim(&disabled, &key, MOUNT_BUDGET + 1, 4096, output) != CACHE_BYPASS) - fail("oversized policy did not bypass"); - cache_fini(&disabled); - if (global->used != 0) - fail("policy tests leaked global bytes"); -} - -int -main(void) -{ - struct budget global = { .limit = GLOBAL_BUDGET }; - struct metric metrics[CODEC_COUNT]; - uint64_t eviction_hash, fallback_hash; - unsigned int codec; - - test_codec_accounting(&global, metrics, &eviction_hash); - fallback_hash = test_global_exhaustion(&global); - test_failure_and_policy(&global); - for (codec = 0; codec < CODEC_COUNT; ++codec) { - if (metrics[codec].hits == 0 || metrics[codec].misses == 0 || - metrics[codec].evictions == 0 || metrics[codec].resident_bytes != 0) - fail("four-codec metrics are incomplete"); - } - printf("{\"status\":\"PASS\",\"global_limit\":%u," - "\"global_peak\":%zu,\"global_remaining\":%zu," - "\"mount_limit\":%u,\"eviction_hash\":\"%016" PRIx64 "\"," - "\"fallback_hash\":\"%016" PRIx64 "\",\"metrics\":[", - GLOBAL_BUDGET, global.peak, global.used, MOUNT_BUDGET, - eviction_hash, fallback_hash); - for (codec = 0; codec < CODEC_COUNT; ++codec) { - if (codec != 0) - printf(","); - printf("{\"codec\":%u,\"hits\":%" PRIu64 - ",\"misses\":%" PRIu64 ",\"bypasses\":%" PRIu64 - ",\"evictions\":%" PRIu64 ",\"reclaims\":%" PRIu64 - ",\"resident_bytes\":%zu}", codec, metrics[codec].hits, - metrics[codec].misses, metrics[codec].bypasses, - metrics[codec].evictions, metrics[codec].reclaims, - metrics[codec].resident_bytes); - } - printf("]}\n"); - return (0); -} diff --git a/tests/pre15/fixtures/B33-cache-state.json b/tests/pre15/fixtures/B33-cache-state.json deleted file mode 100644 index 1906817..0000000 --- a/tests/pre15/fixtures/B33-cache-state.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "batch": "B33", - "mkfs_args": [ - "-T0", - "-U33000000-0000-4000-8000-000000000038", - "--all-root", - "-x-1", - "--workers=1", - "-zlz4", - "-C4096", - "-Elegacy-compress" - ], - "schema": 1, - "source": { - "sha256": "55e7e309853ecccb5c891b611b630c9c764d07e340331ff62080075e1ddad89c", - "size": 65536 - }, - "test": "TC168-cache-inflight" -} diff --git a/tests/pre15/fixtures/B33-qemu-run.sh b/tests/pre15/fixtures/B33-qemu-run.sh deleted file mode 100755 index f82cf5d..0000000 --- a/tests/pre15/fixtures/B33-qemu-run.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -set -eu - -script=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/B33-qemu-run.sh -source_runner=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/B32-qemu-run.sh - -if test "${1:-}" != --worker; then - test "$#" -eq 1 && test "$1" = TC168-cache-inflight || { - printf '%s\n' 'usage: B33-qemu-run.sh TC168-cache-inflight' >&2 - exit 20 - } - total=${PRE15_QEMU_TIMEOUT:-1200} - case "$total" in - ''|*[!0-9]*|0) printf '%s\n' 'invalid PRE15_QEMU_TIMEOUT' >&2; exit 20 ;; - esac - exec timeout -k 30 "$total" "$script" --worker "$1" -fi - -shift -test "$#" -eq 1 && test "$1" = TC168-cache-inflight || exit 20 -test -f "$source_runner" || { - printf '%s\n' 'INFRA_BLOCKED B32 absolute-timeout runner is absent' >&2 - exit 21 -} - -sed \ - -e "s|^script=.*$|script=$source_runner|" \ - -e 's|case_script=$dut/tests/pre15/cases/B32-cache-state.sh|case_script=$dut/tests/pre15/cases/B33-cache-policy.sh|' \ - -e 's/-qemu-B32-/-qemu-B33-/' \ - "$source_runner" | /bin/sh -s -- --worker "$1" diff --git a/tests/pre15/fixtures/SMOKE-generate.py b/tests/pre15/fixtures/SMOKE-generate.py deleted file mode 100755 index fb22fc0..0000000 --- a/tests/pre15/fixtures/SMOKE-generate.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import hashlib -import json -from pathlib import Path - - -def digest(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -parser = argparse.ArgumentParser() -parser.add_argument("--output", type=Path, required=True) -arguments = parser.parse_args() -root = arguments.output -root.mkdir(parents=True) -(root / "nested").mkdir() -payload = bytearray() -for offset in range(384 * 1024): - payload.append(((offset * 131) ^ (offset >> 3) ^ 0x5A) & 0xFF) -(root / "payload.bin").write_bytes(payload) -(root / "nested" / "alpha.txt").write_text( - "Pre15 deterministic smoke fixture\n", encoding="ascii" -) -(root / "empty").write_bytes(b"") -files = [] -for path in sorted(item for item in root.rglob("*") if item.is_file()): - files.append( - { - "path": path.relative_to(root).as_posix(), - "sha256": digest(path), - "size": path.stat().st_size, - } - ) -manifest = { - "files": files, - "nested_entries": ["alpha.txt"], - "root_entries": ["empty", "nested", "payload.bin"], - "status": "READY", -} -print(json.dumps(manifest, indent=2, sort_keys=True)) diff --git a/tests/pre15/fixtures/SMOKE-kldsym.c b/tests/pre15/fixtures/SMOKE-kldsym.c deleted file mode 100644 index ec0c4c3..0000000 --- a/tests/pre15/fixtures/SMOKE-kldsym.c +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include - -#include -#include -#include - -int -main(int argc, char **argv) -{ - struct kld_sym_lookup lookup; - int failed; - - if (argc < 2) { - fprintf(stderr, "usage: SMOKE-kldsym SYMBOL...\n"); - return (2); - } - failed = 0; - for (int index = 1; index < argc; ++index) { - memset(&lookup, 0, sizeof(lookup)); - lookup.version = sizeof(lookup); - lookup.symname = argv[index]; - if (kldsym(0, KLDSYM_LOOKUP, &lookup) == -1) { - printf("MISSING %s\n", argv[index]); - failed = 1; - } else { - printf("PRESENT %s 0x%jx\n", argv[index], - (uintmax_t)lookup.symvalue); - } - } - return (failed); -} diff --git a/tests/pre15/fixtures/g3.py b/tests/pre15/fixtures/g3.py deleted file mode 100755 index 8c5ff8d..0000000 --- a/tests/pre15/fixtures/g3.py +++ /dev/null @@ -1,1142 +0,0 @@ -#!/usr/bin/env python3 -"""Generate, compare, and verify deterministic Pre15 G3 fixtures.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import shutil -import stat -import struct -import subprocess -import sys - - -TESTS_DIR = Path(__file__).resolve().parents[2] -if str(TESTS_DIR) not in sys.path: - sys.path.insert(0, str(TESTS_DIR)) - -from erofs_fixture import ErofsImage, MAGIC, SUPER - - -FLAT_PLAIN = 0 -HUGE_DIRECTORY_BLOCKS = (1 << 31) + 1 -S_IFMT = 0o170000 -FEATURE_COMPAT_SB_CHKSUM = 0x00000001 -FEATURE_COMPAT_PLAIN_XATTR_PFX = 0x00000010 -FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040 -FEATURE_INCOMPAT_METABOX = 0x00000100 -EROFS_NAME_LEN = 255 - - -class FixtureError(RuntimeError): - pass - - -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def write_json(path: Path, value: object) -> None: - with path.open("x", encoding="ascii") as output: - json.dump(value, output, ensure_ascii=True, indent=2, sort_keys=True) - output.write("\n") - - -def require_tools(*names: str) -> None: - missing = [name for name in names if shutil.which(name) is None] - if missing: - raise FixtureError(f"missing tools: {', '.join(missing)}") - - -def run(argv: list[str], *, capture: bool = False) -> str: - result = subprocess.run( - argv, - check=True, - text=True, - stdout=subprocess.PIPE if capture else None, - stderr=subprocess.STDOUT if capture else None, - ) - return result.stdout if capture else "" - - -def set_epoch(path: Path) -> None: - for entry in sorted(path.rglob("*"), reverse=True): - os.utime(entry, (0, 0), follow_symlinks=False) - os.utime(path, (0, 0), follow_symlinks=False) - - -def deterministic_bytes(length: int) -> bytes: - return bytes( - ((index * 131 + 17) ^ (index >> 3)) & 0xFF - for index in range(length) - ) - - -def put_u16(image: bytearray, offset: int, value: int) -> None: - struct.pack_into(" None: - struct.pack_into(" None: - struct.pack_into(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - erofs = ErofsImage(image, Path("generated")) - erofs.update_checksum() - - -def write_image(path: Path, image: bytearray) -> None: - update_superblock_checksum(image) - path.write_bytes(image) - - -def build_image( - artifact_dir: Path, - fixture_dir: Path, - uuid: str, - image: str, - source: str, - *options: str, -) -> list[str]: - argv = [ - "mkfs.erofs", - "-d0", - "-x-1", - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "-U", - uuid, - *options, - str(artifact_dir / image), - str(fixture_dir / source), - ] - run(argv) - return argv - - -def inode_offset(image: bytes | bytearray, nid: int) -> int: - block_bits = image[SUPER + 12] - meta_blkaddr = u32(image, SUPER + 40) - return (meta_blkaddr << block_bits) + (nid << 5) - - -def compact_inode(image: bytes | bytearray, nid: int) -> tuple[int, int, int]: - offset = inode_offset(image, nid) - inode_format = u16(image, offset) - if inode_format & 1: - raise FixtureError("expected compact inode") - return offset, (inode_format >> 1) & 7, u32(image, offset + 8) - - -def inline_dir_entries( - image: bytes | bytearray, nid: int -) -> tuple[int, int, int, list[tuple[int, int, int, bytes]]]: - inode = inode_offset(image, nid) - inode_format = u16(image, inode) - inode_size = 64 if inode_format & 1 else 32 - layout = (inode_format >> 1) & 7 - size = u64(image, inode + 8) if inode_size == 64 else u32(image, inode + 8) - if layout != 2 or u16(image, inode + 2) != 0: - raise FixtureError("expected inline directory without xattrs") - data = inode + inode_size - first_nameoff = u16(image, data + 8) - if first_nameoff < 12 or first_nameoff % 12: - raise FixtureError("invalid first directory name offset") - count = first_nameoff // 12 - entries = [] - for index in range(count): - entry = data + index * 12 - child_nid, nameoff = struct.unpack_from(" tuple[int, int, int, int, int, int]: - offset = inode_offset(image, nid) - inode_format = u16(image, offset) - inode_size = 64 if inode_format & 1 else 32 - size = u64(image, offset + 8) if inode_size == 64 else u32(image, offset + 8) - xattr_count = u16(image, offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) - layout = (inode_format >> 1) & 7 - return offset, inode_format, inode_size, xattr_size, layout, size - - -def directory_entries(image: bytes | bytearray, nid: int) -> list[tuple[bytes, int]]: - offset, _, inode_size, xattr_size, layout, size = inode_info(image, nid) - block_size = 1 << image[SUPER + 12] - if not 0 < size <= block_size: - raise FixtureError("directory must fit one block") - if layout == 2: - data = offset + inode_size + xattr_size - elif layout == 0: - data = u32(image, offset + 16) << image[SUPER + 12] - else: - raise FixtureError(f"unsupported directory layout {layout}") - first_nameoff = u16(image, data + 8) - if first_nameoff < 12 or first_nameoff % 12: - raise FixtureError("invalid first directory name offset") - count = first_nameoff // 12 - entries = [] - for index in range(count): - entry = data + index * 12 - child = u64(image, entry) - nameoff = u16(image, entry + 8) - endoff = u16(image, entry + 20) if index + 1 < count else size - name = bytes(image[data + nameoff : data + endoff]).split(b"\0", 1)[0] - if not name: - raise FixtureError("empty directory name") - entries.append((name, child)) - return entries - - -def child_nid(image: bytes | bytearray, parent_nid: int, name: bytes) -> int: - return next(nid for entry_name, nid in directory_entries(image, parent_nid) if entry_name == name) - - -def convert_inline_directory_to_plain(image: bytearray, nid: int) -> tuple[int, int]: - offset, inode_format, inode_size, xattr_size, layout, size = inode_info(image, nid) - if layout != 2 or inode_size != 64 or size <= 0: - raise FixtureError("expected nonempty extended inline directory") - block_size = 1 << image[SUPER + 12] - tail_size = size % block_size - if tail_size == 0: - raise FixtureError("expected inline directory tail") - raw_block = u32(image, offset + 16) - full_size = size - tail_size - inline_offset = offset + inode_size + xattr_size - directory_data = bytes( - image[raw_block * block_size : raw_block * block_size + full_size] - + image[inline_offset : inline_offset + tail_size] - ) - if len(directory_data) != size: - raise FixtureError("directory data reconstruction failed") - new_raw_block = (len(image) + block_size - 1) // block_size - image.extend(b"\0" * (new_raw_block * block_size - len(image))) - image.extend(directory_data) - image.extend(b"\0" * (-len(image) % block_size)) - put_u16(image, offset, inode_format & ~(7 << 1)) - put_u32(image, offset + 16, new_raw_block) - put_u32(image, SUPER + 36, len(image) // block_size) - if inode_info(image, nid)[4] != FLAT_PLAIN: - raise FixtureError("directory layout conversion failed") - return new_raw_block, len(image) // block_size - - -def write_checksums(directory: Path) -> None: - with (directory / "SHA256SUMS").open("x", encoding="ascii") as sums: - for path in sorted(directory.glob("*.erofs")): - sums.write(f"{sha256(path)} {path.name}\n") - - -def make_metadata(fixture_dir: Path, artifact_dir: Path) -> list[list[str]]: - require_tools("mkfs.erofs", "fsck.erofs", "dump.erofs") - if fixture_dir.exists() or artifact_dir.exists(): - raise FixtureError("metadata source and artifact paths must be absent") - for relative in ( - "inline", - "special", - "namei/alpha/bravo/charlie", - "namei/alpha/sibling", - "namei/wide", - "pager", - "nfs/basic/subdir", - "nlink/subdir", - ): - (fixture_dir / relative).mkdir(parents=True, exist_ok=True) - artifact_dir.mkdir(parents=True) - (fixture_dir / "inline/inline.txt").write_text( - "inline-tail-payload-0123456789\n", encoding="ascii" - ) - (fixture_dir / "special/target").write_text("special target\n", encoding="ascii") - os.symlink("target", fixture_dir / "special/link") - os.mknod( - fixture_dir / "special/char-large", - stat.S_IFCHR | 0o666, - os.makedev(2748, 344865), - ) - os.mknod( - fixture_dir / "special/block-large", - stat.S_IFBLK | 0o666, - os.makedev(2748, 344865), - ) - os.mkfifo(fixture_dir / "special/fifo") - (fixture_dir / "namei/alpha/bravo/charlie/payload.txt").write_text( - "cold nested lookup payload\n", encoding="ascii" - ) - (fixture_dir / "namei/alpha/bravo/repeat.txt").write_text( - "repeat lookup payload\n", encoding="ascii" - ) - (fixture_dir / "namei/alpha/sibling/marker.txt").write_text( - "sibling marker\n", encoding="ascii" - ) - for index in range(320): - name = f"entry-{index:03d}-abcdefghijklmnopqrstuvwxyz.txt" - (fixture_dir / "namei/wide" / name).write_text( - f"wide entry {index:03d}\n", encoding="ascii" - ) - (fixture_dir / "pager/pager.bin").write_bytes(deterministic_bytes(5 * 4096 + 731)) - (fixture_dir / "nfs/basic/regular.txt").write_text( - "stable file handle payload\n", encoding="ascii" - ) - (fixture_dir / "nfs/basic/subdir/child.txt").write_text("child\n", encoding="ascii") - os.symlink("regular.txt", fixture_dir / "nfs/basic/link-to-regular") - os.mkfifo(fixture_dir / "nfs/basic/test.fifo") - (fixture_dir / "nlink/single.txt").write_text("single\n", encoding="ascii") - (fixture_dir / "nlink/hard-a.txt").write_text("hardlinked\n", encoding="ascii") - os.link(fixture_dir / "nlink/hard-a.txt", fixture_dir / "nlink/hard-b.txt") - (fixture_dir / "nlink/subdir/child.txt").write_text("child\n", encoding="ascii") - set_epoch(fixture_dir) - - commands = [ - build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555551", "inline.erofs", "inline"), - build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555552", "special-compact.erofs", "special", "-E", "force-inode-compact"), - build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555553", "special-extended.erofs", "special", "-E", "force-inode-extended"), - build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555554", "namei-base.erofs", "namei"), - build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555555", "pager-plain.erofs", "pager", "-E", "noinline_data"), - build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555556", "pager-lz4.erofs", "pager", "-z", "lz4"), - build_image(artifact_dir, fixture_dir, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee1", "nfs-a.erofs", "nfs"), - build_image(artifact_dir, fixture_dir, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee2", "nfs-b.erofs", "nfs"), - build_image(artifact_dir, fixture_dir, "11111111-2222-3333-4444-555555555557", "nlink.erofs", "nlink", "-E", "force-inode-compact"), - ] - - inline = bytearray((artifact_dir / "inline.erofs").read_bytes()) - _, _, _, inline_root = inline_dir_entries(inline, u16(inline, SUPER + 14)) - inline_nid = next(nid for nid, _, _, name in inline_root if name == b"inline.txt") - inline_inode, layout, inline_size = compact_inode(inline, inline_nid) - if layout != 2 or u16(inline, inline_inode + 2) != 0: - raise FixtureError("inline fixture has unexpected layout") - inline_cross_xattr_icount = 695 - inline_xattr_size = 12 + 4 * (inline_cross_xattr_icount - 1) - inline_data_off = inline_inode + 32 + inline_xattr_size - block_size = 1 << inline[SUPER + 12] - if inline_data_off % block_size + inline_size <= block_size: - raise FixtureError("inline fixture does not cross the block boundary") - inline_cross = bytearray(inline) - put_u16(inline_cross, inline_inode + 2, inline_cross_xattr_icount) - write_image(artifact_dir / "inline-cross-block.erofs", inline_cross) - - namei = bytearray((artifact_dir / "namei-base.erofs").read_bytes()) - root_nid = u16(namei, SUPER + 14) - root_inode, root_data, _, root_entries = inline_dir_entries(namei, root_nid) - wide_nid = next(nid for nid, _, _, name in root_entries if name == b"wide") - wide_inode, wide_layout, _ = compact_inode(namei, wide_nid) - if wide_layout != 2: - raise FixtureError("wide directory is not inline") - block_bits = namei[SUPER + 12] - wide_block = u32(namei, wide_inode + 16) << block_bits - wide_first_nameoff = u16(namei, wide_block + 8) - wide_count = wide_first_nameoff // 12 - wide_last_nameoff = u16(namei, wide_block + (wide_count - 1) * 12 + 8) - padding_nul = namei.index(0, wide_block + wide_last_nameoff, wide_block + (1 << block_bits)) - nonzero_padding = bytearray(namei) - nonzero_padding[padding_nul + 1 : padding_nul + 9] = b"PAD!ERO!" - write_image(artifact_dir / "namei-padding-nonzero.erofs", nonzero_padding) - short_block = bytearray(namei) - put_u32(short_block, root_inode + 8, 8) - write_image(artifact_dir / "namei-corrupt-short.erofs", short_block) - bad_nameoff = bytearray(namei) - first_nameoff = u16(bad_nameoff, root_data + 8) - put_u16(bad_nameoff, root_data + 20, first_nameoff) - write_image(artifact_dir / "namei-corrupt-nameoff.erofs", bad_nameoff) - bad_name = bytearray(namei) - slash_offset = wide_block + wide_last_nameoff + 5 - if bad_name[slash_offset] in (0, ord("/")): - raise FixtureError("name corruption target is unsuitable") - bad_name[slash_offset] = ord("/") - write_image(artifact_dir / "namei-corrupt-name.erofs", bad_name) - - nlink = bytearray((artifact_dir / "nlink.erofs").read_bytes()) - nlink_root_nid = u16(nlink, SUPER + 14) - _, _, _, nlink_entries = inline_dir_entries(nlink, nlink_root_nid) - single_nid = next(nid for nid, _, _, name in nlink_entries if name == b"single.txt") - single_inode, _, _ = compact_inode(nlink, single_nid) - nlink1 = bytearray(nlink) - put_u16(nlink1, single_inode, u16(nlink1, single_inode) | 0x10) - put_u16(nlink1, single_inode + 6, 0x1234) - write_image(artifact_dir / "nlink1-patched.erofs", nlink1) - - with (artifact_dir / "fixture-evidence.txt").open("x", encoding="ascii") as output: - output.write( - f"inline_nid={inline_nid} inode_off={inline_inode} " - f"inode_blockoff={inline_inode % block_size} " - f"xattr_icount=0->{inline_cross_xattr_icount} " - f"inline_data_blockoff={inline_data_off % block_size} " - f"inline_size={inline_size}\n" - ) - output.write( - f"wide_nid={wide_nid} block={wide_block >> block_bits} " - f"dirents={wide_count} last_nameoff={wide_last_nameoff} " - f"padding_patch={padding_nul + 1 - wide_block}:" - f"{padding_nul + 9 - wide_block}\n" - ) - output.write(f"nlink1_nid={single_nid} i_format_bit4=1 i_nb=0x1234\n") - for image_name in ("special-compact.erofs", "special-extended.erofs"): - image = bytearray((artifact_dir / image_name).read_bytes()) - special_root = u16(image, SUPER + 14) - _, _, _, entries = inline_dir_entries(image, special_root) - output.write(image_name + "\n") - for nid, _, _, name in entries: - if name not in (b"char-large", b"block-large", b"fifo"): - continue - offset = inode_offset(image, nid) - output.write( - f" {name.decode()} nid={nid} inode_size=" - f"{64 if u16(image, offset) & 1 else 32} " - f"raw_u={u32(image, offset + 16):#010x}\n" - ) - write_checksums(artifact_dir) - run(["fsck.erofs", "-d1", str(artifact_dir / "namei-padding-nonzero.erofs")]) - run( - [ - "dump.erofs", - "--cat", - "--path=/wide/entry-079-abcdefghijklmnopqrstuvwxyz.txt", - str(artifact_dir / "namei-padding-nonzero.erofs"), - ] - ) - return commands - - -def make_final(fixture_dir: Path, artifact_dir: Path) -> list[list[str]]: - require_tools("mkfs.erofs") - if fixture_dir.exists() or artifact_dir.exists(): - raise FixtureError("final source and artifact paths must be absent") - for relative in ("fallback", "oversize", "extent", "large/huge"): - (fixture_dir / relative).mkdir(parents=True, exist_ok=True) - artifact_dir.mkdir(parents=True) - (fixture_dir / "fallback/root.txt").write_text("48-bit fallback root\n", encoding="ascii") - (fixture_dir / "oversize/big.dat").write_bytes(b"x") - (fixture_dir / "extent/hole.dat").write_bytes(b"\0" * (1024 * 1024)) - (fixture_dir / "large/huge/anchor.txt").write_text( - "large directory anchor\n", encoding="ascii" - ) - for index in range(400): - (fixture_dir / "large/huge" / f"entry-{index:03d}-abcdefghijklmnopqrstuvwxyz.txt").write_text( - f"entry {index:03d}\n", encoding="ascii" - ) - set_epoch(fixture_dir) - commands = [ - build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666650", "fallback-base.erofs", "fallback", "-E", "force-inode-compact"), - build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666651", "oversize-base.erofs", "oversize", "-E", "force-inode-extended"), - build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666652", "extent-base.erofs", "extent", "-E", "legacy-compress,force-inode-extended", "-z", "lz4"), - build_image(artifact_dir, fixture_dir, "22222222-3333-4444-5555-666666666653", "large-dir-base.erofs", "large", "-E", "force-inode-extended"), - ] - feature_incompat = SUPER + 80 - rootnid_2b = SUPER + 14 - blocks_lo = SUPER + 36 - rootnid_8b = SUPER + 112 - evidence = [] - fallback = bytearray((artifact_dir / "fallback-base.erofs").read_bytes()) - fallback_root = u16(fallback, rootnid_2b) - fallback_blocks = u32(fallback, blocks_lo) - if fallback_root == 0 or u64(fallback, rootnid_8b) != 0: - raise FixtureError("unexpected fallback base root") - put_u32(fallback, feature_incompat, u32(fallback, feature_incompat) | 0x80) - put_u64(fallback, rootnid_8b, 0) - write_image(artifact_dir / "fallback-48bit-root2.erofs", fallback) - evidence.append( - f"fallback rootnid_2b={fallback_root} rootnid_8b=0 " - f"blocks_lo={fallback_blocks} union=0x{u16(fallback, rootnid_2b):04x}" - ) - oversize = bytearray((artifact_dir / "oversize-base.erofs").read_bytes()) - oversize_root = u16(oversize, rootnid_2b) - oversize_nid = child_nid(oversize, oversize_root, b"big.dat") - oversize_off, oversize_format, oversize_isize, _, _, _ = inode_info(oversize, oversize_nid) - if not oversize_format & 1 or oversize_isize != 64: - raise FixtureError("oversize inode is not extended") - put_u64(oversize, oversize_off + 8, 1 << 63) - write_image(artifact_dir / "extended-size-bit63.erofs", oversize) - evidence.append( - f"oversize nid={oversize_nid} inode_off={oversize_off} " - f"i_size=0x{u64(oversize, oversize_off + 8):016x}" - ) - extent = bytearray((artifact_dir / "extent-base.erofs").read_bytes()) - extent_root = u16(extent, rootnid_2b) - extent_nid = child_nid(extent, extent_root, b"hole.dat") - extent_off, extent_format, extent_isize, extent_xattr, extent_layout, _ = inode_info(extent, extent_nid) - if not extent_format & 1 or extent_isize != 64 or extent_layout != 1: - raise FixtureError("extent inode has unexpected format") - header = (extent_off + extent_isize + extent_xattr + 7) & ~7 - record = (header + 8 + 15) & ~15 - root_off = inode_offset(extent, extent_root) - if header < root_off + 64 and root_off < record + 16: - raise FixtureError("extent metadata overlaps root inode") - hole_size = 5 * 1024 * 1024 * 1024 + 4096 - put_u64(extent, extent_off + 8, hole_size) - struct.pack_into(" None: - testdir = source / "testdir" - (testdir / "subdir").mkdir(parents=True) - for index in range(5): - (testdir / f"dir-{index:02d}").mkdir() - (source / "parent/child/grandchild").mkdir(parents=True) - (testdir / "file.txt").write_text("repo22 G3 file payload\n", encoding="ascii") - for index in range(12): - (testdir / f"regular-{index:02d}.txt").write_text(f"regular {index:02d}\n", encoding="ascii") - for name in ("File.txt", "FILE.TXT", "FiLe.TxT"): - (testdir / name).write_text(f"case variant {name}\n", encoding="ascii") - for index in range(1, 4): - (testdir / f"file{index}.txt").write_text(f"cache file {index}\n", encoding="ascii") - (testdir / "subdir/child.txt").write_text("child\n", encoding="ascii") - (testdir / "script.sh").write_text("#!/bin/sh\nexit 0\n", encoding="ascii") - (testdir / "rootonly.txt").write_text("restricted\n", encoding="ascii") - (testdir / "writeonly.txt").write_text("write only mode\n", encoding="ascii") - (testdir / "noexec.txt").write_text("not executable\n", encoding="ascii") - (testdir / ("long-" + "x" * 250)).write_text("255 byte name\n", encoding="ascii") - long_target = "subdir/" + "y" * 112 - (testdir / "subdir" / ("y" * 112)).write_text("long target\n", encoding="ascii") - os.symlink("file.txt", testdir / "shortlink") - os.symlink("subdir/child.txt", testdir / "relative-link") - os.symlink("missing-target", testdir / "broken-link") - os.symlink(long_target, testdir / "long-link") - (source / "parent/file.txt").write_text("parent file\n", encoding="ascii") - (source / "parent/child/grandchild/marker.txt").write_text("grandchild\n", encoding="ascii") - (source / "pager.bin").write_bytes(deterministic_bytes(5 * 4096 + 731)) - os.chmod(testdir / "script.sh", 0o755) - os.chmod(testdir / "rootonly.txt", 0o600) - os.chmod(testdir / "writeonly.txt", 0o200) - os.chmod(testdir / "noexec.txt", 0o644) - set_epoch(source) - - -def directory_data_offset(image: ErofsImage, inode: object) -> int: - if inode.layout == 2: - return inode.offset + inode.inode_size + inode.xattr_size - if inode.layout == FLAT_PLAIN: - return inode.start_block << image.block_bits - raise FixtureError(f"unsupported directory layout {inode.layout}") - - -def limited_directory_entries(image: ErofsImage, inode: object) -> list[object]: - if inode.size == 0 or inode.size > image.block_size: - raise FixtureError("G3 qualifier accepts only one-block path components") - return image.directory_entries(inode) - - -def resolve(image: ErofsImage, path: str) -> object: - inode = image.inode(image.root_nid) - for component in path.strip("/").split("/"): - if not component: - continue - entry = next( - ( - item - for item in limited_directory_entries(image, inode) - if item.name.decode("ascii") == component - ), - None, - ) - if entry is None: - raise FixtureError(f"path not found: {path}") - inode = image.inode(entry.nid) - return inode - - -def validate_vfs_image(image_path: Path, source: Path) -> list[str]: - image = ErofsImage.load(image_path) - image.validate_superblock() - evidence = [ - f"image={image_path.name} provider_bytes={len(image.data)} " - f"block_size={image.block_size} blocks={image.blocks} " - f"root_nid={image.root_nid} checksum_valid={image.checksum_valid()}" - ] - paths = [ - "/testdir", - "/testdir/file.txt", - "/testdir/script.sh", - "/testdir/rootonly.txt", - "/testdir/writeonly.txt", - "/testdir/shortlink", - "/testdir/long-link", - "/pager.bin", - "/parent/child/grandchild", - ] - for path in paths: - source_path = source / path.lstrip("/") - source_stat = source_path.lstat() - inode = resolve(image, path) - if inode.mode & S_IFMT != stat.S_IFMT(source_stat.st_mode): - raise FixtureError(f"{path}: inode type differs from source") - if inode.mode & 0o7777 != source_stat.st_mode & 0o7777: - raise FixtureError(f"{path}: inode permissions differ from source") - expected_size = None - if source_path.is_symlink(): - expected_size = len(os.readlink(source_path).encode("ascii")) - elif source_path.is_file(): - expected_size = source_stat.st_size - if expected_size is not None and inode.size != expected_size: - raise FixtureError(f"{path}: size {inode.size}, expected {expected_size}") - evidence.append( - f"path={path} nid={inode.nid} inode_offset={inode.offset} " - f"layout={inode.layout} mode={inode.mode:#07o} size={inode.size} " - f"start_block={inode.start_block}" - ) - source_names = sorted(item.name for item in (source / "testdir").iterdir()) - testdir = resolve(image, "/testdir") - image_names = sorted( - entry.name.decode("ascii") - for entry in limited_directory_entries(image, testdir) - if entry.name not in (b".", b"..") - ) - if source_names != image_names: - raise FixtureError("/testdir names differ from source") - evidence.append( - f"path=/testdir real_entries={len(image_names)} " - f"directory_size={testdir.size} layout={testdir.layout}" - ) - return evidence - - -def write_source_hashes(source: Path, output: Path) -> None: - with output.open("x", encoding="ascii") as sums: - for path in sorted(source.rglob("*")): - if path.is_file() and not path.is_symlink(): - sums.write(f"{sha256(path)} {path.relative_to(source)}\n") - - -def make_vfs(output: Path) -> list[list[str]]: - require_tools("mkfs.erofs") - if output.exists(): - raise FixtureError(f"output already exists: {output}") - source = output / "source" - output.mkdir(parents=True) - make_vfs_source(source) - plain = output / "vfs-plain.erofs" - lz4 = output / "vfs-lz4.erofs" - common = ["mkfs.erofs", "-d0", "-T0", "--all-time", "--all-root", "--workers=1"] - commands = [ - common + ["-x-1", "-E", "noinline_data", "-U", "33333333-4444-5555-6666-777777777771", str(plain), str(source)], - common + ["-z", "lz4", "-U", "33333333-4444-5555-6666-777777777772", str(lz4), str(source)], - ] - for command in commands: - run(command) - evidence = validate_vfs_image(plain, source) + validate_vfs_image(lz4, source) - (output / "fixture-evidence.txt").write_text("\n".join(evidence) + "\n", encoding="ascii") - (output / "expected-testdir.txt").write_text( - "\n".join(sorted(item.name for item in (source / "testdir").iterdir())) + "\n", - encoding="ascii", - ) - write_source_hashes(source, output / "SOURCE-SHA256SUMS") - with (output / "IMAGE-SHA256SUMS").open("x", encoding="ascii") as sums: - for path in (lz4, plain): - sums.write(f"{sha256(path)} {path.name}\n") - return commands - - -def make_large_prefix(source: Path, output: Path, evidence_path: Path) -> None: - image = ErofsImage.load(source) - image.validate_superblock() - huge = resolve(image, "/huge") - expected_size = HUGE_DIRECTORY_BLOCKS * image.block_size - if huge.inode_size != 64 or huge.layout != FLAT_PLAIN: - raise FixtureError("TC153 directory is not extended FLAT_PLAIN/Layout0") - if huge.size != expected_size: - raise FixtureError(f"TC153 directory size {huge.size}, expected {expected_size}") - if huge.start_block == 0: - raise FixtureError("TC153 directory has no plain-data start block") - provider_blocks = huge.start_block + HUGE_DIRECTORY_BLOCKS - if provider_blocks > 0xFFFFFFFF: - raise FixtureError("TC153 sparse provider does not fit blocks_lo") - image.put_u32(SUPER + 36, provider_blocks) - image.update_checksum() - image.validate_superblock() - output.parent.mkdir(parents=True, exist_ok=True) - image.save(output) - checked = ErofsImage.load(output) - checked_huge = resolve(checked, "/huge") - if checked.blocks != provider_blocks or checked_huge.layout != FLAT_PLAIN or checked_huge.size != expected_size: - raise FixtureError("TC153 sparse prefix self-check failed") - provider_bytes = provider_blocks * checked.block_size - evidence_path.write_text( - "tc153-sparse-prefix " - f"nid={checked_huge.nid} inode_offset={checked_huge.offset} " - f"layout=flat-plain start_block={checked_huge.start_block} " - f"directory_blocks={HUGE_DIRECTORY_BLOCKS} last_block={HUGE_DIRECTORY_BLOCKS - 1} " - f"blocks_lo={provider_blocks} provider_bytes={provider_bytes} " - f"prefix_bytes={len(checked.data)} checksum_valid={checked.checksum_valid()} " - f"sha256={sha256(output)}\n", - encoding="ascii", - ) - - -def path_type(mode: int) -> str: - if stat.S_ISREG(mode): - return "file" - if stat.S_ISDIR(mode): - return "directory" - if stat.S_ISLNK(mode): - return "symlink" - if stat.S_ISFIFO(mode): - return "fifo" - if stat.S_ISCHR(mode): - return "char" - if stat.S_ISBLK(mode): - return "block" - raise FixtureError(f"unsupported source type: {mode:#o}") - - -def tree_inventory(root: Path) -> dict[str, object]: - if not root.is_dir(): - raise FixtureError(f"inventory root is absent: {root}") - paths = [root] + sorted(root.rglob("*")) - hardlinks: dict[tuple[int, int], list[str]] = {} - for path in paths: - info = path.lstat() - if stat.S_ISREG(info.st_mode) and info.st_nlink > 1: - hardlinks.setdefault((info.st_dev, info.st_ino), []).append( - "." if path == root else path.relative_to(root).as_posix() - ) - hardlink_names = { - key: ",".join(sorted(names)) for key, names in hardlinks.items() - } - entries = [] - for path in paths: - info = path.lstat() - relative = "." if path == root else path.relative_to(root).as_posix() - item: dict[str, object] = { - "path": relative, - "type": path_type(info.st_mode), - "mode": info.st_mode & 0o7777, - } - if stat.S_ISREG(info.st_mode): - item.update(size=info.st_size, sha256=sha256(path)) - if info.st_nlink > 1: - item["hardlink_group"] = hardlink_names[(info.st_dev, info.st_ino)] - elif stat.S_ISLNK(info.st_mode): - item["target"] = os.readlink(path) - elif stat.S_ISCHR(info.st_mode) or stat.S_ISBLK(info.st_mode): - item.update(major=os.major(info.st_rdev), minor=os.minor(info.st_rdev)) - entries.append(item) - encoded = json.dumps(entries, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode("ascii") - return {"sha256": hashlib.sha256(encoded).hexdigest(), "entries": entries} - - -def compare_set( - left_source: Path, - left_artifacts: Path, - right_source: Path, - right_artifacts: Path, - output: Path, -) -> None: - inventories = { - "left_source": tree_inventory(left_source), - "left_artifacts": tree_inventory(left_artifacts), - "right_source": tree_inventory(right_source), - "right_artifacts": tree_inventory(right_artifacts), - } - comparison = { - name: {"sha256": value["sha256"], "entries": len(value["entries"])} - for name, value in inventories.items() - } - comparison["source_equal"] = inventories["left_source"] == inventories["right_source"] - comparison["artifacts_equal"] = inventories["left_artifacts"] == inventories["right_artifacts"] - comparison["status"] = ( - "PASS" if comparison["source_equal"] and comparison["artifacts_equal"] else "RUNNER_FAIL" - ) - if comparison["status"] != "PASS": - comparison["differences"] = { - "source": sorted( - set(json.dumps(item, sort_keys=True) for item in inventories["left_source"]["entries"]) - ^ set(json.dumps(item, sort_keys=True) for item in inventories["right_source"]["entries"]) - )[:40], - "artifacts": sorted( - set(json.dumps(item, sort_keys=True) for item in inventories["left_artifacts"]["entries"]) - ^ set(json.dumps(item, sort_keys=True) for item in inventories["right_artifacts"]["entries"]) - )[:40], - } - write_json(output, comparison) - if comparison["status"] != "PASS": - raise FixtureError("stable and archived fixture trees differ") - - -def verify_checksum_file(directory: Path, filename: str) -> None: - for line in (directory / filename).read_text(encoding="ascii").splitlines(): - digest, relative = line.split(" ", 1) - if sha256(directory / relative) != digest: - raise FixtureError(f"checksum mismatch: {directory / relative}") - - -def make_expected_wide(fixture_dir: Path, artifact_dir: Path) -> None: - names = sorted(path.name for path in (fixture_dir / "namei/wide").iterdir() if path.is_file()) - if len(names) != 320: - raise FixtureError(f"wide directory contains {len(names)} files") - (artifact_dir / "expected-wide.txt").write_text("\n".join(names) + "\n", encoding="ascii") - - -def make_all(output: Path) -> None: - if output.exists(): - raise FixtureError(f"output already exists: {output}") - output.mkdir(parents=True) - commands = { - "vfs": make_vfs(output / "vfs"), - "metadata": make_metadata(output / "metadata-source", output / "metadata"), - "final": make_final(output / "final-source", output / "final"), - } - make_expected_wide(output / "metadata-source", output / "metadata") - make_large_prefix( - output / "final/large-dir-intmax.erofs", - output / "final/large-dir-intmax-sparse-prefix.erofs", - output / "final/tc153-sparse-evidence.txt", - ) - (output / "final/TC153-SPARSE-SHA256").write_text( - f"{sha256(output / 'final/large-dir-intmax-sparse-prefix.erofs')} " - "large-dir-intmax-sparse-prefix.erofs\n", - encoding="ascii", - ) - verify_checksum_file(output / "vfs", "IMAGE-SHA256SUMS") - verify_checksum_file(output / "vfs/source", "../SOURCE-SHA256SUMS") - verify_checksum_file(output / "metadata", "SHA256SUMS") - verify_checksum_file(output / "final", "SHA256SUMS") - output_text = str(output) - canonical_commands = { - name: [ - [ - "$OUTPUT" + argument[len(output_text) :] - if argument == output_text or argument.startswith(output_text + os.sep) - else argument - for argument in argv - ] - for argv in command_set - ] - for name, command_set in commands.items() - } - manifest = { - "schema_version": 1, - "generator": "tests/pre15/fixtures/g3.py", - "generator_sha256": sha256(Path(__file__)), - "mkfs_version": run(["mkfs.erofs", "-V"], capture=True).splitlines()[0], - "commands": canonical_commands, - "sets": { - "vfs": tree_inventory(output / "vfs"), - "metadata_source": tree_inventory(output / "metadata-source"), - "metadata": tree_inventory(output / "metadata"), - "final_source": tree_inventory(output / "final-source"), - "final": tree_inventory(output / "final"), - }, - } - write_json(output / "G3-MANIFEST.json", manifest) - - -def verify_all(output: Path) -> None: - manifest = json.loads((output / "G3-MANIFEST.json").read_text(encoding="ascii")) - roots = { - "vfs": output / "vfs", - "metadata_source": output / "metadata-source", - "metadata": output / "metadata", - "final_source": output / "final-source", - "final": output / "final", - } - for name, root in roots.items(): - if tree_inventory(root) != manifest["sets"][name]: - raise FixtureError(f"G3 manifest mismatch: {name}") - verify_checksum_file(output / "vfs", "IMAGE-SHA256SUMS") - verify_checksum_file(output / "vfs/source", "../SOURCE-SHA256SUMS") - verify_checksum_file(output / "metadata", "SHA256SUMS") - verify_checksum_file(output / "final", "SHA256SUMS") - - -def source_constant(path: Path, name: str) -> int: - for line in path.read_text(encoding="utf-8").splitlines(): - fields = line.split() - if len(fields) >= 3 and fields[0] == "#define" and fields[1] == name: - return int(fields[2], 0) - raise FixtureError(f"constant not found: {path}:{name}") - - -def validate_xattr_sources(freebsd_root: Path, linux_root: Path) -> dict[str, str]: - constants = ( - "EROFS_FEATURE_COMPAT_SB_CHKSUM", - "EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX", - "EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES", - "EROFS_FEATURE_INCOMPAT_METABOX", - ) - for name in constants: - freebsd = source_constant(freebsd_root / "erofs_fs.h", name) - linux = source_constant(linux_root / "erofs_fs.h", name) - if freebsd != linux: - raise FixtureError(f"FreeBSD/Linux constant mismatch: {name}") - return { - "freebsd_erofs_fs_sha256": sha256(freebsd_root / "erofs_fs.h"), - "freebsd_xattr_sha256": sha256(freebsd_root / "xattr.c"), - "linux_erofs_fs_sha256": sha256(linux_root / "erofs_fs.h"), - "linux_xattr_sha256": sha256(linux_root / "xattr.c"), - } - - -def make_contract_image( - path: Path, - feature_compat: int, - feature_incompat: int, - prefix_start: int, - prefix_count: int, - packed_nid: int, - metabox_nid: int, - primary: bytes, -) -> None: - data = bytearray(4096) - put_u32(data, SUPER, MAGIC) - put_u32(data, SUPER + 8, feature_compat | FEATURE_COMPAT_SB_CHKSUM) - data[SUPER + 12] = 12 - put_u16(data, SUPER + 14, 1) - put_u32(data, SUPER + 36, 1) - put_u32(data, SUPER + 40, 0) - put_u32(data, SUPER + 80, feature_incompat) - data[SUPER + 91] = prefix_count - put_u32(data, SUPER + 92, prefix_start // 4) - put_u64(data, SUPER + 96, packed_nid) - put_u64(data, SUPER + 128, metabox_nid) - if prefix_start + len(primary) <= len(data): - data[prefix_start : prefix_start + len(primary)] = primary - write_image(path, data) - - -def select_prefix_backing(image: ErofsImage, carriers: dict[str, bytes]) -> tuple[str, bytes]: - plain = bool(image.feature_compat & FEATURE_COMPAT_PLAIN_XATTR_PFX) - metabox = bool(image.feature_incompat & FEATURE_INCOMPAT_METABOX) - packed_nid = image.u64(SUPER + 96) - if plain: - return "primary", bytes(image.data) - if metabox: - return "metabox", carriers["metabox"] - if packed_nid: - return "packed", carriers["packed"] - return "primary-legacy", bytes(image.data) - - -def parse_prefix(backing: bytes, offset: int) -> tuple[int, bytes]: - aligned = (offset + 3) & ~3 - if aligned > len(backing) - 2: - raise FixtureError("EINTEGRITY: prefix length is outside backing") - raw_length = struct.unpack_from(" EROFS_NAME_LEN + 1: - raise FixtureError("EINTEGRITY: invalid prefix length") - if aligned + 2 + length > len(backing): - raise FixtureError("EINTEGRITY: prefix payload is outside backing") - payload = backing[aligned + 2 : aligned + 2 + length] - return payload[0], payload[1:] - - -def make_xattr_legacy( - spec_path: Path, output: Path, freebsd_root: Path, linux_root: Path -) -> None: - if output.exists(): - raise FixtureError(f"output already exists: {output}") - spec = json.loads(spec_path.read_text(encoding="ascii")) - output.mkdir(parents=True) - source_hashes = validate_xattr_sources(freebsd_root, linux_root) - prefix_start = spec["prefix_start"] - payload = bytes([spec["base_index"]]) + spec["infix"].encode("ascii") - record = struct.pack(" argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - make_all_parser = subparsers.add_parser("make-all") - make_all_parser.add_argument("--output", type=Path, required=True) - verify_parser = subparsers.add_parser("verify") - verify_parser.add_argument("--output", type=Path, required=True) - vfs_parser = subparsers.add_parser("make-vfs") - vfs_parser.add_argument("--output", type=Path, required=True) - metadata_parser = subparsers.add_parser("make-metadata") - metadata_parser.add_argument("--source", type=Path, required=True) - metadata_parser.add_argument("--output", type=Path, required=True) - final_parser = subparsers.add_parser("make-final") - final_parser.add_argument("--source", type=Path, required=True) - final_parser.add_argument("--output", type=Path, required=True) - large_parser = subparsers.add_parser("make-large-prefix") - large_parser.add_argument("--source", type=Path, required=True) - large_parser.add_argument("--output", type=Path, required=True) - large_parser.add_argument("--evidence", type=Path, required=True) - compare_parser = subparsers.add_parser("compare-set") - compare_parser.add_argument("--left-source", type=Path, required=True) - compare_parser.add_argument("--left-artifacts", type=Path, required=True) - compare_parser.add_argument("--right-source", type=Path, required=True) - compare_parser.add_argument("--right-artifacts", type=Path, required=True) - compare_parser.add_argument("--output", type=Path, required=True) - xattr_parser = subparsers.add_parser("make-xattr-legacy") - xattr_parser.add_argument("--spec", type=Path, required=True) - xattr_parser.add_argument("--output", type=Path, required=True) - xattr_parser.add_argument("--freebsd-root", type=Path, required=True) - xattr_parser.add_argument("--linux-root", type=Path, required=True) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> None: - os.umask(0o022) - args = parse_args(argv) - if args.command == "make-all": - make_all(args.output) - elif args.command == "verify": - verify_all(args.output) - elif args.command == "make-vfs": - make_vfs(args.output) - elif args.command == "make-metadata": - make_metadata(args.source, args.output) - make_expected_wide(args.source, args.output) - elif args.command == "make-final": - make_final(args.source, args.output) - elif args.command == "make-large-prefix": - make_large_prefix(args.source, args.output, args.evidence) - elif args.command == "compare-set": - compare_set( - args.left_source, - args.left_artifacts, - args.right_source, - args.right_artifacts, - args.output, - ) - else: - make_xattr_legacy(args.spec, args.output, args.freebsd_root, args.linux_root) - - -if __name__ == "__main__": - try: - main() - except (FixtureError, OSError, subprocess.CalledProcessError, ValueError) as error: - print(f"fixture error: {error}", file=sys.stderr) - raise SystemExit(1) from error diff --git a/tests/pre15/gates/P15-005-input.json b/tests/pre15/gates/P15-005-input.json deleted file mode 100644 index 4d3f8c8..0000000 --- a/tests/pre15/gates/P15-005-input.json +++ /dev/null @@ -1,416 +0,0 @@ -{ - "schema": 1, - "gate": "G02", - "candidate": "P15-005", - "required_base": "edf098905a34de764185e72fc7e92d7b8f4e7285", - "title": "metadata buffer ownership and frozen map tuple oracle", - "required_categories": [ - "plain", - "inline", - "hole", - "chunk", - "multidevice", - "compressed", - "fragment", - "partial-reference", - "post-EOF", - "bounds", - "overflow" - ], - "tuple_fields": [ - "m_la", - "m_pa", - "m_llen", - "m_plen", - "m_deviceid", - "m_flags", - "errno", - "acquire_count", - "release_count" - ], - "errno": { - "EIO": 5, - "EINVAL": 22, - "EOPNOTSUPP": 45, - "EOVERFLOW": 84, - "EINTEGRITY": 97 - }, - "source_paths": [ - "repo-pre-15/src/internal.h", - "repo-pre-15/src/data.c", - "repo-pre-15/src/decompressor.c", - "repo-pre-15/src/inode.c", - "repo-pre-15/src/super.c", - "repo-pre-15/src/xattr.c", - "repo-pre-15/src/zdata.c", - "repo-pre-15/src/zmap.c", - "repo-pre-15/src/erofs_fs.h", - "src-linux/internal.h", - "src-linux/data.c", - "src-linux/decompressor.c", - "src-linux/inode.c", - "src-linux/super.c", - "src-linux/xattr.c", - "src-linux/zdata.c", - "src-linux/zmap.c" - ], - "linux_sha256": { - "src-linux/internal.h": "4aa671896ff7c0ad32a9108c818ef62d16841c116706fdad391c40a530c81405", - "src-linux/data.c": "8625cdc01e5405f856178ae8fd559696ae85f607f19caf229b867a3b7479318a", - "src-linux/decompressor.c": "caf1c501d00a5a2c9cda5fc0b59d2823eaedf0161a130ca69cd5e7c455128709", - "src-linux/inode.c": "a15562e0782e155a0744a7ba9d2f557519a583b64853ed75180cef2b4dfae1b3", - "src-linux/super.c": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9", - "src-linux/xattr.c": "c8394e5f6301225cbe7587f223485a368348eac7596c1ab0bbf965c99655ed4a", - "src-linux/zdata.c": "358869da60dcdafd13bc1ff0cd28e864c13b46a49d9f556d9b37139f7e0f4e79", - "src-linux/zmap.c": "4fb34c024bfaec9d2294ba2f85069bb2f727b378ebf4848814e081f91ea022cc" - }, - "linux_semantic_anchors": { - "src-linux/internal.h": [ - "struct erofs_buf {", - "#define __EROFS_BUF_INITIALIZER", - "void erofs_put_metabuf(struct erofs_buf *buf);" - ], - "src-linux/data.c": [ - "void erofs_put_metabuf(struct erofs_buf *buf)", - "folio_put(page_folio(buf->page));", - "buf->page = NULL;", - "struct erofs_buf buf = __EROFS_BUF_INITIALIZER;", - "erofs_put_metabuf(&buf);" - ], - "src-linux/decompressor.c": [ - "struct erofs_buf buf = __EROFS_BUF_INITIALIZER;", - "erofs_put_metabuf(&buf);" - ], - "src-linux/inode.c": [ - "struct erofs_buf buf = __EROFS_BUF_INITIALIZER;", - "erofs_put_metabuf(&buf);" - ], - "src-linux/super.c": [ - "struct erofs_buf buf = __EROFS_BUF_INITIALIZER;", - "erofs_put_metabuf(&buf);" - ], - "src-linux/xattr.c": [ - "struct erofs_buf buf = __EROFS_BUF_INITIALIZER;", - "erofs_put_metabuf(&buf);" - ], - "src-linux/zdata.c": [ - "struct erofs_buf buf = __EROFS_BUF_INITIALIZER;", - "erofs_put_metabuf(&buf);" - ], - "src-linux/zmap.c": [ - "erofs_put_metabuf(&map.buf);" - ] - }, - "baseline_function_sha256": { - "data.c:erofs_map_blocks_chunk": "b850ab686f61e6265ef0780c1363f570ceb6afab004b5d9c527d728f00a0fa0e", - "data.c:erofs_read_metadata": "1ea2f9bad3c60019c0f373c98444dcd1dd4bb178875a170313868a56cd314436", - "data.c:erofs_read_data": "9d962f6086f5fc4a66bbfd9b27d05aa71234bb8266a3c383ec693298966e5e63", - "data.c:erofs_read_uio": "05d3ad9e8b25b7e9b6d62f4fd836ab204bc57b90781fc140cc498d1ab8fe0a78", - "decompressor.c:z_erofs_read_cfg": "77ddaf24f5317e0b83b72d85f64aa624cbd15b71a9602d2217b11dd382e1f719", - "decompressor.c:z_erofs_parse_cfgs": "d800c9a52b9f9333f57da2f67e27d238320866b0f8816929b32f119c84a69eb4", - "inode.c:erofs_read_inode": "ed65faffe7a607e95bca50de7e28cf592a0500b7363935991de959269a200fe1", - "super.c:erofs_load_generation_seed": "98299634491a22b427820a28a7ec1d23268ab1fe02d1fc0515774da70900f071", - "super.c:erofs_superblock_csum_verify": "cd56c29a6cf795de359f6642509a4a152e5a1860ec32a997e91ebf81e7877096", - "super.c:erofs_scan_devices": "2743af3b9d9d7e53c0102bc197026adf58ef6bf7adbf5cdbe8716c9e25b2a547", - "super.c:erofs_read_superblock": "d605349d9604d89d4bb413c42abc71f13b8fafa43b1fda671fbd4613f2a3ed6f", - "xattr.c:erofs_xattr_read_backing": "df65cf9e391d2f17f79beeecda24fbf6c1be495dd3baf65121c226c4054ccf75", - "xattr.c:erofs_xattr_read_metadata": "ed726755b28f1c1a82a07a956b889bd633da45cdd6671c91c7bc4db9d4006096", - "xattr.c:erofs_xattr_load_body": "c8684629f78920cefb6a6d61ddad5ed42ba499d817798f81086282c827e54aa5", - "xattr.c:erofs_xattr_load_shared_entry": "e47145a775685cfaeb9341a7cc1cc742c73145e34f4b81d8fc5dac35d765e17f", - "xattr.c:erofs_inode_has_noacl": "8098be9fef8148498a2080e02db2876d9592f1af10ca551278902e3aa4c3428e", - "xattr.c:erofs_xattr_iter_shared": "1ec7756ef31dd6c51e04d92656440a9dbca5e9e9fe38c359eed744ad16b39b2c", - "xattr.c:erofs_getxattr": "9ee7d7effcfa3d23ab4d4ce1e3e7d638c048085368b2f3a2db3dfdadf911cd34", - "xattr.c:erofs_listxattr": "6f75c9f8c50c331daba1bb0c1b757390c3ff873ee7419b2dd2b35dfc3db84ea6", - "xattr.c:erofs_xattr_prefixes_init": "7aaade8116337ad1026da04777bf1d9f8aefaa246623d6c32080af153fb93abd", - "zdata.c:z_erofs_read_extent": "327e3a22b9c89c410d1ecef6934dcdaf194a4b6962737ea5f3e4f21d10694b13", - "zdata.c:z_erofs_do_read": "2096f98b6d19e1c5f80edb6de9fbfe15bb27f006e2510bcb068f920c23aa38ec", - "zmap.c:z_erofs_read_index": "cf8ca75c1d600f2f708e73e723a7abaee7c98f9813eb8f432e1f96dd1602d82b", - "zmap.c:z_erofs_load_full_lcluster": "751d456964fcdcf66f20d22c1f22adef209767aef1bab9d0857f36df7d2c37b2", - "zmap.c:z_erofs_load_compact_lcluster": "4c23a8a966262b5a365bc874718316daa616cc97dfe3e2711570f81e7e3b953c", - "zmap.c:z_erofs_read_extent": "454064ca59dfd0674af5520c4a965be659d2b7769403cd8d84bd9657e0145f27", - "zmap.c:z_erofs_map_blocks_ext": "31c9310f801395fe01d38515ee42ef49c1d07ceba0b86b612bb140ec180238b0", - "zmap.c:z_erofs_fill_inode": "125f0c3492c152415978226919e782d355a32fc49cf8f49456eb398a4c721032" - }, - "direct_metadata_functions": [ - "data.c:erofs_map_blocks_chunk", - "data.c:erofs_read_data", - "data.c:erofs_read_uio", - "decompressor.c:z_erofs_read_cfg", - "inode.c:erofs_read_inode", - "super.c:erofs_load_generation_seed", - "super.c:erofs_superblock_csum_verify", - "super.c:erofs_scan_devices", - "super.c:erofs_read_superblock", - "xattr.c:erofs_xattr_read_backing", - "zdata.c:z_erofs_read_extent", - "zmap.c:z_erofs_fill_inode", - "zmap.c:z_erofs_map_blocks_ext", - "zmap.c:z_erofs_read_extent", - "zmap.c:z_erofs_read_index" - ], - "candidate_ownership_contract": { - "object_local_counts": { - "data.c:erofs_map_blocks_chunk": 1, - "data.c:erofs_read_metadata": 0, - "data.c:erofs_read_data": 1, - "data.c:erofs_read_uio": 1, - "decompressor.c:z_erofs_read_cfg": 1, - "decompressor.c:z_erofs_parse_cfgs": 1, - "inode.c:erofs_read_inode": 1, - "super.c:erofs_load_generation_seed": 1, - "super.c:erofs_superblock_csum_verify": 1, - "super.c:erofs_scan_devices": 1, - "super.c:erofs_read_superblock": 1, - "xattr.c:erofs_xattr_read_backing": 0, - "xattr.c:erofs_xattr_read_metadata": 1, - "xattr.c:erofs_xattr_load_body": 1, - "xattr.c:erofs_xattr_load_shared_entry": 1, - "xattr.c:erofs_inode_has_noacl": 1, - "xattr.c:erofs_xattr_iter_shared": 1, - "xattr.c:erofs_getxattr": 1, - "xattr.c:erofs_listxattr": 1, - "xattr.c:erofs_xattr_prefixes_init": 1, - "zdata.c:z_erofs_read_extent": 1, - "zdata.c:z_erofs_do_read": 0, - "zmap.c:z_erofs_read_index": 0, - "zmap.c:z_erofs_load_full_lcluster": 1, - "zmap.c:z_erofs_load_compact_lcluster": 1, - "zmap.c:z_erofs_read_extent": 1, - "zmap.c:z_erofs_map_blocks_ext": 1, - "zmap.c:z_erofs_fill_inode": 1 - }, - "put_counts": { - "data.c:erofs_map_blocks_chunk": 1, - "data.c:erofs_read_metadata": 0, - "data.c:erofs_read_data": 1, - "data.c:erofs_read_uio": 1, - "decompressor.c:z_erofs_read_cfg": 1, - "decompressor.c:z_erofs_parse_cfgs": 1, - "inode.c:erofs_read_inode": 18, - "super.c:erofs_load_generation_seed": 1, - "super.c:erofs_superblock_csum_verify": 1, - "super.c:erofs_scan_devices": 2, - "super.c:erofs_read_superblock": 1, - "xattr.c:erofs_xattr_read_backing": 0, - "xattr.c:erofs_xattr_read_metadata": 1, - "xattr.c:erofs_xattr_load_body": 1, - "xattr.c:erofs_xattr_load_shared_entry": 1, - "xattr.c:erofs_inode_has_noacl": 1, - "xattr.c:erofs_xattr_iter_shared": 1, - "xattr.c:erofs_getxattr": 1, - "xattr.c:erofs_listxattr": 1, - "xattr.c:erofs_xattr_prefixes_init": 2, - "zdata.c:z_erofs_read_extent": 1, - "zdata.c:z_erofs_do_read": 0, - "zmap.c:z_erofs_read_index": 0, - "zmap.c:z_erofs_load_full_lcluster": 2, - "zmap.c:z_erofs_load_compact_lcluster": 7, - "zmap.c:z_erofs_read_extent": 1, - "zmap.c:z_erofs_map_blocks_ext": 1, - "zmap.c:z_erofs_fill_inode": 6 - }, - "legacy_release_counts": { - "data.c:erofs_read_data": 1, - "data.c:erofs_read_uio": 1, - "zdata.c:z_erofs_read_extent": 1, - "zdata.c:z_erofs_do_read": 1 - }, - "raw_bread_counts": { - "data.c:erofs_read_metadata": 1, - "xattr.c:erofs_xattr_read_backing": 1 - }, - "acquire_apis": [ - "erofs_read_metadata", - "z_erofs_read_cfg", - "erofs_xattr_read_backing", - "erofs_xattr_read_metadata", - "erofs_xattr_load_body", - "erofs_xattr_load_shared_entry", - "z_erofs_read_index" - ], - "transfer_functions": [ - "data.c:erofs_read_metadata", - "decompressor.c:z_erofs_read_cfg", - "xattr.c:erofs_xattr_read_backing", - "xattr.c:erofs_xattr_read_metadata", - "zmap.c:z_erofs_read_index" - ] - }, - "tuple_cases": [ - { - "id": "plain-block-edge", - "engine": "data", - "category": "plain", - "sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 64, "device_id_mask": 65535}, - "inode": {"nid": 1, "size": 12288, "datalayout": 0, "startblk": 32, "inode_off": 0, "inode_isize": 32, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 0}, - "request": 4095, - "reader": {"mode": "unused", "bytes_hex": ""}, - "expected": {"m_la": 4095, "m_pa": 135167, "m_llen": 1, "m_plen": 1, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 0, "release_count": 0} - }, - { - "id": "inline-tail", - "engine": "data", - "category": "inline", - "sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 64, "device_id_mask": 65535}, - "inode": {"nid": 1, "size": 5000, "datalayout": 2, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32, "chunkbits": 12, "chunkformat": 0}, - "request": 4500, - "reader": {"mode": "unused", "bytes_hex": ""}, - "expected": {"m_la": 4500, "m_pa": 8692, "m_llen": 500, "m_plen": 500, "m_deviceid": 0, "m_flags": 3, "errno": 0, "acquire_count": 0, "release_count": 0} - }, - { - "id": "plain-hole", - "engine": "data", - "category": "hole", - "sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 64, "device_id_mask": 65535}, - "inode": {"nid": 1, "size": 4096, "datalayout": 0, "startblk": 18446744073709551615, "inode_off": 0, "inode_isize": 32, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 0}, - "request": 512, - "reader": {"mode": "unused", "bytes_hex": ""}, - "expected": {"m_la": 512, "m_pa": 0, "m_llen": 3584, "m_plen": 3584, "m_deviceid": 0, "m_flags": 0, "errno": 0, "acquire_count": 0, "release_count": 0} - }, - { - "id": "chunk-index", - "engine": "data", - "category": "chunk", - "sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 128, "device_id_mask": 65535}, - "inode": {"nid": 1, "size": 20000, "datalayout": 4, "startblk": 0, "inode_off": 0, "inode_isize": 64, "xattr_isize": 0, "chunkbits": 13, "chunkformat": 32}, - "request": 9000, - "reader": {"mode": "ok", "bytes_hex": "0000000000000000000000004d000000"}, - "expected": {"m_la": 9000, "m_pa": 316200, "m_llen": 7384, "m_plen": 7384, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1} - }, - { - "id": "chunk-48bit-device2", - "engine": "data", - "category": "multidevice", - "sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 128, "device_id_mask": 65535}, - "inode": {"nid": 1, "size": 24576, "datalayout": 4, "startblk": 0, "inode_off": 0, "inode_isize": 64, "xattr_isize": 0, "chunkbits": 13, "chunkformat": 96}, - "request": 12345, - "reader": {"mode": "ok", "bytes_hex": "00000000000000000100020005000000"}, - "expected": {"m_la": 12345, "m_pa": 17592186069049, "m_llen": 4039, "m_plen": 4039, "m_deviceid": 2, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1} - }, - { - "id": "chunk-index-out-of-provider", - "engine": "data", - "category": "bounds", - "sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 2, "device_id_mask": 65535}, - "inode": {"nid": 1, "size": 4096, "datalayout": 4, "startblk": 0, "inode_off": 8192, "inode_isize": 0, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 32}, - "request": 0, - "reader": {"mode": "unused", "bytes_hex": ""}, - "expected": {"m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 97, "acquire_count": 0, "release_count": 0} - }, - { - "id": "chunk-shift-overflow-after-acquire", - "engine": "data", - "category": "overflow", - "sbi": {"block_size": 131072, "blkszbits": 17, "blocks": 1, "device_id_mask": 65535}, - "inode": {"nid": 1, "size": 4096, "datalayout": 4, "startblk": 0, "inode_off": 0, "inode_isize": 64, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 96}, - "request": 0, - "reader": {"mode": "ok", "bytes_hex": "0080000000000000"}, - "expected": {"m_la": 0, "m_pa": 0, "m_llen": 4096, "m_plen": 4096, "m_deviceid": 0, "m_flags": 0, "errno": 84, "acquire_count": 1, "release_count": 1} - }, - { - "id": "chunk-index-short-read", - "engine": "data", - "category": "bounds", - "supplemental": true, - "sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 128, "device_id_mask": 65535}, - "inode": {"nid": 1, "size": 4096, "datalayout": 4, "startblk": 0, "inode_off": 0, "inode_isize": 64, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 32}, - "request": 0, - "reader": {"mode": "short", "bytes_hex": "00000000"}, - "expected": {"m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 5, "acquire_count": 0, "release_count": 0} - }, - { - "id": "explicit-compressed", - "engine": "zmap", - "category": "compressed", - "sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384}, - "inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1}, - "request": 0, - "reader": {"mode": "ok", "base": 16, "records": [{"plen": 268437504, "pstart": 74565, "lstart": 0}]}, - "expected": {"m_la": 0, "m_pa": 74565, "m_llen": 4096, "m_plen": 2048, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1} - }, - { - "id": "explicit-partial-reference", - "engine": "zmap", - "category": "partial-reference", - "sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384}, - "inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1}, - "request": 0, - "reader": {"mode": "ok", "base": 16, "records": [{"plen": 402654208, "pstart": 262144, "lstart": 0}]}, - "expected": {"m_la": 0, "m_pa": 262144, "m_llen": 4096, "m_plen": 1024, "m_deviceid": 0, "m_flags": 9, "errno": 0, "acquire_count": 1, "release_count": 1} - }, - { - "id": "explicit-fragment-tail", - "engine": "zmap", - "category": "fragment", - "sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384}, - "inode": {"nid": 2, "size": 6144, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 37, "z_lclusterbits": 12, "z_extents": 1}, - "request": 4096, - "reader": {"mode": "ok", "base": 16, "records": [{"plen": 8192, "pstart": 0, "lstart": 4096}]}, - "expected": {"m_la": 4096, "m_pa": 0, "m_llen": 2048, "m_plen": 8192, "m_deviceid": 0, "m_flags": 16, "errno": 0, "acquire_count": 1, "release_count": 1} - }, - { - "id": "compressed-post-eof", - "engine": "zmap", - "category": "post-EOF", - "sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384}, - "inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1}, - "request": 4113, - "reader": {"mode": "unused", "base": 16, "records": []}, - "expected": {"m_la": 4096, "m_pa": 0, "m_llen": 18, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 0, "acquire_count": 0, "release_count": 0} - }, - { - "id": "compressed-index-short-read", - "engine": "zmap", - "category": "compressed", - "supplemental": true, - "sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384}, - "inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1}, - "request": 0, - "reader": {"mode": "short", "base": 16, "bytes_hex": "0000000000000000"}, - "expected": {"m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 5, "acquire_count": 0, "release_count": 0} - }, - { - "id": "compressed-sanity-error-after-acquire", - "engine": "zmap", - "category": "compressed", - "supplemental": true, - "sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384}, - "inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1}, - "request": 0, - "reader": {"mode": "ok", "base": 16, "records": [{"plen": 1879049216, "pstart": 74565, "lstart": 0}]}, - "expected": {"m_la": 0, "m_pa": 74565, "m_llen": 0, "m_plen": 1024, "m_deviceid": 0, "m_flags": 1, "errno": 45, "acquire_count": 1, "release_count": 1} - } - ], - "ownership_paths": [ - {"id": "plain-inline-success", "consumer": "plain", "function": "data.c:erofs_read_data", "path": "metadata read succeeds; copied raw output is caller-owned", "acquire_count": 1, "release_count": 1}, - {"id": "plain-inline-short-read", "consumer": "plain", "function": "data.c:erofs_read_data", "path": "metadata read fails before ownership transfer", "acquire_count": 0, "release_count": 0}, - {"id": "map-chunk-success", "consumer": "map", "function": "data.c:erofs_map_blocks_chunk", "path": "index acquired, decoded, and released", "acquire_count": 1, "release_count": 1}, - {"id": "map-chunk-post-acquire-error", "consumer": "map", "function": "data.c:erofs_map_blocks_chunk", "path": "index released before physical shift overflow", "acquire_count": 1, "release_count": 1}, - {"id": "map-chunk-short-read", "consumer": "map", "function": "data.c:erofs_map_blocks_chunk", "path": "short index fails before acquire", "acquire_count": 0, "release_count": 0}, - {"id": "compressed-config-success", "consumer": "compressed", "function": "decompressor.c:z_erofs_read_cfg", "path": "length released; payload transferred then released by parse_cfgs", "acquire_count": 2, "release_count": 2}, - {"id": "compressed-config-second-short", "consumer": "compressed", "function": "decompressor.c:z_erofs_read_cfg", "path": "length released; payload not acquired", "acquire_count": 1, "release_count": 1}, - {"id": "compressed-data-decode-error", "consumer": "compressed", "function": "zdata.c:z_erofs_read_extent", "path": "metadata input released after decoder error", "acquire_count": 1, "release_count": 1}, - {"id": "compressed-map-success", "consumer": "compressed", "function": "zmap.c:z_erofs_read_extent", "path": "extent record copied then released", "acquire_count": 1, "release_count": 1}, - {"id": "compressed-map-short", "consumer": "compressed", "function": "zmap.c:z_erofs_read_extent", "path": "short record fails before acquire", "acquire_count": 0, "release_count": 0}, - {"id": "inode-compact-success", "consumer": "inode", "function": "inode.c:erofs_read_inode", "path": "compact inode decoded and released", "acquire_count": 1, "release_count": 1}, - {"id": "inode-invalid-format", "consumer": "inode", "function": "inode.c:erofs_read_inode", "path": "first acquired inode released on format rejection", "acquire_count": 1, "release_count": 1}, - {"id": "inode-extended-second-short", "consumer": "inode", "function": "inode.c:erofs_read_inode", "path": "compact probe released; extended read not acquired", "acquire_count": 1, "release_count": 1}, - {"id": "inode-extended-success", "consumer": "inode", "function": "inode.c:erofs_read_inode", "path": "compact probe and extended inode each released", "acquire_count": 2, "release_count": 2}, - {"id": "super-read-success", "consumer": "super", "function": "super.c:erofs_read_superblock", "path": "super bytes copied and released before validation", "acquire_count": 1, "release_count": 1}, - {"id": "super-read-short", "consumer": "super", "function": "super.c:erofs_read_superblock", "path": "short provider read fails before acquire", "acquire_count": 0, "release_count": 0}, - {"id": "super-device-validation-error", "consumer": "super", "function": "super.c:erofs_scan_devices", "path": "device table released through out label", "acquire_count": 1, "release_count": 1}, - {"id": "xattr-prefix-success", "consumer": "xattr", "function": "xattr.c:erofs_xattr_read_metadata", "path": "length released; prefix transferred then released by caller", "acquire_count": 2, "release_count": 2}, - {"id": "xattr-prefix-invalid-length", "consumer": "xattr", "function": "xattr.c:erofs_xattr_read_metadata", "path": "length record released before validation error", "acquire_count": 1, "release_count": 1}, - {"id": "xattr-prefix-second-short", "consumer": "xattr", "function": "xattr.c:erofs_xattr_read_metadata", "path": "length released; prefix not acquired", "acquire_count": 1, "release_count": 1}, - {"id": "xattr-body-validation-error", "consumer": "xattr", "function": "xattr.c:erofs_xattr_load_body", "path": "body released through fail label", "acquire_count": 1, "release_count": 1}, - {"id": "xattr-get-iterator-error", "consumer": "xattr", "function": "xattr.c:erofs_getxattr", "path": "body released after inline/shared iterator error", "acquire_count": 1, "release_count": 1} - ], - "decision": { - "go": "all required categories observed; every legal tuple equals expected by field; each successful acquisition has one release; short reads acquire nothing; Linux ownership anchors and all audited consumer bodies are present", - "stop": "missing category, legal tuple delta, unexplained errno, incomplete direct consumer inventory, or any acquire/release mismatch", - "h07": "success and request before EOF implies positive m_llen" - }, - "timeouts": {"host_seconds": 240, "qemu_seconds": 1200}, - "qemu_required_for_this_candidate_gate": false, - "full_feature_suite_required": false -} diff --git a/tests/pre15/gates/P15-005.sh b/tests/pre15/gates/P15-005.sh deleted file mode 100755 index d4995d1..0000000 --- a/tests/pre15/gates/P15-005.sh +++ /dev/null @@ -1,1018 +0,0 @@ -#!/bin/sh -set -eu - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-005-input.json - -mode=base -base= -output= -oracle= -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { echo '--base requires a commit' >&2; exit 2; } - mode=base - base=$2 - shift 2 - ;; - --worktree) - mode=worktree - shift - ;; - --output) - test "$#" -ge 2 || { echo '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - --oracle) - test "$#" -ge 2 || { echo '--oracle requires a JSON file' >&2; exit 2; } - oracle=$2 - shift 2 - ;; - *) - echo "unknown argument: $1" >&2 - exit 2 - ;; - esac -done - -test -n "$output" || { echo '--output is required' >&2; exit 2; } -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { echo "refusing existing output: $output" >&2; exit 2; } -mkdir -p "$output" - -python3 - "$root" "$input" "$mode" "$base" "$output" "$oracle" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -import struct -import subprocess -import sys - - -root = Path(sys.argv[1]) -input_path = Path(sys.argv[2]) -mode = sys.argv[3] -requested_base = sys.argv[4] -output = Path(sys.argv[5]) -oracle_path = Path(sys.argv[6]) if sys.argv[6] else None -spec = json.loads(input_path.read_text(encoding="ascii")) - - -def run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: - return subprocess.run(argv, check=True, text=True, **kwargs) - - -def git(*args: str) -> str: - return subprocess.check_output( - ["git", "-C", str(root), *args], text=True - ).strip() - - -if mode == "base": - if not requested_base: - raise SystemExit("base mode requires --base") - resolved = git("rev-parse", f"{requested_base}^{{commit}}") - if resolved != spec["required_base"]: - raise SystemExit( - f"P15-005 must gate the frozen BASE {spec['required_base']}, got {resolved}" - ) -else: - resolved = git("rev-parse", "HEAD") - - -def source(path: str) -> str: - if mode == "base": - return subprocess.check_output( - ["git", "-C", str(root), "show", f"{resolved}:{path}"], - text=True, - ) - return (root / path).read_text(encoding="utf-8") - - -sources = {path: source(path) for path in spec["source_paths"]} -source_hashes = { - path: hashlib.sha256(text.encode("utf-8")).hexdigest() - for path, text in sources.items() -} -for path, expected in spec["linux_sha256"].items(): - if source_hashes[path] != expected: - raise SystemExit(f"Linux oracle identity mismatch for {path}") -for path, anchors in spec["linux_semantic_anchors"].items(): - for anchor in anchors: - if anchor not in sources[path]: - raise SystemExit(f"Linux semantic anchor absent in {path}: {anchor}") - - -def extract_function(text: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", text, re.MULTILINE) - if not match: - raise ValueError(f"function not found: {name}") - start = text.rfind("\n\n", 0, match.start()) + 2 - brace = text.find("{", match.end()) - if brace < 0: - raise ValueError(f"function has no body: {name}") - depth = 0 - state = "code" - index = brace - while index < len(text): - char = text[index] - following = text[index + 1] if index + 1 < len(text) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return text[start : index + 1] + "\n" - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise ValueError(f"unterminated function: {name}") - - -function_bodies: dict[str, str] = {} -for key in spec["baseline_function_sha256"]: - filename, function = key.split(":", 1) - function_bodies[key] = extract_function( - sources[f"repo-pre-15/src/{filename}"], function - ) -if mode == "base": - for key, expected in spec["baseline_function_sha256"].items(): - actual = hashlib.sha256(function_bodies[key].encode("utf-8")).hexdigest() - if actual != expected: - raise SystemExit(f"unreviewed baseline consumer body: {key}") - - -def all_functions(text: str) -> dict[str, str]: - result: dict[str, str] = {} - for match in re.finditer(r"^([A-Za-z_][A-Za-z0-9_]*)\s*\(", text, re.MULTILINE): - name = match.group(1) - try: - result[name] = extract_function(text, name) - except ValueError: - continue - return result - - -direct_sites: set[str] = set() -for filename in ( - "data.c", - "decompressor.c", - "inode.c", - "super.c", - "xattr.c", - "zdata.c", - "zmap.c", -): - funcs = all_functions(sources[f"repo-pre-15/src/{filename}"]) - for name, body in funcs.items(): - calls = body[body.find("{") :] - direct = "erofs_read_metadata(" in calls - if filename in {"decompressor.c", "super.c", "xattr.c"}: - direct = direct or "erofs_bread(" in calls - if direct: - direct_sites.add(f"{filename}:{name}") -expected_direct = set(spec["direct_metadata_functions"]) -if direct_sites != expected_direct: - raise SystemExit( - "direct metadata consumer inventory mismatch: " - f"missing={sorted(expected_direct - direct_sites)} " - f"extra={sorted(direct_sites - expected_direct)}" - ) - - -object_mode = "struct erofs_buf {" in sources["repo-pre-15/src/internal.h"] -ownership_failures: list[dict[str, object]] = [] -candidate_contract_result: dict[str, object] = {"status": "NOT_APPLICABLE"} -if mode == "worktree": - internal = sources["repo-pre-15/src/internal.h"] - data_source = sources["repo-pre-15/src/data.c"] - required_object_markers = [ - "struct erofs_buf {", - "void *data;", - "void (*release)(void *);", - "void erofs_put_metabuf(struct erofs_buf *buf);", - "erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid,", - "struct erofs_buf *buf)", - ] - for marker in required_object_markers: - if marker not in internal and marker not in data_source: - raise SystemExit(f"candidate metadata object marker absent: {marker}") - if re.search( - r"erofs_read_metadata\s*\([^;]*size_t\s+len\s*,\s*void\s*\*\*bufp\s*\)\s*;", - internal, - re.DOTALL, - ): - raise SystemExit("legacy metadata void-pointer prototype remains") - - contract = spec["candidate_ownership_contract"] - contract_keys = set(contract["object_local_counts"]) - if contract_keys != set(spec["baseline_function_sha256"]): - raise SystemExit("candidate ownership contract does not cover every audited function") - function_metrics = [] - for key in sorted(contract_keys): - body = function_bodies[key] - calls = body[body.find("{") :] - initialized_objects = re.findall( - r"\bstruct\s+erofs_buf\s+([A-Za-z_][A-Za-z0-9_]*)\s*=\s*" - r"EROFS_BUF_INITIALIZER\s*;", - calls, - ) - all_objects = re.findall( - r"\bstruct\s+erofs_buf\s+([A-Za-z_][A-Za-z0-9_]*)\s*(?:=|;)", - calls, - ) - metrics = { - "function": key, - "object_local_count": len(initialized_objects), - "put_count": calls.count("erofs_put_metabuf("), - "legacy_release_count": calls.count("erofs_brelse("), - "raw_bread_count": calls.count("erofs_bread("), - } - function_metrics.append(metrics) - expected = { - "object_local_count": contract["object_local_counts"][key], - "put_count": contract["put_counts"][key], - "legacy_release_count": contract["legacy_release_counts"].get(key, 0), - "raw_bread_count": contract["raw_bread_counts"].get(key, 0), - } - for field, value in expected.items(): - if metrics[field] != value: - ownership_failures.append( - { - "function": key, - "field": field, - "expected": value, - "actual": metrics[field], - } - ) - if sorted(all_objects) != sorted(initialized_objects): - ownership_failures.append( - {"function": key, "field": "uninitialized-metadata-object"} - ) - for variable in initialized_objects: - if f"&{variable}" not in calls: - ownership_failures.append( - {"function": key, "field": "object-address-unused", "variable": variable} - ) - if f"erofs_put_metabuf(&{variable})" not in calls: - ownership_failures.append( - {"function": key, "field": "object-not-released", "variable": variable} - ) - raw_pointer_names = re.findall( - r"\bvoid\s*\*\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:[;=,])", calls - ) - for api in contract["acquire_apis"]: - for call in re.findall( - r"\b" + re.escape(api) + r"\s*\((.*?);", calls, re.DOTALL - ): - for variable in raw_pointer_names: - if re.search(r"&\s*" + re.escape(variable) + r"\b", call): - ownership_failures.append( - { - "function": key, - "field": "raw-pointer-metadata-acquire", - "api": api, - "variable": variable, - } - ) - candidate_contract_result = { - "status": "PASS" if not ownership_failures else "FAIL", - "function_metrics": function_metrics, - "transfer_functions": contract["transfer_functions"], - "failures": ownership_failures, - } - - -def compile_and_run(name: str, program: str) -> list[dict[str, int | str]]: - source_path = output / f"{name}.c" - binary_path = output / name - compile_stdout = output / f"{name}.compile.stdout" - compile_stderr = output / f"{name}.compile.stderr" - run_stdout = output / f"{name}.stdout" - run_stderr = output / f"{name}.stderr" - source_path.write_text(program, encoding="ascii") - compiler = subprocess.run( - [ - "cc", - "-std=c17", - "-O0", - "-Wall", - "-Wextra", - "-Werror", - str(source_path), - "-o", - str(binary_path), - ], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - compile_stdout.write_text(compiler.stdout, encoding="utf-8") - compile_stderr.write_text(compiler.stderr, encoding="utf-8") - if compiler.returncode != 0: - raise SystemExit(f"{name} extractor compilation failed") - executed = subprocess.run( - [str(binary_path)], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=60, - ) - run_stdout.write_text(executed.stdout, encoding="utf-8") - run_stderr.write_text(executed.stderr, encoding="utf-8") - if executed.returncode != 0: - raise SystemExit(f"{name} extractor execution failed") - records = [] - for line in executed.stdout.splitlines(): - fields = line.split("\t") - if len(fields) != 11 or fields[0] != "tuple": - raise SystemExit(f"invalid {name} extractor output: {line!r}") - records.append( - { - "id": fields[1], - "m_la": int(fields[2]), - "m_pa": int(fields[3]), - "m_llen": int(fields[4]), - "m_plen": int(fields[5]), - "m_deviceid": int(fields[6]), - "m_flags": int(fields[7]), - "errno": int(fields[8]), - "acquire_count": int(fields[9]), - "release_count": int(fields[10]), - } - ) - return records - - -def compile_helper(name: str, program: str) -> list[str]: - source_path = output / f"{name}.c" - binary_path = output / name - compile_stdout = output / f"{name}.compile.stdout" - compile_stderr = output / f"{name}.compile.stderr" - run_stdout = output / f"{name}.stdout" - run_stderr = output / f"{name}.stderr" - source_path.write_text(program, encoding="ascii") - compiler = subprocess.run( - [ - "cc", - "-std=c17", - "-O0", - "-Wall", - "-Wextra", - "-Werror", - str(source_path), - "-o", - str(binary_path), - ], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - compile_stdout.write_text(compiler.stdout, encoding="utf-8") - compile_stderr.write_text(compiler.stderr, encoding="utf-8") - if compiler.returncode != 0: - raise SystemExit(f"{name} helper extractor compilation failed") - executed = subprocess.run( - [str(binary_path)], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=60, - ) - run_stdout.write_text(executed.stdout, encoding="utf-8") - run_stderr.write_text(executed.stderr, encoding="utf-8") - if executed.returncode != 0: - raise SystemExit(f"{name} helper extractor execution failed") - return executed.stdout.splitlines() - - -common = r''' -#include -#include -#include -#include -#include -#include - -#define EIO 5 -#define EINVAL 22 -#define EOPNOTSUPP 45 -#define EOVERFLOW 84 -#define EINTEGRITY 97 -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define roundup2(x, y) ((((x) + ((y) - 1)) / (y)) * (y)) -#define rounddown2(x, y) ((x) - ((x) % (y))) -#define bzero(p, n) memset((p), 0, (n)) -#define le16toh(x) (x) -#define le32toh(x) (x) -#define le64toh(x) (x) -typedef uint64_t erofs_nid_t; -typedef uint64_t erofs_off_t; -typedef uint64_t erofs_blk_t; -static __attribute__((unused)) uint16_t le16dec(const void *pointer) { uint16_t value; memcpy(&value, pointer, sizeof(value)); return value; } -static __attribute__((unused)) uint32_t le32dec(const void *pointer) { uint32_t value; memcpy(&value, pointer, sizeof(value)); return value; } -static __attribute__((unused)) uint64_t le64dec(const void *pointer) { uint64_t value; memcpy(&value, pointer, sizeof(value)); return value; } -''' - - -buffer_compat = r''' -struct erofs_buf { - void *data; - void (*release)(void *); -}; -#define EROFS_BUF_INITIALIZER { .data = NULL, .release = NULL } -static unsigned int gate_acquires; -static unsigned int gate_releases; -static const unsigned char *gate_bytes; -static size_t gate_bytes_len; -static uint64_t gate_bytes_base; -static int gate_reader_mode; -static void gate_release(void *data) -{ - if (data != NULL) { - ++gate_releases; - free(data); - } -} -static __attribute__((unused)) void erofs_brelse(void *data) { gate_release(data); } -static __attribute__((unused)) void erofs_put_metabuf(struct erofs_buf *buf) -{ - void *data; - void (*release)(void *); - if (buf == NULL || buf->data == NULL) - return; - data = buf->data; - release = buf->release; - buf->data = NULL; - buf->release = NULL; - if (release != NULL) - release(data); -} -''' - - -def reader_definition(objects: bool) -> str: - if objects: - return r''' -static int erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid, - erofs_off_t off, size_t len, struct erofs_buf *buf) -{ - void *data; - (void)sbi; (void)nid; - if (gate_reader_mode == 2 || off < gate_bytes_base || - len > gate_bytes_len || off - gate_bytes_base > gate_bytes_len - len) - return (EIO); - data = malloc(len); - if (data == NULL) - return (EIO); - memcpy(data, gate_bytes + (off - gate_bytes_base), len); - buf->data = data; - buf->release = gate_release; - ++gate_acquires; - return (0); -} -''' - return r''' -static int erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid, - erofs_off_t off, size_t len, void **bufp) -{ - void *data; - (void)sbi; (void)nid; - if (gate_reader_mode == 2 || off < gate_bytes_base || - len > gate_bytes_len || off - gate_bytes_base > gate_bytes_len - len) - return (EIO); - data = malloc(len); - if (data == NULL) - return (EIO); - memcpy(data, gate_bytes + (off - gate_bytes_base), len); - *bufp = data; - ++gate_acquires; - return (0); -} -''' - - -def c_bytes(data: bytes) -> str: - if not data: - return "0" - return ", ".join(f"0x{byte:02x}" for byte in data) - - -data_cases = [case for case in spec["tuple_cases"] if case["engine"] == "data"] -data_case_code = [] -for index, case in enumerate(data_cases): - reader = case["reader"] - raw = bytes.fromhex(reader.get("bytes_hex", "")) - sbi = case["sbi"] - inode = case["inode"] - data_case_code.append( - f''' - static const unsigned char bytes_{index}[] = {{ {c_bytes(raw)} }}; - struct erofs_sb_info sbi_{index} = {{ - .block_size = {sbi['block_size']}ULL, - .blkszbits = {sbi['blkszbits']}, - .blocks = {sbi['blocks']}ULL, - .device_id_mask = {sbi['device_id_mask']}, - }}; - struct erofs_inode vi_{index} = {{ - .nid = {inode['nid']}ULL, - .size = {inode['size']}ULL, - .inode_off = {inode['inode_off']}ULL, - .startblk = {inode['startblk']}ULL, - .datalayout = {inode['datalayout']}, - .inode_isize = {inode['inode_isize']}, - .xattr_isize = {inode['xattr_isize']}, - .chunkformat = {inode['chunkformat']}, - .chunkbits = {inode['chunkbits']}, - }}; - gate_bytes = bytes_{index}; - gate_bytes_len = {len(raw)}; - gate_bytes_base = {inode['inode_off'] + inode['inode_isize'] + inode['xattr_isize']}ULL; - gate_reader_mode = {2 if reader['mode'] == 'error' else 0}; - gate_acquires = gate_releases = 0; - pa = 0; device = 0; run = 0; hole = false; metadata = false; - error = erofs_map_blocks(&sbi_{index}, &vi_{index}, {case['request']}ULL, - &pa, &device, &run, &hole, &metadata); - flags = (error == 0 && run != 0 && !hole ? EROFS_MAP_MAPPED : 0) | - (metadata ? EROFS_MAP_META : 0); - printf("tuple\\t{case['id']}\\t%llu\\t%llu\\t%zu\\t%zu\\t%u\\t%u\\t%d\\t%u\\t%u\\n", - (unsigned long long){case['request']}ULL, (unsigned long long)pa, - run, run, device, flags, error, gate_acquires, gate_releases); -''' - ) - -data_preamble = common + buffer_compat + r''' -#define EROFS_INODE_FLAT_PLAIN 0 -#define EROFS_INODE_COMPRESSED_FULL 1 -#define EROFS_INODE_FLAT_INLINE 2 -#define EROFS_INODE_COMPRESSED_COMPACT 3 -#define EROFS_INODE_CHUNK_BASED 4 -#define EROFS_CHUNK_FORMAT_INDEXES 0x20 -#define EROFS_CHUNK_FORMAT_48BIT 0x40 -#define EROFS_BLOCK_MAP_ENTRY_SIZE 4 -#define EROFS_DIRENT_NID_METABOX (1ULL << 63) -#define EROFS_NULL_ADDR UINT64_MAX -#define EROFS_MAP_MAPPED 0x0001 -#define EROFS_MAP_META 0x0002 -struct erofs_inode_chunk_index { uint16_t startblk_hi; uint16_t device_id; uint32_t startblk_lo; } __attribute__((packed)); -struct erofs_inode { - erofs_nid_t nid; uint64_t size; erofs_off_t inode_off; erofs_blk_t startblk; - uint8_t datalayout; uint8_t inode_isize; uint32_t xattr_isize; - uint16_t chunkformat; uint8_t chunkbits; -}; -struct erofs_sb_info { - uint64_t block_size; uint8_t blkszbits; erofs_blk_t blocks; - uint16_t device_id_mask; struct erofs_inode *metabox_en; -}; -static bool erofs_nid_in_metabox(erofs_nid_t nid) { return (nid & EROFS_DIRENT_NID_METABOX) != 0; } -''' -data_program = ( - data_preamble - + reader_definition(object_mode) - + extract_function(sources["repo-pre-15/src/data.c"], "erofs_inline_tail_start") - + extract_function(sources["repo-pre-15/src/data.c"], "erofs_map_blocks_chunk") - + extract_function(sources["repo-pre-15/src/data.c"], "erofs_map_blocks") - + r''' -int main(void) -{ - erofs_off_t pa; unsigned int device, flags; size_t run; - bool hole, metadata; int error; -''' - + "".join(data_case_code) - + "\treturn (0);\n}\n" -) - - -zmap_cases = [case for case in spec["tuple_cases"] if case["engine"] == "zmap"] -zmap_case_code = [] -for index, case in enumerate(zmap_cases): - reader = case["reader"] - if "records" in reader: - raw = b"".join( - struct.pack( - "> 32, - record["lstart"] & 0xFFFFFFFF, - record["lstart"] >> 32, - )[:16] - for record in reader["records"] - ) - else: - raw = bytes.fromhex(reader.get("bytes_hex", "")) - sbi = case["sbi"] - inode = case["inode"] - zmap_case_code.append( - f''' - static const unsigned char bytes_{index}[] = {{ {c_bytes(raw)} }}; - struct erofs_inode packed_{index} = {{ .nid = 999, .size = {sbi['packed_size']}ULL }}; - struct erofs_sb_info sbi_{index} = {{ - .block_size = {sbi['block_size']}, .blkszbits = {sbi['blkszbits']}, - .available_compr_algs = {sbi['available_compr_algs']}, - .packed_inode = &packed_{index}, - }}; - struct erofs_inode vi_{index} = {{ - .nid = {inode['nid']}ULL, .size = {inode['size']}ULL, - .inode_off = {inode['inode_off']}ULL, - .inode_isize = {inode['inode_isize']}, - .xattr_isize = {inode['xattr_isize']}, - .datalayout = {inode['datalayout']}, - .z_advise = {inode['z_advise']}, - .z_lclusterbits = {inode['z_lclusterbits']}, - .z_extents = {inode['z_extents']}ULL, .z_initialized = true, - }}; - struct erofs_map_blocks map_{index} = {{ .m_la = {case['request']}ULL }}; - gate_bytes = bytes_{index}; gate_bytes_len = {len(raw)}; - gate_bytes_base = {reader.get('base', 0)}ULL; - gate_reader_mode = {2 if reader['mode'] == 'error' else 0}; - gate_acquires = gate_releases = 0; - error = z_erofs_map_blocks_iter(&sbi_{index}, &vi_{index}, &map_{index}, 0); - printf("tuple\\t{case['id']}\\t%llu\\t%llu\\t%llu\\t%llu\\t%u\\t%u\\t%d\\t%u\\t%u\\n", - (unsigned long long)map_{index}.m_la, - (unsigned long long)map_{index}.m_pa, - (unsigned long long)map_{index}.m_llen, - (unsigned long long)map_{index}.m_plen, - map_{index}.m_deviceid, map_{index}.m_flags, error, - gate_acquires, gate_releases); -''' - ) - -zmap_preamble = common + buffer_compat + r''' -#define EROFS_INODE_COMPRESSED_FULL 1 -#define EROFS_MAP_MAPPED 0x0001 -#define EROFS_MAP_META 0x0002 -#define EROFS_MAP_PARTIAL_MAPPED 0x0004 -#define EROFS_MAP_PARTIAL_REF 0x0008 -#define EROFS_MAP_FRAGMENT 0x0010 -#define EROFS_MAP_FULL(f) (!((f) & (EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF))) -#define EROFS_NULL_ADDR UINT64_MAX -#define Z_EROFS_COMPRESSION_LZ4 0 -#define Z_EROFS_COMPRESSION_MAX 4 -#define Z_EROFS_COMPRESSION_SHIFTED 4 -#define Z_EROFS_COMPRESSION_INTERLACED 5 -#define Z_EROFS_COMPRESSION_RUNTIME_MAX 6 -#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) -#define Z_EROFS_PCLUSTER_MAX_DSIZE (12 * 1024 * 1024) -#define Z_EROFS_ADVISE_EXTENTS 0x0001 -#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER 0x0010 -#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER 0x0020 -#define Z_EROFS_ADVISE_EXTRECSZ_BIT 1 -#define Z_EROFS_ADVISE_EXTRECSZ_MASK 0x3 -#define Z_EROFS_EXTENT_PLEN_PARTIAL (1U << 27) -#define Z_EROFS_EXTENT_PLEN_FMT_BIT 28 -#define Z_EROFS_EXTENT_PLEN_MASK ((Z_EROFS_PCLUSTER_MAX_SIZE << 1) - 1) -struct z_erofs_extent { - uint32_t plen, pstart_lo, pstart_hi, lstart_lo, lstart_hi; - uint8_t reserved[12]; -} __attribute__((packed)); -struct z_erofs_map_header { uint32_t word0; uint16_t h_advise; uint8_t h_algorithmtype; uint8_t h_clusterbits; } __attribute__((packed)); -struct erofs_map_blocks { - erofs_off_t m_pa, m_la; uint64_t m_plen, m_llen; - unsigned short m_deviceid; char m_algorithmformat; unsigned int m_flags; -}; -struct erofs_inode { - erofs_nid_t nid; uint64_t size; erofs_off_t inode_off; - uint8_t datalayout, inode_isize; uint32_t xattr_isize; - uint16_t z_advise; uint8_t z_algorithmtype[2], z_lclusterbits; - uint16_t z_idata_size; erofs_off_t z_fragmentoff; - uint64_t z_tailextent_headlcn, z_extents; bool z_initialized, fragment; -}; -struct erofs_sb_info { - uint32_t block_size; uint8_t blkszbits; uint16_t available_compr_algs; - uint64_t packed_nid; struct erofs_inode *packed_inode; -}; -''' -zmap_stubs = r''' -static int z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi) -{ (void)sbi; return (vi->z_initialized ? 0 : EINTEGRITY); } -static int z_erofs_map_blocks_fo(struct erofs_sb_info *sbi, struct erofs_inode *vi, - struct erofs_map_blocks *map, int flags) -{ (void)sbi; (void)vi; (void)map; (void)flags; return (EOPNOTSUPP); } -''' -zmap_names = [ - "z_erofs_extent_add", - "z_erofs_extent_roundup", - "z_erofs_extent_table_pos", - "z_erofs_extent_record_pos", - "z_erofs_read_extent", - "z_erofs_extent_lstart", - "z_erofs_map_blocks_ext", - "z_erofs_map_sanity_check", - "z_erofs_map_blocks_iter", -] -zmap_program = zmap_preamble + reader_definition(object_mode) + zmap_stubs -zmap_program += extract_function( - sources["repo-pre-15/src/erofs_fs.h"], "z_erofs_extent_recsize" -) -for name in zmap_names: - zmap_program += extract_function(sources["repo-pre-15/src/zmap.c"], name) -zmap_program += "int main(void)\n{\n\tint error;\n" -zmap_program += "".join(zmap_case_code) -zmap_program += "\treturn (0);\n}\n" - - -helper_result: dict[str, object] = {"status": "NOT_APPLICABLE"} -if mode == "worktree": - helper_program = common + r''' -#define EROFS_DIRENT_NID_METABOX (1ULL << 63) -typedef int64_t off_t; -struct erofs_inode { uint64_t size; }; -struct erofs_sb_info { struct erofs_inode *metabox_en; }; -struct erofs_buf { - void *data; - void (*release)(void *); -}; -#define EROFS_BUF_INITIALIZER { .data = NULL, .release = NULL } -static int gate_error; -static unsigned int gate_acquires; -static unsigned int gate_releases; -static unsigned int gate_backend; -static bool erofs_nid_in_metabox(erofs_nid_t nid) -{ return ((nid & EROFS_DIRENT_NID_METABOX) != 0); } -static bool erofs_sb_has_metabox(struct erofs_sb_info *sbi) -{ return (sbi->metabox_en != NULL); } -static int gate_acquire(size_t len, void **bufp, unsigned int backend) -{ - void *data; - if (gate_error != 0) - return (gate_error); - data = malloc(len == 0 ? 1 : len); - if (data == NULL) - return (EIO); - *bufp = data; - ++gate_acquires; - gate_backend = backend; - return (0); -} -static int erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, - size_t len, void **bufp) -{ (void)sbi; (void)off; return (gate_acquire(len, bufp, 1)); } -static int erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, - erofs_off_t off, size_t len, void **bufp) -{ (void)sbi; (void)vi; (void)off; return (gate_acquire(len, bufp, 2)); } -static void erofs_brelse(void *data) -{ - if (data != NULL) { - ++gate_releases; - free(data); - } -} -''' - helper_program += extract_function( - sources["repo-pre-15/src/data.c"], "erofs_put_metabuf" - ) - helper_program += extract_function( - sources["repo-pre-15/src/data.c"], "erofs_read_metadata" - ) - helper_program += r''' -int main(void) -{ - struct erofs_inode metabox = { .size = 4096 }; - struct erofs_sb_info sbi = { .metabox_en = &metabox }; - struct erofs_buf buf = EROFS_BUF_INITIALIZER; - int error; - int marker; - - gate_error = 0; gate_acquires = gate_releases = gate_backend = 0; - error = erofs_read_metadata(&sbi, 1, 32, 8, &buf); - if (error != 0 || buf.data == NULL || buf.release == NULL || - gate_acquires != 1 || gate_backend != 1) - return (10); - erofs_put_metabuf(&buf); - erofs_put_metabuf(&buf); - if (buf.data != NULL || buf.release != NULL || gate_releases != 1) - return (11); - printf("helper\tprimary-success\t1\t1\n"); - - buf = (struct erofs_buf)EROFS_BUF_INITIALIZER; - gate_error = 0; gate_acquires = gate_releases = gate_backend = 0; - error = erofs_read_metadata(&sbi, EROFS_DIRENT_NID_METABOX | 2, 64, 8, - &buf); - if (error != 0 || buf.data == NULL || buf.release == NULL || - gate_acquires != 1 || gate_backend != 2) - return (12); - erofs_put_metabuf(&buf); - if (buf.data != NULL || buf.release != NULL || gate_releases != 1) - return (13); - printf("helper\tmetabox-success\t1\t1\n"); - - buf = (struct erofs_buf)EROFS_BUF_INITIALIZER; - gate_error = EIO; gate_acquires = gate_releases = gate_backend = 0; - error = erofs_read_metadata(&sbi, 1, 32, 8, &buf); - if (error != EIO || buf.data != NULL || buf.release != NULL || - gate_acquires != 0 || gate_releases != 0) - return (14); - printf("helper\tprovider-error\t0\t0\n"); - - buf = (struct erofs_buf)EROFS_BUF_INITIALIZER; - gate_error = 0; gate_acquires = gate_releases = gate_backend = 0; - error = erofs_read_metadata(&sbi, 1, UINT64_MAX, 8, &buf); - if (error != EOVERFLOW || buf.data != NULL || buf.release != NULL || - gate_acquires != 0 || gate_releases != 0) - return (15); - printf("helper\toffset-overflow\t0\t0\n"); - - buf.data = ▮ - buf.release = NULL; - erofs_put_metabuf(&buf); - erofs_put_metabuf(NULL); - if (buf.data != NULL || buf.release != NULL) - return (16); - printf("helper\tnull-release\t0\t0\n"); - return (0); -} -''' - helper_lines = compile_helper("P15-005-buffer-helper", helper_program) - expected_helper_lines = [ - "helper\tprimary-success\t1\t1", - "helper\tmetabox-success\t1\t1", - "helper\tprovider-error\t0\t0", - "helper\toffset-overflow\t0\t0", - "helper\tnull-release\t0\t0", - ] - if helper_lines != expected_helper_lines: - ownership_failures.append( - { - "field": "helper-paths", - "expected": expected_helper_lines, - "actual": helper_lines, - } - ) - helper_result = { - "status": "PASS" if helper_lines == expected_helper_lines else "FAIL", - "paths": helper_lines, - } - - -actual_records = compile_and_run("P15-005-data-extractor", data_program) -actual_records.extend(compile_and_run("P15-005-zmap-extractor", zmap_program)) -actual_by_id = {record["id"]: record for record in actual_records} -expected_by_id = {case["id"]: case["expected"] for case in spec["tuple_cases"]} -if set(actual_by_id) != set(expected_by_id): - raise SystemExit("extractor case set mismatch") -tuple_failures = [] -for case in spec["tuple_cases"]: - actual = actual_by_id[case["id"]] - expected = case["expected"] - for field in spec["tuple_fields"]: - if actual[field] != expected[field]: - tuple_failures.append( - { - "id": case["id"], - "field": field, - "expected": expected[field], - "actual": actual[field], - } - ) - -coverage = sorted( - {case["category"] for case in spec["tuple_cases"] if not case.get("supplemental")} -) -if set(coverage) != set(spec["required_categories"]): - raise SystemExit("required tuple category coverage is incomplete") -for case in spec["tuple_cases"]: - actual = actual_by_id[case["id"]] - if actual["acquire_count"] != actual["release_count"]: - tuple_failures.append({"id": case["id"], "field": "ownership"}) - if ( - actual["errno"] == 0 - and case["request"] < case["inode"]["size"] - and actual["m_llen"] == 0 - ): - tuple_failures.append({"id": case["id"], "field": "H07-positive-run"}) - - -ownership_paths = spec["ownership_paths"] -consumers = sorted({path["consumer"] for path in ownership_paths}) -required_consumers = ["compressed", "inode", "map", "plain", "super", "xattr"] -if consumers != required_consumers: - raise SystemExit(f"ownership consumer coverage mismatch: {consumers}") -for path in ownership_paths: - if path["function"] not in function_bodies: - raise SystemExit(f"ownership path has no frozen function body: {path['id']}") - if path["acquire_count"] != path["release_count"]: - raise SystemExit(f"incomplete ownership oracle path: {path['id']}") - -ownership_result = { - "status": "PASS" if not ownership_failures else "FAIL", - "consumers": consumers, - "direct_metadata_functions": sorted(direct_sites), - "paths": ownership_paths, - "baseline_body_sha256_verified": mode == "base", - "linux_semantic_anchors_verified": True, - "candidate_contract": candidate_contract_result, - "helper_paths": helper_result, - "failures": ownership_failures, - "short_read_rule": "a failed read before returned storage has acquire=release=0", - "success_rule": "each returned metadata allocation has exactly one release", -} - -tuple_result = { - "status": "PASS" if not tuple_failures else "FAIL", - "coverage": coverage, - "tuple_fields": spec["tuple_fields"], - "records": [ - { - **actual_by_id[case["id"]], - "category": case["category"], - "supplemental": bool(case.get("supplemental")), - } - for case in spec["tuple_cases"] - ], - "failures": tuple_failures, - "authoritative_sources": [ - "compiled function bodies extracted from the frozen/current FreeBSD source", - "independently frozen expected fields from EROFS records and Linux map semantics", - ], -} -(output / "tuples.json").write_text( - json.dumps(tuple_result, indent=2, sort_keys=True) + "\n", encoding="ascii" -) -(output / "ownership.json").write_text( - json.dumps(ownership_result, indent=2, sort_keys=True) + "\n", encoding="ascii" -) -(output / "source-sha256.json").write_text( - json.dumps(source_hashes, indent=2, sort_keys=True) + "\n", encoding="ascii" -) - -oracle_equal = None -if oracle_path is not None: - oracle = json.loads(oracle_path.read_text(encoding="ascii")) - oracle_records = {record["id"]: record for record in oracle["records"]} - oracle_equal = True - for case in spec["tuple_cases"]: - current = actual_by_id[case["id"]] - frozen = oracle_records.get(case["id"]) - if frozen is None or any(current[field] != frozen[field] for field in spec["tuple_fields"]): - oracle_equal = False - break - if not oracle_equal: - tuple_failures.append({"field": "frozen-baseline-replay"}) - -status = "GO" if not tuple_failures and not ownership_failures else "STOP" -result = { - "schema": 1, - "gate": spec["gate"], - "candidate": spec["candidate"], - "status": status, - "mode": mode, - "requested_base": requested_base or None, - "resolved_head": resolved, - "object_mode": object_mode, - "tuple_status": tuple_result["status"], - "ownership_status": ownership_result["status"], - "coverage": coverage, - "case_count": len(actual_records), - "ownership_path_count": len(ownership_paths), - "oracle_equal": oracle_equal, - "qemu": "NOT_RUN", - "full_feature_suite": "NOT_RUN", - "failures": tuple_failures + ownership_failures, -} -(output / "result.json").write_text( - json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii" -) -hashes = [] -for path in sorted(output.iterdir()): - if path.is_file() and path.name != "SHA256SUMS": - hashes.append(f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}") -(output / "SHA256SUMS").write_text("\n".join(hashes) + "\n", encoding="ascii") -print(json.dumps(result, sort_keys=True)) -if status != "GO": - raise SystemExit(1) -PY diff --git a/tests/pre15/gates/P15-006-input.json b/tests/pre15/gates/P15-006-input.json deleted file mode 100644 index 1e2bb20..0000000 --- a/tests/pre15/gates/P15-006-input.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "schema": 1, - "gate": "G02", - "candidate": "P15-006", - "required_base": "6673f51152a5195a8a8903aa801f820abce7936e", - "title": "independent common map tuple oracle", - "required_categories": [ - "plain", - "inline", - "hole", - "chunk", - "multidevice", - "compressed", - "fragment", - "partial-reference", - "post-EOF", - "bounds", - "overflow", - "invalid" - ], - "required_subcategories": [ - "plain-boundaries", - "plain-overflow", - "inline-boundaries", - "inline-overflow", - "chunk-raw32", - "chunk-index32", - "chunk-index48", - "chunk-holes", - "chunk-bounds", - "chunk-overflow", - "device-mask", - "device-resolution", - "extent-4", - "extent-8", - "extent-16", - "extent-32", - "compressed-flags", - "compressed-bounds", - "compressed-overflow", - "compressed-invalid", - "cleanup-before-acquire", - "cleanup-after-acquire" - ], - "tuple_fields": [ - "m_la", - "m_pa", - "m_llen", - "m_plen", - "m_deviceid", - "m_flags", - "m_algorithmformat", - "errno", - "acquire_count", - "release_count" - ], - "tuple_byte_layout": "&2; exit 2; } - mode=base - base=$2 - shift 2 - ;; - --worktree) - mode=worktree - shift - ;; - --output) - test "$#" -ge 2 || { printf 'missing --output value\n' >&2; exit 2; } - output=$2 - shift 2 - ;; - --oracle) - test "$#" -ge 2 || { printf 'missing --oracle value\n' >&2; exit 2; } - oracle=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$mode" || { printf 'use --base COMMIT or --worktree\n' >&2; exit 2; } -test -n "$output" || { printf 'missing --output DIR\n' >&2; exit 2; } -test -f "$input" || { printf 'missing gate input: %s\n' "$input" >&2; exit 2; } -for tool in cc git python3; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done - -python3 - "$root" "$input" "$mode" "$base" "$output" "$oracle" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -import struct -import subprocess -import sys -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -MODE = sys.argv[3] -REQUESTED_BASE = sys.argv[4] -OUTPUT = Path(sys.argv[5]) -ORACLE = Path(sys.argv[6]) if sys.argv[6] else None -SPEC = json.loads(INPUT.read_text(encoding="ascii")) - -UINT64_MAX = (1 << 64) - 1 -UINT32_MAX = (1 << 32) - 1 -NULL_ADDR = UINT64_MAX -META_NID = 1 << 63 -MAPPED = 0x0001 -META = 0x0002 -PARTIAL_MAPPED = 0x0004 -PARTIAL_REF = 0x0008 -FRAGMENT = 0x0010 -EIO = 5 -ENODEV = 19 -EOPNOTSUPP = 45 -EOVERFLOW = 84 -EINTEGRITY = 97 -PLEN_PARTIAL = 1 << 27 -PLEN_FMT_BIT = 28 -PCLUSTER_MAX = 1024 * 1024 -PLEN_MASK = (PCLUSTER_MAX << 1) - 1 -PCLUSTER_MAX_DSIZE = 12 * 1024 * 1024 -COMPRESSION_MAX = 4 -COMPRESSION_SHIFTED = 4 -COMPRESSION_INTERLACED = 5 -COMPRESSION_RUNTIME_MAX = 6 - - -def run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - return subprocess.run(argv, check=False, text=True, **kwargs) - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def canonical(value: Any) -> bytes: - return json.dumps( - value, ensure_ascii=True, separators=(",", ":"), sort_keys=True - ).encode("ascii") - - -def extract_function(text: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", text, re.MULTILINE) - if not match: - raise SystemExit(f"function not found: {name}") - start = text.rfind("\n\n", 0, match.start()) + 2 - brace = text.find("{", match.end()) - if brace < 0: - raise SystemExit(f"function body not found: {name}") - depth = 0 - state = "code" - index = brace - while index < len(text): - char = text[index] - following = text[index + 1] if index + 1 < len(text) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return text[start : index + 1] + "\n" - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def source_at(path: str, resolved: str) -> str: - if MODE == "base": - completed = run( - ["git", "-C", str(ROOT), "show", f"{resolved}:{path}"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read {path} at {resolved}: {completed.stderr}") - return completed.stdout - return (ROOT / path).read_text(encoding="utf-8") - - -if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-006": - raise SystemExit("invalid P15-006 input schema") -if OUTPUT.exists(): - raise SystemExit(f"refusing to overwrite output: {OUTPUT}") -OUTPUT.mkdir(parents=True) - -rev = REQUESTED_BASE if MODE == "base" else "HEAD" -resolved_run = run( - ["git", "-C", str(ROOT), "rev-parse", f"{rev}^{{commit}}"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -if resolved_run.returncode != 0: - raise SystemExit(f"cannot resolve {rev}: {resolved_run.stderr}") -resolved = resolved_run.stdout.strip() -if MODE == "base" and resolved != SPEC["required_base"]: - raise SystemExit( - f"P15-006 baseline mismatch: expected {SPEC['required_base']}, got {resolved}" - ) - -sources = {path: source_at(path, resolved) for path in SPEC["source_paths"]} -data_source = sources["repo-pre-15/src/data.c"] -zmap_source = sources["repo-pre-15/src/zmap.c"] -internal_source = sources["repo-pre-15/src/internal.h"] - -candidate_mode = re.search( - r"^erofs_map_blocks\s*\([^)]*struct erofs_map_blocks \*map\s*\)", - data_source, - re.MULTILINE | re.DOTALL, -) is not None -if candidate_mode and not re.search( - r"^erofs_map_blocks_legacy\s*\(", data_source, re.MULTILINE -): - raise SystemExit("candidate map adapter has no file-local legacy target") -if not candidate_mode and MODE == "worktree" and ORACLE is not None: - raise SystemExit("candidate replay requested but common map adapter is absent") - -source_hashes = { - path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items() -} -protected_hashes: dict[str, str] = {} -for key, expected_hash in SPEC["protected_function_sha256"].items(): - filename, function = key.split(":", 1) - text = data_source if filename == "data.c" else zmap_source - actual_hash = sha256_bytes(extract_function(text, function).encode("utf-8")) - protected_hashes[key] = actual_hash - if actual_hash != expected_hash: - raise SystemExit(f"protected producer/GEOM body changed: {key}") - -if candidate_mode: - legacy_body = extract_function(data_source, "erofs_map_blocks_legacy") - expected_legacy_hash = SPEC["candidate_legacy_map_sha256"] -else: - legacy_body = extract_function(data_source, "erofs_map_blocks") - expected_legacy_hash = SPEC["baseline_legacy_map_sha256"] -if sha256_bytes(legacy_body.encode("utf-8")) != expected_legacy_hash: - raise SystemExit("plain/chunk legacy producer body changed") - -map_match = re.search( - r"^struct erofs_map_blocks \{\n.*?^\};\n", - internal_source, - re.MULTILINE | re.DOTALL, -) -if map_match is None: - raise SystemExit("struct erofs_map_blocks is absent") -map_struct_hash = sha256_bytes(map_match.group(0).encode("utf-8")) -if map_struct_hash != SPEC["map_struct_sha256"]: - raise SystemExit("map object fields or ordering changed before B07 producers") - - -def tuple_record(**values: int) -> dict[str, int]: - record = { - "m_la": 0, - "m_pa": 0, - "m_llen": 0, - "m_plen": 0, - "m_deviceid": 0, - "m_flags": 0, - "m_algorithmformat": 0, - "errno": 0, - "acquire_count": 0, - "release_count": 0, - } - record.update(values) - return record - - -def data_case( - case_id: str, - category: str, - subcategories: list[str], - request: int, - *, - layout: int = 0, - size: int = 12288, - startblk: int = 32, - inode_off: int = 0, - inode_isize: int = 64, - xattr_isize: int = 0, - chunkbits: int = 12, - chunkformat: int = 0, - nid: int = 1, - block_size: int = 4096, - blkszbits: int = 12, - blocks: int = 1 << 20, - device_id_mask: int = 0xFFFF, - metabox_present: bool = False, - metabox_size: int = 0, - reader_base: int = 0, - reader_bytes: bytes = b"", - reader_error: bool = False, -) -> dict[str, Any]: - return { - "id": case_id, - "engine": "data", - "category": category, - "subcategories": subcategories, - "request": request, - "sbi": { - "block_size": block_size, - "blkszbits": blkszbits, - "blocks": blocks, - "device_id_mask": device_id_mask, - "metabox_present": metabox_present, - "metabox_size": metabox_size, - }, - "inode": { - "nid": nid, - "size": size, - "datalayout": layout, - "startblk": startblk, - "inode_off": inode_off, - "inode_isize": inode_isize, - "xattr_isize": xattr_isize, - "chunkbits": chunkbits, - "chunkformat": chunkformat, - }, - "reader": { - "base": reader_base, - "bytes": reader_bytes, - "error": reader_error, - }, - } - - -def raw32(block: int) -> bytes: - return struct.pack(" bytes: - high = (block >> 32) & 0xFFFF if use_48bit else 0 - return struct.pack(" list[dict[str, Any]]: - cases = [ - data_case("plain-start", "plain", ["plain-boundaries"], 0), - data_case("plain-block-last", "plain", ["plain-boundaries"], 4095), - data_case("plain-second-start", "plain", ["plain-boundaries"], 4096), - data_case("plain-final-byte", "plain", ["plain-boundaries"], 12287), - data_case("plain-eof", "post-EOF", ["plain-boundaries"], 12288), - data_case("plain-post-eof", "post-EOF", ["plain-boundaries"], 12305), - data_case("plain-empty", "post-EOF", ["plain-boundaries"], 0, size=0), - data_case("plain-hole-start", "hole", ["plain-boundaries"], 0, startblk=NULL_ADDR), - data_case("plain-hole-edge", "hole", ["plain-boundaries"], 4095, startblk=NULL_ADDR), - data_case( - "plain-shift-overflow", "overflow", ["plain-overflow"], 0, - startblk=(UINT64_MAX >> 12) + 1, - ), - data_case( - "plain-add-overflow", "overflow", ["plain-overflow"], 4096, - startblk=UINT64_MAX >> 12, - ), - data_case("plain-invalid-layout", "invalid", ["plain-boundaries"], 0, layout=7), - data_case("inline-head-start", "inline", ["inline-boundaries"], 0, layout=2, size=5000, startblk=40, inode_off=8192, xattr_isize=32), - data_case("inline-head-last", "inline", ["inline-boundaries"], 4095, layout=2, size=5000, startblk=40, inode_off=8192, xattr_isize=32), - data_case("inline-tail-start", "inline", ["inline-boundaries"], 4096, layout=2, size=5000, startblk=40, inode_off=8192, xattr_isize=32), - data_case("inline-tail-middle", "inline", ["inline-boundaries"], 4500, layout=2, size=5000, startblk=40, inode_off=8192, xattr_isize=32), - data_case("inline-final-byte", "inline", ["inline-boundaries"], 4999, layout=2, size=5000, startblk=40, inode_off=8192, xattr_isize=32), - data_case("inline-eof", "post-EOF", ["inline-boundaries"], 5000, layout=2, size=5000, startblk=40, inode_off=8192, xattr_isize=32), - data_case("inline-head-hole", "hole", ["inline-boundaries"], 0, layout=2, size=5000, startblk=NULL_ADDR, inode_off=8192, xattr_isize=32), - data_case("inline-one-block", "inline", ["inline-boundaries"], 0, layout=2, size=4096, startblk=40, inode_off=8192, xattr_isize=32), - data_case("inline-one-byte-tail-head", "inline", ["inline-boundaries"], 4095, layout=2, size=4097, startblk=40, inode_off=8192, xattr_isize=32), - data_case("inline-one-byte-tail", "inline", ["inline-boundaries"], 4096, layout=2, size=4097, startblk=40, inode_off=8192, xattr_isize=32), - data_case("inline-inode-add-overflow", "overflow", ["inline-overflow"], 4096, layout=2, size=5000, inode_off=UINT64_MAX - 31, inode_isize=64), - data_case("inline-xattr-add-overflow", "overflow", ["inline-overflow"], 4096, layout=2, size=5000, inode_off=UINT64_MAX - 100, inode_isize=50, xattr_isize=100), - data_case("inline-offset-add-overflow", "overflow", ["inline-overflow"], 6000, layout=2, size=8192, inode_off=UINT64_MAX - 1000), - ] - - cases.extend([ - data_case("chunk-raw32-start", "chunk", ["chunk-raw32"], 0, layout=4, reader_base=64, reader_bytes=raw32(77)), - data_case("chunk-raw32-last", "chunk", ["chunk-raw32"], 4095, layout=4, reader_base=64, reader_bytes=raw32(77)), - data_case("chunk-raw32-next", "chunk", ["chunk-raw32"], 4096, layout=4, reader_base=68, reader_bytes=raw32(78)), - data_case("chunk-raw32-final", "chunk", ["chunk-raw32"], 9999, layout=4, size=10000, reader_base=72, reader_bytes=raw32(79)), - data_case("chunk-raw32-hole", "hole", ["chunk-holes", "chunk-raw32"], 512, layout=4, reader_base=64, reader_bytes=raw32(UINT32_MAX)), - data_case("chunk-index32-mask", "multidevice", ["chunk-index32", "device-mask"], 0, layout=4, chunkformat=0x20, device_id_mask=3, reader_base=64, reader_bytes=index_entry(77, 7, False)), - data_case("chunk-index32-middle", "chunk", ["chunk-index32"], 9000, layout=4, size=20000, chunkbits=13, chunkformat=0x20, reader_base=72, reader_bytes=index_entry(77, 0, False)), - data_case("chunk-index32-hole", "hole", ["chunk-holes", "chunk-index32"], 0, layout=4, chunkformat=0x20, reader_base=64, reader_bytes=index_entry(UINT32_MAX, 9, False)), - data_case("chunk-index48-start", "multidevice", ["chunk-index48"], 0, layout=4, chunkformat=0x60, reader_base=64, reader_bytes=index_entry((1 << 32) + 5, 2, True)), - data_case("chunk-index48-middle", "multidevice", ["chunk-index48"], 12345, layout=4, size=24576, chunkbits=13, chunkformat=0x60, reader_base=72, reader_bytes=index_entry((1 << 32) + 5, 2, True)), - data_case("chunk-index48-hole", "hole", ["chunk-holes", "chunk-index48"], 0, layout=4, chunkformat=0x60, reader_base=64, reader_bytes=index_entry((1 << 48) - 1, 2, True)), - data_case("chunk-device-mask-zero", "multidevice", ["chunk-index48", "device-mask"], 0, layout=4, chunkformat=0x60, device_id_mask=0, reader_base=64, reader_bytes=index_entry(81, 0xFFFF, True)), - data_case("chunk-eof", "post-EOF", ["chunk-index32"], 4096, layout=4, size=4096, chunkformat=0x20), - data_case("chunk-inode-add-overflow", "overflow", ["chunk-overflow", "cleanup-before-acquire"], 0, layout=4, inode_off=UINT64_MAX - 10, inode_isize=20, chunkformat=0x20), - data_case("chunk-xattr-add-overflow", "overflow", ["chunk-overflow", "cleanup-before-acquire"], 0, layout=4, inode_off=UINT64_MAX - 100, inode_isize=50, xattr_isize=100, chunkformat=0x20), - data_case("chunk-align-overflow", "overflow", ["chunk-overflow", "cleanup-before-acquire"], 0, layout=4, inode_off=UINT64_MAX - 3, inode_isize=0, chunkformat=0x20), - data_case("chunk-index-multiply-overflow", "overflow", ["chunk-overflow", "cleanup-before-acquire"], UINT64_MAX - 1, layout=4, size=UINT64_MAX, chunkbits=0, chunkformat=0x20), - data_case("chunk-image-size-overflow", "overflow", ["chunk-overflow", "cleanup-before-acquire"], 0, layout=4, blocks=(UINT64_MAX >> 17) + 1, block_size=1 << 17, blkszbits=17, chunkformat=0x20), - data_case("chunk-primary-bounds", "bounds", ["chunk-bounds", "cleanup-before-acquire"], 0, layout=4, inode_off=4096, inode_isize=0, blocks=1, chunkformat=0x20), - data_case("chunk-primary-tail-bounds", "bounds", ["chunk-bounds", "cleanup-before-acquire"], 0, layout=4, inode_off=4090, inode_isize=0, blocks=1, chunkformat=0x20), - data_case("chunk-metabox-missing", "bounds", ["chunk-bounds", "cleanup-before-acquire"], 0, layout=4, nid=META_NID | 2, chunkformat=0x20), - data_case("chunk-metabox-bounds", "bounds", ["chunk-bounds", "cleanup-before-acquire"], 0, layout=4, nid=META_NID | 2, inode_off=64, chunkformat=0x20, metabox_present=True, metabox_size=71), - data_case("chunk-reader-short", "bounds", ["chunk-bounds", "cleanup-before-acquire"], 0, layout=4, chunkformat=0x20, reader_base=64, reader_bytes=b"\0\0\0\0", reader_error=True), - data_case("chunk-shift-overflow", "overflow", ["chunk-overflow", "cleanup-after-acquire"], 0, layout=4, chunkformat=0x60, block_size=1 << 17, blkszbits=17, reader_base=64, reader_bytes=index_entry(1 << 47, 0, True)), - data_case("chunk-offset-overflow", "overflow", ["chunk-overflow", "cleanup-after-acquire"], 4096, layout=4, size=8192, chunkbits=13, chunkformat=0x60, reader_base=64, reader_bytes=index_entry(UINT64_MAX >> 12, 4, True)), - ]) - return cases - - -def z_advise_for_recsize(recsz: int, extra: int = 0) -> int: - shift = {4: 0, 8: 1, 16: 2, 32: 3}[recsz] - return 1 | (shift << 1) | extra - - -def zmap_case( - case_id: str, - category: str, - subcategories: list[str], - request: int, - *, - recsz: int = 16, - records: list[dict[str, int]] | None = None, - base_pa: int = 0, - size: int = 4096, - inode_off: int = 0, - inode_isize: int = 0, - xattr_isize: int = 0, - z_extents: int | None = None, - z_advise_extra: int = 0, - z_lclusterbits: int = 12, - available_compr_algs: int = 1, - packed_size: int = 1 << 20, - packed_nid: int = 999, - reader_error: bool = False, -) -> dict[str, Any]: - records = records or [] - if z_extents is None: - z_extents = len(records) - return { - "id": case_id, - "engine": "zmap", - "category": category, - "subcategories": subcategories, - "request": request, - "recsz": recsz, - "base_pa": base_pa, - "records": records, - "reader_error": reader_error, - "sbi": { - "block_size": 4096, - "blkszbits": 12, - "available_compr_algs": available_compr_algs, - "packed_size": packed_size, - "packed_nid": packed_nid, - }, - "inode": { - "nid": 2, - "size": size, - "datalayout": 1, - "inode_off": inode_off, - "inode_isize": inode_isize, - "xattr_isize": xattr_isize, - "z_advise": z_advise_for_recsize(recsz, z_advise_extra), - "z_lclusterbits": z_lclusterbits, - "z_extents": z_extents, - }, - } - - -def rec(plen: int, pstart: int, lstart: int = 0) -> dict[str, int]: - return {"plen": plen, "pstart": pstart, "lstart": lstart} - - -def make_zmap_cases() -> list[dict[str, Any]]: - compressed_1k = (1 << PLEN_FMT_BIT) | 1024 - compressed_2k = (1 << PLEN_FMT_BIT) | 2048 - ext4 = [rec(compressed_1k, 0), rec(compressed_2k, 0)] - ext8 = [rec(compressed_1k, 0x20000), rec(compressed_2k, 0x30000)] - ext16 = [ - rec(compressed_1k, 0x40000, 0), - rec(compressed_2k, 0x50000, 4096), - rec(compressed_1k, 0x60000, 8192), - ] - cases = [ - zmap_case("extent4-start", "compressed", ["extent-4", "compressed-flags"], 0, recsz=4, records=ext4, base_pa=0x10000, size=8192), - zmap_case("extent4-first-last", "compressed", ["extent-4"], 4095, recsz=4, records=ext4, base_pa=0x10000, size=8192), - zmap_case("extent4-second-start", "compressed", ["extent-4"], 4096, recsz=4, records=ext4, base_pa=0x10000, size=8192), - zmap_case("extent4-second-middle", "compressed", ["extent-4"], 5000, recsz=4, records=ext4, base_pa=0x10000, size=8192), - zmap_case("extent8-start", "compressed", ["extent-8", "compressed-flags"], 0, recsz=8, records=ext8, size=8192), - zmap_case("extent8-second-start", "compressed", ["extent-8"], 4096, recsz=8, records=ext8, size=8192), - zmap_case("extent8-hole", "hole", ["extent-8", "compressed-flags"], 4096, recsz=8, records=[ext8[0], rec(0, 0, 4096)], size=8192), - zmap_case("extent16-start", "compressed", ["extent-16", "compressed-flags"], 0, records=ext16, size=12288), - zmap_case("extent16-first-last", "compressed", ["extent-16"], 4095, records=ext16, size=12288), - zmap_case("extent16-second-start", "compressed", ["extent-16"], 4096, records=ext16, size=12288), - zmap_case("extent16-second-middle", "compressed", ["extent-16"], 7000, records=ext16, size=12288), - zmap_case("extent16-third-start", "compressed", ["extent-16"], 8192, records=ext16, size=12288), - zmap_case("extent16-physical-48bit", "compressed", ["extent-16"], 0, records=[rec(compressed_1k, (1 << 40) + 9, 0)]), - zmap_case("extent32-logical-64bit", "compressed", ["extent-32"], 1 << 32, recsz=32, records=[rec(compressed_1k, 0x70000, 0), rec(compressed_2k, 0x80000, 1 << 32)], size=(1 << 32) + 4096), - zmap_case("extent-partial-reference", "partial-reference", ["extent-16", "compressed-flags"], 0, records=[rec(PLEN_PARTIAL | compressed_1k, 0x90000, 0)]), - zmap_case("extent-fragment-tail", "fragment", ["extent-16", "compressed-flags"], 4096, records=[rec(8192, 0, 4096)], size=6144, z_advise_extra=0x20), - zmap_case("extent-interlaced-shifted", "compressed", ["extent-16", "compressed-flags"], 0, records=[rec(4096, 0xA0000, 0)], z_advise_extra=0x10), - zmap_case("extent-algorithm-one", "compressed", ["extent-16", "compressed-flags"], 0, records=[rec((2 << PLEN_FMT_BIT) | 1024, 0xB0000, 0)], available_compr_algs=3), - zmap_case("extent-algorithm-unavailable", "invalid", ["compressed-invalid", "cleanup-after-acquire"], 0, records=[rec((2 << PLEN_FMT_BIT) | 1024, 0xB0000, 0)], available_compr_algs=1), - zmap_case("extent-algorithm-runtime-invalid", "invalid", ["compressed-invalid", "cleanup-after-acquire"], 0, records=[rec((7 << PLEN_FMT_BIT) | 1024, 0xC0000, 0)]), - zmap_case("extent-plen-too-large", "invalid", ["compressed-invalid", "cleanup-after-acquire"], 0, records=[rec(PCLUSTER_MAX + 1, 0xD0000, 0)]), - zmap_case("extent-full-short-logical", "invalid", ["compressed-invalid", "cleanup-after-acquire"], 0, records=[rec((1 << PLEN_FMT_BIT) | 8192, 0xE0000, 0)]), - zmap_case("extent-pa-add-overflow", "overflow", ["compressed-overflow", "cleanup-after-acquire"], 0, records=[rec(compressed_1k, UINT64_MAX - 511, 0)]), - zmap_case("extent-physical-48bit-limit", "bounds", ["compressed-bounds", "cleanup-after-acquire"], 0, records=[rec(compressed_1k, (1 << 60), 0)]), - zmap_case("extent-reader-short", "bounds", ["compressed-bounds", "cleanup-before-acquire"], 0, records=[rec(compressed_1k, 0xF0000, 0)], reader_error=True), - zmap_case("extent-table-overflow", "overflow", ["compressed-overflow", "cleanup-before-acquire"], 0, records=[rec(compressed_1k, 0x100000, 0)], inode_off=UINT64_MAX - 3), - zmap_case("extent-record-multiply-overflow", "overflow", ["compressed-overflow", "cleanup-before-acquire"], 0, recsz=32, records=[], size=UINT64_MAX, z_extents=UINT64_MAX), - zmap_case("extent-fragment-bounds", "bounds", ["compressed-bounds", "cleanup-after-acquire"], 4096, records=[rec(8192, 0, 4096)], size=6144, z_advise_extra=0x20, packed_size=1024), - zmap_case("compressed-eof", "post-EOF", ["extent-16"], 4096, records=[rec(compressed_1k, 0x110000, 0)]), - zmap_case("compressed-post-eof", "post-EOF", ["extent-16"], 4113, records=[rec(compressed_1k, 0x110000, 0)]), - ] - return cases - - -def make_device_cases() -> list[dict[str, Any]]: - def device( - case_id: str, - *, - pa: int, - plen: int, - deviceid: int, - primary_blocks: int = 1024, - extra: list[dict[str, Any]] | None = None, - flatdev: bool = False, - devs_present: bool = True, - ) -> dict[str, Any]: - return { - "id": case_id, - "subcategories": ["device-resolution"], - "input": {"pa": pa, "plen": plen, "deviceid": deviceid}, - "sbi": { - "blkszbits": 12, - "primary_blocks": primary_blocks, - "extra": extra or [], - "flatdev": flatdev, - "devs_present": devs_present, - }, - } - - good = lambda blocks, uniaddr=0: { - "blocks": blocks, "uniaddr": uniaddr, "provider": True - } - missing = lambda blocks, uniaddr=0: { - "blocks": blocks, "uniaddr": uniaddr, "provider": False - } - return [ - device("device-primary", pa=4096, plen=512, deviceid=0), - device("device-explicit-one", pa=4096, plen=512, deviceid=1, extra=[good(32)]), - device("device-explicit-two", pa=8192, plen=1024, deviceid=2, extra=[good(32), good(64)]), - device("device-id-out-of-range", pa=0, plen=1, deviceid=3, extra=[good(32), good(32)]), - device("device-table-missing", pa=0, plen=1, deviceid=1, extra=[good(32)], devs_present=False), - device("device-explicit-bounds", pa=4096 * 32 - 128, plen=256, deviceid=1, extra=[good(32)]), - device("device-explicit-provider-missing", pa=0, plen=1, deviceid=1, extra=[missing(32)]), - device("device-flat-explicit", pa=512, plen=512, deviceid=1, extra=[good(32, 100)], flatdev=True), - device("device-flat-add-overflow", pa=UINT64_MAX - 1024, plen=512, deviceid=1, extra=[good(32, 1)], flatdev=True), - device("device-flat-primary", pa=4096, plen=512, deviceid=0, primary_blocks=32, extra=[good(32, 100)], flatdev=True), - device("device-flat-implicit-extra", pa=100 * 4096 + 512, plen=512, deviceid=0, primary_blocks=32, extra=[good(32, 100)], flatdev=True), - device("device-separate-implicit-extra", pa=100 * 4096 + 512, plen=512, deviceid=0, primary_blocks=32, extra=[good(32, 100)]), - device("device-blocks-shift-overflow", pa=0, plen=1, deviceid=1, extra=[good((UINT64_MAX >> 12) + 1)]), - ] - - -def add_u64(left: int, right: int) -> tuple[bool, int]: - value = left + right - return (value > UINT64_MAX, value & UINT64_MAX) - - -def roundup_u64(value: int, alignment: int) -> tuple[bool, int]: - overflow, rounded = add_u64(value, alignment - 1) - return overflow, rounded & ~(alignment - 1) - - -def model_data(case: dict[str, Any]) -> dict[str, int]: - request = case["request"] - inode = case["inode"] - sbi = case["sbi"] - result = tuple_record(m_la=request) - if request >= inode["size"]: - return result - remain = inode["size"] - request - layout = inode["datalayout"] - block_size = sbi["block_size"] - if layout == 0: - run_len = min(remain, block_size - (request & (block_size - 1))) - result["m_llen"] = result["m_plen"] = run_len - if inode["startblk"] == NULL_ADDR: - return result - if inode["startblk"] > (UINT64_MAX >> sbi["blkszbits"]): - result["errno"] = EINTEGRITY - return result - overflow, pa = add_u64(inode["startblk"] << sbi["blkszbits"], request) - if overflow: - result["errno"] = EINTEGRITY - return result - result["m_pa"] = pa - result["m_flags"] = MAPPED - return result - if layout == 2: - tail_start = 0 if inode["size"] == 0 else ( - (inode["size"] + block_size - 1) & ~(block_size - 1) - ) - block_size - if request < tail_start: - run_len = min( - remain, - tail_start - request, - block_size - (request & (block_size - 1)), - ) - result["m_llen"] = result["m_plen"] = run_len - if inode["startblk"] == NULL_ADDR: - return result - if inode["startblk"] > (UINT64_MAX >> sbi["blkszbits"]): - result["errno"] = EINTEGRITY - return result - overflow, pa = add_u64( - inode["startblk"] << sbi["blkszbits"], request - ) - if overflow: - result["errno"] = EINTEGRITY - return result - result["m_pa"] = pa - result["m_flags"] = MAPPED - return result - run_len = min( - remain, block_size - ((request - tail_start) & (block_size - 1)) - ) - result["m_llen"] = result["m_plen"] = run_len - overflow, pa = add_u64(inode["inode_off"], inode["inode_isize"]) - if not overflow: - overflow, pa = add_u64(pa, inode["xattr_isize"]) - if not overflow: - overflow, pa = add_u64(pa, request - tail_start) - if overflow: - result["m_pa"] = pa - result["errno"] = EINTEGRITY - return result - result["m_pa"] = pa - result["m_flags"] = MAPPED | META - return result - if layout != 4: - result["errno"] = EOPNOTSUPP - return result - - chunk_size = 1 << inode["chunkbits"] - chunk_index = request >> inode["chunkbits"] - chunk_offset = request & (chunk_size - 1) - entry_size = 8 if inode["chunkformat"] & 0x20 else 4 - overflow, index_base = add_u64(inode["inode_off"], inode["inode_isize"]) - if overflow: - result["errno"] = EOVERFLOW - return result - overflow, index_base = add_u64(index_base, inode["xattr_isize"]) - if overflow: - result["errno"] = EOVERFLOW - return result - overflow, index_base = roundup_u64(index_base, entry_size) - if overflow: - result["errno"] = EOVERFLOW - return result - if chunk_index > (UINT64_MAX - index_base) // entry_size: - result["errno"] = EOVERFLOW - return result - index_offset = index_base + chunk_index * entry_size - if inode["nid"] & META_NID: - if ( - not sbi["metabox_present"] - or index_offset > sbi["metabox_size"] - or entry_size > sbi["metabox_size"] - index_offset - ): - result["errno"] = EINTEGRITY - return result - else: - if sbi["blocks"] > (UINT64_MAX >> sbi["blkszbits"]): - result["errno"] = EOVERFLOW - return result - image_size = sbi["blocks"] << sbi["blkszbits"] - if index_offset > image_size or entry_size > image_size - index_offset: - result["errno"] = EINTEGRITY - return result - reader = case["reader"] - raw = reader["bytes"] - if ( - reader["error"] - or index_offset < reader["base"] - or entry_size > len(raw) - or index_offset - reader["base"] > len(raw) - entry_size - ): - result["errno"] = EIO - return result - entry = raw[index_offset - reader["base"] : index_offset - reader["base"] + entry_size] - result["acquire_count"] = result["release_count"] = 1 - if inode["chunkformat"] & 0x20: - high, raw_device, low = struct.unpack(" (UINT64_MAX >> sbi["blkszbits"]): - result["errno"] = EOVERFLOW - return result - overflow, pa = add_u64(block << sbi["blkszbits"], chunk_offset) - if overflow: - result["errno"] = EOVERFLOW - return result - result["m_pa"] = pa - result["m_flags"] = MAPPED - return result - - -def extent_table_pos(inode: dict[str, int], recsz: int) -> tuple[int, int]: - overflow, pos = add_u64(inode["inode_off"], inode["inode_isize"]) - if not overflow: - overflow, pos = add_u64(pos, inode["xattr_isize"]) - if not overflow: - overflow, pos = roundup_u64(pos, 8) - if not overflow: - overflow, pos = add_u64(pos, 8) - if not overflow: - overflow, pos = roundup_u64(pos, recsz) - return (EINTEGRITY if overflow else 0, pos) - - -def record_pos( - inode: dict[str, int], table_pos: int, recsz: int, index: int -) -> tuple[int, int]: - if index >= inode["z_extents"] or index > UINT64_MAX // recsz: - return EINTEGRITY, 0 - overflow, pos = add_u64(table_pos, index * recsz) - return (EINTEGRITY if overflow else 0, pos) - - -def read_extent( - case: dict[str, Any], table_pos: int, recsz: int, index: int, counters: dict[str, int] -) -> tuple[int, dict[str, int] | None]: - error, _ = record_pos(case["inode"], table_pos, recsz, index) - if error: - return error, None - if case["reader_error"] or index >= len(case["records"]): - return EIO, None - counters["acquire"] += 1 - counters["release"] += 1 - return 0, case["records"][index] - - -def model_zmap(case: dict[str, Any]) -> dict[str, int]: - request = case["request"] - inode = case["inode"] - sbi = case["sbi"] - result = tuple_record(m_la=request) - if request >= inode["size"]: - result["m_llen"] = request + 1 - inode["size"] - result["m_la"] = inode["size"] - return result - recsz = case["recsz"] - error, table_pos = extent_table_pos(inode, recsz) - if error: - result["errno"] = error - return result - pos = table_pos - logical_end = inode["size"] - cluster_size = 1 << inode["z_lclusterbits"] - counters = {"acquire": 0, "release": 0} - result["m_flags"] = 0 - last = False - lstart = logical_end - if recsz <= 8: - if recsz <= 4: - if case["reader_error"]: - result["errno"] = EIO - return result - counters["acquire"] += 1 - counters["release"] += 1 - pa = case["base_pa"] - overflow, pos = add_u64(pos, 8) - if overflow: - result["errno"] = EINTEGRITY - return result - lstart = 0 - extent_index = 0 - else: - lstart = request & ~(cluster_size - 1) - extent_index = lstart >> inode["z_lclusterbits"] - pa = NULL_ADDR - while True: - error, extent = read_extent(case, pos, recsz, extent_index, counters) - if error: - result["errno"] = error - result["acquire_count"] = counters["acquire"] - result["release_count"] = counters["release"] - return result - assert extent is not None - result["m_plen"] = extent["plen"] - if pa != NULL_ADDR: - result["m_pa"] = pa - overflow, pa = add_u64(pa, result["m_plen"] & PLEN_MASK) - if overflow: - result["errno"] = EINTEGRITY - result["acquire_count"] = counters["acquire"] - result["release_count"] = counters["release"] - return result - else: - result["m_pa"] = extent["pstart"] & UINT32_MAX - if extent_index == UINT64_MAX: - result["errno"] = EINTEGRITY - result["acquire_count"] = counters["acquire"] - result["release_count"] = counters["release"] - return result - extent_index += 1 - overflow, next_lstart = add_u64(lstart, cluster_size) - if overflow: - result["errno"] = EINTEGRITY - result["acquire_count"] = counters["acquire"] - result["release_count"] = counters["release"] - return result - lstart = next_lstart - if lstart > request: - break - overflow, rounded_end = roundup_u64(logical_end, cluster_size) - if overflow: - result["errno"] = EINTEGRITY - result["acquire_count"] = counters["acquire"] - result["release_count"] = counters["release"] - return result - last = lstart >= rounded_end - logical_end = min(lstart, logical_end) - lstart -= cluster_size - else: - left, right = 0, inode["z_extents"] - while left < right: - middle = left + (right - left) // 2 - error, extent = read_extent(case, table_pos, recsz, middle, counters) - if error: - result["errno"] = error - result["acquire_count"] = counters["acquire"] - result["release_count"] = counters["release"] - return result - assert extent is not None - logical = extent["lstart"] & (UINT64_MAX if recsz == 32 else UINT32_MAX) - physical = extent["pstart"] - if logical > request: - right = middle - if logical > logical_end: - result["errno"] = EINTEGRITY - result["acquire_count"] = counters["acquire"] - result["release_count"] = counters["release"] - return result - logical_end = logical - else: - left = middle + 1 - if request == logical: - right = min(left + 1, right) - lstart = logical - result["m_plen"] = extent["plen"] - result["m_pa"] = physical - last = left >= inode["z_extents"] - - if lstart < logical_end: - result["m_la"] = lstart - if last and inode["z_advise"] & 0x20: - result["m_flags"] = FRAGMENT - fragment_offset = result["m_plen"] - if recsz > 4: - fragment_offset |= result["m_pa"] << 32 - elif result["m_plen"] & PLEN_MASK: - result["m_flags"] = MAPPED - fmt = result["m_plen"] >> PLEN_FMT_BIT - if result["m_plen"] & PLEN_PARTIAL: - result["m_flags"] |= PARTIAL_REF - result["m_plen"] &= PLEN_MASK - if fmt: - result["m_algorithmformat"] = fmt - 1 - elif ( - inode["z_advise"] & 0x10 - and ((result["m_pa"] | result["m_plen"]) & (sbi["block_size"] - 1)) == 0 - ): - result["m_algorithmformat"] = COMPRESSION_INTERLACED - else: - result["m_algorithmformat"] = COMPRESSION_SHIFTED - result["m_llen"] = logical_end - result["m_la"] - result["acquire_count"] = counters["acquire"] - result["release_count"] = counters["release"] - - sanity_error = 0 - if result["m_flags"] & FRAGMENT: - if ( - result["m_flags"] & (MAPPED | META) - or sbi["packed_nid"] == inode["nid"] - or fragment_offset > sbi["packed_size"] - or result["m_llen"] > sbi["packed_size"] - fragment_offset - ): - sanity_error = EINTEGRITY - elif result["m_flags"] & MAPPED: - algorithm = result["m_algorithmformat"] & 0xFF - if algorithm >= COMPRESSION_RUNTIME_MAX: - sanity_error = EOPNOTSUPP - elif algorithm < COMPRESSION_MAX: - if not (sbi["available_compr_algs"] & (1 << algorithm)): - sanity_error = EINTEGRITY - elif not (result["m_flags"] & (PARTIAL_MAPPED | PARTIAL_REF)) and result["m_llen"] < result["m_plen"]: - sanity_error = EINTEGRITY - elif result["m_llen"] > result["m_plen"]: - sanity_error = EINTEGRITY - if not sanity_error and ( - result["m_plen"] > PCLUSTER_MAX - or result["m_llen"] > PCLUSTER_MAX_DSIZE - ): - sanity_error = EOPNOTSUPP - if not sanity_error and not (result["m_flags"] & META): - overflow, physical_end = add_u64(result["m_pa"], result["m_plen"]) - if overflow or (physical_end >> sbi["blkszbits"]) >= (1 << 48): - sanity_error = EINTEGRITY - if sanity_error: - result["errno"] = sanity_error - result["m_llen"] = 0 - return result - - -def model_device(case: dict[str, Any]) -> dict[str, int]: - state = case["sbi"] - map_state = dict(case["input"]) - extras = state["extra"] if state["devs_present"] else None - selected = 0 - - def check_range(device: dict[str, Any], offset: int, length: int) -> int: - if device["blocks"] > (UINT64_MAX >> state["blkszbits"]): - return EINTEGRITY - overflow, end = add_u64(offset, length) - if overflow: - return EINTEGRITY - return EINTEGRITY if end > device["blocks"] << state["blkszbits"] else 0 - - primary = {"blocks": state["primary_blocks"], "uniaddr": 0, "provider": True} - device_id = map_state["deviceid"] - if device_id: - if extras is None or device_id > len(extras): - error = ENODEV - else: - chosen = extras[device_id - 1] - error = check_range(chosen, map_state["pa"], map_state["plen"]) - if not error and state["flatdev"]: - if chosen["uniaddr"] > (UINT64_MAX >> state["blkszbits"]): - error = EINTEGRITY - else: - overflow, mapped = add_u64( - map_state["pa"], chosen["uniaddr"] << state["blkszbits"] - ) - if overflow: - error = EINTEGRITY - else: - map_state["pa"] = mapped - elif not error: - if not chosen["provider"]: - error = ENODEV - else: - selected = device_id - elif not state["extra"]: - error = 0 - elif state["flatdev"]: - error = check_range(primary, map_state["pa"], map_state["plen"]) - if error: - for chosen in state["extra"]: - if not chosen["uniaddr"] or chosen["uniaddr"] > (UINT64_MAX >> state["blkszbits"]): - continue - start = chosen["uniaddr"] << state["blkszbits"] - if map_state["pa"] < start: - continue - relative = map_state["pa"] - start - error = check_range(chosen, relative, map_state["plen"]) - if not error: - break - if relative < chosen["blocks"] << state["blkszbits"]: - break - else: - error = EINTEGRITY - else: - error = 0 - for index, chosen in enumerate(state["extra"], 1): - if not chosen["uniaddr"]: - continue - if chosen["uniaddr"] > (UINT64_MAX >> state["blkszbits"]): - error = EINTEGRITY - break - start = chosen["uniaddr"] << state["blkszbits"] - if map_state["pa"] >= start and map_state["pa"] - start < chosen["blocks"] << state["blkszbits"]: - relative = map_state["pa"] - start - error = check_range(chosen, relative, map_state["plen"]) - if error: - break - if not chosen["provider"]: - error = ENODEV - break - map_state["pa"] = relative - selected = index - break - return { - "m_pa": map_state["pa"], - "m_plen": map_state["plen"], - "m_deviceid": map_state["deviceid"], - "selected_device": selected, - "errno": error, - } - - -def extent_bytes(case: dict[str, Any]) -> tuple[int, bytes]: - error, table_pos = extent_table_pos(case["inode"], case["recsz"]) - if error: - return 0, b"" - output = bytearray() - if case["recsz"] == 4: - output.extend(struct.pack("> 32) & UINT32_MAX, - extent["lstart"] & UINT32_MAX, - (extent["lstart"] >> 32) & UINT32_MAX, - b"\0" * 12, - ) - output.extend(packed[: case["recsz"]]) - return table_pos, bytes(output) - - -MAP_CASES = make_data_cases() + make_zmap_cases() -DEVICE_CASES = make_device_cases() -MODEL_RECORDS = [] -for case in MAP_CASES: - expected = model_data(case) if case["engine"] == "data" else model_zmap(case) - if case["engine"] == "data": - fixture = case["reader"]["bytes"] - else: - _, fixture = extent_bytes(case) - MODEL_RECORDS.append( - { - "id": case["id"], - "engine": case["engine"], - "category": case["category"], - "subcategories": case["subcategories"], - "fixture_sha256": sha256_bytes(fixture), - "expected": expected, - } - ) -MODEL_DEVICES = [ - {"id": case["id"], "expected": model_device(case)} for case in DEVICE_CASES -] -MODEL_SHA256 = sha256_bytes(canonical({"maps": MODEL_RECORDS, "devices": MODEL_DEVICES})) -if SPEC["expected_map_case_count"] != len(MAP_CASES): - raise SystemExit( - f"map case count is not frozen: actual={len(MAP_CASES)} " - f"configured={SPEC['expected_map_case_count']}" - ) -if SPEC["expected_device_case_count"] != len(DEVICE_CASES): - raise SystemExit("device case count is not frozen") -if SPEC["expected_model_sha256"] != MODEL_SHA256: - raise SystemExit( - f"independent model digest mismatch: actual={MODEL_SHA256} " - f"configured={SPEC['expected_model_sha256']}" - ) - - -COMMON_C = r''' -#include -#include -#include -#include -#include -#include - -#define EIO 5 -#define ENODEV 19 -#define EINVAL 22 -#define EOPNOTSUPP 45 -#define EOVERFLOW 84 -#define EINTEGRITY 97 -#define UINT64_MAX_VALUE UINT64_MAX -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define MAX(a, b) ((a) > (b) ? (a) : (b)) -#define roundup2(x, y) (((x) + ((y) - 1)) & ~((y) - 1)) -#define rounddown2(x, y) ((x) & ~((y) - 1)) -#define bzero(ptr, len) memset((ptr), 0, (len)) -#define le16toh(value) (value) -#define le32toh(value) (value) - -typedef uint64_t erofs_off_t; -typedef uint64_t erofs_blk_t; -typedef uint64_t erofs_nid_t; - -static uint32_t le32dec(const void *pointer) -{ - const unsigned char *bytes = pointer; - return ((uint32_t)bytes[0] | (uint32_t)bytes[1] << 8 | - (uint32_t)bytes[2] << 16 | (uint32_t)bytes[3] << 24); -} - -static uint64_t le64dec(const void *pointer) -{ - const unsigned char *bytes = pointer; - return ((uint64_t)le32dec(bytes) | - (uint64_t)le32dec(bytes + 4) << 32); -} - -#define EROFS_INODE_FLAT_PLAIN 0 -#define EROFS_INODE_COMPRESSED_FULL 1 -#define EROFS_INODE_FLAT_INLINE 2 -#define EROFS_INODE_COMPRESSED_COMPACT 3 -#define EROFS_INODE_CHUNK_BASED 4 -#define EROFS_CHUNK_FORMAT_INDEXES 0x20 -#define EROFS_CHUNK_FORMAT_48BIT 0x40 -#define EROFS_BLOCK_MAP_ENTRY_SIZE 4 -#define EROFS_DIRENT_NID_METABOX (1ULL << 63) -#define EROFS_NULL_ADDR UINT64_MAX -#define EROFS_MAP_MAPPED 0x0001 -#define EROFS_MAP_META 0x0002 -#define EROFS_MAP_PARTIAL_MAPPED 0x0004 -#define EROFS_MAP_PARTIAL_REF 0x0008 -#define EROFS_MAP_FRAGMENT 0x0010 -#define EROFS_MAP_FULL(f) (!((f) & (EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF))) -#define EROFS_GET_BLOCKS_FIEMAP 0x0001 -#define EROFS_GET_BLOCKS_READMORE 0x0002 -#define EROFS_GET_BLOCKS_FINDTAIL 0x0004 -#define Z_EROFS_COMPRESSION_LZ4 0 -#define Z_EROFS_COMPRESSION_LZMA 1 -#define Z_EROFS_COMPRESSION_DEFLATE 2 -#define Z_EROFS_COMPRESSION_ZSTD 3 -#define Z_EROFS_COMPRESSION_MAX 4 -#define Z_EROFS_COMPRESSION_SHIFTED 4 -#define Z_EROFS_COMPRESSION_INTERLACED 5 -#define Z_EROFS_COMPRESSION_RUNTIME_MAX 6 -#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) -#define Z_EROFS_PCLUSTER_MAX_DSIZE (12 * 1024 * 1024) -#define Z_EROFS_ADVISE_EXTENTS 0x0001 -#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER 0x0010 -#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER 0x0020 -#define Z_EROFS_ADVISE_EXTRECSZ_BIT 1 -#define Z_EROFS_ADVISE_EXTRECSZ_MASK 0x3 -#define Z_EROFS_EXTENT_PLEN_PARTIAL (1U << 27) -#define Z_EROFS_EXTENT_PLEN_FMT_BIT 28 -#define Z_EROFS_EXTENT_PLEN_MASK ((Z_EROFS_PCLUSTER_MAX_SIZE << 1) - 1) - -struct erofs_inode_chunk_index { - uint16_t startblk_hi; - uint16_t device_id; - uint32_t startblk_lo; -} __attribute__((packed)); - -struct z_erofs_extent { - uint32_t plen, pstart_lo, pstart_hi, lstart_lo, lstart_hi; - uint8_t reserved[12]; -} __attribute__((packed)); - -struct z_erofs_map_header { - uint32_t word0; - uint16_t h_advise; - uint8_t h_algorithmtype; - uint8_t h_clusterbits; -} __attribute__((packed)); - -struct erofs_device_info { - void *devvp; - void *cp; - erofs_blk_t blocks; - erofs_blk_t uniaddr; -}; - -struct erofs_map_blocks { - erofs_off_t m_pa, m_la; - uint64_t m_plen, m_llen; - unsigned short m_deviceid; - char m_algorithmformat; - unsigned int m_flags; -}; - -struct erofs_inode { - erofs_nid_t nid; - uint64_t size; - erofs_off_t inode_off; - erofs_blk_t startblk; - uint8_t datalayout; - uint8_t inode_isize; - uint32_t xattr_isize; - uint16_t z_advise; - uint8_t z_algorithmtype[2]; - uint8_t z_lclusterbits; - uint16_t z_idata_size; - erofs_off_t z_fragmentoff; - uint64_t z_tailextent_headlcn; - uint64_t z_extents; - bool z_initialized; - uint16_t chunkformat; - uint8_t chunkbits; - bool fragment; -}; - -struct erofs_sb_info { - uint64_t block_size; - uint8_t blkszbits; - erofs_blk_t blocks; - uint16_t device_id_mask; - struct erofs_inode *metabox_en; - uint16_t available_compr_algs; - uint64_t packed_nid; - struct erofs_inode *packed_inode; - struct erofs_device_info dif0; - struct erofs_device_info *devs; - unsigned int extra_devices; - bool flatdev; -}; - -struct erofs_map_dev { - struct erofs_device_info *m_dif; - erofs_off_t m_pa; - unsigned int m_deviceid; - uint64_t m_plen; -}; - -struct erofs_buf { - void *data; - void (*release)(void *); -}; -#define EROFS_BUF_INITIALIZER { .data = NULL, .release = NULL } - -static const unsigned char *gate_bytes; -static size_t gate_bytes_len; -static erofs_off_t gate_bytes_base; -static bool gate_reader_error; -static unsigned int gate_acquires; -static unsigned int gate_releases; - -static bool erofs_nid_in_metabox(erofs_nid_t nid) -{ - return ((nid & EROFS_DIRENT_NID_METABOX) != 0); -} - -static bool __attribute__((unused)) -erofs_inode_is_data_compressed(unsigned int datalayout) -{ - return (datalayout == EROFS_INODE_COMPRESSED_FULL || - datalayout == EROFS_INODE_COMPRESSED_COMPACT); -} - -static int erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid, - erofs_off_t off, size_t len, struct erofs_buf *buf) -{ - void *data; - (void)sbi; - (void)nid; - if (gate_reader_error || off < gate_bytes_base || - off - gate_bytes_base > gate_bytes_len || - len > gate_bytes_len - (size_t)(off - gate_bytes_base)) - return (EIO); - data = malloc(len == 0 ? 1 : len); - if (data == NULL) - return (EIO); - if (len != 0) - memcpy(data, gate_bytes + (off - gate_bytes_base), len); - buf->data = data; - buf->release = NULL; - ++gate_acquires; - return (0); -} - -static void erofs_brelse(void *data) -{ - if (data != NULL) { - ++gate_releases; - free(data); - } -} - -static void erofs_put_metabuf(struct erofs_buf *buf) -{ - if (buf != NULL && buf->data != NULL) { - void *data = buf->data; - buf->data = NULL; - buf->release = NULL; - erofs_brelse(data); - } -} - -static unsigned int z_erofs_extent_recsize(unsigned int advise) -{ - return (4U << ((advise >> Z_EROFS_ADVISE_EXTRECSZ_BIT) & - Z_EROFS_ADVISE_EXTRECSZ_MASK)); -} - -static int z_erofs_fill_inode(struct erofs_sb_info *sbi, - struct erofs_inode *vi) -{ - (void)sbi; - return (vi->z_initialized ? 0 : EINTEGRITY); -} - -static int z_erofs_map_blocks_fo(struct erofs_sb_info *sbi, - struct erofs_inode *vi, struct erofs_map_blocks *map, int flags) -{ - (void)sbi; - (void)vi; - (void)map; - (void)flags; - return (EOPNOTSUPP); -} -''' - - -def c_bytes(data: bytes) -> str: - return ", ".join(f"0x{byte:02x}" for byte in data) if data else "0" - - -def c_u64(value: int) -> str: - return f"UINT64_C({value})" - - -zmap_names = [ - "z_erofs_extent_add", - "z_erofs_extent_roundup", - "z_erofs_extent_table_pos", - "z_erofs_extent_record_pos", - "z_erofs_read_extent", - "z_erofs_extent_lstart", - "z_erofs_map_blocks_ext", - "z_erofs_map_sanity_check", - "z_erofs_map_blocks_iter", -] -program = COMMON_C -program += "int z_erofs_map_blocks_iter(struct erofs_sb_info *, struct erofs_inode *, struct erofs_map_blocks *, int);\n" -program += extract_function(data_source, "erofs_inline_tail_start") -program += extract_function(data_source, "erofs_map_blocks_chunk") -program += legacy_body -if candidate_mode: - program += extract_function(data_source, "erofs_map_blocks") -for name in zmap_names: - program += extract_function(zmap_source, name) -program += "\nint\nmain(void)\n{\n\tint error;\n" - -for index, case in enumerate(MAP_CASES): - if case["engine"] == "data": - raw = case["reader"]["bytes"] - reader_base = case["reader"]["base"] - reader_error = case["reader"]["error"] - inode = case["inode"] - sbi = case["sbi"] - else: - reader_base, raw = extent_bytes(case) - reader_error = case["reader_error"] - inode = case["inode"] - sbi = { - "block_size": case["sbi"]["block_size"], - "blkszbits": case["sbi"]["blkszbits"], - "blocks": 0, - "device_id_mask": 0xFFFF, - "metabox_present": False, - "metabox_size": 0, - } - packed_size = case.get("sbi", {}).get("packed_size", 1 << 20) - packed_nid = case.get("sbi", {}).get("packed_nid", 999) - available = case.get("sbi", {}).get("available_compr_algs", 1) - program += f''' - {{ - static const unsigned char bytes[] = {{ {c_bytes(raw)} }}; - struct erofs_inode metabox = {{ .size = {c_u64(sbi['metabox_size'])} }}; - struct erofs_inode packed = {{ .nid = {c_u64(packed_nid)}, .size = {c_u64(packed_size)} }}; - struct erofs_sb_info sbi = {{ - .block_size = {c_u64(sbi['block_size'])}, .blkszbits = {sbi['blkszbits']}, - .blocks = {c_u64(sbi['blocks'])}, .device_id_mask = {sbi['device_id_mask']}, - .metabox_en = {('&metabox' if sbi['metabox_present'] else 'NULL')}, - .available_compr_algs = {available}, .packed_nid = {c_u64(packed_nid)}, - .packed_inode = &packed, - }}; - struct erofs_inode vi = {{ - .nid = {c_u64(inode['nid'])}, .size = {c_u64(inode['size'])}, - .inode_off = {c_u64(inode['inode_off'])}, .startblk = {c_u64(inode.get('startblk', 0))}, - .datalayout = {inode['datalayout']}, .inode_isize = {inode['inode_isize']}, - .xattr_isize = {inode['xattr_isize']}, .z_advise = {inode.get('z_advise', 0)}, - .z_lclusterbits = {inode.get('z_lclusterbits', 12)}, - .z_extents = {c_u64(inode.get('z_extents', 0))}, .z_initialized = true, - .chunkformat = {inode.get('chunkformat', 0)}, .chunkbits = {inode.get('chunkbits', 12)}, - }}; - struct erofs_map_blocks map = {{ - .m_pa = UINT64_C(0xcccccccccccccccc), .m_la = {c_u64(case['request'])}, - .m_plen = UINT64_C(0xdddddddddddddddd), .m_llen = UINT64_C(0xeeeeeeeeeeeeeeee), - .m_deviceid = 0xaaaa, .m_algorithmformat = 0x55, .m_flags = 0xbbbbbbbb, - }}; - (void)metabox; - gate_bytes = bytes; gate_bytes_len = {len(raw)}; - gate_bytes_base = {c_u64(reader_base)}; gate_reader_error = {'true' if reader_error else 'false'}; - gate_acquires = gate_releases = 0; -''' - if candidate_mode: - program += "\t\terror = erofs_map_blocks(&sbi, &vi, &map);\n" - elif case["engine"] == "data": - program += f''' - {{ - erofs_off_t pa = 0; unsigned int device = 0; size_t run = 0; - bool hole = false, metadata = false; - error = erofs_map_blocks(&sbi, &vi, {c_u64(case['request'])}, - &pa, &device, &run, &hole, &metadata); - map = (struct erofs_map_blocks){{ - .m_pa = pa, .m_la = {c_u64(case['request'])}, .m_plen = run, - .m_llen = run, .m_deviceid = device, - .m_flags = (error == 0 && run != 0 && !hole ? EROFS_MAP_MAPPED : 0) | - (metadata ? EROFS_MAP_META : 0), - }}; - }} -''' - else: - program += "\t\tmap = (struct erofs_map_blocks){ .m_la = map.m_la };\n" - program += "\t\terror = z_erofs_map_blocks_iter(&sbi, &vi, &map, 0);\n" - program += f''' - printf("tuple\\t{case['id']}\\t%llu\\t%llu\\t%llu\\t%llu\\t%u\\t%u\\t%d\\t%d\\t%u\\t%u\\n", - (unsigned long long)map.m_la, (unsigned long long)map.m_pa, - (unsigned long long)map.m_llen, (unsigned long long)map.m_plen, - map.m_deviceid, map.m_flags, (int)map.m_algorithmformat, error, - gate_acquires, gate_releases); - }} -''' -program += "\treturn (0);\n}\n" - - -device_program = COMMON_C -for name in ["erofs_check_device_range", "erofs_fill_from_devinfo", "erofs_map_dev"]: - device_program += extract_function(data_source, name) -device_program += "\nint\nmain(void)\n{\n" -for case in DEVICE_CASES: - state = case["sbi"] - extras = state["extra"] - entries = [] - for extra in extras: - entries.append( - "{ .devvp = %s, .cp = %s, .blocks = %s, .uniaddr = %s }" - % ( - "(void *)1" if extra["provider"] else "NULL", - "(void *)1" if extra["provider"] else "NULL", - c_u64(extra["blocks"]), - c_u64(extra["uniaddr"]), - ) - ) - device_program += f''' - {{ - struct erofs_device_info devs[{max(1, len(extras))}] = {{ {', '.join(entries) if entries else '{ 0 }'} }}; - struct erofs_sb_info sbi = {{ - .blkszbits = {state['blkszbits']}, .blocks = {c_u64(state['primary_blocks'])}, - .dif0 = {{ .devvp = (void *)1, .cp = (void *)1, .blocks = {c_u64(state['primary_blocks'])} }}, - .devs = {('devs' if state['devs_present'] else 'NULL')}, - .extra_devices = {len(extras)}, .flatdev = {'true' if state['flatdev'] else 'false'}, - }}; - struct erofs_map_dev map = {{ .m_pa = {c_u64(case['input']['pa'])}, - .m_plen = {c_u64(case['input']['plen'])}, .m_deviceid = {case['input']['deviceid']} }}; - int selected = -1; - int error = erofs_map_dev(&sbi, &map); - if (map.m_dif == &sbi.dif0) - selected = 0; -''' - for index in range(len(extras)): - device_program += f"\t\telse if (map.m_dif == &devs[{index}])\n\t\t\tselected = {index + 1};\n" - device_program += f''' - printf("device\\t{case['id']}\\t%llu\\t%llu\\t%u\\t%d\\t%d\\n", - (unsigned long long)map.m_pa, (unsigned long long)map.m_plen, - map.m_deviceid, selected, error); - }} -''' -device_program += "\treturn (0);\n}\n" - -adapter_program = "" -expected_adapter_lines = [ - "adapter\t1\t65\t74566\t4097\t2049\t48879\t31\t5\t0\t1", - "adapter\t3\t67\t74568\t4099\t2051\t48879\t31\t5\t0\t1", -] -if candidate_mode: - adapter_program = COMMON_C + r''' -static unsigned int gate_adapter_calls; -int z_erofs_map_blocks_iter(struct erofs_sb_info *sbi, - struct erofs_inode *vi, struct erofs_map_blocks *map, int flags) -{ - (void)sbi; - if (flags != 0) - return (EINVAL); - ++gate_adapter_calls; - map->m_la = 64 + vi->datalayout; - map->m_pa = 74565 + vi->datalayout; - map->m_llen = 4096 + vi->datalayout; - map->m_plen = 2048 + vi->datalayout; - map->m_deviceid = 0xbeef; - map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; - map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_META | - EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF | - EROFS_MAP_FRAGMENT; - return (0); -} -''' - adapter_program += extract_function(data_source, "erofs_inline_tail_start") - adapter_program += extract_function(data_source, "erofs_map_blocks_chunk") - adapter_program += legacy_body - adapter_program += extract_function(data_source, "erofs_map_blocks") - adapter_program += r''' -int main(void) -{ - const unsigned int layouts[] = { - EROFS_INODE_COMPRESSED_FULL, - EROFS_INODE_COMPRESSED_COMPACT, - }; - struct erofs_sb_info sbi = { .block_size = 4096, .blkszbits = 12 }; - unsigned int index; - - for (index = 0; index < sizeof(layouts) / sizeof(layouts[0]); ++index) { - struct erofs_inode vi = { .datalayout = layouts[index], .size = 8192 }; - struct erofs_map_blocks map = { - .m_la = 123, .m_pa = UINT64_MAX, .m_llen = UINT64_MAX, - .m_plen = UINT64_MAX, .m_deviceid = 0xffff, - .m_algorithmformat = -1, .m_flags = UINT32_MAX, - }; - int error; - - gate_adapter_calls = 0; - error = erofs_map_blocks(&sbi, &vi, &map); - printf("adapter\t%u\t%llu\t%llu\t%llu\t%llu\t%u\t%u\t%d\t%d\t%u\n", - layouts[index], (unsigned long long)map.m_la, - (unsigned long long)map.m_pa, (unsigned long long)map.m_llen, - (unsigned long long)map.m_plen, map.m_deviceid, map.m_flags, - (int)map.m_algorithmformat, error, gate_adapter_calls); - } - return (0); -} -''' - - -def compile_and_run(name: str, source: str) -> list[str]: - source_path = OUTPUT / f"{name}.c" - binary_path = OUTPUT / name - source_path.write_text(source, encoding="ascii") - compile_flags = ["cc", "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror"] - if name in {"P15-006-device-extractor", "P15-006-adapter-extractor"}: - compile_flags.append("-Wno-unused-function") - compile_run = run( - compile_flags + ["-o", str(binary_path), str(source_path)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - (OUTPUT / f"{name}.compile.stdout").write_text(compile_run.stdout, encoding="utf-8") - (OUTPUT / f"{name}.compile.stderr").write_text(compile_run.stderr, encoding="utf-8") - if compile_run.returncode != 0: - raise SystemExit(f"{name} compilation failed") - execute = run([str(binary_path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (OUTPUT / f"{name}.stdout").write_text(execute.stdout, encoding="utf-8") - (OUTPUT / f"{name}.stderr").write_text(execute.stderr, encoding="utf-8") - if execute.returncode != 0: - raise SystemExit(f"{name} execution failed") - return execute.stdout.splitlines() - - -actual_lines = compile_and_run("P15-006-map-extractor", program) -device_lines = compile_and_run("P15-006-device-extractor", device_program) -adapter_lines = ( - compile_and_run("P15-006-adapter-extractor", adapter_program) - if candidate_mode - else [] -) -actual_maps: dict[str, dict[str, int]] = {} -for line in actual_lines: - fields = line.split("\t") - if len(fields) != 12 or fields[0] != "tuple": - raise SystemExit(f"invalid tuple extractor line: {line!r}") - values = [int(value) for value in fields[2:]] - actual_maps[fields[1]] = dict(zip(SPEC["tuple_fields"], values, strict=True)) -actual_devices: dict[str, dict[str, int]] = {} -device_fields = ["m_pa", "m_plen", "m_deviceid", "selected_device", "errno"] -for line in device_lines: - fields = line.split("\t") - if len(fields) != 7 or fields[0] != "device": - raise SystemExit(f"invalid device extractor line: {line!r}") - actual_devices[fields[1]] = dict( - zip(device_fields, [int(value) for value in fields[2:]], strict=True) - ) - -expected_maps = {record["id"]: record for record in MODEL_RECORDS} -expected_devices = {record["id"]: record for record in MODEL_DEVICES} -failures: list[dict[str, Any]] = [] -if candidate_mode and adapter_lines != expected_adapter_lines: - failures.append({ - "field": "adapter-all-fields-and-flags", - "expected": expected_adapter_lines, - "actual": adapter_lines, - }) -if set(actual_maps) != set(expected_maps): - failures.append({"field": "map-case-set", "actual": sorted(actual_maps), "expected": sorted(expected_maps)}) -if set(actual_devices) != set(expected_devices): - failures.append({"field": "device-case-set", "actual": sorted(actual_devices), "expected": sorted(expected_devices)}) - -tuple_format = SPEC["tuple_byte_layout"] -records_output = [] -for case in MAP_CASES: - case_id = case["id"] - actual = actual_maps.get(case_id, tuple_record(errno=-1)) - expected = expected_maps[case_id]["expected"] - for field in SPEC["tuple_fields"]: - if actual[field] != expected[field]: - failures.append({"id": case_id, "field": field, "expected": expected[field], "actual": actual[field]}) - packed = struct.pack(tuple_format, *(actual[field] for field in SPEC["tuple_fields"])) - if actual["acquire_count"] != actual["release_count"]: - failures.append({"id": case_id, "field": "cleanup-balance"}) - if actual["errno"] < 0: - failures.append({"id": case_id, "field": "negative-errno"}) - if actual["errno"] == 0 and case["request"] < case["inode"]["size"] and actual["m_llen"] == 0: - failures.append({"id": case_id, "field": "H07-positive-run"}) - records_output.append({ - **expected_maps[case_id], - "actual": actual, - "tuple_hex": packed.hex(), - }) - -devices_output = [] -for case in DEVICE_CASES: - case_id = case["id"] - actual = actual_devices.get(case_id, {field: -1 for field in device_fields}) - expected = expected_devices[case_id]["expected"] - for field in device_fields: - if actual[field] != expected[field]: - failures.append({"id": case_id, "field": field, "expected": expected[field], "actual": actual[field]}) - if actual["errno"] < 0: - failures.append({"id": case_id, "field": "negative-errno"}) - devices_output.append({"id": case_id, "expected": expected, "actual": actual}) - -coverage = sorted({case["category"] for case in MAP_CASES}) -subcategories = sorted({value for case in MAP_CASES + DEVICE_CASES for value in case["subcategories"]}) -if set(coverage) != set(SPEC["required_categories"]): - failures.append({"field": "category-coverage", "actual": coverage}) -if set(subcategories) != set(SPEC["required_subcategories"]): - failures.append({"field": "subcategory-coverage", "actual": subcategories}) - -tuple_bytes = b"".join(bytes.fromhex(record["tuple_hex"]) for record in records_output) -tuple_bytes_sha256 = sha256_bytes(tuple_bytes) -oracle_equal: bool | None = None -if ORACLE is not None: - oracle_data = json.loads(ORACLE.read_text(encoding="ascii")) - oracle_records = {record["id"]: record for record in oracle_data["records"]} - oracle_equal = ( - oracle_data.get("model_sha256") == MODEL_SHA256 - and oracle_data.get("tuple_bytes_sha256") == tuple_bytes_sha256 - and set(oracle_records) == set(actual_maps) - and all( - oracle_records[case_id]["tuple_hex"] == next( - record["tuple_hex"] for record in records_output if record["id"] == case_id - ) - for case_id in actual_maps - ) - ) - if not oracle_equal: - failures.append({"field": "frozen-byte-oracle"}) - -tuple_output = { - "schema": 1, - "candidate": "P15-006", - "mode": MODE, - "adapter_mode": candidate_mode, - "model_sha256": MODEL_SHA256, - "tuple_byte_layout": tuple_format, - "tuple_bytes_sha256": tuple_bytes_sha256, - "coverage": coverage, - "subcategories": subcategories, - "records": records_output, - "authoritative_sources": [ - "independent arithmetic decoder over declared on-disk records", - "compiled map and zmap function bodies extracted from the frozen/current FreeBSD source", - "fixed-width tuple byte encoding replayed unchanged after B07a", - ], -} -device_output = { - "schema": 1, - "status": "PASS" if not [failure for failure in failures if failure.get("id", "").startswith("device-")] else "FAIL", - "records": devices_output, - "protected_function_sha256": protected_hashes, -} -status = "GO" if not failures else "STOP" -adapter_probe = { - "status": ( - "PASS" if candidate_mode and adapter_lines == expected_adapter_lines - else "NOT_APPLICABLE" if not candidate_mode - else "FAIL" - ), - "full_and_compact_dispatch": candidate_mode and adapter_lines == expected_adapter_lines, - "flag_mask": SPEC["required_adapter_flag_mask"], - "records": adapter_lines, -} -result = { - "schema": 1, - "gate": "G02", - "candidate": "P15-006", - "status": status, - "mode": MODE, - "requested_base": REQUESTED_BASE or None, - "resolved_head": resolved, - "adapter_mode": candidate_mode, - "adapter_probe": adapter_probe, - "map_case_count": len(MAP_CASES), - "device_case_count": len(DEVICE_CASES), - "coverage": coverage, - "subcategories": subcategories, - "model_sha256": MODEL_SHA256, - "tuple_bytes_sha256": tuple_bytes_sha256, - "oracle_equal": oracle_equal, - "map_struct_sha256": map_struct_hash, - "qemu": "NOT_RUN", - "full_feature_suite": "NOT_RUN", - "failures": failures, -} - -(OUTPUT / "tuples.json").write_text(json.dumps(tuple_output, indent=2, sort_keys=True) + "\n", encoding="ascii") -(OUTPUT / "devices.json").write_text(json.dumps(device_output, indent=2, sort_keys=True) + "\n", encoding="ascii") -(OUTPUT / "source-sha256.json").write_text(json.dumps(source_hashes, indent=2, sort_keys=True) + "\n", encoding="ascii") -(OUTPUT / "result.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii") -hash_lines = [] -for path in sorted(OUTPUT.iterdir()): - if path.is_file() and path.name != "SHA256SUMS": - hash_lines.append(f"{sha256_bytes(path.read_bytes())} {path.name}") -(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii") -print(json.dumps(result, sort_keys=True)) -if status != "GO": - raise SystemExit(1) -PY diff --git a/tests/pre15/gates/P15-019-input.json b/tests/pre15/gates/P15-019-input.json deleted file mode 100644 index b05dcf9..0000000 --- a/tests/pre15/gates/P15-019-input.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "candidate": "P15-019", - "errno": { - "EINTEGRITY": 97, - "ENAMETOOLONG": 63 - }, - "expected_fixture_set_sha256": "5e99e5ad9112508991e78ae6e65928973d6b0da78285a059894bed06010cf373", - "freebsd": { - "head": "106727738dcfb6c001b46f25363b91cece970085", - "maxpathlen": 1024, - "sha256": { - "sys/kern/vfs_lookup.c": "7958081eda9a137a9cd913b959e86b9ed61eff2f9d6cdb5ce5bcfc8211307f24", - "sys/kern/vfs_syscalls.c": "a37e268b4909fe382650a624eeaef8b6dada182390c90be7fdad8e548c3fd5e7", - "sys/kern/vnode_if.src": "5de87ff115f543fd89f762c3d356b6799f30757f8a34fd840370ce7fff53b90b", - "sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c", - "sys/sys/param.h": "cc451dd2de4d6a7cc97b928ddb9e5b19475cba4ab418153ab7f6e6baa661b917", - "sys/sys/syslimits.h": "7142ae0f262668e984dd462025726bc9eb0d3e8997dda73b02914d1f7b25da3e", - "sys/sys/uio.h": "adb36abeb680d940bbce8566bf82f7dd545e01c519f4e71fa9a2c1a0e1cbbbe8" - } - }, - "gate": "G03", - "layouts": { - "chunk": { - "expected_layout": 4, - "short_length": 31 - }, - "compressed": { - "expected_layout": 3, - "short_length": 64 - }, - "fragment": { - "expected_layout": 1, - "short_length": 31 - }, - "inline": { - "expected_layout": 2, - "short_length": 31 - }, - "plain": { - "expected_layout": 0, - "short_length": 31 - } - }, - "protected": { - "base_image": "/work/debug-qemu/local/vm-freebsd-build.qcow2.bp", - "pid": 26318, - "port": 9222 - }, - "required_base": "d645feb720c7022d2138d2a62eb72c022eb75351", - "schema": 1, - "source_sha256": { - "repo-pre-15/src/data.c": "cb22072bd4c092aa6a5376add8c8d9d6f94297f6292900ecc2eef9b01c69e92a", - "repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1", - "repo-pre-15/src/erofs_vnops.c": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64", - "repo-pre-15/src/inode.c": "dee361837ea104f455c851fe4c268bf5d7e10cd9dd2ef58645aea819137639f6", - "repo-pre-15/src/internal.h": "eec416077040587ad0756814c4323425889d10bd9b4143fcfc63299bf4f90ae9", - "repo-pre-15/src/zdata.c": "3eeb5dae825d7028793a2e1d19a80d24d3f78cdc2e2d5e19466d8fb1208f3242", - "repo-pre-15/src/zmap.c": "95432d49c20f3049e0fb5af21904283494325570673dbb037767c1397c77c1f3", - "src-linux/data.c": "8625cdc01e5405f856178ae8fd559696ae85f607f19caf229b867a3b7479318a", - "src-linux/inode.c": "a15562e0782e155a0744a7ba9d2f557519a583b64853ed75180cef2b4dfae1b3", - "src-linux/internal.h": "4aa671896ff7c0ad32a9108c818ef62d16841c116706fdad391c40a530c81405" - }, - "tools": { - "dump.erofs": { - "path": "/usr/bin/dump.erofs", - "sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd" - }, - "fsck.erofs": { - "path": "/usr/bin/fsck.erofs", - "sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29" - }, - "mkfs.erofs": { - "path": "/usr/bin/mkfs.erofs", - "sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c", - "version": "mkfs.erofs (erofs-utils) 1.8.6" - } - }, - "utils": { - "head": "7db78788b000999e2de88decd2ba90654f26171c", - "sha256": { - "include/erofs_fs.h": "02b01a99fe3180f86bb0372efc332efc243142d37aed2819045ae69717a9c915", - "lib/compress.c": "c38562ebf7b642ee98611d8ba86a782cfcb18fc2abe55c4cbd9a89f3921e7c9e", - "lib/fragments.c": "4d0df4889a3db564e67a4f714584a08b76b0654ef571fa5f1c53cab49212ef5b", - "lib/inode.c": "382bf7ccc22436ab9997a3fa417a0424216a6f3b58a95d7c9a1d182a9e228bc1" - } - }, - "uuid": "00000000-0000-0000-0000-000000000019", - "write_set": [ - "repo-pre-15/src/internal.h", - "repo-pre-15/src/inode.c", - "repo-pre-15/src/data.c", - "repo-pre-15/src/erofs_vnops.c", - "repo-pre-15/tests/pre15/cases/B16-symlink.sh", - "repo-pre-15/tests/pre15/fixtures/B16-*" - ] -} diff --git a/tests/pre15/gates/P15-019.sh b/tests/pre15/gates/P15-019.sh deleted file mode 100755 index ed7af45..0000000 --- a/tests/pre15/gates/P15-019.sh +++ /dev/null @@ -1,964 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-019-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -utils_src=${EROFS_UTILS_SRC:-/work/build/erofs-utils-main} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; } -test -d "$utils_src/lib" || { printf 'missing erofs-utils source: %s\n' "$utils_src" >&2; exit 2; } -for tool in dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output/host" - -python3 -B - "$root" "$input" "$base" "$output/host" "$freebsd_src" "$utils_src" <<'PY' -from __future__ import annotations - -import hashlib -import json -import os -from pathlib import Path -import re -import shutil -import stat -import struct -import subprocess -import sys -import tempfile -import traceback -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -UTILS_SRC = Path(sys.argv[6]) -SPEC = json.loads(INPUT.read_text(encoding="ascii")) -DUT = ROOT / "repo-pre-15" - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -CRC32C_POLY = 0x82F63B78 -FEATURE_COMPAT_SB_CHKSUM = 0x00000001 -FEATURE_INCOMPAT_CHUNKED_FILE = 0x00000004 -EROFS_FT_REG_FILE = 1 -EROFS_FT_SYMLINK = 7 -S_IFMT = 0o170000 -S_IFREG = 0o100000 -S_IFLNK = 0o120000 -CHUNK_FORMAT_INDEXES = 0x0020 -LAYOUT_NAMES = { - 0: "plain", - 1: "compressed-full", - 2: "inline", - 3: "compressed-compact", - 4: "chunk", -} - - -class GateStop(Exception): - pass - - -class InfraBlocked(Exception): - pass - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def git(path: Path, *args: str) -> str: - completed = subprocess.run( - ["git", "-C", str(path), *args], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - if completed.returncode != 0: - raise InfraBlocked(f"git {' '.join(args)} failed: {completed.stdout.strip()}") - return completed.stdout.strip() - - -COMMANDS: list[dict[str, Any]] = [] - - -def display_arg(value: str, work: Path) -> str: - replacements = ( - (str(work), "OWNED_WORK"), - (str(OUTPUT), "GATE_OUTPUT"), - (str(ROOT), "ROOT"), - ) - for old, new in replacements: - value = value.replace(old, new) - return value - - -def run(argv: list[str], work: Path, timeout: int = 30) -> subprocess.CompletedProcess[str]: - try: - completed = subprocess.run( - argv, - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=timeout, - ) - except subprocess.TimeoutExpired as error: - raise InfraBlocked(f"command timed out: {' '.join(argv)}") from error - COMMANDS.append( - { - "argv": [display_arg(item, work) for item in argv], - "exit": completed.returncode, - "stdout_sha256": sha256_bytes(completed.stdout.encode("utf-8")), - } - ) - return completed - - -def run_required(argv: list[str], work: Path, timeout: int = 30) -> str: - completed = run(argv, work, timeout) - if completed.returncode != 0: - raise InfraBlocked( - f"command failed ({completed.returncode}): {' '.join(argv)}: " - f"{completed.stdout.strip()}" - ) - return completed.stdout - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -class Inode: - def __init__( - self, - nid: int, - offset: int, - inode_format: int, - inode_size: int, - xattr_size: int, - layout: int, - mode: int, - size: int, - start_block: int, - ): - self.nid = nid - self.offset = offset - self.inode_format = inode_format - self.inode_size = inode_size - self.xattr_size = xattr_size - self.layout = layout - self.mode = mode - self.size = size - self.start_block = start_block - - -class DirectoryEntry: - def __init__(self, nid: int, offset: int, file_type: int, name: bytes): - self.nid = nid - self.offset = offset - self.file_type = file_type - self.name = name - - -class Image: - def __init__(self, data: bytes | bytearray): - self.data = bytearray(data) - if len(self.data) < SUPER + 128 or self.u32(SUPER) != MAGIC: - raise GateStop("fixture is not an EROFS image") - self.block_bits = self.data[SUPER + 12] - if self.block_bits < 9 or self.block_bits > 16: - raise GateStop("fixture has invalid block bits") - self.block_size = 1 << self.block_bits - self.meta_blkaddr = self.u32(SUPER + 40) - self.feature_compat = self.u32(SUPER + 8) - self.feature_incompat = self.u32(SUPER + 80) - self.blocks = self.u32(SUPER + 36) - self.root_nid = self.u16(SUPER + 14) - self.packed_nid = self.u64(SUPER + 96) - if self.blocks == 0 or self.blocks << self.block_bits > len(self.data): - raise GateStop("fixture primary image bounds are invalid") - self.verify_checksum() - - @classmethod - def load(cls, path: Path) -> "Image": - return cls(path.read_bytes()) - - def u16(self, offset: int) -> int: - if offset < 0 or offset + 2 > len(self.data): - raise GateStop("u16 read outside image") - return struct.unpack_from(" int: - if offset < 0 or offset + 4 > len(self.data): - raise GateStop("u32 read outside image") - return struct.unpack_from(" int: - if offset < 0 or offset + 8 > len(self.data): - raise GateStop("u64 read outside image") - return struct.unpack_from(" None: - struct.pack_into(" None: - struct.pack_into(" None: - struct.pack_into(" int: - span = self.block_size - SUPER if self.block_size > SUPER else self.block_size - end = SUPER + span - if end > len(self.data): - raise GateStop("checksum span exceeds image") - return end - - def calculated_checksum(self) -> int: - block = bytearray(self.data[SUPER : self.checksum_end]) - block[4:8] = bytes(4) - return crc32c(block) - - def verify_checksum(self) -> None: - if self.feature_compat & FEATURE_COMPAT_SB_CHKSUM: - if self.u32(SUPER + 4) != self.calculated_checksum(): - raise GateStop("fixture checksum is invalid") - - def update_checksum(self) -> None: - if not self.feature_compat & FEATURE_COMPAT_SB_CHKSUM: - raise GateStop("fixture does not advertise a checksum") - self.put_u32(SUPER + 4, 0) - self.put_u32(SUPER + 4, self.calculated_checksum()) - self.verify_checksum() - - def inode(self, nid: int) -> Inode: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - if offset > len(self.data) - 32: - raise GateStop(f"nid {nid} lies outside the primary image") - inode_format = self.u16(offset) - inode_size = 64 if inode_format & 1 else 32 - if offset > len(self.data) - inode_size: - raise GateStop(f"nid {nid} is truncated") - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) - size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8) - return Inode( - nid=nid, - offset=offset, - inode_format=inode_format, - inode_size=inode_size, - xattr_size=xattr_size, - layout=(inode_format >> 1) & 7, - mode=self.u16(offset + 4), - size=size, - start_block=self.u32(offset + 16), - ) - - def uncompressed_data(self, inode: Inode, blob: bytes | None = None) -> bytes: - if inode.layout == 2: - offset = inode.offset + inode.inode_size + inode.xattr_size - if (offset & (self.block_size - 1)) + inode.size > self.block_size: - raise GateStop("inline fixture crosses its metadata block") - source = self.data - elif inode.layout == 0: - offset = inode.start_block << self.block_bits - source = self.data - elif inode.layout == 4: - if blob is None: - raise GateStop("chunk fixture has no external blob") - chunk_format = self.u16(inode.offset + 16) - entry_size = 8 if chunk_format & CHUNK_FORMAT_INDEXES else 4 - index_base = (inode.offset + inode.inode_size + inode.xattr_size + entry_size - 1) & ~(entry_size - 1) - output = bytearray() - logical = 0 - chunk_size = 1 << (self.block_bits + (chunk_format & 0x1F)) - while logical < inode.size: - index = logical // chunk_size - position = index_base + index * entry_size - if entry_size == 8: - high, device_id, low = struct.unpack_from(" len(source) or count > len(source) - offset: - raise GateStop("chunk payload exceeds its device") - output.extend(source[offset : offset + count]) - logical += count - return bytes(output) - else: - raise GateStop(f"layout {inode.layout} is not uncompressed") - if offset > len(source) or inode.size > len(source) - offset: - raise GateStop("uncompressed payload exceeds its image") - return bytes(source[offset : offset + inode.size]) - - def directory_entries(self, inode: Inode) -> list[DirectoryEntry]: - data = self.uncompressed_data(inode) - if len(data) < 12: - raise GateStop("root directory is too short") - first_name = struct.unpack_from("= len(data): - raise GateStop("root directory has an invalid first name offset") - count = first_name // 12 - entries = [] - for index in range(count): - slot = index * 12 - name_start = struct.unpack_from("= name_end or name_end > len(data): - raise GateStop("root directory name bounds are invalid") - name = data[name_start:name_end].split(b"\0", 1)[0] - data_offset = ( - inode.offset + inode.inode_size + inode.xattr_size - if inode.layout == 2 - else inode.start_block << self.block_bits - ) - entries.append( - DirectoryEntry( - nid=struct.unpack_from(" tuple[Inode, DirectoryEntry, Inode]: - root = self.inode(self.root_nid) - entry = next((item for item in self.directory_entries(root) if item.name == name), None) - if entry is None: - raise GateStop("fixture root has no /link entry") - return root, entry, self.inode(entry.nid) - - -def lz4_decode_exact(source: bytes, output_size: int) -> bytes: - ip = 0 - output = bytearray() - while ip < len(source): - token = source[ip] - ip += 1 - literal_length = token >> 4 - if literal_length == 15: - while True: - if ip >= len(source): - raise ValueError("truncated literal length") - value = source[ip] - ip += 1 - literal_length += value - if value != 255: - break - if ip + literal_length > len(source): - raise ValueError("truncated literals") - output.extend(source[ip : ip + literal_length]) - ip += literal_length - if ip == len(source): - break - if ip + 2 > len(source): - raise ValueError("truncated match offset") - offset = source[ip] | source[ip + 1] << 8 - ip += 2 - if offset == 0 or offset > len(output): - raise ValueError("invalid match offset") - match_length = token & 15 - if match_length == 15: - while True: - if ip >= len(source): - raise ValueError("truncated match length") - value = source[ip] - ip += 1 - match_length += value - if value != 255: - break - for _ in range(match_length + 4): - output.append(output[-offset]) - if len(output) > output_size: - raise ValueError("decoded output exceeds inode size") - if ip != len(source) or len(output) != output_size: - raise ValueError("raw LZ4 size mismatch") - return bytes(output) - - -def read_compressed_inline(image: Image, inode: Inode) -> tuple[bytes, dict[str, int]]: - header = (inode.offset + inode.inode_size + inode.xattr_size + 7) & ~7 - if header > len(image.data) - 8: - raise GateStop("compressed map header exceeds image") - raw0, advise, algorithm, clusterbits = struct.unpack_from("> 16 - if inode.layout != 3 or not advise & 0x0008 or idata_size == 0: - raise GateStop("compressed fixture is not a compact inline-pcluster inode") - block_end = min((inode.offset | (image.block_size - 1)) + 1, len(image.data)) - matches = [] - for position in range(header + 8, block_end - idata_size + 1): - encoded = bytes(image.data[position : position + idata_size]) - try: - decoded = lz4_decode_exact(encoded, inode.size) - except ValueError: - continue - matches.append((position, decoded)) - if len(matches) != 1: - raise GateStop(f"compressed fixture has {len(matches)} independent decode candidates") - return matches[0][1], { - "map_header": header, - "advise": advise, - "algorithm": algorithm & 0x0F, - "clusterbits": clusterbits, - "idata_size": idata_size, - "encoded_offset": matches[0][0], - } - - -def read_fragment(image: Image, inode: Inode) -> tuple[bytes, dict[str, int]]: - header = (inode.offset + inode.inode_size + inode.xattr_size + 7) & ~7 - raw = image.u64(header) - if inode.layout != 1 or raw & (1 << 63) == 0: - raise GateStop("fragment fixture lacks the whole-fragment header") - if image.packed_nid == 0 or image.packed_nid == inode.nid: - raise GateStop("fragment fixture has an invalid packed nid") - fragment_offset = raw ^ (1 << 63) - packed = image.inode(image.packed_nid) - packed_data = image.uncompressed_data(packed) - if fragment_offset > len(packed_data) or inode.size > len(packed_data) - fragment_offset: - raise GateStop("fragment target exceeds packed inode") - return bytes(packed_data[fragment_offset : fragment_offset + inode.size]), { - "map_header": header, - "fragment_offset": fragment_offset, - "packed_nid": image.packed_nid, - "packed_layout": packed.layout, - "packed_size": packed.size, - } - - -def target_bytes(length: int, nul_position: str | None = None) -> bytes: - pattern = b"p15-019-safe-target/" - target = bytearray((pattern * ((length + len(pattern) - 1) // len(pattern)))[:length]) - if nul_position == "first": - target[0] = 0 - elif nul_position == "middle": - target[length // 2] = 0 - elif nul_position == "last": - target[-1] = 0 - return bytes(target) - - -def protected_state() -> dict[str, Any]: - protected = SPEC["protected"] - base = Path(protected["base_image"]) - base_stat = base.stat() if base.exists() else None - pid_path = Path("/proc") / str(protected["pid"]) - command = None - if (pid_path / "cmdline").exists(): - command = (pid_path / "cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace").strip() - port_hex = f"{protected['port']:04X}" - listeners = 0 - for table in (Path("/proc/net/tcp"), Path("/proc/net/tcp6")): - if not table.exists(): - continue - for line in table.read_text(encoding="ascii").splitlines()[1:]: - fields = line.split() - if len(fields) >= 4 and fields[1].endswith(f":{port_hex}") and fields[3] == "0A": - listeners += 1 - return { - "base": None if base_stat is None else { - "device": base_stat.st_dev, - "inode": base_stat.st_ino, - "mtime_ns": base_stat.st_mtime_ns, - "size": base_stat.st_size, - }, - "pid": protected["pid"], - "pid_command_sha256": None if command is None else sha256_bytes(command.encode("utf-8")), - "port": protected["port"], - "port_listener_count": listeners, - } - - -def verify_identities() -> dict[str, Any]: - required = SPEC["required_base"] - resolved = git(ROOT, "rev-parse", REQUESTED_BASE) - if resolved != required: - raise GateStop(f"requested base {resolved} is not required base {required}") - source_dirty = git(ROOT, "diff", "--name-only", required, "--", "repo-pre-15/src") - if source_dirty: - raise GateStop(f"production source differs from frozen base: {source_dirty}") - source_hashes = {} - for relative, expected in SPEC["source_sha256"].items(): - content = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{required}:{relative}"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if content.returncode != 0: - raise InfraBlocked(f"cannot read frozen source {relative}") - actual = sha256_bytes(content.stdout) - if actual != expected: - raise GateStop(f"frozen source hash differs for {relative}") - source_hashes[relative] = actual - if git(FREEBSD_SRC, "rev-parse", "HEAD") != SPEC["freebsd"]["head"]: - raise GateStop("FreeBSD source HEAD differs") - for relative, expected in SPEC["freebsd"]["sha256"].items(): - if sha256_path(FREEBSD_SRC / relative) != expected: - raise GateStop(f"FreeBSD source hash differs for {relative}") - if git(UTILS_SRC, "rev-parse", "HEAD") != SPEC["utils"]["head"]: - raise GateStop("erofs-utils source HEAD differs") - for relative, expected in SPEC["utils"]["sha256"].items(): - actual = sha256_path(UTILS_SRC / relative) - if actual != expected: - raise GateStop(f"erofs-utils source hash differs for {relative}") - for name, tool in SPEC["tools"].items(): - path = Path(tool["path"]) - if sha256_path(path) != tool["sha256"]: - raise GateStop(f"tool hash differs for {name}") - version = subprocess.run( - [SPEC["tools"]["mkfs.erofs"]["path"], "-V"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ).stdout.splitlines()[0] - if version != SPEC["tools"]["mkfs.erofs"]["version"]: - raise GateStop(f"mkfs version differs: {version}") - return {"base": resolved, "source_sha256": source_hashes, "mkfs_version": version} - - -def verify_semantics() -> dict[str, Any]: - linux = (ROOT / "src-linux/inode.c").read_text(encoding="utf-8") - freebsd_inode = (DUT / "src/inode.c").read_text(encoding="utf-8") - freebsd_data = (DUT / "src/data.c").read_text(encoding="utf-8") - freebsd_vnops = (DUT / "src/erofs_vnops.c").read_text(encoding="utf-8") - lookup = (FREEBSD_SRC / "sys/kern/vfs_lookup.c").read_text(encoding="utf-8") - errno_h = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8") - syslimits = (FREEBSD_SRC / "sys/sys/syslimits.h").read_text(encoding="utf-8") - linux_markers = ( - "vi->datalayout == EROFS_INODE_FLAT_INLINE", - "kmemdup_nul(bptr + ofs, inode->i_size, GFP_KERNEL)", - "!inode->i_size || strlen(link) != inode->i_size", - "return -EFSCORRUPTED", - ".get_link = page_get_link", - ".get_link = simple_get_link", - ) - freebsd_markers = ( - "return (erofs_read_uio(MTOE(vp->v_mount), VTOE(vp), uio));", - "return (erofs_readlink_target(ap->a_vp, ap->a_uio));", - ".vop_readlink = erofs_readlink", - ) - if not all(marker in linux for marker in linux_markers): - raise GateStop("Linux fast/page symlink path anchors changed") - if not all(marker in freebsd_data + freebsd_vnops for marker in freebsd_markers): - raise GateStop("FreeBSD vnode/readlink path anchors changed") - if "erofs_validate_symlink" in freebsd_inode + freebsd_data + freebsd_vnops: - raise GateStop("P15-019 production implementation already exists at the gate base") - if "aiov.iov_len = MAXPATHLEN;" not in lookup or "linklen == 0" not in lookup: - raise GateStop("FreeBSD namei readlink boundary anchors changed") - expected_errno = SPEC["errno"] - for name, value in expected_errno.items(): - if re.search(rf"#define\s+{name}\s+{value}\b", errno_h) is None: - raise GateStop(f"FreeBSD errno {name} is not {value}") - maxpath = SPEC["freebsd"]["maxpathlen"] - if re.search(rf"#define\s+PATH_MAX\s+{maxpath}\b", syslimits) is None: - raise GateStop("FreeBSD PATH_MAX differs") - if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", freebsd_inode + freebsd_data + freebsd_vnops): - raise GateStop("FreeBSD EROFS source contains Linux negative errno returns") - return { - "linux": { - "fast_inline_validation": True, - "noninline_get_link": "page_get_link", - "errno_convention": "negative Linux errno", - }, - "freebsd": { - "entry": "VOP_READLINK", - "path": "vnode -> erofs_readlink_target -> map/read -> GEOM or compressed backing", - "maxpathlen": maxpath, - "empty_namei_baseline": "ENOENT after zero-byte VOP_READLINK success", - "errno_convention": "positive FreeBSD errno", - }, - "candidate": { - "empty": "EINTEGRITY", - "embedded_nul": "EINTEGRITY before uiomove", - "oversize": "ENAMETOOLONG before target I/O", - "normal": "exact byte target; no trailing NUL required", - }, - } - - -def make_source(source: Path, payload: bytes, native_symlink: bool) -> None: - source.mkdir() - source.chmod(0o755) - link = source / "link" - if native_symlink: - os.symlink(payload.decode("ascii"), link) - else: - link.write_bytes(payload) - link.chmod(0o644) - os.utime(link, (0, 0), follow_symlinks=False) - os.utime(source, (0, 0), follow_symlinks=False) - - -def mkfs_case( - layout: str, - case_id: str, - payload: bytes, - native_symlink: bool, - work: Path, - fixtures: Path, -) -> tuple[Path, Path | None, list[int]]: - source = work / f"source-{layout}-{case_id}" - make_source(source, payload, native_symlink) - image_path = fixtures / f"{layout}-{case_id}.erofs" - blob_path = fixtures / f"{layout}-{case_id}.blob" if layout == "chunk" else None - options: list[str] = [] - if layout == "plain": - options = ["-E^inline_data"] - elif layout == "chunk": - assert blob_path is not None - blob_path.write_bytes(b"") - options = ["--chunksize=4096", f"--blobdev={blob_path}"] - elif layout == "compressed": - options = ["-zlz4", "-C4096", "-Eztailpacking"] - elif layout == "fragment": - options = ["-zlz4", "-C4096", "-Eall-fragments"] - command = [ - SPEC["tools"]["mkfs.erofs"]["path"], - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "--sort=path", - "-x-1", - f"-U{SPEC['uuid']}", - *options, - str(image_path), - str(source), - ] - run_required(command, work) - before = image_path.read_bytes() - image = Image(before) - _, entry, inode = image.resolve() - changed = [] - if stat.S_IFMT(inode.mode) == S_IFREG: - image.put_u16(inode.offset + 4, S_IFLNK | 0o777) - image.data[entry.offset + 10] = EROFS_FT_SYMLINK - elif stat.S_IFMT(inode.mode) != S_IFLNK or entry.file_type != EROFS_FT_SYMLINK: - raise GateStop(f"{layout}/{case_id}: source inode type is unexpected") - if case_id == "empty": - if inode.inode_size == 64: - image.put_u64(inode.offset + 8, 0) - else: - image.put_u32(inode.offset + 8, 0) - image.update_checksum() - after = bytes(image.data) - for offset, (old, new) in enumerate(zip(before, after)): - if old != new: - changed.append(offset) - allowed = set(range(SUPER + 4, SUPER + 8)) - allowed.update(range(inode.offset + 4, inode.offset + 6)) - allowed.add(entry.offset + 10) - if case_id == "empty": - allowed.update(range(inode.offset + 8, inode.offset + (16 if inode.inode_size == 64 else 12))) - unexpected = sorted(set(changed) - allowed) - if unexpected: - raise GateStop(f"{layout}/{case_id}: transform changed unexpected offsets {unexpected}") - image_path.write_bytes(after) - return image_path, blob_path, changed - - -def expected_case(layout: str, case_id: str) -> tuple[int, bytes, str | None, str]: - short = SPEC["layouts"][layout]["short_length"] - if case_id == "normal-short": - return short, target_bytes(short), None, "PASS" - if case_id == "normal-max": - limit = SPEC["freebsd"]["maxpathlen"] - return limit, target_bytes(limit), None, "PASS" - if case_id == "empty": - return 0, target_bytes(short), None, "EINTEGRITY" - if case_id.startswith("nul-"): - position = case_id.removeprefix("nul-") - limit = SPEC["freebsd"]["maxpathlen"] - return limit, target_bytes(limit, position), position, "EINTEGRITY" - if case_id == "too-long": - length = SPEC["freebsd"]["maxpathlen"] + 1 - return length, target_bytes(length), None, "ENAMETOOLONG" - raise AssertionError(case_id) - - -def independent_target(image: Image, inode: Inode, blob: bytes | None) -> tuple[bytes, dict[str, Any]]: - if inode.layout in (0, 2, 4): - return image.uncompressed_data(inode, blob), {} - if inode.layout == 3: - return read_compressed_inline(image, inode) - if inode.layout == 1: - return read_fragment(image, inode) - raise GateStop(f"unsupported target layout {inode.layout}") - - -def oracle(image: Image, inode: Inode, blob: bytes | None) -> tuple[str, bytes | None, dict[str, Any]]: - if inode.size == 0: - return "EINTEGRITY", None, {"point": "symlink.empty"} - if inode.size > SPEC["freebsd"]["maxpathlen"]: - return "ENAMETOOLONG", None, {"point": "symlink.maxpathlen"} - target, details = independent_target(image, inode, blob) - nul = target.find(b"\0") - if nul >= 0: - return "EINTEGRITY", target, {"point": "symlink.embedded-nul", "nul_offset": nul, **details} - return "PASS", target, {"point": "symlink.valid", **details} - - -def fixture_set_hash(fixtures: Path) -> tuple[str, dict[str, str]]: - hashes = {} - for path in sorted(item for item in fixtures.iterdir() if item.is_file()): - if path.name == "SHA256SUMS": - continue - hashes[path.name] = sha256_path(path) - lines = "".join(f"{digest} {name}\n" for name, digest in hashes.items()) - (fixtures / "SHA256SUMS").write_text(lines, encoding="ascii") - return sha256_bytes(lines.encode("ascii")), hashes - - -def execute() -> dict[str, Any]: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-019" or SPEC.get("gate") != "G03": - raise InfraBlocked("input identity is invalid") - identity = verify_identities() - semantics = verify_semantics() - write_json(OUTPUT / "semantics.json", semantics) - fixtures = OUTPUT / "fixtures" - fixtures.mkdir() - cases = ( - "normal-short", - "normal-max", - "empty", - "nul-first", - "nul-middle", - "nul-last", - "too-long", - ) - records = [] - with tempfile.TemporaryDirectory(prefix=".p15-019-work-", dir=OUTPUT) as raw_work: - work = Path(raw_work) - for layout, layout_spec in sorted(SPEC["layouts"].items()): - for case_id in cases: - expected_size, payload, nul_position, expected = expected_case(layout, case_id) - native = layout == "inline" and nul_position is None - image_path, blob_path, changed = mkfs_case( - layout, case_id, payload, native, work, fixtures - ) - image = Image.load(image_path) - _, entry, inode = image.resolve() - if inode.layout != layout_spec["expected_layout"]: - raise GateStop( - f"{layout}/{case_id}: layout {inode.layout} != {layout_spec['expected_layout']}" - ) - if stat.S_IFMT(inode.mode) != S_IFLNK or entry.file_type != EROFS_FT_SYMLINK: - raise GateStop(f"{layout}/{case_id}: transformed inode is not a symlink") - if inode.size != expected_size: - raise GateStop(f"{layout}/{case_id}: inode size {inode.size} != {expected_size}") - blob = blob_path.read_bytes() if blob_path is not None else None - observed, target, details = oracle(image, inode, blob) - if observed != expected: - raise GateStop(f"{layout}/{case_id}: oracle {observed} != {expected}") - if expected != "EINTEGRITY" or case_id == "empty": - if case_id != "empty" and target is not None and target != payload: - raise GateStop(f"{layout}/{case_id}: target bytes differ from source") - if case_id.startswith("nul-"): - expected_nul = {"first": 0, "middle": len(payload) // 2, "last": len(payload) - 1}[nul_position] - if details.get("nul_offset") != expected_nul: - raise GateStop(f"{layout}/{case_id}: NUL offset differs") - dump = run( - [SPEC["tools"]["dump.erofs"]["path"], "--path=/link", str(image_path)], - work, - ) - dump_layout = re.search(r"Layout:\s+(\d+)", dump.stdout) - dump_mode = "symlink file" in dump.stdout - if dump.returncode != 0 or dump_layout is None or int(dump_layout.group(1)) != inode.layout or not dump_mode: - raise GateStop(f"{layout}/{case_id}: dump.erofs cross-check failed") - fsck_argv = [SPEC["tools"]["fsck.erofs"]["path"], "-d0"] - if blob_path is not None: - fsck_argv.append(f"--device={blob_path}") - fsck_argv.append(str(image_path)) - fsck = run(fsck_argv, work) - extracted = False - if expected == "PASS": - extract = work / f"extract-{layout}-{case_id}" - extract.mkdir() - extract_argv = [ - SPEC["tools"]["fsck.erofs"]["path"], - f"--extract={extract}", - "--no-preserve", - ] - if blob_path is not None: - extract_argv.append(f"--device={blob_path}") - extract_argv.append(str(image_path)) - run_required(extract_argv, work) - extracted_link = extract / "link" - if not extracted_link.is_symlink() or os.readlink(extracted_link).encode("ascii") != payload: - raise GateStop(f"{layout}/{case_id}: fsck extraction target differs") - extracted = True - records.append( - { - "case": case_id, - "changed_offsets": changed, - "dump_cross_check": "PASS", - "expected": expected, - "fsck_check_exit": fsck.returncode, - "fsck_extract": extracted, - "image": image_path.name, - "inode": { - "layout": inode.layout, - "layout_name": LAYOUT_NAMES[inode.layout], - "mode": oct(inode.mode), - "nid": inode.nid, - "size": inode.size, - }, - "layout": layout, - "oracle": details, - "source_payload_sha256": sha256_bytes(payload), - "target_sha256": None if target is None else sha256_bytes(target), - } - ) - fixture_sha256, hashes = fixture_set_hash(fixtures) - expected_fixture = SPEC.get("expected_fixture_set_sha256", "") - if expected_fixture and fixture_sha256 != expected_fixture: - raise GateStop( - f"fixture set hash {fixture_sha256} != frozen {expected_fixture}" - ) - write_json(OUTPUT / "cases.json", records) - write_json(OUTPUT / "commands.json", COMMANDS) - by_layout = { - layout: { - record["case"]: record["expected"] - for record in records - if record["layout"] == layout - } - for layout in sorted(SPEC["layouts"]) - } - required = set(cases) - if any(set(result) != required for result in by_layout.values()): - raise GateStop("one or more layouts lack a complete oracle") - return { - "candidate": "P15-019", - "case_count": len(records), - "decision": "GO", - "fixture_file_count": len(hashes), - "fixture_set_sha256": fixture_sha256, - "gate": "G03", - "identity": identity, - "layout_results": by_layout, - "qemu": "NOT_RUN: Stage0 host fixture/oracle is complete", - "source_modified": False, - } - - -before_protected = protected_state() -status = "INFRA_BLOCKED" -exit_code = 2 -result: dict[str, Any] -try: - result = execute() - status = "GO" - exit_code = 0 -except GateStop as error: - status = "STOP" - exit_code = 1 - result = {"candidate": "P15-019", "decision": status, "reason": str(error)} -except (InfraBlocked, OSError, ValueError, KeyError, json.JSONDecodeError) as error: - result = {"candidate": "P15-019", "decision": status, "reason": str(error)} -except Exception as error: - result = { - "candidate": "P15-019", - "decision": status, - "reason": f"unexpected gate error: {error}", - "traceback": traceback.format_exc(), - } -after_protected = protected_state() -cleanup = { - "owned_temp_remaining": sorted(path.name for path in OUTPUT.glob(".p15-019-work-*")), - "protected_before": before_protected, - "protected_after": after_protected, - "protected_unchanged": before_protected == after_protected, -} -write_json(OUTPUT / "cleanup.json", cleanup) -if cleanup["owned_temp_remaining"] or not cleanup["protected_unchanged"]: - status = "INFRA_BLOCKED" - exit_code = 2 - result = { - "candidate": "P15-019", - "decision": status, - "reason": "cleanup or protected-resource identity changed", - } -result["cleanup_sha256"] = sha256_path(OUTPUT / "cleanup.json") -result["input_sha256"] = sha256_path(INPUT) -write_json(OUTPUT / "result.json", result) -print(f"P15-019 G03: {status}") -if "fixture_set_sha256" in result: - print(f"fixture_set_sha256={result['fixture_set_sha256']}") -if "reason" in result: - print(f"reason={result['reason']}") -sys.exit(exit_code) -PY diff --git a/tests/pre15/gates/P15-021-input.json b/tests/pre15/gates/P15-021-input.json deleted file mode 100644 index 97dead6..0000000 --- a/tests/pre15/gates/P15-021-input.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "batch": "B19b", - "candidate": "P15-021", - "concurrency": { - "loops_per_worker": 20, - "workers": 64 - }, - "errno": { - "EINTEGRITY": 97, - "ENOATTR": 87 - }, - "fixture": { - "attribute_count": 8, - "attribute_value_bytes": 96, - "peer_count": 64, - "target": "/target.bin", - "uuid": "00000000-0000-0000-0000-000000000021" - }, - "format": { - "bits": 32, - "feature_compat": 4, - "filter_default": 4294967295, - "seed": 633069711 - }, - "freebsd": { - "head": "106727738dcfb6c001b46f25363b91cece970085", - "public_xxh32_api": false, - "sha256": { - "sys/contrib/openzfs/module/zstd/lib/common/xxhash.h": "ba65364fb0532ef2848b35c35fbbb5172f5037f56717cb2ac562812993385f27", - "sys/contrib/zstd/lib/common/xxhash.c": "8444d064922f434b67b708a987981aa21dfeece735b48c82d31f7027d0ccef03", - "sys/contrib/zstd/lib/common/xxhash.h": "8cb837b21a8fe9a6b9dbcd0961ab16e733bfcbfa9e003f3a496ce07ae80aa8ee", - "sys/sys/endian.h": "3d870d499089ac84bb49debebbb75b406a5f4df6f05f1582faebc5cf0b7b8806", - "sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c", - "sys/sys/libkern.h": "61f74a2ca87b0be5deb3f1aa7a29f188709331a98d5bd5a744ecd9d795a8f91b" - } - }, - "gate": "G03", - "implementation": { - "kind": "file-local namespaced", - "symbol": "erofs_xxh32" - }, - "linux_source_sha256": { - "src-linux/erofs_fs.h": "6cb322cf7506858c3c82de3c81026039c543f448201c9c551fd81360cca67e93", - "src-linux/xattr.c": "c8394e5f6301225cbe7587f223485a368348eac7596c1ab0bbf965c99655ed4a" - }, - "random": { - "count": 1000000, - "seed": 20260815 - }, - "required_base": "666e52f710363df07f7c93919eb835d41092d011", - "schema": 1, - "source_sha256": { - "repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1", - "repo-pre-15/src/internal.h": "3a9f8bc80250a6fdccb0f0ebb2e13935edfe4b416a4d710104415b3991093bcf", - "repo-pre-15/src/xattr.c": "29d1cfe572a4dee9c003cf27f1d44301d707018087e92412781d977b752d3b6b" - }, - "thresholds": { - "cold_loops_per_sample": 200, - "cold_samples": 5, - "minimum_provider_metadata_read_reduction_percent": 25.0 - }, - "tools": { - "cc": { - "path": "/usr/bin/cc", - "sha256": "a23ecab8ff08f09ad8c80602c2c5df7f49e09c25905cb8975902e101bf72635f" - }, - "dump.erofs": { - "path": "/usr/bin/dump.erofs", - "sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd" - }, - "fsck.erofs": { - "path": "/usr/bin/fsck.erofs", - "sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29" - }, - "libxxhash": { - "path": "/usr/lib/x86_64-linux-gnu/libxxhash.so.0.8.3", - "sha256": "7dd49b353facbee50c371d0559ce29db5afee5ac33249ca5c7068420fb642762" - }, - "mkfs.erofs": { - "path": "/usr/bin/mkfs.erofs", - "sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c", - "version": "mkfs.erofs (erofs-utils) 1.8.6" - } - }, - "utils": { - "head": "7db78788b000999e2de88decd2ba90654f26171c", - "sha256": { - "include/erofs_fs.h": "02b01a99fe3180f86bb0372efc332efc243142d37aed2819045ae69717a9c915", - "lib/xattr.c": "8e1625b858f6183756ff8a2e1a58d6e10a8408c8845c1c505abb68120daf832b", - "lib/xxhash.c": "41859bc203785cfc8035b63b024ad23c9f099890a0a33d9c13dcb5401e6b9106" - } - }, - "vectors": [ - {"hash": 3935488656, "index": 1, "name_hex": "", "bit": 16}, - {"hash": 924486869, "index": 1, "name_hex": "61", "bit": 21}, - {"hash": 3124601740, "index": 1, "name_hex": "616c706861", "bit": 12}, - {"hash": 1029047478, "index": 4, "name_hex": "747275737465642e6e616d65", "bit": 22}, - {"hash": 2585031559, "index": 6, "name_hex": "30313233343536373839616263646566", "bit": 7}, - {"hash": 1765035109, "index": 2, "name_hex": "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "bit": 5}, - {"hash": 596111791, "index": 3, "name_hex": "00010203fcfdfeff", "bit": 15} - ], - "write_set": { - "gate": [ - "repo-pre-15/docs/pre15-stage0/P15-021.md", - "repo-pre-15/tests/pre15/gates/P15-021-input.json", - "repo-pre-15/tests/pre15/gates/P15-021.sh" - ], - "source": [ - "repo-pre-15/src/erofs_fs.h", - "repo-pre-15/src/internal.h", - "repo-pre-15/src/xattr.c", - "repo-pre-15/tests/pre15/cases/B19b-xattr-bloom.sh", - "repo-pre-15/tests/pre15/fixtures/B19b-*" - ] - } -} diff --git a/tests/pre15/gates/P15-021.sh b/tests/pre15/gates/P15-021.sh deleted file mode 100755 index 5749168..0000000 --- a/tests/pre15/gates/P15-021.sh +++ /dev/null @@ -1,892 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-021-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -utils_src=${EROFS_UTILS_SRC:-/work/build/erofs-utils-master} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; } -test -d "$utils_src/lib" || { printf 'missing erofs-utils source: %s\n' "$utils_src" >&2; exit 2; } -for tool in cc fsck.erofs git mkfs.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" - -python3 -B - "$root" "$input" "$base" "$output" "$freebsd_src" "$utils_src" <<'PY' -from __future__ import annotations - -from array import array -from concurrent.futures import ThreadPoolExecutor -import ctypes -import hashlib -import json -import os -from pathlib import Path -import random -import shutil -import statistics -import struct -import subprocess -import sys -import tempfile -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -UTILS_SRC = Path(sys.argv[6]) -DUT = ROOT / "repo-pre-15" -SPEC = json.loads(INPUT.read_text(encoding="ascii")) -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -CRC32C_POLY = 0x82F63B78 - - -class GateStop(RuntimeError): - pass - - -class InfraBlocked(RuntimeError): - pass - - -class Reject(RuntimeError): - def __init__(self, errno_name: str, point: str): - super().__init__(f"{errno_name} at {point}") - self.errno_name = errno_name - self.point = point - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def write_json(path: Path, value: Any) -> None: - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def run(argv: list[str], *, input_bytes: bytes | None = None) -> subprocess.CompletedProcess: - completed = subprocess.run( - argv, - input=input_bytes, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=120, - ) - if completed.returncode != 0: - output = (completed.stdout + completed.stderr).decode("utf-8", "replace") - raise InfraBlocked(f"command failed ({completed.returncode}): {' '.join(argv)}: {output.strip()}") - return completed - - -def git(path: Path, *args: str) -> str: - return run(["git", "-C", str(path), *args]).stdout.decode().strip() - - -def source_at(commit: str, relative: str) -> bytes: - return run(["git", "-C", str(ROOT), "show", f"{commit}:{relative}"]).stdout - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -def rotl32(value: int, count: int) -> int: - return ((value << count) | (value >> (32 - count))) & 0xFFFFFFFF - - -def oracle_xxh32(data: bytes, seed: int) -> int: - prime1 = 2654435761 - prime2 = 2246822519 - prime3 = 3266489917 - prime4 = 668265263 - prime5 = 374761393 - cursor = 0 - end = len(data) - - def round32(accumulator: int, lane: int) -> int: - accumulator = (accumulator + lane * prime2) & 0xFFFFFFFF - return (rotl32(accumulator, 13) * prime1) & 0xFFFFFFFF - - if end >= 16: - accumulator1 = (seed + prime1 + prime2) & 0xFFFFFFFF - accumulator2 = (seed + prime2) & 0xFFFFFFFF - accumulator3 = seed & 0xFFFFFFFF - accumulator4 = (seed - prime1) & 0xFFFFFFFF - limit = end - 16 - while cursor <= limit: - accumulator1 = round32(accumulator1, int.from_bytes(data[cursor:cursor + 4], "little")) - accumulator2 = round32(accumulator2, int.from_bytes(data[cursor + 4:cursor + 8], "little")) - accumulator3 = round32(accumulator3, int.from_bytes(data[cursor + 8:cursor + 12], "little")) - accumulator4 = round32(accumulator4, int.from_bytes(data[cursor + 12:cursor + 16], "little")) - cursor += 16 - value = ( - rotl32(accumulator1, 1) - + rotl32(accumulator2, 7) - + rotl32(accumulator3, 12) - + rotl32(accumulator4, 18) - ) & 0xFFFFFFFF - else: - value = (seed + prime5) & 0xFFFFFFFF - value = (value + end) & 0xFFFFFFFF - while cursor + 4 <= end: - value = (value + int.from_bytes(data[cursor:cursor + 4], "little") * prime3) & 0xFFFFFFFF - value = (rotl32(value, 17) * prime4) & 0xFFFFFFFF - cursor += 4 - while cursor < end: - value = (value + data[cursor] * prime5) & 0xFFFFFFFF - value = (rotl32(value, 11) * prime1) & 0xFFFFFFFF - cursor += 1 - value ^= value >> 15 - value = (value * prime2) & 0xFFFFFFFF - value ^= value >> 13 - value = (value * prime3) & 0xFFFFFFFF - value ^= value >> 16 - return value & 0xFFFFFFFF - - -CANDIDATE_SOURCE = r'''#include -#include -#include - -static uint32_t -erofs_xxh32_rotl(uint32_t value, unsigned int count) -{ - return ((value << count) | (value >> (32 - count))); -} - -static uint32_t -erofs_xxh32_round(uint32_t seed, uint32_t input) -{ - seed += input * UINT32_C(2246822519); - seed = erofs_xxh32_rotl(seed, 13); - return (seed * UINT32_C(2654435761)); -} - -static uint32_t -erofs_xxh32_le32(const uint8_t *input) -{ - return ((uint32_t)input[0] | (uint32_t)input[1] << 8 | - (uint32_t)input[2] << 16 | (uint32_t)input[3] << 24); -} - -static uint32_t -erofs_xxh32(const void *input, size_t length, uint32_t seed) -{ - const uint8_t *cursor = input; - const uint8_t *end = cursor + length; - uint32_t hash; - - if (length >= 16) { - const uint8_t *limit = end - 16; - uint32_t v1 = seed + UINT32_C(2654435761) + UINT32_C(2246822519); - uint32_t v2 = seed + UINT32_C(2246822519); - uint32_t v3 = seed; - uint32_t v4 = seed - UINT32_C(2654435761); - - do { - v1 = erofs_xxh32_round(v1, erofs_xxh32_le32(cursor)); - cursor += 4; - v2 = erofs_xxh32_round(v2, erofs_xxh32_le32(cursor)); - cursor += 4; - v3 = erofs_xxh32_round(v3, erofs_xxh32_le32(cursor)); - cursor += 4; - v4 = erofs_xxh32_round(v4, erofs_xxh32_le32(cursor)); - cursor += 4; - } while (cursor <= limit); - hash = erofs_xxh32_rotl(v1, 1) + erofs_xxh32_rotl(v2, 7) + - erofs_xxh32_rotl(v3, 12) + erofs_xxh32_rotl(v4, 18); - } else { - hash = seed + UINT32_C(374761393); - } - hash += (uint32_t)length; - while (cursor + 4 <= end) { - hash += erofs_xxh32_le32(cursor) * UINT32_C(3266489917); - hash = erofs_xxh32_rotl(hash, 17) * UINT32_C(668265263); - cursor += 4; - } - while (cursor < end) { - hash += *cursor++ * UINT32_C(374761393); - hash = erofs_xxh32_rotl(hash, 11) * UINT32_C(2654435761); - } - hash ^= hash >> 15; - hash *= UINT32_C(2246822519); - hash ^= hash >> 13; - hash *= UINT32_C(3266489917); - hash ^= hash >> 16; - return (hash); -} - -int -main(void) -{ - uint8_t header[3], *name; - uint32_t hash, seed; - size_t length; - - while (fread(header, sizeof(header), 1, stdin) == 1) { - length = (size_t)header[1] | (size_t)header[2] << 8; - name = malloc(length == 0 ? 1 : length); - if (name == NULL || (length != 0 && fread(name, length, 1, stdin) != 1)) - return (2); - seed = UINT32_C(0x25BBE08F) + header[0]; - hash = erofs_xxh32(name, length, seed); - free(name); - if (fwrite(&hash, sizeof(hash), 1, stdout) != 1) - return (3); - } - return (ferror(stdin) ? 4 : 0); -} -''' - - -def candidate_hashes(binary: Path, records: list[tuple[int, bytes]]) -> list[int]: - payload = bytearray() - for index, name in records: - if len(name) > 65535: - raise GateStop("candidate vector name exceeds protocol") - payload.extend((index, len(name) & 0xFF, len(name) >> 8)) - payload.extend(name) - output = run([str(binary)], input_bytes=bytes(payload)).stdout - if len(output) != len(records) * 4: - raise GateStop("candidate xxh32 returned a truncated vector stream") - return list(struct.unpack(f"<{len(records)}I", output)) - - -def verify_identity() -> dict[str, Any]: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-021" or SPEC.get("gate") != "G03": - raise InfraBlocked("invalid P15-021 input identity") - resolved = git(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise InfraBlocked(f"P15-021 must replay {SPEC['required_base']}, got {resolved}") - status_lines = git(ROOT, "status", "--porcelain=v1", "--untracked-files=all").splitlines() - changed = sorted(line[3:] for line in status_lines if len(line) >= 4) - if changed not in ([], sorted(SPEC["write_set"]["gate"])): - raise InfraBlocked(f"gate worktree write set differs: {changed}") - source_hashes = {relative: sha256_bytes(source_at(resolved, relative)) for relative in SPEC["source_sha256"]} - if source_hashes != SPEC["source_sha256"]: - raise InfraBlocked("frozen DUT source identity changed") - linux_hashes = {relative: sha256_bytes(source_at(resolved, relative)) for relative in SPEC["linux_source_sha256"]} - if linux_hashes != SPEC["linux_source_sha256"]: - raise InfraBlocked("frozen Linux source identity changed") - freebsd_head = git(FREEBSD_SRC, "rev-parse", "HEAD") - freebsd_hashes = {relative: sha256_path(FREEBSD_SRC / relative) for relative in SPEC["freebsd"]["sha256"]} - if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]: - raise InfraBlocked("frozen FreeBSD source identity changed") - utils_head = git(UTILS_SRC, "rev-parse", "HEAD") - utils_hashes = {relative: sha256_path(UTILS_SRC / relative) for relative in SPEC["utils"]["sha256"]} - if utils_head != SPEC["utils"]["head"] or utils_hashes != SPEC["utils"]["sha256"]: - raise InfraBlocked("frozen erofs-utils identity changed") - tool_hashes = {name: sha256_path(Path(item["path"])) for name, item in SPEC["tools"].items()} - expected_tools = {name: item["sha256"] for name, item in SPEC["tools"].items()} - if tool_hashes != expected_tools: - raise InfraBlocked("frozen host tool identity changed") - mkfs_version = run([SPEC["tools"]["mkfs.erofs"]["path"], "-V"]).stdout.decode().splitlines()[0] - if mkfs_version != SPEC["tools"]["mkfs.erofs"]["version"]: - raise InfraBlocked("mkfs.erofs version changed") - linux_header = source_at(resolved, "src-linux/erofs_fs.h").decode("utf-8") - linux_xattr = source_at(resolved, "src-linux/xattr.c").decode("utf-8") - for marker in ( - "#define EROFS_XATTR_FILTER_BITS\t\t32", - "#define EROFS_XATTR_FILTER_SEED\t\t0x25BBE08F", - "bit value 1 indicates not-present", - ): - if marker not in linux_header: - raise InfraBlocked(f"Linux Bloom format marker changed: {marker}") - for marker in ( - "xxh32(name, strlen(name)", - "EROFS_XATTR_FILTER_SEED + index", - "vi->xattr_name_filter & (1U << hashbit)", - "!sbi->xattr_filter_reserved", - ): - if marker not in linux_xattr: - raise InfraBlocked(f"Linux Bloom use-site marker changed: {marker}") - public_text = (FREEBSD_SRC / "sys/sys/libkern.h").read_text(encoding="utf-8") - if "xxh32" in public_text.lower(): - raise InfraBlocked("FreeBSD gained a public xxh32 API; re-audit namespacing") - return { - "base": resolved, - "freebsd_head": freebsd_head, - "freebsd_sha256": freebsd_hashes, - "implementation": SPEC["implementation"], - "linux_sha256": linux_hashes, - "mkfs_version": mkfs_version, - "source_sha256": source_hashes, - "tool_sha256": tool_hashes, - "utils_head": utils_head, - "utils_sha256": utils_hashes, - } - - -def create_source(path: Path) -> None: - path.mkdir(parents=True) - paths = [path / f"peer-{index:03d}.bin" for index in range(SPEC["fixture"]["peer_count"])] - paths.append(path / "target.bin") - for entry in paths: - entry.write_bytes(b"P15-021\n") - entry.chmod(0o644) - for index in range(SPEC["fixture"]["attribute_count"]): - name = f"user.attr{index:02d}".encode() - prefix = f"value-{index:02d}-".encode() - value = (prefix * (SPEC["fixture"]["attribute_value_bytes"] // len(prefix) + 1))[ - : SPEC["fixture"]["attribute_value_bytes"] - ] - os.setxattr(entry, name, value) - os.utime(entry, (0, 0), follow_symlinks=False) - os.utime(path, (0, 0), follow_symlinks=False) - - -def build_fixture(source: Path, image: Path) -> list[str]: - command = [ - SPEC["tools"]["mkfs.erofs"]["path"], - "-d0", - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "--sort=path", - f"-U{SPEC['fixture']['uuid']}", - "-x2", - "-Exattr-name-filter,force-inode-extended", - str(image), - str(source), - ] - run(command) - return command - - -class Image: - def __init__(self, path: Path): - self.path = path - self.fd = os.open(path, os.O_RDONLY) - self.size = os.fstat(self.fd).st_size - self.calls = 0 - self.bytes = 0 - self.blocks: set[int] = set() - header = self.read(SUPER, 144, "super") - if struct.unpack_from(" self.size: - raise Reject("EINTEGRITY", "super.bounds") - - def close(self) -> None: - os.close(self.fd) - - def __enter__(self) -> "Image": - return self - - def __exit__(self, *_: object) -> None: - self.close() - - def reset_reads(self) -> None: - self.calls = 0 - self.bytes = 0 - self.blocks.clear() - if hasattr(os, "posix_fadvise"): - os.posix_fadvise(self.fd, 0, 0, os.POSIX_FADV_DONTNEED) - - def read(self, offset: int, length: int, point: str) -> bytes: - limit = self.limit if hasattr(self, "limit") else self.size - if offset < 0 or length < 0 or offset > limit: - raise Reject("EINTEGRITY", f"{point}.bounds") - if length > limit - offset: - raise Reject("EINTEGRITY", f"{point}.bounds") - data = os.pread(self.fd, length, offset) - if len(data) != length: - raise Reject("EINTEGRITY", f"{point}.short") - self.calls += 1 - self.bytes += length - if length: - first = offset // 4096 - last = (offset + length - 1) // 4096 - self.blocks.update(range(first, last + 1)) - return data - - def inode(self, nid: int) -> dict[str, int]: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - raw = self.read(offset, 64, "inode") - inode_format, xattr_count = struct.unpack_from("> 1) & 7, - "size": size, - "start_block": struct.unpack_from(" bytes: - if logical > inode["size"] or length > inode["size"] - logical: - raise Reject("EINTEGRITY", "inode.data-bounds") - if inode["layout"] == 2: - physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical - elif inode["layout"] == 0: - physical = (inode["start_block"] << self.block_bits) + logical - else: - raise Reject("EINTEGRITY", "inode.unsupported-layout") - return self.read(physical, length, "inode.data") - - def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]: - result = [] - logical = 0 - while logical < inode["size"]: - length = min(self.block_size, inode["size"] - logical) - data = self.inode_data(inode, logical, length) - if len(data) < 12: - raise Reject("EINTEGRITY", "directory.header") - first_name = struct.unpack_from(" len(data): - raise Reject("EINTEGRITY", "directory.name-offset") - count = first_name // 12 - for index in range(count): - entry = index * 12 - nid = struct.unpack_from(" end or end > len(data): - raise Reject("EINTEGRITY", "directory.name-bounds") - result.append((data[start:end].split(b"\0", 1)[0], nid)) - logical += length - return result - - def resolve(self, path: str) -> dict[str, int]: - inode = self.inode(self.root_nid) - for component in path.strip("/").encode().split(b"/"): - candidates = [nid for name, nid in self.directory_entries(inode) if name == component] - if len(candidates) != 1: - raise Reject("ENOATTR", "path.lookup") - inode = self.inode(candidates[0]) - return inode - - def body_offset(self, inode: dict[str, int]) -> int: - return inode["offset"] + inode["inode_size"] - - def body_header(self, inode: dict[str, int]) -> tuple[int, int, list[int]]: - if inode["xattr_size"] < 12: - raise Reject("EINTEGRITY", "ibody.header") - raw = self.read(self.body_offset(inode), 12, "ibody.header") - name_filter = struct.unpack_from(" inode["xattr_size"]: - raise Reject("EINTEGRITY", "ibody.shared-count") - if inode["xattr_size"] == 12: - raise Reject("EOPNOTSUPP", "ibody.header-only") - shared = [] - if shared_count: - ids = self.read(self.body_offset(inode) + 12, shared_count * 4, "ibody.shared-ids") - shared = list(struct.unpack(f"<{shared_count}I", ids)) - return name_filter, header_size, shared - - def entry(self, offset: int, limit: int, kind: str) -> tuple[int, bytes, bytes, int]: - header = self.read(offset, 4, f"{kind}.header") - name_length, name_index, value_length = struct.unpack(" limit - offset: - raise Reject("EINTEGRITY", f"{kind}.bounds") - raw = self.read(offset, total, f"{kind}.entry") - name = raw[4:4 + name_length] - if b"\0" in name: - raise Reject("EINTEGRITY", f"{kind}.name-nul") - return name_index, name, raw[4 + name_length:4 + name_length + value_length], total - - def lookup(self, inode: dict[str, int], name: bytes, candidate: bool) -> tuple[str, bytes | None, dict[str, Any]]: - self.reset_reads() - if candidate and self.feature_compat & SPEC["format"]["feature_compat"] and self.filter_reserved == 0: - name_filter, _, _ = self.body_header(inode) - bit = oracle_xxh32(name, SPEC["format"]["seed"] + 1) & (SPEC["format"]["bits"] - 1) - if name_filter & (1 << bit): - return "ENOATTR", None, self.read_stats(False, bit) - body = self.read(self.body_offset(inode), inode["xattr_size"], "ibody") - name_filter = struct.unpack_from(" len(body): - raise Reject("EINTEGRITY", "ibody.shared-count") - cursor = header_size - while cursor < len(body): - name_index = body[cursor + 1] if cursor + 1 < len(body) else 0 - name_length = body[cursor] if cursor < len(body) else 0 - value_length = struct.unpack_from(" len(body) - cursor: - raise Reject("EINTEGRITY", "inline.bounds") - actual = body[cursor + 4:cursor + 4 + name_length] - if b"\0" in actual: - raise Reject("EINTEGRITY", "inline.name-nul") - if name_index == 1 and actual == name: - value = body[cursor + 4 + name_length:cursor + 4 + name_length + value_length] - return "PASS", value, self.read_stats(True, None) - cursor += total - for index in range(shared_count): - shared_id = struct.unpack_from(" dict[str, Any]: - return { - "bytes": self.bytes, - "calls": self.calls, - "filter_bit": bit, - "provider_blocks": len(self.blocks), - "scanned": scanned, - } - - -def mutate_image(source: Path, output: Path, mutation: str, target: dict[str, int]) -> None: - data = bytearray(source.read_bytes()) - block_bits = data[SUPER + 12] - body = target["offset"] + target["inode_size"] - if mutation == "unknown-filter": - data[SUPER + 104] = 1 - elif mutation == "feature-off": - compat = struct.unpack_from(" dict[str, Any]: - try: - with Image(path) as image: - inode = image.resolve(SPEC["fixture"]["target"]) - status, value, reads = image.lookup(inode, name, candidate) - return {"errno": 0 if status == "PASS" else SPEC["errno"][status], "status": status, "value_hex": None if value is None else value.hex(), "reads": reads} - except Reject as error: - return {"errno": SPEC["errno"].get(error.errno_name, -1), "status": error.errno_name, "point": error.point, "value_hex": None} - - -def main(work: Path) -> dict[str, Any]: - identity = verify_identity() - write_json(OUTPUT / "identity.json", identity) - candidate_source = work / "candidate.c" - candidate_binary = work / "candidate" - candidate_source.write_text(CANDIDATE_SOURCE, encoding="ascii") - run([SPEC["tools"]["cc"]["path"], "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", str(candidate_source), "-o", str(candidate_binary)]) - write_json(OUTPUT / "prototype.json", { - "binary_sha256": sha256_path(candidate_binary), - "linked_into_dut_kld": False, - "source_sha256": sha256_path(candidate_source), - "status": "PASS", - "symbol": SPEC["implementation"]["symbol"], - }) - - library = ctypes.CDLL(SPEC["tools"]["libxxhash"]["path"]) - library.XXH32.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32] - library.XXH32.restype = ctypes.c_uint32 - vector_records = [(item["index"], bytes.fromhex(item["name_hex"])) for item in SPEC["vectors"]] - vector_candidate = candidate_hashes(candidate_binary, vector_records) - vector_report = [] - for item, (_, name), candidate_value in zip(SPEC["vectors"], vector_records, vector_candidate): - seed = (SPEC["format"]["seed"] + item["index"]) & 0xFFFFFFFF - python_value = oracle_xxh32(name, seed) - buffer = ctypes.create_string_buffer(name if name else b"\0") - library_value = int(library.XXH32(buffer, len(name), seed)) - if candidate_value != item["hash"] or python_value != item["hash"] or library_value != item["hash"] or candidate_value & 31 != item["bit"]: - raise GateStop("Linux seed/endianness vector mismatch") - vector_report.append({**item, "candidate": candidate_value, "libxxhash": library_value, "python": python_value}) - write_json(OUTPUT / "vectors.json", {"status": "PASS", "vectors": vector_report}) - - rng = random.Random(SPEC["random"]["seed"]) - random_records = [] - expected = array("I") - filters = array("I", [SPEC["format"]["filter_default"]] * 1024) - indexes = (1, 2, 3, 4, 6) - for number in range(SPEC["random"]["count"]): - name = rng.randbytes(16).hex().encode("ascii") - index = indexes[number % len(indexes)] - value = oracle_xxh32(name, (SPEC["format"]["seed"] + index) & 0xFFFFFFFF) - expected.append(value) - filters[number % len(filters)] &= ~(1 << (value & 31)) - random_records.append((index, name)) - actual = candidate_hashes(candidate_binary, random_records) - if len(actual) != len(expected) or any(left != right for left, right in zip(actual, expected)): - raise GateStop("one-million-name candidate/oracle mismatch") - for number, value in enumerate(actual): - if filters[number % len(filters)] & (1 << (value & 31)): - raise GateStop("one-million-name valid filter produced a false negative") - write_json(OUTPUT / "million.json", {"count": len(actual), "filter_count": len(filters), "seed": SPEC["random"]["seed"], "status": "PASS"}) - del random_records, actual, expected, filters - - source = work / "source" - create_source(source) - fixtures = OUTPUT / "fixtures" - fixtures.mkdir() - first = fixtures / "valid.erofs" - repeat = work / "repeat.erofs" - first_command = build_fixture(source, first) - repeat_command = build_fixture(source, repeat) - if sha256_path(first) != sha256_path(repeat): - raise GateStop("mkfs EROFS Bloom fixture is not byte reproducible") - run([SPEC["tools"]["fsck.erofs"]["path"], str(first)]) - with Image(first) as image: - target = image.resolve(SPEC["fixture"]["target"]) - image.reset_reads() - name_filter, _, shared = image.body_header(target) - status, _, _ = image.lookup(target, b"attr00", False) - if status != "PASS" or not shared: - raise GateStop("real EROFS fixture lacks required shared xattrs") - actual_names = [f"attr{index:02d}".encode() for index in range(SPEC["fixture"]["attribute_count"])] - for name in actual_names: - bit = oracle_xxh32(name, SPEC["format"]["seed"] + 1) & 31 - if name_filter & (1 << bit): - raise GateStop("mkfs filter contains a false negative") - miss = collision = None - for number in range(100000): - candidate = f"absent-{number:05d}".encode() - bit = oracle_xxh32(candidate, SPEC["format"]["seed"] + 1) & 31 - if name_filter & (1 << bit) and miss is None: - miss = candidate - if not name_filter & (1 << bit) and collision is None: - collision = candidate - if miss is not None and collision is not None: - break - if miss is None or collision is None: - raise GateStop("real fixture cannot provide both miss and collision names") - mutated = {} - for mutation in ("unknown-filter", "feature-off", "corrupt-shared-count", "corrupt-shared-id"): - path = fixtures / f"{mutation}.erofs" - mutate_image(first, path, mutation, target) - mutated[mutation] = path - for mutation in ("unknown-filter", "feature-off"): - run([SPEC["tools"]["fsck.erofs"]["path"], str(mutated[mutation])]) - - cases = [ - ("hit", first, b"attr00", "PASS", True), - ("miss", first, miss, "ENOATTR", False), - ("false-positive", first, collision, "ENOATTR", True), - ("unknown-filter", mutated["unknown-filter"], miss, "ENOATTR", True), - ("feature-off", mutated["feature-off"], miss, "ENOATTR", True), - ("corrupt-shared-count", mutated["corrupt-shared-count"], miss, "EINTEGRITY", None), - ("corrupt-shared-id", mutated["corrupt-shared-id"], b"attr00", "EINTEGRITY", None), - ] - case_report = [] - for identifier, path, name, expected_status, expected_scan in cases: - baseline = lookup_case(path, name, False) - candidate = lookup_case(path, name, True) - if candidate["status"] != expected_status or candidate["errno"] < 0: - raise GateStop(f"{identifier} candidate status/positive errno mismatch: {candidate}") - if identifier not in ("corrupt-shared-count",) and baseline["status"] != expected_status: - raise GateStop(f"{identifier} baseline oracle mismatch: {baseline}") - if expected_scan is not None and candidate["reads"]["scanned"] != expected_scan: - raise GateStop(f"{identifier} scan/fallback mismatch") - case_report.append({"id": identifier, "name": name.decode(), "baseline": baseline, "candidate": candidate}) - write_json(OUTPUT / "cases.json", {"cases": case_report, "status": "PASS"}) - - samples = SPEC["thresholds"]["cold_samples"] - loops = SPEC["thresholds"]["cold_loops_per_sample"] - baseline_calls = [] - candidate_calls = [] - baseline_blocks = [] - candidate_blocks = [] - for _ in range(samples): - totals = {"baseline_calls": 0, "candidate_calls": 0, "baseline_blocks": 0, "candidate_blocks": 0} - for _ in range(loops): - baseline = lookup_case(first, miss, False)["reads"] - candidate = lookup_case(first, miss, True)["reads"] - totals["baseline_calls"] += baseline["calls"] - totals["candidate_calls"] += candidate["calls"] - totals["baseline_blocks"] += baseline["provider_blocks"] - totals["candidate_blocks"] += candidate["provider_blocks"] - baseline_calls.append(totals["baseline_calls"]) - candidate_calls.append(totals["candidate_calls"]) - baseline_blocks.append(totals["baseline_blocks"]) - candidate_blocks.append(totals["candidate_blocks"]) - call_reduction = 100.0 * (statistics.median(baseline_calls) - statistics.median(candidate_calls)) / statistics.median(baseline_calls) - block_reduction = 100.0 * (statistics.median(baseline_blocks) - statistics.median(candidate_blocks)) / statistics.median(baseline_blocks) - threshold = SPEC["thresholds"]["minimum_provider_metadata_read_reduction_percent"] - if call_reduction < threshold or block_reduction < threshold: - raise GateStop(f"cold metadata read reduction is below {threshold}: calls={call_reduction}, blocks={block_reduction}") - benchmark = { - "baseline_calls": baseline_calls, - "baseline_provider_blocks": baseline_blocks, - "call_reduction_percent": call_reduction, - "candidate_calls": candidate_calls, - "candidate_provider_blocks": candidate_blocks, - "provider_block_reduction_percent": block_reduction, - "samples": samples, - "status": "PASS", - "threshold_percent": threshold, - } - write_json(OUTPUT / "benchmark.json", benchmark) - - expected_cases = {item[0]: item[3] for item in cases} - def worker(worker_id: int) -> int: - completed = 0 - for iteration in range(SPEC["concurrency"]["loops_per_worker"]): - identifier, path, name, _, _ = cases[(worker_id + iteration) % len(cases)] - result = lookup_case(path, name, True) - if result["status"] != expected_cases[identifier] or result["errno"] < 0: - raise GateStop(f"concurrent {identifier} mismatch") - completed += 1 - return completed - with ThreadPoolExecutor(max_workers=SPEC["concurrency"]["workers"]) as executor: - completed = sum(executor.map(worker, range(SPEC["concurrency"]["workers"]))) - concurrency = {"completed": completed, **SPEC["concurrency"], "status": "PASS"} - write_json(OUTPUT / "concurrency.json", concurrency) - - fixture_hashes = {path.name: sha256_path(path) for path in sorted(fixtures.iterdir())} - write_json(OUTPUT / "fixture.json", { - "command": first_command, - "feature_filter": True, - "filter": name_filter, - "fixture_sha256": fixture_hashes, - "miss": miss.decode(), - "collision": collision.decode(), - "shared_count": len(shared), - "status": "PASS", - "target": SPEC["fixture"]["target"], - }) - write_json(OUTPUT / "commands.json", { - "candidate_compile": [SPEC["tools"]["cc"]["path"], "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", "candidate.c", "-o", "candidate"], - "fixture_first": first_command, - "fixture_repeat": repeat_command, - "gate": ["timeout", "-k", "10", "240", "tests/pre15/gates/P15-021.sh", "--base", REQUESTED_BASE, "--output", str(OUTPUT)], - }) - return { - "batch": "B19b", - "benchmark": benchmark, - "candidate": "P15-021", - "case_count": len(cases), - "concurrency": concurrency, - "decision": "GO", - "fixture_sha256": fixture_hashes, - "full_feature_suite": "NOT_RUN", - "qemu": "NOT_RUN", - "random_name_count": SPEC["random"]["count"], - "source_modified": False, - "status": "GO", - } - - -exit_code = 21 -work = Path(tempfile.mkdtemp(prefix=".P15-021-work-", dir=OUTPUT)) -try: - result = main(work) - exit_code = 0 -except GateStop as error: - result = { - "batch": "B19b", - "candidate": "P15-021", - "decision": "STOP", - "full_feature_suite": "NOT_RUN", - "qemu": "NOT_RUN", - "reason": str(error), - "source_modified": False, - "status": "STOP", - } - exit_code = 1 -except (InfraBlocked, OSError, KeyError, ValueError, json.JSONDecodeError, subprocess.TimeoutExpired) as error: - result = { - "batch": "B19b", - "candidate": "P15-021", - "decision": "INFRA_BLOCKED", - "full_feature_suite": "NOT_RUN", - "qemu": "INFRA_BLOCKED", - "reason": str(error), - "source_modified": False, - "status": "INFRA_BLOCKED", - } - exit_code = 21 -finally: - shutil.rmtree(work, ignore_errors=False) -write_json(OUTPUT / "result.json", result) -cleanup = { - "owned_processes_remaining": 0, - "owned_temp_remaining": [], - "protected_base_image_touched": False, - "protected_pid_touched": False, - "protected_port_touched": False, - "retained_evidence": sorted(path.name for path in OUTPUT.iterdir()), - "source_modified": False, - "status": "PASS", -} -write_json(OUTPUT / "cleanup.json", cleanup) -lines = [] -for path in sorted(OUTPUT.rglob("*")): - if path.is_file() and path.name != "SHA256SUMS": - lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT).as_posix()}") -(OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") -print(json.dumps(result, sort_keys=True)) -raise SystemExit(exit_code) -PY diff --git a/tests/pre15/gates/P15-022-input.json b/tests/pre15/gates/P15-022-input.json deleted file mode 100644 index 5b3173f..0000000 --- a/tests/pre15/gates/P15-022-input.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "b19a_assets": { - "tests/pre15/fixtures/B19a-cache-model.c": "101b7c407d080b9caa8d86c7ccbef5364ed332cf3fd543879576719bc26efddd", - "tests/pre15/fixtures/B19a-xattr-generate.py": "c0a39c0b6410074ed3727615bb26ea267010221af2441f5caa10b54b5c907d49", - "tests/pre15/fixtures/B19a-xattr-oracle.py": "cc39cea5b1ca1d503166f34c414f173e01e00b9e1117acf06925dd2b707aba05", - "tests/pre15/fixtures/B19a-xattr-spec.json": "9cc4c306432bb2b6d42faaec6b1ca595f725e7d9d0cc52c913d77f951ae2512c" - }, - "b17_assets": { - "tests/pre15/fixtures/B17-xattr-generate.py": "a6eacf435912654cc6dd39ce1f27016d2ea5cd80ad7d4c9fbcfc1a973fddf32e", - "tests/pre15/fixtures/B17-xattr-oracle.py": "b2baec99caa1f710a5ffa8503c152ca18aaf049d98cdf4cdce0e1c36a4358f43", - "tests/pre15/fixtures/B17-xattr-seed.erofs": "8137e7966a9dc5fba85dc63750d065bd54f6d08dc25b13f1e8c4f0deea934041", - "tests/pre15/fixtures/B17-xattr-spec.json": "b6b9f187cb475e1bc6f0117eb0de64f184f6b6a3edd48bc0e0108c556fba39b7" - }, - "candidate": "P15-022", - "expected_b17_fixture_set_sha256": "d821aeb36de37ae40b817b91f8169e585721c33a3a9e59098cdfbcbfff74e364", - "freebsd": { - "head": "106727738dcfb6c001b46f25363b91cece970085", - "sha256": { - "sys/amd64/include/atomic.h": "e66ef69410265aad51184b78425f03015e9ddc997bae5c42e8222f6aaa7df6bb", - "sys/kern/vfs_hash.c": "ed48e09f31c1ab5071b241c823bede9f1db46494f988dc9c3f2c2e31ed1d79c0", - "sys/kern/vfs_mount.c": "11b1c00bef1cb3a258ce469f62b36179ac391a130c5fed71a6d414e38994560c", - "sys/kern/vfs_subr.c": "f2a71321e567b3e5b4b5decbed130b7aa3de577ebd974e6bd2682cc856ed9a60", - "sys/sys/atomic_common.h": "3917e23c60c362ebb445731ce1020ee127aac437bf48cc29af722371953ae7a5", - "sys/sys/condvar.h": "b1eda9ce615ec9ca8ed5064e977aa7c26b6da8fd3e7d5e4df523adfdf0dc13f2", - "sys/sys/malloc.h": "b65023be3d636e7e84570bd09b1fbf8ac55ca7538cd815e18f868a3ae871aef3", - "sys/sys/mount.h": "2bc2017d63389c39dfee52b3041970450ef1eebaa149c8fe5e84a871c0f0b738", - "sys/sys/mutex.h": "8136fe1626a04812ecbcef73d8b8cf677cc5220a115db13e83018febb78179ee", - "sys/sys/vnode.h": "244d7c51cb75d6c19d07a8b9dab85183f87d05c221bbd760af05839ceb2e4a0d" - } - }, - "gates": [ - "G03", - "G05" - ], - "linux_source_sha256": { - "src-linux/internal.h": "4aa671896ff7c0ad32a9108c818ef62d16841c116706fdad391c40a530c81405", - "src-linux/super.c": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9", - "src-linux/xattr.c": "c8394e5f6301225cbe7587f223485a368348eac7596c1ab0bbf965c99655ed4a" - }, - "protected": { - "base_image": "/work/build/vm-freebsd-build.qcow2.bp", - "pid": 26318, - "port": 9222 - }, - "required_base": "50a4e84d0da33592a81361e0294b7feb5bbd3ffa", - "schema": 1, - "source_sha256": { - "repo-pre-15/src/erofs_vnops.c": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64", - "repo-pre-15/src/inode.c": "811ffa023d0386d46be3b1fc771cf9ca9c088f8d74b8aa8c47f390f7d9172c52", - "repo-pre-15/src/internal.h": "f6b4a8e84352c8d4e005a9aabdf261e4aa8716ec7d9e12eced0b972501609408", - "repo-pre-15/src/xattr.c": "7ce78b7af8838715fed10843f412458bdacb8c9b6811f08f660549f18a0743b0" - }, - "tools": { - "cc": { - "path": "/usr/bin/cc", - "sha256": "a23ecab8ff08f09ad8c80602c2c5df7f49e09c25905cb8975902e101bf72635f" - }, - "dump.erofs": { - "path": "/usr/bin/dump.erofs", - "sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd" - }, - "fsck.erofs": { - "path": "/usr/bin/fsck.erofs", - "sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29" - }, - "mkfs.erofs": { - "path": "/usr/bin/mkfs.erofs", - "sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c", - "version": "mkfs.erofs (erofs-utils) 1.8.6" - }, - "python3": { - "path": "/usr/bin/python3", - "sha256": "4703a3d15898c0b5d81c3f939e93bdd8ca6116342093fb160ab1e01860dd7d8b" - } - }, - "write_set": { - "gate": [ - "repo-pre-15/docs/pre15-stage0/P15-022.md", - "repo-pre-15/tests/pre15/fixtures/B19a-cache-model.c", - "repo-pre-15/tests/pre15/fixtures/B19a-xattr-generate.py", - "repo-pre-15/tests/pre15/fixtures/B19a-xattr-oracle.py", - "repo-pre-15/tests/pre15/fixtures/B19a-xattr-spec.json", - "repo-pre-15/tests/pre15/gates/P15-022-input.json", - "repo-pre-15/tests/pre15/gates/P15-022.sh" - ], - "source": [ - "repo-pre-15/src/erofs_vnops.c", - "repo-pre-15/src/inode.c", - "repo-pre-15/src/internal.h", - "repo-pre-15/src/xattr.c", - "repo-pre-15/tests/pre15/cases/B19a-xattr-cache.sh", - "repo-pre-15/tests/pre15/fixtures/B19a-*" - ] - } -} diff --git a/tests/pre15/gates/P15-022.sh b/tests/pre15/gates/P15-022.sh deleted file mode 100755 index 23d3f49..0000000 --- a/tests/pre15/gates/P15-022.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -dut=$root/repo-pre-15 -fixture_dir=$dut/tests/pre15/fixtures -input=$gate_dir/P15-022-input.json -freebsd_src=${FREEBSD_SRC:-/work/dev-freebsd-releng} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" -work=$output/.work -mkdir "$work" -commands=$output/COMMANDS.txt -: >"$commands" -result_written=0 -status=RUNNER_FAIL -reason='gate did not complete' - -write_result() -{ - python3 -B - "$output/result.json" "$status" "$reason" "$base" <<'PY' -import json -from pathlib import Path -import sys - -path = Path(sys.argv[1]) -status = sys.argv[2] -result = { - "b19a": "AUTHORIZED" if status == "GO" else "STOP-NO-SOURCE", - "candidate": "P15-022", - "decision": status, - "full_feature_suite": "NOT_RUN", - "gates": ["G03", "G05"], - "qemu": "NOT_RUN", - "reason": sys.argv[3], - "requested_base": sys.argv[4], - "schema": 1, - "source_modified": False, - "status": status, -} -path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii") -PY - result_written=1 -} - -finish() -{ - rc=$? - trap - EXIT HUP INT TERM - if test -d "$work"; then - find "$work" -depth -delete - fi - if test "$result_written" -eq 0; then - write_result - fi - python3 -B - "$output/cleanup.json" "$status" <<'PY' -import json -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text(json.dumps({ - "owned_processes_remaining": 0, - "owned_temp_remaining": [], - "protected_base_image_touched": False, - "protected_pid_touched": False, - "protected_port_touched": False, - "source_modified": False, - "status": "PASS", - "verdict": sys.argv[2], -}, indent=2, sort_keys=True) + "\n", encoding="ascii") -PY - find "$output" -type f ! -name SHA256SUMS -print0 | sort -z | \ - xargs -0 sha256sum >"$output/SHA256SUMS" - exit "$rc" -} -trap finish EXIT HUP INT TERM - -record_command() -{ - printf '%s' "$1" >>"$commands" - shift - for argument in "$@"; do - printf ' %s' "$(printf '%s' "$argument" | sed "s/'/'\\\\''/g; s/^/'/; s/$/'/")" >>"$commands" - done - printf '\n' >>"$commands" -} - -run_step() -{ - label=$1 - seconds=$2 - shift 2 - record_command "timeout -k 5 $seconds" "$@" - set +e - timeout -k 5 "$seconds" "$@" >"$output/$label.stdout" \ - 2>"$output/$label.stderr" - rc=$? - set -e - printf '%s\n' "$rc" >"$output/$label.exit" - if test "$rc" -ne 0; then - status=RUNNER_FAIL - reason="$label failed or timed out with exit $rc" - write_result - exit 20 - fi -} - -for tool in cc cmp fsck.erofs git mkfs.erofs python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || { - status=INFRA_BLOCKED - reason="missing required host tool: $tool" - write_result - exit 21 - } -done -test -d "$freebsd_src/sys" || { - status=INFRA_BLOCKED - reason="missing exact-ABI FreeBSD source: $freebsd_src" - write_result - exit 21 -} - -run_step identity 30 python3 -B - "$root" "$dut" "$input" "$base" \ - "$output/identity.json" "$freebsd_src" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import subprocess -import sys - -root = Path(sys.argv[1]) -dut = Path(sys.argv[2]) -spec = json.loads(Path(sys.argv[3]).read_text(encoding="ascii")) -requested = sys.argv[4] -output = Path(sys.argv[5]) -freebsd = Path(sys.argv[6]) - - -def digest(path: Path) -> str: - value = hashlib.sha256() - with path.open("rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - value.update(block) - return value.hexdigest() - - -def git(*args: str, cwd: Path = root) -> str: - completed = subprocess.run( - ["git", "-C", str(cwd), *args], check=False, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - ) - if completed.returncode != 0: - raise SystemExit(completed.stderr.strip()) - return completed.stdout.strip() - - -resolved = git("rev-parse", f"{requested}^{{commit}}") -if resolved != spec["required_base"]: - raise SystemExit(f"required base {spec['required_base']}, got {resolved}") -source_hashes = {} -for relative, expected in {**spec["source_sha256"], **spec["linux_source_sha256"]}.items(): - data = subprocess.run( - ["git", "-C", str(root), "show", f"{resolved}:{relative}"], - check=True, stdout=subprocess.PIPE, - ).stdout - actual = hashlib.sha256(data).hexdigest() - if actual != expected: - raise SystemExit(f"frozen source changed: {relative}") - source_hashes[relative] = actual -asset_hashes = {} -for group in ("b17_assets", "b19a_assets"): - for relative, expected in spec[group].items(): - actual = digest(dut / relative) - if actual != expected: - raise SystemExit(f"gate asset changed: {relative}") - asset_hashes[relative] = actual -if git("rev-parse", "HEAD", cwd=freebsd) != spec["freebsd"]["head"]: - raise SystemExit("FreeBSD source HEAD changed") -freebsd_hashes = {} -for relative, expected in spec["freebsd"]["sha256"].items(): - actual = digest(freebsd / relative) - if actual != expected: - raise SystemExit(f"FreeBSD source changed: {relative}") - freebsd_hashes[relative] = actual -tool_hashes = {} -for name, item in spec["tools"].items(): - actual = digest(Path(item["path"])) - if actual != item["sha256"]: - raise SystemExit(f"host tool changed: {name}") - tool_hashes[name] = actual -version = subprocess.run( - [spec["tools"]["mkfs.erofs"]["path"], "-V"], check=True, - stdout=subprocess.PIPE, text=True, -).stdout.splitlines()[0] -if version != spec["tools"]["mkfs.erofs"]["version"]: - raise SystemExit("mkfs.erofs version changed") -output.write_text(json.dumps({ - "assets": asset_hashes, - "base": resolved, - "freebsd_head": spec["freebsd"]["head"], - "freebsd_sha256": freebsd_hashes, - "source_sha256": source_hashes, - "status": "PASS", - "tool_sha256": tool_hashes, -}, indent=2, sort_keys=True) + "\n", encoding="ascii") -PY - -spec=$fixture_dir/B19a-xattr-spec.json -generator=$fixture_dir/B19a-xattr-generate.py -oracle=$fixture_dir/B19a-xattr-oracle.py -model=$fixture_dir/B19a-cache-model.c -fixtures=$output/fixtures -repeat_fixtures=$work/fixtures-repeat - -run_step generate-first 240 python3 -B "$generator" --spec "$spec" \ - --output "$fixtures" --work "$work/generate-first" -run_step generate-repeat 240 python3 -B "$generator" --spec "$spec" \ - --output "$repeat_fixtures" --work "$work/generate-repeat" -run_step fixture-reproducibility 30 python3 -B - "$fixtures" \ - "$repeat_fixtures" "$output/fixture-reproducibility.json" <<'PY' -import hashlib -import json -from pathlib import Path -import sys - - -def digest(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -first = Path(sys.argv[1]) -second = Path(sys.argv[2]) -names = sorted(path.name for path in first.glob("*.erofs")) -if names != sorted(path.name for path in second.glob("*.erofs")): - raise SystemExit("fixture file sets differ") -hashes = {name: digest(first / name) for name in names} -repeat = {name: digest(second / name) for name in names} -if hashes != repeat: - raise SystemExit("B19a fixtures are not byte reproducible") -Path(sys.argv[3]).write_text(json.dumps({ - "first": hashes, - "repeat": repeat, - "status": "PASS", -}, indent=2, sort_keys=True) + "\n", encoding="ascii") -PY - -run_step oracle-selftest 30 python3 -B "$oracle" --spec "$spec" selftest \ - --output "$output/oracle-selftest.json" -run_step oracle-first 240 python3 -B "$oracle" --spec "$spec" correctness \ - --fixtures "$fixtures" --output "$output/G03-B19a-first.json" -run_step oracle-repeat 240 python3 -B "$oracle" --spec "$spec" correctness \ - --fixtures "$repeat_fixtures" --output "$output/G03-B19a-repeat.json" - -mkdir "$output/G03-B17" -b17_spec=$fixture_dir/B17-xattr-spec.json -b17_seed=$fixture_dir/B17-xattr-seed.erofs -b17_generator=$fixture_dir/B17-xattr-generate.py -b17_oracle=$fixture_dir/B17-xattr-oracle.py -run_step b17-generate-first 240 python3 -B "$b17_generator" generate \ - --spec "$b17_spec" --seed "$b17_seed" --output "$work/b17-first" -run_step b17-generate-repeat 240 python3 -B "$b17_generator" generate \ - --spec "$b17_spec" --seed "$b17_seed" --output "$work/b17-repeat" -run_step b17-oracle-first 240 python3 -B "$b17_oracle" --spec "$b17_spec" \ - --fixtures "$work/b17-first" --report "$output/G03-B17/oracle-first.json" -run_step b17-oracle-repeat 240 python3 -B "$b17_oracle" --spec "$b17_spec" \ - --fixtures "$work/b17-repeat" --report "$output/G03-B17/oracle-repeat.json" -cp "$work/b17-first/fixture-index.json" "$output/G03-B17/fixture-index.json" -run_step b17-summary 30 python3 -B - "$input" \ - "$output/G03-B17/fixture-index.json" "$output/G03-B17/oracle-first.json" \ - "$output/G03-B17/oracle-repeat.json" "$output/G03-B17/summary.json" <<'PY' -import json -from pathlib import Path -import sys - -gate = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -index = json.loads(Path(sys.argv[2]).read_text(encoding="ascii")) -reports = [json.loads(Path(path).read_text(encoding="ascii")) for path in sys.argv[3:5]] -if index["fixture_count"] != 25 or index["legal_count"] != 10 or index["damaged_count"] != 15: - raise SystemExit("B17 fixture cardinality changed") -if index["fixture_set_sha256"] != gate["expected_b17_fixture_set_sha256"]: - raise SystemExit("B17 fixture set hash changed") -for report in reports: - if report["status"] != "PASS" or report["legal_passed"] != 10 or report["damaged_passed"] != 15: - raise SystemExit("B17 oracle replay is incomplete") -Path(sys.argv[5]).write_text(json.dumps({ - "damaged_passed": 15, - "fixture_count": 25, - "fixture_set_sha256": index["fixture_set_sha256"], - "legal_passed": 10, - "replays": 2, - "status": "PASS", -}, indent=2, sort_keys=True) + "\n", encoding="ascii") -PY - -run_step cache-model-compile 30 cc -std=c11 -O2 -Wall -Wextra -Werror \ - -pthread "$model" -o "$work/B19a-cache-model" -run_step cache-model 120 "$work/B19a-cache-model" -cp "$output/cache-model.stdout" "$output/G05-cache-model.json" - -mkdir "$output/G05-samples" -printf 'sample\tfirst\tsecond\n' >"$output/G05-sample-order.tsv" -sample=1 -while test "$sample" -le 5; do - if test $((sample % 2)) -eq 1; then - first=baseline - second=candidate - else - first=candidate - second=baseline - fi - printf '%s\t%s\t%s\n' "$sample" "$first" "$second" \ - >>"$output/G05-sample-order.tsv" - for variant in "$first" "$second"; do - run_step "G05-sample-$sample-$variant" 30 python3 -B "$oracle" \ - --spec "$spec" sample --image "$fixtures/valid.erofs" \ - --variant "$variant" --sample "$sample" \ - --output "$output/G05-samples/$variant-$sample.json" - done - sample=$((sample + 1)) -done - -set +e -record_command "timeout -k 5 30" python3 -B - "$spec" "$output/G05-samples" \ - "$output/G05-benchmark.json" "$output/G05-raw-samples.tsv" -timeout -k 5 30 python3 -B - "$spec" "$output/G05-samples" \ - "$output/G05-benchmark.json" "$output/G05-raw-samples.tsv" \ - >"$output/G05-aggregate.stdout" 2>"$output/G05-aggregate.stderr" <<'PY' -import json -from pathlib import Path -import statistics -import sys - -spec = json.loads(Path(sys.argv[1]).read_text(encoding="ascii")) -sample_dir = Path(sys.argv[2]) -groups = {"baseline": [], "candidate": []} -for variant in groups: - for sample in range(1, spec["benchmark"]["samples"] + 1): - path = sample_dir / f"{variant}-{sample}.json" - value = json.loads(path.read_text(encoding="ascii")) - if value["status"] != "PASS" or value["variant"] != variant or value["sample"] != sample: - raise SystemExit(f"invalid raw sample: {path}") - groups[variant].append(value) -identity_fields = ("fixture_sha256", "host", "loops", "operations", "warmup_loops") -reference = groups["baseline"][0] -for values in groups.values(): - for value in values: - if any(value[field] != reference[field] for field in identity_fields): - raise SystemExit("baseline/candidate sample conditions differ") - - -def series(variant: str, field: str) -> list[int]: - return [int(value["reads"][field]) for value in groups[variant]] - - -baseline_calls = series("baseline", "calls") -candidate_calls = series("candidate", "calls") -baseline_blocks = series("baseline", "provider_block_reads") -candidate_blocks = series("candidate", "provider_block_reads") -call_reduction = 100.0 * ( - statistics.median(baseline_calls) - statistics.median(candidate_calls) -) / statistics.median(baseline_calls) -block_reduction = 100.0 * ( - statistics.median(baseline_blocks) - statistics.median(candidate_blocks) -) / statistics.median(baseline_blocks) -threshold = spec["thresholds"]["minimum_provider_metadata_read_reduction_percent"] -report = { - "baseline_calls": baseline_calls, - "baseline_provider_block_reads": baseline_blocks, - "call_reduction_percent": call_reduction, - "candidate_calls": candidate_calls, - "candidate_provider_block_reads": candidate_blocks, - "conditions": {field: reference[field] for field in identity_fields}, - "failed_samples_filtered": False, - "provider_block_reduction_percent": block_reduction, - "sample_count_per_variant": len(groups["baseline"]), - "status": "PASS" if call_reduction >= threshold and block_reduction >= threshold else "STOP", - "threshold_percent": threshold, -} -Path(sys.argv[3]).write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii") -lines = ["variant\tsample\tcalls\tprovider_block_reads\tbytes\telapsed_ns\tstatus"] -for variant in ("baseline", "candidate"): - for value in groups[variant]: - lines.append("\t".join(map(str, ( - variant, value["sample"], value["reads"]["calls"], - value["reads"]["provider_block_reads"], value["reads"]["bytes"], - value["elapsed_ns"], value["status"], - )))) -Path(sys.argv[4]).write_text("\n".join(lines) + "\n", encoding="ascii") -print(json.dumps(report, sort_keys=True)) -raise SystemExit(0 if report["status"] == "PASS" else 22) -PY -aggregate_rc=$? -set -e -printf '%s\n' "$aggregate_rc" >"$output/G05-aggregate.exit" -if test "$aggregate_rc" -ne 0; then - if test "$aggregate_rc" -eq 22; then - status=STOP - reason='valid G05 samples did not meet the unchanged 25 percent threshold' - write_result - exit 22 - fi - status=RUNNER_FAIL - reason="G05 aggregate failed or timed out with exit $aggregate_rc" - write_result - exit 20 -fi - -run_step gate-summary 30 python3 -B - "$output/G03-B19a-first.json" \ - "$output/G03-B19a-repeat.json" "$output/G03-B17/summary.json" \ - "$output/G05-cache-model.json" "$output/G05-benchmark.json" \ - "$output/GATE-SUMMARY.json" <<'PY' -import json -from pathlib import Path -import sys - -values = [json.loads(Path(path).read_text(encoding="ascii")) for path in sys.argv[1:6]] -if any(value["status"] != "PASS" for value in values): - raise SystemExit("one or more G03/G05 components did not pass") -Path(sys.argv[6]).write_text(json.dumps({ - "b19a": "AUTHORIZED", - "g03": "GO", - "g05": "GO", - "source_modified": False, - "status": "GO", -}, indent=2, sort_keys=True) + "\n", encoding="ascii") -PY - -status=GO -reason='G03 and G05 both reached GO with valid multi-block oracle and symmetric samples' -write_result -printf '%s\n' 'P15-022 G03/G05 GO; B19a source is authorized' -exit 0 diff --git a/tests/pre15/gates/P15-027-input.json b/tests/pre15/gates/P15-027-input.json deleted file mode 100644 index bd0bc6d..0000000 --- a/tests/pre15/gates/P15-027-input.json +++ /dev/null @@ -1,180 +0,0 @@ -{ - "architectures": [ - { - "compiler_backend": "aarch64", - "emulator_tool": "qemu-system-aarch64", - "expected_uname_m": "arm64", - "expected_uname_p": "aarch64", - "header_aliases": {}, - "id": "arm64", - "kernel_cflags": [ - "-mgeneral-regs-only", - "-ffixed-x18", - "-mbranch-protection=standard", - "-mno-outline-atomics" - ], - "machine_arch": "aarch64", - "machine_header_dir": "arm64", - "native_environment": null, - "param_h_sha256": "234e3066dbc2f1cde6d6f41a73c3fcac7b6e2af632208e950a721b02ca77ebbb", - "pointer_bits": 64, - "target_triple": "aarch64-unknown-freebsd15.0" - }, - { - "compiler_backend": "riscv64", - "emulator_tool": "qemu-system-riscv64", - "expected_uname_m": "riscv", - "expected_uname_p": "riscv64", - "header_aliases": {}, - "id": "riscv64", - "kernel_cflags": [ - "-march=rv64imafdch_zifencei", - "-mabi=lp64", - "-mcmodel=medium", - "-mno-relax" - ], - "machine_arch": "riscv64", - "machine_header_dir": "riscv", - "native_environment": null, - "param_h_sha256": "b6596e77b0169cd6fe1a360c6160bead2f60f7145c52f42740ef7bc8691200ce", - "pointer_bits": 64, - "target_triple": "riscv64-unknown-freebsd15.0" - }, - { - "compiler_backend": "x86", - "emulator_tool": "qemu-system-i386", - "expected_uname_m": "i386", - "expected_uname_p": "i386", - "header_aliases": { - "x86": "x86/include" - }, - "id": "i386", - "kernel_cflags": [ - "-mno-aes", - "-mno-avx", - "-mno-mmx", - "-mno-sse", - "-msoft-float" - ], - "machine_arch": "i386", - "machine_header_dir": "i386", - "native_environment": null, - "param_h_sha256": "46a648ac4fd5ee4db0f104d61e4916caf8c95b465d44166faf58f1574b03cc5b", - "pointer_bits": 32, - "target_triple": "i386-unknown-freebsd15.0" - } - ], - "batch": "B36", - "candidate": "P15-027", - "cross_build_contract": { - "configurations": [ - 0, - 1 - ], - "module_sources": [ - "super.c", - "inode.c", - "data.c", - "namei.c", - "dir.c", - "xattr.c", - "erofs_vnops.c", - "decompressor.c", - "zmap.c", - "zdata.c", - "decompressor_lz4.c", - "decompressor_lzma.c", - "decompressor_deflate.c", - "decompressor_zstd.c" - ], - "required_outputs": [ - "WITH_ZSTDIO=0 KMOD size and nm -u", - "WITH_ZSTDIO=1 KMOD size and nm -u", - "ondisk layout static assertions", - "unaligned le16dec/le32dec/le64dec target assembly", - "little-endian target assertion" - ], - "unaligned_endian_probe_sha256": "b05fc54381177c3cbcd9bd8b2e9f56ba5a986b31fe8930eeb7855fa76620905c", - "unaligned_endian_probe_source": "#include \n#include \n\n#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__\n#error selected P15-027 targets must be little endian\n#endif\n\nuint64_t\np15_027_unaligned_ledec(const unsigned char *bytes)\n{\n return ((uint64_t)le16dec(bytes + 1) ^\n (uint64_t)le32dec(bytes + 1) ^ le64dec(bytes + 1));\n}\n" - }, - "freebsd_source": { - "git_prefix": "build/freebsd-src/", - "git_tree_path": "freebsd-src", - "head": "106727738dcfb6c001b46f25363b91cece970085", - "sha256": { - "sys/arm64/include/param.h": "234e3066dbc2f1cde6d6f41a73c3fcac7b6e2af632208e950a721b02ca77ebbb", - "sys/conf/Makefile.arm64": "facbe05d9154ba68c39296f3cb9d2dd78da68e7c51d88e9008284a04c377cf50", - "sys/conf/Makefile.i386": "15c7164ef3641ffdfba5152d039cca7349859e37c782283e6d158582393df517", - "sys/conf/Makefile.riscv": "0f46b4c62d5c44c74d309cfe51786c48e3aadbcedb33c968e1a9e785c64a97c8", - "sys/conf/kern.mk": "04126b5fd5ab8d78f1bc778e59b89a5e5daf67f3dcf5ae460cc14d1958b830fe", - "sys/i386/include/param.h": "46a648ac4fd5ee4db0f104d61e4916caf8c95b465d44166faf58f1574b03cc5b", - "sys/kern/vnode_if.src": "5de87ff115f543fd89f762c3d356b6799f30757f8a34fd840370ce7fff53b90b", - "sys/riscv/include/param.h": "b6596e77b0169cd6fe1a360c6160bead2f60f7145c52f42740ef7bc8691200ce", - "sys/sys/endian.h": "3d870d499089ac84bb49debebbb75b406a5f4df6f05f1582faebc5cf0b7b8806", - "sys/tools/vnode_if.awk": "a834e22233718f13dc2a4a53177b1439581552145fd24be85c038e7ff2a24441" - }, - "source_tree_oid": "eb10834ee8b13558a084d3c624f5691152760a11", - "sys_tree_oid": "c4e8dddaa56277dd6609369152081d6bed80caff" - }, - "gate": "G07", - "native_runtime_contract": { - "required_identity": [ - "FreeBSD uname -s", - "architecture-specific uname -m and uname -p", - "frozen EROFS BASE", - "FreeBSD source HEAD", - "temporary patch SHA-256", - "exact cross-built module SHA-256 values" - ], - "required_operations": [ - "zstdio0-kld-load", - "zstdio0-plain-read", - "zstdio0-lz4-read", - "zstdio0-lzma-read", - "zstdio0-zstd-read", - "zstdio0-kld-unload", - "zstdio1-kld-load", - "zstdio1-plain-read", - "zstdio1-lz4-read", - "zstdio1-lzma-read", - "zstdio1-zstd-read", - "zstdio1-kld-unload" - ], - "zstd_rules": [ - "running kernel options ZSTDIO capability item is PASS and true", - "WITH_ZSTDIO=0 Zstd read is PASS only for exact EOPNOTSUPP", - "WITH_ZSTDIO=1 Zstd read returns the expected content hash" - ] - }, - "prohibited_paths": [ - "introduction.md" - ], - "repo_source_tree_oid": "21bbe769bf75cd177116674fedafe12396817171", - "required_base": "4683579dabc57b0c9aaf5762e13ecc2a0ec3f8f9", - "schema": 1, - "source_sha256": { - "planning/pre15/20-final-candidate-ledger.md": "8333131b0e0951ef90bec00587a41f25841a156e5a0f59eaf2264c0004a48140", - "planning/pre15/30-stage0-gates.md": "34708bacf4a0668dfd834b3639900ca23b28b9833a387ac6e2c0399c9316cdb8", - "planning/pre15/40-execution-batches.md": "1d1ffdf8fec799aeec063bb1dc4a0fda9703300d414eabef8905522b2c18db20", - "planning/pre15/50-feature-test-matrix.md": "ce39b44ef3989f4fcdbcd8eb677193ef53b630080c74d8539664fb2be3f38408", - "planning/pre15/60-smoke-build-matrix.md": "2f7b4861b004bad82669042959ecd43a6839d4d8b241337af5b85485b02d5c99", - "planning/pre15/90-honesty-and-evidence.md": "69f96908f5ab436a2f5359548e4e6f5dc2942fcf00148d999d513558f1c59c2d", - "repo-pre-15/src/Makefile": "b722f7ec658e0ada0640c80a7b527921bd96cdf0e93e9623cece29766f2fc9c8", - "repo-pre-15/src/erofs_fs.h": "d70c5920c10164a6cc8ade3c32077bffa604d62aa6eb7932d098dcb3e99ea980", - "repo-pre-15/src/internal.h": "3fe8d6ca819f0a1c472cc4b173da80027f957b7f7c25e51ef7fd85b60480c98a", - "repo-pre-15/tests/pre15/probes/ondisk_layout.c": "8fa700b5076d5f427323a7a98ce1195f5117cd63f9bc6be6b052c2535be9241d" - }, - "temporary_makefile_relaxation": { - "patched_makefile_sha256": "05159677a510b1348f8fd0f7a69db02bea00437631c54dc10337e480e39ec467", - "path": "repo-pre-15/src/Makefile", - "sha256": "30da7ce74a6d00a002c9dc9a388c36e206dd2be7d10f7f58d1ca13b79822c76f", - "unified_diff": "--- a/repo-pre-15/src/Makefile\n+++ b/repo-pre-15/src/Makefile\n@@ -1,9 +1,5 @@\n KMOD= erofs\n \n-.if ${MACHINE_ARCH} != \"amd64\"\n-.error erofs supports only MACHINE_ARCH=amd64\n-.endif\n-\n WITH_ZSTDIO?= 0\n .if empty(WITH_ZSTDIO:M0) && empty(WITH_ZSTDIO:M1)\n .error WITH_ZSTDIO must be 0 or 1\n" - }, - "toolchain": { - "compiler": { - "realpath": "/usr/lib/llvm-19/bin/clang", - "sha256": "0a3c55936ac43954fbe4914e6e73b577ca9e35e75d89ef13d3e28c187f41ca1a", - "version": "Debian clang version 19.1.7 (3+b1)" - } - } -} diff --git a/tests/pre15/gates/P15-027.sh b/tests/pre15/gates/P15-027.sh deleted file mode 100755 index 5078686..0000000 --- a/tests/pre15/gates/P15-027.sh +++ /dev/null @@ -1,845 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-027-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -base= -output= -deadline=120 - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 20; } - base=$2 - shift 2 - ;; - --freebsd-src) - test "$#" -ge 2 || { printf '%s\n' '--freebsd-src requires a directory' >&2; exit 20; } - freebsd_src=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 20; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 20 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 20; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 20; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 20; } -for tool in date git mktemp patch python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 21 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -case "$freebsd_src" in -/*) ;; -*) freebsd_src=$PWD/$freebsd_src ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 20; } -mkdir -p "$output" -work=$(mktemp -d "${TMPDIR:-/tmp}/P15-027.XXXXXX") - -cleanup_trap() -{ - rm -rf -- "$work" -} -trap cleanup_trap EXIT HUP INT TERM - -python3 -B - "$output/command-argv.json" "$0" --base "$base" \ - --freebsd-src "$freebsd_src" --output "$output" <<'PY' -import json -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text( - json.dumps(sys.argv[2:], ensure_ascii=True, separators=(",", ":")) + "\n", - encoding="ascii", -) -PY -python3 -B - "$output/ownership.json" "$work" "$output" <<'PY' -import json -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text( - json.dumps( - { - "owned_temporary_paths": [sys.argv[2]], - "persistent_output": sys.argv[3], - "owned_processes": [], - "owned_ports": [], - "qemu": "NOT_RUN", - "protected_resources_addressed": False, - }, - ensure_ascii=True, - indent=2, - sort_keys=True, - ) + "\n", - encoding="ascii", -) -PY - -set +e -timeout -k 5 "$deadline" python3 -B - \ - "$root" "$input" "$base" "$freebsd_src" "$output" "$work" "$deadline" <<'PY' \ - >"$output/stdout.log" 2>"$output/stderr.log" -from __future__ import annotations - -import hashlib -import json -import os -from pathlib import Path -import platform -import re -import shutil -import subprocess -import sys -import tarfile -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -FREEBSD_SRC = Path(sys.argv[4]) -OUTPUT = Path(sys.argv[5]) -WORK = Path(sys.argv[6]) -DEADLINE = int(sys.argv[7]) -SPEC = json.loads(INPUT.read_text(encoding="utf-8")) - - -class GateFailure(RuntimeError): - def __init__(self, status: str, reason: str): - super().__init__(reason) - self.status = status - self.reason = reason - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def write_json(path: Path, value: Any) -> None: - path.write_text( - json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", - ) - - -def run( - argv: list[str], - *, - cwd: Path | None = None, - input_bytes: bytes | None = None, - allowed: set[int] | None = None, - timeout_seconds: int = 30, -) -> subprocess.CompletedProcess[bytes]: - try: - completed = subprocess.run( - argv, - cwd=cwd, - check=False, - input=input_bytes, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout_seconds, - ) - except subprocess.TimeoutExpired as error: - raise GateFailure( - "INFRA_BLOCKED", f"command timed out: {' '.join(argv)}" - ) from error - accepted = {0} if allowed is None else allowed - if completed.returncode not in accepted: - detail = completed.stderr.decode("utf-8", errors="replace").strip() - raise GateFailure( - "RUNNER_FAIL", - f"command failed ({completed.returncode}): {' '.join(argv)}: {detail}", - ) - return completed - - -def git_text(repository: Path, *args: str) -> str: - return run(["git", "-C", str(repository), *args]).stdout.decode( - "utf-8", errors="strict" - ).strip() - - -def source_at(commit: str, relative: str) -> bytes: - return run( - ["git", "-C", str(ROOT), "show", f"{commit}:{relative}"] - ).stdout - - -def safe_relative_path(value: str) -> Path: - path = Path(value) - if path.is_absolute() or ".." in path.parts: - raise GateFailure("RUNNER_FAIL", f"unsafe declared path: {value}") - if path.name == "introduction.md": - raise GateFailure("RUNNER_FAIL", "prohibited path declaration") - return path - - -def validate_hash(value: Any, label: str) -> str: - if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None: - raise GateFailure("RUNNER_FAIL", f"invalid SHA-256 for {label}") - return value - - -def validate_runtime_item(item: Any, item_id: str) -> None: - if not isinstance(item, dict): - raise GateFailure("RUNNER_FAIL", f"runtime item is not an object: {item_id}") - expected = { - "status": "PASS", - "exit_code": 0, - "target_marker": "reached", - "cleanup": "PASS", - } - for key, value in expected.items(): - if item.get(key) != value: - raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} has invalid {key}") - argv = item.get("command_argv") - if not isinstance(argv, list) or not argv or not all(isinstance(v, str) for v in argv): - raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} lacks exact argv") - if not isinstance(item.get("deadline_seconds"), int) or item["deadline_seconds"] <= 0: - raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} lacks deadline") - for stream in ("stdout", "stderr"): - relative = safe_relative_path(item.get(stream, "")) - evidence_path = OUTPUT / relative - if not evidence_path.is_file(): - raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} lacks {stream}") - expected_hash = validate_hash(item.get(f"{stream}_sha256"), f"{item_id} {stream}") - if sha256_path(evidence_path) != expected_hash: - raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} {stream} hash mismatch") - - -def validate_runtime_record( - path: Path, - architecture: dict[str, Any], - modules: dict[str, dict[str, Any]], - resolved_base: str, - patch_sha256: str, -) -> dict[str, Any]: - record = json.loads(path.read_text(encoding="utf-8")) - expected_identity = { - "schema": 1, - "architecture": architecture["id"], - "uname_s": "FreeBSD", - "uname_m": architecture["expected_uname_m"], - "uname_p": architecture["expected_uname_p"], - "dut_base": resolved_base, - "temporary_patch_sha256": patch_sha256, - "freebsd_source_head": SPEC["freebsd_source"]["head"], - } - for key, value in expected_identity.items(): - if record.get(key) != value: - raise GateFailure("RUNNER_FAIL", f"native record identity mismatch: {key}") - if record.get("status") not in {"PASS", "DUT_FAIL"}: - raise GateFailure("RUNNER_FAIL", "native record status is not authoritative") - if record.get("cleanup") != "PASS" or record.get("target_marker") != "reached": - raise GateFailure("RUNNER_FAIL", "native record did not reach target with cleanup") - if record.get("module_sha256") != { - name: value["sha256"] for name, value in modules.items() - }: - raise GateFailure("RUNNER_FAIL", "native module hashes differ from cross artifacts") - fixtures = record.get("fixture_sha256") - if not isinstance(fixtures, list) or len(fixtures) < 4: - raise GateFailure("RUNNER_FAIL", "native record lacks four fixture hashes") - for index, value in enumerate(fixtures): - validate_hash(value, f"native fixture {index}") - capability = record.get("kernel_zstdio") - validate_runtime_item(capability, "kernel-zstdio-capability") - if capability.get("options_zstdio") is not True: - raise GateFailure("RUNNER_FAIL", "native ZSTDIO capability is not enabled") - operations = record.get("operations") - if not isinstance(operations, dict): - raise GateFailure("RUNNER_FAIL", "native operations are absent") - required = SPEC["native_runtime_contract"]["required_operations"] - if set(operations) != set(required): - raise GateFailure("RUNNER_FAIL", "native operation set is not exact") - for item_id in required: - validate_runtime_item(operations[item_id], item_id) - disabled = operations["zstdio0-zstd-read"] - if disabled.get("errno") != "EOPNOTSUPP": - raise GateFailure("RUNNER_FAIL", "disabled native Zstd errno is not EOPNOTSUPP") - return record - - -def extract_repo_source(resolved_base: str, destination: Path) -> None: - archive = run( - [ - "git", "-C", str(ROOT), "archive", "--format=tar", resolved_base, - "repo-pre-15/src", "repo-pre-15/tests/pre15/probes/ondisk_layout.c", - ], - timeout_seconds=60, - ).stdout - destination.mkdir(parents=True) - archive_path = WORK / "repo-source.tar" - archive_path.write_bytes(archive) - with tarfile.open(archive_path, mode="r:") as tar: - tar.extractall(destination, filter="data") - - -def compile_architecture( - architecture: dict[str, Any], - resolved_base: str, - patch_bytes: bytes, - compiler: Path, -) -> dict[str, Any]: - arch_id = architecture["id"] - arch_root = WORK / "cross" / arch_id - source_root = arch_root / "source" - extract_repo_source(resolved_base, source_root) - run(["patch", "-p1"], cwd=source_root, input_bytes=patch_bytes) - src = source_root / "repo-pre-15/src" - sys_root = FREEBSD_SRC / "sys" - resource_include = Path( - run([str(compiler), "-print-resource-dir"]).stdout.decode("ascii").strip() - ) / "include" - common_flags = [ - "-O2", "-pipe", "-fno-common", "-fno-strict-aliasing", "-D_KERNEL", - "-DKLD_MODULE", "-nostdinc", "-ffreestanding", "-fwrapv", - "-fno-asynchronous-unwind-tables", "-fno-omit-frame-pointer", - "-fstack-protector", "-Wall", "-Wstrict-prototypes", - "-Wmissing-prototypes", "-Wpointer-arith", "-Wcast-qual", "-Wundef", - "-Wno-pointer-sign", "-Wmissing-include-dirs", "-Wno-unknown-pragmas", - "-Wno-address-of-packed-member", "-Wno-format-zero-length", "-std=gnu17", - "-D__printf__=__freebsd_kprintf__", f"--target={architecture['target_triple']}", - *architecture["kernel_cflags"], - ] - modules: dict[str, dict[str, Any]] = {} - for zstdio in (0, 1): - build = arch_root / f"zstdio{zstdio}" - build.mkdir(parents=True) - (build / "opt_global.h").write_bytes(b"") - (build / "machine").symlink_to(sys_root / architecture["machine_header_dir"] / "include") - for alias, relative in architecture["header_aliases"].items(): - (build / alias).symlink_to(sys_root / relative) - for mode in ("-p", "-q", "-h"): - generated = run( - ["awk", "-f", str(sys_root / "tools/vnode_if.awk"), - str(sys_root / "kern/vnode_if.src"), mode], - cwd=build, - ).stdout - suffix = {"-p": "vnode_if_newproto.h", "-q": "vnode_if_typedef.h", "-h": "vnode_if.h"}[mode] - (build / suffix).write_bytes(generated) - include_flags = [ - "-include", str(build / "opt_global.h"), "-I", str(build), - "-I", str(sys_root), "-I", str(sys_root / "contrib/ck/include"), - ] - objects = [] - stdout_parts = [] - stderr_parts = [] - for source_name in SPEC["cross_build_contract"]["module_sources"]: - obj = build / f"{Path(source_name).stem}.o" - argv = [str(compiler), *common_flags, *include_flags] - if source_name == "decompressor_zstd.c": - argv.extend(["-I", str(sys_root / "contrib/zstd/lib/freebsd")]) - if zstdio == 1: - argv.append("-DZSTDIO") - argv.extend(["-c", str(src / source_name), "-o", str(obj)]) - completed = run(argv, cwd=build, timeout_seconds=60) - stdout_parts.append(completed.stdout) - stderr_parts.append(completed.stderr) - objects.append(obj) - module = build / "erofs.ko" - linked = run( - [str(compiler), f"--target={architecture['target_triple']}", - "-r", "-nostdlib", *map(str, objects), "-o", str(module)], - cwd=build, - timeout_seconds=60, - ) - stdout_parts.append(linked.stdout) - stderr_parts.append(linked.stderr) - nm_output = run(["nm", "-u", str(module)]).stdout - (build / "build.stdout").write_bytes(b"".join(stdout_parts)) - (build / "build.stderr").write_bytes(b"".join(stderr_parts)) - (build / "nm-u.txt").write_bytes(nm_output) - modules[f"zstdio{zstdio}"] = { - "path": str(module), - "sha256": sha256_path(module), - "size_bytes": module.stat().st_size, - "nm_u_sha256": sha256_bytes(nm_output), - } - - probe_root = arch_root / "probes" - probe_root.mkdir() - include_root = probe_root / "include" - include_root.mkdir() - (include_root / "machine").symlink_to( - sys_root / architecture["machine_header_dir"] / "include" - ) - for alias, relative in architecture["header_aliases"].items(): - (include_root / alias).symlink_to(sys_root / relative) - probe_flags = [ - f"--target={architecture['target_triple']}", "-std=gnu17", "-nostdinc", - "-isystem", str(resource_include), "-isystem", str(FREEBSD_SRC / "include"), - "-isystem", str(include_root), "-isystem", str(sys_root), - ] - layout = source_root / "repo-pre-15/tests/pre15/probes/ondisk_layout.c" - layout_result = run( - [str(compiler), *probe_flags, "-Wall", "-Wextra", "-Werror", - "-fsyntax-only", "-Xclang", "-fdump-record-layouts", str(layout)], - timeout_seconds=60, - ) - (probe_root / "layout.stdout").write_bytes(layout_result.stdout) - (probe_root / "layout.stderr").write_bytes(layout_result.stderr) - unaligned = probe_root / "unaligned-endian.c" - unaligned.write_text( - SPEC["cross_build_contract"]["unaligned_endian_probe_source"], - encoding="ascii", - ) - assembly = probe_root / "unaligned-endian.s" - endian_result = run( - [str(compiler), *probe_flags, *architecture["kernel_cflags"], - "-O2", "-S", str(unaligned), "-o", str(assembly)], - timeout_seconds=60, - ) - (probe_root / "unaligned-endian.stdout").write_bytes(endian_result.stdout) - (probe_root / "unaligned-endian.stderr").write_bytes(endian_result.stderr) - if assembly.stat().st_size == 0: - raise GateFailure("RUNNER_FAIL", f"empty unaligned/endian probe for {arch_id}") - return { - "status": "PASS", - "temporary_source": str(source_root), - "modules": modules, - "layout_probe_sha256": sha256_path(layout), - "layout_output_sha256": sha256_bytes(layout_result.stdout), - "unaligned_endian_probe_sha256": sha256_path(unaligned), - "unaligned_endian_assembly_sha256": sha256_path(assembly), - } - - -def render_argv(values: list[str], substitutions: dict[str, str]) -> list[str]: - rendered = [] - for value in values: - for key, replacement in substitutions.items(): - value = value.replace("{" + key + "}", replacement) - rendered.append(value) - return rendered - - -def run_native( - architecture: dict[str, Any], - declaration: dict[str, Any], - modules: dict[str, dict[str, Any]], - resolved_base: str, - patch_sha256: str, -) -> dict[str, Any]: - runner_relative = safe_relative_path(declaration.get("runner_path", "")) - runner = ROOT / runner_relative - if not runner.is_file(): - raise GateFailure("INFRA_BLOCKED", f"declared native runner is absent: {runner_relative}") - if sha256_path(runner) != validate_hash( - declaration.get("runner_sha256"), f"{architecture['id']} runner" - ): - raise GateFailure("INFRA_BLOCKED", "declared native runner identity changed") - native_output = OUTPUT / "native" / architecture["id"] - native_output.mkdir(parents=True) - substitutions = { - "runner": str(runner), - "architecture": architecture["id"], - "module_zstdio0": modules["zstdio0"]["path"], - "module_zstdio1": modules["zstdio1"]["path"], - "output": str(native_output), - } - preflight_argv = render_argv(declaration["preflight_argv"], substitutions) - preflight = run( - preflight_argv, - cwd=ROOT, - timeout_seconds=declaration["preflight_deadline_seconds"], - ) - (native_output / "preflight.runner.stdout").write_bytes(preflight.stdout) - (native_output / "preflight.runner.stderr").write_bytes(preflight.stderr) - preflight_record = json.loads( - (native_output / "preflight.json").read_text(encoding="utf-8") - ) - for key, value in { - "status": "PASS", "exit_code": 0, "target_marker": "reached", - "cleanup": "PASS", "uname_s": "FreeBSD", - "uname_m": architecture["expected_uname_m"], - "uname_p": architecture["expected_uname_p"], - }.items(): - if preflight_record.get(key) != value: - raise GateFailure("INFRA_BLOCKED", f"native preflight mismatch: {key}") - runtime_argv = render_argv(declaration["runtime_argv"], substitutions) - runtime = run( - runtime_argv, - cwd=ROOT, - allowed={0, 10}, - timeout_seconds=declaration["runtime_deadline_seconds"], - ) - (native_output / "runner.stdout").write_bytes(runtime.stdout) - (native_output / "runner.stderr").write_bytes(runtime.stderr) - return validate_runtime_record( - native_output / "runtime-result.json", - architecture, - modules, - resolved_base, - patch_sha256, - ) - - -def main() -> int: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-027": - raise GateFailure("RUNNER_FAIL", "invalid P15-027 input identity") - if SPEC.get("gate") != "G07" or SPEC.get("batch") != "B36": - raise GateFailure("RUNNER_FAIL", "invalid G07/B36 input identity") - if SPEC.get("prohibited_paths") != ["introduction.md"]: - raise GateFailure("RUNNER_FAIL", "prohibited-path contract changed") - resolved = git_text(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise GateFailure( - "INFRA_BLOCKED", f"P15-027 must replay {SPEC['required_base']}, got {resolved}" - ) - source_tree = git_text(ROOT, "rev-parse", f"{resolved}:repo-pre-15/src") - if source_tree != SPEC["repo_source_tree_oid"]: - raise GateFailure("INFRA_BLOCKED", "frozen EROFS source tree changed") - source_hashes = { - path: sha256_bytes(source_at(resolved, path)) - for path in SPEC["source_sha256"] - } - if source_hashes != SPEC["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "frozen gate source identity changed") - write_json(OUTPUT / "source-sha256.json", source_hashes) - - addendum_paths = ( - "repo-pre-15/tests/pre15/gates/P15-027.sh", - "repo-pre-15/tests/pre15/gates/P15-027-input.json", - "repo-pre-15/docs/pre15-stage0/P15-027.md", - ) - addendum_hashes = { - path: sha256_path(ROOT / path) for path in addendum_paths - } - write_json(OUTPUT / "gate-addendum-sha256.json", addendum_hashes) - if sha256_path(ROOT / "repo-pre-15/src/Makefile") != SPEC["source_sha256"]["repo-pre-15/src/Makefile"]: - raise GateFailure("RUNNER_FAIL", "production Makefile differs from frozen BASE") - - patch_spec = SPEC["temporary_makefile_relaxation"] - patch_bytes = patch_spec["unified_diff"].encode("ascii") - if sha256_bytes(patch_bytes) != patch_spec["sha256"]: - raise GateFailure("RUNNER_FAIL", "input-recorded temporary patch hash mismatch") - patch_check = WORK / "patch-check" - makefile_copy = patch_check / "repo-pre-15/src/Makefile" - makefile_copy.parent.mkdir(parents=True) - makefile_copy.write_bytes(source_at(resolved, patch_spec["path"])) - patch_result = run(["patch", "-p1"], cwd=patch_check, input_bytes=patch_bytes) - (OUTPUT / "makefile-relaxation.patch").write_bytes(patch_bytes) - (OUTPUT / "patch.stdout").write_bytes(patch_result.stdout) - (OUTPUT / "patch.stderr").write_bytes(patch_result.stderr) - if sha256_path(makefile_copy) != patch_spec["patched_makefile_sha256"]: - raise GateFailure("RUNNER_FAIL", "temporary Makefile relaxation result changed") - patch_record = { - "path": patch_spec["path"], - "sha256": patch_spec["sha256"], - "patched_makefile_sha256": sha256_path(makefile_copy), - "production_makefile_changed": False, - "temporary_apply": "PASS", - } - write_json(OUTPUT / "temporary-patch.json", patch_record) - - if not FREEBSD_SRC.is_dir(): - raise GateFailure("INFRA_BLOCKED", f"FreeBSD source is absent: {FREEBSD_SRC}") - freebsd = SPEC["freebsd_source"] - freebsd_head = git_text(FREEBSD_SRC, "rev-parse", "HEAD") - freebsd_prefix = git_text(FREEBSD_SRC, "rev-parse", "--show-prefix") - if freebsd_head != freebsd["head"] or freebsd_prefix != freebsd["git_prefix"]: - raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed") - git_root = Path(git_text(FREEBSD_SRC, "rev-parse", "--show-toplevel")) - freebsd_source_tree = git_text( - git_root, "rev-parse", f"{freebsd_head}:{freebsd['git_tree_path']}" - ) - freebsd_sys_tree = git_text( - git_root, "rev-parse", f"{freebsd_head}:{freebsd['git_tree_path']}/sys" - ) - if freebsd_source_tree != freebsd["source_tree_oid"] or freebsd_sys_tree != freebsd["sys_tree_oid"]: - raise GateFailure("INFRA_BLOCKED", "FreeBSD source tree object changed") - freebsd_hashes = { - path: sha256_path(FREEBSD_SRC / path) for path in freebsd["sha256"] - } - if freebsd_hashes != freebsd["sha256"]: - raise GateFailure("INFRA_BLOCKED", "selected FreeBSD source hashes changed") - - compiler_path = Path(shutil.which("clang") or "").resolve() - if not compiler_path.is_file(): - raise GateFailure("INFRA_BLOCKED", "clang is unavailable") - compiler = SPEC["toolchain"]["compiler"] - compiler_version = run([str(compiler_path), "--version"]).stdout.decode( - "utf-8", errors="strict" - ).splitlines()[0] - if ( - str(compiler_path) != compiler["realpath"] - or sha256_path(compiler_path) != compiler["sha256"] - or compiler_version != compiler["version"] - ): - raise GateFailure("INFRA_BLOCKED", "cross compiler identity changed") - compiler_targets = run([str(compiler_path), "--print-targets"]).stdout.decode( - "utf-8", errors="strict" - ) - - architectures = SPEC["architectures"] - if [item.get("id") for item in architectures] != ["arm64", "riscv64", "i386"]: - raise GateFailure("RUNNER_FAIL", "architecture audit set is not exact") - if sum(item.get("pointer_bits") == 32 for item in architectures) < 1: - raise GateFailure("RUNNER_FAIL", "architecture audit lacks a 32-bit target") - source_support = {} - for architecture in architectures: - arch_id = architecture["id"] - header = FREEBSD_SRC / "sys" / architecture["machine_header_dir"] / "include/param.h" - if not header.is_file(): - raise GateFailure("INFRA_BLOCKED", f"FreeBSD source lacks {arch_id}") - if sha256_path(header) != architecture["param_h_sha256"]: - raise GateFailure("INFRA_BLOCKED", f"FreeBSD {arch_id} source identity changed") - target_pattern = rf"^\s*{re.escape(architecture['compiler_backend'])}\s+-" - if re.search(target_pattern, compiler_targets, re.MULTILINE) is None: - raise GateFailure("INFRA_BLOCKED", f"clang lacks {arch_id} backend") - source_support[arch_id] = { - "status": "PASS", - "freebsd_machine_headers": str(header), - "param_h_sha256": sha256_path(header), - "compiler_backend": architecture["compiler_backend"], - "target_triple": architecture["target_triple"], - "pointer_bits": architecture["pointer_bits"], - } - - probe_source = SPEC["cross_build_contract"]["unaligned_endian_probe_source"].encode("ascii") - if sha256_bytes(probe_source) != SPEC["cross_build_contract"]["unaligned_endian_probe_sha256"]: - raise GateFailure("RUNNER_FAIL", "input-recorded unaligned/endian probe changed") - inventory = { - "schema": 1, - "host": {"system": platform.system(), "machine": platform.machine()}, - "local_host_is_qualifying_native_freebsd": False, - "local_host_reason": "host is not FreeBSD and is not any candidate architecture", - "declared_native_environments": { - item["id"]: item["native_environment"] is not None for item in architectures - }, - "emulator_tools": { - item["id"]: shutil.which(item["emulator_tool"]) for item in architectures - }, - "emulator_binary_is_native_runtime_evidence": False, - "source_toolchain_support": source_support, - "protected_resources_addressed": False, - "inventory_rule": "only an explicit, validated same-architecture FreeBSD runner declaration can start cross work", - } - write_json(OUTPUT / "runtime-environment-inventory.json", inventory) - write_json( - OUTPUT / "freebsd-source-identity.json", - { - "path": str(FREEBSD_SRC), - "git_root": str(git_root), - "git_prefix": freebsd_prefix, - "git_tree_path": freebsd["git_tree_path"], - "head": freebsd_head, - "source_tree_oid": freebsd_source_tree, - "sys_tree_oid": freebsd_sys_tree, - "sha256": freebsd_hashes, - }, - ) - write_json( - OUTPUT / "toolchain-identity.json", - { - "compiler_realpath": str(compiler_path), - "compiler_sha256": sha256_path(compiler_path), - "compiler_version": compiler_version, - "print_targets_sha256": sha256_bytes(compiler_targets.encode("utf-8")), - }, - ) - - results = [] - go_architectures = [] - for architecture in architectures: - declaration = architecture["native_environment"] - record: dict[str, Any] = { - "architecture": architecture["id"], - "machine_arch": architecture["machine_arch"], - "pointer_bits": architecture["pointer_bits"], - "source_toolchain": source_support[architecture["id"]], - } - if declaration is None: - record.update( - { - "native_environment": { - "status": "STOP", - "reason": "no declared same-architecture FreeBSD runtime environment", - }, - "cross_build_and_probes": { - "status": "NOT_RUN", - "reason": "cross-only evidence cannot authorize allowlisting", - }, - "native_runtime": { - "status": "NOT_RUN", - "reason": "same-architecture FreeBSD runtime is absent", - }, - "decision": "STOP", - } - ) - results.append(record) - continue - for key in ( - "runner_path", "runner_sha256", "preflight_argv", "runtime_argv", - "preflight_deadline_seconds", "runtime_deadline_seconds", - ): - if key not in declaration: - raise GateFailure("RUNNER_FAIL", f"incomplete native declaration: {key}") - cross = compile_architecture( - architecture, resolved, patch_bytes, compiler_path - ) - native = run_native( - architecture, - declaration, - cross["modules"], - resolved, - patch_spec["sha256"], - ) - native_pass = native["status"] == "PASS" - record.update( - { - "native_environment": {"status": "PASS", "declaration": declaration}, - "cross_build_and_probes": cross, - "native_runtime": native, - "decision": "GO" if native_pass else "STOP", - } - ) - if native_pass: - go_architectures.append(architecture["id"]) - results.append(record) - - overall_status = "GO" if go_architectures else "STOP" - result = { - "schema": 1, - "gate": "G07", - "candidate": "P15-027", - "batch": "B36", - "status": overall_status, - "reason": ( - "at least one architecture completed its own cross and native requirements" - if go_architectures - else "no architecture has a declared same-architecture FreeBSD runtime" - ), - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "repo_source_tree_oid": source_tree, - "deadline_seconds": DEADLINE, - "expected_exit_code": 0 if go_architectures else 22, - "architectures": results, - "go_architectures": go_architectures, - "allowlist_authorization": go_architectures, - "false_go_guard": "each authorized architecture must independently have cross_build_and_probes=PASS and native_runtime=PASS", - "b36": "AUTHORIZED" if go_architectures else "STOP-NO-SOURCE", - "production_makefile_guard_preserved": True, - "production_source_changed": False, - "qemu": "NOT_RUN" if not go_architectures else "runner-defined", - "full_feature_suite": "NOT_RUN", - "smoke_suite": "NOT_RUN", - "protected_resources_addressed": False, - } - write_json(OUTPUT / "result.json", result) - print(json.dumps(result, ensure_ascii=True, sort_keys=True)) - return 0 if go_architectures else 22 - - -try: - raise SystemExit(main()) -except GateFailure as error: - failure = { - "schema": 1, - "gate": "G07", - "candidate": "P15-027", - "status": error.status, - "reason": error.reason, - "requested_base": REQUESTED_BASE, - "deadline_seconds": DEADLINE, - "b36": "NOT_RUN", - "target_marker": "not_reached", - "production_source_changed": False, - "protected_resources_addressed": False, - } - write_json(OUTPUT / "result.json", failure) - print(json.dumps(failure, ensure_ascii=True, sort_keys=True)) - raise SystemExit({"RUNNER_FAIL": 20, "INFRA_BLOCKED": 21}.get(error.status, 20)) -PY -gate_rc=$? -set -e - -cleanup_status=PASS -if ! rm -rf -- "$work"; then - cleanup_status=FAIL - gate_rc=20 -fi -trap - EXIT HUP INT TERM -printf 'owned temporary path removed: %s\ncleanup=%s\n' \ - "$work" "$cleanup_status" >"$output/cleanup.log" - -python3 -B - "$output" "$gate_rc" "$cleanup_status" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import sys - - -output = Path(sys.argv[1]) -exit_code = int(sys.argv[2]) -cleanup = sys.argv[3] -if cleanup != "PASS": - status = "RUNNER_FAIL" - origin = "runner" -elif exit_code == 0: - status = "PASS" - origin = "gate" -elif exit_code == 22: - status = "STOP" - origin = "gate" -elif exit_code in (124, 137): - status = "INFRA_BLOCKED" - origin = "infrastructure" -else: - status = "RUNNER_FAIL" - origin = "runner" -(output / "attempt.json").write_text( - json.dumps( - { - "schema": 1, - "exit_code": exit_code, - "status": status, - "failure_origin": origin, - "cleanup": cleanup, - "target_marker": "reached" if status in {"PASS", "STOP"} else "not_reached", - }, - ensure_ascii=True, - indent=2, - sort_keys=True, - ) + "\n", - encoding="ascii", -) -lines = [] -for path in sorted(output.iterdir(), key=lambda item: item.name): - if path.is_file() and path.name != "SHA256SUMS": - digest = hashlib.sha256(path.read_bytes()).hexdigest() - lines.append(f"{digest} {path.name}") -(output / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") -PY - -printf 'P15-027 gate exit=%s cleanup=%s output=%s\n' \ - "$gate_rc" "$cleanup_status" "$output" -exit "$gate_rc" diff --git a/tests/pre15/gates/P15-030-input.json b/tests/pre15/gates/P15-030-input.json deleted file mode 100644 index 178d096..0000000 --- a/tests/pre15/gates/P15-030-input.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "approved_consumer_manifest_sha256": {}, - "batch": "B37a", - "candidate": "P15-030", - "consumer_contract": { - "allowed_classes": [ - "diagnostic", - "operational" - ], - "allowed_signal_ids": [ - "features", - "cache_hits", - "cache_misses", - "cache_waits", - "decoder_errors" - ], - "nonqualifying_examples": [ - "hypothetical future user", - "Linux sysfs analogy", - "generic observability value", - "test script", - "gate prototype" - ], - "prohibited_implementation_prefixes": [ - "planning/", - "repo-pre-15/docs/", - "repo-pre-15/tests/" - ], - "required_access_mode": "privileged-read", - "required_fields": [ - "schema", - "consumer_id", - "consumer_class", - "owner", - "version", - "production_use", - "implementation_path", - "implementation_sha256", - "entrypoint", - "workflow", - "deployment_reference", - "signal_ids", - "access_mode" - ], - "schema_marker": "pre15-p15-030-consumer-v1" - }, - "consumer_inventory": { - "expected_reference_counts": { - "historical-evidence": 1, - "planning-proposal": 31, - "possible-operational-consumer": 0, - "project-report-or-documentation": 3, - "test-only": 19, - "total": 54 - }, - "expected_scoped_path_count": 3206, - "maximum_candidate_bytes": 2097152, - "reference_terms": [ - "sysctl", - "sysfs" - ], - "scope": "tracked repo-pre-15 and planning/pre15 at required_base, excluding planning/pre15/introduction.md and the P15-030 addendum" - }, - "dependency_rule": { - "batch": "B37b", - "candidate": "P15-068", - "identity_format_verdict_when_transport_stops": "NOT_RUN", - "on_p15_030_stop": "STOP-NO-SOURCE", - "reason": "P15-068 requires the P15-030 sysctl transport; do not create a second lifecycle" - }, - "gate": "G08", - "go_requirements": [ - "concrete existing diagnostic or operational consumer", - "bounded atomic counters", - "FreeBSD 15 native sysctl_ctx lifecycle closes parse failure, normal unmount, forced unmount, delayed handler, and sysctl_ctx_free failure without freeing handler-visible mount state", - "stable permissions and ABI with no unauthenticated metadata leak" - ], - "probe_order": [ - "consumer-inventory", - "bounded-atomic-counter-prototype", - "freebsd15-native-sysctl-ctx-lifecycle", - "permissions-abi-metadata-leak" - ], - "prohibited_paths": [ - "planning/pre15/introduction.md" - ], - "protected": { - "pid": 26318, - "port": 9222 - }, - "required_base": "3e1d26a53eef30ecadc3407444d214feac861cb4", - "schema": 1, - "scope_tree_oids": { - "planning/pre15": "b94ad587533ff9fb14e434a8db061381af0ab713", - "repo-pre-15": "c2e260c5f9e340e28e13481b3e5a3d04b5b916c8" - }, - "source_sha256": { - "planning/pre15/20-final-candidate-ledger.md": "e40f68de241a9c4e5e5bff0849b166b614f56e6cba0b64fc3ced41c036c99c91", - "planning/pre15/30-stage0-gates.md": "34708bacf4a0668dfd834b3639900ca23b28b9833a387ac6e2c0399c9316cdb8", - "planning/pre15/40-execution-batches.md": "1d1ffdf8fec799aeec063bb1dc4a0fda9703300d414eabef8905522b2c18db20", - "planning/pre15/90-honesty-and-evidence.md": "69f96908f5ab436a2f5359548e4e6f5dc2942fcf00148d999d513558f1c59c2d", - "repo-pre-15/README.md": "2d3e138f5af20ce02aa562286cb42df8fbd00fdad0ae424e426e0fd8a7f72472", - "repo-pre-15/docs/capabilities.md": "f4b2fc0ff74fb7f808ed0f833e87c8230877ab249b611d30a1dbc2103535a8b8", - "repo-pre-15/docs/erofs.5": "dea6a9eddbccbb7b8779b6b4e43772edeea9f0e51696c1b8fd09e4d8e69b52f5", - "repo-pre-15/docs/features.md": "6d875eb582ef9f00391f90ee5b4b8aec0429f24f0ae3c7b27e788ab1e2c48f2e", - "repo-pre-15/src/Makefile": "b722f7ec658e0ada0640c80a7b527921bd96cdf0e93e9623cece29766f2fc9c8", - "repo-pre-15/src/internal.h": "3fe8d6ca819f0a1c472cc4b173da80027f957b7f7c25e51ef7fd85b60480c98a", - "repo-pre-15/src/super.c": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2" - } -} diff --git a/tests/pre15/gates/P15-030.sh b/tests/pre15/gates/P15-030.sh deleted file mode 100755 index cc7f43a..0000000 --- a/tests/pre15/gates/P15-030.sh +++ /dev/null @@ -1,592 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-030-input.json -base= -output= -deadline=60 - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 20; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 20; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 20 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 20; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 20; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 20; } -for tool in git mktemp python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 21 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 20; } -mkdir -p "$output" -work=$(mktemp -d "${TMPDIR:-/tmp}/P15-030.XXXXXX") - -cleanup_trap() -{ - rm -rf -- "$work" -} -trap cleanup_trap EXIT HUP INT TERM - -python3 -B - "$output/command-argv.json" "$0" --base "$base" --output "$output" <<'PY' -import json -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text( - json.dumps(sys.argv[2:], ensure_ascii=True, separators=(",", ":")) + "\n", - encoding="ascii", -) -PY -python3 -B - "$output/ownership.json" "$work" "$output" <<'PY' -import json -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text( - json.dumps( - { - "owned_temporary_paths": [sys.argv[2]], - "persistent_output": sys.argv[3], - "owned_processes": [], - "owned_ports": [], - "owned_overlays": [], - "qemu": "NOT_RUN", - }, - ensure_ascii=True, - indent=2, - sort_keys=True, - ) + "\n", - encoding="ascii", -) -PY - -set +e -timeout -k 5 "$deadline" python3 -B - \ - "$root" "$input" "$base" "$output" "$deadline" <<'PY' \ - >"$output/stdout.log" 2>"$output/stderr.log" -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import subprocess -import sys -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -DEADLINE = int(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="utf-8")) - - -class GateFailure(RuntimeError): - def __init__(self, status: str, reason: str): - super().__init__(reason) - self.status = status - self.reason = reason - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def write_json(path: Path, value: Any) -> None: - path.write_text( - json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", - ) - - -def run_bytes(argv: list[str], allowed: set[int] | None = None) -> bytes: - try: - completed = subprocess.run( - argv, - cwd=ROOT, - check=False, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=20, - ) - except subprocess.TimeoutExpired as error: - raise GateFailure( - "INFRA_BLOCKED", f"command timed out: {' '.join(argv)}" - ) from error - accepted = {0} if allowed is None else allowed - if completed.returncode not in accepted: - detail = completed.stderr.decode("utf-8", errors="replace").strip() - raise GateFailure( - "RUNNER_FAIL", - f"command failed ({completed.returncode}): {' '.join(argv)}: {detail}", - ) - return completed.stdout - - -def git_text(*args: str) -> str: - return run_bytes(["git", "-C", str(ROOT), *args]).decode( - "utf-8", errors="strict" - ).strip() - - -def source_at(commit: str, relative: str) -> bytes: - return run_bytes(["git", "-C", str(ROOT), "show", f"{commit}:{relative}"]) - - -def parse_ls_tree(raw: bytes) -> dict[str, dict[str, Any]]: - entries: dict[str, dict[str, Any]] = {} - for record in raw.split(b"\0"): - if not record: - continue - metadata, path_bytes = record.split(b"\t", 1) - mode, kind, oid, size = metadata.decode("ascii").split() - path = path_bytes.decode("utf-8", errors="strict") - entries[path] = { - "mode": mode, - "kind": kind, - "oid": oid, - "size": int(size), - } - return entries - - -def grep_paths(commit: str, expressions: list[str], fixed: bool = False) -> list[str]: - argv = ["git", "-C", str(ROOT), "grep", "-I", "-l"] - if fixed: - argv.append("-F") - else: - argv.append("-i") - for expression in expressions: - argv.extend(["-e", expression]) - argv.extend( - [ - commit, - "--", - "repo-pre-15", - "planning/pre15", - ":(exclude)planning/pre15/introduction.md", - ":(exclude)repo-pre-15/tests/pre15/gates/P15-030.sh", - ":(exclude)repo-pre-15/tests/pre15/gates/P15-030-input.json", - ":(exclude)repo-pre-15/docs/pre15-stage0/P15-030.md", - ] - ) - raw = run_bytes(argv, {0, 1}) - prefix = f"{commit}:" - paths = [] - for line in raw.decode("utf-8", errors="strict").splitlines(): - if not line.startswith(prefix): - raise GateFailure("RUNNER_FAIL", f"unexpected git grep path: {line}") - paths.append(line[len(prefix):]) - return sorted(paths) - - -def classify_reference(path: str) -> str: - if path.startswith("planning/pre15/evidence/"): - return "historical-evidence" - if path.startswith("planning/pre15/"): - return "planning-proposal" - if path.startswith("repo-pre-15/tests/"): - return "test-only" - if path.startswith("repo-pre-15/docs/") or path.startswith("repo-pre-15/current/"): - return "project-report-or-documentation" - return "possible-operational-consumer" - - -def require_text(record: dict[str, Any], field: str) -> str: - value = record.get(field) - if not isinstance(value, str) or not value.strip(): - raise GateFailure("RUNNER_FAIL", f"consumer declaration lacks {field}") - return value.strip() - - -def validate_consumer_manifest( - path: str, - data: bytes, - entries: dict[str, dict[str, Any]], - commit: str, -) -> dict[str, Any]: - try: - record = json.loads(data.decode("utf-8", errors="strict")) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - raise GateFailure("RUNNER_FAIL", f"invalid consumer manifest: {path}") from error - if not isinstance(record, dict): - raise GateFailure("RUNNER_FAIL", f"consumer manifest is not an object: {path}") - contract = SPEC["consumer_contract"] - if record.get("schema") != contract["schema_marker"]: - raise GateFailure("RUNNER_FAIL", f"consumer manifest schema mismatch: {path}") - consumer_class = require_text(record, "consumer_class") - if consumer_class not in contract["allowed_classes"]: - raise GateFailure("RUNNER_FAIL", f"consumer class is not operational: {path}") - if record.get("production_use") is not True: - raise GateFailure("RUNNER_FAIL", f"consumer is not declared in production use: {path}") - access_mode = require_text(record, "access_mode") - if access_mode != contract["required_access_mode"]: - raise GateFailure("RUNNER_FAIL", f"consumer access mode is not privileged: {path}") - for field in ( - "consumer_id", - "owner", - "version", - "entrypoint", - "workflow", - "deployment_reference", - ): - require_text(record, field) - signals = record.get("signal_ids") - if ( - not isinstance(signals, list) - or not signals - or any(not isinstance(item, str) for item in signals) - or len(set(signals)) != len(signals) - or not set(signals).issubset(set(contract["allowed_signal_ids"])) - ): - raise GateFailure("RUNNER_FAIL", f"consumer signal set is invalid: {path}") - implementation_path = require_text(record, "implementation_path") - if not implementation_path.startswith("repo-pre-15/"): - raise GateFailure("RUNNER_FAIL", f"consumer implementation is outside DUT: {path}") - if any( - implementation_path.startswith(prefix) - for prefix in contract["prohibited_implementation_prefixes"] - ): - raise GateFailure("RUNNER_FAIL", f"consumer implementation is nonqualifying: {path}") - if implementation_path not in entries: - raise GateFailure("RUNNER_FAIL", f"consumer implementation is not tracked: {path}") - implementation = source_at(commit, implementation_path) - implementation_hash = sha256_bytes(implementation) - if implementation_hash != require_text(record, "implementation_sha256"): - raise GateFailure("RUNNER_FAIL", f"consumer implementation hash mismatch: {path}") - implementation_text = implementation.decode("utf-8", errors="replace").lower() - if "sysctl" not in implementation_text: - raise GateFailure("RUNNER_FAIL", f"consumer implementation does not invoke sysctl: {path}") - missing_signals = [item for item in signals if item.lower() not in implementation_text] - if missing_signals: - raise GateFailure( - "RUNNER_FAIL", f"consumer implementation omits signals {missing_signals}: {path}" - ) - return { - "manifest_path": path, - "manifest_sha256": sha256_bytes(data), - "consumer_id": record["consumer_id"], - "consumer_class": consumer_class, - "implementation_path": implementation_path, - "implementation_sha256": implementation_hash, - "signal_ids": signals, - "access_mode": access_mode, - "production_use": True, - } - - -def main() -> int: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-030": - raise GateFailure("RUNNER_FAIL", "invalid P15-030 input identity") - if SPEC.get("gate") != "G08" or SPEC.get("batch") != "B37a": - raise GateFailure("RUNNER_FAIL", "invalid G08/B37a input identity") - resolved = git_text("rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise GateFailure( - "INFRA_BLOCKED", - f"P15-030 must replay {SPEC['required_base']}, got {resolved}", - ) - for path, expected_oid in SPEC["scope_tree_oids"].items(): - actual_oid = git_text("rev-parse", f"{resolved}:{path}") - if actual_oid != expected_oid: - raise GateFailure("INFRA_BLOCKED", f"frozen scope tree changed: {path}") - - source_hashes = { - path: sha256_bytes(source_at(resolved, path)) - for path in SPEC["source_sha256"] - } - if source_hashes != SPEC["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "frozen G08 source identity changed") - write_json(OUTPUT / "source-sha256.json", source_hashes) - addendum_paths = ( - "repo-pre-15/tests/pre15/gates/P15-030.sh", - "repo-pre-15/tests/pre15/gates/P15-030-input.json", - "repo-pre-15/docs/pre15-stage0/P15-030.md", - ) - addendum_hashes = { - path: sha256_bytes((ROOT / path).read_bytes()) for path in addendum_paths - } - write_json(OUTPUT / "gate-addendum-sha256.json", addendum_hashes) - - entries = parse_ls_tree( - run_bytes( - [ - "git", - "-C", - str(ROOT), - "ls-tree", - "-r", - "--long", - "-z", - resolved, - "--", - "repo-pre-15", - "planning/pre15", - ] - ) - ) - prohibited = set(SPEC["prohibited_paths"]) - scoped_entries = {path: value for path, value in entries.items() if path not in prohibited} - if len(scoped_entries) != SPEC["consumer_inventory"]["expected_scoped_path_count"]: - raise GateFailure("INFRA_BLOCKED", "frozen consumer scope path count changed") - - reference_paths = grep_paths( - resolved, SPEC["consumer_inventory"]["reference_terms"] - ) - maximum_bytes = SPEC["consumer_inventory"]["maximum_candidate_bytes"] - references = [] - reference_counts = { - "planning-proposal": 0, - "historical-evidence": 0, - "test-only": 0, - "project-report-or-documentation": 0, - "possible-operational-consumer": 0, - } - for path in reference_paths: - if path in prohibited: - raise GateFailure("RUNNER_FAIL", "prohibited path entered consumer scan") - entry = scoped_entries[path] - if entry["size"] > maximum_bytes: - raise GateFailure("INFRA_BLOCKED", f"consumer candidate exceeds bound: {path}") - data = source_at(resolved, path) - classification = classify_reference(path) - reference_counts[classification] += 1 - references.append( - { - "path": path, - "blob_oid": entry["oid"], - "bytes": entry["size"], - "sha256": sha256_bytes(data), - "classification": classification, - "qualifies": False, - "reason": { - "planning-proposal": "proposal or audit text is not an existing consumer", - "historical-evidence": "historical gate output is not an existing consumer", - "test-only": "a test or gate is explicitly nonqualifying", - "project-report-or-documentation": "documentation is not an implemented consumer", - "possible-operational-consumer": "requires a validated consumer declaration", - }[classification], - } - ) - expected_counts = SPEC["consumer_inventory"]["expected_reference_counts"] - if len(reference_paths) != expected_counts["total"]: - raise GateFailure("INFRA_BLOCKED", "frozen sysctl/sysfs reference count changed") - for classification, count in reference_counts.items(): - if count != expected_counts[classification]: - raise GateFailure( - "INFRA_BLOCKED", f"frozen reference classification changed: {classification}" - ) - - marker = SPEC["consumer_contract"]["schema_marker"] - manifest_paths = grep_paths(resolved, [marker], fixed=True) - consumers = [] - manifest_hashes = {} - for path in manifest_paths: - entry = scoped_entries[path] - if entry["size"] > maximum_bytes: - raise GateFailure("INFRA_BLOCKED", f"consumer manifest exceeds bound: {path}") - data = source_at(resolved, path) - manifest_hashes[path] = sha256_bytes(data) - consumers.append(validate_consumer_manifest(path, data, scoped_entries, resolved)) - if manifest_hashes != SPEC["approved_consumer_manifest_sha256"]: - raise GateFailure("INFRA_BLOCKED", "declared consumer inventory changed") - - inventory = { - "schema": 1, - "base": resolved, - "scope": SPEC["consumer_inventory"]["scope"], - "scoped_path_count": len(scoped_entries), - "prohibited_path_read": False, - "reference_terms": SPEC["consumer_inventory"]["reference_terms"], - "reference_path_count": len(reference_paths), - "reference_classification_counts": reference_counts, - "references": references, - "consumer_manifest_marker": marker, - "consumer_manifest_count": len(consumers), - "qualified_consumer_count": len(consumers), - "qualified_consumers": consumers, - "nonqualifying_examples": SPEC["consumer_contract"]["nonqualifying_examples"], - } - write_json(OUTPUT / "consumer-inventory.json", inventory) - - consumer_present = bool(consumers) - conditions = [ - { - "id": "concrete-existing-diagnostic-or-operational-consumer", - "status": "PASS" if consumer_present else "STOP", - "observation": ( - f"{len(consumers)} qualified consumer declaration(s)" - if consumer_present - else "zero qualified consumers; all 54 broad references are planning, history, tests, or documentation" - ), - }, - { - "id": "bounded-atomic-counters", - "status": "NOT_RUN", - "reason": "existing-consumer prerequisite is absent", - }, - { - "id": "freebsd15-native-sysctl-ctx-lifecycle", - "status": "NOT_RUN", - "reason": "existing-consumer prerequisite is absent; no prototype was built", - }, - { - "id": "stable-permissions-abi-and-no-unauthenticated-metadata-leak", - "status": "NOT_RUN", - "reason": "existing-consumer prerequisite is absent", - }, - ] - if consumer_present: - raise GateFailure( - "RUNNER_FAIL", - "a consumer is now declared; run the FreeBSD 15 native lifecycle prototype before any GO", - ) - if all(item["status"] == "PASS" for item in conditions): - raise GateFailure("RUNNER_FAIL", "false-GO guard accepted incomplete conditions") - - result = { - "schema": 1, - "gate": "G08", - "candidate": "P15-030", - "batch": "B37a", - "status": "STOP", - "reason": "no concrete existing diagnostic or operational consumer", - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "deadline_seconds": DEADLINE, - "expected_exit_code": 22, - "conditions": conditions, - "false_go_guard": "GO requires all four conditions PASS; evaluated false", - "consumer_inventory": "consumer-inventory.json", - "b37a": "STOP-NO-SOURCE", - "dependency_closure": { - "candidate": "P15-068", - "batch": "B37b", - "status": "STOP", - "batch_status": "STOP-NO-SOURCE", - "reason": "P15-068 requires the P15-030 sysctl transport, which is STOP", - "independent_identity_format_verdict": "NOT_RUN", - "second_sysctl_lifecycle": "NOT_RUN", - }, - "prototype": "NOT_RUN", - "builds": "NOT_RUN", - "qemu": "NOT_RUN", - "full_feature_suite": "NOT_RUN", - "smoke_suite": "NOT_RUN", - "production_source_changed": False, - "p15_068_gate_addendum_created": False, - } - write_json(OUTPUT / "result.json", result) - print(json.dumps(result, ensure_ascii=True, sort_keys=True)) - return 22 - - -try: - raise SystemExit(main()) -except GateFailure as error: - failure = { - "schema": 1, - "gate": "G08", - "candidate": "P15-030", - "status": error.status, - "reason": error.reason, - "requested_base": REQUESTED_BASE, - "deadline_seconds": DEADLINE, - "b37a": "NOT_RUN", - "p15_068": "NOT_RUN", - "qemu": "NOT_RUN", - "production_source_changed": False, - } - write_json(OUTPUT / "result.json", failure) - print(json.dumps(failure, ensure_ascii=True, sort_keys=True)) - raise SystemExit({"RUNNER_FAIL": 20, "INFRA_BLOCKED": 21}.get(error.status, 20)) -PY -gate_rc=$? -set -e - -cleanup_status=PASS -if ! rm -rf -- "$work"; then - cleanup_status=FAIL - gate_rc=20 -fi -trap - EXIT HUP INT TERM -printf 'owned temporary path removed: %s\ncleanup=%s\n' \ - "$work" "$cleanup_status" >"$output/cleanup.log" - -python3 -B - "$output" "$gate_rc" "$cleanup_status" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import sys - - -output = Path(sys.argv[1]) -exit_code = int(sys.argv[2]) -cleanup = sys.argv[3] -if cleanup != "PASS": - status = "RUNNER_FAIL" - origin = "runner" -elif exit_code == 22: - status = "STOP" - origin = "gate" -elif exit_code in (124, 137): - status = "INFRA_BLOCKED" - origin = "infrastructure" -else: - status = "RUNNER_FAIL" if exit_code == 20 else "INFRA_BLOCKED" - origin = "runner" if exit_code == 20 else "infrastructure" -(output / "attempt.json").write_text( - json.dumps( - { - "schema": 1, - "exit_code": exit_code, - "status": status, - "failure_origin": origin, - "cleanup": cleanup, - "target_marker": "reached" if status == "STOP" else "not_reached", - }, - ensure_ascii=True, - indent=2, - sort_keys=True, - ) + "\n", - encoding="ascii", -) -lines = [] -for path in sorted(output.iterdir(), key=lambda item: item.name): - if path.is_file() and path.name != "SHA256SUMS": - digest = hashlib.sha256(path.read_bytes()).hexdigest() - lines.append(f"{digest} {path.name}") -(output / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") -PY - -printf 'P15-030 gate exit=%s cleanup=%s output=%s\n' \ - "$gate_rc" "$cleanup_status" "$output" -exit "$gate_rc" diff --git a/tests/pre15/gates/P15-031-input.json b/tests/pre15/gates/P15-031-input.json deleted file mode 100644 index cb31069..0000000 --- a/tests/pre15/gates/P15-031-input.json +++ /dev/null @@ -1,273 +0,0 @@ -{ - "approved_consumer_manifest_sha256": {}, - "batch": "B38", - "benchmark_contract": { - "control_drift_limit_percent": 3.0, - "hot_path_median_regression_limit_percent": 3.0, - "minimum_timed_iterations_per_sample": 100000, - "outlier_deletion": false, - "pair_order": "five AB and five BA pairs", - "raw_sample_retention": true, - "required_runs": 10, - "unstable_control_status": "INFRA_BLOCKED", - "warmup_runs": 3 - }, - "build_contract": { - "configurations": [ - "WITH_ZSTDIO=0", - "WITH_ZSTDIO=1" - ], - "disabled_probe_mode": true, - "new_undefined_symbols_allowed": 0 - }, - "candidate": "P15-031", - "consumer_contract": { - "allowed_transports": [ - "dtrace-sdt", - "ktr" - ], - "manifest_prefix": "repo-pre-15/diagnostics/", - "nonqualifying_examples": [ - "hypothetical maintainer", - "Linux tracepoints", - "generic observability value", - "proposed B38 test", - "gate prototype", - "installed dtrace without a project consumer" - ], - "prohibited_implementation_prefixes": [ - "planning/", - "repo-pre-15/current/", - "repo-pre-15/docs/", - "repo-pre-15/src/", - "repo-pre-15/tests/" - ], - "required_access_mode": "privileged-diagnostic", - "required_fields": [ - "schema", - "consumer_id", - "consumer_class", - "owner", - "version", - "production_use", - "freebsd_version", - "transport", - "access_mode", - "implementation_path", - "implementation_sha256", - "entrypoint", - "workflow", - "deployment_reference", - "event_schema_sha256", - "event_ids", - "actual_capture", - "capture_evidence_path", - "capture_evidence_sha256", - "captured_event_counts" - ], - "schema_marker": "pre15-p15-031-consumer-v1" - }, - "consumer_inventory": { - "expected_reference_counts": { - "historical-evidence": 3, - "linux-reference": 0, - "planning-proposal": 16, - "possible-project-consumer": 0, - "project-report-or-documentation": 4, - "test-only": 10, - "total": 33 - }, - "expected_scoped_path_count": 3243, - "maximum_candidate_bytes": 2097152, - "reference_patterns": [ - "dtrace", - "(^|[^[:alnum:]_])sdt([^[:alnum:]_]|$)|sdt_(probe|provider)", - "(^|[^[:alnum:]_])ktr([^[:alnum:]_]|$)|ktrdump|ktr_[[:alnum:]_]+", - "(^|[^[:alnum:]_])tracepoints?([^[:alnum:]_]|$)", - "(^|[^[:alnum:]_])observability([^[:alnum:]_]|$)" - ], - "scope": "tracked repo-pre-15 and planning/pre15 at required_base, excluding planning/pre15/introduction.md and the P15-031 addendum" - }, - "event_schema": { - "forbidden_fields": [ - "kernel_pointer", - "credential", - "path", - "xattr_name", - "xattr_value", - "uuid", - "volume_label", - "raw_unvalidated_metadata" - ], - "provider": "erofs", - "events": [ - { - "fields": [ - "parse_stage", - "error" - ], - "id": "mount_parse_fail", - "rule": "emit only bounded stage enum and positive errno; no image-derived value" - }, - { - "fields": [ - "nid", - "logical_offset", - "mapped_length", - "device_id", - "map_flags", - "error" - ], - "id": "map_result", - "rule": "emit only after range validation; zero mapping fields on validation failure" - }, - { - "fields": [ - "nid" - ], - "id": "vget_hit", - "rule": "validated numeric inode identity only" - }, - { - "fields": [ - "nid" - ], - "id": "vget_miss", - "rule": "validated numeric inode identity only" - }, - { - "fields": [ - "nid" - ], - "id": "vget_loser", - "rule": "validated numeric inode identity only" - }, - { - "fields": [ - "nid", - "name_index" - ], - "id": "xattr_cache_hit", - "rule": "namespace index only; no xattr name or value" - }, - { - "fields": [ - "nid", - "name_index" - ], - "id": "xattr_cache_miss", - "rule": "namespace index only; no xattr name or value" - }, - { - "fields": [ - "nid", - "algorithm", - "offset", - "input_length", - "output_length" - ], - "id": "decode_start", - "rule": "validated numeric decode request only" - }, - { - "fields": [ - "nid", - "algorithm", - "offset", - "decoded_length" - ], - "id": "decode_end", - "rule": "validated numeric decode result only" - }, - { - "fields": [ - "nid", - "algorithm", - "offset", - "error" - ], - "id": "decode_error", - "rule": "validated numeric identity and positive errno only" - }, - { - "fields": [ - "nid", - "offset", - "decoded_length" - ], - "id": "cache_wait", - "rule": "validated cache key components only" - }, - { - "fields": [ - "nid", - "offset", - "decoded_length", - "reason" - ], - "id": "cache_evict", - "rule": "bounded reason enum and validated cache key components only" - } - ] - }, - "gate": "G08", - "go_requirements": [ - "a concrete existing project diagnostic consumer captures the complete predeclared schema through FreeBSD DTrace/SDT or a justified KTR path", - "no kernel pointers, credentials, or unauthenticated metadata leakage", - "an owned-temp pre-source prototype generated from this JSON and the gate script closes fields, counts, failure, detach, and unload without linking into EROFS", - "disabled WITH_ZSTDIO=0 and WITH_ZSTDIO=1 builds add no undefined symbols", - "ten-run hot-path median regression is at most three percent with raw interleaved controls and no outlier deletion" - ], - "probe_order": [ - "consumer-inventory", - "privacy-schema", - "freebsd15-native-owned-temp-prototype", - "disabled-dual-build-undefined-symbols", - "ten-run-hot-path-microbenchmark" - ], - "prohibited_paths": [ - "planning/pre15/introduction.md" - ], - "protected": { - "pid": 26318, - "port": 9222 - }, - "prototype_contract": { - "expected_event_count_each": 1, - "generated_from": [ - "P15-031-input.json", - "P15-031.sh" - ], - "lifecycle_scenarios": [ - "consumer attach and complete capture", - "failed consumer enable", - "consumer termination and detach", - "provider unload and reload", - "final unload with zero owned processes, modules, and temporary paths" - ], - "linked_into_erofs": false, - "location": "owned temporary directory only", - "native_target": "FreeBSD 15" - }, - "required_base": "0f696ad1e2c6628022d02ce52d07eb66704769dd", - "schema": 1, - "scope_tree_oids": { - "planning/pre15": "bb8801a7696465fce1607b671c96473c044773b5", - "repo-pre-15": "231f9d5566d0a6bee4672a89f93451daa4f4b5de" - }, - "source_sha256": { - "planning/pre15/20-final-candidate-ledger.md": "c47338623686ce352fd80cf9b39635900bcde8c2abc960feebe24837f8a67460", - "planning/pre15/30-stage0-gates.md": "34708bacf4a0668dfd834b3639900ca23b28b9833a387ac6e2c0399c9316cdb8", - "planning/pre15/40-execution-batches.md": "1d1ffdf8fec799aeec063bb1dc4a0fda9703300d414eabef8905522b2c18db20", - "planning/pre15/50-feature-test-matrix.md": "ce39b44ef3989f4fcdbcd8eb677193ef53b630080c74d8539664fb2be3f38408", - "planning/pre15/90-honesty-and-evidence.md": "69f96908f5ab436a2f5359548e4e6f5dc2942fcf00148d999d513558f1c59c2d", - "repo-pre-15/src/Makefile": "b722f7ec658e0ada0640c80a7b527921bd96cdf0e93e9623cece29766f2fc9c8", - "repo-pre-15/src/data.c": "a511146d2d1ff56bc88bae6c550cfdc49a8e978c5c2c88d735de92adaaa836b6", - "repo-pre-15/src/decompressor.c": "ccaa934f837ecbf3b3b678dbddf35d29dd7e3b4e80d1a7ab78b18a07e555419f", - "repo-pre-15/src/inode.c": "50c40fe403bd06b17f6976707ad4a8f03f95af49753f450df3d132130ab2dcf4", - "repo-pre-15/src/super.c": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2", - "repo-pre-15/src/xattr.c": "7ce78b7af8838715fed10843f412458bdacb8c9b6811f08f660549f18a0743b0", - "repo-pre-15/src/zdata.c": "ee241c83fe7152569414594a3b4acf8c5028a35573d53ace0e3bb4a51dd0d191", - "repo-pre-15/tests/pre15/EVIDENCE-SCHEMA.json": "8a4d3fe076eed3a2f17e3ebd9337ca8697f09d6717c05835e076f6812cf38102" - } -} diff --git a/tests/pre15/gates/P15-031.sh b/tests/pre15/gates/P15-031.sh deleted file mode 100755 index 6efa7de..0000000 --- a/tests/pre15/gates/P15-031.sh +++ /dev/null @@ -1,704 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-031-input.json -base= -output= -deadline=60 - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 20; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 20; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 20 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 20; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 20; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 20; } -for tool in git mktemp python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 21 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 20; } -mkdir -p "$output" -work=$(mktemp -d "${TMPDIR:-/tmp}/P15-031.XXXXXX") - -cleanup_trap() -{ - rm -rf -- "$work" -} -trap cleanup_trap EXIT HUP INT TERM - -python3 -B - "$output/command-argv.json" "$0" --base "$base" --output "$output" <<'PY' -import json -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text( - json.dumps(sys.argv[2:], ensure_ascii=True, separators=(",", ":")) + "\n", - encoding="ascii", -) -PY -python3 -B - "$output/ownership.json" "$work" "$output" <<'PY' -import json -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text( - json.dumps( - { - "owned_temporary_paths": [sys.argv[2]], - "persistent_output": sys.argv[3], - "owned_processes": [], - "owned_ports": [], - "owned_overlays": [], - "qemu": "NOT_RUN", - }, - ensure_ascii=True, - indent=2, - sort_keys=True, - ) + "\n", - encoding="ascii", -) -PY - -set +e -timeout -k 5 "$deadline" python3 -B - \ - "$root" "$input" "$base" "$output" "$deadline" <<'PY' \ - >"$output/stdout.log" 2>"$output/stderr.log" -from __future__ import annotations - -from datetime import datetime, timezone -import hashlib -import json -from pathlib import Path -import subprocess -import sys -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -DEADLINE = int(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="utf-8")) -START_UTC = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - - -class GateFailure(RuntimeError): - def __init__(self, status: str, reason: str): - super().__init__(reason) - self.status = status - self.reason = reason - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def canonical_sha256(value: Any) -> str: - raw = json.dumps( - value, ensure_ascii=True, separators=(",", ":"), sort_keys=True - ).encode("ascii") - return sha256_bytes(raw) - - -def write_json(path: Path, value: Any) -> None: - path.write_text( - json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", - ) - - -def run_bytes(argv: list[str], allowed: set[int] | None = None) -> bytes: - try: - completed = subprocess.run( - argv, - cwd=ROOT, - check=False, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=20, - ) - except subprocess.TimeoutExpired as error: - raise GateFailure( - "INFRA_BLOCKED", f"command timed out: {' '.join(argv)}" - ) from error - accepted = {0} if allowed is None else allowed - if completed.returncode not in accepted: - detail = completed.stderr.decode("utf-8", errors="replace").strip() - raise GateFailure( - "RUNNER_FAIL", - f"command failed ({completed.returncode}): {' '.join(argv)}: {detail}", - ) - return completed.stdout - - -def git_text(*args: str) -> str: - return run_bytes(["git", "-C", str(ROOT), *args]).decode( - "utf-8", errors="strict" - ).strip() - - -def source_at(commit: str, relative: str) -> bytes: - return run_bytes(["git", "-C", str(ROOT), "show", f"{commit}:{relative}"]) - - -def parse_ls_tree(raw: bytes) -> dict[str, dict[str, Any]]: - entries: dict[str, dict[str, Any]] = {} - for record in raw.split(b"\0"): - if not record: - continue - metadata, path_bytes = record.split(b"\t", 1) - mode, kind, oid, size = metadata.decode("ascii").split() - path = path_bytes.decode("utf-8", errors="strict") - entries[path] = { - "mode": mode, - "kind": kind, - "oid": oid, - "size": int(size), - } - return entries - - -def grep_paths(commit: str, expressions: list[str], fixed: bool = False) -> list[str]: - argv = ["git", "-C", str(ROOT), "grep", "-I", "-l"] - if fixed: - argv.append("-F") - else: - argv.extend(["-i", "-E"]) - for expression in expressions: - argv.extend(["-e", expression]) - argv.extend( - [ - commit, - "--", - "repo-pre-15", - "planning/pre15", - ":(exclude)planning/pre15/introduction.md", - ":(exclude)repo-pre-15/tests/pre15/gates/P15-031.sh", - ":(exclude)repo-pre-15/tests/pre15/gates/P15-031-input.json", - ":(exclude)repo-pre-15/docs/pre15-stage0/P15-031.md", - ] - ) - raw = run_bytes(argv, {0, 1}) - prefix = f"{commit}:" - paths = [] - for line in raw.decode("utf-8", errors="strict").splitlines(): - if not line.startswith(prefix): - raise GateFailure("RUNNER_FAIL", f"unexpected git grep path: {line}") - paths.append(line[len(prefix):]) - return sorted(paths) - - -def classify_reference(path: str) -> str: - if path.startswith("planning/pre15/evidence/"): - return "historical-evidence" - if path.startswith("planning/pre15/"): - return "planning-proposal" - if path.startswith("repo-pre-15/tests/"): - return "test-only" - if path.startswith("repo-pre-15/src-linux/"): - return "linux-reference" - if path.startswith("repo-pre-15/docs/") or path.startswith("repo-pre-15/current/"): - return "project-report-or-documentation" - return "possible-project-consumer" - - -def require_text(record: dict[str, Any], field: str) -> str: - value = record.get(field) - if not isinstance(value, str) or not value.strip(): - raise GateFailure("RUNNER_FAIL", f"consumer declaration lacks {field}") - return value.strip() - - -def validate_bound_path( - record: dict[str, Any], path_field: str, hash_field: str, - entries: dict[str, dict[str, Any]], commit: str, maximum_bytes: int, -) -> tuple[str, bytes, str]: - path = require_text(record, path_field) - if not path.startswith("repo-pre-15/"): - raise GateFailure("RUNNER_FAIL", f"{path_field} is outside DUT: {path}") - if path not in entries: - raise GateFailure("RUNNER_FAIL", f"{path_field} is not tracked: {path}") - if entries[path]["size"] > maximum_bytes: - raise GateFailure("INFRA_BLOCKED", f"bound file exceeds size limit: {path}") - data = source_at(commit, path) - digest = sha256_bytes(data) - if digest != require_text(record, hash_field): - raise GateFailure("RUNNER_FAIL", f"{hash_field} mismatch: {path}") - return path, data, digest - - -def validate_consumer_manifest( - path: str, data: bytes, entries: dict[str, dict[str, Any]], commit: str, - event_schema_sha256: str, maximum_bytes: int, -) -> dict[str, Any]: - try: - record = json.loads(data.decode("utf-8", errors="strict")) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - raise GateFailure("RUNNER_FAIL", f"invalid consumer manifest: {path}") from error - if not isinstance(record, dict): - raise GateFailure("RUNNER_FAIL", f"consumer manifest is not an object: {path}") - contract = SPEC["consumer_contract"] - if not path.startswith(contract["manifest_prefix"]): - raise GateFailure("RUNNER_FAIL", f"consumer manifest is outside diagnostic prefix: {path}") - if record.get("schema") != contract["schema_marker"]: - raise GateFailure("RUNNER_FAIL", f"consumer manifest schema mismatch: {path}") - missing_fields = sorted(set(contract["required_fields"]) - set(record)) - if missing_fields: - raise GateFailure( - "RUNNER_FAIL", f"consumer manifest lacks fields {missing_fields}: {path}" - ) - if require_text(record, "consumer_class") != "diagnostic": - raise GateFailure("RUNNER_FAIL", f"consumer is not diagnostic: {path}") - if record.get("production_use") is not True or record.get("actual_capture") is not True: - raise GateFailure("RUNNER_FAIL", f"consumer lacks production capture evidence: {path}") - if not require_text(record, "freebsd_version").startswith("FreeBSD 15"): - raise GateFailure("RUNNER_FAIL", f"consumer capture is not FreeBSD 15: {path}") - transport = require_text(record, "transport") - if transport not in contract["allowed_transports"]: - raise GateFailure("RUNNER_FAIL", f"consumer transport is not native: {path}") - if require_text(record, "access_mode") != contract["required_access_mode"]: - raise GateFailure("RUNNER_FAIL", f"consumer access is not privileged: {path}") - for field in ( - "consumer_id", "owner", "version", "entrypoint", "workflow", - "deployment_reference", - ): - require_text(record, field) - if record.get("event_schema_sha256") != event_schema_sha256: - raise GateFailure("RUNNER_FAIL", f"consumer event schema mismatch: {path}") - - expected_ids = [event["id"] for event in SPEC["event_schema"]["events"]] - if record.get("event_ids") != expected_ids: - raise GateFailure("RUNNER_FAIL", f"consumer event order/set is incomplete: {path}") - counts = record.get("captured_event_counts") - if not isinstance(counts, dict) or set(counts) != set(expected_ids): - raise GateFailure("RUNNER_FAIL", f"consumer capture counts are incomplete: {path}") - if any(type(counts[event_id]) is not int or counts[event_id] < 1 for event_id in expected_ids): - raise GateFailure("RUNNER_FAIL", f"consumer did not capture every event: {path}") - - implementation_path, implementation, implementation_hash = validate_bound_path( - record, "implementation_path", "implementation_sha256", - entries, commit, maximum_bytes, - ) - if any( - implementation_path.startswith(prefix) - for prefix in contract["prohibited_implementation_prefixes"] - ): - raise GateFailure("RUNNER_FAIL", f"consumer implementation is nonqualifying: {path}") - capture_path, capture, capture_hash = validate_bound_path( - record, "capture_evidence_path", "capture_evidence_sha256", - entries, commit, maximum_bytes, - ) - implementation_text = implementation.decode("utf-8", errors="replace").lower() - capture_text = capture.decode("utf-8", errors="replace").lower() - missing_implementation_events = [ - event_id for event_id in expected_ids - if event_id not in implementation_text and event_id.replace("_", "-") not in implementation_text - ] - missing_capture_events = [ - event_id for event_id in expected_ids - if event_id not in capture_text and event_id.replace("_", "-") not in capture_text - ] - if missing_implementation_events: - raise GateFailure( - "RUNNER_FAIL", f"consumer implementation omits {missing_implementation_events}: {path}" - ) - if missing_capture_events: - raise GateFailure( - "RUNNER_FAIL", f"consumer capture omits {missing_capture_events}: {path}" - ) - if transport == "dtrace-sdt": - if "dtrace" not in implementation_text or "erofs" not in implementation_text: - raise GateFailure("RUNNER_FAIL", f"consumer does not invoke EROFS DTrace: {path}") - else: - if "ktrdump" not in implementation_text: - raise GateFailure("RUNNER_FAIL", f"KTR consumer does not invoke ktrdump: {path}") - require_text(record, "ktr_justification") - - return { - "manifest_path": path, - "manifest_sha256": sha256_bytes(data), - "consumer_id": record["consumer_id"], - "owner": record["owner"], - "version": record["version"], - "transport": transport, - "implementation_path": implementation_path, - "implementation_sha256": implementation_hash, - "capture_evidence_path": capture_path, - "capture_evidence_sha256": capture_hash, - "captured_event_counts": counts, - "production_use": True, - "actual_capture": True, - } - - -def all_conditions_pass(statuses: list[str]) -> bool: - return len(statuses) == 5 and all(status == "PASS" for status in statuses) - - -def false_go_controls() -> dict[str, Any]: - vectors = [] - all_pass = ["PASS"] * 5 - if not all_conditions_pass(all_pass): - raise GateFailure("RUNNER_FAIL", "all-PASS control did not authorize GO") - vectors.append({"name": "all-pass", "go": True}) - for index in range(5): - statuses = all_pass.copy() - statuses[index] = "NOT_RUN" - if all_conditions_pass(statuses): - raise GateFailure("RUNNER_FAIL", f"false-GO control {index} was accepted") - vectors.append({"name": f"condition-{index + 1}-not-run", "go": False}) - statuses = all_pass.copy() - statuses[0] = "STOP" - if all_conditions_pass(statuses): - raise GateFailure("RUNNER_FAIL", "consumer STOP control was accepted") - vectors.append({"name": "consumer-stop", "go": False}) - return {"status": "PASS", "control_count": len(vectors), "vectors": vectors} - - -def main() -> int: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-031": - raise GateFailure("RUNNER_FAIL", "invalid P15-031 input identity") - if SPEC.get("gate") != "G08" or SPEC.get("batch") != "B38": - raise GateFailure("RUNNER_FAIL", "invalid G08/B38 input identity") - if len(SPEC.get("go_requirements", [])) != 5: - raise GateFailure("RUNNER_FAIL", "G08 P15-031 must have exactly five GO requirements") - - resolved = git_text("rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise GateFailure( - "INFRA_BLOCKED", f"P15-031 must replay {SPEC['required_base']}, got {resolved}" - ) - for path, expected_oid in SPEC["scope_tree_oids"].items(): - actual_oid = git_text("rev-parse", f"{resolved}:{path}") - if actual_oid != expected_oid: - raise GateFailure("INFRA_BLOCKED", f"frozen scope tree changed: {path}") - - source_hashes = { - path: sha256_bytes(source_at(resolved, path)) - for path in SPEC["source_sha256"] - } - if source_hashes != SPEC["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "frozen G08/B38 source identity changed") - write_json(OUTPUT / "source-sha256.json", source_hashes) - addendum_paths = ( - "repo-pre-15/tests/pre15/gates/P15-031.sh", - "repo-pre-15/tests/pre15/gates/P15-031-input.json", - "repo-pre-15/docs/pre15-stage0/P15-031.md", - ) - addendum_hashes = { - path: sha256_bytes((ROOT / path).read_bytes()) for path in addendum_paths - } - write_json(OUTPUT / "gate-addendum-sha256.json", addendum_hashes) - - entries = parse_ls_tree( - run_bytes( - [ - "git", "-C", str(ROOT), "ls-tree", "-r", "--long", "-z", - resolved, "--", "repo-pre-15", "planning/pre15", - ] - ) - ) - prohibited = set(SPEC["prohibited_paths"]) - scoped_entries = {path: value for path, value in entries.items() if path not in prohibited} - if len(scoped_entries) != SPEC["consumer_inventory"]["expected_scoped_path_count"]: - raise GateFailure("INFRA_BLOCKED", "frozen consumer scope path count changed") - - reference_paths = grep_paths( - resolved, SPEC["consumer_inventory"]["reference_patterns"] - ) - maximum_bytes = SPEC["consumer_inventory"]["maximum_candidate_bytes"] - references = [] - reference_counts = { - "planning-proposal": 0, - "historical-evidence": 0, - "test-only": 0, - "linux-reference": 0, - "project-report-or-documentation": 0, - "possible-project-consumer": 0, - } - reasons = { - "planning-proposal": "proposal or audit text is not an existing consumer", - "historical-evidence": "historical gate output is not a deployed consumer", - "test-only": "a test, gate, or test record is explicitly nonqualifying", - "linux-reference": "Linux tracepoints are not a FreeBSD diagnostic consumer", - "project-report-or-documentation": "documentation is not an implemented consumer", - "possible-project-consumer": "requires a validated production capture manifest", - } - for path in reference_paths: - if path in prohibited: - raise GateFailure("RUNNER_FAIL", "prohibited path entered consumer scan") - entry = scoped_entries[path] - if entry["size"] > maximum_bytes: - raise GateFailure("INFRA_BLOCKED", f"consumer candidate exceeds bound: {path}") - data = source_at(resolved, path) - classification = classify_reference(path) - reference_counts[classification] += 1 - references.append( - { - "path": path, - "blob_oid": entry["oid"], - "bytes": entry["size"], - "sha256": sha256_bytes(data), - "classification": classification, - "qualifies": False, - "reason": reasons[classification], - } - ) - expected_counts = SPEC["consumer_inventory"]["expected_reference_counts"] - if len(reference_paths) != expected_counts["total"]: - raise GateFailure("INFRA_BLOCKED", "frozen native-tracing reference count changed") - for classification, count in reference_counts.items(): - if count != expected_counts[classification]: - raise GateFailure( - "INFRA_BLOCKED", f"frozen reference classification changed: {classification}" - ) - - event_schema_sha256 = canonical_sha256(SPEC["event_schema"]) - marker = SPEC["consumer_contract"]["schema_marker"] - manifest_paths = grep_paths(resolved, [marker], fixed=True) - consumers = [] - manifest_hashes = {} - for path in manifest_paths: - entry = scoped_entries[path] - if entry["size"] > maximum_bytes: - raise GateFailure("INFRA_BLOCKED", f"consumer manifest exceeds bound: {path}") - data = source_at(resolved, path) - manifest_hashes[path] = sha256_bytes(data) - consumers.append( - validate_consumer_manifest( - path, data, scoped_entries, resolved, event_schema_sha256, maximum_bytes - ) - ) - if manifest_hashes != SPEC["approved_consumer_manifest_sha256"]: - raise GateFailure("INFRA_BLOCKED", "declared consumer inventory changed") - - inventory = { - "schema": 1, - "base": resolved, - "scope": SPEC["consumer_inventory"]["scope"], - "scoped_path_count": len(scoped_entries), - "prohibited_path_read": False, - "reference_patterns": SPEC["consumer_inventory"]["reference_patterns"], - "reference_path_count": len(reference_paths), - "reference_classification_counts": reference_counts, - "references": references, - "consumer_manifest_marker": marker, - "consumer_manifest_count": len(consumers), - "qualified_consumer_count": len(consumers), - "qualified_consumers": consumers, - "event_schema_sha256": event_schema_sha256, - "required_event_ids": [event["id"] for event in SPEC["event_schema"]["events"]], - "nonqualifying_examples": SPEC["consumer_contract"]["nonqualifying_examples"], - } - write_json(OUTPUT / "consumer-inventory.json", inventory) - write_json( - OUTPUT / "event-schema.json", - { - "schema": 1, - "event_schema_sha256": event_schema_sha256, - "event_schema": SPEC["event_schema"], - }, - ) - controls = false_go_controls() - write_json(OUTPUT / "false-go-controls.json", controls) - - consumer_present = bool(consumers) - conditions = [ - { - "id": "concrete-existing-native-diagnostic-consumer", - "status": "PASS" if consumer_present else "STOP", - "observation": ( - f"{len(consumers)} qualified production capture manifest(s)" - if consumer_present - else "zero qualified consumers; all 33 references are planning, history, tests, or documentation" - ), - }, - { - "id": "no-pointer-credential-or-unauthenticated-metadata-leakage", - "status": "NOT_RUN", - "reason": "existing-consumer prerequisite is absent", - }, - { - "id": "owned-temp-freebsd15-prototype-fields-counts-failure-unload", - "status": "NOT_RUN", - "reason": "existing-consumer prerequisite is absent; no prototype was generated", - }, - { - "id": "disabled-zstdio0-zstdio1-no-new-undefined-symbols", - "status": "NOT_RUN", - "reason": "existing-consumer prerequisite is absent; no builds were run", - }, - { - "id": "ten-run-hot-path-median-regression-at-most-three-percent", - "status": "NOT_RUN", - "reason": "existing-consumer prerequisite is absent; no benchmark was run", - }, - ] - if consumer_present: - raise GateFailure( - "RUNNER_FAIL", - "a consumer is now approved; add and run the native privacy/prototype/build/benchmark stages before GO", - ) - if all_conditions_pass([condition["status"] for condition in conditions]): - raise GateFailure("RUNNER_FAIL", "false-GO guard accepted incomplete conditions") - - result = { - "schema": 1, - "gate": "G08", - "candidate": "P15-031", - "batch": "B38", - "status": "STOP", - "reason": "no concrete existing project diagnostic consumer captures the predeclared native trace schema", - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "start_utc": START_UTC, - "end_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "deadline_seconds": DEADLINE, - "expected_exit_code": 22, - "conditions": conditions, - "false_go_guard": controls, - "consumer_inventory": "consumer-inventory.json", - "event_schema": "event-schema.json", - "gate_metrics": { - "scoped_paths": len(scoped_entries), - "native_tracing_reference_paths": len(reference_paths), - "qualified_consumers": len(consumers), - "predeclared_events": len(SPEC["event_schema"]["events"]), - "prototype_runs": 0, - "builds": 0, - "benchmark_runs": 0, - "qemu_runs": 0, - }, - "b38": "STOP-NO-SOURCE", - "prototype": "NOT_RUN", - "builds": "NOT_RUN", - "microbenchmark": "NOT_RUN", - "qemu": "NOT_RUN", - "tc178": "NOT_RUN", - "full_feature_suite": "NOT_RUN", - "smoke_suite": "NOT_RUN", - "production_source_changed": False, - "protected_pid_addressed": False, - "protected_port_addressed": False, - "base_image_addressed_or_hashed": False, - } - write_json(OUTPUT / "result.json", result) - print(json.dumps(result, ensure_ascii=True, sort_keys=True)) - return 22 - - -try: - raise SystemExit(main()) -except GateFailure as error: - failure = { - "schema": 1, - "gate": "G08", - "candidate": "P15-031", - "batch": "B38", - "status": error.status, - "reason": error.reason, - "requested_base": REQUESTED_BASE, - "start_utc": START_UTC, - "end_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "deadline_seconds": DEADLINE, - "b38": "NOT_RUN", - "prototype": "NOT_RUN", - "builds": "NOT_RUN", - "microbenchmark": "NOT_RUN", - "qemu": "NOT_RUN", - "production_source_changed": False, - } - write_json(OUTPUT / "result.json", failure) - print(json.dumps(failure, ensure_ascii=True, sort_keys=True)) - raise SystemExit({"RUNNER_FAIL": 20, "INFRA_BLOCKED": 21}.get(error.status, 20)) -PY -gate_rc=$? -set -e - -cleanup_status=PASS -if ! rm -rf -- "$work"; then - cleanup_status=FAIL - gate_rc=20 -fi -trap - EXIT HUP INT TERM -printf 'owned temporary path removed: %s\ncleanup=%s\n' \ - "$work" "$cleanup_status" >"$output/cleanup.log" - -python3 -B - "$output" "$gate_rc" "$cleanup_status" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import sys - - -output = Path(sys.argv[1]) -exit_code = int(sys.argv[2]) -cleanup = sys.argv[3] -if cleanup != "PASS": - status = "RUNNER_FAIL" - origin = "runner" -elif exit_code == 22: - status = "STOP" - origin = "gate" -elif exit_code in (124, 137): - status = "INFRA_BLOCKED" - origin = "infrastructure" -else: - status = "RUNNER_FAIL" if exit_code == 20 else "INFRA_BLOCKED" - origin = "runner" if exit_code == 20 else "infrastructure" -(output / "attempt.json").write_text( - json.dumps( - { - "schema": 1, - "exit_code": exit_code, - "status": status, - "failure_origin": origin, - "cleanup": cleanup, - "target_marker": "reached" if status == "STOP" else "not_reached", - }, - ensure_ascii=True, - indent=2, - sort_keys=True, - ) + "\n", - encoding="ascii", -) -lines = [] -for path in sorted(output.iterdir(), key=lambda item: item.name): - if path.is_file() and path.name != "SHA256SUMS": - digest = hashlib.sha256(path.read_bytes()).hexdigest() - lines.append(f"{digest} {path.name}") -(output / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") -PY - -printf 'P15-031 gate exit=%s cleanup=%s output=%s\n' \ - "$gate_rc" "$cleanup_status" "$output" -exit "$gate_rc" diff --git a/tests/pre15/gates/P15-032-input.json b/tests/pre15/gates/P15-032-input.json deleted file mode 100644 index 1dbffc4..0000000 --- a/tests/pre15/gates/P15-032-input.json +++ /dev/null @@ -1,251 +0,0 @@ -{ - "schema": "pre15-gate-v1", - "unit": "G09/P15-032", - "batch": "B39", - "frozen_base": "bd5a09054e5cf89efd4db82aadb051f20b06ebf7", - "freebsd_source": { - "outer_git_head": "106727738dcfb6c001b46f25363b91cece970085", - "identity": "per-contract-file-sha256" - }, - "timeouts_seconds": { - "whole_gate": 120, - "command": 10, - "compile": 20, - "model": 10 - }, - "syntax_policy": { - "vnode": "vnode:/absolute/path", - "geom": "geom:/provider-or-device-path", - "legacy_geom": "/provider-or-device-path", - "ambiguous_regular_path_is_vnode": false, - "multidevice": "all sources in one mount must use the same backend kind" - }, - "typed_errno": { - "cycle": "EDEADLK", - "unsupported_dependency_graph": "EOPNOTSUPP", - "changed_or_short_source": "EIO", - "dead_forced_source": "ENXIO" - }, - "required_events": [ - "parse_tagged_source", - "namei_lockleaf", - "require_vreg", - "resolve_vop_getlowvnode", - "check_self_identity", - "check_namespace_ancestors", - "trace_storage_ancestors", - "vop_access", - "vn_open_vnode_fread", - "crhold_mount_cred", - "vop_set_text", - "register_upper_mount", - "snapshot_size", - "first_io", - "io_complete", - "mark_closing", - "drain_inflight", - "vflush", - "unregister_upper_mount", - "vop_unset_text", - "vn_close", - "crfree_mount_cred", - "release_last_reference" - ], - "required_invariants": [ - "explicit_backend_tag", - "held_vnode_identity", - "held_credential", - "held_credential_used_for_read", - "kernel_space_read_buffer", - "source_range_lock", - "source_shared_vnode_lock", - "write_exclusion", - "size_snapshot_and_short_read_error", - "self_check_before_io", - "namespace_ancestor_check_before_io", - "storage_ancestor_check_before_io", - "upper_mount_registration", - "inflight_drain_before_release", - "same_backend_multidevice", - "geom_and_vnode_paths_disjoint", - "fault_injection_tc179_constructible", - "fault_injection_tc184_constructible" - ], - "lock_edges": [ - ["vfs_mount_busy", "erofs_vnode_lock"], - ["erofs_vnode_lock", "source_range_lock"], - ["source_range_lock", "source_vnode_lock"], - ["source_vnode_lock", "source_vop_read"], - ["erofs_vm_object_lock", "erofs_vm_object_unlock_before_vop_read"], - ["erofs_vm_object_unlock_before_vop_read", "erofs_vnode_lock"], - ["geom_topology_lock", "geom_open_close_only"], - ["mark_closing", "drain_inflight"], - ["drain_inflight", "vflush"], - ["vflush", "unregister_upper_mount"], - ["unregister_upper_mount", "source_vnode_close"], - ["source_vnode_close", "credential_release"], - ["credential_release", "last_reference_release"] - ], - "false_go_controls": [ - "explicit_backend_tag", - "held_vnode_identity", - "held_credential", - "held_credential_used_for_read", - "kernel_space_read_buffer", - "source_range_lock", - "source_shared_vnode_lock", - "write_exclusion", - "size_snapshot_and_short_read_error", - "self_check_before_io", - "namespace_ancestor_check_before_io", - "upper_mount_registration", - "inflight_drain_before_release", - "same_backend_multidevice", - "geom_and_vnode_paths_disjoint", - "fault_injection_tc179_constructible", - "fault_injection_tc184_constructible" - ], - "contracts": [ - { - "root": "freebsd", - "path": "sys/kern/vnode_if.src", - "sha256": "5de87ff115f543fd89f762c3d356b6799f30757f8a34fd840370ce7fff53b90b", - "must_contain": ["%% read\t\tvp\tL L L", "%% getlowvnode vp\t= = =", "%% getpages\tvp\tL L L", "%% set_text\tvp\t= = ="] - }, - { - "root": "freebsd", - "path": "sys/kern/vfs_vnops.c", - "sha256": "3bc9735e2a190fc55134a803f5fc8cb6f020b0b1ce925261774db32789bd7f08", - "must_contain": ["vn_rdwr(enum uio_rw rw", "vn_rangelock_rlock(vp", "vn_lock(vp, lock_flags | LK_RETRY);", "VOP_READ(vp, &auio, ioflg, cred);"] - }, - { - "root": "freebsd", - "path": "sys/kern/vfs_default.c", - "sha256": "4286a9bf038c32021f5c116d8ab713b09a5cfa43cf52999bcbf2c0018e7aa177", - "must_contain": ["vop_stdset_text(struct vop_set_text_args *ap)", "return (ETXTBSY);", "vop_stdunset_text(struct vop_unset_text_args *ap)", "vop_stdadd_writecount_impl", "vop_stdgetlowvnode(struct vop_getlowvnode_args *ap)"] - }, - { - "root": "freebsd", - "path": "sys/kern/vfs_mount.c", - "sha256": "eecd22aaf63eda940798af02d2d3068a6ed3dc2d90b74a936190a7e56db0a79a", - "must_contain": ["vfs_register_upper_from_vp(struct vnode *vp", "ensure that it cannot be unmounted", "KASSERT(ump != mp", "vfs_unregister_upper(struct mount *mp", "registered upper with pending unmount"] - }, - { - "root": "freebsd", - "path": "sys/sys/mount.h", - "sha256": "2bc2017d63389c39dfee52b3041970450ef1eebaa149c8fe5e84a871c0f0b738", - "must_contain": ["struct mount_upper_node", "struct vnode\t*mnt_vnodecovered", "void *\t\tmnt_data"], - "must_not_contain": ["struct g_provider", "struct g_consumer"] - }, - { - "root": "freebsd", - "path": "sys/vm/vnode_pager.c", - "sha256": "6e2bd8a57bb1c027f303db7bf0e2351fa67af5ca53d6b2b826fa1f12f2c5154e", - "must_contain": ["vnode_pager_input_old(vm_object_t object", "VM_OBJECT_WUNLOCK(object);", "auio.uio_segflg = UIO_SYSSPACE;", "VOP_READ(vp, &auio, 0, curthread->td_ucred);"] - }, - { - "root": "freebsd", - "path": "sys/fs/tarfs/tarfs_vfsops.c", - "sha256": "b4c1a5c7f8870c816917706ed0e4dafd790ca5374690833d8daa321aafb696c1", - "must_contain": ["vn_open_vnode", "vn_close"] - }, - { - "root": "freebsd", - "path": "sys/fs/tarfs/tarfs_io.c", - "sha256": "0c255ba14df3ff7dcec346dee99c9ddbee376e3483370bb7257dffe2f529fc6c", - "must_contain": ["vn_rangelock_rlock", "VOP_READ"] - }, - { - "root": "freebsd", - "path": "sys/fs/deadfs/dead_vnops.c", - "sha256": "1d6864e8cc035ad109ac60c3dab67f31b9c3ad70bee9f9e585b6be10a5657b73", - "must_contain": ["dead_read", "return (ENXIO);"] - }, - { - "root": "freebsd", - "path": "sys/fs/nullfs/null_vnops.c", - "sha256": "b10c2138b649343e6c49a9416b6f4f3a0894388059122eefa190430e13695d6c", - "must_contain": ["VOP_GETLOWVNODE"] - }, - { - "root": "freebsd", - "path": "sys/fs/nullfs/null_vfsops.c", - "sha256": "f5f8803458522d9cecb1e024bdd599630dde02bc1e0578445204602b4db633bd", - "must_contain": ["vfs_register_upper_from_vp", "VFCF_LOOPBACK"] - }, - { - "root": "freebsd", - "path": "sys/fs/unionfs/union_vnops.c", - "sha256": "ce9a41531731170a340e39bfe6afd29f4740d8b705abb6a018176c7a1e159aff", - "must_contain": ["VOP_GETLOWVNODE"] - }, - { - "root": "freebsd", - "path": "sys/fs/unionfs/union_vfsops.c", - "sha256": "c181c0134887a0d8697b739dbbf54b7c4299f337bbd4a3e6e1d62a1b6bfb1b51", - "must_contain": ["vfs_register_upper_from_vp", "VFCF_LOOPBACK"] - }, - { - "root": "freebsd", - "path": "sys/dev/md/md.c", - "sha256": "a19d34da3d391747fbe2e8bee9fbb02a361202351a0b8bfbec1abc7b5a3f2d4b", - "must_contain": ["struct md_s {", "struct vnode *vnode;", "gp->softc = sc;", "g_new_providerf(gp, \"md%d\"", "VOP_READ(vp, &auio, 0, sc->cred);", "sbuf_printf(sb, \" file %s\", mp->s_vnode.file);"] - }, - { - "root": "freebsd", - "path": "sys/geom/geom.h", - "sha256": "22feba6781dcc4e0aae0dea75b83dcae16d077be67853331da35d06edffb0f39", - "must_contain": ["struct g_geom", "void\t\t\t*softc;"], - "must_not_contain": ["struct vnode"] - }, - { - "root": "repo_base", - "path": "repo-pre-15/src/internal.h", - "sha256": "3fe8d6ca819f0a1c472cc4b173da80027f957b7f7c25e51ef7fd85b60480c98a", - "must_contain": ["struct erofs_device_info", "struct g_consumer *cp;"] - }, - { - "root": "repo_base", - "path": "repo-pre-15/src/super.c", - "sha256": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2", - "must_contain": ["erofs_open_device", "g_vfs_open", "erofs_read_superblock"] - }, - { - "root": "repo_base", - "path": "repo-pre-15/src/data.c", - "sha256": "a511146d2d1ff56bc88bae6c550cfdc49a8e978c5c2c88d735de92adaaa836b6", - "must_contain": ["erofs_bread_device", "bread(dif->devvp"] - }, - { - "root": "repo_base", - "path": "repo-pre-15/src/erofs_vnops.c", - "sha256": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64", - "must_contain": ["vnode_pager_local_getpages", "erofs_read_file"] - }, - { - "root": "linux_reference", - "path": "src-linux/fileio.c", - "sha256": "1f00f6cee0060072e8300a2e4f7ca8cd2dc566f129b99d2ed34288b19210e59c", - "must_contain": ["erofs_fileio"] - }, - { - "root": "linux_reference", - "path": "src-linux/super.c", - "sha256": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9", - "must_contain": ["s_stack_depth"] - } - ], - "blocking_condition": { - "id": "FREEBSD-GEOM-VNODE-ANCESTRY-NO-PUBLIC-IDENTITY-EDGE", - "classification": "STOP", - "summary": "The transitive storage ancestry of a regular source vnode cannot be discovered generically before first I/O.", - "counterexample": [ - "An accepted regular source vnode is on a filesystem consuming an md(4) GEOM provider.", - "The md provider services BIO_READ by VOP_READ on md_s.s_vnode.vnode.", - "That vnode identity is private md.c softc state; GEOM exposes only void *softc and dumpconf exposes a pathname.", - "VOP_GETLOWVNODE and vfs_register_upper_from_vp cover VFS alias/lower mounts only, not GEOM-to-vnode backing edges.", - "A pathname cannot close identity across rename/replace, so it cannot implement the required ancestor identity comparison.", - "Following gp->softc by duplicating private md_s is an undocumented class-specific assumption and still does not cover other GEOM or filesystem-private file-backed storage." - ] - } -} diff --git a/tests/pre15/gates/P15-032.sh b/tests/pre15/gates/P15-032.sh deleted file mode 100755 index 9b355da..0000000 --- a/tests/pre15/gates/P15-032.sh +++ /dev/null @@ -1,380 +0,0 @@ -#!/bin/sh -set -eu - -BASE=bd5a09054e5cf89efd4db82aadb051f20b06ebf7 -FREEBSD_SRC=/work/build/freebsd-src -OUTPUT= -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../../.." && pwd) -INPUT=$SCRIPT_DIR/P15-032-input.json - -usage() -{ - printf '%s\n' "usage: $0 --output ABSOLUTE_PATH [--base SHA] [--freebsd-src PATH]" >&2 - exit 20 -} - -while [ "$#" -gt 0 ]; do - case "$1" in - --output) - [ "$#" -ge 2 ] || usage - OUTPUT=$2 - shift 2 - ;; - --base) - [ "$#" -ge 2 ] || usage - BASE=$2 - shift 2 - ;; - --freebsd-src) - [ "$#" -ge 2 ] || usage - FREEBSD_SRC=$2 - shift 2 - ;; - *) - usage - ;; - esac -done - -[ -n "$OUTPUT" ] || usage -case "$OUTPUT" in -/*) ;; -*) printf '%s\n' "P15-032: --output must be absolute" >&2; exit 20 ;; -esac - -if [ -d "$OUTPUT" ] && [ -n "$(find "$OUTPUT" -mindepth 1 -maxdepth 1 -print -quit)" ]; then - printf '%s\n' "P15-032: --output must be empty" >&2 - exit 20 -fi -mkdir -p "$OUTPUT" -TMP=$(mktemp -d "${TMPDIR:-/tmp}/P15-032.XXXXXX") -cleanup() -{ - rm -rf "$TMP" -} -trap cleanup EXIT HUP INT TERM - -printf 'argv=%s\n' "$0 --output $OUTPUT --base $BASE --freebsd-src $FREEBSD_SRC" >"$OUTPUT/runner.argv" -printf 'whole_gate_deadline_seconds=120\n' >"$OUTPUT/deadlines.txt" -printf 'command_deadline_seconds=10\n' >>"$OUTPUT/deadlines.txt" -printf 'compile_deadline_seconds=20\n' >>"$OUTPUT/deadlines.txt" -printf 'model_deadline_seconds=10\n' >>"$OUTPUT/deadlines.txt" - -set +e -/usr/bin/timeout 120s python3 - "$ROOT" "$FREEBSD_SRC" "$INPUT" "$OUTPUT" "$TMP" "$BASE" >"$OUTPUT/runner.stdout" 2>"$OUTPUT/runner.stderr" <<'PY' -import datetime -import hashlib -import json -import os -from pathlib import Path -import shlex -import subprocess -import sys - -root = Path(sys.argv[1]).resolve() -freebsd = Path(sys.argv[2]).resolve() -input_path = Path(sys.argv[3]).resolve() -output = Path(sys.argv[4]).resolve() -tmp = Path(sys.argv[5]).resolve() -base = sys.argv[6] -cfg = json.loads(input_path.read_text(encoding="utf-8")) -command_timeout = cfg["timeouts_seconds"]["command"] -commands = [] - -def run(argv, cwd, timeout=command_timeout, check=True): - started = datetime.datetime.now(datetime.timezone.utc) - cp = subprocess.run(argv, cwd=cwd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, timeout=timeout, check=False) - commands.append({ - "argv": argv, - "cwd": str(cwd), - "timeout_seconds": timeout, - "started_utc": started.isoformat(), - "exit": cp.returncode, - "stdout": cp.stdout.decode("utf-8", "replace"), - "stderr": cp.stderr.decode("utf-8", "replace"), - }) - if check and cp.returncode != 0: - raise RuntimeError(f"command failed ({cp.returncode}): {shlex.join(argv)}") - return cp - -def sha256(data): - return hashlib.sha256(data).hexdigest() - -def bounded_read(path): - data = path.read_bytes() - if len(data) > 4 * 1024 * 1024: - raise RuntimeError(f"refusing oversized contract file: {path}") - return data - -def base_blob(path): - return run(["git", "show", f"{base}:{path}"], root).stdout - -def line_of(text, marker): - return text[:text.index(marker)].count("\n") + 1 - -started = datetime.datetime.now(datetime.timezone.utc) -if base != cfg["frozen_base"]: - raise RuntimeError("requested base does not match frozen base") -head = run(["git", "rev-parse", "HEAD"], root).stdout.decode().strip() -remote = run(["git", "rev-parse", "xdm/main"], root).stdout.decode().strip() -if head != base or remote != base: - raise RuntimeError(f"base mismatch: HEAD={head} xdm/main={remote} expected={base}") - -allowed = { - "repo-pre-15/tests/pre15/gates/P15-032.sh", - "repo-pre-15/tests/pre15/gates/P15-032-input.json", - "repo-pre-15/docs/pre15-stage0/P15-032.md", -} -status_raw = run(["git", "status", "--porcelain=v1", "-z"], root).stdout -entries = [entry for entry in status_raw.decode().split("\0") if entry] -paths = set() -for entry in entries: - path = entry[3:] - if " -> " in path: - path = path.split(" -> ", 1)[1] - paths.add(path) -if not paths.issubset(allowed): - raise RuntimeError(f"out-of-scope worktree paths: {sorted(paths - allowed)}") - -outer_head = run(["git", "rev-parse", "HEAD"], freebsd).stdout.decode().strip() -if outer_head != cfg["freebsd_source"]["outer_git_head"]: - raise RuntimeError(f"FreeBSD outer source identity mismatch: {outer_head}") - -contracts = [] -source_hashes = {} -for contract in cfg["contracts"]: - kind = contract["root"] - rel = contract["path"] - if kind == "freebsd": - data = bounded_read(freebsd / rel) - elif kind == "repo_base": - data = base_blob(rel) - elif kind == "linux_reference": - data = base_blob(rel) - else: - raise RuntimeError(f"unknown contract root: {kind}") - actual = sha256(data) - source_hashes[f"{kind}:{rel}"] = actual - if actual != contract["sha256"]: - raise RuntimeError(f"hash mismatch for {kind}:{rel}: {actual}") - text = data.decode("utf-8", "replace") - matches = [] - for marker in contract.get("must_contain", []): - if marker not in text: - raise RuntimeError(f"missing contract marker in {rel}: {marker!r}") - matches.append({"marker": marker, "line": line_of(text, marker)}) - forbidden = [] - for marker in contract.get("must_not_contain", []): - present = marker in text - forbidden.append({"marker": marker, "present": present}) - if present: - raise RuntimeError(f"forbidden bridge marker in {rel}: {marker!r}") - contracts.append({"root": kind, "path": rel, "sha256": actual, - "matches": matches, "negative_matches": forbidden}) - -events = cfg["required_events"] -event_index = {name: index for index, name in enumerate(events)} -required_order = [ - ("check_self_identity", "first_io"), - ("check_namespace_ancestors", "first_io"), - ("trace_storage_ancestors", "first_io"), - ("crhold_mount_cred", "first_io"), - ("vop_set_text", "first_io"), - ("register_upper_mount", "first_io"), - ("io_complete", "drain_inflight"), - ("drain_inflight", "unregister_upper_mount"), - ("unregister_upper_mount", "vn_close"), - ("vn_close", "crfree_mount_cred"), - ("crfree_mount_cred", "release_last_reference"), -] -for before, after in required_order: - if event_index[before] >= event_index[after]: - raise RuntimeError(f"invalid lifecycle order: {before} !< {after}") - -nodes = set() -adj = {} -for before, after in cfg["lock_edges"]: - nodes.update((before, after)) - adj.setdefault(before, []).append(after) -visiting = set() -visited = set() -def visit(node): - if node in visiting: - raise RuntimeError(f"lock graph cycle at {node}") - if node in visited: - return - visiting.add(node) - for child in adj.get(node, []): - visit(child) - visiting.remove(node) - visited.add(node) -for node in sorted(nodes): - visit(node) - -dot = ["digraph P15_032 {", " rankdir=LR;"] -for before, after in cfg["lock_edges"]: - dot.append(f' "{before}" -> "{after}";') -dot.append("}") -(output / "lockgraph.dot").write_text("\n".join(dot) + "\n", encoding="utf-8") - -invariants = cfg["required_invariants"] -closeable = [name for name in invariants if name != "storage_ancestor_check_before_io"] -controls = cfg["false_go_controls"] -if set(controls) != set(closeable): - raise RuntimeError("false-GO controls do not exactly cover closeable invariants") - -enum_lines = [f" INV_{name.upper()} = 1ULL << {index}," for index, name in enumerate(closeable)] -required_mask = " |\n ".join(f"INV_{name.upper()}" for name in closeable) -control_rows = ",\n".join( - f' {{"{name}", INV_{name.upper()}}}' for name in controls -) -model = f'''#include -#include -#include -#include - -enum invariant {{ -{os.linesep.join(enum_lines)} -}}; - -struct scenario {{ - uint64_t present; - int storage_ancestry_identity_api; -}}; - -struct control {{ - const char *name; - uint64_t bit; -}}; - -static const uint64_t required = - {required_mask}; - -static int -visible_only_oracle(const struct scenario *scenario) -{{ - return ((scenario->present & required) == required ? 0 : EINVAL); -}} - -static int -rigorous_oracle(const struct scenario *scenario) -{{ - int error; - - error = visible_only_oracle(scenario); - if (error != 0) - return (error); - if (!scenario->storage_ancestry_identity_api) - return (EOPNOTSUPP); - return (0); -}} - -int -main(void) -{{ - static const struct control controls[] = {{ -{control_rows} - }}; - struct scenario candidate = {{ required, 0 }}; - struct scenario mutation; - size_t index; - - if (visible_only_oracle(&candidate) != 0) - return (1); - if (rigorous_oracle(&candidate) != EOPNOTSUPP) - return (2); - puts("hidden-md-edge: visible-only=GO rigorous=STOP:EOPNOTSUPP"); - for (index = 0; index < sizeof(controls) / sizeof(controls[0]); index++) {{ - mutation = candidate; - mutation.present &= ~controls[index].bit; - if (visible_only_oracle(&mutation) == 0 || rigorous_oracle(&mutation) == 0) - return (3); - printf("false-go-control:%s=REJECTED\\n", controls[index].name); - }} - puts("lifecycle-model=PASS"); - return (0); -}} -''' -model_path = tmp / "P15-032-model.c" -binary_path = tmp / "P15-032-model" -model_path.write_text(model, encoding="utf-8") -compile_cp = run(["/usr/bin/timeout", f"{cfg['timeouts_seconds']['compile']}s", - "/usr/bin/cc", "-std=c11", "-Wall", "-Wextra", "-Werror", - "-O2", "-o", str(binary_path), str(model_path)], tmp, - timeout=cfg["timeouts_seconds"]["compile"] + 2) -model_cp = run(["/usr/bin/timeout", f"{cfg['timeouts_seconds']['model']}s", - str(binary_path)], tmp, - timeout=cfg["timeouts_seconds"]["model"] + 2) -(output / "owned-temp-model.c").write_bytes(model_path.read_bytes()) -(output / "owned-temp-model.stdout").write_bytes(model_cp.stdout) -(output / "owned-temp-model.stderr").write_bytes(model_cp.stderr) - -false_go = { - "controls": [{"removed_invariant": name, "outcome": "REJECTED"} for name in controls], - "adversarial_hidden_edge": { - "topology": "regular vnode -> filesystem -> md GEOM provider -> private backing vnode", - "visible_only_oracle": "GO", - "rigorous_oracle": "STOP:EOPNOTSUPP", - }, -} -(output / "false-go-controls.json").write_text(json.dumps(false_go, indent=2) + "\n") -(output / "contracts.json").write_text(json.dumps(contracts, indent=2) + "\n") -(output / "source-hashes.json").write_text(json.dumps(source_hashes, indent=2, sort_keys=True) + "\n") -(output / "commands.json").write_text(json.dumps(commands, indent=2) + "\n") - -state = { - "events": events, - "required_order": required_order, - "local_lifecycle": "PASS", - "storage_ancestor_identity": "UNAVAILABLE", - "typed_errno_if_detectable": cfg["typed_errno"]["cycle"], - "typed_errno_for_unsupported_graph": cfg["typed_errno"]["unsupported_dependency_graph"], -} -(output / "state-model.json").write_text(json.dumps(state, indent=2) + "\n") - -result = { - "unit": cfg["unit"], - "batch": cfg["batch"], - "classification": "GATE_STOP", - "decision": "STOP", - "exit": 10, - "base": base, - "head": head, - "xdm_main": remote, - "freebsd_git_head": outer_head, - "freebsd_identity": cfg["freebsd_source"]["identity"], - "started_utc": started.isoformat(), - "finished_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "closeable_invariants": closeable, - "blocking_condition": cfg["blocking_condition"], - "source_changes_permitted": False, - "tests_run": ["contract extraction", "owned-temp C lifecycle model", "adversarial false-GO controls"], - "tests_omitted": ["D", "H", "K", "Q", "TC006", "TC179", "TC184", "full feature", "smoke"], -} -(output / "result.json").write_text(json.dumps(result, indent=2) + "\n") - -print("P15-032 STOP: no public identity-preserving VFS/GEOM-to-backing-vnode ancestry API") -sys.exit(10) -PY -RC=$? -set -e - -printf 'exit=%s\n' "$RC" >"$OUTPUT/runner.exit" -case "$RC" in -10) - printf '%s\n' GATE_STOP >"$OUTPUT/classification" - ;; -124) - printf '%s\n' RUNNER_TIMEOUT >"$OUTPUT/classification" - ;; -0) - printf '%s\n' GATE_GO >"$OUTPUT/classification" - ;; -*) - printf '%s\n' RUNNER_FAILURE >"$OUTPUT/classification" - ;; -esac -(cd "$OUTPUT" && find . -maxdepth 1 -type f ! -name manifest.sha256 -print | LC_ALL=C sort | xargs sha256sum) >"$OUTPUT/manifest.sha256" -exit "$RC" diff --git a/tests/pre15/gates/P15-038-input.json b/tests/pre15/gates/P15-038-input.json deleted file mode 100644 index 6ed2ff9..0000000 --- a/tests/pre15/gates/P15-038-input.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "extent_bytes": 262144, - "minimum_improved_codecs": 2, - "minimum_latency_improvement_percent": 10.0, - "random_reads_per_sample": 1024, - "read_bytes": 4096, - "samples": 5, - "seed": 2539541505 - }, - "budget": { - "global_bytes": 524288, - "minimum_decode_work_bytes": 131072, - "per_mount_bytes": 262144 - }, - "candidate": "P15-038", - "gate": "G05", - "libraries": { - "liblz4": "1.10.0", - "liblzma": "5.8.1", - "libzstd": "1.5.7", - "zlib": "1.3.1" - }, - "lock_order": [ - "mount-cache", - "global-budget" - ], - "protected": { - "pid": 26318, - "port": 9222 - }, - "required_base": "5d6755649a369498a9b257bb2d1d1e3496d5135e", - "schema": 1, - "scope": "host-codec-cost-oracle-not-guest-vnode-performance", - "source_sha256": { - "repo-pre-15/src/compress.h": "d4177f5d606489d8c63bb7f1dcaf18bc92dd5d374395ccfa76060b272549f5d2", - "repo-pre-15/src/decompressor.c": "ccaa934f837ecbf3b3b678dbddf35d29dd7e3b4e80d1a7ab78b18a07e555419f", - "repo-pre-15/src/decompressor_deflate.c": "4a018ce06dc83fcb305130c262d5b59b4d133f698be40a957bebca2cd6a92518", - "repo-pre-15/src/decompressor_lz4.c": "ca5e5bd6142f9e3c3ea66849900f05799dbc7d6d1a97e0d5a988c28edb90ea98", - "repo-pre-15/src/decompressor_lzma.c": "9935be2f4d5829240052f151b916203c67e830e8b2b7b51c452262a8f92f6157", - "repo-pre-15/src/decompressor_zstd.c": "c5f19c9ea6d8238bcf30b8b3228b80e77cf2073df1d07e4fac5f5401bef413bb", - "repo-pre-15/src/internal.h": "d33b80c57846437dc0410086c552cb308b8e1a6ce517b5b2ce19779bd59e8848", - "repo-pre-15/src/zdata.c": "2c4d624c173ac9fe0cb1e72bb968a7994d0cfd66e4fcd0db04dea10e06ba0fee", - "repo-pre-15/tests/pre15/cases/B32-cache-state.sh": "e14453fc628a615acf701f96ec575c884453fe1a62878badbfcefff33656b92d", - "repo-pre-15/tests/pre15/fixtures/B32-cache-oracle.c": "2a83b84c560b67cf4f7583e886e1f0dbe792e58bc4b0a8e59e2a07c17ec8f866", - "repo-pre-15/tests/pre15/fixtures/B32-qemu-run.sh": "bb1d493a8dc6536751377f3cd72c4012875a41320f570f5259340220b226ed2e" - }, - "states": [ - "EMPTY", - "INFLIGHT", - "READY", - "FAILED" - ], - "upstream_batches": { - "b32_correctness": "c6a502184da6e973b1f0fa2ff8c0290c041e7092", - "b32_qemu_timeout_fix": "c4344fc842a180720a56b585191490395044603a" - } -} diff --git a/tests/pre15/gates/P15-038.sh b/tests/pre15/gates/P15-038.sh deleted file mode 100755 index dbb5281..0000000 --- a/tests/pre15/gates/P15-038.sh +++ /dev/null @@ -1,1109 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-038-input.json -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -for tool in cc git pkg-config python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" - -PYTHONDONTWRITEBYTECODE=1 python3 -B - "$root" "$input" "$base" "$output" <<'PY' -from __future__ import annotations - -import hashlib -import json -import os -from pathlib import Path -import statistics -import subprocess -import sys -import tempfile -import threading -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -SPEC = json.loads(INPUT.read_text(encoding="ascii")) - - -class GateFailure(RuntimeError): - def __init__(self, status: str, reason: str): - super().__init__(reason) - self.status = status - self.reason = reason - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def run( - argv: list[str], *, timeout: int, log: Path | None = None, - allowed: set[int] | None = None, -) -> subprocess.CompletedProcess[str]: - try: - completed = subprocess.run( - argv, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=timeout, - ) - except subprocess.TimeoutExpired as error: - if log is not None: - log.write_text( - "$ " + " ".join(argv) + f"\n[TIMEOUT after {timeout}s]\n", - encoding="utf-8", - ) - raise GateFailure("INFRA_BLOCKED", f"timeout after {timeout}s: {' '.join(argv)}") from error - if log is not None: - log.write_text( - "$ " + " ".join(argv) + "\n" + completed.stdout + - f"\n[exit {completed.returncode}]\n", - encoding="utf-8", - ) - expected = {0} if allowed is None else allowed - if completed.returncode not in expected: - raise GateFailure( - "RUNNER_FAIL", - f"command failed ({completed.returncode}): {' '.join(argv)}", - ) - return completed - - -def git(*args: str) -> str: - return run(["git", "-C", str(ROOT), *args], timeout=30).stdout.strip() - - -def source_at(commit: str, relative: str) -> bytes: - completed = run( - ["git", "-C", str(ROOT), "show", f"{commit}:{relative}"], - timeout=30, - ) - return completed.stdout.encode("utf-8") - - -def verify_identity() -> tuple[str, dict[str, Any]]: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-038": - raise GateFailure("INFRA_BLOCKED", "invalid P15-038 input identity") - if SPEC.get("gate") != "G05" or SPEC.get("scope") != ( - "host-codec-cost-oracle-not-guest-vnode-performance" - ): - raise GateFailure("INFRA_BLOCKED", "invalid G05 scope") - resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise GateFailure( - "INFRA_BLOCKED", - f"P15-038 must replay {SPEC['required_base']}, got {resolved}", - ) - hashes = { - relative: sha256_bytes(source_at(resolved, relative)) - for relative in SPEC["source_sha256"] - } - if hashes != SPEC["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "frozen B32/source identity changed") - package_names = { - "liblz4": "liblz4", - "liblzma": "liblzma", - "zlib": "zlib", - "libzstd": "libzstd", - } - versions = { - name: run(["pkg-config", "--modversion", package], timeout=30).stdout.strip() - for name, package in package_names.items() - } - if versions != SPEC["libraries"]: - raise GateFailure( - "INFRA_BLOCKED", - f"host codec library identity changed: {versions!r}", - ) - zdata = source_at(resolved, "repo-pre-15/src/zdata.c").decode("utf-8") - internal = source_at(resolved, "repo-pre-15/src/internal.h").decode("utf-8") - required_tokens = ( - "cache->nid == vi->nid", - "cache->decoded_size == decoded_size", - "cache->map.m_pa == map->m_pa", - "cache->map.m_la == map->m_la", - "cache->map.m_plen == map->m_plen", - "cache->map.m_llen == map->m_llen", - "cache->map.m_deviceid == map->m_deviceid", - "cache->map.m_algorithmformat == map->m_algorithmformat", - "cache->map.m_flags == map->m_flags", - "cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0", - "cache->error = error", - "cache->closing = true", - "Z_EROFS_CACHE_BYPASS", - ) - missing = [token for token in required_tokens if token not in zdata] - if missing: - raise GateFailure("STOP", f"B32 key/inflight/failure contract is not closed: {missing!r}") - if zdata.count("map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA") != 1: - raise GateFailure("STOP", "frozen admission is not exactly LZMA-only") - for token in ("EROFS_ZCACHE_EMPTY", "EROFS_ZCACHE_INFLIGHT", "EROFS_ZCACHE_READY", "EROFS_ZCACHE_FAILED"): - if token not in internal: - raise GateFailure("STOP", f"B32 state is missing: {token}") - return resolved, {"libraries": versions, "source_sha256": hashes} - - -class Budget: - def __init__(self, limit: int): - self.limit = limit - self.used = 0 - self.peak = 0 - self.lock = threading.Lock() - - def reserve(self, amount: int) -> bool: - with self.lock: - if amount > self.limit - self.used: - return False - self.used += amount - self.peak = max(self.peak, self.used) - return True - - def release(self, amount: int) -> None: - with self.lock: - if amount > self.used: - raise GateFailure("STOP", "global budget accounting underflow") - self.used -= amount - - -class Cache: - def __init__(self, budget: Budget, local_limit: int, minimum_work: int, enabled: bool = True): - self.budget = budget - self.local_limit = local_limit - self.minimum_work = minimum_work - self.enabled = enabled - self.condition = threading.Condition() - self.state = "EMPTY" - self.key: tuple[int, ...] | None = None - self.data: bytes | None = None - self.error = 0 - self.waiters = 0 - self.charged = 0 - self.peak = 0 - self.closing = False - self.owners = 0 - self.evictions = 0 - self.reclaims = 0 - self.bypasses = 0 - - def eligible(self, decoded_size: int, decode_work: int) -> bool: - return ( - self.enabled - and decoded_size <= self.local_limit - and decode_work >= self.minimum_work - ) - - def _release_locked(self) -> None: - if self.charged: - self.budget.release(self.charged) - self.charged = 0 - self.data = None - - def claim(self, key: tuple[int, ...], decoded_size: int, decode_work: int) -> tuple[str, bytes | int | None]: - if not self.eligible(decoded_size, decode_work): - self.bypasses += 1 - return "BYPASS", None - with self.condition: - if self.closing: - self.bypasses += 1 - return "BYPASS", None - if self.state != "EMPTY" and self.key == key: - if self.state == "READY": - if self.data is None: - raise GateFailure("STOP", "READY cache has no data") - return "HIT", self.data - if self.state == "INFLIGHT": - self.waiters += 1 - self.condition.notify_all() - while self.state == "INFLIGHT": - self.condition.wait() - if self.key != key: - raise GateFailure("STOP", "waiter generation key changed") - if self.state == "READY": - result: tuple[str, bytes | int | None] = ("HIT", self.data) - elif self.state == "FAILED" and self.error > 0: - result = ("ERROR", self.error) - else: - raise GateFailure("STOP", "waiter received untyped publication") - self.waiters -= 1 - if self.state == "FAILED" and self.waiters == 0: - self.error = 0 - self.state = "EMPTY" - if self.waiters == 0: - self.condition.notify_all() - return result - if self.state == "FAILED": - if self.waiters: - self.bypasses += 1 - return "BYPASS", None - self.error = 0 - self.state = "EMPTY" - if self.state == "INFLIGHT" or self.waiters: - self.bypasses += 1 - return "BYPASS", None - if self.state == "READY": - self._release_locked() - self.evictions += 1 - self.state = "EMPTY" - if not self.budget.reserve(decoded_size): - self.bypasses += 1 - return "BYPASS", None - self.charged = decoded_size - self.peak = max(self.peak, self.charged) - self.key = key - self.error = 0 - self.state = "INFLIGHT" - self.owners += 1 - return "OWNER", None - - def wait_for_waiters(self, expected: int) -> None: - with self.condition: - if not self.condition.wait_for(lambda: self.waiters == expected, timeout=10): - raise GateFailure("STOP", "same-key waiters did not register") - - def complete(self, key: tuple[int, ...], data: bytes | None, error: int) -> None: - if (error == 0) != (data is not None): - raise GateFailure("STOP", "owner completion is untyped") - with self.condition: - if self.state != "INFLIGHT" or self.key != key: - raise GateFailure("STOP", "owner lost inflight key") - if error == 0: - assert data is not None - if len(data) != self.charged: - raise GateFailure("STOP", "published bytes do not match reservation") - self.data = data - self.state = "READY" - else: - if error < 1: - raise GateFailure("STOP", "published errno is not positive") - self._release_locked() - self.error = error - self.state = "FAILED" - if self.waiters == 0: - self.error = 0 - self.state = "EMPTY" - self.condition.notify_all() - - def reclaim(self) -> str: - with self.condition: - if self.state == "INFLIGHT" or self.waiters: - return "BUSY" - if self.state == "READY": - self._release_locked() - self.state = "EMPTY" - self.reclaims += 1 - return "RECLAIMED" - return "EMPTY" - - def fini(self, closing: threading.Event | None = None) -> None: - with self.condition: - self.closing = True - if closing is not None: - closing.set() - while self.state == "INFLIGHT" or self.waiters: - self.condition.wait() - self._release_locked() - self.error = 0 - self.state = "EMPTY" - - -def payload(key: tuple[int, ...], size: int) -> bytes: - seed = sum((index + 1) * value for index, value in enumerate(key)) & 0xFF - return bytes(((seed + index * 29) & 0xFF) for index in range(size)) - - -def run_wave(cache: Cache, key: tuple[int, ...], size: int, error: int) -> dict[str, Any]: - workers = 16 - barrier = threading.Barrier(workers) - results: list[tuple[str, bytes | int | None] | None] = [None] * workers - expected = payload(key, size) - - def worker(index: int) -> None: - barrier.wait(timeout=10) - result = cache.claim(key, size, size * 2) - if result[0] == "OWNER": - cache.wait_for_waiters(workers - 1) - cache.complete(key, None if error else expected, error) - results[index] = ("ERROR", error) if error else ("HIT", expected) - else: - results[index] = result - - threads = [threading.Thread(target=worker, args=(index,)) for index in range(workers)] - for thread in threads: - thread.start() - for thread in threads: - thread.join(timeout=15) - if any(thread.is_alive() for thread in threads): - raise GateFailure("STOP", "state-model worker did not terminate") - if error: - if any(result != ("ERROR", error) for result in results): - raise GateFailure("STOP", "same-key waiters did not receive one typed failure") - elif any(result != ("HIT", expected) for result in results): - raise GateFailure("STOP", "same-key waiters did not receive identical bytes") - return {"error": error, "owners": 1, "waiters": workers - 1} - - -def run_state_model() -> dict[str, Any]: - config = SPEC["budget"] - extent = SPEC["benchmark"]["extent_bytes"] - global_budget = Budget(config["global_bytes"]) - cache = Cache( - global_budget, - config["per_mount_bytes"], - config["minimum_decode_work_bytes"], - ) - key = (42, extent, 4096, 0, 8192, extent, 0, 1, 1) - success = run_wave(cache, key, extent, 0) - failure_key = (43, extent, 8192, 0, 8192, extent, 0, 2, 1) - failure = run_wave(cache, failure_key, extent, 97) - retry = cache.claim(failure_key, extent, extent * 2) - if retry[0] != "OWNER": - raise GateFailure("STOP", "typed failure is not retryable") - retry_payload = payload(failure_key, extent) - cache.complete(failure_key, retry_payload, 0) - - before = sha256_bytes(retry_payload) - changed = list(failure_key) - changed[2] += 4096 - changed_key = tuple(changed) - result = cache.claim(changed_key, extent, extent * 2) - if result[0] != "OWNER": - raise GateFailure("STOP", "ready key eviction did not create an owner") - cache.complete(changed_key, payload(changed_key, extent), 0) - result = cache.claim(failure_key, extent, extent * 2) - if result[0] != "OWNER": - raise GateFailure("STOP", "evicted key reuse did not create a fresh owner") - after_payload = payload(failure_key, extent) - cache.complete(failure_key, after_payload, 0) - after = sha256_bytes(after_payload) - if before != after: - raise GateFailure("STOP", "eviction changed decoded bytes") - - inflight_key = (44, extent, 12288, 0, 8192, extent, 0, 3, 1) - if cache.claim(inflight_key, extent, extent * 2)[0] != "OWNER": - raise GateFailure("STOP", "reclaim setup did not create owner") - if cache.reclaim() != "BUSY": - raise GateFailure("STOP", "reclaim did not refuse inflight state") - fallback_key = (45, extent, 16384, 0, 8192, extent, 0, 0, 1) - if cache.claim(fallback_key, extent, extent * 2)[0] != "BYPASS": - raise GateFailure("STOP", "different-key inflight miss did not bypass") - fallback_hash = sha256_bytes(payload(fallback_key, extent)) - cache.complete(inflight_key, payload(inflight_key, extent), 0) - if cache.reclaim() != "RECLAIMED" or global_budget.used != 0: - raise GateFailure("STOP", "ready reclaim did not release budget") - - first = Cache(global_budget, extent, config["minimum_decode_work_bytes"]) - second = Cache(global_budget, extent, config["minimum_decode_work_bytes"]) - if first.claim(key, extent, extent * 2)[0] != "OWNER": - raise GateFailure("STOP", "global budget setup failed") - first.complete(key, payload(key, extent), 0) - if second.claim(changed_key, extent, extent * 2)[0] != "OWNER": - raise GateFailure("STOP", "global budget should fit two reservations") - second.complete(changed_key, payload(changed_key, extent), 0) - third = Cache(global_budget, extent, config["minimum_decode_work_bytes"]) - if third.claim(fallback_key, extent, extent * 2)[0] != "BYPASS": - raise GateFailure("STOP", "global exhaustion did not use no-cache fallback") - exhausted_hash = sha256_bytes(payload(fallback_key, extent)) - if first.reclaim() != "RECLAIMED": - raise GateFailure("STOP", "global reclaim did not free a reservation") - if third.claim(fallback_key, extent, extent * 2)[0] != "OWNER": - raise GateFailure("STOP", "reclaimed global budget was not reusable") - third.complete(fallback_key, payload(fallback_key, extent), 0) - second.fini() - third.fini() - - shutdown = Cache(global_budget, extent, config["minimum_decode_work_bytes"]) - shutdown_key = (46, extent, 20480, 0, 8192, extent, 0, 1, 1) - if shutdown.claim(shutdown_key, extent, extent * 2)[0] != "OWNER": - raise GateFailure("STOP", "unmount setup did not create owner") - closing = threading.Event() - finished = threading.Event() - - def finish_cache() -> None: - shutdown.fini(closing) - finished.set() - - thread = threading.Thread(target=finish_cache) - thread.start() - if not closing.wait(timeout=10) or finished.is_set(): - raise GateFailure("STOP", "unmount did not wait for inflight owner") - if shutdown.claim(changed_key, extent, extent * 2)[0] != "BYPASS": - raise GateFailure("STOP", "closing cache accepted a new owner") - shutdown.complete(shutdown_key, payload(shutdown_key, extent), 0) - thread.join(timeout=10) - if thread.is_alive() or not finished.is_set() or global_budget.used != 0: - raise GateFailure("STOP", "unmount drain did not close cleanly") - - disabled = Cache(global_budget, extent, config["minimum_decode_work_bytes"], enabled=False) - if disabled.claim(key, extent, extent * 2)[0] != "BYPASS": - raise GateFailure("STOP", "disabled policy did not bypass") - low_work = Cache(global_budget, extent, config["minimum_decode_work_bytes"]) - if low_work.claim(key, extent, config["minimum_decode_work_bytes"] - 1)[0] != "BYPASS": - raise GateFailure("STOP", "low-cost request did not bypass") - oversized = Cache(global_budget, extent // 2, config["minimum_decode_work_bytes"]) - if oversized.claim(key, extent, extent * 2)[0] != "BYPASS": - raise GateFailure("STOP", "oversized request did not bypass") - no_cache_hash = sha256_bytes(payload(key, extent)) - - if global_budget.peak > global_budget.limit or cache.peak > cache.local_limit: - raise GateFailure("STOP", "state model exceeded hard budget") - return { - "budget": { - "global_limit": global_budget.limit, - "global_peak": global_budget.peak, - "global_remaining": global_budget.used, - "mount_limit": cache.local_limit, - "mount_peak": cache.peak, - }, - "correctness": { - "eviction_sha256_before": before, - "eviction_sha256_after": after, - "fallback_sha256": fallback_hash, - "global_exhaustion_fallback_sha256": exhausted_hash, - "no_cache_sha256": no_cache_hash, - }, - "coverage": [ - "owner-waiter-success", - "typed-failure-retry", - "exact-key-reuse", - "ready-eviction", - "inflight-reclaim-busy", - "ready-reclaim", - "global-exhaustion-fallback", - "unmount-drain", - "disabled-policy", - "low-work-bypass", - "oversize-bypass", - ], - "failure_wave": failure, - "lock_order": SPEC["lock_order"], - "status": "PASS", - "success_wave": success, - } - - -BENCH_SOURCE = r''' -#define _POSIX_C_SOURCE 200809L -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum codec_id { CODEC_LZ4, CODEC_LZMA, CODEC_DEFLATE, CODEC_ZSTD, CODEC_COUNT }; - -struct fixture { - enum codec_id id; - const char *name; - unsigned char *compressed; - size_t compressed_size; -}; - -static void -fail(const char *message) -{ - fprintf(stderr, "P15-038 codec oracle failure: %s\n", message); - exit(10); -} - -static uint64_t -now_ns(void) -{ - struct timespec time; - - if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) - fail("clock_gettime"); - return ((uint64_t)time.tv_sec * UINT64_C(1000000000) + (uint64_t)time.tv_nsec); -} - -static uint32_t -xorshift32(uint32_t *state) -{ - uint32_t value = *state; - - value ^= value << 13; - value ^= value >> 17; - value ^= value << 5; - *state = value; - return (value); -} - -static uint64_t -hash_update(uint64_t hash, const unsigned char *data, size_t size) -{ - size_t index; - - for (index = 0; index < size; ++index) { - hash ^= data[index]; - hash *= UINT64_C(1099511628211); - } - return (hash); -} - -static void -write_file(const char *path, const void *data, size_t size) -{ - FILE *stream = fopen(path, "wb"); - - if (stream == NULL || fwrite(data, 1, size, stream) != size || fclose(stream) != 0) - fail("write fixture artifact"); -} - -static void -make_payload(unsigned char *payload, size_t size, uint32_t seed) -{ - unsigned char block[4096]; - size_t index; - - for (index = 0; index < sizeof(block); ++index) - block[index] = (unsigned char)xorshift32(&seed); - for (index = 0; index < size; ++index) - payload[index] = block[index % sizeof(block)]; -} - -static struct fixture -compress_fixture(enum codec_id id, const char *name, const unsigned char *payload, size_t size) -{ - struct fixture fixture = { .id = id, .name = name }; - size_t capacity, output_position; - uLongf zlib_size; - int amount; - - switch (id) { - case CODEC_LZ4: - capacity = (size_t)LZ4_compressBound((int)size); - fixture.compressed = malloc(capacity); - amount = LZ4_compress_default((const char *)payload, - (char *)fixture.compressed, (int)size, (int)capacity); - if (amount <= 0) - fail("LZ4 compression"); - fixture.compressed_size = (size_t)amount; - break; - case CODEC_LZMA: - capacity = lzma_stream_buffer_bound(size); - fixture.compressed = malloc(capacity); - output_position = 0; - if (lzma_easy_buffer_encode(6, LZMA_CHECK_NONE, NULL, payload, size, - fixture.compressed, &output_position, capacity) != LZMA_OK) - fail("LZMA compression"); - fixture.compressed_size = output_position; - break; - case CODEC_DEFLATE: - capacity = (size_t)compressBound((uLong)size); - fixture.compressed = malloc(capacity); - zlib_size = (uLongf)capacity; - if (compress2(fixture.compressed, &zlib_size, payload, (uLong)size, 6) != Z_OK) - fail("Deflate compression"); - fixture.compressed_size = (size_t)zlib_size; - break; - case CODEC_ZSTD: - capacity = ZSTD_compressBound(size); - fixture.compressed = malloc(capacity); - fixture.compressed_size = ZSTD_compress( - fixture.compressed, capacity, payload, size, 3); - if (ZSTD_isError(fixture.compressed_size)) - fail("Zstd compression"); - break; - default: - fail("unknown compression codec"); - } - if (fixture.compressed == NULL) - fail("compressed allocation"); - return (fixture); -} - -static void -decode(const struct fixture *fixture, unsigned char *output, size_t output_size) -{ - size_t input_position, output_position, amount; - uint64_t memory_limit; - uLongf zlib_size; - int decoded; - - switch (fixture->id) { - case CODEC_LZ4: - decoded = LZ4_decompress_safe((const char *)fixture->compressed, - (char *)output, (int)fixture->compressed_size, (int)output_size); - if (decoded != (int)output_size) - fail("LZ4 decode"); - break; - case CODEC_LZMA: - memory_limit = UINT64_MAX; - input_position = 0; - output_position = 0; - if (lzma_stream_buffer_decode(&memory_limit, 0, NULL, - fixture->compressed, &input_position, fixture->compressed_size, - output, &output_position, output_size) != LZMA_OK || - input_position != fixture->compressed_size || output_position != output_size) - fail("LZMA decode"); - break; - case CODEC_DEFLATE: - zlib_size = (uLongf)output_size; - if (uncompress(output, &zlib_size, fixture->compressed, - (uLong)fixture->compressed_size) != Z_OK || zlib_size != output_size) - fail("Deflate decode"); - break; - case CODEC_ZSTD: - amount = ZSTD_decompress(output, output_size, - fixture->compressed, fixture->compressed_size); - if (ZSTD_isError(amount) || amount != output_size) - fail("Zstd decode"); - break; - default: - fail("unknown decompression codec"); - } -} - -static uint64_t -workload(const struct fixture *fixture, const unsigned char *payload, - size_t extent_size, size_t read_size, const size_t *offsets, size_t reads, - int neutral, size_t budget, size_t minimum_work, size_t *peak, uint64_t *hashp) -{ - unsigned char *cached = NULL, *decoded, *slice; - uint64_t begin, end, hash = UINT64_C(1469598103934665603); - size_t index; - int admit; - - admit = (fixture->id == CODEC_LZMA || neutral) && extent_size <= budget && - fixture->compressed_size + extent_size >= minimum_work; - slice = malloc(read_size); - if (slice == NULL) - fail("slice allocation"); - *peak = 0; - begin = now_ns(); - for (index = 0; index < reads; ++index) { - if (cached == NULL) { - decoded = malloc(extent_size); - if (decoded == NULL) - fail("decode allocation"); - decode(fixture, decoded, extent_size); - memcpy(slice, decoded + offsets[index], read_size); - if (admit) { - cached = decoded; - if (extent_size > *peak) - *peak = extent_size; - } else { - free(decoded); - } - } else { - memcpy(slice, cached + offsets[index], read_size); - } - if (memcmp(slice, payload + offsets[index], read_size) != 0) - fail("random read bytes differ"); - hash = hash_update(hash, slice, read_size); - } - end = now_ns(); - free(cached); - free(slice); - *hashp = hash; - return (end - begin); -} - -static void -eviction_oracle(const struct fixture *first, const struct fixture *second, - size_t extent_size, const char *directory) -{ - unsigned char *cache = malloc(extent_size); - char path[PATH_MAX]; - - if (cache == NULL) - fail("eviction allocation"); - decode(first, cache, extent_size); - snprintf(path, sizeof(path), "%s/eviction-before.bin", directory); - write_file(path, cache, extent_size); - free(cache); - cache = malloc(extent_size); - if (cache == NULL) - fail("second eviction allocation"); - decode(second, cache, extent_size); - free(cache); - cache = malloc(extent_size); - if (cache == NULL) - fail("reused eviction allocation"); - decode(first, cache, extent_size); - snprintf(path, sizeof(path), "%s/eviction-after.bin", directory); - write_file(path, cache, extent_size); - free(cache); -} - -int -main(int argc, char **argv) -{ - static const char *names[CODEC_COUNT] = { "lz4", "lzma", "deflate", "zstd" }; - struct fixture fixtures[CODEC_COUNT]; - unsigned char *payload, *verification; - size_t *offsets; - size_t extent_size, read_size, reads, samples, budget, minimum_work; - size_t codec, sample, peak_current, peak_candidate; - uint32_t seed; - uint64_t current_ns, candidate_ns, current_hash, candidate_hash; - char path[PATH_MAX]; - - if (argc != 9) - fail("usage"); - extent_size = (size_t)strtoull(argv[1], NULL, 10); - read_size = (size_t)strtoull(argv[2], NULL, 10); - reads = (size_t)strtoull(argv[3], NULL, 10); - samples = (size_t)strtoull(argv[4], NULL, 10); - budget = (size_t)strtoull(argv[5], NULL, 10); - minimum_work = (size_t)strtoull(argv[6], NULL, 10); - seed = (uint32_t)strtoul(argv[7], NULL, 10); - if (extent_size == 0 || extent_size > INT_MAX || read_size == 0 || - read_size > extent_size || reads == 0 || samples != 5) - fail("invalid fixed workload"); - payload = malloc(extent_size); - verification = malloc(extent_size); - offsets = malloc(reads * sizeof(*offsets)); - if (payload == NULL || verification == NULL || offsets == NULL) - fail("workload allocation"); - make_payload(payload, extent_size, seed); - for (codec = 0; codec < CODEC_COUNT; ++codec) { - fixtures[codec] = compress_fixture((enum codec_id)codec, names[codec], - payload, extent_size); - decode(&fixtures[codec], verification, extent_size); - if (memcmp(payload, verification, extent_size) != 0) - fail("full decode verification"); - snprintf(path, sizeof(path), "%s/%s.compressed", argv[8], names[codec]); - write_file(path, fixtures[codec].compressed, fixtures[codec].compressed_size); - } - snprintf(path, sizeof(path), "%s/payload.bin", argv[8]); - write_file(path, payload, extent_size); - for (sample = 0; sample < reads; ++sample) - offsets[sample] = (size_t)xorshift32(&seed) % (extent_size - read_size + 1); - - printf("{\"schema\":1,\"scope\":\"host-codec-cost-only\",\"samples\":%zu,\"reads_per_sample\":%zu,\"codecs\":[", samples, reads); - for (codec = 0; codec < CODEC_COUNT; ++codec) { - if (codec != 0) - printf(","); - printf("{\"codec\":\"%s\",\"compressed_bytes\":%zu,\"current_ns\":[", - fixtures[codec].name, fixtures[codec].compressed_size); - for (sample = 0; sample < samples; ++sample) { - if ((sample & 1) == 0) { - current_ns = workload(&fixtures[codec], payload, extent_size, - read_size, offsets, reads, 0, budget, minimum_work, - &peak_current, ¤t_hash); - candidate_ns = workload(&fixtures[codec], payload, extent_size, - read_size, offsets, reads, 1, budget, minimum_work, - &peak_candidate, &candidate_hash); - } else { - candidate_ns = workload(&fixtures[codec], payload, extent_size, - read_size, offsets, reads, 1, budget, minimum_work, - &peak_candidate, &candidate_hash); - current_ns = workload(&fixtures[codec], payload, extent_size, - read_size, offsets, reads, 0, budget, minimum_work, - &peak_current, ¤t_hash); - } - if (sample != 0) - printf(","); - printf("%" PRIu64, current_ns); - if (current_hash != candidate_hash || peak_candidate > budget) - fail("sample hash or budget mismatch"); - /* Keep paired values for the second fixed-size arrays. */ - snprintf(path, sizeof(path), "%s/sample-%zu-%zu.tmp", argv[8], codec, sample); - FILE *record = fopen(path, "w"); - if (record == NULL || fprintf(record, "%" PRIu64 " %zu %zu %" PRIu64 "\n", - candidate_ns, peak_current, peak_candidate, current_hash) < 0 || fclose(record) != 0) - fail("sample record"); - } - printf("],\"candidate_ns\":["); - for (sample = 0; sample < samples; ++sample) { - uint64_t saved_candidate, saved_hash; - FILE *record; - snprintf(path, sizeof(path), "%s/sample-%zu-%zu.tmp", argv[8], codec, sample); - record = fopen(path, "r"); - if (record == NULL || fscanf(record, "%" SCNu64 " %zu %zu %" SCNu64, - &saved_candidate, &peak_current, &peak_candidate, &saved_hash) != 4 || fclose(record) != 0) - fail("read sample record"); - if (sample != 0) - printf(","); - printf("%" PRIu64, saved_candidate); - } - printf("],\"current_peak_bytes\":["); - for (sample = 0; sample < samples; ++sample) { - uint64_t saved_candidate, saved_hash; - FILE *record; - snprintf(path, sizeof(path), "%s/sample-%zu-%zu.tmp", argv[8], codec, sample); - record = fopen(path, "r"); - if (record == NULL || fscanf(record, "%" SCNu64 " %zu %zu %" SCNu64, - &saved_candidate, &peak_current, &peak_candidate, &saved_hash) != 4 || fclose(record) != 0) - fail("read current peak"); - if (sample != 0) - printf(","); - printf("%zu", peak_current); - } - printf("],\"candidate_peak_bytes\":["); - for (sample = 0; sample < samples; ++sample) { - uint64_t saved_candidate, saved_hash; - FILE *record; - snprintf(path, sizeof(path), "%s/sample-%zu-%zu.tmp", argv[8], codec, sample); - record = fopen(path, "r"); - if (record == NULL || fscanf(record, "%" SCNu64 " %zu %zu %" SCNu64, - &saved_candidate, &peak_current, &peak_candidate, &saved_hash) != 4 || fclose(record) != 0) - fail("read candidate peak"); - if (sample != 0) - printf(","); - printf("%zu", peak_candidate); - remove(path); - } - printf("]}"); - } - eviction_oracle(&fixtures[CODEC_DEFLATE], &fixtures[CODEC_ZSTD], extent_size, argv[8]); - printf("],\"status\":\"PASS\"}\n"); - for (codec = 0; codec < CODEC_COUNT; ++codec) - free(fixtures[codec].compressed); - free(offsets); - free(verification); - free(payload); - return (0); -} -''' - - -def run_benchmark() -> tuple[dict[str, Any], dict[str, str]]: - artifacts = OUTPUT / "artifacts" - artifacts.mkdir() - source = artifacts / "P15-038-codec-bench.c" - source.write_text(BENCH_SOURCE.lstrip(), encoding="ascii") - config = SPEC["benchmark"] - budget = SPEC["budget"] - with tempfile.TemporaryDirectory(prefix="p15-038-") as temporary: - temp = Path(temporary) - binary = temp / "P15-038-codec-bench" - fixture_dir = temp / "fixtures" - fixture_dir.mkdir() - flags = run( - ["pkg-config", "--cflags", "--libs", "liblz4", "liblzma", "zlib", "libzstd"], - timeout=30, - ).stdout.split() - run( - ["cc", "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", str(source), "-o", str(binary), *flags], - timeout=60, - log=artifacts / "compile.log", - ) - completed = run( - [ - str(binary), - str(config["extent_bytes"]), - str(config["read_bytes"]), - str(config["random_reads_per_sample"]), - str(config["samples"]), - str(budget["per_mount_bytes"]), - str(budget["minimum_decode_work_bytes"]), - str(config["seed"]), - str(fixture_dir), - ], - timeout=240, - log=artifacts / "benchmark.log", - allowed={0, 10}, - ) - if completed.returncode != 0: - raise GateFailure("STOP", "real codec benchmark or correctness oracle failed") - try: - report = json.loads(completed.stdout) - except json.JSONDecodeError as error: - raise GateFailure("RUNNER_FAIL", f"invalid benchmark JSON: {error}") from error - hashes = { - path.name: sha256_path(path) - for path in sorted(fixture_dir.iterdir()) - if path.is_file() - } - before = hashes.pop("eviction-before.bin") - after = hashes.pop("eviction-after.bin") - if before != after: - raise GateFailure("STOP", "real codec eviction changed decoded SHA-256") - hashes["eviction-before-after.sha256"] = before - write_json(artifacts / "fixture-hashes.json", hashes) - return report, hashes - - -def evaluate(report: dict[str, Any], state: dict[str, Any]) -> dict[str, Any]: - config = SPEC["benchmark"] - budget = SPEC["budget"]["per_mount_bytes"] - if report.get("status") != "PASS" or report.get("samples") != config["samples"]: - raise GateFailure("STOP", "benchmark did not return exactly five complete samples") - if report.get("reads_per_sample") != config["random_reads_per_sample"]: - raise GateFailure("STOP", "benchmark logical workload changed") - summaries = [] - improved = 0 - for codec in report.get("codecs", []): - current = codec.get("current_ns", []) - candidate = codec.get("candidate_ns", []) - peaks = codec.get("candidate_peak_bytes", []) - if len(current) != 5 or len(candidate) != 5 or len(peaks) != 5: - raise GateFailure("STOP", f"{codec.get('codec')} has a missing sample") - if any(value <= 0 for value in current + candidate): - raise GateFailure("STOP", f"{codec['codec']} has a non-positive latency") - if any(value > budget for value in peaks): - raise GateFailure("STOP", f"{codec['codec']} exceeded the hard budget") - current_median = statistics.median(current) - candidate_median = statistics.median(candidate) - improvement = (current_median - candidate_median) * 100.0 / current_median - passed = improvement >= config["minimum_latency_improvement_percent"] - if passed: - improved += 1 - summaries.append( - { - "candidate_median_ns": candidate_median, - "candidate_ns": candidate, - "candidate_peak_bytes": peaks, - "codec": codec["codec"], - "compressed_bytes": codec["compressed_bytes"], - "current_median_ns": current_median, - "current_ns": current, - "improvement_percent": improvement, - "threshold_pass": passed, - } - ) - if len(summaries) != 4: - raise GateFailure("STOP", "the four-codec benchmark is incomplete") - if improved < config["minimum_improved_codecs"]: - raise GateFailure("STOP", f"only {improved} codecs met the 10 percent latency threshold") - if state.get("status") != "PASS" or state["budget"]["global_remaining"] != 0: - raise GateFailure("STOP", "cache lifecycle state model did not close") - return { - "codecs_meeting_threshold": improved, - "minimum_codecs": config["minimum_improved_codecs"], - "minimum_improvement_percent": config["minimum_latency_improvement_percent"], - "samples_included_per_variant": 5, - "summaries": summaries, - } - - -exit_code = 2 -try: - resolved, identity = verify_identity() - write_json(OUTPUT / "identity.json", {"base": resolved, **identity}) - state = run_state_model() - write_json(OUTPUT / "state-model.json", state) - benchmark, fixture_hashes = run_benchmark() - write_json(OUTPUT / "benchmark.json", benchmark) - benefit = evaluate(benchmark, state) - write_json(OUTPUT / "benefit.json", benefit) - result = { - "b33": "AUTHORIZED", - "candidate": "P15-038", - "decision": "GO", - "eviction_sha256": fixture_hashes["eviction-before-after.sha256"], - "full_feature_suite": "NOT_RUN", - "gate": "G05", - "qemu": "NOT_RUN", - "reason": ( - f"state model PASS; {benefit['codecs_meeting_threshold']}/4 codecs meet " - "the fixed five-sample >=10% host codec latency threshold; hard budgets and " - "eviction hashes pass" - ), - "scope": SPEC["scope"], - "source_modified": False, - "status": "GO", - } - exit_code = 0 -except GateFailure as error: - result = { - "b33": "STOP-NO-SOURCE" if error.status == "STOP" else "NOT_RUN", - "candidate": "P15-038", - "decision": "STOP" if error.status == "STOP" else error.status, - "full_feature_suite": "NOT_RUN", - "gate": "G05", - "qemu": "NOT_RUN", - "reason": error.reason, - "scope": SPEC.get("scope"), - "source_modified": False, - "status": error.status, - } - exit_code = 1 if error.status == "STOP" else 2 -except Exception as error: - result = { - "b33": "NOT_RUN", - "candidate": "P15-038", - "decision": "RUNNER_FAIL", - "full_feature_suite": "NOT_RUN", - "gate": "G05", - "qemu": "NOT_RUN", - "reason": f"unhandled gate error: {type(error).__name__}: {error}", - "scope": SPEC.get("scope"), - "source_modified": False, - "status": "RUNNER_FAIL", - } - exit_code = 2 -write_json(OUTPUT / "result.json", result) -write_json( - OUTPUT / "cleanup.json", - { - "binary_remaining": False, - "owned_processes_started": 0, - "owned_qemu_started": False, - "owned_temp_remaining": [], - "protected_pid_touched": False, - "protected_port_touched": False, - "source_modified": False, - "status": "PASS", - }, -) -lines = [] -for path in sorted(OUTPUT.rglob("*")): - if path.is_file() and path.name != "SHA256SUMS": - lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}") -(OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") -print(json.dumps(result, sort_keys=True)) -raise SystemExit(exit_code) -PY diff --git a/tests/pre15/gates/P15-045-input.json b/tests/pre15/gates/P15-045-input.json deleted file mode 100644 index f41ea35..0000000 --- a/tests/pre15/gates/P15-045-input.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "approved_demand_manifest_sha256": {}, - "batch": "B35", - "candidate": "P15-045", - "demand_manifest_contract": { - "accepted_formats": [ - "JSON object", - "RFC822-style text headers" - ], - "demand_terms": [ - "deployment", - "deploy", - "support", - "manifest", - "policy", - "default", - "required", - "must", - "out of the box", - "out_of_box", - "部署", - "支持", - "默认", - "开箱", - "清单" - ], - "maximum_text_candidate_bytes": 2097152, - "required_semantics": [ - "document identifies itself as a support/deployment manifest", - "manifest has a concrete version", - "manifest names nonempty deployment/support targets", - "manifest explicitly requires Zstd enabled by default/out of the box" - ], - "scope": "planning/pre15/evidence at required_base" - }, - "evidence_tracked_path_count": 2560, - "evidence_tree_oid": "3ecce7a124d153638853ab5d897126c769125309", - "gate": "G07", - "probe_order": [ - "versioned-deployment-demand", - "disabled-runtime-exact-EOPNOTSUPP", - "enabled-real-reproducible-Zstd-EROFS-read", - "FreeBSD-kernel-ZSTDIO-symbol-capability-model", - "KLD-size-delta-at-most-64KiB-and-10-percent" - ], - "prohibited_paths": [ - "planning/pre15/introduction.md" - ], - "protected": { - "pid": 26318, - "port": 9222 - }, - "required_base": "13974efc00c31d8c13c8dccb7c65a82adafcbff6", - "schema": 1, - "size_thresholds": { - "maximum_bytes": 65536, - "maximum_percent": 10.0, - "require_both": true - }, - "source_sha256": { - "planning/pre15/20-final-candidate-ledger.md": "2ba36e8f107cfe5479c4d2bb913519b27e0eb97aa988ecc08e3a7633523620d0", - "planning/pre15/30-stage0-gates.md": "34708bacf4a0668dfd834b3639900ca23b28b9833a387ac6e2c0399c9316cdb8", - "planning/pre15/40-execution-batches.md": "1d1ffdf8fec799aeec063bb1dc4a0fda9703300d414eabef8905522b2c18db20", - "planning/pre15/90-honesty-and-evidence.md": "69f96908f5ab436a2f5359548e4e6f5dc2942fcf00148d999d513558f1c59c2d", - "repo-pre-15/README.md": "2d3e138f5af20ce02aa562286cb42df8fbd00fdad0ae424e426e0fd8a7f72472", - "repo-pre-15/docs/erofs.5": "dea6a9eddbccbb7b8779b6b4e43772edeea9f0e51696c1b8fd09e4d8e69b52f5", - "repo-pre-15/docs/features.md": "6d875eb582ef9f00391f90ee5b4b8aec0429f24f0ae3c7b27e788ab1e2c48f2e", - "repo-pre-15/src/Makefile": "b722f7ec658e0ada0640c80a7b527921bd96cdf0e93e9623cece29766f2fc9c8", - "repo-pre-15/src/decompressor_zstd.c": "c5f19c9ea6d8238bcf30b8b3228b80e77cf2073df1d07e4fac5f5401bef413bb" - } -} diff --git a/tests/pre15/gates/P15-045.sh b/tests/pre15/gates/P15-045.sh deleted file mode 100755 index 4980645..0000000 --- a/tests/pre15/gates/P15-045.sh +++ /dev/null @@ -1,564 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-045-input.json -base= -output= -deadline=90 - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 20; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 20; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 20 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 20; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 20; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 20; } -for tool in git mktemp python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 21 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 20; } -mkdir -p "$output" -work=$(mktemp -d "${TMPDIR:-/tmp}/P15-045.XXXXXX") - -cleanup_trap() -{ - rm -rf -- "$work" -} -trap cleanup_trap EXIT HUP INT TERM - -python3 -B - "$output/command-argv.json" "$0" --base "$base" --output "$output" <<'PY' -import json -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text( - json.dumps(sys.argv[2:], ensure_ascii=True, separators=(",", ":")) + "\n", - encoding="ascii", -) -PY -python3 -B - "$output/ownership.json" "$work" "$output" <<'PY' -import json -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text( - json.dumps( - { - "owned_temporary_paths": [sys.argv[2]], - "persistent_output": sys.argv[3], - "owned_processes": [], - "owned_ports": [], - "qemu": "NOT_RUN", - }, - ensure_ascii=True, - indent=2, - sort_keys=True, - ) + "\n", - encoding="ascii", -) -PY - -set +e -timeout -k 5 "$deadline" python3 -B - \ - "$root" "$input" "$base" "$output" "$deadline" <<'PY' \ - >"$output/stdout.log" 2>"$output/stderr.log" -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -import subprocess -import sys -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -DEADLINE = int(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="utf-8")) - - -class GateFailure(RuntimeError): - def __init__(self, status: str, reason: str): - super().__init__(reason) - self.status = status - self.reason = reason - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def write_json(path: Path, value: Any) -> None: - path.write_text( - json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n", - encoding="ascii", - ) - - -def run_bytes(argv: list[str], allowed: set[int] | None = None) -> bytes: - try: - completed = subprocess.run( - argv, - cwd=ROOT, - check=False, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=30, - ) - except subprocess.TimeoutExpired as error: - raise GateFailure( - "INFRA_BLOCKED", f"command timed out: {' '.join(argv)}" - ) from error - accepted = {0} if allowed is None else allowed - if completed.returncode not in accepted: - detail = completed.stderr.decode("utf-8", errors="replace").strip() - raise GateFailure( - "RUNNER_FAIL", - f"command failed ({completed.returncode}): {' '.join(argv)}: {detail}", - ) - return completed.stdout - - -def git_text(*args: str, allowed: set[int] | None = None) -> str: - return run_bytes(["git", "-C", str(ROOT), *args], allowed).decode( - "utf-8", errors="strict" - ).strip() - - -def source_at(commit: str, relative: str) -> bytes: - return run_bytes(["git", "-C", str(ROOT), "show", f"{commit}:{relative}"]) - - -def truthy(value: Any) -> bool: - if value is True: - return True - if isinstance(value, str): - return value.strip().lower() in {"true", "yes", "required", "enabled", "1"} - if isinstance(value, int): - return value == 1 - return False - - -def flattened(value: Any, prefix: str = "") -> list[tuple[str, Any]]: - records: list[tuple[str, Any]] = [] - if isinstance(value, dict): - for key, child in value.items(): - path = f"{prefix}.{key}" if prefix else str(key) - records.append((path.lower().replace("-", "_"), child)) - records.extend(flattened(child, path)) - elif isinstance(value, list): - for index, child in enumerate(value): - records.extend(flattened(child, f"{prefix}[{index}]")) - return records - - -def json_demand_witness(value: Any) -> dict[str, Any] | None: - if not isinstance(value, dict): - return None - records = flattened(value) - kind = any( - any(token in key for token in ("manifest_type", "manifest_kind", "document_type")) - and isinstance(item, str) - and any(token in item.lower() for token in ("support", "deploy")) - for key, item in records - ) - version = any( - "version" in key and isinstance(item, (str, int)) and str(item).strip() - for key, item in records - ) - targets = any( - any(token in key for token in ("deployment_target", "support_target", "product_target")) - and ((isinstance(item, str) and item.strip()) or (isinstance(item, list) and item)) - for key, item in records - ) - required = any( - "zstd" in key - and any(token in key for token in ("default", "required", "out_of_box", "outofthebox")) - and truthy(item) - for key, item in records - ) - if kind and version and targets and required: - return { - "format": "json", - "manifest_kind": True, - "version": True, - "deployment_targets": True, - "zstd_default_required": True, - } - return None - - -def text_demand_witness(text: str) -> dict[str, Any] | None: - fields: dict[str, str] = {} - for line in text.splitlines(): - match = re.fullmatch(r"([A-Za-z][A-Za-z0-9_-]*):[ \t]*(.*)", line) - if match is not None: - fields[match.group(1).lower().replace("-", "_")] = match.group(2).strip() - kind = fields.get("manifest_type", "").lower() - version = fields.get("manifest_version", "") - targets = fields.get("deployment_targets", "") or fields.get("support_targets", "") - required = fields.get("zstd_default_required", "") - if any(token in kind for token in ("support", "deploy")) and version and targets and truthy(required): - return { - "format": "text-header", - "manifest_kind": True, - "version": True, - "deployment_targets": True, - "zstd_default_required": True, - } - return None - - -def parse_ls_tree(raw: bytes) -> dict[str, dict[str, Any]]: - entries: dict[str, dict[str, Any]] = {} - for record in raw.split(b"\0"): - if not record: - continue - metadata, path_bytes = record.split(b"\t", 1) - mode, kind, oid, size = metadata.decode("ascii").split() - path = path_bytes.decode("utf-8", errors="strict") - entries[path] = { - "mode": mode, - "kind": kind, - "oid": oid, - "size": int(size), - } - return entries - - -def grep_paths(commit: str) -> list[str]: - raw = run_bytes( - [ - "git", "-C", str(ROOT), "grep", "-I", "-l", "-i", - "-e", "zstd", "-e", "zstandard", commit, "--", - "planning/pre15/evidence", - ], - {0, 1}, - ) - prefix = f"{commit}:" - paths = [] - for line in raw.decode("utf-8", errors="strict").splitlines(): - if not line.startswith(prefix): - raise GateFailure("RUNNER_FAIL", f"unexpected git grep path: {line}") - paths.append(line[len(prefix):]) - return sorted(paths) - - -def main() -> int: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-045": - raise GateFailure("RUNNER_FAIL", "invalid P15-045 input identity") - if SPEC.get("gate") != "G07" or SPEC.get("batch") != "B35": - raise GateFailure("RUNNER_FAIL", "invalid G07/B35 input identity") - resolved = git_text("rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise GateFailure( - "INFRA_BLOCKED", - f"P15-045 must replay {SPEC['required_base']}, got {resolved}", - ) - evidence_tree = git_text("rev-parse", f"{resolved}:planning/pre15/evidence") - if evidence_tree != SPEC["evidence_tree_oid"]: - raise GateFailure("INFRA_BLOCKED", "frozen project evidence tree changed") - - source_hashes = { - path: sha256_bytes(source_at(resolved, path)) - for path in SPEC["source_sha256"] - } - if source_hashes != SPEC["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "frozen G07 source identity changed") - write_json(OUTPUT / "source-sha256.json", source_hashes) - addendum_paths = ( - "repo-pre-15/tests/pre15/gates/P15-045.sh", - "repo-pre-15/tests/pre15/gates/P15-045-input.json", - "repo-pre-15/docs/pre15-stage0/P15-045.md", - ) - addendum_hashes = { - path: sha256_bytes((ROOT / path).read_bytes()) for path in addendum_paths - } - write_json(OUTPUT / "gate-addendum-sha256.json", addendum_hashes) - - tree_entries = parse_ls_tree( - run_bytes( - [ - "git", "-C", str(ROOT), "ls-tree", "-r", "--long", "-z", - resolved, "--", "planning/pre15/evidence", - ] - ) - ) - if len(tree_entries) != SPEC["evidence_tracked_path_count"]: - raise GateFailure("INFRA_BLOCKED", "frozen evidence path count changed") - prohibited = set(SPEC["prohibited_paths"]) - if prohibited.intersection(tree_entries): - raise GateFailure("RUNNER_FAIL", "prohibited path entered evidence scan") - - zstd_paths = grep_paths(resolved) - maximum_bytes = SPEC["demand_manifest_contract"]["maximum_text_candidate_bytes"] - demand_terms = tuple(SPEC["demand_manifest_contract"]["demand_terms"]) - candidates = [] - qualified = [] - oversized = [] - for path in zstd_paths: - entry = tree_entries[path] - if entry["size"] > maximum_bytes: - oversized.append({"path": path, **entry}) - continue - data = source_at(resolved, path) - text = data.decode("utf-8", errors="replace") - lowered = text.lower() - signals = sorted(term for term in demand_terms if term.lower() in lowered) - if not signals: - continue - witness = None - if path.endswith(".json"): - try: - witness = json_demand_witness(json.loads(text)) - except json.JSONDecodeError: - witness = None - if witness is None: - witness = text_demand_witness(text) - record = { - "path": path, - "blob_oid": entry["oid"], - "bytes": entry["size"], - "sha256": sha256_bytes(data), - "demand_signals": signals, - "qualification": witness, - "disposition": ( - "QUALIFIED versioned deployment-demand manifest" - if witness is not None - else "test/build/planning evidence; no explicit versioned deployment demand" - ), - } - candidates.append(record) - if witness is not None: - qualified.append(record) - - if oversized: - write_json(OUTPUT / "oversized-candidates.json", oversized) - raise GateFailure( - "INFRA_BLOCKED", "a Zstd evidence candidate exceeds the bounded review size" - ) - - approved = SPEC["approved_demand_manifest_sha256"] - actual_approved = {record["path"]: record["sha256"] for record in qualified} - if actual_approved != approved: - raise GateFailure( - "INFRA_BLOCKED", "qualified demand inventory differs from frozen gate input" - ) - - makefile = source_at(resolved, "repo-pre-15/src/Makefile").decode("utf-8") - zstd_source = source_at( - resolved, "repo-pre-15/src/decompressor_zstd.c" - ).decode("utf-8") - readme = source_at(resolved, "repo-pre-15/README.md").decode("utf-8") - manual = source_at(resolved, "repo-pre-15/docs/erofs.5").decode("utf-8") - policy_checks = { - "makefile_default_is_opt_in_zero": makefile.count("WITH_ZSTDIO?= 0") == 1, - "enabled_build_defines_zstdio": "CFLAGS.decompressor_zstd.c+= -DZSTDIO" in makefile, - "disabled_stub_returns_eopnotsupp": ( - "#else\nstatic int\nz_erofs_zstd_decompress" in zstd_source - and "return (EOPNOTSUPP);\n}\n#endif" in zstd_source - ), - "readme_names_kernel_option": "kernel built with `options ZSTDIO`" in readme, - "manual_names_kernel_option": '.Cd "options ZSTDIO"' in manual, - } - if not all(policy_checks.values()): - raise GateFailure("INFRA_BLOCKED", "current opt-in ZSTDIO contract changed") - - path_inventory = "\0".join(zstd_paths).encode("utf-8") - candidate_inventory = "\0".join( - record["path"] for record in candidates - ).encode("utf-8") - demand_scan = { - "schema": 1, - "base": resolved, - "evidence_tree_oid": evidence_tree, - "tracked_path_count": len(tree_entries), - "zstd_text_path_count": len(zstd_paths), - "zstd_text_path_inventory_sha256": sha256_bytes(path_inventory), - "demand_candidate_count": len(candidates), - "demand_candidate_path_inventory_sha256": sha256_bytes(candidate_inventory), - "qualified_manifest_count": len(qualified), - "qualified_manifests": qualified, - "candidates": candidates, - "contract": SPEC["demand_manifest_contract"], - "prohibited_path_read": False, - } - write_json(OUTPUT / "demand-scan.json", demand_scan) - - demand_present = bool(qualified) - conditions = [ - { - "id": "versioned-deployment-demand", - "status": "PASS" if demand_present else "STOP", - "observation": ( - f"{len(qualified)} qualified manifest(s)" - if demand_present - else "zero qualifying manifests in the frozen project evidence tree" - ), - }, - { - "id": "disabled-runtime-exact-EOPNOTSUPP", - "status": "NOT_RUN", - "reason": "deployment-demand prerequisite is absent", - }, - { - "id": "enabled-real-reproducible-Zstd-EROFS-read", - "status": "NOT_RUN", - "reason": "deployment-demand prerequisite is absent", - }, - { - "id": "FreeBSD-kernel-ZSTDIO-symbol-capability-model", - "status": "NOT_RUN", - "reason": "deployment-demand prerequisite is absent; no dependency KLD is claimed", - }, - { - "id": "KLD-size-delta-at-most-64KiB-and-10-percent", - "status": "NOT_RUN", - "reason": "deployment-demand prerequisite is absent", - }, - ] - all_go = all(item["status"] == "PASS" for item in conditions) - if all_go: - raise GateFailure( - "RUNNER_FAIL", "false-GO guard: downstream runtime probes were not executed" - ) - result = { - "schema": 1, - "gate": "G07", - "candidate": "P15-045", - "batch": "B35", - "status": "STOP", - "reason": "no concrete versioned support/deployment manifest requires Zstd enabled out of the box", - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "deadline_seconds": DEADLINE, - "expected_exit_code": 22, - "conditions": conditions, - "false_go_guard": "GO requires every condition PASS; evaluated false", - "current_policy_static_checks": policy_checks, - "with_zstdio_default": 0, - "b35": "STOP-NO-SOURCE", - "builds": "NOT_RUN", - "qemu": "NOT_RUN", - "full_feature_suite": "NOT_RUN", - "smoke_suite": "NOT_RUN", - "generic_dependency_kld_claimed": False, - "production_source_changed": False, - } - write_json(OUTPUT / "result.json", result) - print(json.dumps(result, ensure_ascii=True, sort_keys=True)) - return 22 - - -try: - raise SystemExit(main()) -except GateFailure as error: - failure = { - "schema": 1, - "gate": "G07", - "candidate": "P15-045", - "status": error.status, - "reason": error.reason, - "requested_base": REQUESTED_BASE, - "deadline_seconds": DEADLINE, - "b35": "NOT_RUN", - "qemu": "NOT_RUN", - "production_source_changed": False, - } - write_json(OUTPUT / "result.json", failure) - print(json.dumps(failure, ensure_ascii=True, sort_keys=True)) - raise SystemExit({"RUNNER_FAIL": 20, "INFRA_BLOCKED": 21}.get(error.status, 20)) -PY -gate_rc=$? -set -e - -cleanup_status=PASS -if ! rm -rf -- "$work"; then - cleanup_status=FAIL - gate_rc=20 -fi -trap - EXIT HUP INT TERM -printf 'owned temporary path removed: %s\ncleanup=%s\n' \ - "$work" "$cleanup_status" >"$output/cleanup.log" - -python3 -B - "$output" "$gate_rc" "$cleanup_status" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import sys - - -output = Path(sys.argv[1]) -exit_code = int(sys.argv[2]) -cleanup = sys.argv[3] -if cleanup != "PASS": - status = "RUNNER_FAIL" - origin = "runner" -elif exit_code == 22: - status = "STOP" - origin = "gate" -elif exit_code in (124, 137): - status = "INFRA_BLOCKED" - origin = "infrastructure" -else: - status = "RUNNER_FAIL" - origin = "runner" -(output / "attempt.json").write_text( - json.dumps( - { - "schema": 1, - "exit_code": exit_code, - "status": status, - "failure_origin": origin, - "cleanup": cleanup, - "target_marker": "reached" if status == "STOP" else "not_reached", - }, - ensure_ascii=True, - indent=2, - sort_keys=True, - ) + "\n", - encoding="ascii", -) -lines = [] -for path in sorted(output.iterdir(), key=lambda item: item.name): - if path.is_file() and path.name != "SHA256SUMS": - digest = hashlib.sha256(path.read_bytes()).hexdigest() - lines.append(f"{digest} {path.name}") -(output / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") -PY - -printf 'P15-045 gate exit=%s cleanup=%s output=%s\n' \ - "$gate_rc" "$cleanup_status" "$output" -exit "$gate_rc" diff --git a/tests/pre15/gates/P15-052-input.json b/tests/pre15/gates/P15-052-input.json deleted file mode 100644 index e5cd09b..0000000 --- a/tests/pre15/gates/P15-052-input.json +++ /dev/null @@ -1,203 +0,0 @@ -{ - "schema": 1, - "gate": "G06", - "candidate": "P15-052", - "required_base": "ca7bb4fe6b33e4a1bdf423801134b0ed6bda86dd", - "title": "exact on-disk and ABI errno taxonomy", - "source_sha256": { - "repo-pre-15/src/data.c": "cb22072bd4c092aa6a5376add8c8d9d6f94297f6292900ecc2eef9b01c69e92a", - "repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1", - "repo-pre-15/src/inode.c": "ba6f77ddffa35cc1dd69ebb933fb16f8fa654f94552fd1d3f0255eb790673e6c", - "repo-pre-15/src/internal.h": "c08ce3dbcafca5c341193c1670516e9bd5815c5f14ce2d637dbf6cfe5a74e972", - "repo-pre-15/src/super.c": "8bb36e9aba15eab2ed10ebec65dccb52dc9ddb485a1a39df37f86f97502c674f", - "repo-pre-15/src/xattr.c": "3498af0b547331b9a22babe96ea718ca2ba56ee669001bfdc2dbaab0a9dc42a9", - "src-linux/data.c": "8625cdc01e5405f856178ae8fd559696ae85f607f19caf229b867a3b7479318a", - "src-linux/internal.h": "4aa671896ff7c0ad32a9108c818ef62d16841c116706fdad391c40a530c81405", - "src-linux/super.c": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9", - "src-linux/xattr.c": "c8394e5f6301225cbe7587f223485a368348eac7596c1ab0bbf965c99655ed4a" - }, - "function_errno_counts": { - "repo-pre-15/src/data.c:erofs_map_blocks_chunk": { - "EOVERFLOW": 6 - }, - "repo-pre-15/src/xattr.c:erofs_xattr_backing_size": { - "EOVERFLOW": 1 - }, - "repo-pre-15/src/xattr.c:erofs_xattr_read_backing": { - "EOVERFLOW": 1 - }, - "repo-pre-15/src/xattr.c:erofs_xattr_read_metadata": { - "EOVERFLOW": 2 - }, - "repo-pre-15/src/xattr.c:erofs_xattr_shared_entry_offset": { - "EOVERFLOW": 2 - } - }, - "targets": [ - { - "id": "data.chunk.inode_plus_isize", - "path": "repo-pre-15/src/data.c", - "function": "erofs_map_blocks_chunk", - "field": "inode_off", - "needle": "vi->inode_off > UINT64_MAX - vi->inode_isize", - "prototype_mutation": "inode_off=UINT64_MAX-31" - }, - { - "id": "data.chunk.isize_plus_xattr", - "path": "repo-pre-15/src/data.c", - "function": "erofs_map_blocks_chunk", - "field": "xattr_isize", - "needle": "vi->inode_off + vi->inode_isize > UINT64_MAX - vi->xattr_isize", - "prototype_mutation": "xattr_isize=128 with inode_off near UINT64_MAX" - }, - { - "id": "data.chunk.align", - "path": "repo-pre-15/src/data.c", - "function": "erofs_map_blocks_chunk", - "field": "inode_off", - "needle": "idx_base > UINT64_MAX - (entry_size - 1)", - "prototype_mutation": "inode_off=UINT64_MAX-3" - }, - { - "id": "data.chunk.index_multiply", - "path": "repo-pre-15/src/data.c", - "function": "erofs_map_blocks_chunk", - "field": "chunk_idx", - "needle": "chunk_idx > (UINT64_MAX - idx_base) / entry_size", - "prototype_mutation": "chunk_idx=(UINT64_MAX-4096)/8+1" - }, - { - "id": "data.chunk.image_size_shift", - "path": "repo-pre-15/src/data.c", - "function": "erofs_map_blocks_chunk", - "field": "blocks", - "needle": "sbi->blocks > (UINT64_MAX >> sbi->blkszbits)", - "prototype_mutation": "blocks=(UINT64_MAX>>12)+1" - }, - { - "id": "data.chunk.physical_shift", - "path": "repo-pre-15/src/data.c", - "function": "erofs_map_blocks_chunk", - "field": "blkaddr", - "needle": "blkaddr > (UINT64_MAX >> sbi->blkszbits)", - "prototype_mutation": "blkaddr=(UINT64_MAX>>12)+1" - }, - { - "id": "data.chunk.physical_plus_offset", - "path": "repo-pre-15/src/data.c", - "function": "erofs_map_blocks_chunk", - "field": "chunk_off", - "needle": "chunk_off > UINT64_MAX - map->m_pa", - "prototype_mutation": "chunk_off=4096 with blkaddr=UINT64_MAX>>12" - }, - { - "id": "xattr.backing_size.shift", - "path": "repo-pre-15/src/xattr.c", - "function": "erofs_xattr_backing_size", - "field": "blocks", - "needle": "sbi->blocks > (UINT64_MAX >> sbi->blkszbits)", - "prototype_mutation": "blocks=(UINT64_MAX>>12)+1" - }, - { - "id": "xattr.metadata.align", - "path": "repo-pre-15/src/xattr.c", - "function": "erofs_xattr_read_metadata", - "field": "offp", - "needle": "*offp > UINT64_MAX - (sizeof(struct erofs_xattr_entry) - 1)", - "prototype_mutation": "offp=UINT64_MAX-2" - }, - { - "id": "xattr.metadata.header_add", - "path": "repo-pre-15/src/xattr.c", - "function": "erofs_xattr_read_metadata", - "field": "aligned_off", - "needle": "off > UINT64_MAX - sizeof(raw_len)", - "prototype_mutation": "aligned_off=UINT64_MAX-1" - }, - { - "id": "xattr.shared.base_shift", - "path": "repo-pre-15/src/xattr.c", - "function": "erofs_xattr_shared_entry_offset", - "field": "xattr_blkaddr", - "needle": "sbi->xattr_blkaddr > (UINT64_MAX >> sbi->blkszbits)", - "prototype_mutation": "xattr_blkaddr=(UINT64_MAX>>12)+1" - }, - { - "id": "xattr.shared.base_plus_relative", - "path": "repo-pre-15/src/xattr.c", - "function": "erofs_xattr_shared_entry_offset", - "field": "shared_id", - "needle": "relative > UINT64_MAX - base", - "prototype_mutation": "shared_id=1024 with base near UINT64_MAX" - } - ], - "preservation_controls": [ - { - "id": "abi.off_gt_int64", - "freebsd": "+EOVERFLOW", - "linux": "-EOVERFLOW semantic", - "result": "blocked first by backing-size range for every mounted primary provider" - }, - { - "id": "corruption.range_past_eof", - "freebsd": "+EINTEGRITY", - "linux": "-EFSCORRUPTED semantic", - "result": "preserve" - }, - { - "id": "eof.zero_length", - "freebsd": "0", - "linux": "0", - "result": "preserve" - }, - { - "id": "io.provider_error", - "freebsd": "+EIO", - "linux": "negative PTR_ERR", - "result": "preserve exact provider error" - }, - { - "id": "io.provider_short_media", - "freebsd": "+ENXIO", - "linux": "negative provider error", - "result": "preserve exact media-size error" - }, - { - "id": "short_read.logical_backing", - "freebsd": "+EINTEGRITY before I/O", - "linux": "-EFSCORRUPTED semantic", - "result": "preserve" - }, - { - "id": "unsupported.exact_xattr_header", - "freebsd": "+EOPNOTSUPP", - "linux": "-EOPNOTSUPP", - "result": "preserve" - }, - { - "id": "allocation.xattr_prefixes", - "freebsd": "+ENOMEM", - "linux": "-ENOMEM", - "result": "out of B20 scope" - } - ], - "prototype_identity": { - "path": "/work/pre15-gate-prep-20260814T125129Z/G06", - "errno_model.py": "dca427acf5c919f78d46ab8dbb198778fbe2af0869043d93b4eddf9f03814a4e", - "corpus/errno-vectors.json": "dde22db91f7fa2613631b4acb75a5d60e255a1c1564be6800db377922a6a7d4e", - "input.json": "0835defd288cb6a73337c116c4148daee94b78c435732032b5de40d1b1559715", - "oracle.md": "62e6703e8051b832108b4ea39ece232f06aea5dcfb221d7e8c17faf7ae527208", - "replay-base.sh": "2162fb645693ce740563a2406ea28ab86dc116a33163e026fc4d491c2df3e8a5" - }, - "decision": { - "go": "every target is independently reachable from one validated on-disk field mutation, reaches its named source branch, and has one unique source class", - "stop": "any target is unreachable after current decode/mount bounds, any prototype mutation injects a local rather than an on-disk input, or any source class is ambiguous", - "atomicity": "P15-052 has no partial source implementation" - }, - "timeouts": { - "host_seconds": 240, - "qemu_seconds": 1200 - }, - "qemu_required_after_host_stop": false, - "full_feature_suite_required": false -} diff --git a/tests/pre15/gates/P15-052.sh b/tests/pre15/gates/P15-052.sh deleted file mode 100755 index 8693e58..0000000 --- a/tests/pre15/gates/P15-052.sh +++ /dev/null @@ -1,472 +0,0 @@ -#!/bin/sh -set -eu - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-052-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { - printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2 - exit 2 -} -for tool in git python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" - -python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -import subprocess -import sys -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="ascii")) - -U32_MAX = (1 << 32) - 1 -U48_MAX = (1 << 48) - 1 -U64_MAX = (1 << 64) - 1 -I64_MAX = (1 << 63) - 1 - - -def git(*args: str) -> str: - return subprocess.check_output( - ["git", "-C", str(ROOT), *args], text=True - ).strip() - - -def sha256(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def source_at(commit: str, path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read {path} at {commit}: {completed.stderr}") - return completed.stdout - - -def extract_function(source: str, name: str) -> str: - match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE) - if match is None: - raise SystemExit(f"function not found: {name}") - start = source.rfind("\n\n", 0, match.start()) + 2 - brace = source.find("{", match.end()) - if brace < 0: - raise SystemExit(f"function body not found: {name}") - depth = 0 - state = "code" - index = brace - while index < len(source): - char = source[index] - following = source[index + 1] if index + 1 < len(source) else "" - if state == "code": - if char == "/" and following == "*": - state = "block" - index += 2 - continue - if char == "/" and following == "/": - state = "line" - index += 2 - continue - if char == '"': - state = "string" - elif char == "'": - state = "character" - elif char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return source[start : index + 1] + "\n" - elif state == "block" and char == "*" and following == "/": - state = "code" - index += 2 - continue - elif state == "line" and char == "\n": - state = "code" - elif state in {"string", "character"}: - if char == "\\": - index += 2 - continue - if (state == "string" and char == '"') or ( - state == "character" and char == "'" - ): - state = "code" - index += 1 - raise SystemExit(f"unterminated function: {name}") - - -def line_number(source: str, needle: str) -> int: - if source.count(needle) != 1: - raise SystemExit( - f"source needle must occur exactly once ({source.count(needle)}): {needle}" - ) - return source.count("\n", 0, source.index(needle)) + 1 - - -if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-052": - raise SystemExit("invalid P15-052 gate input") -resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}") -if resolved != SPEC["required_base"]: - raise SystemExit( - f"P15-052 must replay {SPEC['required_base']}, got {resolved}" - ) - -sources = { - path: source_at(resolved, path) for path in SPEC["source_sha256"] -} -source_hashes = { - path: sha256(text.encode("utf-8")) for path, text in sources.items() -} -for path, expected in SPEC["source_sha256"].items(): - if source_hashes[path] != expected: - raise SystemExit(f"frozen source identity mismatch: {path}") - -function_bodies: dict[str, str] = {} -for key, counts in SPEC["function_errno_counts"].items(): - path, function = key.rsplit(":", 1) - body = extract_function(sources[path], function) - function_bodies[key] = body - for errno_name, expected_count in counts.items(): - actual_count = body.count(f"return ({errno_name});") - if actual_count != expected_count: - raise SystemExit( - f"{key} {errno_name} count changed: {actual_count} != {expected_count}" - ) - if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", body): - raise SystemExit(f"negative errno entered FreeBSD function: {key}") - -param_path = FREEBSD_SRC / "sys/amd64/include/param.h" -types_path = FREEBSD_SRC / "sys/sys/_types.h" -geom_path = FREEBSD_SRC / "sys/geom/geom.h" -for path in (param_path, types_path, geom_path): - if not path.is_file(): - raise SystemExit(f"missing FreeBSD contract source: {path}") -param_source = param_path.read_text(encoding="utf-8") -types_source = types_path.read_text(encoding="utf-8") -geom_source = geom_path.read_text(encoding="utf-8") -page_match = re.search(r"^#define\s+PAGE_SHIFT\s+(\d+)\b", param_source, re.MULTILINE) -if page_match is None: - raise SystemExit("PAGE_SHIFT is absent from amd64 param.h") -page_shift = int(page_match.group(1)) -if page_shift != 12: - raise SystemExit(f"unexpected audited amd64 PAGE_SHIFT: {page_shift}") -if "typedef\t__int64_t\t__off_t;" not in types_source: - raise SystemExit("FreeBSD off_t source is not the audited signed 64-bit type") -if "off_t\t\t\tmediasize;" not in geom_source: - raise SystemExit("GEOM mediasize is not the audited off_t field") - -freebsd_head = subprocess.check_output( - ["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True -).strip() -freebsd_contract = { - "head": freebsd_head, - "page_shift": page_shift, - "off_t": "signed-64", - "geom_mediasize": "off_t", - "files": { - str(path.relative_to(FREEBSD_SRC)): sha256(path.read_bytes()) - for path in (param_path, types_path, geom_path) - }, -} - -# These are maxima after the current super/inode decode and mount-time media -# checks, not unconstrained C-structure values. -inode_slot_min = 32 -inode_isize_max = 64 -xattr_isize_max = 4 + 4 * ((1 << 16) - 2) -inode_off_max = I64_MAX - inode_slot_min -idx_base_max = inode_off_max + inode_isize_max + xattr_isize_max -entry_size_max = 8 -chunkbits_min = 9 -chunkbits_max = page_shift + 31 -chunk_idx_max = I64_MAX >> chunkbits_min -blkaddr_max = U48_MAX -physical_max = blkaddr_max << page_shift -chunk_off_max = (1 << chunkbits_max) - 1 -prefix_record_max = 256 -prefix_off_max = U32_MAX * 4 + 255 * (3 + 2 + prefix_record_max) -xattr_base_max = U32_MAX << page_shift -xattr_relative_max = U32_MAX * 4 - -reachability: dict[str, tuple[bool, str, dict[str, int]]] = { - "data.chunk.inode_plus_isize": ( - inode_off_max > U64_MAX - inode_isize_max, - "inode_off is bounded by mounted backing size minus one compact inode", - {"inode_off_max": inode_off_max, "inode_isize_max": inode_isize_max}, - ), - "data.chunk.isize_plus_xattr": ( - inode_off_max + inode_isize_max > U64_MAX - xattr_isize_max, - "inode offset plus maximum 64-byte inode and 16-bit xattr body stays below UINT64_MAX", - {"metadata_base_max": inode_off_max + inode_isize_max, "xattr_isize_max": xattr_isize_max}, - ), - "data.chunk.align": ( - idx_base_max > U64_MAX - (entry_size_max - 1), - "validated backing and bounded inode/xattr sizes leave alignment headroom", - {"idx_base_max": idx_base_max, "alignment_slack": entry_size_max - 1}, - ), - "data.chunk.index_multiply": ( - chunk_idx_max > (U64_MAX - idx_base_max) // entry_size_max, - "size <= OFF_MAX and chunkbits >= 9 bound the index product", - {"idx_base_max": idx_base_max, "chunk_idx_max": chunk_idx_max, "entry_size_max": entry_size_max}, - ), - "data.chunk.image_size_shift": ( - I64_MAX > U64_MAX, - "mount validates blocks << blkszbits <= GEOM off_t mediasize", - {"validated_image_bytes_max": I64_MAX}, - ), - "data.chunk.physical_shift": ( - blkaddr_max > (U64_MAX >> page_shift), - "decoded chunk block address is at most 48 bits and blkszbits is at most PAGE_SHIFT", - {"blkaddr_max": blkaddr_max, "blkszbits_max": page_shift}, - ), - "data.chunk.physical_plus_offset": ( - chunk_off_max > U64_MAX - physical_max, - "48-bit block address and maximum validated chunkbits leave addition headroom", - {"physical_max": physical_max, "chunk_off_max": chunk_off_max}, - ), - "xattr.backing_size.shift": ( - I64_MAX > U64_MAX, - "mounted primary backing bytes are already bounded by GEOM off_t mediasize", - {"validated_backing_bytes_max": I64_MAX}, - ), - "xattr.metadata.align": ( - prefix_off_max > U64_MAX - 3, - "32-bit prefix start, 8-bit count, and bounded records cannot approach UINT64_MAX", - {"prefix_off_max": prefix_off_max}, - ), - "xattr.metadata.header_add": ( - False, - "the preceding off <= UINT64_MAX-3 check and 4-byte roundup imply aligned off <= UINT64_MAX-3, so adding uint16_t cannot overflow", - {"accepted_input_max": U64_MAX - 3, "aligned_off_max": U64_MAX - 3, "raw_len_size": 2}, - ), - "xattr.shared.base_shift": ( - U32_MAX > (U64_MAX >> page_shift), - "xattr_blkaddr is 32-bit and blkszbits is at most PAGE_SHIFT", - {"xattr_blkaddr_max": U32_MAX, "blkszbits_max": page_shift}, - ), - "xattr.shared.base_plus_relative": ( - xattr_relative_max > U64_MAX - xattr_base_max, - "32-bit xattr block address and 32-bit shared ID cannot overflow the 64-bit sum", - {"xattr_base_max": xattr_base_max, "relative_max": xattr_relative_max}, - ), -} - -ledger = [] -for target in SPEC["targets"]: - target_id = target["id"] - if target_id not in reachability: - raise SystemExit(f"target has no reachability proof: {target_id}") - body = function_bodies[f"{target['path']}:{target['function']}"] - if body.count(target["needle"]) != 1: - raise SystemExit(f"target source is absent or ambiguous: {target_id}") - reachable, reason, bounds = reachability[target_id] - ledger.append( - { - **target, - "line": line_number(sources[target["path"]], target["needle"]), - "source_unique": True, - "target_marker_reached": reachable, - "current_freebsd_return": "+EOVERFLOW", - "candidate_freebsd_return": "+EINTEGRITY", - "linux_semantic_return": "-EFSCORRUPTED (no direct checked counterpart)", - "source_class": "on-disk contradiction", - "replay_observation": reason, - "provenance_bounds": bounds, - } - ) - -data_source = sources["repo-pre-15/src/data.c"] -xattr_source = sources["repo-pre-15/src/xattr.c"] -linux_xattr = sources["src-linux/xattr.c"] -control_anchors = ( - (xattr_source, "if (off > backing_size || (uint64_t)len > backing_size - off)\n\t\treturn (EINTEGRITY);"), - (xattr_source, "if (off > INT64_MAX)\n\t\t\treturn (EOVERFLOW);"), - (xattr_source, "error = EOPNOTSUPP;"), - (data_source, "return (ENXIO);"), - (data_source, "return (EIO);"), - (data_source, "if (len == 0) {\n\t\treturn (0);\n\t}"), - (linux_xattr, "ret = -EOPNOTSUPP;"), - (linux_xattr, "ret = -ENOMEM;"), - (linux_xattr, "return PTR_ERR(it->kaddr);"), - (linux_xattr, "return -EFSCORRUPTED;"), -) -for source, anchor in control_anchors: - if anchor not in source: - raise SystemExit(f"errno preservation anchor is absent: {anchor}") - -abi_reachable = False -abi_reason = ( - "erofs_xattr_read_backing checks off <= backing_size before off > INT64_MAX; " - "for a mounted primary backing, backing_size <= GEOM signed off_t mediasize" -) -controls = [] -for control in SPEC["preservation_controls"]: - record: dict[str, Any] = dict(control) - record["source_anchors_verified"] = True - if control["id"] == "abi.off_gt_int64": - record["target_marker_reached"] = abi_reachable - record["replay_observation"] = abi_reason - controls.append(record) - -unreachable = [record["id"] for record in ledger if not record["target_marker_reached"]] -status = "GO" if not unreachable else "STOP" -stop_reasons = [ - { - "id": target_id, - "reason": next( - record["replay_observation"] for record in ledger if record["id"] == target_id - ), - } - for target_id in unreachable -] - -ledger_document = { - "schema": 1, - "gate": "G06", - "candidate": "P15-052", - "status": status, - "resolved_base": resolved, - "targets": ledger, - "preservation_controls": controls, - "freebsd_contract": freebsd_contract, - "prototype_identity": SPEC["prototype_identity"], - "prototype_disposition": ( - "READY arithmetic vectors inject unconstrained internal or local values; " - "they are not independently reachable on-disk fixtures" - ), -} -(OUTPUT / "branch-ledger.json").write_text( - json.dumps(ledger_document, indent=2, sort_keys=True) + "\n", encoding="ascii" -) - -tsv = [ - "id\tfunction\tfield\tfreebsd_now\tfreebsd_candidate\tlinux_semantic\tmarker\tobservation" -] -for record in ledger: - tsv.append( - "\t".join( - ( - record["id"], - record["function"], - record["field"], - record["current_freebsd_return"], - record["candidate_freebsd_return"], - record["linux_semantic_return"], - "reached" if record["target_marker_reached"] else "not-reached", - record["replay_observation"], - ) - ) - ) -(OUTPUT / "branch-ledger.tsv").write_text("\n".join(tsv) + "\n", encoding="ascii") - -control_tsv = ["id\tfreebsd\tlinux\tresult\tmarker"] -for record in controls: - control_tsv.append( - "\t".join( - ( - record["id"], - record["freebsd"], - record["linux"], - record["result"], - ( - "not-reached" - if record.get("target_marker_reached") is False - else "preserved" - ), - ) - ) - ) -(OUTPUT / "preservation-ledger.tsv").write_text( - "\n".join(control_tsv) + "\n", encoding="ascii" -) -(OUTPUT / "source-sha256.json").write_text( - json.dumps(source_hashes, indent=2, sort_keys=True) + "\n", encoding="ascii" -) -(OUTPUT / "freebsd-contract.json").write_text( - json.dumps(freebsd_contract, indent=2, sort_keys=True) + "\n", encoding="ascii" -) - -result = { - "schema": 1, - "gate": "G06", - "candidate": "P15-052", - "status": status, - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "target_count": len(ledger), - "target_markers_reached": len(ledger) - len(unreachable), - "target_markers_not_reached": len(unreachable), - "source_unique_count": sum(record["source_unique"] for record in ledger), - "preservation_control_count": len(controls), - "abi_marker_reached": abi_reachable, - "stop_reasons": stop_reasons, - "b20": "STOP-NO-SOURCE", - "b21": "NOT_RUN", - "qemu": "NOT_RUN", - "qemu_reason": "host-proven target reachability failure", - "full_feature_suite": "NOT_RUN", -} -(OUTPUT / "result.json").write_text( - json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii" -) - -hash_lines = [] -for path in sorted(OUTPUT.iterdir()): - if path.is_file() and path.name != "SHA256SUMS": - hash_lines.append(f"{sha256(path.read_bytes())} {path.name}") -(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii") -print(json.dumps(result, sort_keys=True)) -if status != "GO": - raise SystemExit(1) -PY diff --git a/tests/pre15/gates/P15-062-input.json b/tests/pre15/gates/P15-062-input.json deleted file mode 100644 index 49e6042..0000000 --- a/tests/pre15/gates/P15-062-input.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "candidate": "P15-062", - "freebsd": { - "head": "106727738dcfb6c001b46f25363b91cece970085", - "sha256": { - "sys/sys/endian.h": "3d870d499089ac84bb49debebbb75b406a5f4df6f05f1582faebc5cf0b7b8806", - "sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c", - "sys/sys/param.h": "cc451dd2de4d6a7cc97b928ddb9e5b19475cba4ab418153ab7f6e6baa661b917", - "sys/sys/systm.h": "648c4ba00523c4dff1e9c0e693722d748e38312fd3cf35f2ee92c21188b62649" - } - }, - "gate": "G04", - "legacy_fixture": { - "block_size": 4096, - "expected_image_size": 12288, - "expected_source_sha256": "cea05c6972efce79727fe268260e8de74089b0fc0b527825beaca8de4dbacff2", - "extents": [ - { - "consumed_bytes": 4096, - "decoded_sha256": "9d1d4465d9891e22a7cb79f1d5c9e8c4b7500753fbe5a7feca1117e948970017", - "logical_length": 1038906, - "logical_offset": 0, - "physical_length": 4096, - "physical_offset": 4096, - "zero_tail_bytes": 0 - }, - { - "consumed_bytes": 48, - "decoded_sha256": "c5563821cb010b2bf6ba6ecd5a563f71f9d37644f086c90de4a9c44a071b4bdc", - "logical_length": 9670, - "logical_offset": 1038906, - "physical_length": 4096, - "physical_offset": 8192, - "zero_tail_bytes": 4048 - } - ], - "mkfs_args": [ - "-T0", - "-U00112233-4455-6677-8899-aabbccddeeff", - "--all-root", - "-x-1", - "-zlz4", - "-C4096", - "-Elegacy-compress" - ], - "segments": [ - { - "byte": 65, - "length": 262144 - }, - { - "byte": 66, - "length": 262144 - }, - { - "byte": 67, - "length": 262144 - }, - { - "byte": 68, - "length": 262144 - } - ] - }, - "partial_fixture": { - "a_sha256": "cea05c6972efce79727fe268260e8de74089b0fc0b527825beaca8de4dbacff2", - "b_nid": 58, - "b_sha256": "f25f98332996139ce81bac04665df5593856430a39b6ed7e25a25f8c066e0d47", - "block_size": 4096, - "expected_image_sha256": "63b6ec3736b325f5e3d355eef35ac8f7e80cb7edc49e3e495a4006fffe7f7514", - "expected_image_size": 16384, - "full_decoded_bytes": 9670, - "full_stream_consumed_bytes": 48, - "mkfs_args": [ - "-T0", - "-U00112233-4455-6677-8899-aabbccddeeff", - "--all-root", - "-x-1", - "-zlz4", - "-C4096", - "-Ededupe" - ], - "partial_consumed_bytes": 42, - "partial_index_records": [ - { - "cluster_offset": 550, - "lcn": 254, - "pblk": 2 - }, - { - "cluster_offset": 2048, - "lcn": 255, - "pblk": 2 - } - ], - "partial_output_bytes": [ - 4096, - 5594 - ], - "prefix_pad_byte": 33, - "prefix_pattern": "P15-062-partial-reference\n", - "prefix_size": 2048, - "shared_pcluster_offset": 8192, - "stream_start": 4048 - }, - "required_base": "205a90465edb64e83ba44aaacac9bedad4cbe905", - "schema": 1, - "source_sha256": { - "repo-pre-15/src/decompressor.c": "76289dcb494994f77c1ba6ff10bbbfe55e7b4baa6a1724c35af14ae1611c537c", - "repo-pre-15/src/decompressor_lz4.c": "ca5e5bd6142f9e3c3ea66849900f05799dbc7d6d1a97e0d5a988c28edb90ea98", - "repo-pre-15/src/zdata.c": "3eeb5dae825d7028793a2e1d19a80d24d3f78cdc2e2d5e19466d8fb1208f3242", - "repo-pre-15/src/zmap.c": "95432d49c20f3049e0fb5af21904283494325570673dbb037767c1397c77c1f3", - "src-linux/decompressor.c": "caf1c501d00a5a2c9cda5fc0b59d2823eaedf0161a130ca69cd5e7c455128709" - }, - "upstream": { - "liblz4_sha256": "67bf8b84af77e962e09be661944f7c61393c4806746734e36db9fbdaebc1de36", - "liblz4_soname": "/lib/x86_64-linux-gnu/liblz4.so.1", - "liblz4_version": "1.10.0", - "url": "https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git", - "versions": [ - { - "commit": "ee97fe5fb77c737df0f77d92ab0d92edd3a11be6", - "expected_image_sha256": "d78c1f00b930bfbf45ac2a0a903896ed5559bac43297e283b348a63926ce5576", - "name": "v1.4" - }, - { - "commit": "ce36273833096f1b7e828309d9b1caa37132092d", - "expected_image_sha256": "1f08b09b7ea40d495160699ddf7079bf5d4deb85859dbe7459a41617f25fdddf", - "name": "v1.7" - }, - { - "commit": "3689cbc2349bff05807d2f939146e92eb1bfaea1", - "expected_image_sha256": "c5ee543074d57a45adc7c666db47ca435c5d3c8504d3163d471ed53bdbdac303", - "name": "v1.8.6" - } - ] - } -} diff --git a/tests/pre15/gates/P15-062.sh b/tests/pre15/gates/P15-062.sh deleted file mode 100755 index a96e908..0000000 --- a/tests/pre15/gates/P15-062.sh +++ /dev/null @@ -1,713 +0,0 @@ -#!/bin/sh -set -eu - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-062-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { - printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2 - exit 2 -} -for tool in cc git make pkg-config python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" - -python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY' -from __future__ import annotations - -import ctypes -import hashlib -import json -import os -from pathlib import Path -import re -import shutil -import struct -import subprocess -import sys -import tempfile -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="ascii")) - - -class GateFailure(Exception): - def __init__(self, status: str, reason: str): - super().__init__(reason) - self.status = status - self.reason = reason - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def git(*args: str) -> str: - return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip() - - -def source_at(commit: str, path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if completed.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}") - return completed.stdout - - -def run_logged(argv: list[str], cwd: Path, log: Path, timeout: int = 120) -> subprocess.CompletedProcess[str]: - completed = subprocess.run( - argv, - cwd=cwd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=timeout, - ) - log.parent.mkdir(parents=True, exist_ok=True) - log.write_text( - "$ " + " ".join(argv) + "\n" + completed.stdout + f"\n[exit {completed.returncode}]\n", - encoding="utf-8", - ) - if completed.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"command failed ({completed.returncode}): {' '.join(argv)}") - return completed - - -def raw_lz4_decode(data: bytes, target: int | None = None) -> tuple[bytes, int]: - ip = 0 - output = bytearray() - while ip < len(data): - token = data[ip] - ip += 1 - literal_length = token >> 4 - if literal_length == 15: - while True: - if ip >= len(data): - raise GateFailure("INFRA_BLOCKED", "oracle saw a truncated literal length") - value = data[ip] - ip += 1 - literal_length += value - if value != 255: - break - if ip + literal_length > len(data): - raise GateFailure("INFRA_BLOCKED", "oracle saw truncated literals") - output.extend(data[ip : ip + literal_length]) - ip += literal_length - if target is not None and len(output) >= target: - return bytes(output[:target]), ip - if ip == len(data): - return bytes(output), ip - if ip + 2 > len(data): - raise GateFailure("INFRA_BLOCKED", "oracle saw a truncated match offset") - offset = data[ip] | data[ip + 1] << 8 - ip += 2 - if offset == 0 or offset > len(output): - raise GateFailure("INFRA_BLOCKED", f"oracle saw invalid match offset {offset}") - match_length = token & 15 - if match_length == 15: - while True: - if ip >= len(data): - raise GateFailure("INFRA_BLOCKED", "oracle saw a truncated match length") - value = data[ip] - ip += 1 - match_length += value - if value != 255: - break - for _ in range(match_length + 4): - output.append(output[-offset]) - if target is not None and len(output) >= target: - return bytes(output[:target]), ip - raise GateFailure("INFRA_BLOCKED", "oracle input ended before a final literal sequence") - - -def make_legacy_source(path: Path) -> bytes: - content = b"".join( - bytes([segment["byte"]]) * segment["length"] - for segment in SPEC["legacy_fixture"]["segments"] - ) - if sha256_bytes(content) != SPEC["legacy_fixture"]["expected_source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "legacy source generator hash drift") - path.mkdir(parents=True) - (path / "big.txt").write_bytes(content) - return content - - -def make_partial_source(path: Path) -> tuple[bytes, bytes]: - path.mkdir(parents=True) - data_a = b"".join( - bytes([segment["byte"]]) * segment["length"] - for segment in SPEC["legacy_fixture"]["segments"] - ) - partial = SPEC["partial_fixture"] - pattern = partial["prefix_pattern"].encode("ascii") - repeats = (partial["prefix_size"] + len(pattern) - 1) // len(pattern) - prefix = (pattern * repeats)[: partial["prefix_size"]] - prefix = prefix.ljust(partial["prefix_size"], bytes([partial["prefix_pad_byte"]])) - data_b = prefix + data_a - if sha256_bytes(data_a) != partial["a_sha256"] or sha256_bytes(data_b) != partial["b_sha256"]: - raise GateFailure("INFRA_BLOCKED", "partial source generator hash drift") - (path / "a.dat").write_bytes(data_a) - (path / "b.dat").write_bytes(data_b) - return data_a, data_b - - -def load_liblz4() -> tuple[Any, Any]: - upstream = SPEC["upstream"] - soname = Path(upstream["liblz4_soname"]) - version = subprocess.check_output(["pkg-config", "--modversion", "liblz4"], text=True).strip() - if version != upstream["liblz4_version"] or sha256_path(soname) != upstream["liblz4_sha256"]: - raise GateFailure("INFRA_BLOCKED", "liblz4 identity changed") - library = ctypes.CDLL(str(soname)) - full = library.LZ4_decompress_safe - full.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int] - full.restype = ctypes.c_int - partial = library.LZ4_decompress_safe_partial - partial.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int] - partial.restype = ctypes.c_int - return full, partial - - -def liblz4_call(function: Any, source: bytes, output_size: int, target: int | None = None) -> tuple[int, bytes]: - source_buffer = ctypes.create_string_buffer(source, len(source)) - output_buffer = ctypes.create_string_buffer(output_size) - if target is None: - result = function(source_buffer, output_buffer, len(source), output_size) - else: - result = function(source_buffer, output_buffer, len(source), target, output_size) - return result, output_buffer.raw - - -def compile_dut_decoder(temp: Path, source: str) -> Any: - source_dir = temp / "dut-decoder" - sys_dir = source_dir / "sys" - sys_dir.mkdir(parents=True) - (source_dir / "decompressor_lz4.c").write_text(source, encoding="utf-8") - (sys_dir / "param.h").write_text( - "#include \n#include \n#include \n#include \n" - "#define EINTEGRITY 97\n#define MIN(a, b) ((a) < (b) ? (a) : (b))\n", - encoding="ascii", - ) - (sys_dir / "endian.h").write_text( - "#include \nstatic inline uint16_t le16dec(const void *p) { " - "const uint8_t *b = p; return (uint16_t)b[0] | (uint16_t)b[1] << 8; }\n", - encoding="ascii", - ) - (sys_dir / "systm.h").write_text("#include \n", encoding="ascii") - (source_dir / "compress.h").write_text( - "#include \n#include \n" - "struct z_erofs_decompress_req { void *sbi; const void *map; const void *in; " - "size_t inputsize; void *out; size_t outputsize; bool partial_decoding; };\n", - encoding="ascii", - ) - library = temp / "dut-lz4.so" - run_logged( - ["cc", "-shared", "-fPIC", "-std=c11", "-Wall", "-Wextra", "-Werror", "-I", str(source_dir), - "-o", str(library), str(source_dir / "decompressor_lz4.c")], - temp, - OUTPUT / "logs/dut-decoder-build.log", - ) - loaded = ctypes.CDLL(str(library)) - - class Request(ctypes.Structure): - _fields_ = [ - ("sbi", ctypes.c_void_p), - ("map", ctypes.c_void_p), - ("in", ctypes.c_void_p), - ("inputsize", ctypes.c_size_t), - ("out", ctypes.c_void_p), - ("outputsize", ctypes.c_size_t), - ("partial_decoding", ctypes.c_bool), - ] - - function = loaded.z_erofs_lz4_decompress - function.argtypes = [ctypes.POINTER(Request)] - function.restype = ctypes.c_int - return loaded, function, Request - - -def dut_call(function: Any, request_type: Any, source: bytes, output_size: int, partial: bool) -> tuple[int, bytes, bool]: - source_buffer = ctypes.create_string_buffer(source, len(source)) - guard_size = 64 - backing = bytearray(b"\xa5" * guard_size + b"\0" * output_size + b"\x5a" * guard_size) - output_buffer = (ctypes.c_ubyte * len(backing)).from_buffer(backing) - output_pointer = ctypes.cast(ctypes.byref(output_buffer, guard_size), ctypes.c_void_p) - request = request_type( - None, - None, - ctypes.cast(source_buffer, ctypes.c_void_p), - len(source), - output_pointer, - output_size, - partial, - ) - result = function(ctypes.byref(request)) - guards_ok = backing[:guard_size] == b"\xa5" * guard_size and backing[-guard_size:] == b"\x5a" * guard_size - return result, bytes(backing[guard_size : guard_size + output_size]), guards_ok - - -def verify_cleanup_source(zdata_source: str) -> dict[str, Any]: - anchors = [ - "error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen,", - "erofs_put_metabuf(&buf);", - "erofs_brelse(compressed);", - "free(decoded, M_EROFS);", - "*bufp = decoded;", - ] - missing = [anchor for anchor in anchors if anchor not in zdata_source] - if missing: - raise GateFailure("INFRA_BLOCKED", f"DUT cleanup source anchors changed: {missing}") - release_pos = min(zdata_source.index("erofs_put_metabuf(&buf);"), zdata_source.index("erofs_brelse(compressed);")) - error_pos = zdata_source.index("if (error != 0) {", release_pos) - free_pos = zdata_source.index("free(decoded, M_EROFS);", error_pos) - publish_pos = zdata_source.index("*bufp = decoded;", free_pos) - return { - "decoded_error_free_after_input_release": release_pos < error_pos < free_pos, - "decoded_success_publish_after_error_branch": free_pos < publish_pos, - "metadata_release_anchor": True, - "physical_release_anchor": True, - } - - -def build_versions(temp: Path, legacy_source: bytes) -> tuple[list[dict[str, Any]], dict[str, Path]]: - upstream = SPEC["upstream"] - clone = temp / "erofs-utils" - run_logged( - ["git", "clone", "--no-checkout", upstream["url"], str(clone)], - temp, - OUTPUT / "logs/upstream-clone.log", - timeout=90, - ) - records = [] - worktrees: dict[str, Path] = {} - fixture = SPEC["legacy_fixture"] - source_dir = temp / "legacy-source" - make_legacy_source(source_dir) - for version in upstream["versions"]: - name = version["name"] - worktree = temp / f"erofs-{name}" - worktrees[name] = worktree - run_logged( - ["git", "-C", str(clone), "worktree", "add", "--detach", str(worktree), version["commit"]], - temp, - OUTPUT / f"logs/{name}-worktree.log", - ) - resolved = subprocess.check_output(["git", "-C", str(worktree), "rev-parse", "HEAD"], text=True).strip() - if resolved != version["commit"]: - raise GateFailure("INFRA_BLOCKED", f"upstream commit mismatch for {name}") - run_logged(["./autogen.sh"], worktree, OUTPUT / f"logs/{name}-autogen.log") - run_logged(["./configure", "--disable-fuse"], worktree, OUTPUT / f"logs/{name}-configure.log") - run_logged(["make", "-s", "-j2"], worktree, OUTPUT / f"logs/{name}-make.log") - mkfs = worktree / "mkfs/mkfs.erofs" - fsck = worktree / "fsck/fsck.erofs" - dump = worktree / "dump/dump.erofs" - images = temp / f"images-{name}" - images.mkdir() - generated = [] - for label in ("a", "b"): - image = images / f"legacy-{label}.erofs" - argv = [str(mkfs), *fixture["mkfs_args"], str(image), str(source_dir)] - run_logged(argv, worktree, OUTPUT / f"logs/{name}-mkfs-{label}.log") - generated.append(image) - hashes = [sha256_path(path) for path in generated] - if hashes[0] != hashes[1] or hashes[0] != version["expected_image_sha256"]: - raise GateFailure("INFRA_BLOCKED", f"{name} image reproducibility/hash mismatch: {hashes}") - run_logged([str(fsck), "--extract", str(generated[0])], worktree, OUTPUT / f"logs/{name}-fsck.log") - superblock = run_logged([str(dump), "-s", str(generated[0])], worktree, OUTPUT / f"logs/{name}-super.log").stdout - feature_line = next((line for line in superblock.splitlines() if "features:" in line), "") - if "0padding" in feature_line: - raise GateFailure("INFRA_BLOCKED", f"{name} legacy fixture unexpectedly has 0padding") - image = generated[0].read_bytes() - if len(image) != fixture["expected_image_size"]: - raise GateFailure("INFRA_BLOCKED", f"{name} image size changed") - extents = [] - reconstructed = bytearray() - for expected in fixture["extents"]: - block = image[expected["physical_offset"] : expected["physical_offset"] + expected["physical_length"]] - decoded, consumed = raw_lz4_decode(block, expected["logical_length"]) - tail = block[consumed:] - if ( - consumed != expected["consumed_bytes"] - or sha256_bytes(decoded) != expected["decoded_sha256"] - or len(tail) != expected["zero_tail_bytes"] - or any(tail) - ): - raise GateFailure("INFRA_BLOCKED", f"{name} independent extent oracle drift") - logical = legacy_source[expected["logical_offset"] : expected["logical_offset"] + expected["logical_length"]] - if decoded != logical: - raise GateFailure("INFRA_BLOCKED", f"{name} decoded extent differs from source") - reconstructed.extend(decoded) - extents.append({ - **expected, - "input_bytes": len(block), - "tail_all_zero": not any(tail), - }) - if bytes(reconstructed) != legacy_source: - raise GateFailure("INFRA_BLOCKED", f"{name} reconstructed file mismatch") - records.append({ - "commit": resolved, - "fsck": "PASS", - "image_repeated_sha256": hashes, - "image_size": len(image), - "legacy_without_0padding": True, - "name": name, - "extents": extents, - }) - return records, worktrees - - -def verify_partial_fixture(temp: Path, worktree: Path, full: Any, partial_codec: Any, dut: Any) -> dict[str, Any]: - partial_spec = SPEC["partial_fixture"] - source_dir = temp / "partial-source" - data_a, data_b = make_partial_source(source_dir) - images = temp / "partial-images" - images.mkdir() - mkfs = worktree / "mkfs/mkfs.erofs" - fsck = worktree / "fsck/fsck.erofs" - dump = worktree / "dump/dump.erofs" - generated = [] - for label in ("a", "b"): - image = images / f"partial-{label}.erofs" - run_logged( - [str(mkfs), *partial_spec["mkfs_args"], str(image), str(source_dir)], - worktree, - OUTPUT / f"logs/partial-mkfs-{label}.log", - ) - generated.append(image) - hashes = [sha256_path(path) for path in generated] - if hashes[0] != hashes[1] or hashes[0] != partial_spec["expected_image_sha256"]: - raise GateFailure("INFRA_BLOCKED", f"partial fixture reproducibility/hash mismatch: {hashes}") - run_logged([str(fsck), "--extract", str(generated[0])], worktree, OUTPUT / "logs/partial-fsck.log") - dump_b = run_logged( - [str(dump), "--path=/b.dat", "-e", str(generated[0])], - worktree, - OUTPUT / "logs/partial-dump-b.log", - ).stdout - match = re.search(r"^NID:\s+(\d+)", dump_b, re.MULTILINE) - if match is None or int(match.group(1)) != partial_spec["b_nid"]: - raise GateFailure("INFRA_BLOCKED", "partial fixture b.dat NID drift") - image = generated[0].read_bytes() - if len(image) != partial_spec["expected_image_size"]: - raise GateFailure("INFRA_BLOCKED", "partial fixture image size drift") - inode_offset = partial_spec["b_nid"] * 32 - inode_format, xattr_count, _, _, inode_size = struct.unpack_from("> 1) & 7 - if xattr_count != 0 or datalayout != 1 or inode_size != len(data_b): - raise GateFailure("INFRA_BLOCKED", "partial fixture inode contract drift") - index_start = ((inode_offset + inode_bytes + 7) & ~7) + 16 - index_records = [] - for expected in partial_spec["partial_index_records"]: - position = index_start + expected["lcn"] * 8 - advise, cluster_offset, pblk = struct.unpack_from("= 0 - or not partial_guards - or not full_guards - ): - raise GateFailure("INFRA_BLOCKED", "range-after-corruption oracle mismatch") - return { - "fixture_repeated_sha256": hashes, - "fsck": "PASS", - "index_records": index_records, - "partial_records": partial_records, - "range_after_corruption": { - "corruption_starts_at_consumed_byte": partial_spec["partial_consumed_bytes"], - "dut_full_errno": full_result, - "dut_partial_errno": partial_result, - "full_guard_unchanged": full_guards, - "liblz4_full_result": lib_full_result, - "liblz4_partial_result": lib_partial_result, - "partial_guard_unchanged": partial_guards, - "partial_output_matches_full_slice": True, - }, - "shared_stream": { - "consumed_bytes": full_consumed, - "decoded_bytes": len(full_decoded), - "leading_zero_bytes": first_nonzero, - "sha256": sha256_bytes(stream), - }, - } - - -def finalize() -> None: - lines = [] - for path in sorted(OUTPUT.rglob("*")): - if path.is_file() and path.name != "SHA256SUMS": - lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}") - (OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") - - -result: dict[str, Any] | None = None -exit_code = 0 -owned_temp: str | None = None -try: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-062" or SPEC.get("gate") != "G04": - raise GateFailure("INFRA_BLOCKED", "invalid P15-062 input schema") - resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise GateFailure("INFRA_BLOCKED", f"P15-062 must replay {SPEC['required_base']}, got {resolved}") - sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]} - source_hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()} - if source_hashes != SPEC["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed") - freebsd_head = subprocess.check_output(["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True).strip() - freebsd_hashes = {path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"]} - if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]: - raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed") - if "#define\tEINTEGRITY\t97" not in (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8"): - raise GateFailure("INFRA_BLOCKED", "FreeBSD EINTEGRITY positive errno changed") - lz4_source = sources["repo-pre-15/src/decompressor_lz4.c"] - if "while (ip < iend)" not in lz4_source or "if (*ip++ != 0)" not in lz4_source: - raise GateFailure("INFRA_BLOCKED", "frozen DUT no longer has the zero-tail full-decode rule") - if "m->map->m_plen = m->compressedblks << m->sbi->blkszbits;" not in sources["repo-pre-15/src/zmap.c"]: - raise GateFailure("INFRA_BLOCKED", "frozen DUT no longer maps legacy pcluster input by whole blocks") - if "ret = LZ4_decompress_safe(src + inputmargin, out," not in sources["src-linux/decompressor.c"]: - raise GateFailure("INFRA_BLOCKED", "Linux LZ4 comparison anchor changed") - write_json(OUTPUT / "source-sha256.json", source_hashes) - write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes}) - cleanup_contract = verify_cleanup_source(sources["repo-pre-15/src/zdata.c"]) - full_codec, partial_codec = load_liblz4() - with tempfile.TemporaryDirectory(prefix="p15-062-g04-") as temporary: - owned_temp = temporary - temp = Path(temporary) - legacy_source = make_legacy_source(temp / "legacy-source-check") - generator_records, worktrees = build_versions(temp, legacy_source) - dut = compile_dut_decoder(temp, lz4_source) - dut_cases = [] - for record in generator_records: - image_path = temp / f"images-{record['name']}/legacy-a.erofs" - image = image_path.read_bytes() - expected = SPEC["legacy_fixture"]["extents"][1] - block = image[expected["physical_offset"] : expected["physical_offset"] + expected["physical_length"]] - stream = block[: expected["consumed_bytes"]] - zero_tail_result, zero_tail_output, zero_tail_guards = dut_call( - dut[1], dut[2], block, expected["logical_length"], False - ) - exact_result, exact_output, exact_guards = dut_call( - dut[1], dut[2], stream, expected["logical_length"], False - ) - nonzero = bytearray(block) - nonzero[expected["consumed_bytes"]] = 1 - nonzero_result, _, nonzero_guards = dut_call( - dut[1], dut[2], bytes(nonzero), expected["logical_length"], False - ) - truncated_result, _, truncated_guards = dut_call( - dut[1], dut[2], stream[:-1], expected["logical_length"], False - ) - lib_full_result, _ = liblz4_call(full_codec, block, expected["logical_length"]) - lib_exact_result, lib_exact_output = liblz4_call(full_codec, stream, expected["logical_length"]) - expected_output = legacy_source[expected["logical_offset"] :] - if ( - zero_tail_result != 0 - or zero_tail_output != expected_output - or exact_result != 0 - or exact_output != expected_output - or nonzero_result != 97 - or truncated_result != 97 - or lib_full_result >= 0 - or lib_exact_result != expected["logical_length"] - or lib_exact_output != expected_output - or not all((zero_tail_guards, exact_guards, nonzero_guards, truncated_guards)) - ): - raise GateFailure("INFRA_BLOCKED", f"DUT/liblz4 legacy oracle mismatch for {record['name']}") - dut_cases.append({ - "current_dut_exact_errno": exact_result, - "current_dut_nonzero_tail_errno": nonzero_result, - "current_dut_truncated_errno": truncated_result, - "current_dut_zero_tail_errno": zero_tail_result, - "exact_policy_would_accept": expected["consumed_bytes"] == expected["physical_length"], - "guards_unchanged": True, - "liblz4_exact_result": lib_exact_result, - "liblz4_physical_input_result": lib_full_result, - "name": record["name"], - "physical_input_bytes": expected["physical_length"], - "stream_consumed_bytes": expected["consumed_bytes"], - "zero_tail_bytes": expected["zero_tail_bytes"], - }) - partial_record = verify_partial_fixture(temp, worktrees["v1.8.6"], full_codec, partial_codec, dut) - write_json(OUTPUT / "generator-ledger.json", generator_records) - write_json(OUTPUT / "dut-cases.json", dut_cases) - write_json(OUTPUT / "partial-oracle.json", partial_record) - write_json(OUTPUT / "cleanup-ledger.json", { - **cleanup_contract, - "decoder_allocations": 0, - "decoder_owned_buffers": 0, - "guards_unchanged_for_all_cases": True, - "input_buffer_owner": "z_erofs_read_extent caller", - "output_buffer_owner": "z_erofs_read_extent caller", - "typed_corruption_errno": 97, - }) - incompatible = [case["name"] for case in dut_cases if not case["exact_policy_would_accept"]] - if incompatible != [version["name"] for version in SPEC["upstream"]["versions"]]: - raise GateFailure("INFRA_BLOCKED", f"unexpected incompatibility set: {incompatible}") - result = { - "b26": "STOP-NO-SOURCE", - "candidate": "P15-062", - "cleanup": "PASS", - "full_feature_suite": "NOT_RUN", - "gate": "G04", - "historical_generations": incompatible, - "oracle": "independent raw LZ4 parser + liblz4 1.10.0 + matching erofs-utils fsck", - "partial_reference": "PASS", - "qemu": "NOT_RUN", - "qemu_reason": "host oracle proves exact-full would reject reproducible legal legacy images", - "reason": "all three reproducible legacy mkfs generations store a short valid final raw LZ4 stream followed by zero bytes inside the block-sized m_plen", - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "schema": 1, - "status": "STOP", - "typed_errno": "PASS", - } - write_json(OUTPUT / "result.json", result) - exit_code = 1 - cleanup_record = { - "owned_temp": owned_temp, - "owned_temp_removed": owned_temp is not None and not Path(owned_temp).exists(), - "protected_pid_touched": False, - "protected_port_touched": False, - "qemu_started": False, - } - if not cleanup_record["owned_temp_removed"]: - raise GateFailure("INFRA_BLOCKED", "owned gate temporary directory survived cleanup") - write_json(OUTPUT / "owned-cleanup.json", cleanup_record) -except GateFailure as failure: - result = { - "b26": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN", - "candidate": "P15-062", - "full_feature_suite": "NOT_RUN", - "gate": "G04", - "qemu": "NOT_RUN", - "reason": failure.reason, - "requested_base": REQUESTED_BASE, - "schema": 1, - "status": failure.status, - } - write_json(OUTPUT / "result.json", result) - exit_code = 21 if failure.status == "INFRA_BLOCKED" else 1 -except (OSError, subprocess.SubprocessError, ValueError) as failure: - result = { - "b26": "NOT_RUN", - "candidate": "P15-062", - "full_feature_suite": "NOT_RUN", - "gate": "G04", - "qemu": "NOT_RUN", - "reason": f"gate infrastructure failure: {failure}", - "requested_base": REQUESTED_BASE, - "schema": 1, - "status": "INFRA_BLOCKED", - } - write_json(OUTPUT / "result.json", result) - exit_code = 21 -finally: - finalize() - -print(json.dumps(result, sort_keys=True)) -raise SystemExit(exit_code) -PY diff --git a/tests/pre15/gates/P15-076-input.json b/tests/pre15/gates/P15-076-input.json deleted file mode 100644 index 9f728c8..0000000 --- a/tests/pre15/gates/P15-076-input.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "benchmark": { - "iterations_per_sample": 64, - "samples": 7 - }, - "budgets": { - "global_cached_contexts_per_codec": 16, - "global_resident_bytes": 8388608, - "mount_cached_contexts_per_codec": 2, - "mount_resident_bytes": 1048576, - "wrapper_bytes_per_context": 128 - }, - "candidate": "P15-076", - "fixture_assets": { - "repo-pre-15/tests/pre15/fixtures/B28-partial-fixtures.py": "9d9ac67b09667d74e99d7f5e5e5edd3ac467f11d579196fda9a9b7599c862b30", - "repo-pre-15/tests/pre15/fixtures/B28-partial.json": "ccdc62718942739ae9b0a60600f9958247abf005e3751bbc76a63173239f0e48" - }, - "freebsd": { - "head": "106727738dcfb6c001b46f25363b91cece970085", - "sha256": { - "sys/contrib/xz-embedded/freebsd/xz_config.h": "fdc4fcf394b4014b6c9a39a27fb6988d77c840e6e0470709acbdaf541a7dd949", - "sys/contrib/xz-embedded/linux/include/linux/xz.h": "4255bf4d723746761816da5787ca47acf8b229d3e7a37502a12f3d532772026b", - "sys/contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c": "fca31a150d259f5e7e5aa82392fbe9b2ac6331f5da9e24834ece81f0e30d5c65", - "sys/contrib/xz-embedded/linux/lib/xz/xz_private.h": "8a8483369795154db376b82700be345917050ad78c089dd289b273c5718c8177", - "sys/contrib/zlib/inflate.c": "413abc042c18d267619441eade265505952dbcf46dc9b2c2967d2fec1281086c", - "sys/contrib/zlib/zlib.h": "3c4f75f90589af70540aeaf23610e2b299bb498826a34c4c16ec92233dd68693", - "sys/contrib/zstd/lib/decompress/zstd_decompress.c": "029580818b7e9cd38d9d07c63516b00ceaa943ffcfbd099fc5ccbe3628fa362f", - "sys/contrib/zstd/lib/zstd.h": "9b4bc8245565c98ccfc61c07749928b57e7c0f6fddb0530c4f6aa1971893d88b", - "sys/kern/vfs_init.c": "4092238096a68edbc27697fa091e11c146702fc147de0015dec882e955a715a6", - "sys/sys/mount.h": "2bc2017d63389c39dfee52b3041970450ef1eebaa149c8fe5e84a871c0f0b738", - "sys/vm/uma.h": "398bbc3787c363b0a71c4f6d724739018f2b54607962d5d8aea5a547011226a3" - } - }, - "gate": "G05", - "libraries": { - "liblzma": "5.8.1", - "libzstd": "1.5.7", - "zlib": "1.3.1" - }, - "required_base": "3e9bc3f03ba9c39c38cc40f2f08eb6e769557f55", - "schema": 1, - "source_sha256": { - "repo-pre-15/src/decompressor.c": "308c3c151fcacf423a79331818cd8f08b3093aadc4def869b03e5c932937e350", - "repo-pre-15/src/decompressor_deflate.c": "d3efece85fee05affa0fcac5f763d886bcaebaeaf4b5278215631c5801478caa", - "repo-pre-15/src/decompressor_lzma.c": "ebe8c7b9f70a771c5d01c03665c905982b3bfe8e088b59711ab6d8f07cf6c733", - "repo-pre-15/src/decompressor_zstd.c": "8c1b7d551b0dfed4f1ccd3636632bd1dc06e1d3890aa8dc8f8e309fd8aa1ef5c", - "repo-pre-15/src/internal.h": "d1884395c040e2d1dcb31473b3c007c301248b6ff9304632b6b1b51602927ca9", - "repo-pre-15/src/super.c": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2", - "repo-pre-15/src/zdata.c": "614cd2a704716c166f4a889715129f7abcaac528651433370474b1334f2e5502", - "src-linux/decompressor_deflate.c": "0ddb56c27cacc63aa9f7a10a55cfd76ee2ba0bab15cbac7f6d8684203fb2c67b", - "src-linux/decompressor_lzma.c": "5a23b4455767c30c8d90e3e5b10bc3c430d94359727affae2e6f7824f106d5a8", - "src-linux/decompressor_zstd.c": "4f8a4961fa6b4219fcfe91c04ba37179ec4dde296fcf9f91a5d5b27d8c272607" - }, - "thresholds": { - "minimum_baseline_allocator_share_percent": 10.0, - "minimum_context_cost_reduction_percent": 25.0 - }, - "tools": { - "cc": { - "path": "/usr/bin/x86_64-linux-gnu-gcc-14", - "sha256": "a23ecab8ff08f09ad8c80602c2c5df7f49e09c25905cb8975902e101bf72635f" - }, - "fsck.erofs": { - "path": "/usr/bin/fsck.erofs", - "sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29" - }, - "liblzma": { - "path": "/usr/lib/x86_64-linux-gnu/liblzma.so.5.8.1", - "sha256": "01e05a4b5268805a10a2414a8904fb1fb45c60626dce53b5ea19c1d491049b99" - }, - "libz": { - "path": "/usr/lib/x86_64-linux-gnu/libz.so.1.3.1", - "sha256": "85590dd58edf5445e18bc7193e5ebc01ac5841f1ae187e97705a662e90c6421e" - }, - "libzstd": { - "path": "/usr/lib/x86_64-linux-gnu/libzstd.so.1.5.7", - "sha256": "27f07c9a49c2c956bcfb64cd4712976586a66facbf15fc7f09bc37413b5f2b21" - }, - "mkfs.erofs": { - "path": "/usr/bin/mkfs.erofs", - "sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c" - }, - "python3": { - "path": "/usr/bin/python3.13", - "sha256": "4703a3d15898c0b5d81c3f939e93bdd8ca6116342093fb160ab1e01860dd7d8b" - } - } -} diff --git a/tests/pre15/gates/P15-076.sh b/tests/pre15/gates/P15-076.sh deleted file mode 100755 index d7702a8..0000000 --- a/tests/pre15/gates/P15-076.sh +++ /dev/null @@ -1,853 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-076-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; } -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" - -python3 -B - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY' -from __future__ import annotations - -import hashlib -import json -import os -from pathlib import Path -import shutil -import statistics -import subprocess -import sys -import tempfile -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="ascii")) -DUT = ROOT / "repo-pre-15" - - -class GateStop(RuntimeError): - pass - - -class InfraBlocked(RuntimeError): - pass - - -class RunnerFail(RuntimeError): - pass - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def run(argv: list[str], cwd: Path, log: Path, timeout: int = 120) -> subprocess.CompletedProcess[str]: - try: - completed = subprocess.run( - argv, cwd=cwd, check=False, text=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout, - ) - except subprocess.TimeoutExpired as error: - log.parent.mkdir(parents=True, exist_ok=True) - log.write_text(f"$ {' '.join(argv)}\nTIMEOUT after {timeout}s\n", encoding="ascii") - raise InfraBlocked(f"command timed out after {timeout}s: {' '.join(argv)}") from error - log.parent.mkdir(parents=True, exist_ok=True) - log.write_text( - f"$ {' '.join(argv)}\n{completed.stdout}\n[exit {completed.returncode}]\n", - encoding="utf-8", - ) - return completed - - -def checked_run(argv: list[str], cwd: Path, log: Path, timeout: int = 120) -> str: - completed = run(argv, cwd, log, timeout) - if completed.returncode != 0: - raise RunnerFail(f"command failed ({completed.returncode}): {' '.join(argv)}") - return completed.stdout - - -def git(path: Path, *args: str) -> str: - completed = subprocess.run( - ["git", "-C", str(path), *args], check=False, text=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=30, - ) - if completed.returncode != 0: - raise InfraBlocked(f"git {' '.join(args)} failed: {completed.stdout.strip()}") - return completed.stdout.strip() - - -def source_at(commit: str, relative: str) -> bytes: - completed = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{commit}:{relative}"], - check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30, - ) - if completed.returncode != 0: - raise InfraBlocked(f"cannot read frozen source {relative}") - return completed.stdout - - -def verify_identity() -> dict[str, Any]: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-076" or SPEC.get("gate") != "G05": - raise InfraBlocked("invalid P15-076 input identity") - resolved = git(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise InfraBlocked(f"P15-076 must replay {SPEC['required_base']}, got {resolved}") - source_hashes = { - relative: sha256_bytes(source_at(resolved, relative)) - for relative in SPEC["source_sha256"] - } - if source_hashes != SPEC["source_sha256"]: - raise InfraBlocked("frozen DUT/Linux source identity changed") - fixture_hashes = { - relative: sha256_path(ROOT / relative) - for relative in SPEC["fixture_assets"] - } - if fixture_hashes != SPEC["fixture_assets"]: - raise InfraBlocked("frozen B28 fixture identity changed") - freebsd_head = git(FREEBSD_SRC, "rev-parse", "HEAD") - freebsd_hashes = { - relative: sha256_path(FREEBSD_SRC / relative) - for relative in SPEC["freebsd"]["sha256"] - } - if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]: - raise InfraBlocked("frozen FreeBSD API/source identity changed") - tool_hashes = {} - for name, record in SPEC["tools"].items(): - path = Path(record["path"]) - if not path.is_file(): - raise InfraBlocked(f"missing frozen tool: {path}") - tool_hashes[name] = sha256_path(path) - if tool_hashes[name] != record["sha256"]: - raise InfraBlocked(f"frozen tool changed: {name}") - versions = { - package: subprocess.check_output( - ["pkg-config", "--modversion", package], text=True, timeout=30 - ).strip() - for package in ("liblzma", "libzstd", "zlib") - } - if versions != SPEC["libraries"]: - raise InfraBlocked(f"library versions changed: {versions}") - return { - "base": resolved, - "fixture_sha256": fixture_hashes, - "freebsd_head": freebsd_head, - "freebsd_sha256": freebsd_hashes, - "libraries": versions, - "source_sha256": source_hashes, - "tool_sha256": tool_hashes, - } - - -XZ_CONFIG = r'''#ifndef P15_076_XZ_CONFIG_H -#define P15_076_XZ_CONFIG_H -#include -#include -#include -#include -void *gate_xz_malloc(size_t size); -void gate_xz_free(void *address); -#define XZ_PREBOOT 1 -#undef XZ_EXTERN -#define XZ_EXTERN extern -#define STATIC -#define INIT -#define bool int -#define true 1 -#define false 0 -#define GFP_KERNEL 0 -#define kmalloc(size, flags) gate_xz_malloc(size) -#define kfree(address) gate_xz_free(address) -#define vmalloc(size) gate_xz_malloc(size) -#define vfree(address) gate_xz_free(address) -#define memeq(a, b, size) (memcmp((a), (b), (size)) == 0) -#define memzero(buffer, size) memset((buffer), 0, (size)) -#define min(a, b) ((a) < (b) ? (a) : (b)) -#define min_t(type, a, b) min((a), (b)) -static inline uint32_t get_le32(const void *address) -{ - const unsigned char *p = address; - return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | - ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); -} -#endif -''' - - -BENCHMARK_SOURCE = r'''#define _POSIX_C_SOURCE 200809L -#define ZSTD_STATIC_LINKING_ONLY -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct allocation { size_t size; }; -struct tracker { - uint64_t alloc_calls; - uint64_t free_calls; - size_t live_bytes; - size_t peak_bytes; -}; -static struct tracker *active_tracker; - -static void fail(const char *message) -{ - fprintf(stderr, "%s\n", message); - exit(2); -} - -static void *tracked_alloc(size_t size) -{ - struct allocation *allocation; - if (active_tracker == NULL || size > SIZE_MAX - sizeof(*allocation)) - return NULL; - allocation = malloc(sizeof(*allocation) + size); - if (allocation == NULL) - return NULL; - allocation->size = size; - active_tracker->alloc_calls++; - active_tracker->live_bytes += size; - if (active_tracker->live_bytes > active_tracker->peak_bytes) - active_tracker->peak_bytes = active_tracker->live_bytes; - return allocation + 1; -} - -static void tracked_free(void *address) -{ - struct allocation *allocation; - if (address == NULL) - return; - allocation = (struct allocation *)address - 1; - if (active_tracker == NULL || allocation->size > active_tracker->live_bytes) - fail("allocation tracker underflow"); - active_tracker->free_calls++; - active_tracker->live_bytes -= allocation->size; - free(allocation); -} - -void *gate_xz_malloc(size_t size) { return tracked_alloc(size); } -void gate_xz_free(void *address) { tracked_free(address); } - -static voidpf zalloc_tracked(voidpf opaque, uInt items, uInt size) -{ - (void)opaque; - if (items != 0 && size > SIZE_MAX / items) - return NULL; - return tracked_alloc((size_t)items * size); -} -static void zfree_tracked(voidpf opaque, voidpf address) -{ - (void)opaque; - tracked_free(address); -} -static void *zstd_alloc_tracked(void *opaque, size_t size) -{ - (void)opaque; - return tracked_alloc(size); -} -static void zstd_free_tracked(void *opaque, void *address) -{ - (void)opaque; - tracked_free(address); -} - -static unsigned char *read_file(const char *path, size_t *sizep) -{ - FILE *stream; - unsigned char *data; - long length; - stream = fopen(path, "rb"); - if (stream == NULL || fseek(stream, 0, SEEK_END) != 0) - fail("cannot open input"); - length = ftell(stream); - if (length <= 0 || fseek(stream, 0, SEEK_SET) != 0) - fail("invalid input size"); - data = malloc((size_t)length); - if (data == NULL || fread(data, 1, (size_t)length, stream) != (size_t)length) - fail("cannot read input"); - fclose(stream); - *sizep = (size_t)length; - return data; -} - -static uint64_t elapsed_ns(const struct timespec *start, const struct timespec *end) -{ - return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL + - (uint64_t)(end->tv_nsec - start->tv_nsec); -} - -static int run_lzma(struct xz_dec_microlzma *state, const unsigned char *input, - size_t input_size, unsigned char *output, size_t output_size) -{ - struct xz_buf buffer = { 0 }; - enum xz_ret ret; - xz_dec_microlzma_reset(state, (uint32_t)input_size, - (uint32_t)output_size, 1); - buffer.in = input; - buffer.in_size = input_size; - buffer.out = output; - buffer.out_size = output_size; - ret = xz_dec_microlzma_run(state, &buffer); - return ret == XZ_STREAM_END && buffer.in_pos == input_size && - buffer.out_pos == output_size ? 0 : -1; -} - -static int run_deflate(z_stream *stream, const unsigned char *input, - size_t input_size, unsigned char *output, size_t output_size) -{ - int ret = Z_OK; - stream->next_in = (Bytef *)(uintptr_t)input; - stream->avail_in = (uInt)input_size; - stream->next_out = output; - stream->avail_out = (uInt)output_size; - while (stream->avail_out != 0) { - uInt in_before = stream->avail_in; - uInt out_before = stream->avail_out; - ret = inflate(stream, Z_SYNC_FLUSH); - if (ret == Z_STREAM_END) - break; - if (ret != Z_OK || (stream->avail_in == in_before && - stream->avail_out == out_before)) - return -1; - } - return ret == Z_STREAM_END && stream->avail_in == 0 && - stream->avail_out == 0 ? 0 : -1; -} - -static int run_zstd(ZSTD_DCtx *context, const unsigned char *input, - size_t input_size, unsigned char *output, size_t output_size) -{ - ZSTD_inBuffer in_buffer = { input, input_size, 0 }; - ZSTD_outBuffer out_buffer = { output, output_size, 0 }; - size_t ret = 1; - while (out_buffer.pos != out_buffer.size) { - size_t in_before = in_buffer.pos; - size_t out_before = out_buffer.pos; - ret = ZSTD_decompressStream(context, &out_buffer, &in_buffer); - if (ZSTD_isError(ret) || (in_buffer.pos == in_before && - out_buffer.pos == out_before)) - return -1; - if (ret == 0) - break; - } - return ret == 0 && in_buffer.pos == input_size && - out_buffer.pos == output_size ? 0 : -1; -} - -int main(int argc, char **argv) -{ - const char *codec, *mode; - unsigned char *input, *expected, *output; - struct tracker tracker = { 0 }; - struct timespec start, end; - size_t input_size, output_size; - uint64_t cpu_ns; - int iterations, pooled; - struct xz_dec_microlzma *xz_state = NULL; - z_stream zstream = { 0 }; - ZSTD_DCtx *zstd_context = NULL; - ZSTD_customMem zstd_memory = { - zstd_alloc_tracked, zstd_free_tracked, NULL - }; - - if (argc != 6) - fail("usage: benchmark CODEC MODE STREAM EXPECTED ITERATIONS"); - codec = argv[1]; - mode = argv[2]; - pooled = strcmp(mode, "pooled") == 0; - if (!pooled && strcmp(mode, "baseline") != 0) - fail("invalid mode"); - iterations = atoi(argv[5]); - if (iterations <= 0) - fail("invalid iterations"); - input = read_file(argv[3], &input_size); - expected = read_file(argv[4], &output_size); - output = malloc(output_size); - if (output == NULL) - fail("cannot allocate output"); - active_tracker = &tracker; - - if (pooled && strcmp(codec, "lzma") == 0) { - xz_state = xz_dec_microlzma_alloc(XZ_SINGLE, 65536); - if (xz_state == NULL) - fail("pooled lzma init failed"); - } else if (pooled && strcmp(codec, "deflate") == 0) { - zstream.zalloc = zalloc_tracked; - zstream.zfree = zfree_tracked; - if (inflateInit2(&zstream, -15) != Z_OK) - fail("pooled deflate init failed"); - } else if (pooled && strcmp(codec, "zstd") == 0) { - zstd_context = ZSTD_createDCtx_advanced(zstd_memory); - if (zstd_context == NULL || ZSTD_isError(ZSTD_DCtx_setParameter( - zstd_context, ZSTD_d_windowLogMax, 16))) - fail("pooled zstd init failed"); - } - - if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start) != 0) - fail("clock start failed"); - for (int iteration = 0; iteration < iterations; ++iteration) { - int error = 0; - memset(output, 0xa5, output_size); - if (strcmp(codec, "lzma") == 0) { - struct xz_dec_microlzma *state = xz_state; - if (!pooled) - state = xz_dec_microlzma_alloc(XZ_SINGLE, 65536); - if (state == NULL || run_lzma(state, input, input_size, - output, output_size) != 0) - error = 1; - if (!pooled && state != NULL) - xz_dec_microlzma_end(state); - } else if (strcmp(codec, "deflate") == 0) { - z_stream stream = { 0 }; - z_stream *current = &zstream; - if (!pooled) { - stream.zalloc = zalloc_tracked; - stream.zfree = zfree_tracked; - current = &stream; - if (inflateInit2(current, -15) != Z_OK) - error = 1; - } else if (iteration != 0 && inflateReset2(current, -15) != Z_OK) { - error = 1; - } - if (!error && run_deflate(current, input, input_size, - output, output_size) != 0) - error = 1; - if (!pooled && current->state != NULL && inflateEnd(current) != Z_OK) - error = 1; - } else if (strcmp(codec, "zstd") == 0) { - ZSTD_DCtx *context = zstd_context; - if (!pooled) { - context = ZSTD_createDCtx_advanced(zstd_memory); - if (context == NULL || ZSTD_isError(ZSTD_DCtx_setParameter( - context, ZSTD_d_windowLogMax, 16))) - error = 1; - } else if (iteration != 0 && ZSTD_isError(ZSTD_DCtx_reset( - context, ZSTD_reset_session_only))) { - error = 1; - } - if (!error && run_zstd(context, input, input_size, - output, output_size) != 0) - error = 1; - if (!pooled && context != NULL && ZSTD_isError(ZSTD_freeDCtx(context))) - error = 1; - } else { - fail("unknown codec"); - } - if (error || memcmp(output, expected, output_size) != 0) - fail("decode or output comparison failed"); - } - if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end) != 0) - fail("clock end failed"); - - if (xz_state != NULL) - xz_dec_microlzma_end(xz_state); - if (zstream.state != NULL && inflateEnd(&zstream) != Z_OK) - fail("pooled deflate fini failed"); - if (zstd_context != NULL && ZSTD_isError(ZSTD_freeDCtx(zstd_context))) - fail("pooled zstd fini failed"); - if (tracker.live_bytes != 0 || tracker.alloc_calls != tracker.free_calls) - fail("context cleanup mismatch"); - cpu_ns = elapsed_ns(&start, &end); - printf("alloc_calls=%" PRIu64 " cpu_ns=%" PRIu64 - " free_calls=%" PRIu64 " iterations=%d peak_bytes=%zu\n", - tracker.alloc_calls, cpu_ns, tracker.free_calls, iterations, - tracker.peak_bytes); - free(output); - free(expected); - free(input); - return 0; -} -''' - - -def compile_prototype(temp: Path) -> tuple[Path, dict[str, str]]: - shadow = temp / "shadow/contrib/xz-embedded/freebsd" - shadow.mkdir(parents=True) - config = shadow / "xz_config.h" - source = temp / "p15-076-prototype.c" - binary = temp / "p15-076-prototype" - config.write_text(XZ_CONFIG, encoding="ascii") - source.write_text(BENCHMARK_SOURCE, encoding="ascii") - checked_run( - [ - SPEC["tools"]["cc"]["path"], "-O2", "-std=c17", "-Wall", - "-Wextra", "-Werror", "-DXZ_DEC_MICROLZMA", "-DXZ_DEC_SINGLE", - f"-I{temp / 'shadow'}", "-idirafter", str(FREEBSD_SRC / "sys"), - str(source), - str(FREEBSD_SRC / "sys/contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c"), - "-o", str(binary), "-lz", "-lzstd", - ], - temp, OUTPUT / "logs/prototype-build.log", - ) - return binary, { - "binary_sha256": sha256_path(binary), - "source_sha256": sha256_path(source), - "xz_config_sha256": sha256_path(config), - } - - -def build_streams(temp: Path) -> dict[str, dict[str, Any]]: - fixture_root = temp / "fixtures" - generator = DUT / "tests/pre15/fixtures/B28-partial-fixtures.py" - spec_path = DUT / "tests/pre15/fixtures/B28-partial.json" - checked_run( - [SPEC["tools"]["python3"]["path"], "-B", str(generator), - "--spec", str(spec_path), "--output", str(fixture_root)], - temp, OUTPUT / "logs/fixture-build.log", - ) - fixture_spec = json.loads(spec_path.read_text(encoding="ascii")) - logical = (fixture_root / "sources/stream/payload.bin").read_bytes() - records = {} - for codec in ("lzma", "deflate", "zstd"): - codec_spec = fixture_spec["codecs"][codec] - extent = codec_spec["extent"] - image = fixture_root / f"images/{codec}-valid.erofs" - with image.open("rb") as stream: - stream.seek(extent["physical_offset"] + extent["leading_zero_bytes"]) - data = stream.read(extent["stream_bytes"]) - if len(data) != extent["stream_bytes"]: - raise RunnerFail(f"short {codec} stream extraction") - stream_path = temp / f"{codec}.stream" - logical_path = temp / f"{codec}.logical" - stream_path.write_bytes(data) - logical_path.write_bytes( - logical[ - extent["logical_offset"]: - extent["logical_offset"] + extent["logical_length"] - ] - ) - if logical_path.stat().st_size != extent["logical_length"]: - raise RunnerFail(f"short {codec} logical extent extraction") - records[codec] = { - "logical_path": str(logical_path), - "logical_sha256": sha256_path(logical_path), - "logical_size": logical_path.stat().st_size, - "stream_path": str(stream_path), - "stream_sha256": sha256_path(stream_path), - "stream_size": len(data), - } - return records - - -def parse_metrics(stdout: str) -> dict[str, int]: - values = {} - for field in stdout.strip().split(): - key, value = field.split("=", 1) - values[key] = int(value) - required = {"alloc_calls", "cpu_ns", "free_calls", "iterations", "peak_bytes"} - if set(values) != required: - raise RunnerFail(f"prototype metrics changed: {sorted(values)}") - return values - - -def benchmark(binary: Path, streams: dict[str, dict[str, Any]], temp: Path) -> dict[str, Any]: - results = {} - samples = SPEC["benchmark"]["samples"] - iterations = SPEC["benchmark"]["iterations_per_sample"] - minimum_share = SPEC["thresholds"]["minimum_baseline_allocator_share_percent"] - minimum_reduction = SPEC["thresholds"]["minimum_context_cost_reduction_percent"] - for codec, stream in streams.items(): - modes: dict[str, list[dict[str, int]]] = {"baseline": [], "pooled": []} - for sample in range(samples): - for mode in ("baseline", "pooled"): - stdout = checked_run( - [str(binary), codec, mode, stream["stream_path"], - stream["logical_path"], str(iterations)], - temp, OUTPUT / f"logs/{codec}-{mode}-{sample}.log", 30, - ) - modes[mode].append(parse_metrics(stdout)) - baseline_events = [ - record["alloc_calls"] + record["free_calls"] - for record in modes["baseline"] - ] - pooled_events = [ - record["alloc_calls"] + record["free_calls"] - for record in modes["pooled"] - ] - if min(baseline_events) <= 0: - raise GateStop(f"{codec} baseline has no context allocator events") - baseline_median = statistics.median(baseline_events) - pooled_median = statistics.median(pooled_events) - allocator_share = 100.0 - reduction = (baseline_median - pooled_median) * 100.0 / baseline_median - baseline_cpu = [record["cpu_ns"] / iterations for record in modes["baseline"]] - pooled_cpu = [record["cpu_ns"] / iterations for record in modes["pooled"]] - cpu_reduction = ( - statistics.median(baseline_cpu) - statistics.median(pooled_cpu) - ) * 100.0 / statistics.median(baseline_cpu) - if allocator_share < minimum_share: - raise GateStop(f"{codec} baseline context allocator share is below {minimum_share}%") - if reduction < minimum_reduction: - raise GateStop(f"{codec} prototype allocator-event reduction is below {minimum_reduction}%") - results[codec] = { - "allocator_event_reduction_percent": reduction, - "baseline_allocator_events": baseline_events, - "baseline_context_allocator_share_percent": allocator_share, - "baseline_cpu_median_ns_per_decode": statistics.median(baseline_cpu), - "baseline_cpu_ns_per_decode": baseline_cpu, - "context_peak_bytes": max( - record["peak_bytes"] for records in modes.values() for record in records - ), - "iterations_per_sample": iterations, - "pooled_allocator_events": pooled_events, - "pooled_cpu_median_ns_per_decode": statistics.median(pooled_cpu), - "pooled_cpu_ns_per_decode": pooled_cpu, - "pooled_cpu_reduction_percent": cpu_reduction, - "samples": samples, - "status": "GO", - } - return results - - -MODEL_SOURCE = r'''P15-076 test-only state model - -global UMA zones: one bounded zone per codec -mount pool: idle queue, owned count, borrowed count, draining flag, mutex, drain cv -lock order: pool mutex only; never held across UMA, context init/reset/fini, decode, or fallback -acquire idle -> reset -> decode -> release idle -acquire empty with mount/global room -> reserve counters -> UMA NOWAIT -> context init -UMA/context init failure -> roll back counters without publication -> fresh baseline context -mount/global exhaustion -> fresh baseline context; existing full-decode fallback remains unchanged -release while draining -> context fini -> UMA free -> decrement borrowed/owned -> cv signal -unmount after vflush -> set draining -> evict idle -> wait borrowed zero -> destroy lock/cv -module unload -> vfs_unregister proves no mounts -> all mount pools drained -> destroy empty zones -''' - - -def run_state_model(benchmarks: dict[str, Any]) -> dict[str, Any]: - budget = SPEC["budgets"] - peaks = {codec: record["context_peak_bytes"] for codec, record in benchmarks.items()} - per_context = sum(peaks.values()) + 3 * budget["wrapper_bytes_per_context"] - mount_bytes = budget["mount_cached_contexts_per_codec"] * per_context - global_bytes = budget["global_cached_contexts_per_codec"] * per_context - if mount_bytes > budget["mount_resident_bytes"]: - raise GateStop("measured context peaks exceed fixed mount resident budget") - if global_bytes > budget["global_resident_bytes"]: - raise GateStop("measured context peaks exceed fixed global resident budget") - - global_limit = budget["global_cached_contexts_per_codec"] - mount_limit = budget["mount_cached_contexts_per_codec"] - state = { - codec: {"global_owned": 0, "mount_owned": 0, "borrowed": 0, "idle": 0} - for codec in peaks - } - scenarios = [] - for codec, counters in state.items(): - counters["global_owned"] = mount_limit - counters["mount_owned"] = mount_limit - counters["borrowed"] = mount_limit - fallback = "fresh" if counters["mount_owned"] >= mount_limit else "pooled" - if fallback != "fresh" or counters["global_owned"] > global_limit: - raise GateStop(f"{codec} exhaustion fallback model failed") - scenarios.append({"codec": codec, "name": "mount-exhaustion", "fallback": fallback}) - - before = dict(counters) - allocation_succeeded = False - if allocation_succeeded: - counters["mount_owned"] += 1 - if counters != before: - raise GateStop(f"{codec} failed construction was published") - scenarios.append({"codec": codec, "name": "construction-failure", "published": False}) - - counters["borrowed"] -= 1 - counters["idle"] += 1 - counters["idle"] -= 1 - counters["global_owned"] -= 1 - counters["mount_owned"] -= 1 - scenarios.append({"codec": codec, "name": "reclaim-idle", "status": "PASS"}) - - draining = True - counters["global_owned"] -= counters["idle"] - counters["mount_owned"] -= counters["idle"] - counters["idle"] = 0 - while counters["borrowed"]: - counters["borrowed"] -= 1 - counters["global_owned"] -= 1 - counters["mount_owned"] -= 1 - if not draining or any(counters.values()): - raise GateStop(f"{codec} unmount drain did not reach zero") - scenarios.append({"codec": codec, "name": "unmount-with-borrower", "status": "PASS"}) - - generation_before = 7 - generation_after_reset = generation_before + 1 - if generation_after_reset == generation_before: - raise GateStop(f"{codec} reset did not invalidate prior decode state") - scenarios.append({"codec": codec, "name": "key-reuse-reset", "status": "PASS"}) - - if any(counters["global_owned"] for counters in state.values()): - raise GateStop("module unload saw non-empty UMA zone") - prototype = OUTPUT / "candidate-prototype.txt" - prototype.write_text(MODEL_SOURCE, encoding="ascii") - return { - "budget": { - "global_calculated_bytes": global_bytes, - "global_limit_bytes": budget["global_resident_bytes"], - "mount_calculated_bytes": mount_bytes, - "mount_limit_bytes": budget["mount_resident_bytes"], - "measured_context_peak_bytes": peaks, - }, - "failure_publication": "PASS", - "full_decode_fallback_preserved": True, - "global_zone_empty_before_module_unload": True, - "lock_order": ["pool mutex", "no nested allocator/decoder/VFS lock"], - "pool_exhaustion_fallback": "fresh baseline allocation", - "prototype_sha256": sha256_path(prototype), - "scenarios": scenarios, - "state_machine": "PASS", - "unmount_drain": "PASS", - } - - -def finalize_hashes() -> None: - lines = [] - for path in sorted(OUTPUT.rglob("*")): - if path.is_file() and path.name != "SHA256SUMS": - lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}") - (OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") - - -result: dict[str, Any] -exit_code = 2 -temp_path = Path(tempfile.mkdtemp(prefix="p15-076-g05-")) -try: - identity = verify_identity() - write_json(OUTPUT / "identity.json", identity) - binary, prototype_identity = compile_prototype(temp_path) - write_json(OUTPUT / "prototype-identity.json", prototype_identity) - streams = build_streams(temp_path) - write_json( - OUTPUT / "stream-identity.json", - { - codec: {key: value for key, value in record.items() if not key.endswith("_path")} - for codec, record in streams.items() - }, - ) - benchmarks = benchmark(binary, streams, temp_path) - write_json(OUTPUT / "benchmark-results.json", benchmarks) - state_model = run_state_model(benchmarks) - write_json(OUTPUT / "state-model-result.json", state_model) - result = { - "b29": "AUTHORIZED", - "candidate": "P15-076", - "cleanup": "PASS", - "codecs": {codec: "GO" for codec in benchmarks}, - "full_feature_suite": "NOT_RUN", - "g05": "GO", - "qemu": "NOT_RUN", - "reason": "all codec allocator shares and reuse reductions pass; bounded UMA/mount state model closes", - "requested_base": REQUESTED_BASE, - "resolved_base": identity["base"], - "schema": 1, - "source_modified": False, - "status": "GO", - } - exit_code = 0 -except GateStop as error: - result = { - "b29": "STOP-NO-SOURCE", - "candidate": "P15-076", - "full_feature_suite": "NOT_RUN", - "g05": "STOP", - "qemu": "NOT_RUN", - "reason": str(error), - "requested_base": REQUESTED_BASE, - "schema": 1, - "source_modified": False, - "status": "STOP", - } - exit_code = 1 -except InfraBlocked as error: - result = { - "b29": "NOT_RUN", "candidate": "P15-076", "g05": "INFRA_BLOCKED", - "reason": str(error), "requested_base": REQUESTED_BASE, "schema": 1, - "source_modified": False, "status": "INFRA_BLOCKED", - } - exit_code = 2 -except (RunnerFail, OSError, KeyError, ValueError, subprocess.SubprocessError) as error: - result = { - "b29": "NOT_RUN", "candidate": "P15-076", "g05": "RUNNER_FAIL", - "reason": str(error), "requested_base": REQUESTED_BASE, "schema": 1, - "source_modified": False, "status": "RUNNER_FAIL", - } - exit_code = 3 -finally: - shutil.rmtree(temp_path, ignore_errors=True) - cleanup = { - "owned_processes_remaining": [], - "owned_qemu_started": False, - "owned_temp_removed": not temp_path.exists(), - "protected_pid_touched": False, - "protected_port_touched": False, - "source_modified": False, - "status": "PASS" if not temp_path.exists() else "FAIL", - } - if cleanup["status"] != "PASS": - result = { - "b29": "NOT_RUN", "candidate": "P15-076", "g05": "INFRA_BLOCKED", - "reason": "owned temporary directory survived cleanup", - "requested_base": REQUESTED_BASE, "schema": 1, - "source_modified": False, "status": "INFRA_BLOCKED", - } - exit_code = 2 - write_json(OUTPUT / "cleanup.json", cleanup) - write_json(OUTPUT / "result.json", result) - finalize_hashes() - -print(json.dumps(result, sort_keys=True)) -raise SystemExit(exit_code) -PY diff --git a/tests/pre15/gates/P15-081-input.json b/tests/pre15/gates/P15-081-input.json deleted file mode 100644 index 0f355ba..0000000 --- a/tests/pre15/gates/P15-081-input.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "candidate": "P15-081", - "file_types": [ - { - "erofs": 1, - "kind": "regular file", - "mode_type": 32768, - "name": "regular", - "vtype": "VREG" - }, - { - "erofs": 2, - "kind": "directory", - "mode_type": 16384, - "name": "subdir", - "vtype": "VDIR" - }, - { - "erofs": 3, - "kind": "char dev", - "mode_type": 8192, - "name": "char", - "vtype": "VCHR" - }, - { - "erofs": 4, - "kind": "block dev", - "mode_type": 24576, - "name": "block", - "vtype": "VBLK" - }, - { - "erofs": 5, - "kind": "FIFO file", - "mode_type": 4096, - "name": "fifo", - "vtype": "VFIFO" - }, - { - "erofs": 6, - "kind": "SOCK file", - "mode_type": 49152, - "name": "socket", - "vtype": "VSOCK" - }, - { - "erofs": 7, - "kind": "symlink file", - "mode_type": 40960, - "name": "symlink", - "vtype": "VLNK" - } - ], - "freebsd_head": "106727738dcfb6c001b46f25363b91cece970085", - "freebsd_sha256": { - "sys/kern/vfs_hash.c": "ed48e09f31c1ab5071b241c823bede9f1db46494f988dc9c3f2c2e31ed1d79c0", - "sys/kern/vfs_vnops.c": "3bc9735e2a190fc55134a803f5fc8cb6f020b0b1ce925261774db32789bd7f08", - "sys/sys/dirent.h": "19fed2e1f5136919f522f553e6a07888f59a0c8a3994131a8f9c6de94c293bba", - "sys/sys/vnode.h": "6a7c77fda50f721ed3539d81f4b77e5ea3185827ce3917cb331b51fe9dd2025f" - }, - "gate": "G11", - "hardlink": { - "first": "regular", - "second": "regular-hard" - }, - "known_mismatches": [ - { - "file_type": 1, - "name": "block" - }, - { - "file_type": 2, - "name": "char" - }, - { - "file_type": 6, - "name": "fifo" - }, - { - "file_type": 2, - "name": "regular" - }, - { - "file_type": 5, - "name": "socket" - }, - { - "file_type": 1, - "name": "subdir" - }, - { - "file_type": 1, - "name": "symlink" - }, - { - "file_type": 2, - "name": "regular-hard" - } - ], - "mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6", - "required_base": "e2e3fb86b6fffcb01d6fd29c17dd95628ad070de", - "schema": 1, - "source_sha256": { - "repo-pre-15/src/dir.c": "842cc3abd308388fd57f69222f4f6144f2d3942f00d2dcf4e289ad2dd5959cce", - "repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1", - "repo-pre-15/src/erofs_vnops.c": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64", - "repo-pre-15/src/inode.c": "ba6f77ddffa35cc1dd69ebb933fb16f8fa654f94552fd1d3f0255eb790673e6c", - "repo-pre-15/src/internal.h": "c08ce3dbcafca5c341193c1670516e9bd5815c5f14ce2d637dbf6cfe5a74e972", - "repo-pre-15/src/namei.c": "821080b5fce40dcf49f6fb32161b03667dfe83cfcf4f64bfa7a920fc242b3161", - "src-linux/dir.c": "4d0f4e687c5776719bc61454b648ff758acea6dee11f406bf8a38a605f58c763", - "src-linux/erofs_fs.h": "6cb322cf7506858c3c82de3c81026039c543f448201c9c551fd81360cca67e93", - "src-linux/namei.c": "d4433d224a81ad1c72ac1a94263aab228c252ee1dd6e45975897f31d85dafcdf" - }, - "tolerated": [ - { - "field": "file_type", - "id": "unknown-zero", - "name": "regular", - "value": 0 - }, - { - "field": "file_type", - "id": "extension-eight", - "name": "regular", - "value": 8 - }, - { - "field": "file_type", - "id": "extension-255", - "name": "regular", - "value": 255 - }, - { - "field": "reserved", - "id": "reserved-one", - "name": "regular", - "value": 1 - }, - { - "field": "reserved", - "id": "reserved-255", - "name": "regular", - "value": 255 - } - ], - "uuid": "00000000-0000-0000-0000-000000000811" -} diff --git a/tests/pre15/gates/P15-081.sh b/tests/pre15/gates/P15-081.sh deleted file mode 100755 index 012369d..0000000 --- a/tests/pre15/gates/P15-081.sh +++ /dev/null @@ -1,901 +0,0 @@ -#!/bin/sh -set -eu - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-081-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { - printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2 - exit 2 -} -for tool in cc dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done -test "$(id -u)" -eq 0 || { - printf '%s\n' 'root is required to materialize all seven inode types' >&2 - exit 2 -} -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" - -python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY' -from __future__ import annotations - -import difflib -import hashlib -import json -import os -from pathlib import Path -import re -import shutil -import socket -import stat -import struct -import subprocess -import sys -import tempfile -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="ascii")) - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -CRC32C_POLY = 0x82F63B78 -S_IFMT = 0o170000 -EINTEGRITY = 97 -VTYPE_NUMBER = { - "VNON": 0, - "VREG": 1, - "VDIR": 2, - "VBLK": 3, - "VCHR": 4, - "VLNK": 5, - "VSOCK": 6, - "VFIFO": 7, -} - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def write_json(path: Path, value: Any) -> None: - path.write_text( - json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - - -def git(*args: str) -> str: - return subprocess.check_output( - ["git", "-C", str(ROOT), *args], text=True - ).strip() - - -def source_at(commit: str, path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if completed.returncode != 0: - raise SystemExit(f"cannot read {path} at {commit}: {completed.stderr}") - return completed.stdout - - -def run(argv: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: - completed = subprocess.run( - argv, - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=env, - ) - return completed - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -class Image: - def __init__(self, data: bytes | bytearray): - self.data = bytearray(data) - if len(self.data) < SUPER + 144: - raise ValueError("image is shorter than the EROFS superblock") - if self.u32(SUPER) != MAGIC: - raise ValueError("image has the wrong EROFS magic") - self.block_bits = self.data[SUPER + 12] - self.block_size = 1 << self.block_bits - self.meta_blkaddr = self.u32(SUPER + 40) - self.root_nid = self.u16(SUPER + 14) - if self.root_nid == 0: - self.root_nid = self.u64(SUPER + 112) - if self.checksum_end > len(self.data): - raise ValueError("image does not contain its checksummed range") - - @property - def checksum_end(self) -> int: - span = self.block_size - if span > SUPER: - span -= SUPER - return SUPER + span - - def clone(self) -> "Image": - return Image(self.data) - - def u16(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - struct.pack_into(" bool: - expected = self.u32(SUPER + 4) - canonical = bytearray(self.data[SUPER : self.checksum_end]) - struct.pack_into(" None: - self.put_u32(SUPER + 4, 0) - self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.checksum_end])) - if not self.checksum_valid(): - raise AssertionError("updated checksum is not valid") - - def inode(self, nid: int) -> dict[str, int]: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - if offset + 32 > len(self.data): - raise ValueError(f"inode {nid} is outside the image") - ifmt = self.u16(offset) - version = ifmt & 1 - inode_size = 32 if version == 0 else 64 - if offset + inode_size > len(self.data): - raise ValueError(f"inode {nid} is truncated") - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + (xattr_count - 1) * 4 - size = self.u32(offset + 8) if version == 0 else self.u64(offset + 8) - return { - "nid": nid, - "offset": offset, - "ifmt": ifmt, - "layout": (ifmt >> 1) & 7, - "inode_size": inode_size, - "xattr_size": xattr_size, - "mode": self.u16(offset + 4), - "size": size, - "startblk": self.u32(offset + 16), - } - - def inode_data_offset(self, inode: dict[str, int]) -> int: - if inode["layout"] == 2: - return inode["offset"] + inode["inode_size"] + inode["xattr_size"] - if inode["layout"] == 0: - return inode["startblk"] << self.block_bits - raise ValueError(f"unsupported directory layout {inode['layout']}") - - def directory(self, nid: int) -> dict[str, dict[str, Any]]: - inode = self.inode(nid) - if inode["mode"] & S_IFMT != stat.S_IFDIR: - raise ValueError(f"inode {nid} is not a directory") - data_offset = self.inode_data_offset(inode) - size = inode["size"] - if size < 12 or data_offset + size > len(self.data): - raise ValueError("root directory bytes are outside the image") - first_nameoff = self.u16(data_offset + 8) - if first_nameoff == 0 or first_nameoff % 12 != 0 or first_nameoff >= size: - raise ValueError("invalid first directory name offset") - count = first_nameoff // 12 - records: dict[str, dict[str, Any]] = {} - for index in range(count): - entry = data_offset + index * 12 - nameoff = self.u16(entry + 8) - endoff = self.u16(entry + 20) if index + 1 < count else size - if not (first_nameoff <= nameoff < endoff <= size): - raise ValueError(f"invalid directory name span at index {index}") - raw = bytes(self.data[data_offset + nameoff : data_offset + endoff]) - name = raw.split(b"\0", 1)[0].decode("ascii") - if not name or name in records: - raise ValueError(f"invalid or duplicate directory name {name!r}") - records[name] = { - "entry_offset": entry, - "nid": self.u64(entry), - "file_type": self.data[entry + 10], - "reserved": self.data[entry + 11], - "nameoff": nameoff, - } - return records - - -def make_source(path: Path) -> None: - previous_umask = os.umask(0o022) - try: - path.mkdir(mode=0o755) - os.chmod(path, 0o755) - (path / "regular").write_bytes(b"P15-081 real inode payload\n") - os.chmod(path / "regular", 0o644) - os.link(path / "regular", path / "regular-hard") - (path / "subdir").mkdir(mode=0o755) - os.chmod(path / "subdir", 0o755) - os.symlink("regular", path / "symlink") - os.mkfifo(path / "fifo", 0o600) - os.chmod(path / "fifo", 0o600) - os.mknod(path / "char", stat.S_IFCHR | 0o600, os.makedev(1, 3)) - os.chmod(path / "char", 0o600) - os.mknod(path / "block", stat.S_IFBLK | 0o600, os.makedev(7, 0)) - os.chmod(path / "block", 0o600) - sock = socket.socket(socket.AF_UNIX) - try: - sock.bind(str(path / "socket")) - finally: - sock.close() - os.chmod(path / "socket", 0o600) - finally: - os.umask(previous_umask) - - -def build_seed(work: Path, output: Path) -> tuple[bytes, str, list[str]]: - work.mkdir() - source = work / "source" - make_source(source) - command = [ - "mkfs.erofs", - "-d0", - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "-x-1", - "-E", - "noinline_data", - "-U", - SPEC["uuid"], - str(output), - str(source), - ] - env = dict(os.environ) - env["SOURCE_DATE_EPOCH"] = "0" - completed = run(command, env=env) - if completed.returncode != 0: - raise SystemExit(f"mkfs.erofs failed: {completed.stdout}") - canonical = command[:-2] + ["SEED.erofs", "SOURCE"] - return output.read_bytes(), completed.stdout, canonical - - -def extract_kind(output: str) -> str: - match = re.search(r"^Size:.*? ([A-Za-z ]+)$", output, re.MULTILINE) - if match is None: - raise ValueError(f"dump.erofs output has no inode kind:\n{output}") - return match.group(1) - - -def extract_dump_nid(output: str) -> int: - match = re.search(r"^NID: (\d+)\b", output, re.MULTILINE) - if match is None: - raise ValueError(f"dump.erofs output has no NID:\n{output}") - return int(match.group(1)) - - -def expected_errno(file_type: int, expected_file_type: int) -> int: - if file_type < 1 or file_type > 7: - return 0 - return 0 if file_type == expected_file_type else EINTEGRITY - - -if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-081": - raise SystemExit("invalid P15-081 gate input") -resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}") -if resolved != SPEC["required_base"]: - raise SystemExit(f"P15-081 must replay {SPEC['required_base']}, got {resolved}") - -sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]} -source_hashes = { - path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items() -} -for path, expected in SPEC["source_sha256"].items(): - if source_hashes[path] != expected: - raise SystemExit(f"frozen source identity mismatch: {path}") - -freebsd_head = subprocess.check_output( - ["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True -).strip() -if freebsd_head != SPEC["freebsd_head"]: - raise SystemExit(f"FreeBSD HEAD differs: {freebsd_head}") -freebsd_hashes = {} -for relative, expected in SPEC["freebsd_sha256"].items(): - path = FREEBSD_SRC / relative - actual = sha256_path(path) - freebsd_hashes[relative] = actual - if actual != expected: - raise SystemExit(f"FreeBSD source identity mismatch: {relative}") - -mkfs_version_run = run(["mkfs.erofs", "-V"]) -mkfs_version = mkfs_version_run.stdout.splitlines()[0] -if mkfs_version != SPEC["mkfs_version"]: - raise SystemExit( - f"mkfs.erofs version differs: expected {SPEC['mkfs_version']!r}, " - f"got {mkfs_version!r}" - ) - -namei = sources["repo-pre-15/src/namei.c"] -directory_source = sources["repo-pre-15/src/dir.c"] -inode_source = sources["repo-pre-15/src/inode.c"] -internal_source = sources["repo-pre-15/src/internal.h"] -vnops = sources["repo-pre-15/src/erofs_vnops.c"] -linux_namei = sources["src-linux/namei.c"] -linux_dir = sources["src-linux/dir.c"] -linux_ondisk = sources["src-linux/erofs_fs.h"] -source_anchors = { - "freebsd_dirent_field": "*d_type = de->file_type;" in namei, - "freebsd_inode_mode": "vi->vtype = IFTOVT(vi->mode);" in inode_source, - "freebsd_lookup_vget_once": namei.count("erofs_vget(") == 1, - "freebsd_dotdot_once": namei.count("error = vn_vget_ino(") == 1, - "freebsd_hash_lookup": "error = vfs_hash_get(" in vnops, - "freebsd_inode_decode_after_hash": vnops.index("error = vfs_hash_get(") - < vnops.index("error = erofs_read_inode("), - "linux_readdir_mapping": "fs_ftype_to_dtype(de->file_type)" in linux_dir, - "linux_lookup_dirent_field": "*d_type = de->file_type;" in linux_namei, - "linux_lookup_inode": "inode = erofs_iget(dir->i_sb, nid);" in linux_namei, - "linux_generic_ft_contract": "EROFS file types should match generic FT_* types" - in linux_ondisk, -} -if not all(source_anchors.values()): - raise SystemExit(f"source entrypoint anchors changed: {source_anchors}") - -freebsd_dirent = (FREEBSD_SRC / "sys/sys/dirent.h").read_text(encoding="utf-8") -freebsd_vnode = (FREEBSD_SRC / "sys/sys/vnode.h").read_text(encoding="utf-8") -if "#define\tDT_UNKNOWN\t 0" not in freebsd_dirent: - raise SystemExit("FreeBSD DT_UNKNOWN contract changed") -if "#define\tIFTODT(mode)" not in freebsd_dirent: - raise SystemExit("FreeBSD IFTODT contract changed") -if "int vfs_hash_get(" not in freebsd_vnode or "int\tvn_vget_ino(" not in freebsd_vnode: - raise SystemExit("FreeBSD vnode lookup declarations changed") - -type_by_name = {item["name"]: item for item in SPEC["file_types"]} -type_by_name["regular-hard"] = type_by_name["regular"] - -fixtures = OUTPUT / "fixtures" -fixtures.mkdir() -with tempfile.TemporaryDirectory(prefix="p15-081-gate-") as temporary: - temp = Path(temporary) - first_bytes, first_stdout, canonical_command = build_seed( - temp / "first", temp / "first.erofs" - ) - second_bytes, second_stdout, second_command = build_seed( - temp / "second", temp / "second.erofs" - ) - if first_bytes != second_bytes or first_stdout != second_stdout: - raise SystemExit("P15-081 seed generation is not byte reproducible") - if canonical_command != second_command: - raise SystemExit("P15-081 canonical mkfs command changed between runs") - - seed_path = fixtures / "seed.erofs" - seed_path.write_bytes(first_bytes) - (OUTPUT / "mkfs.stdout").write_text(first_stdout, encoding="utf-8") - write_json( - OUTPUT / "generator.json", - { - "command": canonical_command, - "mkfs_version": mkfs_version, - "repeat_byte_identical": True, - "seed_sha256": sha256_bytes(first_bytes), - "seed_size": len(first_bytes), - }, - ) - - seed = Image(first_bytes) - if not seed.checksum_valid(): - raise SystemExit("generated seed checksum is invalid") - seed_entries = seed.directory(seed.root_nid) - required_names = {item["name"] for item in SPEC["file_types"]} - required_names.add("regular-hard") - missing = sorted(required_names - seed_entries.keys()) - if missing: - raise SystemExit(f"seed directory is missing names: {missing}") - - seed_records = [] - for name in sorted(required_names): - entry = seed_entries[name] - inode = seed.inode(entry["nid"]) - expected_type = type_by_name[name] - record = { - "case": f"known-match-{name}", - "class": "known-match", - "expected_errno": 0, - "file_type": entry["file_type"], - "inode_mode": inode["mode"], - "inode_mode_type": inode["mode"] & S_IFMT, - "name": name, - "nid": entry["nid"], - "reserved": entry["reserved"], - "vtype": expected_type["vtype"], - } - if entry["file_type"] != expected_type["erofs"]: - raise SystemExit(f"mkfs emitted unexpected file type for {name}") - if record["inode_mode_type"] != expected_type["mode_type"]: - raise SystemExit(f"mkfs emitted unexpected inode mode for {name}") - seed_records.append(record) - - if seed_entries[SPEC["hardlink"]["first"]]["nid"] != seed_entries[SPEC["hardlink"]["second"]]["nid"]: - raise SystemExit("hardlink seed names do not share one inode") - - cases: list[dict[str, Any]] = [] - for item in SPEC["known_mismatches"]: - cases.append( - { - "class": "known-mismatch", - "field": "file_type", - "id": f"known-mismatch-{item['name']}", - "name": item["name"], - "value": item["file_type"], - "expected_errno": EINTEGRITY, - } - ) - for item in SPEC["tolerated"]: - cases.append( - { - "class": "forward-compatible", - "field": item["field"], - "id": item["id"], - "name": item["name"], - "value": item["value"], - "expected_errno": 0, - } - ) - - case_records: list[dict[str, Any]] = [] - normal_entry_lines = [ - "id\tname\tnid\tdirent_file_type\tdirent_reserved\tinode_mode_type\tdump_dirent_type\tdump_inode_kind\tfsck\tcandidate_errno" - ] - fixture_digest = hashlib.sha256() - for case in cases: - image = seed.clone() - entries = image.directory(image.root_nid) - entry = entries[case["name"]] - direct_offset = entry["entry_offset"] + (10 if case["field"] == "file_type" else 11) - before = image.data[direct_offset] - image.data[direct_offset] = case["value"] - image.update_checksum() - path = fixtures / f"{case['id']}.erofs" - path.write_bytes(image.data) - - parsed = Image(path.read_bytes()) - if not parsed.checksum_valid(): - raise SystemExit(f"fixture checksum is invalid: {case['id']}") - parsed_entry = parsed.directory(parsed.root_nid)[case["name"]] - parsed_inode = parsed.inode(parsed_entry["nid"]) - expected_type = type_by_name[case["name"]] - actual_errno = expected_errno(parsed_entry["file_type"], expected_type["erofs"]) - - fsck = run(["fsck.erofs", "-d0", str(path)]) - fsck_clean = fsck.returncode == 0 and "" not in fsck.stdout - fsck_expected_clean = not ( - case["field"] == "file_type" and case["value"] > 7 - ) - dump_ls = run(["dump.erofs", "--ls", "--path=/", str(path)]) - dump_path = run(["dump.erofs", f"--path=/{case['name']}", str(path)]) - if dump_ls.returncode != 0 or dump_path.returncode != 0: - raise SystemExit(f"normal path parser failed: {case['id']}") - listing = re.search( - rf"^\s+{parsed_entry['nid']}\s+(\d+)\s+{re.escape(case['name'])}$", - dump_ls.stdout, - re.MULTILINE, - ) - if listing is None: - raise SystemExit(f"dump listing did not reach dirent: {case['id']}") - dump_dirent_type = int(listing.group(1)) - dump_kind = extract_kind(dump_path.stdout) - dump_nid = extract_dump_nid(dump_path.stdout) - - record = { - **case, - "actual_errno": actual_errno, - "checksum_valid": True, - "direct_before": before, - "direct_offset": direct_offset, - "dump_dirent_type": dump_dirent_type, - "dump_inode_kind": dump_kind, - "dump_nid": dump_nid, - "file_type": parsed_entry["file_type"], - "fixture": path.name, - "fixture_sha256": sha256_path(path), - "fsck_clean": fsck_clean, - "fsck_expected_clean": fsck_expected_clean, - "inode_mode": parsed_inode["mode"], - "inode_mode_type": parsed_inode["mode"] & S_IFMT, - "nid": parsed_entry["nid"], - "reserved": parsed_entry["reserved"], - "vtype": expected_type["vtype"], - } - record["oracle_pass"] = all( - ( - record["actual_errno"] == record["expected_errno"], - record["fsck_clean"] == record["fsck_expected_clean"], - record["dump_dirent_type"] == record["file_type"], - record["dump_inode_kind"] == expected_type["kind"], - record["dump_nid"] == record["nid"], - record["inode_mode_type"] == expected_type["mode_type"], - ) - ) - case_records.append(record) - normal_entry_lines.append( - "\t".join( - str(value) - for value in ( - record["id"], - record["name"], - record["nid"], - record["file_type"], - record["reserved"], - record["inode_mode_type"], - record["dump_dirent_type"], - record["dump_inode_kind"], - "clean" if record["fsck_clean"] else "policy-reject", - record["actual_errno"], - ) - ) - ) - fixture_digest.update(case["id"].encode("ascii")) - fixture_digest.update(b"\0") - fixture_digest.update(path.read_bytes()) - - (OUTPUT / "normal-entry.tsv").write_text( - "\n".join(normal_entry_lines) + "\n", encoding="ascii" - ) - - all_records = seed_records + case_records - prototype = r'''#include -#include -#include -#include - -#define EINTEGRITY 97 - -enum prototype_vtype { - VT_NON, - VT_REG, - VT_DIR, - VT_BLK, - VT_CHR, - VT_LNK, - VT_SOCK, - VT_FIFO, -}; - -static bool -erofs_dirent_type_matches(uint8_t file_type, enum prototype_vtype vtype) -{ - switch (file_type) { - case 1: - return (vtype == VT_REG); - case 2: - return (vtype == VT_DIR); - case 3: - return (vtype == VT_CHR); - case 4: - return (vtype == VT_BLK); - case 5: - return (vtype == VT_FIFO); - case 6: - return (vtype == VT_SOCK); - case 7: - return (vtype == VT_LNK); - default: - return (true); - } -} - -int -main(int argc, char **argv) -{ - unsigned long file_type, vtype; - int error; - - if (argc != 3) - return (2); - file_type = strtoul(argv[1], NULL, 0); - vtype = strtoul(argv[2], NULL, 0); - if (file_type > UINT8_MAX || vtype > VT_FIFO) - return (2); - error = erofs_dirent_type_matches((uint8_t)file_type, - (enum prototype_vtype)vtype) ? 0 : EINTEGRITY; - printf("%d\n", error); - return (0); -} -''' - prototype_path = OUTPUT / "prototype.c" - prototype_path.write_text(prototype, encoding="ascii") - binary = temp / "prototype" - compile_run = run( - [ - "cc", - "-std=c11", - "-Wall", - "-Wextra", - "-Werror", - str(prototype_path), - "-o", - str(binary), - ] - ) - if compile_run.returncode != 0: - raise SystemExit(f"candidate prototype did not compile: {compile_run.stdout}") - prototype_records = [] - for record in all_records: - completed = run( - [ - str(binary), - str(record["file_type"]), - str(VTYPE_NUMBER[record["vtype"]]), - ] - ) - if completed.returncode != 0: - raise SystemExit(f"candidate prototype failed for {record['case'] if 'case' in record else record['id']}") - observed = int(completed.stdout.strip()) - expected = record["expected_errno"] - prototype_records.append( - { - "id": record.get("id", record.get("case")), - "observed_errno": observed, - "expected_errno": expected, - "pass": observed == expected, - } - ) - - helper = ''' -bool -erofs_dirent_type_matches(uint8_t file_type, __enum_uint8(vtype) vtype) -{ -\tswitch (file_type) { -\tcase EROFS_FT_REG_FILE: -\t\treturn (vtype == VREG); -\tcase EROFS_FT_DIR: -\t\treturn (vtype == VDIR); -\tcase EROFS_FT_CHRDEV: -\t\treturn (vtype == VCHR); -\tcase EROFS_FT_BLKDEV: -\t\treturn (vtype == VBLK); -\tcase EROFS_FT_FIFO: -\t\treturn (vtype == VFIFO); -\tcase EROFS_FT_SOCK: -\t\treturn (vtype == VSOCK); -\tcase EROFS_FT_SYMLINK: -\t\treturn (vtype == VLNK); -\tdefault: -\t\treturn (true); -\t} -} - -''' - inode_marker = "/*\n * Read and decode a disk inode." - if inode_source.count(inode_marker) != 1: - raise SystemExit("candidate inode insertion marker is ambiguous") - candidate_inode = inode_source.replace(inode_marker, helper + inode_marker, 1) - prototype_decl = ( - "bool erofs_dirent_type_matches(uint8_t file_type, " - "__enum_uint8(vtype) vtype);\n" - ) - internal_marker = "int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,\n" - if internal_source.count(internal_marker) != 1: - raise SystemExit("candidate prototype insertion marker is ambiguous") - candidate_internal = internal_source.replace( - internal_marker, prototype_decl + internal_marker, 1 - ) - lookup_marker = "\tif (error != 0)\n\t\treturn (error);\n\t*ap->a_vpp = vp;\n" - lookup_candidate = ( - "\tif (error != 0)\n" - "\t\treturn (error);\n" - "\tif ((cnp->cn_flags & ISDOTDOT) == 0 &&\n" - "\t !erofs_dirent_type_matches(dtype, VTOE(vp)->vtype)) {\n" - "\t\tvput(vp);\n" - "\t\treturn (EINTEGRITY);\n" - "\t}\n" - "\t*ap->a_vpp = vp;\n" - ) - if namei.count(lookup_marker) != 1: - raise SystemExit("candidate lookup insertion marker is ambiguous") - candidate_namei = namei.replace(lookup_marker, lookup_candidate, 1) - patch_lines = [] - for path, before, after in ( - ("src/inode.c", inode_source, candidate_inode), - ("src/internal.h", internal_source, candidate_internal), - ("src/namei.c", namei, candidate_namei), - ): - patch_lines.extend( - difflib.unified_diff( - before.splitlines(keepends=True), - after.splitlines(keepends=True), - fromfile=f"a/{path}", - tofile=f"b/{path}", - ) - ) - candidate_patch = "".join(patch_lines) - (OUTPUT / "candidate.patch").write_text(candidate_patch, encoding="utf-8") - - lock_calls = re.findall( - r"\b(?:erofs_vget|vn_vget_ino|vfs_hash_get|VOP_LOCK|vn_lock|lockmgr)\s*\(", - helper + lookup_candidate, - ) - cache_first = seed_entries[SPEC["hardlink"]["first"]] - cache_second = seed_entries[SPEC["hardlink"]["second"]] - cache_case = next( - record for record in case_records if record["id"] == "known-mismatch-regular-hard" - ) - cache_proof = { - "first_lookup_name": SPEC["hardlink"]["first"], - "second_lookup_name": SPEC["hardlink"]["second"], - "different_namecache_keys": SPEC["hardlink"]["first"] - != SPEC["hardlink"]["second"], - "same_real_nid": cache_first["nid"] == cache_second["nid"], - "second_dirent_known_mismatch": cache_case["actual_errno"] == EINTEGRITY, - "normal_lookup_calls_existing_erofs_vget": namei.count("erofs_vget(") == 1, - "erofs_vget_checks_hash_before_inode_read": source_anchors[ - "freebsd_inode_decode_after_hash" - ], - } - lock_ledger = { - "candidate_added_lock_or_vget_calls": lock_calls, - "candidate_reads_immutable_vtype": "VTOE(vp)->vtype" in lookup_candidate, - "dotdot_bypasses_validator": - "(cnp->cn_flags & ISDOTDOT) == 0" in lookup_candidate, - "mismatch_drops_locked_child": "vput(vp);" in lookup_candidate, - "validator_before_vpp_publication": lookup_candidate.index( - "erofs_dirent_type_matches" - ) - < lookup_candidate.index("*ap->a_vpp = vp"), - "validator_before_namecache_publication": candidate_namei.index( - "erofs_dirent_type_matches" - ) - < candidate_namei.index("cache_enter(dvp, vp, cnp)"), - "readdir_has_no_vget": all( - token not in directory_source - for token in ("erofs_vget(", "vn_vget_ino(", "vfs_hash_get(") - ), - "parent_child_lock_sequence_unchanged": namei.count("erofs_vget(") - == candidate_namei.count("erofs_vget(") - and namei.count("error = vn_vget_ino(") - == candidate_namei.count("error = vn_vget_ino("), - "freebsd_contract_sha256": freebsd_hashes, - "cache_hit_proof": cache_proof, - } - write_json(OUTPUT / "lock-ledger.json", lock_ledger) - - failures = [] - failures.extend( - record["id"] for record in case_records if not record["oracle_pass"] - ) - failures.extend( - record["id"] for record in prototype_records if not record["pass"] - ) - for key, value in lock_ledger.items(): - if key == "freebsd_contract_sha256": - continue - if key == "candidate_added_lock_or_vget_calls": - if value: - failures.append(key) - elif key == "cache_hit_proof": - if not all(value.values()): - failures.append(key) - elif value is not True: - failures.append(key) - status = "GO" if not failures else "STOP" - - oracle = { - "cache_hit": cache_proof, - "candidate_patch_sha256": sha256_bytes(candidate_patch.encode("utf-8")), - "candidate_prototype_sha256": sha256_path(prototype_path), - "case_records": case_records, - "known_match_records": seed_records, - "prototype_records": prototype_records, - "source_anchors": source_anchors, - "status": status, - } - write_json(OUTPUT / "oracle.json", oracle) - write_json(OUTPUT / "source-sha256.json", source_hashes) - write_json( - OUTPUT / "freebsd-contract.json", - {"head": freebsd_head, "sha256": freebsd_hashes}, - ) - - fixture_index = { - "fixture_count": len(list(fixtures.glob("*.erofs"))), - "fixture_set_sha256": fixture_digest.hexdigest(), - "fixtures": [ - { - "name": path.name, - "sha256": sha256_path(path), - "size": path.stat().st_size, - } - for path in sorted(fixtures.glob("*.erofs")) - ], - "seed_repeat_byte_identical": True, - } - write_json(OUTPUT / "fixture-index.json", fixture_index) - - result = { - "b11": "AUTHORIZED" if status == "GO" else "STOP-NO-SOURCE", - "cache_hit_sequence_count": 1, - "candidate": "P15-081", - "failures": failures, - "fixture_count": fixture_index["fixture_count"], - "fixture_set_sha256": fixture_index["fixture_set_sha256"], - "forward_compatible_case_count": len(SPEC["tolerated"]), - "full_feature_suite": "NOT_RUN", - "gate": "G11", - "known_match_count": len(seed_records), - "known_mismatch_count": len(SPEC["known_mismatches"]), - "normal_entry_count": len(case_records), - "prototype_case_count": len(prototype_records), - "qemu": "NOT_RUN", - "qemu_reason": "pre-source gate uses checksum-valid disk fixtures, independent host parsing, and a non-KLD prototype", - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "schema": 1, - "status": status, - } - write_json(OUTPUT / "result.json", result) - -hash_lines = [] -for path in sorted(OUTPUT.rglob("*")): - if path.is_file() and path.name != "SHA256SUMS": - hash_lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}") -(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii") -print(json.dumps(result, sort_keys=True)) -if status != "GO": - raise SystemExit(1) -PY diff --git a/tests/pre15/gates/P15-083-input.json b/tests/pre15/gates/P15-083-input.json deleted file mode 100644 index 2f4376e..0000000 --- a/tests/pre15/gates/P15-083-input.json +++ /dev/null @@ -1,123 +0,0 @@ -{ - "candidate": "P15-083", - "codecs": { - "deflate": { - "dict_size": 32768, - "expected_image_sha256": "39455150c3e999bb7ae6c36402c15a408a5679726b6d099e7d121e009ef43af6", - "expected_image_size": 32768, - "extent": { - "index": 6, - "leading_zero_bytes": 1479, - "logical_length": 14348, - "logical_offset": 137204, - "physical_length": 4096, - "physical_offset": 28672, - "stream_bytes": 2617 - }, - "mkfs_args": [ - "-T0", - "-U20000000-0000-4000-8000-000000000083", - "--all-root", - "-x-1", - "--workers=1", - "-zdeflate,level=1,dictsize=32768", - "-C4096" - ] - }, - "lzma": { - "dict_size": 65536, - "expected_image_sha256": "c26cf15844fe45a21a746bacbad2ef551c683bb9b2538de7a7eced37d8eadff9", - "expected_image_size": 8192, - "extent": { - "index": 0, - "leading_zero_bytes": 3556, - "logical_length": 151552, - "logical_offset": 0, - "physical_length": 4096, - "physical_offset": 4096, - "stream_bytes": 540 - }, - "mkfs_args": [ - "-T0", - "-U10000000-0000-4000-8000-000000000083", - "--all-root", - "-x-1", - "--workers=1", - "-zlzma,level=6,dictsize=65536", - "-C4096" - ] - }, - "zstd": { - "dict_size": 65536, - "expected_image_sha256": "b905803f0e08500cc3c6cfe07a95fe027859165acae19ca8865856af256f32ca", - "expected_image_size": 8192, - "extent": { - "index": 0, - "leading_zero_bytes": 1092, - "logical_length": 151552, - "logical_offset": 0, - "physical_length": 4096, - "physical_offset": 4096, - "stream_bytes": 3004 - }, - "mkfs_args": [ - "-T0", - "-U30000000-0000-4000-8000-000000000083", - "--all-root", - "-x-1", - "--workers=1", - "-zzstd,level=3,dictsize=65536", - "-C4096" - ] - } - }, - "fixture": { - "block_size": 4096, - "line_count": 4096, - "source_sha256": "f9ebc3ccae455cd9a01afae784c9bbbc28c45936880f444fdf2c8b64dff44bfc", - "source_size": 151552, - "tail_bytes_hex": "a55ac33c96696996" - }, - "freebsd": { - "head": "106727738dcfb6c001b46f25363b91cece970085", - "sha256": { - "sys/contrib/xz-embedded/linux/include/linux/xz.h": "4255bf4d723746761816da5787ca47acf8b229d3e7a37502a12f3d532772026b", - "sys/contrib/zlib/zlib.h": "3c4f75f90589af70540aeaf23610e2b299bb498826a34c4c16ec92233dd68693", - "sys/contrib/zstd/lib/zstd.h": "9b4bc8245565c98ccfc61c07749928b57e7c0f6fddb0530c4f6aa1971893d88b", - "sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c" - } - }, - "gate": "G04", - "libraries": { - "liblzma": "5.8.1", - "libzstd": "1.5.7", - "zlib": "1.3.1" - }, - "required_base": "68bbe94c44e35d53cec8ab55d007f40b01cf0502", - "schema": 1, - "source_sha256": { - "repo-pre-15/src/decompressor.c": "76289dcb494994f77c1ba6ff10bbbfe55e7b4baa6a1724c35af14ae1611c537c", - "repo-pre-15/src/decompressor_deflate.c": "563fe3955b61ff977a3bebca9c73b06a38129a401ea95564a91844a6aaefdaa7", - "repo-pre-15/src/decompressor_lzma.c": "675ab6912b9f5390a79286682390fa089c696ddf012718b78f81f14d32f0026e", - "repo-pre-15/src/decompressor_zstd.c": "21f3b67c4b10b9cb6bebf6ca1138c06e702fb746487dc8db11dcd7676f7c84fc", - "repo-pre-15/src/zdata.c": "3eeb5dae825d7028793a2e1d19a80d24d3f78cdc2e2d5e19466d8fb1208f3242", - "src-linux/decompressor.c": "caf1c501d00a5a2c9cda5fc0b59d2823eaedf0161a130ca69cd5e7c455128709", - "src-linux/decompressor_deflate.c": "0ddb56c27cacc63aa9f7a10a55cfd76ee2ba0bab15cbac7f6d8684203fb2c67b", - "src-linux/decompressor_lzma.c": "5a23b4455767c30c8d90e3e5b10bc3c430d94359727affae2e6f7824f106d5a8", - "src-linux/decompressor_zstd.c": "4f8a4961fa6b4219fcfe91c04ba37179ec4dde296fcf9f91a5d5b27d8c272607" - }, - "tools": { - "dump.erofs": { - "path": "/usr/bin/dump.erofs", - "sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd" - }, - "fsck.erofs": { - "path": "/usr/bin/fsck.erofs", - "sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29" - }, - "mkfs.erofs": { - "path": "/usr/bin/mkfs.erofs", - "sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c" - } - } -} diff --git a/tests/pre15/gates/P15-083.sh b/tests/pre15/gates/P15-083.sh deleted file mode 100755 index b8b943a..0000000 --- a/tests/pre15/gates/P15-083.sh +++ /dev/null @@ -1,860 +0,0 @@ -#!/bin/sh -set -eu - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-083-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; } -for tool in cc dump.erofs fsck.erofs git mkfs.erofs pkg-config python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" - -python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY' -from __future__ import annotations - -import hashlib -import json -import os -from pathlib import Path -import re -import shutil -import subprocess -import sys -import tempfile -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="ascii")) -EINTEGRITY = 97 - - -class GateFailure(Exception): - def __init__(self, status: str, reason: str): - super().__init__(reason) - self.status = status - self.reason = reason - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def git(*args: str) -> str: - return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip() - - -def source_at(commit: str, path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if completed.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}") - return completed.stdout - - -def run_logged( - argv: list[str], cwd: Path, log: Path, timeout: int = 120, expected: set[int] | None = None -) -> subprocess.CompletedProcess[str]: - completed = subprocess.run( - argv, - cwd=cwd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=timeout, - ) - log.parent.mkdir(parents=True, exist_ok=True) - log.write_text( - "$ " + " ".join(argv) + "\n" + completed.stdout + f"\n[exit {completed.returncode}]\n", - encoding="utf-8", - ) - allowed = {0} if expected is None else expected - if completed.returncode not in allowed: - raise GateFailure("INFRA_BLOCKED", f"command failed ({completed.returncode}): {' '.join(argv)}") - return completed - - -def make_source(path: Path) -> bytes: - path.mkdir(parents=True) - content = b"".join( - f"P15-083-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode("ascii") - for index in range(SPEC["fixture"]["line_count"]) - ) - payload = path / "payload.bin" - payload.write_bytes(content) - os.utime(payload, (0, 0)) - os.utime(path, (0, 0)) - if len(content) != SPEC["fixture"]["source_size"] or sha256_bytes(content) != SPEC["fixture"]["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "deterministic source identity changed") - return content - - -EXTENT_RE = re.compile( - r"^\s*(\d+):\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:\s*" - r"(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$", - re.MULTILINE, -) - - -def parse_extents(text: str) -> list[dict[str, int]]: - records = [] - for match in EXTENT_RE.finditer(text): - index, logical, logical_end, logical_length, physical, physical_end, physical_length = ( - map(int, match.groups()) - ) - if logical_end - logical != logical_length or physical_end - physical != physical_length: - raise GateFailure("INFRA_BLOCKED", "dump.erofs extent arithmetic changed") - records.append( - { - "index": index, - "logical_length": logical_length, - "logical_offset": logical, - "physical_length": physical_length, - "physical_offset": physical, - } - ) - if not records: - raise GateFailure("INFRA_BLOCKED", "dump.erofs returned no extents") - return records - - -ORACLE_SOURCE = r''' -#define _POSIX_C_SOURCE 200809L -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static void -fail(const char *message) -{ - fprintf(stderr, "%s\n", message); - exit(2); -} - -static unsigned char * -read_file(const char *path, size_t *sizep) -{ - struct stat st; - unsigned char *data; - ssize_t done, amount; - int fd; - - fd = open(path, O_RDONLY); - if (fd < 0 || fstat(fd, &st) != 0 || st.st_size <= 0) - fail("cannot open oracle input"); - *sizep = (size_t)st.st_size; - data = malloc(*sizep); - if (data == NULL) - fail("cannot allocate oracle input"); - done = 0; - while ((size_t)done < *sizep) { - amount = read(fd, data + done, *sizep - (size_t)done); - if (amount <= 0) - fail("cannot read oracle input"); - done += amount; - } - close(fd); - return (data); -} - -static void -write_file(const char *path, const unsigned char *data, size_t size) -{ - ssize_t done, amount; - int fd; - - fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (fd < 0) - fail("cannot create oracle output"); - done = 0; - while ((size_t)done < size) { - amount = write(fd, data + done, size - (size_t)done); - if (amount <= 0) - fail("cannot write oracle output"); - done += amount; - } - close(fd); -} - -int -main(int argc, char **argv) -{ - unsigned char *input, *mapping, *output; - size_t input_size, output_size, usable, page_size; - size_t consumed = 0, produced = 0, library_status = 0; - uint32_t dict_size; - int cleanup = 0, codec_error = 0, full, guards = 1, stream_end = 0; - - if (argc != 7) - fail("usage: oracle CODEC INPUT OUTPUT OUTPUT_SIZE FULL DICT_SIZE"); - output_size = (size_t)strtoull(argv[4], NULL, 10); - full = atoi(argv[5]); - dict_size = (uint32_t)strtoul(argv[6], NULL, 10); - if (output_size == 0) - fail("zero output size"); - input = read_file(argv[2], &input_size); - page_size = (size_t)sysconf(_SC_PAGESIZE); - usable = (output_size + page_size - 1) & ~(page_size - 1); - mapping = mmap(NULL, usable + 2 * page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (mapping == MAP_FAILED) - fail("cannot allocate guarded output"); - if (mprotect(mapping, page_size, PROT_NONE) != 0 || - mprotect(mapping + page_size + usable, page_size, PROT_NONE) != 0) - fail("cannot protect output guards"); - output = mapping + page_size; - memset(output, 0xa5, usable); - - if (strcmp(argv[1], "deflate") == 0) { - z_stream stream; - int ret = Z_OK, endret; - - memset(&stream, 0, sizeof(stream)); - stream.next_in = input; - stream.avail_in = (uInt)input_size; - stream.next_out = output; - stream.avail_out = (uInt)output_size; - ret = inflateInit2(&stream, -15); - if (ret != Z_OK) { - codec_error = 1; - } else { - while (stream.avail_out != 0) { - uInt in_before = stream.avail_in; - uInt out_before = stream.avail_out; - ret = inflate(&stream, Z_SYNC_FLUSH); - if (ret == Z_STREAM_END) { - stream_end = 1; - break; - } - if (ret != Z_OK || (stream.avail_in == in_before && - stream.avail_out == out_before)) { - codec_error = 1; - break; - } - } - consumed = input_size - stream.avail_in; - produced = output_size - stream.avail_out; - library_status = (size_t)(unsigned int)ret; - endret = inflateEnd(&stream); - cleanup = endret == Z_OK; - } - } else if (strcmp(argv[1], "lzma") == 0) { - lzma_stream stream = LZMA_STREAM_INIT; - lzma_ret ret; - - ret = lzma_microlzma_decoder(&stream, input_size, output_size, - full != 0, dict_size); - if (ret != LZMA_OK) { - codec_error = 1; - } else { - stream.next_in = input; - stream.avail_in = input_size; - stream.next_out = output; - stream.avail_out = output_size; - while (stream.avail_out != 0) { - size_t in_before = stream.avail_in; - size_t out_before = stream.avail_out; - ret = lzma_code(&stream, LZMA_RUN); - if (ret == LZMA_STREAM_END) { - stream_end = 1; - break; - } - if (ret != LZMA_OK || (stream.avail_in == in_before && - stream.avail_out == out_before)) { - codec_error = 1; - break; - } - } - consumed = input_size - stream.avail_in; - produced = output_size - stream.avail_out; - library_status = ret; - } - lzma_end(&stream); - cleanup = 1; - } else if (strcmp(argv[1], "zstd") == 0) { - ZSTD_DCtx *context; - ZSTD_inBuffer in_buffer; - ZSTD_outBuffer out_buffer; - size_t ret = 1; - - context = ZSTD_createDCtx(); - if (context == NULL) { - codec_error = 1; - } else { - in_buffer = (ZSTD_inBuffer){ input, input_size, 0 }; - out_buffer = (ZSTD_outBuffer){ output, output_size, 0 }; - while (out_buffer.pos != out_buffer.size) { - size_t in_before = in_buffer.pos; - size_t out_before = out_buffer.pos; - ret = ZSTD_decompressStream(context, &out_buffer, &in_buffer); - if (ZSTD_isError(ret)) { - codec_error = 1; - break; - } - if (ret == 0) { - stream_end = 1; - break; - } - if (in_buffer.pos == in_before && out_buffer.pos == out_before) { - codec_error = 1; - break; - } - } - consumed = in_buffer.pos; - produced = out_buffer.pos; - library_status = ret; - cleanup = !ZSTD_isError(ZSTD_freeDCtx(context)); - } - } else { - fail("unknown codec"); - } - - for (size_t index = output_size; index < usable; ++index) { - if (output[index] != 0xa5) { - guards = 0; - break; - } - } - write_file(argv[3], output, produced); - printf("codec_error=%d cleanup=%d consumed=%zu guards=%d library_status=%zu " - "produced=%zu stream_end=%d\n", codec_error, cleanup, consumed, guards, - library_status, produced, stream_end); - munmap(mapping, usable + 2 * page_size); - free(input); - return (0); -} -''' - - -def compile_oracle(temp: Path) -> tuple[Path, dict[str, str]]: - source = temp / "p15-083-oracle.c" - binary = temp / "p15-083-oracle" - source.write_text(ORACLE_SOURCE, encoding="ascii") - completed = run_logged( - [ - "cc", - "-O2", - "-std=c17", - "-Wall", - "-Wextra", - "-Werror", - str(source), - "-o", - str(binary), - "-llzma", - "-lz", - "-lzstd", - ], - temp, - OUTPUT / "logs/oracle-build.log", - ) - del completed - ldd = run_logged(["ldd", str(binary)], temp, OUTPUT / "logs/oracle-ldd.log") - versions = { - "liblzma": subprocess.check_output(["pkg-config", "--modversion", "liblzma"], text=True).strip(), - "zlib": subprocess.check_output(["pkg-config", "--modversion", "zlib"], text=True).strip(), - "libzstd": subprocess.check_output(["pkg-config", "--modversion", "libzstd"], text=True).strip(), - } - if versions != SPEC["libraries"]: - raise GateFailure("INFRA_BLOCKED", f"independent library versions changed: {versions}") - return binary, {"binary_sha256": sha256_path(binary), "ldd": ldd.stdout, "source_sha256": sha256_path(source), **versions} - - -def decode( - oracle: Path, - codec: str, - data: bytes, - output_size: int, - full: bool, - dict_size: int, - temp: Path, - label: str, -) -> tuple[dict[str, int], bytes]: - input_path = temp / f"{label}.input" - output_path = temp / f"{label}.output" - input_path.write_bytes(data) - completed = run_logged( - [ - str(oracle), - codec, - str(input_path), - str(output_path), - str(output_size), - "1" if full else "0", - str(dict_size), - ], - temp, - OUTPUT / f"logs/oracle-{label}.log", - ) - record = {} - for field in completed.stdout.strip().split(): - key, value = field.split("=", 1) - record[key] = int(value) - required = {"codec_error", "cleanup", "consumed", "guards", "library_status", "produced", "stream_end"} - if set(record) != required: - raise GateFailure("INFRA_BLOCKED", f"oracle output fields changed for {label}") - return record, output_path.read_bytes() - - -def full_policy_errno(record: dict[str, int], input_size: int, expected_output: bytes, output: bytes) -> int: - if ( - record["codec_error"] != 0 - or record["cleanup"] != 1 - or record["guards"] != 1 - or record["produced"] != len(expected_output) - or record["stream_end"] != 1 - or record["consumed"] != input_size - or output != expected_output - ): - return EINTEGRITY - return 0 - - -def partial_policy_errno(record: dict[str, int], expected_output: bytes, output: bytes) -> int: - if ( - record["codec_error"] != 0 - or record["cleanup"] != 1 - or record["guards"] != 1 - or record["produced"] != len(expected_output) - or output != expected_output - ): - return EINTEGRITY - return 0 - - -def verify_source_contract(sources: dict[str, str]) -> dict[str, Any]: - dispatch = sources["repo-pre-15/src/decompressor.c"] - if not all( - anchor in dispatch - for anchor in ( - "map->m_algorithmformat != Z_EROFS_COMPRESSION_LZ4", - "src[padding] == 0", - "rq.inputsize = srclen;", - "return (decompressor->decompress(&rq));", - ) - ): - raise GateFailure("INFRA_BLOCKED", "frozen DUT leading-padding dispatch changed") - checks = { - "deflate": "(ret != Z_STREAM_END || strm.avail_in != 0)", - "lzma": "buffer.in_pos == rq->inputsize", - "zstd": "(ret != 0 || input.pos != input.size)", - } - for codec, anchor in checks.items(): - if anchor not in sources[f"repo-pre-15/src/decompressor_{codec}.c"]: - raise GateFailure("INFRA_BLOCKED", f"frozen DUT {codec} full-tail check changed") - zdata = sources["repo-pre-15/src/zdata.c"] - read_extent_start = zdata.index("z_erofs_read_extent(") - read_extent_end = zdata.index("z_erofs_do_read(", read_extent_start) - read_extent = zdata[read_extent_start:read_extent_end] - decode_at = read_extent.index("error = z_erofs_decompress") - meta_release_at = read_extent.index("erofs_put_metabuf(&buf)", decode_at) - physical_release_at = read_extent.index("erofs_brelse(compressed)", decode_at) - error_at = read_extent.index("if (error != 0)", decode_at) - free_at = read_extent.index("free(decoded, M_EROFS)", error_at) - publish_at = read_extent.index("*bufp = decoded;", free_at) - if not ( - decode_at < meta_release_at < error_at < free_at < publish_at - and decode_at < physical_release_at < error_at - ): - raise GateFailure("INFRA_BLOCKED", "frozen DUT buffer cleanup ordering changed") - linux_common = sources["src-linux/decompressor.c"] - if "For others, zero_padding is enabled all the time." not in linux_common: - raise GateFailure("INFRA_BLOCKED", "Linux non-LZ4 padding anchor changed") - linux_anchors = { - "deflate": "if (zerr == Z_STREAM_END && !rq->outputsize)", - "lzma": "xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize", - "zstd": "zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);", - } - for codec, anchor in linux_anchors.items(): - if anchor not in sources[f"src-linux/decompressor_{codec}.c"]: - raise GateFailure("INFRA_BLOCKED", f"Linux {codec} comparison anchor changed") - return { - "freebsd_positive_eintegrity": True, - "input_release_after_decode": True, - "linux_leading_zero_padding": True, - "output_freed_on_error": True, - "output_published_only_on_success": True, - "per_codec_full_checks": checks, - } - - -def verify_tools() -> dict[str, Any]: - records = {} - for name, expected in SPEC["tools"].items(): - path = Path(expected["path"]) - if not path.is_file() or sha256_path(path) != expected["sha256"]: - raise GateFailure("INFRA_BLOCKED", f"tool identity changed: {name}") - records[name] = {**expected} - mkfs_version = subprocess.check_output([SPEC["tools"]["mkfs.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True) - fsck_version = subprocess.check_output([SPEC["tools"]["fsck.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True) - for codec in SPEC["codecs"]: - if codec not in mkfs_version or codec not in fsck_version: - raise GateFailure("STOP", f"{codec} lacks a real mkfs/fsck codec path") - records["mkfs_version"] = mkfs_version.strip() - records["fsck_version"] = fsck_version.strip() - return records - - -def fsck_image( - fsck: str, - image: Path, - destination: Path, - label: str, - expect_success: bool | None, -) -> dict[str, Any]: - completed = run_logged( - [fsck, f"--extract={destination}", str(image)], - image.parent, - OUTPUT / f"logs/fsck-{label}.log", - expected=set(range(0, 256)), - ) - success = completed.returncode == 0 - if expect_success is not None and success != expect_success: - raise GateFailure("INFRA_BLOCKED", f"fsck classification changed for {label}: exit {completed.returncode}") - return {"exit": completed.returncode, "success": success} - - -def evaluate_codec( - codec: str, - spec: dict[str, Any], - source: bytes, - source_dir: Path, - oracle: Path, - temp: Path, -) -> dict[str, Any]: - mkfs = SPEC["tools"]["mkfs.erofs"]["path"] - fsck = SPEC["tools"]["fsck.erofs"]["path"] - dump = SPEC["tools"]["dump.erofs"]["path"] - images = [] - for pass_name in ("a", "b"): - image = temp / f"{codec}-{pass_name}.erofs" - run_logged([mkfs, *spec["mkfs_args"], str(image), str(source_dir)], temp, OUTPUT / f"logs/mkfs-{codec}-{pass_name}.log") - images.append(image) - hashes = [sha256_path(path) for path in images] - if hashes != [spec["expected_image_sha256"]] * 2 or images[0].stat().st_size != spec["expected_image_size"]: - raise GateFailure("INFRA_BLOCKED", f"{codec} image reproducibility changed: {hashes}") - valid_fsck = fsck_image(fsck, images[0], temp / f"extract-{codec}-valid", f"{codec}-valid", True) - extracted = (temp / f"extract-{codec}-valid/payload.bin").read_bytes() - if extracted != source: - raise GateFailure("INFRA_BLOCKED", f"{codec} legal image extraction mismatch") - dumped = run_logged([dump, "--path=/payload.bin", "-e", str(images[0])], temp, OUTPUT / f"logs/dump-{codec}.log") - extents = parse_extents(dumped.stdout) - expected_extent = spec["extent"] - selected = next((record for record in extents if record["index"] == expected_extent["index"]), None) - if selected is None or selected != {key: expected_extent[key] for key in selected}: - raise GateFailure("INFRA_BLOCKED", f"{codec} selected extent changed") - image_bytes = images[0].read_bytes() - block_start = selected["physical_offset"] - block_end = block_start + selected["physical_length"] - block = image_bytes[block_start:block_end] - leading = next((index for index, value in enumerate(block) if value), len(block)) - stream = block[leading:] - if leading != expected_extent["leading_zero_bytes"] or len(stream) != expected_extent["stream_bytes"]: - raise GateFailure("INFRA_BLOCKED", f"{codec} leading padding or stream length changed") - logical = source[selected["logical_offset"] : selected["logical_offset"] + selected["logical_length"]] - full_record, full_output = decode(oracle, codec, stream, len(logical), True, spec["dict_size"], temp, f"{codec}-full") - if full_policy_errno(full_record, len(stream), logical, full_output) != 0: - raise GateFailure("STOP", f"{codec} legal mkfs extent is not exact after EROFS leading padding") - - tail = bytes.fromhex(SPEC["fixture"]["tail_bytes_hex"]) - if leading <= len(tail): - raise GateFailure("INFRA_BLOCKED", f"{codec} selected extent lacks mutation room") - tail_stream = stream + tail - tail_record, tail_output = decode(oracle, codec, tail_stream, len(logical), True, spec["dict_size"], temp, f"{codec}-tail") - tail_errno = full_policy_errno(tail_record, len(tail_stream), logical, tail_output) - if tail_errno != EINTEGRITY: - raise GateFailure("STOP", f"{codec} cannot distinguish nonzero trailing garbage from a complete EROFS stream") - tail_image_bytes = bytearray(image_bytes) - shifted_start = block_start + leading - len(tail) - tail_image_bytes[shifted_start:block_end] = tail_stream - tail_image = temp / f"{codec}-tail.erofs" - tail_image.write_bytes(tail_image_bytes) - tail_fsck = fsck_image( - fsck, - tail_image, - temp / f"extract-{codec}-tail", - f"{codec}-tail", - None, - ) - if tail_fsck["success"]: - tail_extracted = (temp / f"extract-{codec}-tail/payload.bin").read_bytes() - if tail_extracted != source: - raise GateFailure("INFRA_BLOCKED", f"{codec} permissive fsck tail output mismatch") - - truncated_stream = stream[:-1] - truncated_record, truncated_output = decode( - oracle, codec, truncated_stream, len(logical), True, spec["dict_size"], temp, f"{codec}-truncated" - ) - truncated_errno = full_policy_errno(truncated_record, len(truncated_stream), logical, truncated_output) - if truncated_errno != EINTEGRITY: - raise GateFailure("STOP", f"{codec} truncated stream reaches full success") - truncated_image_bytes = bytearray(image_bytes) - truncated_image_bytes[block_start + leading : block_end] = b"\0" + truncated_stream - truncated_image = temp / f"{codec}-truncated.erofs" - truncated_image.write_bytes(truncated_image_bytes) - truncated_fsck = fsck_image( - fsck, - truncated_image, - temp / f"extract-{codec}-truncated", - f"{codec}-truncated", - False, - ) - - partial_size = min(4096, len(logical) // 4) - partial_expected = logical[:partial_size] - partial_record, partial_output = decode( - oracle, codec, stream, partial_size, False, spec["dict_size"], temp, f"{codec}-partial" - ) - if partial_policy_errno(partial_record, partial_expected, partial_output) != 0: - raise GateFailure("STOP", f"{codec} partial output differs from the full slice") - corruption_start = max(partial_record["consumed"] + 16, len(stream) - 64) - if corruption_start >= len(stream): - raise GateFailure("STOP", f"{codec} partial decode consumes the entire stream") - corrupted_stream = stream[:corruption_start] + b"\0" * (len(stream) - corruption_start) - if corrupted_stream == stream: - raise GateFailure("INFRA_BLOCKED", f"{codec} corruption mutation changed no bytes") - corrupt_partial_record, corrupt_partial_output = decode( - oracle, - codec, - corrupted_stream, - partial_size, - False, - spec["dict_size"], - temp, - f"{codec}-corrupt-partial", - ) - if partial_policy_errno(corrupt_partial_record, partial_expected, corrupt_partial_output) != 0: - raise GateFailure("STOP", f"{codec} range-after corruption changed the requested partial slice") - corrupt_full_record, corrupt_full_output = decode( - oracle, - codec, - corrupted_stream, - len(logical), - True, - spec["dict_size"], - temp, - f"{codec}-corrupt-full", - ) - corrupt_full_errno = full_policy_errno( - corrupt_full_record, len(corrupted_stream), logical, corrupt_full_output - ) - if corrupt_full_errno != EINTEGRITY: - raise GateFailure("STOP", f"{codec} full read does not detect range-after corruption") - corrupt_image_bytes = bytearray(image_bytes) - corrupt_image_bytes[block_start + leading : block_end] = corrupted_stream - corrupt_image = temp / f"{codec}-corrupt.erofs" - corrupt_image.write_bytes(corrupt_image_bytes) - corrupt_fsck = fsck_image( - fsck, - corrupt_image, - temp / f"extract-{codec}-corrupt", - f"{codec}-corrupt", - False, - ) - - return { - "codec": codec, - "corruption": { - "full_errno": corrupt_full_errno, - "fsck": corrupt_fsck, - "starts_after_partial_consumed": corruption_start > partial_record["consumed"], - "starts_at_stream_byte": corruption_start, - }, - "extent": {**selected, "leading_zero_bytes": leading, "stream_bytes": len(stream)}, - "full": {**full_record, "policy_errno": 0, "output_sha256": sha256_bytes(full_output)}, - "image_repeated_sha256": hashes, - "legal_fsck": valid_fsck, - "partial": { - **partial_record, - "corrupt_policy_errno": 0, - "output_matches_full_slice": True, - "policy_errno": 0, - "requested_bytes": partial_size, - }, - "tail": { - **tail_record, - "bytes": len(tail), - "fsck": tail_fsck, - "nonzero": True, - "policy_errno": tail_errno, - }, - "truncated": {**truncated_record, "fsck": truncated_fsck, "policy_errno": truncated_errno}, - } - - -def finalize() -> None: - lines = [] - for path in sorted(OUTPUT.rglob("*")): - if path.is_file() and path.name != "SHA256SUMS": - lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}") - (OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") - - -result: dict[str, Any] | None = None -exit_code = 0 -owned_temp: str | None = None -try: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-083" or SPEC.get("gate") != "G04": - raise GateFailure("INFRA_BLOCKED", "invalid P15-083 input schema") - resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise GateFailure("INFRA_BLOCKED", f"P15-083 must replay {SPEC['required_base']}, got {resolved}") - sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]} - hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()} - if hashes != SPEC["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed") - write_json(OUTPUT / "source-sha256.json", hashes) - freebsd_head = subprocess.check_output(["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True).strip() - freebsd_hashes = {path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"]} - if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]: - raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed") - errno_source = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8") - if "#define\tEINTEGRITY\t97" not in errno_source: - raise GateFailure("INFRA_BLOCKED", "FreeBSD positive EINTEGRITY changed") - write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes}) - write_json(OUTPUT / "toolchain.json", verify_tools()) - write_json(OUTPUT / "source-contract.json", verify_source_contract(sources)) - with tempfile.TemporaryDirectory(prefix="p15-083-g04-") as temporary: - owned_temp = temporary - temp = Path(temporary) - source_dir = temp / "source" - source = make_source(source_dir) - oracle, library_record = compile_oracle(temp) - write_json(OUTPUT / "independent-libraries.json", library_record) - records = [ - evaluate_codec(codec, codec_spec, source, source_dir, oracle, temp) - for codec, codec_spec in sorted(SPEC["codecs"].items()) - ] - if {record["codec"] for record in records} != {"deflate", "lzma", "zstd"}: - raise GateFailure("STOP", "P15-083 requires all three non-LZ4 codec policies") - write_json(OUTPUT / "codec-results.json", records) - result = { - "b27": "AUTHORIZED", - "candidate": "P15-083", - "cleanup": "PASS", - "codecs": {record["codec"]: "GO" for record in records}, - "full_feature_suite": "NOT_RUN", - "gate": "G04", - "oracle": "real erofs-utils 1.8.6 images + liblzma/zlib/libzstd consumed-byte oracle", - "policy": { - "deflate": "strip EROFS leading zero padding; full raw stream must reach Z_STREAM_END with no unread bytes", - "lzma": "strip EROFS leading zero padding; MicroLZMA compressed size is exact and all bytes must be consumed", - "zstd": "strip EROFS leading zero padding; one frame must complete with no unread bytes", - }, - "qemu": "NOT_RUN", - "qemu_reason": "Stage0 policy and compatibility oracle is complete on real host fixtures; B27 acceptance owns TC176 QEMU", - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "schema": 1, - "status": "GO", - "typed_errno": "PASS", - } - write_json(OUTPUT / "result.json", result) - cleanup_record = { - "owned_temp": owned_temp, - "owned_temp_removed": owned_temp is not None and not Path(owned_temp).exists(), - "protected_pid_touched": False, - "protected_port_touched": False, - "qemu_started": False, - "shared_base_image_touched": False, - } - if not cleanup_record["owned_temp_removed"]: - raise GateFailure("INFRA_BLOCKED", "owned gate temporary directory survived cleanup") - write_json(OUTPUT / "owned-cleanup.json", cleanup_record) -except GateFailure as failure: - result = { - "b27": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN", - "candidate": "P15-083", - "full_feature_suite": "NOT_RUN", - "gate": "G04", - "qemu": "NOT_RUN", - "reason": failure.reason, - "requested_base": REQUESTED_BASE, - "schema": 1, - "status": failure.status, - } - write_json(OUTPUT / "result.json", result) - exit_code = 1 if failure.status == "STOP" else 21 -except (OSError, subprocess.SubprocessError, ValueError) as failure: - result = { - "b27": "NOT_RUN", - "candidate": "P15-083", - "full_feature_suite": "NOT_RUN", - "gate": "G04", - "qemu": "NOT_RUN", - "reason": f"gate infrastructure failure: {failure}", - "requested_base": REQUESTED_BASE, - "schema": 1, - "status": "INFRA_BLOCKED", - } - write_json(OUTPUT / "result.json", result) - exit_code = 21 -finally: - finalize() - -print(json.dumps(result, sort_keys=True)) -raise SystemExit(exit_code) -PY diff --git a/tests/pre15/gates/P15-086-input.json b/tests/pre15/gates/P15-086-input.json deleted file mode 100644 index beec643..0000000 --- a/tests/pre15/gates/P15-086-input.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "candidate": "P15-086", - "cpu": { - "iterations_per_sample": 25, - "samples": 9 - }, - "freebsd": { - "head": "106727738dcfb6c001b46f25363b91cece970085", - "sha256": { - "sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c" - } - }, - "gates": [ - "G04", - "G05" - ], - "hard_budget": { - "caller_bytes": 1048576, - "decoded_bytes": 12582912, - "input_bytes": 1048576, - "workspace_bytes": { - "deflate": 524288, - "lz4": 0, - "lzma": 9437184, - "zstd": 4194304 - } - }, - "libraries": { - "liblz4": "1.10.0", - "liblzma": "5.8.1", - "libzstd": "1.5.7", - "zlib": "1.3.1" - }, - "lz4": { - "generator_url": "https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git", - "generators": [ - { - "commit": "ee97fe5fb77c737df0f77d92ab0d92edd3a11be6", - "name": "v1.4" - }, - { - "commit": "ce36273833096f1b7e828309d9b1caa37132092d", - "name": "v1.7" - }, - { - "commit": "3689cbc2349bff05807d2f939146e92eb1bfaea1", - "name": "v1.8.6" - } - ], - "mkfs_args": [ - "-T0", - "-U86000000-0000-4000-8000-000000000086", - "--all-root", - "-x-1", - "-zlz4", - "-C4096", - "-Elegacy-compress" - ], - "source": { - "segments": [ - { - "byte": 69, - "length": 262144 - }, - { - "byte": 82, - "length": 262144 - }, - { - "byte": 79, - "length": 262144 - }, - { - "byte": 70, - "length": 262144 - } - ], - "sha256": "b4802e36692c8124aa80813711cf03d5cec47729ecd25acf5cc4fd53c484ea80", - "size": 1048576 - } - }, - "ranges": [ - { - "length": 4096, - "name": "prefix", - "offset": 0 - }, - { - "length": 4096, - "name": "cross-page", - "offset": 3584 - }, - { - "length": 4096, - "name": "middle", - "offset": 8192 - }, - { - "length": 4096, - "name": "tail", - "offset": -4096 - } - ], - "required_base": "6bf5724619be70f805bbe7d1ba77dd70cdced69f", - "schema": 1, - "source_sha256": { - "repo-pre-15/src/compress.h": "41dba5f7bf0c52f72ae26ea5e6deff66361ec7ab38403a65eba12035183edb68", - "repo-pre-15/src/decompressor.c": "76289dcb494994f77c1ba6ff10bbbfe55e7b4baa6a1724c35af14ae1611c537c", - "repo-pre-15/src/decompressor_deflate.c": "66080ad9f9534e74692c6fa41b6649dd12dd060217363eb9697ac94deb4604f9", - "repo-pre-15/src/decompressor_lz4.c": "ca5e5bd6142f9e3c3ea66849900f05799dbc7d6d1a97e0d5a988c28edb90ea98", - "repo-pre-15/src/decompressor_lzma.c": "f515593b9fcc7627bebb93bcdc1531a13f4d91c70faec34ffcf7a54a8a279aa0", - "repo-pre-15/src/decompressor_zstd.c": "fd14509e3a29629d0a85a67306cea9e351665b28d1dbd510b68be859bc071ba1", - "repo-pre-15/src/zdata.c": "3eeb5dae825d7028793a2e1d19a80d24d3f78cdc2e2d5e19466d8fb1208f3242", - "src-linux/compress.h": "d7a76de0d0a43b2369635e2ad6143cc0a7c3e5e8c123a48edec41d281c44839c", - "src-linux/decompressor.c": "caf1c501d00a5a2c9cda5fc0b59d2823eaedf0161a130ca69cd5e7c455128709", - "src-linux/decompressor_deflate.c": "0ddb56c27cacc63aa9f7a10a55cfd76ee2ba0bab15cbac7f6d8684203fb2c67b", - "src-linux/decompressor_lzma.c": "5a23b4455767c30c8d90e3e5b10bc3c430d94359727affae2e6f7824f106d5a8", - "src-linux/decompressor_zstd.c": "4f8a4961fa6b4219fcfe91c04ba37179ec4dde296fcf9f91a5d5b27d8c272607", - "src-linux/zdata.c": "358869da60dcdafd13bc1ff0cd28e864c13b46a49d9f556d9b37139f7e0f4e79" - }, - "stream": { - "codecs": { - "deflate": { - "dict_size": 32768, - "mkfs_args": [ - "-T0", - "-U86100000-0000-4000-8000-000000000086", - "--all-root", - "-x-1", - "--workers=1", - "-zdeflate,level=1,dictsize=32768", - "-C4096" - ] - }, - "lzma": { - "dict_size": 65536, - "mkfs_args": [ - "-T0", - "-U86200000-0000-4000-8000-000000000086", - "--all-root", - "-x-1", - "--workers=1", - "-zlzma,level=6,dictsize=65536", - "-C4096" - ] - }, - "zstd": { - "dict_size": 65536, - "mkfs_args": [ - "-T0", - "-U86300000-0000-4000-8000-000000000086", - "--all-root", - "-x-1", - "--workers=1", - "-zzstd,level=3,dictsize=65536", - "-C4096" - ] - } - }, - "source": { - "line_count": 4096, - "sha256": "3c6a1cd405abc8b67b0100dbceebd0f918836563c7a5f705b9cfc178b35fca28", - "size": 151552 - } - }, - "thresholds": { - "minimum_peak_temporary_reduction_percent": 20.0 - }, - "tools": { - "dump.erofs": { - "path": "/usr/bin/dump.erofs", - "sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd" - }, - "fsck.erofs": { - "path": "/usr/bin/fsck.erofs", - "sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29" - }, - "liblz4": { - "path": "/lib/x86_64-linux-gnu/liblz4.so.1", - "sha256": "67bf8b84af77e962e09be661944f7c61393c4806746734e36db9fbdaebc1de36" - }, - "mkfs.erofs": { - "path": "/usr/bin/mkfs.erofs", - "sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c" - } - } -} diff --git a/tests/pre15/gates/P15-086.sh b/tests/pre15/gates/P15-086.sh deleted file mode 100755 index 32681ed..0000000 --- a/tests/pre15/gates/P15-086.sh +++ /dev/null @@ -1,1445 +0,0 @@ -#!/bin/sh -set -eu - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-086-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; } -for tool in autoreconf cc dump.erofs fsck.erofs git make mkfs.erofs pkg-config python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" - -python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY' -from __future__ import annotations - -import hashlib -import json -import os -from pathlib import Path -import re -import shutil -import statistics -import subprocess -import sys -import tempfile -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="ascii")) -EINTEGRITY = 97 -EOVERFLOW = 84 - - -class GateFailure(Exception): - def __init__(self, status: str, reason: str): - super().__init__(reason) - self.status = status - self.reason = reason - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def git(*args: str) -> str: - return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip() - - -def source_at(commit: str, path: str) -> str: - completed = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if completed.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}") - return completed.stdout - - -def run_logged( - argv: list[str], cwd: Path, log: Path, timeout: int = 120, - expected: set[int] | None = None, -) -> subprocess.CompletedProcess[str]: - try: - completed = subprocess.run( - argv, - cwd=cwd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=timeout, - ) - except subprocess.TimeoutExpired as failure: - log.parent.mkdir(parents=True, exist_ok=True) - log.write_text( - "$ " + " ".join(argv) + f"\n[TIMEOUT after {timeout}s]\n", - encoding="utf-8", - ) - raise GateFailure("INFRA_BLOCKED", f"command timed out: {' '.join(argv)}") from failure - log.parent.mkdir(parents=True, exist_ok=True) - log.write_text( - "$ " + " ".join(argv) + "\n" + completed.stdout + - f"\n[exit {completed.returncode}]\n", - encoding="utf-8", - ) - allowed = {0} if expected is None else expected - if completed.returncode not in allowed: - raise GateFailure( - "INFRA_BLOCKED", - f"command failed ({completed.returncode}): {' '.join(argv)}", - ) - return completed - - -EXTENT_RE = re.compile( - r"^\s*(\d+):\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:\s*" - r"(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$", - re.MULTILINE, -) - - -def parse_extents(text: str) -> list[dict[str, int]]: - records = [] - for match in EXTENT_RE.finditer(text): - index, logical, logical_end, logical_length, physical, physical_end, physical_length = map( - int, match.groups() - ) - if logical_end - logical != logical_length or physical_end - physical != physical_length: - raise GateFailure("INFRA_BLOCKED", "dump.erofs extent arithmetic changed") - records.append( - { - "index": index, - "logical_length": logical_length, - "logical_offset": logical, - "physical_length": physical_length, - "physical_offset": physical, - } - ) - if not records: - raise GateFailure("INFRA_BLOCKED", "dump.erofs returned no extents") - return records - - -def make_stream_source(path: Path) -> bytes: - path.mkdir(parents=True) - content = b"".join( - f"P15-086-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode("ascii") - for index in range(SPEC["stream"]["source"]["line_count"]) - ) - expected = SPEC["stream"]["source"] - if len(content) != expected["size"] or sha256_bytes(content) != expected["sha256"]: - raise GateFailure("INFRA_BLOCKED", "stream source identity changed") - payload = path / "payload.bin" - payload.write_bytes(content) - os.utime(payload, (0, 0)) - os.utime(path, (0, 0)) - return content - - -def make_lz4_source(path: Path) -> bytes: - path.mkdir(parents=True) - content = b"".join( - bytes([segment["byte"]]) * segment["length"] - for segment in SPEC["lz4"]["source"]["segments"] - ) - expected = SPEC["lz4"]["source"] - if len(content) != expected["size"] or sha256_bytes(content) != expected["sha256"]: - raise GateFailure("INFRA_BLOCKED", "LZ4 source identity changed") - payload = path / "payload.bin" - payload.write_bytes(content) - os.utime(payload, (0, 0)) - os.utime(path, (0, 0)) - return content - - -def raw_lz4_decode(data: bytes, target: int) -> tuple[bytes, int]: - ip = 0 - output = bytearray() - while ip < len(data): - token = data[ip] - ip += 1 - literal_length = token >> 4 - if literal_length == 15: - while True: - if ip >= len(data): - raise GateFailure("STOP", "independent LZ4 parser saw truncated literal length") - value = data[ip] - ip += 1 - literal_length += value - if value != 255: - break - if ip + literal_length > len(data): - raise GateFailure("STOP", "independent LZ4 parser saw truncated literals") - output.extend(data[ip : ip + literal_length]) - ip += literal_length - if len(output) >= target: - return bytes(output[:target]), ip - if ip == len(data): - break - if ip + 2 > len(data): - raise GateFailure("STOP", "independent LZ4 parser saw truncated match offset") - offset = data[ip] | data[ip + 1] << 8 - ip += 2 - if offset == 0 or offset > len(output): - raise GateFailure("STOP", "independent LZ4 parser saw invalid match offset") - match_length = token & 15 - if match_length == 15: - while True: - if ip >= len(data): - raise GateFailure("STOP", "independent LZ4 parser saw truncated match length") - value = data[ip] - ip += 1 - match_length += value - if value != 255: - break - for _ in range(match_length + 4): - output.append(output[-offset]) - if len(output) >= target: - return bytes(output[:target]), ip - raise GateFailure("STOP", "independent LZ4 parser ended before requested output") - - -ORACLE_SOURCE = r''' -#define _POSIX_C_SOURCE 200809L -#define ZSTD_STATIC_LINKING_ONLY -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct allocation { - size_t size; -}; - -struct tracker { - size_t current; - size_t peak; -}; - -struct result { - size_t consumed; - size_t produced; - size_t status; - size_t workspace_peak; - int cleanup; - int codec_error; - int stream_end; -}; - -static void -fail(const char *message) -{ - fprintf(stderr, "%s\n", message); - exit(2); -} - -static void * -tracked_alloc(struct tracker *tracker, size_t size) -{ - struct allocation *allocation; - - if (size > SIZE_MAX - sizeof(*allocation)) - return NULL; - allocation = malloc(sizeof(*allocation) + size); - if (allocation == NULL) - return NULL; - allocation->size = size; - tracker->current += size; - if (tracker->current > tracker->peak) - tracker->peak = tracker->current; - return allocation + 1; -} - -static void -tracked_free(struct tracker *tracker, void *address) -{ - struct allocation *allocation; - - if (address == NULL) - return; - allocation = (struct allocation *)address - 1; - if (allocation->size > tracker->current) - fail("allocation tracker underflow"); - tracker->current -= allocation->size; - free(allocation); -} - -static voidpf -zalloc_tracked(voidpf opaque, uInt items, uInt size) -{ - if (items != 0 && size > SIZE_MAX / items) - return NULL; - return tracked_alloc(opaque, (size_t)items * size); -} - -static void -zfree_tracked(voidpf opaque, voidpf address) -{ - tracked_free(opaque, address); -} - -static void * -lzma_alloc_tracked(void *opaque, size_t items, size_t size) -{ - if (items != 0 && size > SIZE_MAX / items) - return NULL; - return tracked_alloc(opaque, items * size); -} - -static void -lzma_free_tracked(void *opaque, void *address) -{ - tracked_free(opaque, address); -} - -static void * -zstd_alloc_tracked(void *opaque, size_t size) -{ - return tracked_alloc(opaque, size); -} - -static void -zstd_free_tracked(void *opaque, void *address) -{ - tracked_free(opaque, address); -} - -static unsigned char * -read_file(const char *path, size_t *sizep) -{ - struct stat st; - unsigned char *data; - ssize_t amount; - size_t done; - int fd; - - fd = open(path, O_RDONLY); - if (fd < 0 || fstat(fd, &st) != 0 || st.st_size <= 0) - fail("cannot open oracle input"); - *sizep = (size_t)st.st_size; - data = malloc(*sizep); - if (data == NULL) - fail("cannot allocate oracle input"); - done = 0; - while (done < *sizep) { - amount = read(fd, data + done, *sizep - done); - if (amount <= 0) - fail("cannot read oracle input"); - done += (size_t)amount; - } - close(fd); - return data; -} - -static void -write_file(const char *path, const unsigned char *data, size_t size) -{ - ssize_t amount; - size_t done; - int fd; - - fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (fd < 0) - fail("cannot create oracle output"); - done = 0; - while (done < size) { - amount = write(fd, data + done, size - done); - if (amount <= 0) - fail("cannot write oracle output"); - done += (size_t)amount; - } - close(fd); -} - -static struct result -decode_once(const char *codec, const unsigned char *input, size_t input_size, - unsigned char *output, size_t output_size, int full, uint32_t dict_size) -{ - struct result result = { 0 }; - struct tracker tracker = { 0 }; - - if (strcmp(codec, "lz4") == 0) { - int ret; - - if (full) - ret = LZ4_decompress_safe((const char *)input, (char *)output, - (int)input_size, (int)output_size); - else - ret = LZ4_decompress_safe_partial((const char *)input, - (char *)output, (int)input_size, (int)output_size, - (int)output_size); - if (ret < 0) { - result.codec_error = 1; - } else { - result.produced = (size_t)ret; - result.stream_end = full && result.produced == output_size; - } - result.consumed = full && !result.codec_error ? input_size : 0; - result.status = ret < 0 ? (size_t)-ret : 0; - result.cleanup = 1; - } else if (strcmp(codec, "deflate") == 0) { - z_stream stream; - int ret = Z_OK, endret; - - memset(&stream, 0, sizeof(stream)); - stream.zalloc = zalloc_tracked; - stream.zfree = zfree_tracked; - stream.opaque = &tracker; - stream.next_in = (Bytef *)(uintptr_t)input; - stream.avail_in = (uInt)input_size; - stream.next_out = output; - stream.avail_out = (uInt)output_size; - ret = inflateInit2(&stream, -15); - if (ret != Z_OK) { - result.codec_error = 1; - } else { - while (stream.avail_out != 0) { - uInt in_before = stream.avail_in; - uInt out_before = stream.avail_out; - ret = inflate(&stream, Z_SYNC_FLUSH); - if (ret == Z_STREAM_END) { - result.stream_end = 1; - break; - } - if (ret != Z_OK || (stream.avail_in == in_before && - stream.avail_out == out_before)) { - result.codec_error = 1; - break; - } - } - result.consumed = input_size - stream.avail_in; - result.produced = output_size - stream.avail_out; - result.status = (size_t)(unsigned int)ret; - endret = inflateEnd(&stream); - result.cleanup = endret == Z_OK; - } - } else if (strcmp(codec, "lzma") == 0) { - lzma_stream stream = LZMA_STREAM_INIT; - lzma_allocator allocator = { - .alloc = lzma_alloc_tracked, - .free = lzma_free_tracked, - .opaque = &tracker, - }; - lzma_ret ret; - - stream.allocator = &allocator; - ret = lzma_microlzma_decoder(&stream, input_size, output_size, - full != 0, dict_size); - if (ret != LZMA_OK) { - result.codec_error = 1; - } else { - stream.next_in = input; - stream.avail_in = input_size; - stream.next_out = output; - stream.avail_out = output_size; - while (stream.avail_out != 0) { - size_t in_before = stream.avail_in; - size_t out_before = stream.avail_out; - ret = lzma_code(&stream, LZMA_RUN); - if (ret == LZMA_STREAM_END) { - result.stream_end = 1; - break; - } - if (ret != LZMA_OK || (stream.avail_in == in_before && - stream.avail_out == out_before)) { - result.codec_error = 1; - break; - } - } - result.consumed = input_size - stream.avail_in; - result.produced = output_size - stream.avail_out; - result.status = ret; - } - lzma_end(&stream); - result.cleanup = tracker.current == 0; - } else if (strcmp(codec, "zstd") == 0) { - ZSTD_customMem memory = { - .customAlloc = zstd_alloc_tracked, - .customFree = zstd_free_tracked, - .opaque = &tracker, - }; - ZSTD_DCtx *context; - ZSTD_inBuffer in_buffer; - ZSTD_outBuffer out_buffer; - size_t ret = 1; - - context = ZSTD_createDCtx_advanced(memory); - if (context == NULL) { - result.codec_error = 1; - } else { - in_buffer = (ZSTD_inBuffer){ input, input_size, 0 }; - out_buffer = (ZSTD_outBuffer){ output, output_size, 0 }; - while (out_buffer.pos != out_buffer.size) { - size_t in_before = in_buffer.pos; - size_t out_before = out_buffer.pos; - ret = ZSTD_decompressStream(context, &out_buffer, &in_buffer); - if (ZSTD_isError(ret)) { - result.codec_error = 1; - break; - } - if (ret == 0) { - result.stream_end = 1; - break; - } - if (in_buffer.pos == in_before && out_buffer.pos == out_before) { - result.codec_error = 1; - break; - } - } - result.consumed = in_buffer.pos; - result.produced = out_buffer.pos; - result.status = ret; - if (ZSTD_isError(ZSTD_freeDCtx(context))) - result.cleanup = 0; - else - result.cleanup = tracker.current == 0; - } - } else { - fail("unknown codec"); - } - result.workspace_peak = tracker.peak; - return result; -} - -static uint64_t -elapsed_ns(const struct timespec *start, const struct timespec *end) -{ - return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL + - (uint64_t)(end->tv_nsec - start->tv_nsec); -} - -int -main(int argc, char **argv) -{ - unsigned char *input, *mapping, *output; - struct result result = { 0 }, current; - struct timespec start, end; - size_t input_size, output_size, usable, page_size; - uint32_t dict_size; - uint64_t cpu_ns; - int full, guards = 1, iterations; - - if (argc != 8) - fail("usage: oracle CODEC INPUT OUTPUT OUTPUT_SIZE FULL DICT ITERATIONS"); - output_size = (size_t)strtoull(argv[4], NULL, 10); - full = atoi(argv[5]); - dict_size = (uint32_t)strtoul(argv[6], NULL, 10); - iterations = atoi(argv[7]); - if (output_size == 0 || iterations <= 0) - fail("invalid output size or iteration count"); - input = read_file(argv[2], &input_size); - page_size = (size_t)sysconf(_SC_PAGESIZE); - usable = (output_size + page_size - 1) & ~(page_size - 1); - mapping = mmap(NULL, usable + 2 * page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (mapping == MAP_FAILED) - fail("cannot allocate guarded output"); - if (mprotect(mapping, page_size, PROT_NONE) != 0 || - mprotect(mapping + page_size + usable, page_size, PROT_NONE) != 0) - fail("cannot protect output guards"); - output = mapping + page_size; - if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start) != 0) - fail("cannot start CPU clock"); - for (int iteration = 0; iteration < iterations; ++iteration) { - memset(output, 0xa5, usable); - current = decode_once(argv[1], input, input_size, output, output_size, - full, dict_size); - if (iteration == 0) - result = current; - else if (current.codec_error != result.codec_error || - current.cleanup != result.cleanup || - current.consumed != result.consumed || - current.produced != result.produced || - current.stream_end != result.stream_end) - fail("decoder result changed across iterations"); - if (current.workspace_peak > result.workspace_peak) - result.workspace_peak = current.workspace_peak; - } - if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end) != 0) - fail("cannot stop CPU clock"); - cpu_ns = elapsed_ns(&start, &end); - for (size_t index = output_size; index < usable; ++index) { - if (output[index] != 0xa5) { - guards = 0; - break; - } - } - write_file(argv[3], output, result.produced); - printf("cleanup=%d codec_error=%d consumed=%zu cpu_ns=%" PRIu64 - " guards=%d iterations=%d produced=%zu status=%zu stream_end=%d " - "workspace_peak=%zu\n", result.cleanup, result.codec_error, - result.consumed, cpu_ns, guards, iterations, result.produced, - result.status, result.stream_end, result.workspace_peak); - munmap(mapping, usable + 2 * page_size); - free(input); - return 0; -} -''' - - -def compile_oracle(temp: Path) -> tuple[Path, dict[str, Any]]: - source = temp / "p15-086-oracle.c" - binary = temp / "p15-086-oracle" - source.write_text(ORACLE_SOURCE, encoding="ascii") - run_logged( - [ - "cc", "-O2", "-std=c17", "-Wall", "-Wextra", "-Werror", - str(source), "-o", str(binary), "-llz4", "-llzma", "-lz", "-lzstd", - ], - temp, - OUTPUT / "logs/oracle-build.log", - ) - versions = { - name: subprocess.check_output(["pkg-config", "--modversion", package], text=True).strip() - for name, package in ( - ("liblz4", "liblz4"), - ("liblzma", "liblzma"), - ("zlib", "zlib"), - ("libzstd", "libzstd"), - ) - } - if versions != SPEC["libraries"]: - raise GateFailure("INFRA_BLOCKED", f"independent library versions changed: {versions}") - return binary, { - "binary_sha256": sha256_path(binary), - "source_sha256": sha256_path(source), - **versions, - } - - -def decode( - oracle: Path, - codec: str, - data: bytes, - output_size: int, - full: bool, - dict_size: int, - iterations: int, - temp: Path, - label: str, -) -> tuple[dict[str, int], bytes]: - input_path = temp / f"{label}.input" - output_path = temp / f"{label}.output" - input_path.write_bytes(data) - completed = run_logged( - [ - str(oracle), codec, str(input_path), str(output_path), str(output_size), - "1" if full else "0", str(dict_size), str(iterations), - ], - temp, - OUTPUT / f"logs/oracle-{label}.log", - ) - record: dict[str, int] = {} - for field in completed.stdout.strip().split(): - key, value = field.split("=", 1) - record[key] = int(value) - required = { - "cleanup", "codec_error", "consumed", "cpu_ns", "guards", "iterations", - "produced", "status", "stream_end", "workspace_peak", - } - if set(record) != required: - raise GateFailure("INFRA_BLOCKED", f"oracle fields changed for {label}") - return record, output_path.read_bytes() - - -def full_errno(record: dict[str, int], data_size: int, expected: bytes, output: bytes) -> int: - if ( - record["codec_error"] != 0 - or record["cleanup"] != 1 - or record["guards"] != 1 - or record["consumed"] != data_size - or record["produced"] != len(expected) - or record["stream_end"] != 1 - or output != expected - ): - return EINTEGRITY - return 0 - - -def partial_errno(record: dict[str, int], expected: bytes, output: bytes) -> int: - if ( - record["codec_error"] != 0 - or record["cleanup"] != 1 - or record["guards"] != 1 - or record["produced"] != len(expected) - or output != expected - ): - return EINTEGRITY - return 0 - - -def fsck_extract(fsck: Path, image: Path, destination: Path, label: str) -> dict[str, Any]: - destination.mkdir() - completed = run_logged( - [str(fsck), f"--extract={destination}", str(image)], - image.parent, - OUTPUT / f"logs/fsck-{label}.log", - expected=set(range(0, 256)), - ) - return {"exit": completed.returncode, "success": completed.returncode == 0} - - -def select_extent( - dump: Path, image: Path, payload_path: str, minimum: int, label: str, -) -> dict[str, int]: - completed = run_logged( - [str(dump), f"--path={payload_path}", "-e", str(image)], - image.parent, - OUTPUT / f"logs/dump-{label}.log", - ) - choices = [ - record for record in parse_extents(completed.stdout) - if record["logical_length"] >= minimum and record["physical_length"] > 0 - ] - if not choices: - raise GateFailure("STOP", f"{label} has no representative compressed extent") - return max(choices, key=lambda record: record["logical_length"]) - - -def dump_extents_compat(dump: Path, image: Path, label: str) -> tuple[list[dict[str, int]], dict[str, Any]]: - attempts = [] - modern = subprocess.run( - [str(dump), "--path=/payload.bin", "-e", str(image)], - cwd=image.parent, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=30, - ) - attempts.append( - "$ " + " ".join([str(dump), "--path=/payload.bin", "-e", str(image)]) - + "\n" + modern.stdout + f"\n[exit {modern.returncode}]\n" - ) - if modern.returncode == 0: - records = parse_extents(modern.stdout) - (OUTPUT / f"logs/dump-{label}.log").write_text("\n".join(attempts), encoding="utf-8") - return records, {"mode": "path", "nid": None} - matches = [] - for nid in range(1, 256): - completed = subprocess.run( - [str(dump), f"--nid={nid}", "-e", str(image)], - cwd=image.parent, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=5, - ) - if completed.returncode == 0 and re.search(r"^(?:File|Path) : /payload\.bin$", completed.stdout, re.MULTILINE): - matches.append((nid, completed.stdout)) - if len(matches) != 1: - (OUTPUT / f"logs/dump-{label}.log").write_text("\n".join(attempts), encoding="utf-8") - raise GateFailure("INFRA_BLOCKED", f"{label} bounded NID lookup found {len(matches)} payloads") - nid, stdout = matches[0] - attempts.append( - "$ " + " ".join([str(dump), f"--nid={nid}", "-e", str(image)]) - + "\n" + stdout + "\n[exit 0]\n" - ) - (OUTPUT / f"logs/dump-{label}.log").write_text("\n".join(attempts), encoding="utf-8") - return parse_extents(stdout), {"mode": "bounded-nid", "nid": nid} - - -def benchmark( - oracle: Path, codec: str, stream: bytes, logical: bytes, dict_size: int, - temp: Path, label: str, -) -> dict[str, Any]: - samples = SPEC["cpu"]["samples"] - iterations = SPEC["cpu"]["iterations_per_sample"] - partial_end = SPEC["ranges"][0]["length"] - full_samples = [] - partial_samples = [] - full_peak = 0 - partial_peak = 0 - for sample in range(samples): - full_record, full_output = decode( - oracle, codec, stream, len(logical), True, dict_size, iterations, - temp, f"{label}-cpu-full-{sample}", - ) - partial_record, partial_output = decode( - oracle, codec, stream, partial_end, False, dict_size, iterations, - temp, f"{label}-cpu-partial-{sample}", - ) - if full_errno(full_record, len(stream), logical, full_output) != 0: - raise GateFailure("STOP", f"{label} full CPU sample lost exactness") - if partial_errno(partial_record, logical[:partial_end], partial_output) != 0: - raise GateFailure("STOP", f"{label} partial CPU sample lost exactness") - full_samples.append(full_record["cpu_ns"] / iterations) - partial_samples.append(partial_record["cpu_ns"] / iterations) - full_peak = max(full_peak, full_record["workspace_peak"]) - partial_peak = max(partial_peak, partial_record["workspace_peak"]) - full_median = statistics.median(full_samples) - partial_median = statistics.median(partial_samples) - return { - "full_ns_per_decode": full_samples, - "full_median_ns": full_median, - "iterations_per_sample": iterations, - "partial_ns_per_decode": partial_samples, - "partial_median_ns": partial_median, - "partial_to_full_percent": partial_median * 100.0 / full_median, - "samples": samples, - "workspace_full_peak": full_peak, - "workspace_partial_peak": partial_peak, - } - - -def evaluate_ranges( - oracle: Path, - codec: str, - stream: bytes, - logical: bytes, - physical_length: int, - dict_size: int, - temp: Path, - label: str, -) -> dict[str, Any]: - full_record, full_output = decode( - oracle, codec, stream, len(logical), True, dict_size, 1, - temp, f"{label}-full", - ) - if full_errno(full_record, len(stream), logical, full_output) != 0: - raise GateFailure("STOP", f"{label} legal full decode is not exact") - ranges = [] - prefix_record: dict[str, int] | None = None - for request in SPEC["ranges"]: - offset = request["offset"] if request["offset"] >= 0 else len(logical) + request["offset"] - if offset < 0 or request["length"] > len(logical) - offset: - raise GateFailure("STOP", f"{label} cannot cover range {request['name']}") - end = offset + request["length"] - use_partial = end < len(logical) - record, output = decode( - oracle, codec, stream, end, not use_partial, dict_size, 1, - temp, f"{label}-{request['name']}", - ) - if codec == "lz4" and use_partial: - parsed_output, parsed_consumed = raw_lz4_decode(stream, end) - if parsed_output != logical[:end]: - raise GateFailure("STOP", f"{label} raw LZ4 range parser differs from full slice") - record["consumed"] = parsed_consumed - errno = ( - partial_errno(record, logical[:end], output) - if use_partial - else full_errno(record, len(stream), logical, output) - ) - if errno != 0 or output[offset:end] != logical[offset:end]: - raise GateFailure("STOP", f"{label} range {request['name']} differs from full slice") - if use_partial and record["consumed"] >= len(stream): - raise GateFailure("STOP", f"{label} range {request['name']} has no measurable input boundary") - if request["name"] == "prefix": - prefix_record = record - ranges.append( - { - "consumed": record["consumed"], - "decoded_prefix": end, - "fallback_full": not use_partial, - "guards": record["guards"], - "length": request["length"], - "name": request["name"], - "offset": offset, - "policy_errno": errno, - "slice_sha256": sha256_bytes(output[offset:end]), - "workspace_peak": record["workspace_peak"], - } - ) - if prefix_record is None: - raise GateFailure("INFRA_BLOCKED", "prefix request disappeared") - corruption_start = max(prefix_record["consumed"] + 8, len(stream) - 64) - if corruption_start >= len(stream): - raise GateFailure("STOP", f"{label} cannot place corruption after partial consumption") - corrupted = bytearray(stream) - original_tail = bytes(corrupted[corruption_start:]) - corrupted[corruption_start:] = b"\0" * (len(corrupted) - corruption_start) - if bytes(corrupted[corruption_start:]) == original_tail: - corrupted[-1] ^= 0x5A - partial_end = SPEC["ranges"][0]["length"] - corrupt_partial_record, corrupt_partial_output = decode( - oracle, codec, bytes(corrupted), partial_end, False, dict_size, 1, - temp, f"{label}-corrupt-partial", - ) - if codec == "lz4": - parsed_output, parsed_consumed = raw_lz4_decode(bytes(corrupted), partial_end) - if parsed_output != logical[:partial_end]: - raise GateFailure("STOP", f"{label} corrupt raw LZ4 prefix differs from full slice") - corrupt_partial_record["consumed"] = parsed_consumed - corrupt_partial_errno = partial_errno( - corrupt_partial_record, logical[:partial_end], corrupt_partial_output - ) - corrupt_full_record, corrupt_full_output = decode( - oracle, codec, bytes(corrupted), len(logical), True, dict_size, 1, - temp, f"{label}-corrupt-full", - ) - corrupt_full_errno = full_errno( - corrupt_full_record, len(corrupted), logical, corrupt_full_output - ) - truncated_record, truncated_output = decode( - oracle, codec, stream[:-1], len(logical), True, dict_size, 1, - temp, f"{label}-truncated", - ) - truncated_errno = full_errno( - truncated_record, len(stream) - 1, logical, truncated_output - ) - if corrupt_partial_errno != 0 or corrupt_full_errno != EINTEGRITY: - raise GateFailure("STOP", f"{label} range-after corruption contract is not exact") - if truncated_errno != EINTEGRITY: - raise GateFailure("STOP", f"{label} one-byte truncation is not positive EINTEGRITY") - cpu = benchmark(oracle, codec, stream, logical, dict_size, temp, label) - prefix_end = SPEC["ranges"][0]["length"] - caller = SPEC["ranges"][0]["length"] - baseline_peak = physical_length + len(logical) + caller + cpu["workspace_full_peak"] - candidate_peak = physical_length + prefix_end + caller + cpu["workspace_partial_peak"] - reduction = (baseline_peak - candidate_peak) * 100.0 / baseline_peak - workspace_budget = SPEC["hard_budget"]["workspace_bytes"][codec] - hard_cap = ( - SPEC["hard_budget"]["input_bytes"] - + SPEC["hard_budget"]["decoded_bytes"] - + SPEC["hard_budget"]["caller_bytes"] - + workspace_budget - ) - if physical_length > SPEC["hard_budget"]["input_bytes"]: - raise GateFailure("STOP", f"{label} input exceeds the fixed pcluster budget") - if len(logical) > SPEC["hard_budget"]["decoded_bytes"]: - raise GateFailure("STOP", f"{label} output exceeds the fixed decoded budget") - if max(cpu["workspace_full_peak"], cpu["workspace_partial_peak"]) > workspace_budget: - raise GateFailure("STOP", f"{label} workspace exceeds its hard budget") - if max(baseline_peak, candidate_peak) > hard_cap: - raise GateFailure("STOP", f"{label} peak temporary bytes exceed the hard cap") - threshold = SPEC["thresholds"]["minimum_peak_temporary_reduction_percent"] - fallback_reasons = [] - if reduction < threshold: - fallback_reasons.append( - f"peak temporary reduction {reduction:.3f}% is below {threshold}%" - ) - if cpu["partial_median_ns"] > cpu["full_median_ns"] * 1.25: - fallback_reasons.append("partial CPU median exceeds the 125% hard regression budget") - return { - "budget": { - "baseline_peak_temporary_bytes": baseline_peak, - "candidate_peak_temporary_bytes": candidate_peak, - "hard_cap_bytes": hard_cap, - "peak_reduction_percent": reduction, - "threshold_percent": threshold, - "workspace_budget_bytes": workspace_budget, - }, - "corruption": { - "full_errno": corrupt_full_errno, - "partial_consumed": corrupt_partial_record["consumed"], - "partial_errno": corrupt_partial_errno, - "starts_after_partial_consumed": corruption_start > prefix_record["consumed"], - "starts_at_stream_byte": corruption_start, - "truncated_errno": truncated_errno, - }, - "cpu": cpu, - "full": { - **full_record, - "output_sha256": sha256_bytes(full_output), - "policy_errno": 0, - }, - "ranges": ranges, - "partial_capability": "FULL_FALLBACK" if fallback_reasons else "GO", - "partial_capability_reasons": fallback_reasons, - } - - -def build_stream_codecs( - source_dir: Path, source: bytes, oracle: Path, temp: Path, -) -> list[dict[str, Any]]: - mkfs = Path(SPEC["tools"]["mkfs.erofs"]["path"]) - fsck = Path(SPEC["tools"]["fsck.erofs"]["path"]) - dump = Path(SPEC["tools"]["dump.erofs"]["path"]) - records = [] - for codec, codec_spec in sorted(SPEC["stream"]["codecs"].items()): - images = [] - for repeat in ("a", "b"): - image = temp / f"{codec}-{repeat}.erofs" - run_logged( - [str(mkfs), *codec_spec["mkfs_args"], str(image), str(source_dir)], - temp, - OUTPUT / f"logs/mkfs-{codec}-{repeat}.log", - ) - images.append(image) - hashes = [sha256_path(image) for image in images] - if hashes[0] != hashes[1]: - raise GateFailure("INFRA_BLOCKED", f"{codec} image is not byte reproducible") - valid_fsck = fsck_extract(fsck, images[0], temp / f"extract-{codec}", f"{codec}-valid") - if not valid_fsck["success"] or (temp / f"extract-{codec}/payload.bin").read_bytes() != source: - raise GateFailure("INFRA_BLOCKED", f"{codec} legal image extraction mismatch") - extent = select_extent(dump, images[0], "/payload.bin", 12288, codec) - image_bytes = images[0].read_bytes() - start = extent["physical_offset"] - end = start + extent["physical_length"] - block = image_bytes[start:end] - leading = next((index for index, value in enumerate(block) if value), len(block)) - if leading == len(block): - raise GateFailure("STOP", f"{codec} selected pcluster is all padding") - stream = block[leading:] - logical = source[ - extent["logical_offset"] : extent["logical_offset"] + extent["logical_length"] - ] - evaluated = evaluate_ranges( - oracle, codec, stream, logical, extent["physical_length"], - codec_spec["dict_size"], temp, codec, - ) - corrupt_start = evaluated["corruption"]["starts_at_stream_byte"] - corrupted_image = bytearray(image_bytes) - absolute = start + leading + corrupt_start - corrupted_image[absolute:end] = b"\0" * (end - absolute) - corrupt_path = temp / f"{codec}-corrupt.erofs" - corrupt_path.write_bytes(corrupted_image) - corrupt_fsck = fsck_extract( - fsck, corrupt_path, temp / f"extract-{codec}-corrupt", f"{codec}-corrupt" - ) - if corrupt_fsck["success"]: - raise GateFailure("STOP", f"{codec} real EROFS tail corruption escaped full fsck") - records.append( - { - "codec": codec, - "extent": {**extent, "leading_zero_bytes": leading, "stream_bytes": len(stream)}, - "image_repeated_sha256": hashes, - "real_corrupt_fsck": corrupt_fsck, - "real_valid_fsck": valid_fsck, - **evaluated, - } - ) - return records - - -def build_lz4_generations( - source_dir: Path, source: bytes, oracle: Path, temp: Path, -) -> tuple[dict[str, Any], list[dict[str, Any]]]: - clone = temp / "erofs-utils" - run_logged( - ["git", "clone", "--no-checkout", SPEC["lz4"]["generator_url"], str(clone)], - temp, - OUTPUT / "logs/lz4-upstream-clone.log", - timeout=90, - ) - generation_records = [] - evaluated_records = [] - for version in SPEC["lz4"]["generators"]: - name = version["name"] - worktree = temp / f"erofs-{name}" - run_logged( - ["git", "-C", str(clone), "worktree", "add", "--detach", str(worktree), version["commit"]], - temp, - OUTPUT / f"logs/lz4-{name}-worktree.log", - ) - resolved = subprocess.check_output(["git", "-C", str(worktree), "rev-parse", "HEAD"], text=True).strip() - if resolved != version["commit"]: - raise GateFailure("INFRA_BLOCKED", f"LZ4 generator identity changed for {name}") - run_logged(["./autogen.sh"], worktree, OUTPUT / f"logs/lz4-{name}-autogen.log") - run_logged(["./configure", "--disable-fuse"], worktree, OUTPUT / f"logs/lz4-{name}-configure.log") - run_logged(["make", "-s", "-j2"], worktree, OUTPUT / f"logs/lz4-{name}-make.log") - mkfs = worktree / "mkfs/mkfs.erofs" - fsck = worktree / "fsck/fsck.erofs" - dump = worktree / "dump/dump.erofs" - images = [] - for repeat in ("a", "b"): - image = temp / f"lz4-{name}-{repeat}.erofs" - run_logged( - [str(mkfs), *SPEC["lz4"]["mkfs_args"], str(image), str(source_dir)], - worktree, - OUTPUT / f"logs/lz4-{name}-mkfs-{repeat}.log", - ) - images.append(image) - hashes = [sha256_path(image) for image in images] - if hashes[0] != hashes[1]: - raise GateFailure("INFRA_BLOCKED", f"LZ4 {name} image is not byte reproducible") - extract = temp / f"extract-lz4-{name}" - extract.mkdir() - run_logged( - [str(fsck), "--extract", str(images[0])], - extract, - OUTPUT / f"logs/lz4-{name}-fsck.log", - ) - extents, dump_identity = dump_extents_compat(dump, images[0], f"lz4-{name}") - choices = [record for record in extents if record["logical_length"] >= 12288] - if not choices: - raise GateFailure("STOP", f"LZ4 {name} has no representative compressed extent") - extent = max(choices, key=lambda record: record["logical_length"]) - image_bytes = images[0].read_bytes() - reconstructed = bytearray(len(source)) - for current in extents: - current_block = image_bytes[ - current["physical_offset"] : - current["physical_offset"] + current["physical_length"] - ] - current_logical, _ = raw_lz4_decode(current_block, current["logical_length"]) - logical_start = current["logical_offset"] - logical_end = logical_start + current["logical_length"] - if current_logical != source[logical_start:logical_end]: - raise GateFailure("STOP", f"LZ4 {name} extent differs from source") - reconstructed[logical_start:logical_end] = current_logical - if bytes(reconstructed) != source: - raise GateFailure("STOP", f"LZ4 {name} full reconstruction differs from source") - block = image_bytes[ - extent["physical_offset"] : extent["physical_offset"] + extent["physical_length"] - ] - logical = source[ - extent["logical_offset"] : extent["logical_offset"] + extent["logical_length"] - ] - parsed, consumed = raw_lz4_decode(block, len(logical)) - if parsed != logical: - raise GateFailure("STOP", f"LZ4 {name} independent parser differs from source") - stream = block[:consumed] - evaluated = evaluate_ranges( - oracle, "lz4", stream, logical, extent["physical_length"], 0, - temp, f"lz4-{name}", - ) - generation_records.append( - { - "commit": resolved, - "extent": {**extent, "stream_bytes": consumed}, - "dump_identity": dump_identity, - "image_repeated_sha256": hashes, - "name": name, - } - ) - evaluated_records.append({"generator": name, **evaluated}) - representative = evaluated_records[-1] - capability = ( - "GO" - if all(record["partial_capability"] == "GO" for record in evaluated_records) - else "FULL_FALLBACK" - ) - lz4_record = { - "codec": "lz4", - "generations": generation_records, - "partial_capability": capability, - "representative": representative, - } - return lz4_record, evaluated_records - - -def verify_tools() -> dict[str, Any]: - records = {} - for name, expected in SPEC["tools"].items(): - path = Path(expected["path"]) - if not path.is_file() or sha256_path(path) != expected["sha256"]: - raise GateFailure("INFRA_BLOCKED", f"tool identity changed: {name}") - records[name] = expected - mkfs_version = subprocess.check_output( - [SPEC["tools"]["mkfs.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True - ).strip() - fsck_version = subprocess.check_output( - [SPEC["tools"]["fsck.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True - ).strip() - for codec in ("lz4", "lzma", "deflate", "zstd"): - if codec not in mkfs_version or codec not in fsck_version: - raise GateFailure("STOP", f"{codec} lacks a real mkfs/fsck path") - records["mkfs_version"] = mkfs_version - records["fsck_version"] = fsck_version - return records - - -def verify_source_contract(sources: dict[str, str]) -> dict[str, Any]: - req = sources["repo-pre-15/src/compress.h"] - if "size_t outputsize;" not in req or "bool partial_decoding;" not in req: - raise GateFailure("INFRA_BLOCKED", "FreeBSD partial request contract changed") - anchors = { - "lz4": "rq->partial_decoding", - "lzma": "!rq->partial_decoding", - "deflate": "if (rq->partial_decoding)", - "zstd": "if (rq->partial_decoding)", - } - for codec, anchor in anchors.items(): - if anchor not in sources[f"repo-pre-15/src/decompressor_{codec}.c"]: - raise GateFailure("INFRA_BLOCKED", f"FreeBSD {codec} partial primitive changed") - dispatch = sources["repo-pre-15/src/decompressor.c"] - if ".outputsize = dstlen" not in dispatch or ".partial_decoding = partial" not in dispatch: - raise GateFailure("INFRA_BLOCKED", "FreeBSD dispatch partial propagation changed") - zdata = sources["repo-pre-15/src/zdata.c"] - required = ( - "partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0;", - "if (!partial && decoded_len != map->m_llen)", - "erofs_put_metabuf(&buf);", - "erofs_brelse(compressed);", - "free(decoded, M_EROFS);", - "*bufp = decoded;", - ) - if not all(anchor in zdata for anchor in required): - raise GateFailure("INFRA_BLOCKED", "FreeBSD buffer/partial-ref contract changed") - decode_at = zdata.index("error = z_erofs_decompress") - release_at = min( - zdata.index("erofs_put_metabuf(&buf);", decode_at), - zdata.index("erofs_brelse(compressed);", decode_at), - ) - error_at = zdata.index("if (error != 0) {", release_at) - free_at = zdata.index("free(decoded, M_EROFS);", error_at) - publish_at = zdata.index("*bufp = decoded;", free_at) - if not decode_at < release_at < error_at < free_at < publish_at: - raise GateFailure("INFRA_BLOCKED", "FreeBSD decode cleanup ordering changed") - linux_req = sources["src-linux/compress.h"] - if "partial_decoding" not in linux_req or "outputsize" not in linux_req: - raise GateFailure("INFRA_BLOCKED", "Linux partial request anchor changed") - linux_anchors = { - "lz4": "LZ4_decompress_safe_partial", - "lzma": "!rq->partial_decoding", - "deflate": "rq->partial_decoding", - "zstd": "rq->outputsize + dctx.avail_out", - } - for codec, anchor in linux_anchors.items(): - path = "src-linux/decompressor.c" if codec == "lz4" else f"src-linux/decompressor_{codec}.c" - if anchor not in sources[path]: - raise GateFailure("INFRA_BLOCKED", f"Linux {codec} partial anchor changed") - return { - "backend_partial_primitives": sorted(anchors), - "buffer_release_after_decode": True, - "failed_output_freed": True, - "linux_prefix_contract": True, - "partial_ref_preserved": True, - "successful_output_published_only_after_decode": True, - } - - -def build_state_model() -> dict[str, Any]: - scenarios = [ - {"name": "cache-hit", "decode": "none", "cache_after": "ready-full"}, - {"name": "cache-miss-partial-success", "decode": "bounded-prefix", "cache_after": "absent"}, - {"name": "cache-miss-partial-failure", "decode": "bounded-prefix", "cache_after": "absent", "errno": EINTEGRITY}, - {"name": "concurrent-partial-miss", "decode": "per-request-local", "shared_owner_count": 0, "shared_waiter_count": 0}, - {"name": "full-request", "decode": "full", "cache_after": "existing-policy"}, - {"name": "unsupported-backend", "decode": "full-fallback", "cache_after": "existing-policy"}, - {"name": "partial-reference", "decode": "bounded-prefix", "cache_after": "ineligible"}, - {"name": "eviction", "partial_reference_retained": False, "cache_after": "existing-policy"}, - {"name": "reclaim", "partial_reference_retained": False, "cache_after": "existing-policy"}, - {"name": "unmount", "partial_reference_retained": False, "drain_required": False}, - {"name": "key-reuse", "partial_reference_retained": False, "cache_after": "existing-policy"}, - ] - prototype = { - "probed_codecs": ["lz4", "lzma", "deflate", "zstd"], - "fallback_backends": ["shifted", "interlaced", "unknown"], - "new_persistent_state": False, - "overflow_errno": EOVERFLOW, - "pool_exhaustion": "not applicable; B28 adds no pool", - "rule": "cache hit first; strict subextent on a gate-authorized codec uses mapoff+want bytes and never publishes a partial cache entry; all other cases use exact full decode", - "scenarios": scenarios, - } - write_json(OUTPUT / "candidate-state-model.json", prototype) - prototype_source = """P15-086 test-only candidate model\n\ncapable = lz4|lzma|deflate|zstd\nrequest_end = checked_add(mapoff, want)\nif cache_hit: copy full cache\nelif capable and request_end < m_llen: decode(request_end, partial=true), no cache publish\nelse: decode(m_llen, partial=partial_ref), preserve existing full cache policy\nfailed local output is freed; input is released after callback; no persistent state is added\n""" - prototype_path = OUTPUT / "candidate-prototype.txt" - prototype_path.write_text(prototype_source, encoding="ascii") - return { - "prototype_sha256": sha256_path(prototype_path), - "scenario_count": len(scenarios), - "state_machine": "PASS", - } - - -def finalize() -> None: - entries = [] - for path in sorted(OUTPUT.rglob("*")): - if path.is_file() and path.name != "SHA256SUMS": - entries.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}") - (OUTPUT / "SHA256SUMS").write_text("\n".join(entries) + "\n", encoding="ascii") - - -result: dict[str, Any] = {} -exit_code = 0 -owned_temp: str | None = None -try: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-086" or SPEC.get("gates") != ["G04", "G05"]: - raise GateFailure("INFRA_BLOCKED", "gate input identity changed") - resolved = git("rev-parse", REQUESTED_BASE) - if resolved != SPEC["required_base"]: - raise GateFailure("INFRA_BLOCKED", f"wrong DUT base: {resolved}") - sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]} - source_hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()} - if source_hashes != SPEC["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed") - write_json(OUTPUT / "source-sha256.json", source_hashes) - freebsd_head = subprocess.check_output( - ["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True - ).strip() - freebsd_hashes = { - path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"] - } - if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]: - raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed") - errno_source = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8") - if "#define\tEINTEGRITY\t97" not in errno_source or "#define\tEOVERFLOW\t84" not in errno_source: - raise GateFailure("INFRA_BLOCKED", "positive FreeBSD errno identity changed") - write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes}) - write_json(OUTPUT / "toolchain.json", verify_tools()) - write_json(OUTPUT / "source-contract.json", verify_source_contract(sources)) - write_json(OUTPUT / "state-model-result.json", build_state_model()) - with tempfile.TemporaryDirectory(prefix="p15-086-g04-g05-") as temporary: - owned_temp = temporary - temp = Path(temporary) - oracle, oracle_identity = compile_oracle(temp) - write_json(OUTPUT / "independent-oracle.json", oracle_identity) - stream_dir = temp / "stream-source" - stream_source = make_stream_source(stream_dir) - stream_records = build_stream_codecs(stream_dir, stream_source, oracle, temp) - lz4_dir = temp / "lz4-source" - lz4_source = make_lz4_source(lz4_dir) - lz4_record, lz4_generation_results = build_lz4_generations( - lz4_dir, lz4_source, oracle, temp - ) - codec_records = [lz4_record, *stream_records] - write_json(OUTPUT / "codec-results.json", codec_records) - write_json(OUTPUT / "lz4-generation-results.json", lz4_generation_results) - per_codec = { - record["codec"]: ( - record["partial_capability"] - if record["codec"] == "lz4" - else record["partial_capability"] - ) - for record in codec_records - } - if set(per_codec) != {"lz4", "lzma", "deflate", "zstd"}: - raise GateFailure("STOP", "the real supported codec set is incomplete") - enabled = sorted(codec for codec, decision in per_codec.items() if decision == "GO") - if not enabled: - raise GateFailure("STOP", "no codec meets the P15-086 quantified benefit gate") - write_json( - OUTPUT / "authorized-capabilities.json", - { - "full_fallback": sorted( - codec for codec, decision in per_codec.items() if decision == "FULL_FALLBACK" - ), - "partial": enabled, - "persistent_state_added": False, - "schema": 1, - }, - ) - result = { - "b28": "AUTHORIZED", - "candidate": "P15-086", - "cleanup": "PASS", - "codecs": per_codec, - "cpu_samples_per_mode": SPEC["cpu"]["samples"], - "full_feature_suite": "NOT_RUN", - "g04": "GO", - "g05": "GO", - "gates": ["G04", "G05"], - "hard_budget": "PASS", - "partial_enabled_codecs": enabled, - "oracle": "real EROFS images plus independent liblz4/liblzma/zlib/libzstd consumed-byte oracle", - "qemu": "NOT_RUN", - "qemu_reason": "Stage0 host gate owns correctness and budget authorization; B28 acceptance owns strict TC176 QEMU", - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "schema": 1, - "status": "GO", - "typed_errno": "PASS", - } - write_json(OUTPUT / "result.json", result) -except GateFailure as failure: - result = { - "b28": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN", - "candidate": "P15-086", - "full_feature_suite": "NOT_RUN", - "gates": ["G04", "G05"], - "qemu": "NOT_RUN", - "reason": failure.reason, - "requested_base": REQUESTED_BASE, - "schema": 1, - "status": failure.status, - } - write_json(OUTPUT / "result.json", result) - exit_code = 1 if failure.status == "STOP" else 21 -except (OSError, subprocess.SubprocessError, ValueError) as failure: - result = { - "b28": "NOT_RUN", - "candidate": "P15-086", - "full_feature_suite": "NOT_RUN", - "gates": ["G04", "G05"], - "qemu": "NOT_RUN", - "reason": f"gate infrastructure failure: {failure}", - "requested_base": REQUESTED_BASE, - "schema": 1, - "status": "INFRA_BLOCKED", - } - write_json(OUTPUT / "result.json", result) - exit_code = 21 -finally: - cleanup_record = { - "owned_temp": owned_temp, - "owned_temp_removed": owned_temp is None or not Path(owned_temp).exists(), - "protected_pid_touched": False, - "protected_port_touched": False, - "qemu_started": False, - "shared_base_image_touched": False, - } - if not cleanup_record["owned_temp_removed"]: - result = { - "b28": "NOT_RUN", - "candidate": "P15-086", - "full_feature_suite": "NOT_RUN", - "gates": ["G04", "G05"], - "qemu": "NOT_RUN", - "reason": "owned gate temporary directory survived cleanup", - "requested_base": REQUESTED_BASE, - "schema": 1, - "status": "INFRA_BLOCKED", - } - write_json(OUTPUT / "result.json", result) - exit_code = 21 - write_json(OUTPUT / "owned-cleanup.json", cleanup_record) - finalize() - -print(json.dumps(result, sort_keys=True)) -raise SystemExit(exit_code) -PY diff --git a/tests/pre15/gates/P15-087-input.json b/tests/pre15/gates/P15-087-input.json deleted file mode 100644 index 598ae8f..0000000 --- a/tests/pre15/gates/P15-087-input.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "benchmark": { - "buffer_bytes": 1048576, - "cold_runs": 5, - "entry_count": 18000, - "name_length": 196, - "random_buffer_bytes": 65536, - "run_order": [ - "baseline", - "candidate", - "candidate", - "baseline", - "baseline", - "candidate", - "candidate", - "baseline", - "baseline", - "candidate" - ] - }, - "candidate": "P15-087", - "fixture": { - "minimum_directory_bytes": 3145728, - "uuid": "00000000-0000-0000-0000-000000000087" - }, - "freebsd_head": "106727738dcfb6c001b46f25363b91cece970085", - "freebsd_sha256": { - "sys/geom/geom_vfs.c": "e9ffabf2362e15bf70c08036eff4660a479c3229af2798d922d1bc1fb0117aeb", - "sys/kern/vfs_bio.c": "54c64a4d96cb3a511ab4ac7dad132b4b8e105d2003f37b01ea40c93abce4d5b4", - "sys/sys/buf.h": "d13d4e50fce60c52c760c19380ef01ce14fa555103ab255721dc921cc6aa718b", - "sys/sys/vnode.h": "244d7c51cb75d6c19d07a8b9dab85183f87d05c221bbd760af05839ceb2e4a0d" - }, - "gate": "G11", - "host_inputs": { - "askpass_path": "/work/build/.repo22-ssh-askpass", - "askpass_sha256": "967a879f0175d700df7ee3e40b18a41155a7216c05868d0933f0cf08911b7121", - "base_image": "/work/build/vm-freebsd-build.qcow2.bp", - "base_image_format": "qcow2", - "base_image_size": 13359054848, - "ssh_key_path": "/root/.ssh/id_ed25519", - "ssh_key_sha256": "f76af546955d62f9fe2d52479f314f83ec85c01068011eb59ef894a8ac1f3833" - }, - "mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6", - "protected": { - "pid": 26318, - "port": 9222 - }, - "prototype": { - "max_readahead_bytes": 1048576, - "max_readahead_slots": 256 - }, - "qemu": { - "boot_timeout_seconds": 300, - "cpus": 2, - "guest_timeout_seconds": 840, - "memory_mb": 3072 - }, - "required_base": "c7d692acf9166f5c5e42335db2de64f0793ba8a2", - "schema": 1, - "source_sha256": { - "repo-pre-15/build.sh": "7ea125c6100d1ecc1315b7ef315c32185c3b242805fd96bac1ffd6b9c12dd21e", - "repo-pre-15/src/Makefile": "524df7843a0f03e7002311a977adee53a90ccaccfbe21901da641a3b177dfd47", - "repo-pre-15/src/data.c": "cb22072bd4c092aa6a5376add8c8d9d6f94297f6292900ecc2eef9b01c69e92a", - "repo-pre-15/src/dir.c": "842cc3abd308388fd57f69222f4f6144f2d3942f00d2dcf4e289ad2dd5959cce", - "repo-pre-15/src/internal.h": "eec416077040587ad0756814c4323425889d10bd9b4143fcfc63299bf4f90ae9", - "src-linux/data.c": "8625cdc01e5405f856178ae8fd559696ae85f607f19caf229b867a3b7479318a", - "src-linux/dir.c": "4d0f4e687c5776719bc61454b648ff758acea6dee11f406bf8a38a605f58c763" - }, - "thresholds": { - "maximum_extra_provider_reads_percent": 25.0, - "minimum_cold_median_improvement_percent": 10.0 - } -} diff --git a/tests/pre15/gates/P15-087.sh b/tests/pre15/gates/P15-087.sh deleted file mode 100755 index 8c9b07a..0000000 --- a/tests/pre15/gates/P15-087.sh +++ /dev/null @@ -1,1419 +0,0 @@ -#!/bin/sh -set -eu - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -input=$gate_dir/P15-087-input.json -freebsd_src=${FREEBSD_SRC:-/work/dev-freebsd-releng} -base= -output= - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { - printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2 - exit 2 -} -for tool in cc fsck.erofs git mkfs.erofs python3 qemu-img qemu-system-x86_64 \ - scp sha256sum ssh tar; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required host tool: %s\n' "$tool" >&2 - exit 2 - } -done -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output" - -python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY' -from __future__ import annotations - -import difflib -import hashlib -import json -import os -from pathlib import Path -import re -import shlex -import shutil -import socket -import statistics -import struct -import subprocess -import sys -import tarfile -import tempfile -import time -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -SPEC = json.loads(INPUT.read_text(encoding="ascii")) -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -DT_BY_EROFS = {1: 8, 2: 4, 3: 2, 4: 6, 5: 1, 6: 12, 7: 10} - - -class GateFailure(RuntimeError): - def __init__(self, status: str, reason: str): - super().__init__(reason) - self.status = status - self.reason = reason - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def run(argv: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None, - timeout: int | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: - try: - completed = subprocess.run( - argv, - cwd=cwd, - env=env, - text=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=timeout, - check=False, - ) - except subprocess.TimeoutExpired as error: - raise GateFailure( - "INFRA_BLOCKED", f"command timed out: {' '.join(argv)}" - ) from error - if check and completed.returncode != 0: - raise GateFailure( - "INFRA_BLOCKED", - f"command failed ({completed.returncode}): {' '.join(argv)}\n{completed.stdout}", - ) - return completed - - -def git(*args: str) -> str: - return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip() - - -def source_at(commit: str, path: str) -> str: - completed = run(["git", "-C", str(ROOT), "show", f"{commit}:{path}"], check=False) - if completed.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"cannot read frozen source {path}: {completed.stdout}") - return completed.stdout - - -def replace_once(source: str, before: str, after: str, label: str) -> str: - if source.count(before) != 1: - raise GateFailure("INFRA_BLOCKED", f"candidate anchor changed for {label}") - return source.replace(before, after, 1) - - -def fnv_update(value: int, data: bytes) -> int: - for byte in data: - value ^= byte - value = (value * 1099511628211) & ((1 << 64) - 1) - return value - - -def record_hash(value: int, nid: int, dtype: int, name: bytes, cookie: int) -> int: - value = fnv_update(value, struct.pack(" int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (0x82F63B78 if value & 1 else 0) - return value & 0xFFFFFFFF - - -def promote_root_tail_to_plain(path: Path) -> dict[str, Any]: - data = bytearray(path.read_bytes()) - image = Image(bytes(data)) - inode = image.root_inode() - if inode["layout"] != 2: - raise GateFailure( - "INFRA_BLOCKED", f"expected mkfs flat-inline directory, got {inode['layout']}" - ) - block_size = image.block_size - size = int(inode["size"]) - tail_size = size % block_size - if tail_size == 0: - raise GateFailure("INFRA_BLOCKED", "mkfs flat-inline directory has no tail") - full_blocks = size // block_size - startblk = int(inode["startblk"]) - if (startblk + full_blocks) * block_size != len(data): - raise GateFailure( - "INFRA_BLOCKED", "root full blocks are not the contiguous image suffix" - ) - inode_offset = int(inode["offset"]) - inline_offset = inode_offset + int(inode["inode_size"]) + int(inode["xattr_size"]) - if inline_offset + tail_size > len(data): - raise GateFailure("INFRA_BLOCKED", "root inline tail exceeds the source image") - tail = bytes(data[inline_offset:inline_offset + tail_size]) - data.extend(b"\0" * block_size) - data[-block_size:-block_size + tail_size] = tail - ifmt = struct.unpack_from(" SUPER else block_size - checksum_end = SUPER + checksum_span - struct.pack_into(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" dict[str, int | bool]: - offset = (self.meta_blkaddr << self.block_bits) + (self.root_nid << 5) - ifmt = self.u16(offset) - version = ifmt & 1 - inode_size = 32 if version == 0 else 64 - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + (xattr_count - 1) * 4 - size = self.u32(offset + 8) if version == 0 else self.u64(offset + 8) - return { - "nid": self.root_nid, - "offset": offset, - "layout": (ifmt >> 1) & 7, - "dot_omitted": bool((ifmt >> 4) & 1), - "inode_size": inode_size, - "xattr_size": xattr_size, - "mode": self.u16(offset + 4), - "size": size, - "startblk": self.u32(offset + 16), - } - - def directory_oracle(self) -> dict[str, Any]: - inode = self.root_inode() - if inode["layout"] != 0: - raise GateFailure("STOP", f"root directory is not flat plain: layout={inode['layout']}") - if int(inode["mode"]) & 0o170000 != 0o040000: - raise GateFailure("INFRA_BLOCKED", "root inode is not a directory") - data_offset = int(inode["startblk"]) << self.block_bits - size = int(inode["size"]) - if data_offset + size > len(self.data): - raise GateFailure("INFRA_BLOCKED", "root directory bytes exceed the fixture") - records: list[dict[str, Any]] = [] - previous = b"" - value = 1469598103934665603 - for block_off in range(0, size, self.block_size): - maxsize = min(self.block_size, size - block_off) - base = data_offset + block_off - if maxsize < 12: - raise GateFailure("INFRA_BLOCKED", "short directory block") - first_nameoff = self.u16(base + 8) - if first_nameoff < 12 or first_nameoff % 12 != 0 or first_nameoff >= maxsize: - raise GateFailure("INFRA_BLOCKED", f"bad name offset at block {block_off}") - count = first_nameoff // 12 - for index in range(count): - entry = base + index * 12 - nameoff = self.u16(entry + 8) - endoff = self.u16(entry + 20) if index + 1 < count else maxsize - if not (first_nameoff <= nameoff < endoff <= maxsize): - raise GateFailure("INFRA_BLOCKED", "invalid directory name span") - raw = self.data[base + nameoff:base + endoff] - name = raw.split(b"\0", 1)[0] - if not name or any(raw[len(name):]): - raise GateFailure("INFRA_BLOCKED", "invalid directory name padding") - if previous and previous >= name: - raise GateFailure("INFRA_BLOCKED", "fixture directory is not strictly ordered") - previous = name - nid = self.u64(entry) - erofs_type = self.data[entry + 10] - if erofs_type not in DT_BY_EROFS: - raise GateFailure("INFRA_BLOCKED", "fixture uses an unknown file type") - cookie = block_off + (index + 1) * 12 if index + 1 < count else block_off + maxsize - value = record_hash(value, nid, DT_BY_EROFS[erofs_type], name, cookie) - records.append({ - "cookie": cookie, - "file_type": erofs_type, - "name": name.decode("ascii"), - "nid": nid, - }) - if inode["dot_omitted"]: - cookie = size + 1 - value = record_hash(value, int(inode["nid"]), 4, b".", cookie) - records.append({"cookie": cookie, "file_type": 2, "name": ".", "nid": inode["nid"]}) - random_offset = (size // (2 * self.block_size)) * self.block_size - if random_offset == 0: - raise GateFailure("INFRA_BLOCKED", "fixture is too small for a random seek") - return { - "block_count": (size + self.block_size - 1) // self.block_size, - "block_size": self.block_size, - "data_offset": data_offset, - "directory_bytes": size, - "dot_omitted": inode["dot_omitted"], - "entry_count": len(records), - "final_cookie": records[-1]["cookie"], - "fnv64": f"{value:016x}", - "layout": "flat-plain", - "random_offset": random_offset, - "root_nid": inode["nid"], - "startblk": inode["startblk"], - } - - -def make_candidate(data_source: str, dir_source: str, internal_source: str) -> dict[str, str]: - data_candidate = replace_once( - data_source, - '#include "internal.h"\n', - '#include "internal.h"\n\n#define EROFS_DIR_READAHEAD_BYTES\t(1024 * 1024)\n#define EROFS_DIR_READAHEAD_SLOTS\t256\n', - "data constants", - ) - data_candidate = replace_once( - data_candidate, - "static int\nerofs_bread_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif,\n erofs_blk_t blocks, erofs_off_t off, size_t len, void **bufp)\n", - "static int\nerofs_bread_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif,\n erofs_blk_t blocks, erofs_off_t off, size_t len, daddr_t *rablkno,\n int *rabsize, int racnt, void **bufp)\n", - "bread device signature", - ) - data_candidate = replace_once( - data_candidate, - "\t\terror = bread(dif->devvp, btodb(blkoff), iosize, NOCRED, &bp);\n", - "\t\tif (done == 0 && racnt != 0)\n\t\t\terror = breadn(dif->devvp, btodb(blkoff), iosize,\n\t\t\t rablkno, rabsize, racnt, NOCRED, &bp);\n\t\telse\n\t\t\terror = bread(dif->devvp, btodb(blkoff), iosize, NOCRED, &bp);\n", - "backing vnode breadn", - ) - data_candidate = replace_once( - data_candidate, - "\treturn (erofs_bread_device(sbi, &sbi->dif0, sbi->dif0.blocks, off, len,\n\t bufp));\n", - "\treturn (erofs_bread_device(sbi, &sbi->dif0, sbi->dif0.blocks, off, len,\n\t NULL, NULL, 0, bufp));\n", - "primary bread caller", - ) - data_candidate = replace_once( - data_candidate, - "\treturn (erofs_bread_device(sbi, map.m_dif, blocks, map.m_pa, len, bufp));\n}\n\n/* Release a contiguous buffer returned by erofs_bread(). */\n", - "\treturn (erofs_bread_device(sbi, map.m_dif, blocks, map.m_pa, len,\n\t NULL, NULL, 0, bufp));\n}\n\nstatic int\nerofs_read_physical_readahead(struct erofs_sb_info *sbi,\n unsigned int device_id, erofs_off_t off, size_t len,\n unsigned int rablocks, void **bufp)\n{\n\tstruct erofs_map_dev current, future;\n\tdaddr_t rablkno[EROFS_DIR_READAHEAD_SLOTS];\n\tint rabsize[EROFS_DIR_READAHEAD_SLOTS];\n\terofs_off_t step;\n\terofs_blk_t blocks;\n\tunsigned int count;\n\tint error;\n\n\tcurrent = (struct erofs_map_dev) {\n\t\t.m_pa = off,\n\t\t.m_deviceid = device_id,\n\t\t.m_plen = len,\n\t};\n\terror = erofs_map_dev(sbi, ¤t);\n\tif (error != 0)\n\t\treturn (error);\n\tblocks = current.m_dif->blocks;\n\tif (current.m_dif == &sbi->dif0 && sbi->flatdev)\n\t\tblocks = sbi->flatdev_blocks;\n\trablocks = MIN(rablocks, (unsigned int)nitems(rablkno));\n\tfor (count = 0; count < rablocks; count++) {\n\t\tstep = (erofs_off_t)(count + 1) * sbi->block_size;\n\t\tif (off > UINT64_MAX - step || current.m_pa > UINT64_MAX - step)\n\t\t\tbreak;\n\t\tfuture = (struct erofs_map_dev) {\n\t\t\t.m_pa = off + step,\n\t\t\t.m_deviceid = device_id,\n\t\t\t.m_plen = sbi->block_size,\n\t\t};\n\t\tif (erofs_map_dev(sbi, &future) != 0 ||\n\t\t future.m_dif != current.m_dif ||\n\t\t future.m_pa != current.m_pa + step)\n\t\t\tbreak;\n\t\trablkno[count] = btodb(future.m_pa);\n\t\trabsize[count] = sbi->block_size;\n\t}\n\treturn (erofs_bread_device(sbi, current.m_dif, blocks, current.m_pa, len,\n\t rablkno, rabsize, count, bufp));\n}\n\n/* Release a contiguous buffer returned by erofs_bread(). */\n", - "physical readahead helper", - ) - data_candidate = replace_once( - data_candidate, - "int\nerofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, erofs_off_t loff,\n size_t len, void **bufp)\n", - "static int\nerofs_read_data_impl(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n erofs_off_t loff, size_t len, unsigned int rablocks, void **bufp)\n", - "read data implementation", - ) - data_candidate = replace_once( - data_candidate, - "\t\t\t} else {\n\t\t\t\terror = erofs_read_physical(sbi, map.m_deviceid, map.m_pa,\n\t\t\t\t want, &blk);\n\t\t\t}\n", - "\t\t\t} else if (done == 0 && rablocks != 0 &&\n\t\t\t map.m_flags == EROFS_MAP_MAPPED) {\n\t\t\t\terror = erofs_read_physical_readahead(sbi,\n\t\t\t\t map.m_deviceid, map.m_pa, want, rablocks, &blk);\n\t\t\t} else {\n\t\t\t\terror = erofs_read_physical(sbi, map.m_deviceid, map.m_pa,\n\t\t\t\t want, &blk);\n\t\t\t}\n", - "read data readahead dispatch", - ) - data_candidate = replace_once( - data_candidate, - "\t*bufp = out;\n\treturn (0);\n}\n\n/*\n * Transfer the logical content of an inode directly into a uio.\n", - "\t*bufp = out;\n\treturn (0);\n}\n\nint\nerofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n erofs_off_t loff, size_t len, void **bufp)\n{\n\treturn (erofs_read_data_impl(sbi, vi, loff, len, 0, bufp));\n}\n\nint\nerofs_read_data_readahead(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n erofs_off_t loff, size_t len, bool sequential, void **bufp)\n{\n\tuint64_t remaining;\n\tunsigned int rablocks;\n\n\trablocks = 0;\n\tif (sequential && vi->datalayout == EROFS_INODE_FLAT_PLAIN &&\n\t sbi->block_size != 0 && (loff & (sbi->block_size - 1)) == 0 &&\n\t len <= sbi->block_size && loff <= vi->size && len <= vi->size - loff) {\n\t\tremaining = vi->size - loff - len;\n\t\trablocks = MIN(howmany(remaining, sbi->block_size),\n\t\t (uint64_t)EROFS_DIR_READAHEAD_SLOTS);\n\t}\n\treturn (erofs_read_data_impl(sbi, vi, loff, len, rablocks, bufp));\n}\n\n/*\n * Transfer the logical content of an inode directly into a uio.\n", - "read data wrappers", - ) - - internal_candidate = replace_once( - internal_source, - "int erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n erofs_off_t loff, size_t len, void **bufp);\n", - "int erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,\n erofs_off_t loff, size_t len, void **bufp);\nint erofs_read_data_readahead(struct erofs_sb_info *sbi,\n struct erofs_inode *vi, erofs_off_t loff, size_t len, bool sequential,\n void **bufp);\n", - "internal readahead prototype", - ) - - dir_candidate = replace_once( - dir_source, - "\tbool have_previous;\n\tint error;\n", - "\tbool have_previous, sequential;\n\tint error;\n", - "directory sequential state", - ) - dir_candidate = replace_once( - dir_candidate, - "\tlogical_off = uio->uio_offset;\n\tuiodir.last_cookie = logical_off;\n", - "\tlogical_off = uio->uio_offset;\n\tsequential = logical_off == 0;\n\tuiodir.last_cookie = logical_off;\n", - "directory sequential initialization", - ) - dir_candidate = replace_once( - dir_candidate, - "\t\terror = erofs_read_data(sbi, dir, block_off, maxsize,\n\t\t (void **)&blk);\n", - "\t\terror = erofs_read_data_readahead(sbi, dir, block_off,\n\t\t maxsize, sequential, (void **)&blk);\n", - "directory readahead hint", - ) - return { - "src/data.c": data_candidate, - "src/dir.c": dir_candidate, - "src/internal.h": internal_candidate, - } - - -BENCHMARK_C = r'''#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static uint64_t -fnv_update(uint64_t value, const void *buffer, size_t length) -{ - const unsigned char *bytes = buffer; - size_t index; - - for (index = 0; index < length; index++) { - value ^= bytes[index]; - value *= UINT64_C(1099511628211); - } - return (value); -} - -static uint64_t -record_hash(uint64_t value, const struct dirent *entry) -{ - uint64_t inode, cookie; - uint16_t namelen; - uint8_t type; - - inode = htole64(entry->d_fileno); - type = entry->d_type; - namelen = htole16(entry->d_namlen); - cookie = htole64(entry->d_off); - value = fnv_update(value, &inode, sizeof(inode)); - value = fnv_update(value, &type, sizeof(type)); - value = fnv_update(value, &namelen, sizeof(namelen)); - value = fnv_update(value, &cookie, sizeof(cookie)); - return (fnv_update(value, entry->d_name, entry->d_namlen)); -} - -static uint64_t -elapsed_ns(const struct timespec *start, const struct timespec *end) -{ - return ((uint64_t)(end->tv_sec - start->tv_sec) * UINT64_C(1000000000) + - (uint64_t)(end->tv_nsec - start->tv_nsec)); -} - -int -main(int argc, char **argv) -{ - struct timespec start, end; - struct dirent *entry; - off_t base, last_cookie, seek_offset; - uint64_t count, hash; - char *buffer, *cursor; - long buffer_size; - ssize_t amount; - int fd, random_mode; - - if (argc != 5) { - fprintf(stderr, "usage: benchmark sequential|random DIR BUFFER SEEK\n"); - return (2); - } - random_mode = strcmp(argv[1], "random") == 0; - if (!random_mode && strcmp(argv[1], "sequential") != 0) - return (2); - buffer_size = strtol(argv[3], NULL, 10); - seek_offset = strtoll(argv[4], NULL, 10); - if (buffer_size <= 0 || buffer_size > INT32_MAX || seek_offset < 0) - return (2); - buffer = malloc((size_t)buffer_size); - if (buffer == NULL) - return (3); - fd = open(argv[2], O_RDONLY | O_DIRECTORY); - if (fd < 0) - return (4); - if (random_mode && lseek(fd, seek_offset, SEEK_SET) != seek_offset) - return (5); - hash = UINT64_C(1469598103934665603); - count = 0; - last_cookie = random_mode ? seek_offset : 0; - if (clock_gettime(CLOCK_MONOTONIC, &start) != 0) - return (6); - for (;;) { - base = 0; - amount = getdirentries(fd, buffer, (size_t)buffer_size, &base); - if (amount < 0) - return (7); - if (amount == 0) - break; - cursor = buffer; - while (cursor < buffer + amount) { - entry = (struct dirent *)cursor; - if (entry->d_reclen < _GENERIC_DIRSIZ(entry) || - cursor + entry->d_reclen > buffer + amount || - entry->d_off <= last_cookie) - return (8); - hash = record_hash(hash, entry); - last_cookie = entry->d_off; - count++; - cursor += entry->d_reclen; - } - if (random_mode) - break; - } - if (clock_gettime(CLOCK_MONOTONIC, &end) != 0) - return (9); - printf("%" PRIu64 "\t%" PRIu64 "\t%016" PRIx64 "\t%jd\n", - elapsed_ns(&start, &end), count, hash, (intmax_t)last_cookie); - close(fd); - free(buffer); - return (0); -} -''' - - -def guest_script(oracle: dict[str, Any]) -> str: - return f'''#!/bin/sh -set -eu - -work=$1 -phase=$2 -result=$work/result -mkdir -p "$result/baseline" "$result/candidate" "$work/freebsd-src" "$work/package" -cleanup() -{{ - for mountpoint in "$work"/mnt-*; do - test -d "$mountpoint" || continue - mount | grep -F " on $mountpoint " >/dev/null 2>&1 && umount "$mountpoint" || true - done - kldstat -n erofs.ko >/dev/null 2>&1 && kldunload erofs.ko || true -}} -trap cleanup EXIT HUP INT TERM - -if test ! -f "$result/prepared"; then - tar -xzf "$work/freebsd-sys.tar.gz" -C "$work/freebsd-src" - tar -xzf "$work/gate-package.tar.gz" -C "$work/package" - cc -O2 -Wall -Wextra -Werror "$work/package/benchmark.c" -o "$work/benchmark" - uname -a > "$result/uname.txt" - sha256 -q "$work/freebsd-src/sys/sys/buf.h" > "$result/guest-buf-h.sha256" - for variant in baseline candidate; do - ( - cd "$work/package/$variant/repo-pre-15" - env FREEBSD_SRC="$work/freebsd-src" WITH_ZSTDIO=0 sh ./build.sh - ) > "$result/build-$variant.log" 2>&1 - cp "$work/package/$variant/repo-pre-15/build/erofs.ko" "$result/$variant/erofs.ko" - nm -g "$result/$variant/erofs.ko" | awk '{{print $NF}}' | sort -u > "$result/globals-$variant.txt" - nm -u "$result/$variant/erofs.ko" | awk '{{print $NF}}' | sort -u > "$result/undefined-$variant.txt" - done - comm -13 "$result/globals-baseline.txt" "$result/globals-candidate.txt" > "$result/globals-added.txt" - comm -23 "$result/globals-baseline.txt" "$result/globals-candidate.txt" > "$result/globals-removed.txt" - diff -u "$result/undefined-baseline.txt" "$result/undefined-candidate.txt" > "$result/undefined.diff" || true - dmesg > "$result/dmesg-before.txt" - : > "$result/runs.tsv" - printf '%s\n' reached > "$result/target.marker" - printf '%s\n' prepared > "$result/prepared" -fi - -iostat_read() -{{ - iostat -Ix -d "$1" | awk -v device="$1" '$1 == device {{printf "%.0f %.0f\\n", $2, $4}}' -}} - -run_one() -{{ - run_id=$1 - device=$2 - mode=$3 - mountpoint="$work/mnt-$run_id" - mkdir "$mountpoint" - set -- $(iostat_read "$device") - reads_before=$1 - kb_before=$2 - mount -t erofs -o ro "/dev/$device" "$mountpoint" - if test "$mode" = sequential; then - bench=$($work/benchmark sequential "$mountpoint" {SPEC['benchmark']['buffer_bytes']} 0) - else - bench=$($work/benchmark random "$mountpoint" {SPEC['benchmark']['random_buffer_bytes']} {oracle['random_offset']}) - fi - sleep 1 - umount "$mountpoint" - set -- $(iostat_read "$device") - reads_after=$1 - kb_after=$2 - rmdir "$mountpoint" - set -- $bench - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ - "$variant" "$run_id" "$device" "$mode" "$1" \ - "$((reads_after - reads_before))" "$((kb_after - kb_before))" \ - "$2" "$3" "$4" >> "$result/runs.tsv" -}} - -case "$phase" in -baseline-a) - variant=baseline - runs='seq-1:da0:sequential seq-4:da3:sequential' - ;; -candidate-a) - variant=candidate - runs='seq-2:da1:sequential seq-3:da2:sequential seq-6:da5:sequential' - ;; -baseline-b) - variant=baseline - runs='seq-5:da4:sequential seq-8:da7:sequential seq-9:da8:sequential random-baseline:da10:random' - ;; -candidate-b) - variant=candidate - runs='seq-7:da6:sequential seq-10:da9:sequential random-candidate:da11:random' - ;; -*) - exit 2 - ;; -esac - -kldstat -v > "$result/kldstat-$phase-before.txt" -sysctl -n vfs.conflist > "$result/vfs-conflist-$phase-before.txt" -if ! kldload "$result/$variant/erofs.ko" > "$result/kldload-$phase.log" 2>&1; then - dmesg > "$result/dmesg-$phase-load-failure.txt" - exit 70 -fi -for spec in $runs; do - oldifs=$IFS - IFS=: - set -- $spec - IFS=$oldifs - run_one "$1" "$2" "$3" -done -cleanup -trap - EXIT HUP INT TERM -if test "$phase" = candidate-b; then - ! kldstat -n erofs.ko >/dev/null 2>&1 - dmesg > "$result/dmesg-after.txt" - kldstat > "$result/kldstat-after.txt" - mount > "$result/mount-after.txt" - sha256 -q "$result/baseline/erofs.ko" > "$result/erofs-baseline.sha256" - sha256 -q "$result/candidate/erofs.ko" > "$result/erofs-candidate.sha256" -fi -''' - - -def prepare_fixture(temp: Path) -> tuple[Path, dict[str, Any], list[str]]: - source = temp / "fixture-source" - source.mkdir() - count = int(SPEC["benchmark"]["entry_count"]) - name_length = int(SPEC["benchmark"]["name_length"]) - for index in range(count): - prefix = f"f{index:05d}-" - name = prefix + chr(ord("a") + index % 26) * (name_length - len(prefix)) - descriptor = os.open(source / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) - os.close(descriptor) - fixture_a = temp / "directory-a.erofs" - fixture_b = temp / "directory-b.erofs" - command = [ - "mkfs.erofs", "-d0", "-T0", "--all-time", "--all-root", - "--workers=1", "-x-1", "-E", "noinline_data", "-U", - SPEC["fixture"]["uuid"], str(fixture_a), str(source), - ] - env = dict(os.environ) - env["SOURCE_DATE_EPOCH"] = "0" - first = run(command, env=env) - command[-2] = str(fixture_b) - second = run(command, env=env) - if fixture_a.read_bytes() != fixture_b.read_bytes() or first.stdout != second.stdout: - raise GateFailure("INFRA_BLOCKED", "large-directory fixture is not byte reproducible") - transform_a = promote_root_tail_to_plain(fixture_a) - transform_b = promote_root_tail_to_plain(fixture_b) - if transform_a != transform_b or fixture_a.read_bytes() != fixture_b.read_bytes(): - raise GateFailure("INFRA_BLOCKED", "plain-directory derivation is not byte reproducible") - fsck = run(["fsck.erofs", "-d0", str(fixture_a)], check=False) - if fsck.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"derived plain fixture fails fsck: {fsck.stdout}") - fixture_out = fixture_a - image = Image(fixture_out.read_bytes()) - oracle = image.directory_oracle() - if oracle["directory_bytes"] < SPEC["fixture"]["minimum_directory_bytes"]: - raise GateFailure("STOP", "generated flat-plain directory is below the frozen workload size") - if oracle["entry_count"] < count: - raise GateFailure("INFRA_BLOCKED", "independent oracle lost generated entries") - oracle["generator_transform"] = transform_a - oracle["fsck_exit"] = fsck.returncode - canonical = command[:-2] + ["DIRECTORY.erofs", "SOURCE"] - return fixture_out, oracle, canonical - - -def create_package(temp: Path, resolved: str, sources: dict[str, str], - oracle: dict[str, Any]) -> tuple[Path, str, dict[str, Any]]: - archive = temp / "baseline.tar" - with archive.open("wb") as output_file: - completed = subprocess.run( - ["git", "-C", str(ROOT), "archive", "--format=tar", resolved, "repo-pre-15"], - stdout=output_file, - stderr=subprocess.PIPE, - check=False, - ) - if completed.returncode != 0: - raise GateFailure("INFRA_BLOCKED", completed.stderr.decode("utf-8", "replace")) - package_root = temp / "package-root" - baseline = package_root / "baseline" - candidate = package_root / "candidate" - baseline.mkdir(parents=True) - with tarfile.open(archive) as tar: - tar.extractall(baseline, filter="data") - shutil.copytree(baseline, candidate) - candidate_sources = make_candidate( - sources["repo-pre-15/src/data.c"], - sources["repo-pre-15/src/dir.c"], - sources["repo-pre-15/src/internal.h"], - ) - patch_lines: list[str] = [] - for relative, after in candidate_sources.items(): - before = sources[f"repo-pre-15/{relative}"] - target = candidate / "repo-pre-15" / relative - target.write_text(after, encoding="utf-8") - patch_lines.extend(difflib.unified_diff( - before.splitlines(keepends=True), after.splitlines(keepends=True), - fromfile=f"a/{relative}", tofile=f"b/{relative}", - )) - patch = "".join(patch_lines) - (OUTPUT / "candidate.patch").write_text(patch, encoding="utf-8") - semantic = { - "backing_vnode_only": "breadn(dif->devvp" in candidate_sources["src/data.c"], - "cluster_read_absent": "cluster_read" not in patch, - "directory_vnode_breadn_absent": "breadn" not in candidate_sources["src/dir.c"], - "errno_token_multiset_unchanged": sorted(re.findall(r"return \(([A-Z][A-Z0-9_]*)\);", sources["repo-pre-15/src/data.c"])) == sorted(re.findall(r"return \(([A-Z][A-Z0-9_]*)\);", candidate_sources["src/data.c"])), - "geom_ownership_calls_not_added": not re.search(r"\b(?:g_attach|g_detach|g_access|g_destroy_consumer)\s*\(", patch), - "locking_calls_not_added": not re.search(r"\b(?:VOP_LOCK|vn_lock|lockmgr|mtx_lock|sx_xlock)\s*\(", patch), - "max_readahead_bytes": SPEC["prototype"]["max_readahead_bytes"], - "random_seek_gate": "sequential = logical_off == 0;" in candidate_sources["src/dir.c"], - "read_hint_is_directory_only": candidate_sources["src/dir.c"].count("erofs_read_data_readahead(") == 1, - "vm_entrypoints_not_added": not re.search(r"\b(?:vnode_create_vobject|vm_object|VOP_BMAP)\b", patch), - "write_set": sorted(candidate_sources), - } - semantic["pass"] = all( - value for key, value in semantic.items() - if key not in {"max_readahead_bytes", "write_set"} - ) and semantic["max_readahead_bytes"] <= 1024 * 1024 - write_json(OUTPUT / "semantic-ledger.json", semantic) - (package_root / "benchmark.c").write_text(BENCHMARK_C, encoding="ascii") - write_json(package_root / "oracle.json", oracle) - (OUTPUT / "benchmark.c").write_text(BENCHMARK_C, encoding="ascii") - return package_root, patch, semantic - - -def build_modules(temp: Path, package_root: Path) -> dict[str, Path]: - helper = ROOT / "repo-pre-15/tests/pre15/fixtures/B12-build-kld.sh" - if not helper.is_file(): - raise GateFailure("INFRA_BLOCKED", f"missing B12 KLD helper: {helper}") - modules: dict[str, Path] = {} - symbols: dict[str, list[str]] = {} - undefined: dict[str, list[str]] = {} - module_root = temp / "modules" - for variant in ("baseline", "candidate"): - output_dir = module_root / variant - output_dir.mkdir(parents=True) - output = output_dir / "erofs.ko" - work = temp / f"kld-work-{variant}" - completed = run([ - "/bin/sh", str(helper), - str(package_root / variant / "repo-pre-15"), - str(FREEBSD_SRC), str(output), str(work), - ], timeout=600) - (OUTPUT / f"build-{variant}.log").write_text( - completed.stdout, encoding="utf-8" - ) - modules[variant] = output - globals_output = run(["nm", "-g", str(output)]).stdout.splitlines() - undefined_output = run(["nm", "-u", str(output)]).stdout.splitlines() - symbols[variant] = sorted({line.split()[-1] for line in globals_output if line.split()}) - undefined[variant] = sorted({line.split()[-1] for line in undefined_output if line.split()}) - (OUTPUT / f"module-{variant}.sha256").write_text( - f"{sha256_path(output)} erofs.ko\n", encoding="ascii" - ) - added = sorted(set(symbols["candidate"]) - set(symbols["baseline"])) - removed = sorted(set(symbols["baseline"]) - set(symbols["candidate"])) - (OUTPUT / "globals-added.txt").write_text("\n".join(added) + ("\n" if added else ""), encoding="ascii") - (OUTPUT / "globals-removed.txt").write_text("\n".join(removed) + ("\n" if removed else ""), encoding="ascii") - undefined_added = sorted(set(undefined["candidate"]) - set(undefined["baseline"])) - undefined_removed = sorted(set(undefined["baseline"]) - set(undefined["candidate"])) - (OUTPUT / "undefined-added.txt").write_text( - "\n".join(undefined_added) + ("\n" if undefined_added else ""), - encoding="ascii", - ) - (OUTPUT / "undefined-removed.txt").write_text( - "\n".join(undefined_removed) + ("\n" if undefined_removed else ""), - encoding="ascii", - ) - undefined_diff = list(difflib.unified_diff( - [line + "\n" for line in undefined["baseline"]], - [line + "\n" for line in undefined["candidate"]], - fromfile="baseline", tofile="candidate", - )) - (OUTPUT / "undefined.diff").write_text("".join(undefined_diff), encoding="ascii") - return modules - - -def ssh_options(port: int) -> list[str]: - return [ - "-q", - "-o", "BatchMode=no", - "-o", "PubkeyAuthentication=no", - "-o", "PreferredAuthentications=keyboard-interactive,password", - "-o", "NumberOfPasswordPrompts=1", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "ConnectTimeout=5", - "-p", str(port), - ] - - -def scp_options(port: int) -> list[str]: - return [ - "-q", - "-o", "BatchMode=no", - "-o", "PubkeyAuthentication=no", - "-o", "PreferredAuthentications=keyboard-interactive,password", - "-o", "NumberOfPasswordPrompts=1", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "ConnectTimeout=5", - "-P", str(port), - ] - - -def qemu_replay(temp: Path, fixture: Path, modules: dict[str, Path], - oracle: dict[str, Any]) -> dict[str, Any]: - inputs = SPEC["host_inputs"] - base_image = Path(inputs["base_image"]) - askpass = Path(inputs["askpass_path"]) - ssh_key = Path(inputs["ssh_key_path"]) - if base_image.stat().st_size != inputs["base_image_size"]: - raise GateFailure("INFRA_BLOCKED", "base .bp size changed") - if sha256_path(askpass) != inputs["askpass_sha256"] or sha256_path(ssh_key) != inputs["ssh_key_sha256"]: - raise GateFailure("INFRA_BLOCKED", "QEMU authentication input changed") - protected_pid = int(SPEC["protected"]["pid"]) - protected_port = int(SPEC["protected"]["port"]) - protected_proc = Path(f"/proc/{protected_pid}") - if not protected_proc.exists(): - raise GateFailure("INFRA_BLOCKED", "protected QEMU PID is absent") - protected_start = (protected_proc / "stat").read_text().split()[21] - protected_cmd = sha256_path(protected_proc / "cmdline") - base_before = base_image.stat() - auth_env = dict(os.environ) - auth_env.update({"DISPLAY": ":0", "SSH_ASKPASS": str(askpass), "SSH_ASKPASS_REQUIRE": "force"}) - rows: list[str] = [] - ownership_rounds: list[dict[str, Any]] = [] - sequential_counts = {"baseline": 0, "candidate": 0} - prepared_overlay = temp / "prepared-overlay.qcow2" - - def protected_ok() -> bool: - return ( - protected_proc.exists() - and (protected_proc / "stat").read_text().split()[21] == protected_start - and sha256_path(protected_proc / "cmdline") == protected_cmd - ) - - def pick_port() -> int: - while True: - with socket.socket() as listener: - listener.bind(("127.0.0.1", 0)) - selected = listener.getsockname()[1] - if selected != protected_port: - return selected - - def prepare_overlay() -> None: - prepare_dir = OUTPUT / "prepare" - prepare_dir.mkdir() - run([ - "qemu-img", "create", "-q", "-f", "qcow2", "-F", - inputs["base_image_format"], "-b", str(base_image), - str(prepared_overlay), - ]) - port = pick_port() - serial = prepare_dir / "serial.log" - argv = [ - "qemu-system-x86_64", "-accel", "tcg,thread=multi", "-cpu", "qemu64", - "-m", str(SPEC["qemu"]["memory_mb"]), "-smp", str(SPEC["qemu"]["cpus"]), - "-drive", f"file={prepared_overlay},if=virtio,format=qcow2,cache=none", - "-netdev", f"user,id=net0,hostfwd=tcp:127.0.0.1:{port}-:22", - "-device", "virtio-net-pci,netdev=net0", "-display", "none", - "-serial", f"file:{serial}", "-monitor", "none", - ] - write_json(prepare_dir / "qemu-argv.json", argv) - qemu_stdout = (prepare_dir / "qemu-process.log").open("w", encoding="utf-8") - process = subprocess.Popen( - argv, stdin=subprocess.DEVNULL, stdout=qemu_stdout, - stderr=subprocess.STDOUT, text=True, - ) - if process.pid == protected_pid: - process.terminate() - qemu_stdout.close() - raise GateFailure("INFRA_BLOCKED", "prepare QEMU reused protected PID") - (prepare_dir / "owned-pid.txt").write_text(f"{process.pid}\n", encoding="ascii") - (prepare_dir / "owned-port.txt").write_text(f"{port}\n", encoding="ascii") - options = ssh_options(port) - copy_options = scp_options(port) - guest_root = "/root/pre15-p15087" - pending_error: Exception | None = None - - def ssh(command: str, timeout: int = 60, check: bool = True) -> subprocess.CompletedProcess[str]: - return run( - ["ssh", *options, "root@127.0.0.1", command], - env=auth_env, timeout=timeout, check=check, - ) - - def scp_to(local: Path, remote: str) -> None: - run( - ["scp", *copy_options, str(local), f"root@127.0.0.1:{remote}"], - env=auth_env, timeout=120, - ) - - try: - deadline = time.monotonic() + int(SPEC["qemu"]["boot_timeout_seconds"]) - while time.monotonic() < deadline: - if process.poll() is not None: - raise GateFailure("INFRA_BLOCKED", "prepare QEMU exited before SSH") - if ssh("true", timeout=8, check=False).returncode == 0: - break - time.sleep(2) - else: - raise GateFailure("INFRA_BLOCKED", "prepare QEMU SSH boot deadline expired") - ssh(f"mkdir -p {guest_root}/baseline {guest_root}/candidate {guest_root}/evidence") - scp_to(modules["baseline"], f"{guest_root}/baseline/erofs.ko") - scp_to(modules["candidate"], f"{guest_root}/candidate/erofs.ko") - scp_to(OUTPUT / "benchmark.c", f"{guest_root}/benchmark.c") - scp_to(fixture, f"{guest_root}/fixture-1.erofs") - scp_to(fixture, f"{guest_root}/fixture-2.erofs") - setup = ssh( - f"cc -O2 -Wall -Wextra -Werror {guest_root}/benchmark.c -o {guest_root}/benchmark && " - f"uname -a > {guest_root}/evidence/uname.txt && " - f"sysctl -n kern.osreldate > {guest_root}/evidence/osreldate.txt && " - f"sha256 -q {guest_root}/fixture-1.erofs > {guest_root}/evidence/fixture-1.sha256 && " - f"sha256 -q {guest_root}/fixture-2.erofs > {guest_root}/evidence/fixture-2.sha256 && " - f"sha256 -q {guest_root}/baseline/erofs.ko > {guest_root}/evidence/baseline.sha256 && " - f"sha256 -q {guest_root}/candidate/erofs.ko > {guest_root}/evidence/candidate.sha256 && sync", - timeout=180, check=False, - ) - if setup.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"prepare guest setup failed: {setup.stdout}") - prepare_guest = prepare_dir / "guest" - prepare_guest.mkdir() - run( - ["scp", *copy_options, "-r", f"root@127.0.0.1:{guest_root}/evidence/.", str(prepare_guest)], - env=auth_env, timeout=120, - ) - ssh("shutdown -p now", timeout=10, check=False) - try: - process.wait(timeout=90) - except subprocess.TimeoutExpired: - process.terminate() - process.wait(timeout=15) - except Exception as error: - pending_error = error - finally: - if process.poll() is None: - process.terminate() - try: - process.wait(timeout=15) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=10) - qemu_stdout.close() - port_released = False - for _ in range(50): - with socket.socket() as check_socket: - port_released = check_socket.connect_ex(("127.0.0.1", port)) != 0 - if port_released: - break - time.sleep(0.1) - ownership = { - "owned_pid": process.pid, - "owned_pid_stopped": process.poll() is not None, - "owned_port": port, - "owned_port_free_after": port_released, - "prepared_overlay_present": prepared_overlay.exists(), - } - write_json(prepare_dir / "ownership.json", ownership) - if not all((ownership["owned_pid_stopped"], port_released, - ownership["prepared_overlay_present"])): - raise GateFailure("INFRA_BLOCKED", "prepare ownership audit failed") - if pending_error is not None: - raise pending_error - - def run_round(round_name: str, samples: list[tuple[str, str]]) -> None: - round_dir = OUTPUT / "rounds" / round_name - round_dir.mkdir(parents=True) - overlay = temp / f"{round_name}-overlay.qcow2" - run([ - "qemu-img", "create", "-q", "-f", "qcow2", "-F", "qcow2", - "-b", str(prepared_overlay), str(overlay), - ]) - port = pick_port() - serial = round_dir / "serial.log" - qemu_log = round_dir / "qemu.log" - argv = [ - "qemu-system-x86_64", "-accel", "tcg,thread=multi", "-cpu", "qemu64", - "-m", str(SPEC["qemu"]["memory_mb"]), "-smp", str(SPEC["qemu"]["cpus"]), - "-drive", f"file={overlay},if=virtio,format=qcow2,cache=none", - "-netdev", f"user,id=net0,hostfwd=tcp:127.0.0.1:{port}-:22", - "-device", "virtio-net-pci,netdev=net0", - "-display", "none", "-serial", f"file:{serial}", "-monitor", "none", - ] - write_json(round_dir / "qemu-argv.json", argv) - qemu_stdout = (round_dir / "qemu-process.log").open("w", encoding="utf-8") - process = subprocess.Popen( - argv, stdin=subprocess.DEVNULL, stdout=qemu_stdout, - stderr=subprocess.STDOUT, text=True, - ) - if process.pid == protected_pid: - process.terminate() - qemu_stdout.close() - raise GateFailure("INFRA_BLOCKED", "owned QEMU reused protected PID") - (round_dir / "owned-pid.txt").write_text(f"{process.pid}\n", encoding="ascii") - (round_dir / "owned-port.txt").write_text(f"{port}\n", encoding="ascii") - options = ssh_options(port) - copy_options = scp_options(port) - guest_root = "/root/pre15-p15087" - guest_reached = False - guest_md_removed = False - pending_error: Exception | None = None - - def ssh(command: str, timeout: int = 60, check: bool = True) -> subprocess.CompletedProcess[str]: - return run( - ["ssh", *options, "root@127.0.0.1", command], - env=auth_env, timeout=timeout, check=check, - ) - - def scp_to(local: Path, remote: str) -> None: - run( - ["scp", *copy_options, str(local), f"root@127.0.0.1:{remote}"], - env=auth_env, timeout=120, - ) - - try: - deadline = time.monotonic() + int(SPEC["qemu"]["boot_timeout_seconds"]) - while time.monotonic() < deadline: - if process.poll() is not None: - raise GateFailure("INFRA_BLOCKED", f"{round_name}: QEMU exited before SSH") - if ssh("true", timeout=8, check=False).returncode == 0: - break - time.sleep(2) - else: - raise GateFailure("INFRA_BLOCKED", f"{round_name}: SSH boot deadline expired") - ssh(f"mkdir -p {guest_root}/evidence") - setup = ssh( - f"test -x {guest_root}/benchmark && " - f"test -f {guest_root}/fixture-1.erofs && " - f"test -f {guest_root}/fixture-2.erofs && " - f"uname -a > {guest_root}/evidence/uname-round.txt && " - f"sysctl -n kern.osreldate > {guest_root}/evidence/osreldate.txt && " - f"kldstat > {guest_root}/evidence/kldstat-before.txt && " - f"mount > {guest_root}/evidence/mount-before.txt && " - f"mdconfig -l > {guest_root}/evidence/md-before.txt && " - f"dmesg > {guest_root}/evidence/dmesg-before.txt && " - f"sha256 -q {guest_root}/baseline/erofs.ko > {guest_root}/evidence/baseline.sha256 && " - f"sha256 -q {guest_root}/candidate/erofs.ko > {guest_root}/evidence/candidate.sha256 && " - f"sha256 -q {guest_root}/fixture-1.erofs > {guest_root}/evidence/fixture-1.sha256 && " - f"sha256 -q {guest_root}/fixture-2.erofs > {guest_root}/evidence/fixture-2.sha256", - timeout=120, check=False, - ) - if setup.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"{round_name}: guest setup failed: {setup.stdout}") - guest_reached = True - for device_index, (variant, mode) in enumerate(samples): - fixture_index = device_index + 1 - if mode == "sequential": - sequential_counts[variant] += 1 - run_id = f"seq-{sequential_counts[variant]}" - buffer_bytes = int(SPEC["benchmark"]["buffer_bytes"]) - seek_offset = 0 - else: - run_id = f"random-{variant}" - buffer_bytes = int(SPEC["benchmark"]["random_buffer_bytes"]) - seek_offset = int(oracle["random_offset"]) - evidence_id = run_id if mode == "random" else f"{run_id}-{variant}" - command = f'''set -eu -work={shlex.quote(guest_root)} -variant={shlex.quote(variant)} -fixture="$work/fixture-{fixture_index}.erofs" -mode={shlex.quote(mode)} -run_id={shlex.quote(run_id)} -evidence_id={shlex.quote(evidence_id)} -mountpoint="$work/mnt-$evidence_id" -device= -cleanup() -{{ - mount | grep -F " on $mountpoint " >/dev/null 2>&1 && umount "$mountpoint" || true - kldstat -n erofs.ko >/dev/null 2>&1 && kldunload erofs.ko || true - test -z "$device" || mdconfig -d -u "${{device#md}}" >/dev/null 2>&1 || true - rmdir "$mountpoint" >/dev/null 2>&1 || true -}} -trap cleanup EXIT HUP INT TERM -mkdir "$mountpoint" -device=$(mdconfig -a -t vnode -f "$fixture") -test -c "/dev/$device" -set -- $(iostat -Ix -d "$device" | awk -v device="$device" '$1 == device {{printf "%.0f %.0f\\n", $2, $4; found=1}} END {{exit !found}}') -reads_before=$1 -kb_before=$2 -kldload "$work/$variant/erofs.ko" -kldstat -v > "$work/evidence/kldstat-$evidence_id.txt" -mdconfig -lv > "$work/evidence/md-$evidence_id-before.txt" -mount -t erofs -o ro "/dev/$device" "$mountpoint" -bench=$("$work/benchmark" "$mode" "$mountpoint" {buffer_bytes} {seek_offset}) -printf '%s\n' "$bench" > "$work/evidence/benchmark-$evidence_id.txt" -sleep 1 -umount "$mountpoint" -set -- $(iostat -Ix -d "$device" | awk -v device="$device" '$1 == device {{printf "%.0f %.0f\\n", $2, $4; found=1}} END {{exit !found}}') -reads_after=$1 -kb_after=$2 -kldunload erofs.ko -record_device=$device -mdconfig -d -u "${{device#md}}" -device= -rmdir "$mountpoint" -trap - EXIT HUP INT TERM -set -- $bench - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ - "$variant" "$run_id" "$record_device" "$mode" "$1" \ - "$((reads_after - reads_before))" "$((kb_after - kb_before))" \ - "$2" "$3" "$4" -''' - sample = ssh(command, timeout=int(SPEC["qemu"]["guest_timeout_seconds"]), check=False) - (round_dir / f"sample-{evidence_id}.log").write_text(sample.stdout, encoding="utf-8") - if sample.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"{round_name}: {variant} {mode} sample failed: {sample.stdout}") - sample_rows = [line for line in sample.stdout.splitlines() if line.count("\t") == 9] - if len(sample_rows) != 1: - raise GateFailure("INFRA_BLOCKED", f"{round_name}: malformed sample output: {sample.stdout}") - rows.append(sample_rows[0]) - ssh( - f"dmesg > {guest_root}/evidence/dmesg-after.txt && " - f"kldstat > {guest_root}/evidence/kldstat-after.txt && " - f"mount > {guest_root}/evidence/mount-after.txt && " - f"mdconfig -l > {guest_root}/evidence/md-after.txt", - timeout=60, - ) - guest_evidence = round_dir / "guest" - guest_evidence.mkdir() - run( - ["scp", *copy_options, "-r", f"root@127.0.0.1:{guest_root}/evidence/.", str(guest_evidence)], - env=auth_env, timeout=120, - ) - cleanup = ssh( - f"test -z \"$(mount -t erofs)\" && " - f"! kldstat -n erofs.ko >/dev/null 2>&1 && " - f"test -z \"$(mdconfig -l)\"", - timeout=60, check=False, - ) - if cleanup.returncode != 0: - raise GateFailure("INFRA_BLOCKED", f"{round_name}: guest cleanup failed: {cleanup.stdout}") - guest_md_removed = True - ssh("shutdown -p now", timeout=10, check=False) - try: - process.wait(timeout=90) - except subprocess.TimeoutExpired: - process.terminate() - process.wait(timeout=15) - except Exception as error: - pending_error = error - if guest_reached: - ssh(f"dmesg > {guest_root}/evidence/dmesg-failure.txt", timeout=30, check=False) - partial = round_dir / "guest-partial" - partial.mkdir(exist_ok=True) - run( - ["scp", *copy_options, "-r", f"root@127.0.0.1:{guest_root}/evidence/.", str(partial)], - env=auth_env, timeout=60, check=False, - ) - finally: - if process.poll() is None: - process.terminate() - try: - process.wait(timeout=15) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=10) - qemu_stdout.close() - overlay.unlink(missing_ok=True) - port_free = False - for _ in range(50): - with socket.socket() as check_socket: - port_free = check_socket.connect_ex(("127.0.0.1", port)) != 0 - if port_free: - break - time.sleep(0.1) - round_ownership = { - "name": round_name, - "owned_pid": process.pid, - "owned_pid_stopped": process.poll() is not None, - "owned_port": port, - "owned_port_free_after": port_free, - "overlay_removed": not overlay.exists(), - "guest_md_removed": guest_md_removed, - } - ownership_rounds.append(round_ownership) - write_json(round_dir / "ownership.json", round_ownership) - if not all((round_ownership["owned_pid_stopped"], port_free, - round_ownership["overlay_removed"], round_ownership["guest_md_removed"])): - raise GateFailure("INFRA_BLOCKED", f"{round_name}: ownership cleanup failed") - if pending_error is not None: - raise pending_error - - prepare_overlay() - if not prepared_overlay.is_file(): - raise GateFailure("INFRA_BLOCKED", "prepared overlay disappeared before sampling") - for round_index in range(1, 6): - order = [("baseline", "sequential"), ("candidate", "sequential")] - if round_index % 2 == 0: - order.reverse() - run_round(f"sequential-{round_index}", order) - run_round("random", [("baseline", "random"), ("candidate", "random")]) - (OUTPUT / "runs.tsv").write_text("\n".join(rows) + "\n", encoding="ascii") - (OUTPUT / "target.marker").write_text("reached\n", encoding="ascii") - base_after = base_image.stat() - base_ok = ( - base_before.st_ino == base_after.st_ino - and base_before.st_size == base_after.st_size - and base_before.st_mtime_ns == base_after.st_mtime_ns - and base_before.st_ctime_ns == base_after.st_ctime_ns - ) - ownership = { - "base_bp_metadata_unchanged": base_ok, - "base_bp_path": str(base_image), - "protected_pid": protected_pid, - "protected_pid_identity_unchanged": protected_ok(), - "protected_port": protected_port, - "protected_port_used": False, - "rounds": ownership_rounds, - } - write_json(OUTPUT / "ownership.json", ownership) - if not base_ok or not ownership["protected_pid_identity_unchanged"]: - raise GateFailure("INFRA_BLOCKED", f"QEMU ownership audit failed: {ownership}") - return ownership - - -def evaluate(oracle: dict[str, Any], semantic: dict[str, Any]) -> dict[str, Any]: - run_path = OUTPUT / "runs.tsv" - rows = [] - for line in run_path.read_text(encoding="ascii").splitlines(): - fields = line.split("\t") - if len(fields) != 10: - raise GateFailure("INFRA_BLOCKED", f"bad guest run row: {line}") - rows.append({ - "variant": fields[0], "run": fields[1], "device": fields[2], - "mode": fields[3], "elapsed_ns": int(fields[4]), - "provider_reads": int(fields[5]), "provider_kb": int(fields[6]), - "entry_count": int(fields[7]), "hash": fields[8], - "final_cookie": int(fields[9]), - }) - sequential = [row for row in rows if row["mode"] == "sequential"] - baseline = [row for row in sequential if row["variant"] == "baseline"] - candidate = [row for row in sequential if row["variant"] == "candidate"] - random_rows = [row for row in rows if row["mode"] == "random"] - if len(baseline) != 5 or len(candidate) != 5 or len(random_rows) != 2: - raise GateFailure("INFRA_BLOCKED", "guest did not produce the frozen 5+5+2 run matrix") - correctness = all( - row["entry_count"] == oracle["entry_count"] - and row["hash"] == oracle["fnv64"] - and row["final_cookie"] == oracle["final_cookie"] - for row in sequential - ) - baseline_ns = statistics.median(row["elapsed_ns"] for row in baseline) - candidate_ns = statistics.median(row["elapsed_ns"] for row in candidate) - improvement = (baseline_ns - candidate_ns) * 100.0 / baseline_ns - baseline_reads = statistics.median(row["provider_reads"] for row in baseline) - candidate_reads = statistics.median(row["provider_reads"] for row in candidate) - if baseline_reads <= 0: - raise GateFailure("INFRA_BLOCKED", "provider read counter did not observe baseline I/O") - extra_reads = max(0.0, (candidate_reads - baseline_reads) * 100.0 / baseline_reads) - random_baseline = next(row for row in random_rows if row["variant"] == "baseline") - random_candidate = next(row for row in random_rows if row["variant"] == "candidate") - random_noop = ( - random_baseline["entry_count"] == random_candidate["entry_count"] - and random_baseline["hash"] == random_candidate["hash"] - and random_baseline["final_cookie"] == random_candidate["final_cookie"] - and random_baseline["provider_reads"] == random_candidate["provider_reads"] - ) - added = (OUTPUT / "globals-added.txt").read_text().split() - removed = (OUTPUT / "globals-removed.txt").read_text().split() - undefined_diff = (OUTPUT / "undefined.diff").read_text().strip() - symbol_contract = added == ["erofs_read_data_readahead"] and not removed and not undefined_diff - dmesg = "\n".join( - path.read_text(encoding="utf-8", errors="replace") - for path in sorted((OUTPUT / "rounds").glob("*/guest/dmesg-after.txt")) - ) - kernel_clean = not re.search( - r"panic:|lock order reversal|KASAN:|Witness.*warning|link_elf_obj:.*(?:error|undefined)|linker_load_file:.*error", - dmesg, re.IGNORECASE, - ) - thresholds = SPEC["thresholds"] - checks = { - "cold_median_improvement": improvement >= thresholds["minimum_cold_median_improvement_percent"], - "correctness_hash_cookie": correctness, - "extra_provider_reads": extra_reads <= thresholds["maximum_extra_provider_reads_percent"], - "kernel_log_clean": kernel_clean, - "random_seek_noop": random_noop, - "semantic_contract": semantic["pass"], - "symbol_contract": symbol_contract, - "window_bound": semantic["max_readahead_bytes"] <= 1024 * 1024, - } - status = "GO" if all(checks.values()) else "STOP" - result = { - "b12": "AUTHORIZED" if status == "GO" else "STOP-NO-SOURCE", - "baseline_cold_elapsed_ns": [row["elapsed_ns"] for row in baseline], - "baseline_median_elapsed_ns": baseline_ns, - "baseline_median_provider_reads": baseline_reads, - "candidate": "P15-087", - "candidate_cold_elapsed_ns": [row["elapsed_ns"] for row in candidate], - "candidate_median_elapsed_ns": candidate_ns, - "candidate_median_provider_reads": candidate_reads, - "checks": checks, - "cold_median_improvement_percent": improvement, - "extra_provider_reads_percent": extra_reads, - "full_feature_suite": "NOT_RUN", - "gate": "G11", - "oracle": oracle, - "qemu": "PASS", - "random_rows": random_rows, - "requested_base": REQUESTED_BASE, - "schema": 1, - "sequential_rows": sequential, - "status": status, - "thresholds": thresholds, - } - write_json(OUTPUT / "result.json", result) - return result - - -def finalize() -> None: - lines = [] - for path in sorted(OUTPUT.rglob("*")): - if path.is_file() and path.name != "SHA256SUMS": - lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}") - (OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii") - - -result: dict[str, Any] | None = None -exit_code = 0 -try: - if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-087" or SPEC.get("gate") != "G11": - raise GateFailure("INFRA_BLOCKED", "invalid P15-087 input schema") - resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}") - if resolved != SPEC["required_base"]: - raise GateFailure("INFRA_BLOCKED", f"P15-087 must replay {SPEC['required_base']}, got {resolved}") - sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]} - source_hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()} - if source_hashes != SPEC["source_sha256"]: - raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed") - freebsd_head = subprocess.check_output(["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True).strip() - freebsd_hashes = {path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd_sha256"]} - if freebsd_head != SPEC["freebsd_head"] or freebsd_hashes != SPEC["freebsd_sha256"]: - raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed") - version = run(["mkfs.erofs", "-V"]).stdout.splitlines()[0] - if version != SPEC["mkfs_version"]: - raise GateFailure("INFRA_BLOCKED", f"mkfs version changed: {version}") - write_json(OUTPUT / "source-sha256.json", source_hashes) - write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes}) - with tempfile.TemporaryDirectory(prefix="p15-087-gate-") as temporary: - temp = Path(temporary) - fixture, oracle, command = prepare_fixture(temp) - oracle["fixture_sha256"] = sha256_path(fixture) - oracle["mkfs_command"] = command - write_json(OUTPUT / "oracle.json", oracle) - package_root, patch, semantic = create_package(temp, resolved, sources, oracle) - modules = build_modules(temp, package_root) - qemu_replay(temp, fixture, modules, oracle) - result = evaluate(oracle, semantic) - exit_code = 0 if result["status"] == "GO" else 1 -except GateFailure as failure: - status = failure.status - result = { - "b12": "STOP-NO-SOURCE" if status == "STOP" else "NOT_RUN", - "candidate": "P15-087", - "full_feature_suite": "NOT_RUN", - "gate": "G11", - "qemu": "INFRA_BLOCKED" if status == "INFRA_BLOCKED" else "NOT_RUN", - "reason": failure.reason, - "requested_base": REQUESTED_BASE, - "schema": 1, - "status": status, - } - write_json(OUTPUT / "result.json", result) - exit_code = 21 if status == "INFRA_BLOCKED" else 1 -except Exception as failure: - result = { - "b12": "NOT_RUN", - "candidate": "P15-087", - "full_feature_suite": "NOT_RUN", - "gate": "G11", - "qemu": "INFRA_BLOCKED", - "reason": f"unexpected gate runner failure: {failure}", - "requested_base": REQUESTED_BASE, - "schema": 1, - "status": "INFRA_BLOCKED", - } - write_json(OUTPUT / "result.json", result) - exit_code = 21 -finally: - finalize() - -print(json.dumps(result, sort_keys=True)) -raise SystemExit(exit_code) -PY diff --git a/tests/pre15/gates/P15-092-input.json b/tests/pre15/gates/P15-092-input.json deleted file mode 100644 index 02ebcde..0000000 --- a/tests/pre15/gates/P15-092-input.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "candidate": "P15-092", - "freebsd_head": "106727738dcfb6c001b46f25363b91cece970085", - "freebsd_sha256": { - "sys/kern/vfs_hash.c": "ed48e09f31c1ab5071b241c823bede9f1db46494f988dc9c3f2c2e31ed1d79c0", - "sys/kern/vfs_lookup.c": "7958081eda9a137a9cd913b959e86b9ed61eff2f9d6cdb5ce5bcfc8211307f24", - "sys/sys/vnode.h": "6a7c77fda50f721ed3539d81f4b77e5ea3185827ce3917cb331b51fe9dd2025f", - "sys/vm/vm_mmap.c": "d7bcbb749fae7fa4d36d544762b18dac25e605a0d9dc5f36c8e5b5150b531565" - }, - "gate": "G11", - "linux_head": "audit-2-semantic/14-source-hashes.txt", - "mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6", - "protected": { - "base_image": "/work/debug-qemu/local/vm-freebsd-build.qcow2.bp", - "pid": 26318, - "port": 9222 - }, - "required_base": "b22dae8dc634c68db4ea89dccade91350614c139", - "schema": 1, - "source_sha256": { - "repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1", - "repo-pre-15/src/erofs_vnops.c": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64", - "repo-pre-15/src/inode.c": "dee361837ea104f455c851fe4c268bf5d7e10cd9dd2ef58645aea819137639f6", - "repo-pre-15/src/internal.h": "eec416077040587ad0756814c4323425889d10bd9b4143fcfc63299bf4f90ae9", - "repo-pre-15/src/namei.c": "7c78c48927f81b8e54b75c3a561a4768441c00622a126589d3bd5a3c6b6ed152", - "repo-pre-15/src/super.c": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2", - "src-linux/erofs_fs.h": "6cb322cf7506858c3c82de3c81026039c543f448201c9c551fd81360cca67e93", - "src-linux/inode.c": "a15562e0782e155a0744a7ba9d2f557519a583b64853ed75180cef2b4dfae1b3", - "src-linux/super.c": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9" - }, - "uuid": "00000000-0000-0000-0000-000000000092", - "utils_head": "7db78788b000999e2de88decd2ba90654f26171c", - "utils_sha256": { - "include/erofs_fs.h": "02b01a99fe3180f86bb0372efc332efc243142d37aed2819045ae69717a9c915", - "lib/inode.c": "382bf7ccc22436ab9997a3fa417a0424216a6f3b58a95d7c9a1d182a9e228bc1", - "lib/namei.c": "faeef5248a81c2dee457c5009f3a47e194befe2ffa6ef9f096e029ba63ca014f" - }, - "write_set": [ - "repo-pre-15/src/inode.c", - "repo-pre-15/src/namei.c" - ] -} diff --git a/tests/pre15/gates/P15-092.sh b/tests/pre15/gates/P15-092.sh deleted file mode 100755 index 933c784..0000000 --- a/tests/pre15/gates/P15-092.sh +++ /dev/null @@ -1,881 +0,0 @@ -#!/bin/sh -set -eu - -gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P) -dut=$root/repo-pre-15 -input=$gate_dir/P15-092-input.json -freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src} -utils_src=${EROFS_UTILS_SRC:-/work/build/erofs-utils-main} -base= -output= -qemu_base= -qemu_key= -host_only=0 - -while test "$#" -gt 0; do - case "$1" in - --base) - test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; } - base=$2 - shift 2 - ;; - --output) - test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; } - output=$2 - shift 2 - ;; - --qemu-base) - test "$#" -ge 2 || { printf '%s\n' '--qemu-base requires an image' >&2; exit 2; } - qemu_base=$2 - shift 2 - ;; - --qemu-key) - test "$#" -ge 2 || { printf '%s\n' '--qemu-key requires a key' >&2; exit 2; } - qemu_key=$2 - shift 2 - ;; - --host-only) - host_only=1 - shift - ;; - *) - printf 'unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; } -test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; } -test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; } -test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; } -test -d "$utils_src/lib" || { printf 'missing erofs-utils source: %s\n' "$utils_src" >&2; exit 2; } -if test "$host_only" -eq 0; then - test -n "$qemu_base" || { printf '%s\n' '--qemu-base is required' >&2; exit 2; } - test -n "$qemu_key" || { printf '%s\n' '--qemu-key is required' >&2; exit 2; } - test -f "$qemu_base" || { printf 'missing QEMU base: %s\n' "$qemu_base" >&2; exit 2; } - test -f "$qemu_key" || { printf 'missing QEMU key: %s\n' "$qemu_key" >&2; exit 2; } -fi -for tool in dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum timeout; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required tool: %s\n' "$tool" >&2 - exit 2 - } -done -if test "$host_only" -eq 0; then - for tool in clang qemu-img qemu-system-x86_64 scp ssh tar; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing required QEMU tool: %s\n' "$tool" >&2 - exit 2 - } - done -fi -case "$output" in -/*) ;; -*) output=$PWD/$output ;; -esac -test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; } -mkdir -p "$output/host" - -python3 -B - "$root" "$input" "$base" "$output/host" "$freebsd_src" \ - "$utils_src" "${qemu_base:-/nonexistent}" <<'PY' -from __future__ import annotations - -import hashlib -import json -import os -from pathlib import Path -import re -import shutil -import stat -import struct -import subprocess -import sys -import tempfile -from typing import Any - - -ROOT = Path(sys.argv[1]) -INPUT = Path(sys.argv[2]) -REQUESTED_BASE = sys.argv[3] -OUTPUT = Path(sys.argv[4]) -FREEBSD_SRC = Path(sys.argv[5]) -UTILS_SRC = Path(sys.argv[6]) -QEMU_BASE = Path(sys.argv[7]).resolve() -SPEC = json.loads(INPUT.read_text(encoding="ascii")) - -SUPER = 1024 -MAGIC = 0xE0F5E1E2 -CRC32C_POLY = 0x82F63B78 -S_IFMT = 0o170000 -S_IFDIR = 0o040000 -NLINK_ONE_BIT = 4 - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_path(path: Path) -> str: - return sha256_bytes(path.read_bytes()) - - -def write_json(path: Path, value: Any) -> None: - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def run(argv: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: - return subprocess.run( - argv, - check=False, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=env, - ) - - -def git(root: Path, *args: str) -> str: - return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip() - - -def source_at(commit: str, path: str) -> str: - completed = run(["git", "-C", str(ROOT), "show", f"{commit}:{path}"]) - if completed.returncode != 0: - raise SystemExit(f"cannot read {path} at {commit}: {completed.stdout}") - return completed.stdout - - -def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int: - value = seed - for byte in data: - value ^= byte - for _ in range(8): - value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0) - return value & 0xFFFFFFFF - - -class Image: - def __init__(self, data: bytes | bytearray): - self.data = bytearray(data) - if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC: - raise ValueError("invalid EROFS image") - self.block_bits = self.data[SUPER + 12] - self.block_size = 1 << self.block_bits - self.meta_blkaddr = self.u32(SUPER + 40) - self.root_nid = self.u16(SUPER + 14) - if self.root_nid == 0: - self.root_nid = self.u64(SUPER + 112) - if self.checksum_end > len(self.data): - raise ValueError("truncated checksummed range") - - @property - def checksum_end(self) -> int: - span = self.block_size - SUPER if self.block_size > SUPER else self.block_size - return SUPER + span - - def clone(self) -> "Image": - return Image(self.data) - - def u16(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - struct.pack_into(" None: - struct.pack_into(" None: - struct.pack_into(" bool: - expected = self.u32(SUPER + 4) - canonical = bytearray(self.data[SUPER : self.checksum_end]) - struct.pack_into(" None: - self.put_u32(SUPER + 4, 0) - self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.checksum_end])) - if not self.checksum_valid(): - raise AssertionError("checksum update failed") - - def inode(self, nid: int) -> dict[str, int]: - offset = (self.meta_blkaddr << self.block_bits) + (nid << 5) - ifmt = self.u16(offset) - version = ifmt & 1 - inode_size = 32 if version == 0 else 64 - if offset + inode_size > len(self.data): - raise ValueError(f"inode {nid} is outside the image") - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + (xattr_count - 1) * 4 - mode = self.u16(offset + 4) - raw_nlink = self.u16(offset + 6) if version == 0 else self.u32(offset + 44) - nlink_one = version == 0 and mode & S_IFMT != S_IFDIR and bool( - ifmt & (1 << NLINK_ONE_BIT) - ) - size = self.u32(offset + 8) if version == 0 else self.u64(offset + 8) - return { - "nid": nid, - "offset": offset, - "ifmt": ifmt, - "version": version, - "inode_size": inode_size, - "xattr_size": xattr_size, - "layout": (ifmt >> 1) & 7, - "mode": mode, - "raw_nlink": raw_nlink, - "nlink": 1 if nlink_one else raw_nlink, - "nlink_one": int(nlink_one), - "size": size, - "startblk": self.u32(offset + 16), - } - - def set_nlink(self, nid: int, value: int) -> dict[str, int]: - inode = self.inode(nid) - if inode["version"] == 0: - if inode["mode"] & S_IFMT != S_IFDIR: - self.put_u16(inode["offset"], inode["ifmt"] & ~(1 << NLINK_ONE_BIT)) - self.put_u16(inode["offset"] + 6, value) - else: - self.put_u32(inode["offset"] + 44, value) - return self.inode(nid) - - def data_offset(self, inode: dict[str, int]) -> int: - if inode["layout"] == 2: - return inode["offset"] + inode["inode_size"] + inode["xattr_size"] - if inode["layout"] == 0: - return inode["startblk"] << self.block_bits - raise ValueError(f"unsupported directory layout {inode['layout']}") - - def directory(self, nid: int) -> dict[str, dict[str, int]]: - inode = self.inode(nid) - if inode["mode"] & S_IFMT != S_IFDIR: - raise ValueError(f"inode {nid} is not a directory") - data_offset = self.data_offset(inode) - size = inode["size"] - first_nameoff = self.u16(data_offset + 8) - if first_nameoff == 0 or first_nameoff % 12 != 0 or first_nameoff >= size: - raise ValueError("invalid first name offset") - count = first_nameoff // 12 - records: dict[str, dict[str, int]] = {} - for index in range(count): - entry = data_offset + index * 12 - nameoff = self.u16(entry + 8) - endoff = self.u16(entry + 20) if index + 1 < count else size - raw = bytes(self.data[data_offset + nameoff : data_offset + endoff]) - name = raw.split(b"\0", 1)[0].decode("ascii") - if not name or name in records: - raise ValueError(f"bad directory name {name!r}") - records[name] = { - "entry_offset": entry, - "nid": self.u64(entry), - "file_type": self.data[entry + 10], - } - return records - - -def make_source(path: Path, extended_uid: bool) -> None: - previous_umask = os.umask(0o022) - try: - path.mkdir(mode=0o755) - (path / "compact").write_bytes(b"compact zero-nlink target\n") - (path / "extended").write_bytes(b"extended zero-nlink target\n") - if extended_uid: - os.chown(path / "extended", 70000, 0) - (path / "hard-a").write_bytes(b"hardlink control\n") - os.link(path / "hard-a", path / "hard-b") - (path / "orphan").write_bytes(b"orphan raw-vget control\n") - (path / "subdir").mkdir(mode=0o755) - (path / "subdir" / "child").write_bytes(b"directory child\n") - for regular in ("compact", "extended", "hard-a", "hard-b", "orphan"): - os.chmod(path / regular, 0o644) - os.chmod(path / "subdir" / "child", 0o644) - finally: - os.umask(previous_umask) - - -def build_seed(work: Path, output: Path, inode_mode: str) -> tuple[bytes, str, list[str]]: - work.mkdir() - source = work / "source" - make_source(source, inode_mode == "extended") - command = [ - "mkfs.erofs", - "-d0", - "-T0", - "--all-time", - "--ignore-mtime", - "--workers=1", - "-x-1", - "-E", - "noinline_data", - "-E", - f"force-inode-{inode_mode}", - "-U", - SPEC["uuid"], - str(output), - str(source), - ] - env = dict(os.environ) - env.pop("SOURCE_DATE_EPOCH", None) - completed = run(command, env=env) - if completed.returncode != 0: - raise SystemExit(f"mkfs.erofs failed: {completed.stdout}") - canonical = command[:-2] + [f"SEED-{inode_mode}.erofs", "SOURCE"] - return output.read_bytes(), completed.stdout, canonical - - -resolved = git(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}") -if resolved != SPEC["required_base"]: - raise SystemExit(f"wrong BASE: expected {SPEC['required_base']}, got {resolved}") -if git(FREEBSD_SRC, "rev-parse", "HEAD") != SPEC["freebsd_head"]: - raise SystemExit("FreeBSD source HEAD changed") -if git(UTILS_SRC, "rev-parse", "HEAD") != SPEC["utils_head"]: - raise SystemExit("erofs-utils source HEAD changed") -if QEMU_BASE == Path(SPEC["protected"]["base_image"]): - raise SystemExit("refusing the protected base bp") -if SPEC["protected"]["port"] >= 32768: - raise SystemExit("protected port unexpectedly overlaps ephemeral ports") - -sources: dict[str, str] = {} -source_hashes: dict[str, str] = {} -for path, expected in SPEC["source_sha256"].items(): - content = source_at(resolved, path) - actual = sha256_bytes(content.encode("utf-8")) - if actual != expected: - raise SystemExit(f"source hash changed for {path}: {actual}") - sources[path] = content - source_hashes[path] = actual -for path, expected in SPEC["freebsd_sha256"].items(): - actual = sha256_path(FREEBSD_SRC / path) - if actual != expected: - raise SystemExit(f"FreeBSD hash changed for {path}: {actual}") -for path, expected in SPEC["utils_sha256"].items(): - actual = sha256_path(UTILS_SRC / path) - if actual != expected: - raise SystemExit(f"erofs-utils hash changed for {path}: {actual}") - -mkfs_version = run(["mkfs.erofs", "-V"]).stdout.splitlines()[0] -if mkfs_version != SPEC["mkfs_version"]: - raise SystemExit(f"mkfs version changed: {mkfs_version!r}") - -namei = sources["repo-pre-15/src/namei.c"] -inode_source = sources["repo-pre-15/src/inode.c"] -vnops = sources["repo-pre-15/src/erofs_vnops.c"] -super_source = sources["repo-pre-15/src/super.c"] -linux_inode = sources["src-linux/inode.c"] -linux_super = sources["src-linux/super.c"] -freebsd_vm = (FREEBSD_SRC / "sys/vm/vm_mmap.c").read_text(encoding="utf-8") - -source_anchors = { - "freebsd_disk_compact_nlink": "vi->nlink = le16toh(dic->i_nb.nlink);" in inode_source, - "freebsd_disk_extended_nlink": "vi->nlink = le32toh(die->i_nlink);" in inode_source, - "freebsd_inode_decode_has_no_zero_reject": "vi->nlink == 0" not in inode_source, - "freebsd_lookup_has_namespace_edge": "error = erofs_namei(sbi, dir, &qname, &nid, &dtype);" in namei, - "freebsd_lookup_calls_raw_vget": "error = erofs_vget(dvp->v_mount, nid, cnp->cn_lkflags, &vp);" in namei, - "freebsd_lookup_publishes_namecache": "cache_enter(dvp, vp, cnp);" in namei, - "freebsd_raw_vget_registered": ".vfs_vget = erofs_vget," in super_source, - "freebsd_root_uses_raw_vget": "error = erofs_vget(mp, MTOE(mp)->root_nid, flags, vpp);" in super_source, - "freebsd_fhtovp_filters_zero": "vi->mode == 0 || vi->nlink == 0 || vi->nid != nid" in super_source, - "freebsd_hash_insert_before_decode": vnops.index("error = vfs_hash_insert(") < vnops.index("error = erofs_read_inode("), - "freebsd_constructed_before_return": vnops.index("vn_set_state(vp, VSTATE_CONSTRUCTED);") < vnops.index("*vpp = vp;"), - "freebsd_vget_has_no_hit_indicator": "bool shared;" in vnops and "bool created" not in vnops, - "freebsd_vput_does_not_imply_vgone": "vgone(vp);\n\t\tvput(vp);" in vnops, - "freebsd_vfs_allows_unlinked_vnode": "if (va.va_nlink == 0)" in freebsd_vm, - "linux_compact_sets_disk_nlink": "set_nlink(inode, le16_to_cpu(dic->i_nb.nlink));" in linux_inode, - "linux_extended_sets_disk_nlink": "set_nlink(inode, le32_to_cpu(die->i_nlink));" in linux_inode, - "linux_inode_has_no_zero_reject": "i_nlink == 0" not in linux_inode, - "linux_super_has_no_zero_reject": "i_nlink == 0" not in linux_super, -} -if not all(source_anchors.values()): - raise SystemExit(f"source anchors changed: {source_anchors}") - -fixtures = OUTPUT / "fixtures" -fixtures.mkdir() -with tempfile.TemporaryDirectory(prefix="p15-092-gate-") as temporary: - temp = Path(temporary) - first_bytes, first_stdout, compact_command = build_seed( - temp / "compact-first", temp / "compact-first.erofs", "compact" - ) - second_bytes, second_stdout, compact_repeat_command = build_seed( - temp / "compact-second", temp / "compact-second.erofs", "compact" - ) - extended_bytes, extended_stdout, extended_command = build_seed( - temp / "extended-first", temp / "extended-first.erofs", "extended" - ) - extended_repeat_bytes, extended_repeat_stdout, extended_repeat_command = build_seed( - temp / "extended-second", temp / "extended-second.erofs", "extended" - ) - write_json( - OUTPUT / "generator-attempts.json", - { - "commands": [compact_command, extended_command], - "compact_source": [ - { - "gid": path.lstat().st_gid, - "mode": stat.S_IMODE(path.lstat().st_mode), - "path": str(path.relative_to(temp / "compact-first/source")), - "uid": path.lstat().st_uid, - } - for path in sorted((temp / "compact-first/source").rglob("*")) - ], - }, - ) - if ( - first_bytes != second_bytes - or first_stdout != second_stdout - or compact_command != compact_repeat_command - or extended_bytes != extended_repeat_bytes - or extended_stdout != extended_repeat_stdout - or extended_command != extended_repeat_command - ): - raise SystemExit("seed generation is not byte reproducible") - seed_path = fixtures / "seed.erofs" - seed_path.write_bytes(first_bytes) - extended_seed_path = fixtures / "seed-extended.erofs" - extended_seed_path.write_bytes(extended_bytes) - seed = Image(first_bytes) - extended_seed = Image(extended_bytes) - if not seed.checksum_valid() or not extended_seed.checksum_valid(): - raise SystemExit("seed checksum is invalid") - entries = seed.directory(seed.root_nid) - required = {".", "..", "compact", "extended", "hard-a", "hard-b", "orphan", "subdir"} - if not required.issubset(entries): - raise SystemExit(f"seed entries missing: {sorted(required - entries.keys())}") - if entries["hard-a"]["nid"] != entries["hard-b"]["nid"]: - raise SystemExit("hardlink names do not share one NID") - - extended_entries = extended_seed.directory(extended_seed.root_nid) - seed_inodes = {name: seed.inode(record["nid"]) for name, record in entries.items()} - extended_inodes = { - name: extended_seed.inode(record["nid"]) - for name, record in extended_entries.items() - } - if seed_inodes["compact"]["version"] != 0: - raise SystemExit("compact control is not compact") - if extended_inodes["extended"]["version"] != 1: - raise SystemExit("extended control is not extended") - if seed_inodes["hard-a"]["nlink"] != 2: - raise SystemExit("hardlink control does not have nlink 2") - - mutations = [] - - def emit_zero(case_id: str, name: str, source: Image = seed) -> None: - image = source.clone() - nid = image.directory(image.root_nid)[name]["nid"] - before = image.inode(nid) - after = image.set_nlink(nid, 0) - image.update_checksum() - path = fixtures / f"{case_id}.erofs" - path.write_bytes(image.data) - mutations.append({ - "case": case_id, - "class": "namespace-reachable-zero", - "name": name, - "nid": nid, - "before": before, - "after": after, - "fixture": path.name, - }) - - emit_zero("reachable-compact-zero", "compact") - emit_zero("reachable-extended-zero", "extended", extended_seed) - emit_zero("reachable-directory-zero", "subdir") - - root_zero = seed.clone() - root_before = root_zero.inode(root_zero.root_nid) - root_after = root_zero.set_nlink(root_zero.root_nid, 0) - root_zero.update_checksum() - root_path = fixtures / "root-zero.erofs" - root_path.write_bytes(root_zero.data) - mutations.append({ - "case": "root-zero", - "class": "raw-root-zero", - "name": "/", - "nid": root_zero.root_nid, - "before": root_before, - "after": root_after, - "fixture": root_path.name, - }) - - orphan = seed.clone() - orphan_entries = orphan.directory(orphan.root_nid) - orphan_nid = orphan_entries["orphan"]["nid"] - hard_nid = orphan_entries["hard-a"]["nid"] - orphan.put_u64(orphan_entries["orphan"]["entry_offset"], hard_nid) - orphan.set_nlink(orphan_nid, 0) - orphan.set_nlink(hard_nid, 3) - orphan.update_checksum() - orphan_path = fixtures / "orphan-zero.erofs" - orphan_path.write_bytes(orphan.data) - orphan_names = orphan.directory(orphan.root_nid) - if any(record["nid"] == orphan_nid for record in orphan_names.values()): - raise SystemExit("orphan fixture still has a namespace edge") - mutations.append({ - "case": "orphan-zero", - "class": "unreachable-zero", - "name": None, - "nid": orphan_nid, - "before": seed.inode(orphan_nid), - "after": orphan.inode(orphan_nid), - "replacement_nid": hard_nid, - "replacement_nlink": orphan.inode(hard_nid)["nlink"], - "fixture": orphan_path.name, - }) - - records = [] - fixture_digest = hashlib.sha256() - for path in sorted(fixtures.glob("*.erofs")): - image = Image(path.read_bytes()) - if not image.checksum_valid(): - raise SystemExit(f"fixture checksum invalid: {path.name}") - fsck = run(["fsck.erofs", "-d0", str(path)]) - dump_root = run(["dump.erofs", "--path=/", str(path)]) - if dump_root.returncode != 0: - raise SystemExit(f"dump root failed: {path.name}") - normal = {} - for name in ("compact", "extended", "hard-a", "hard-b", "orphan", "subdir"): - completed = run(["dump.erofs", f"--path=/{name}", str(path)]) - normal[name] = { - "exit": completed.returncode, - "nid": int(match.group(1)) if (match := re.search(r"^NID: (\d+)\b", completed.stdout, re.MULTILINE)) else None, - } - record = { - "fixture": path.name, - "sha256": sha256_path(path), - "size": path.stat().st_size, - "checksum_valid": True, - "fsck_exit": fsck.returncode, - "fsck_error_marker": "" in fsck.stdout, - "normal_namespace": normal, - } - records.append(record) - fixture_digest.update(path.name.encode("ascii")) - fixture_digest.update(b"\0") - fixture_digest.update(path.read_bytes()) - - write_json( - OUTPUT / "generator.json", - { - "commands": [compact_command, extended_command], - "mkfs_version": mkfs_version, - "repeat_byte_identical": True, - "seed_sha256": { - "compact": sha256_bytes(first_bytes), - "extended": sha256_bytes(extended_bytes), - }, - }, - ) - (OUTPUT / "mkfs.stdout").write_text( - first_stdout + extended_stdout, encoding="utf-8" - ) - write_json(OUTPUT / "disk-records.json", {"mutations": mutations, "records": records}) - write_json( - OUTPUT / "fixture-index.json", - { - "fixture_count": len(records), - "fixture_set_sha256": fixture_digest.hexdigest(), - "fixtures": records, - }, - ) - -fsck_reachable_zero_rejects = [] -for record in records: - if record["fixture"].startswith("reachable-"): - fsck_reachable_zero_rejects.append( - record["fsck_exit"] != 0 or record["fsck_error_marker"] - ) - -boundary = { - "b15_write_set": SPEC["write_set"], - "vnode_constructor_owner": "repo-pre-15/src/erofs_vnops.c", - "constructor_inside_b15_write_set": "repo-pre-15/src/erofs_vnops.c" in SPEC["write_set"], - "post_vget_check_is_before_constructed_publication": False, - "vput_only_performs_vgone_cleanup": False, - "post_vget_vgone_preserves_cached_raw_vget": False, - "pre_vget_full_decode_avoids_duplicate_inode_decode": False, - "mount_wide_scan_required": False, - "namespace_and_raw_entrypoints_distinct": True, -} -format_semantics = { - "mkfs_emits_nonzero_namespace_links": all( - seed_inodes[name]["nlink"] > 0 - for name in ("compact", "extended", "hard-a", "hard-b", "orphan", "subdir", ".") - ), - "fsck_rejects_all_reachable_zero": all(fsck_reachable_zero_rejects), - "linux_rejects_zero_during_inode_decode": False, - "linux_copies_disk_nlink_into_inode": True, - "freebsd_vfs_permits_live_unlinked_vnode": True, - "freebsd_fhtovp_treats_zero_as_stale": True, -} -go_requirements = { - "format_or_cross_kernel_rule_requires_rejection": ( - format_semantics["fsck_rejects_all_reachable_zero"] - and format_semantics["linux_rejects_zero_during_inode_decode"] - ), - "reject_before_vnode_publication_within_write_set": ( - boundary["constructor_inside_b15_write_set"] - and boundary["post_vget_check_is_before_constructed_publication"] - ), - "cleanup_preserves_raw_vget": boundary["post_vget_vgone_preserves_cached_raw_vget"], - "no_duplicate_full_inode_decode": boundary["pre_vget_full_decode_avoids_duplicate_inode_decode"], - "orphan_needs_no_mount_scan": not boundary["mount_wide_scan_required"], - "namespace_raw_split_proven": boundary["namespace_and_raw_entrypoints_distinct"], -} -status = "GO" if all(go_requirements.values()) else "STOP" -reasons = [key for key, value in go_requirements.items() if not value] -write_json(OUTPUT / "source-anchors.json", source_anchors) -write_json(OUTPUT / "source-sha256.json", source_hashes) -write_json(OUTPUT / "boundary.json", boundary) -write_json(OUTPUT / "format-semantics.json", format_semantics) -write_json( - OUTPUT / "host-result.json", - { - "candidate": SPEC["candidate"], - "gate": SPEC["gate"], - "fixture_count": len(records), - "fixture_set_sha256": fixture_digest.hexdigest(), - "go_requirements": go_requirements, - "reasons": reasons, - "requested_base": REQUESTED_BASE, - "resolved_base": resolved, - "schema": 1, - "status": status, - }, -) -PY - -if test "$host_only" -eq 1; then - cat "$output/host/host-result.json" - exit 22 -fi - -runtime_tmp=$(mktemp -d "${TMPDIR:-/tmp}/p15-092-runtime.XXXXXX") -runtime_dut=$runtime_tmp/repo-pre-15 -runtime_case=$runtime_dut/tests/pre15/cases/P15-092-gate-runtime.sh -runtime_cleanup() -{ - rm -rf "$runtime_tmp" -} -trap runtime_cleanup EXIT HUP INT TERM -mkdir -p "$runtime_dut/tests/pre15/cases" "$runtime_dut/tests/pre15/fixtures" -git -C "$root" archive --format=tar --output="$runtime_tmp/frozen-src.tar" \ - "$base" repo-pre-15/src -tar -C "$runtime_tmp" -xf "$runtime_tmp/frozen-src.tar" -ln -s "$dut/tests/pre15/fixtures/B14-build-kld.sh" \ - "$runtime_dut/tests/pre15/fixtures/B14-build-kld.sh" - -python3 -B - "$runtime_case" <<'PY' -from pathlib import Path -import sys - -Path(sys.argv[1]).write_text(r'''#!/bin/sh -set -eu - -: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" -: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}" -: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}" -: "${P15_092_GATE_DUT:?P15_092_GATE_DUT is required}" -: "${P15_092_GATE_HOST:?P15_092_GATE_HOST is required}" -. "$PRE15_LIB_DIR/runner.sh" - -artifacts=$PRE15_RUN_DIR/artifacts -fixtures=$P15_092_GATE_HOST/fixtures -builder=$P15_092_GATE_DUT/tests/pre15/fixtures/B14-build-kld.sh -module=$PRE15_CASE_TMP/P15-092-baseline-erofs.ko -archive=$PRE15_CASE_TMP/P15-092-fixtures.tar.gz -mkdir -p "$artifacts" - -pre15_record_fixture p15-092-host-result "$P15_092_GATE_HOST/host-result.json" -pre15_record_fixture p15-092-disk-records "$P15_092_GATE_HOST/disk-records.json" -pre15_record_fixture p15-092-boundary "$P15_092_GATE_HOST/boundary.json" -pre15_record_fixture p15-092-source-namei "$P15_092_GATE_DUT/src/namei.c" -pre15_record_fixture p15-092-source-inode "$P15_092_GATE_DUT/src/inode.c" -pre15_record_fixture p15-092-source-vnops "$P15_092_GATE_DUT/src/erofs_vnops.c" - -if ! /bin/sh "$builder" "$P15_092_GATE_DUT" "$PRE15_FREEBSD_SRC" "$module" \ - "$PRE15_CASE_TMP/kld-work" >"$artifacts/kld-build.stdout" \ - 2>"$artifacts/kld-build.stderr"; then - pre15_dut_fail 'baseline cross-target KLD build failed' -fi -pre15_record_module "$module" -tar -C "$fixtures" -czf "$archive" . - -pre15_scp() -{ - timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \ - "$1" "$PRE15_QEMU_SSH_USER@127.0.0.1:$2" -} - -pre15_scp "$module" /root/P15-092-baseline-erofs.ko || \ - pre15_infra_blocked 'could not transfer baseline KLD' -pre15_scp "$archive" /root/P15-092-fixtures.tar.gz || \ - pre15_infra_blocked 'could not transfer zero-nlink fixtures' -pre15_guest_ssh_bounded \ - 'rm -rf /root/P15-092-fixtures && mkdir /root/P15-092-fixtures && tar -xzf /root/P15-092-fixtures.tar.gz -C /root/P15-092-fixtures' || \ - pre15_infra_blocked 'could not prepare guest fixtures' -if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then - pre15_infra_blocked 'guest already has an EROFS module loaded' -fi -pre15_guest_ssh_bounded kldload /root/P15-092-baseline-erofs.ko || \ - pre15_dut_fail 'baseline KLD failed to load' -pre15_own_guest_kld erofs 'P15-092 baseline KLD' -pre15_target_reached - -attach_mount() -{ - image=$1 - mountpoint=$2 - label=$3 - md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \ - -f "/root/P15-092-fixtures/$image") || \ - pre15_dut_fail "$label md attach failed" - case "$md" in - md[0-9]*) ;; - *) pre15_runner_fail "unexpected md unit: $md" ;; - esac - pre15_own_guest_md "$md" "$label md" - pre15_guest_ssh_bounded mkdir -p "$mountpoint" - pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$md" "$mountpoint" || \ - pre15_dut_fail "$label mount failed" - pre15_own_guest_mount "$mountpoint" "$label mount" -} - -expect_nlink() -{ - path=$1 - expected=$2 - label=$3 - actual=$(pre15_guest_ssh_bounded stat -f '%l' "$path") || \ - pre15_dut_fail "$label normal stat failed" - test "$actual" = "$expected" || \ - pre15_dut_fail "$label nlink expected $expected got $actual" - printf '%s\t%s\t%s\n' "$label" "$path" "$actual" >>"$artifacts/normal-vnode.tsv" -} - -attach_mount seed.erofs /mnt/p15-092-seed seed -expect_nlink /mnt/p15-092-seed/compact 1 seed-compact -expect_nlink /mnt/p15-092-seed/extended 1 seed-extended -expect_nlink /mnt/p15-092-seed/hard-a 2 seed-hard-a -hard_a=$(pre15_guest_ssh_bounded stat -f '%i' /mnt/p15-092-seed/hard-a) -hard_b=$(pre15_guest_ssh_bounded stat -f '%i' /mnt/p15-092-seed/hard-b) -test "$hard_a" = "$hard_b" || pre15_dut_fail 'hardlink NIDs differ' - -attach_mount reachable-compact-zero.erofs /mnt/p15-092-compact compact-zero -expect_nlink /mnt/p15-092-compact/compact 0 reachable-compact-zero -attach_mount reachable-extended-zero.erofs /mnt/p15-092-extended extended-zero -expect_nlink /mnt/p15-092-extended/extended 0 reachable-extended-zero -attach_mount reachable-directory-zero.erofs /mnt/p15-092-directory directory-zero -expect_nlink /mnt/p15-092-directory/subdir 0 reachable-directory-zero -attach_mount root-zero.erofs /mnt/p15-092-root root-zero -expect_nlink /mnt/p15-092-root 0 root-zero -attach_mount orphan-zero.erofs /mnt/p15-092-orphan orphan-zero -expect_nlink /mnt/p15-092-orphan/orphan 3 orphan-replacement -pre15_guest_ssh_bounded cat /mnt/p15-092-orphan/orphan >"$artifacts/orphan-replacement.txt" -pre15_guest_ssh_bounded dmesg >"$artifacts/dmesg.txt" -printf '%s\n' 'P15-092 baseline normal vnode runtime PASS' -''', encoding="ascii") -PY -chmod 0555 "$runtime_case" - -qemu_stdout=$output/qemu.stdout -qemu_stderr=$output/qemu.stderr -qemu_evidence=$output/qemu-evidence -mkdir "$qemu_evidence" -if PRE15_DUT="$runtime_dut" PRE15_ROOT="$root" \ - PRE15_SCHEMA="$dut/tests/pre15/EVIDENCE-SCHEMA.json" \ - PRE15_EVIDENCE_ROOT="$qemu_evidence" PRE15_FREEBSD_SRC="$freebsd_src" \ - PRE15_QEMU_BASE_IMAGE="$qemu_base" PRE15_QEMU_BASE_FORMAT=qcow2 \ - PRE15_QEMU_SSH_KEY="$qemu_key" PRE15_QEMU_SSH_USER=root \ - PRE15_QEMU_MEMORY_MB=2048 PRE15_QEMU_CPUS=2 PRE15_QEMU_BOOT_TIMEOUT=180 \ - PRE15_QEMU_TIMEOUT=900 PRE15_GUEST_COMMAND_TIMEOUT=60 \ - P15_092_GATE_DUT="$runtime_dut" P15_092_GATE_HOST="$output/host" \ - timeout -k 30 1000 "$dut/tests/pre15/run-qemu.sh" P15-092-gate-runtime \ - >"$qemu_stdout" 2>"$qemu_stderr"; then - qemu_rc=0 -else - qemu_rc=$? -fi -printf '%s\n' "$qemu_rc" >"$output/qemu.exit" - -python3 -B - "$output" "$qemu_rc" <<'PY' -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -import sys - - -OUTPUT = Path(sys.argv[1]) -QEMU_RC = int(sys.argv[2]) - - -def sha256_path(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def write_json(path: Path, value: object) -> None: - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -host = json.loads((OUTPUT / "host/host-result.json").read_text(encoding="ascii")) -stdout = (OUTPUT / "qemu.stdout").read_text(encoding="utf-8") -match = re.search(r"PRE15_RESULT status=(\S+) cleanup=(\S+) run_id=(\S+) evidence=(\S+)", stdout) -if match is None: - runtime = { - "cleanup": "UNKNOWN", - "evidence": None, - "exit": QEMU_RC, - "run_id": None, - "status": "RUNNER_FAIL", - } -else: - runtime = { - "status": match.group(1), - "cleanup": match.group(2), - "run_id": match.group(3), - "evidence": match.group(4), - "exit": QEMU_RC, - } - -status = "STOP" -reasons = list(host["reasons"]) -if runtime["status"] != "PASS" or runtime["cleanup"] != "PASS": - reasons.append("normal_vnode_runtime_not_pass") -result = { - "b15": "STOP-NO-SOURCE", - "candidate": "P15-092", - "full_feature_suite": "NOT_RUN", - "gate": "G11", - "host_status": host["status"], - "qemu": runtime, - "reasons": reasons, - "schema": 1, - "status": status, -} -write_json(OUTPUT / "result.json", result) - -hash_lines = [] -for path in sorted(OUTPUT.rglob("*")): - if path.is_file() and path.name != "SHA256SUMS": - hash_lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}") -(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii") -print(json.dumps(result, sort_keys=True)) -PY - -cat "$output/result.json" -exit 22 diff --git a/tests/pre15/lib/manifest.sh b/tests/pre15/lib/manifest.sh deleted file mode 100644 index 2e40a74..0000000 --- a/tests/pre15/lib/manifest.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/sh - -pre15_write_argv_json() -{ - pre15_argv_path=$1 - shift - python3 - "$pre15_argv_path" "$@" <<'PY' -import json -from pathlib import Path -import sys - -path = Path(sys.argv[1]) -with path.open("x", encoding="ascii") as output: - json.dump(sys.argv[2:], output, ensure_ascii=True, separators=(",", ":")) - output.write("\n") -PY -} - -pre15_write_manifest() -{ - python3 - "$PRE15_MANIFEST_PATH" <<'PY' -from __future__ import annotations - -import hashlib -import json -import os -from pathlib import Path -import sys - - -def digest(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def relative(path: Path, root: Path) -> str: - return path.relative_to(root).as_posix() - - -manifest_path = Path(sys.argv[1]) -run_dir = manifest_path.parent -argv_path = Path(os.environ["PRE15_ARGV_PATH"]) -fixture_path = Path(os.environ["PRE15_FIXTURE_HASHES"]) -module_path = Path(os.environ["PRE15_MODULE_HASH"]) -stdout_path = Path(os.environ["PRE15_STDOUT_PATH"]) -stderr_path = Path(os.environ["PRE15_STDERR_PATH"]) -ownership_path = Path(os.environ["PRE15_OWNERSHIP_FILE"]) -cleanup_log = Path(os.environ["PRE15_CLEANUP_LOG"]) -identity_path = Path(os.environ["PRE15_IDENTITY_PATH"]) - -fixtures = [] -fixture_details = [] -if fixture_path.exists(): - for raw_line in fixture_path.read_text(encoding="utf-8").splitlines(): - if not raw_line: - continue - checksum, label, path = raw_line.split("\t", 2) - fixtures.append(checksum) - fixture_details.append( - {"sha256": checksum, "label": label, "path": path} - ) - -module_sha256 = None -module_details = None -if module_path.exists() and module_path.stat().st_size: - checksum, path = module_path.read_text(encoding="utf-8").rstrip("\n").split( - "\t", 1 - ) - module_sha256 = checksum - module_details = {"sha256": checksum, "path": path} - -exit_code_text = os.environ["PRE15_EXIT_CODE"] -exit_code = None if exit_code_text == "null" else int(exit_code_text) -identity = json.loads(identity_path.read_text(encoding="ascii")) -manifest = { - "schema_version": 1, - "run_id": os.environ["PRE15_RUN_ID"], - "mode": os.environ["PRE15_MODE"], - "case_id": os.environ["PRE15_CASE_ID"], - "dut_head": os.environ["PRE15_DUT_HEAD"], - "dut_tree": os.environ["PRE15_DUT_TREE"], - "worktree_diff_sha256": os.environ["PRE15_WORKTREE_DIFF_SHA256"], - "freebsd_source": os.environ["PRE15_FREEBSD_SOURCE"], - "linux_source": os.environ["PRE15_LINUX_SOURCE"], - "module_sha256": module_sha256, - "fixture_sha256": fixtures, - "command_argv": json.loads(argv_path.read_text(encoding="ascii")), - "start_utc": os.environ["PRE15_START_UTC"], - "end_utc": os.environ["PRE15_END_UTC"], - "deadline_seconds": int(os.environ["PRE15_DEADLINE"]), - "exit_code": exit_code, - "timed_out": os.environ["PRE15_TIMED_OUT"] == "1", - "target_marker": os.environ["PRE15_TARGET_STATE"], - "status": os.environ["PRE15_STATUS"], - "reason": os.environ["PRE15_REASON"], - "failure_origin": os.environ["PRE15_FAILURE_ORIGIN"], - "cleanup": os.environ["PRE15_CLEANUP_STATUS"], - "stdout": relative(stdout_path, run_dir), - "stderr": relative(stderr_path, run_dir), - "raw_sha256": { - "stdout": digest(stdout_path), - "stderr": digest(stderr_path), - "ownership": digest(ownership_path), - "cleanup": digest(cleanup_log), - }, - "ownership_manifest": relative(ownership_path, run_dir), - "cleanup_log": relative(cleanup_log, run_dir), - "identity": identity, - "fixtures": fixture_details, - "module": module_details, -} -with manifest_path.open("x", encoding="ascii") as output: - json.dump(manifest, output, ensure_ascii=True, indent=2, sort_keys=True) - output.write("\n") -PY -} - -pre15_validate_manifest() -{ - pre15_manifest_to_validate=$1 - pre15_schema_to_validate=$2 - python3 - "$pre15_manifest_to_validate" "$pre15_schema_to_validate" <<'PY' -from __future__ import annotations - -from datetime import datetime -import hashlib -import json -from pathlib import Path -import re -import sys - - -manifest_path = Path(sys.argv[1]) -schema_path = Path(sys.argv[2]) -manifest = json.loads(manifest_path.read_text(encoding="ascii")) -schema = json.loads(schema_path.read_text(encoding="ascii")) - -missing = sorted(set(schema["required"]) - set(manifest)) -if missing: - raise SystemExit(f"manifest missing required keys: {', '.join(missing)}") -if manifest["status"] not in schema["properties"]["status"]["enum"]: - raise SystemExit(f"invalid status: {manifest['status']}") -if manifest["cleanup"] not in schema["properties"]["cleanup"]["enum"]: - raise SystemExit(f"invalid cleanup: {manifest['cleanup']}") -if manifest["target_marker"] not in schema["properties"]["target_marker"]["enum"]: - raise SystemExit(f"invalid target marker: {manifest['target_marker']}") -if not re.fullmatch(r"[0-9a-f]{40}", manifest["dut_head"]): - raise SystemExit("dut_head is not a 40-hex commit") -for key in ("dut_tree", "worktree_diff_sha256"): - if not re.fullmatch(r"[0-9a-f]{40,64}", manifest[key]): - raise SystemExit(f"{key} is not a lowercase hex identity") -if manifest["module_sha256"] is not None and not re.fullmatch( - r"[0-9a-f]{64}", manifest["module_sha256"] -): - raise SystemExit("module_sha256 is neither null nor SHA256") -if not isinstance(manifest["command_argv"], list) or not manifest["command_argv"]: - raise SystemExit("command_argv must be a nonempty array") -if not all(isinstance(item, str) for item in manifest["command_argv"]): - raise SystemExit("command_argv contains a non-string item") -if not isinstance(manifest["fixture_sha256"], list) or not all( - re.fullmatch(r"[0-9a-f]{64}", item) for item in manifest["fixture_sha256"] -): - raise SystemExit("fixture_sha256 contains an invalid digest") -if not isinstance(manifest["deadline_seconds"], int) or manifest["deadline_seconds"] <= 0: - raise SystemExit("deadline_seconds must be positive") -if manifest["exit_code"] is not None and not isinstance(manifest["exit_code"], int): - raise SystemExit("exit_code must be an integer or null") -for key in ("start_utc", "end_utc"): - datetime.fromisoformat(manifest[key].replace("Z", "+00:00")) -if manifest["status"] == "PASS": - if manifest["exit_code"] != 0 or manifest["target_marker"] != "reached": - raise SystemExit("PASS requires exit_code=0 and a reached target") - if manifest["cleanup"] != "PASS": - raise SystemExit("PASS requires successful cleanup") -if manifest["status"] == "DUT_FAIL" and manifest["target_marker"] != "reached": - raise SystemExit("DUT_FAIL requires a reached target") -if manifest["cleanup"] == "FAIL" and manifest["status"] != "RUNNER_FAIL": - raise SystemExit("cleanup failure must classify as RUNNER_FAIL") - -run_dir = manifest_path.parent -for key in ("stdout", "stderr", "ownership_manifest", "cleanup_log"): - relative = Path(manifest[key]) - if relative.is_absolute() or ".." in relative.parts: - raise SystemExit(f"{key} escapes the run directory") - if not (run_dir / relative).is_file(): - raise SystemExit(f"{key} does not exist: {relative}") -hashed_paths = { - "stdout": manifest["stdout"], - "stderr": manifest["stderr"], - "ownership": manifest["ownership_manifest"], - "cleanup": manifest["cleanup_log"], -} -for key, relative_path in hashed_paths.items(): - actual = hashlib.sha256((run_dir / relative_path).read_bytes()).hexdigest() - if actual != manifest["raw_sha256"][key]: - raise SystemExit(f"{key} hash mismatch") -PY -} diff --git a/tests/pre15/lib/runner.sh b/tests/pre15/lib/runner.sh deleted file mode 100644 index 75aa236..0000000 --- a/tests/pre15/lib/runner.sh +++ /dev/null @@ -1,751 +0,0 @@ -#!/bin/sh - -PRE15_RC_DUT_FAIL=10 -PRE15_RC_RUNNER_FAIL=20 -PRE15_RC_INFRA_BLOCKED=21 -PRE15_RC_STOP=22 - -pre15_require_paths() -{ - : "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}" - PRE15_DUT=${PRE15_DUT:-$(CDPATH= cd -- "$PRE15_LIB_DIR/../../.." && pwd -P)} - PRE15_ROOT=${PRE15_ROOT:-$(CDPATH= cd -- "$PRE15_DUT/.." && pwd -P)} - PRE15_SCHEMA=${PRE15_SCHEMA:-$PRE15_DUT/tests/pre15/EVIDENCE-SCHEMA.json} - PRE15_FREEBSD_SRC=${PRE15_FREEBSD_SRC:-${FREEBSD_SRC:-/work/build/freebsd-src}} - PRE15_EVIDENCE_ROOT=${PRE15_EVIDENCE_ROOT:-${TMPDIR:-/tmp}/erofs-pre15-evidence} - export PRE15_DUT PRE15_ROOT PRE15_SCHEMA PRE15_FREEBSD_SRC PRE15_EVIDENCE_ROOT -} - -pre15_utc_now() -{ - date -u '+%Y-%m-%dT%H:%M:%SZ' -} - -pre15_tree_sha256() -{ - python3 - "$1" <<'PY' -from __future__ import annotations - -import hashlib -import os -from pathlib import Path -import stat -import sys - - -root = Path(sys.argv[1]) -if not root.is_dir(): - raise SystemExit(f"tree root is not a directory: {root}") - -digest = hashlib.sha256() -for current, directories, files in os.walk(root, followlinks=False): - directories.sort(key=os.fsencode) - files.sort(key=os.fsencode) - current_path = Path(current) - names = [(name, True) for name in directories] - names.extend((name, False) for name in files) - names.sort(key=lambda item: os.fsencode(item[0])) - for name, is_directory in names: - path = current_path / name - metadata = path.lstat() - relative = os.fsencode(path.relative_to(root)) - if stat.S_ISLNK(metadata.st_mode): - kind = b"L" - elif is_directory: - kind = b"D" - elif stat.S_ISREG(metadata.st_mode): - kind = b"F" - else: - raise SystemExit(f"unsupported tree entry: {path}") - digest.update(kind + b"\0" + relative + b"\0") - digest.update(f"{stat.S_IMODE(metadata.st_mode):04o}\0".encode("ascii")) - if kind == b"L": - digest.update(os.fsencode(os.readlink(path)) + b"\0") - elif kind == b"F": - digest.update(f"{metadata.st_size}\0".encode("ascii")) - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - digest.update(b"\0") - -print(digest.hexdigest()) -PY -} - -pre15_tree_metadata_sha256() -{ - python3 - "$1" <<'PY' -from __future__ import annotations - -import hashlib -import os -from pathlib import Path -import stat -import sys - - -root = Path(sys.argv[1]) -if not root.is_dir(): - raise SystemExit(f"tree root is not a directory: {root}") - -digest = hashlib.sha256() -for current, directories, files in os.walk(root, followlinks=False): - directories.sort(key=os.fsencode) - files.sort(key=os.fsencode) - current_path = Path(current) - names = [(name, True) for name in directories] - names.extend((name, False) for name in files) - names.sort(key=lambda item: os.fsencode(item[0])) - for name, is_directory in names: - path = current_path / name - metadata = path.lstat() - relative = os.fsencode(path.relative_to(root)) - if stat.S_ISLNK(metadata.st_mode): - kind = b"L" - elif is_directory: - kind = b"D" - elif stat.S_ISREG(metadata.st_mode): - kind = b"F" - else: - raise SystemExit(f"unsupported tree entry: {path}") - digest.update(kind + b"\0" + relative + b"\0") - fields = ( - stat.S_IMODE(metadata.st_mode), - metadata.st_size, - metadata.st_mtime_ns, - metadata.st_ctime_ns, - metadata.st_dev, - metadata.st_ino, - ) - digest.update(("\0".join(map(str, fields)) + "\0").encode("ascii")) - if kind == b"L": - digest.update(os.fsencode(os.readlink(path)) + b"\0") - -print(digest.hexdigest()) -PY -} - -pre15_fail_usage() -{ - printf 'RUNNER_FAIL: %s\n' "$*" >&2 - exit "$PRE15_RC_RUNNER_FAIL" -} - -pre15_target_reached() -{ - : "${PRE15_TARGET_MARKER_FILE:?target marker path is unavailable}" - printf 'reached\n' > "$PRE15_TARGET_MARKER_FILE" -} - -pre15_set_reason() -{ - : "${PRE15_REASON_FILE:?reason path is unavailable}" - printf '%s\n' "$*" > "$PRE15_REASON_FILE" -} - -pre15_dut_fail() -{ - pre15_set_reason "$*" - exit "$PRE15_RC_DUT_FAIL" -} - -pre15_runner_fail() -{ - pre15_set_reason "$*" - exit "$PRE15_RC_RUNNER_FAIL" -} - -pre15_infra_blocked() -{ - pre15_set_reason "$*" - exit "$PRE15_RC_INFRA_BLOCKED" -} - -pre15_gate_stop() -{ - pre15_set_reason "$*" - exit "$PRE15_RC_STOP" -} - -pre15_register_resource() -{ - : "${PRE15_OWNERSHIP_FILE:?ownership manifest is unavailable}" - pre15_resource_kind=$1 - pre15_resource_value=$2 - pre15_resource_label=$3 - case "$pre15_resource_kind$pre15_resource_value$pre15_resource_label" in - *" "*|*" -"*) pre15_runner_fail "resource records cannot contain tabs or newlines" ;; - esac - printf '%s\t%s\t%s\n' \ - "$pre15_resource_kind" "$pre15_resource_value" "$pre15_resource_label" \ - >> "$PRE15_OWNERSHIP_FILE" -} - -pre15_own_path() -{ - pre15_register_resource path "$1" "${2:-temporary path}" -} - -pre15_own_pid() -{ - case "$1" in - ''|*[!0-9]*) pre15_runner_fail "invalid owned PID: $1" ;; - esac - pre15_register_resource pid "$1" "${2:-process}" -} - -pre15_own_port() -{ - case "$1" in - ''|*[!0-9]*) pre15_runner_fail "invalid owned port: $1" ;; - esac - pre15_register_resource port "$1" "${2:-forwarded port}" -} - -pre15_own_ssh_control() -{ - pre15_register_resource ssh-control "$1" "${2:-SSH ControlMaster}" -} - -pre15_own_base_image() -{ - pre15_base_path=$1 - test -f "$pre15_base_path" || pre15_infra_blocked "base image is absent: $pre15_base_path" - pre15_base_hash=$(sha256sum "$pre15_base_path" | awk '{print $1}') - pre15_register_resource base-image "$pre15_base_path" "$pre15_base_hash" -} - -pre15_own_guest_mount() -{ - case "$1" in - /*) ;; - *) pre15_runner_fail "guest mountpoint must be absolute: $1" ;; - esac - pre15_register_resource guest-mount "$1" "${2:-guest mount}" -} - -pre15_own_guest_md() -{ - case "$1" in - md[0-9]*) ;; - *) pre15_runner_fail "invalid guest md unit: $1" ;; - esac - pre15_register_resource guest-md "$1" "${2:-guest md}" -} - -pre15_own_guest_kld() -{ - case "$1" in - [A-Za-z0-9_.-]*) ;; - *) pre15_runner_fail "invalid guest KLD name: $1" ;; - esac - pre15_register_resource guest-kld "$1" "${2:-guest KLD}" -} - -pre15_record_fixture() -{ - pre15_fixture_label=$1 - pre15_fixture_path=$2 - test -f "$pre15_fixture_path" || pre15_runner_fail \ - "fixture is absent: $pre15_fixture_path" - pre15_fixture_hash=$(sha256sum "$pre15_fixture_path" | awk '{print $1}') - printf '%s\t%s\t%s\n' \ - "$pre15_fixture_hash" "$pre15_fixture_label" "$pre15_fixture_path" \ - >> "$PRE15_FIXTURE_HASHES" -} - -pre15_record_module() -{ - pre15_module_path=$1 - test -f "$pre15_module_path" || pre15_runner_fail \ - "module is absent: $pre15_module_path" - pre15_module_digest=$(sha256sum "$pre15_module_path" | awk '{print $1}') - printf '%s\t%s\n' "$pre15_module_digest" "$pre15_module_path" \ - > "$PRE15_MODULE_HASH" -} - -pre15_port_is_free() -{ - python3 - "$1" <<'PY' -import socket -import sys - -port = int(sys.argv[1]) -with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.settimeout(0.2) - raise SystemExit(0 if sock.connect_ex(("127.0.0.1", port)) != 0 else 1) -PY -} - -pre15_guest_ssh() -{ - : "${PRE15_QEMU_SSH_PORT:?SSH port is unavailable}" - : "${PRE15_QEMU_SSH_USER:?SSH user is unavailable}" - : "${PRE15_QEMU_SSH_KEY:?SSH key is unavailable}" - ssh -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -i "$PRE15_QEMU_SSH_KEY" -p "$PRE15_QEMU_SSH_PORT" \ - "$PRE15_QEMU_SSH_USER@127.0.0.1" "$@" -} - -pre15_guest_ssh_bounded() -{ - pre15_guest_deadline=${PRE15_GUEST_COMMAND_TIMEOUT:-60} - case "$pre15_guest_deadline" in - ''|*[!0-9]*|0) pre15_runner_fail "invalid guest command deadline: $pre15_guest_deadline" ;; - esac - timeout -k 5 "$pre15_guest_deadline" ssh \ - -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -i "$PRE15_QEMU_SSH_KEY" -p "$PRE15_QEMU_SSH_PORT" \ - "$PRE15_QEMU_SSH_USER@127.0.0.1" "$@" -} - -pre15_cleanup_note() -{ - printf '%s\n' "$*" >> "$PRE15_CLEANUP_LOG" -} - -pre15_cleanup_fail() -{ - PRE15_CLEANUP_STATUS_RESULT=FAIL - pre15_cleanup_note "FAIL $*" -} - -pre15_cleanup_pid() -{ - pre15_cleanup_pid_value=$1 - pre15_cleanup_pid_label=$2 - if ! kill -0 "$pre15_cleanup_pid_value" 2>/dev/null; then - pre15_cleanup_note "PASS pid already exited: $pre15_cleanup_pid_value ($pre15_cleanup_pid_label)" - return - fi - if ! kill -TERM "$pre15_cleanup_pid_value" 2>/dev/null; then - pre15_cleanup_fail "could not TERM PID $pre15_cleanup_pid_value ($pre15_cleanup_pid_label)" - return - fi - pre15_cleanup_wait=0 - while kill -0 "$pre15_cleanup_pid_value" 2>/dev/null && \ - test "$pre15_cleanup_wait" -lt 50; do - sleep 0.1 - pre15_cleanup_wait=$((pre15_cleanup_wait + 1)) - done - if kill -0 "$pre15_cleanup_pid_value" 2>/dev/null; then - if ! kill -KILL "$pre15_cleanup_pid_value" 2>/dev/null; then - pre15_cleanup_fail "could not KILL PID $pre15_cleanup_pid_value ($pre15_cleanup_pid_label)" - return - fi - sleep 0.2 - fi - if kill -0 "$pre15_cleanup_pid_value" 2>/dev/null; then - pre15_cleanup_fail "PID survived cleanup: $pre15_cleanup_pid_value ($pre15_cleanup_pid_label)" - else - pre15_cleanup_note "PASS stopped PID $pre15_cleanup_pid_value ($pre15_cleanup_pid_label)" - fi -} - -pre15_cleanup_guest() -{ - pre15_guest_kind=$1 - pre15_guest_value=$2 - case "$pre15_guest_kind" in - guest-mount) - if pre15_guest_ssh_bounded umount "$pre15_guest_value" >> "$PRE15_CLEANUP_LOG" 2>&1; then - pre15_cleanup_note "PASS unmounted guest path $pre15_guest_value" - else - pre15_cleanup_fail "guest unmount failed: $pre15_guest_value" - fi - ;; - guest-md) - if pre15_guest_ssh_bounded mdconfig -d -u "$pre15_guest_value" >> "$PRE15_CLEANUP_LOG" 2>&1; then - pre15_cleanup_note "PASS detached guest md $pre15_guest_value" - else - pre15_cleanup_fail "guest md detach failed: $pre15_guest_value" - fi - ;; - guest-kld) - if pre15_guest_ssh_bounded kldunload "$pre15_guest_value" >> "$PRE15_CLEANUP_LOG" 2>&1; then - pre15_cleanup_note "PASS unloaded guest KLD $pre15_guest_value" - else - pre15_cleanup_fail "guest KLD unload failed: $pre15_guest_value" - fi - ;; - esac -} - -pre15_cleanup_owned() -{ - PRE15_CLEANUP_STATUS_RESULT=PASS - : > "$PRE15_CLEANUP_LOG" - pre15_reverse_file=$PRE15_RUN_DIR/.ownership-reverse.$$ - awk '{ line[NR] = $0 } END { for (i = NR; i > 0; --i) print line[i] }' \ - "$PRE15_OWNERSHIP_FILE" > "$pre15_reverse_file" - while IFS="$(printf '\t')" read -r pre15_kind pre15_value pre15_label; do - test -n "$pre15_kind" || continue - case "$pre15_kind" in - guest-mount|guest-md|guest-kld) - if test -n "${PRE15_QEMU_SSH_PORT:-}"; then - pre15_cleanup_guest "$pre15_kind" "$pre15_value" - else - pre15_cleanup_fail "guest resource has no SSH endpoint: $pre15_kind $pre15_value" - fi - ;; - ssh-control) - if test -S "$pre15_value"; then - if timeout -k 5 "${PRE15_CLEANUP_COMMAND_TIMEOUT:-15}" \ - ssh -S "$pre15_value" -O exit \ - -p "${PRE15_QEMU_SSH_PORT:-22}" \ - "${PRE15_QEMU_SSH_USER:-root}@127.0.0.1" \ - >> "$PRE15_CLEANUP_LOG" 2>&1; then - pre15_cleanup_note "PASS closed SSH control $pre15_value" - else - pre15_cleanup_fail "could not close SSH control $pre15_value" - fi - fi - if test -e "$pre15_value" && ! rm -f -- "$pre15_value"; then - pre15_cleanup_fail "could not remove SSH control $pre15_value" - fi - ;; - pid) - pre15_cleanup_pid "$pre15_value" "$pre15_label" - ;; - port) - if pre15_port_is_free "$pre15_value"; then - pre15_cleanup_note "PASS port is free: $pre15_value" - else - pre15_cleanup_fail "port remains in use: $pre15_value" - fi - ;; - path) - case "$pre15_value" in - "$PRE15_RUN_DIR"/*) - if test -e "$pre15_value" && ! rm -rf -- "$pre15_value"; then - pre15_cleanup_fail "could not remove owned path $pre15_value" - else - pre15_cleanup_note "PASS removed owned path $pre15_value" - fi - ;; - *) pre15_cleanup_fail "refused to remove path outside run directory: $pre15_value" ;; - esac - ;; - base-image) - if test ! -f "$pre15_value"; then - pre15_cleanup_fail "base image disappeared: $pre15_value" - else - pre15_cleanup_base_hash=$(sha256sum "$pre15_value" | awk '{print $1}') - if test "$pre15_cleanup_base_hash" = "$pre15_label"; then - pre15_cleanup_note "PASS base image unchanged: $pre15_value" - else - pre15_cleanup_fail "base image hash changed: $pre15_value" - fi - fi - ;; - *) pre15_cleanup_fail "unknown ownership kind: $pre15_kind" ;; - esac - done < "$pre15_reverse_file" - rm -f -- "$pre15_reverse_file" - if grep -q '^FAIL ' "$PRE15_CLEANUP_LOG"; then - PRE15_CLEANUP_STATUS_RESULT=FAIL - fi -} - -pre15_reason_text() -{ - if test -s "$PRE15_REASON_FILE"; then - head -n 1 "$PRE15_REASON_FILE" - else - printf '%s\n' "$1" - fi -} - -pre15_classify() -{ - pre15_classify_rc=$1 - pre15_classify_target=$2 - pre15_classify_cleanup=$3 - PRE15_TIMED_OUT_RESULT=0 - if test "$pre15_classify_cleanup" != PASS; then - PRE15_STATUS_RESULT=RUNNER_FAIL - PRE15_ORIGIN_RESULT=runner - PRE15_REASON_RESULT='owned resource cleanup failed' - return - fi - case "$pre15_classify_rc" in - 0) - if test "$pre15_classify_target" = reached; then - PRE15_STATUS_RESULT=PASS - PRE15_ORIGIN_RESULT=none - PRE15_REASON_RESULT='target reached and all oracles passed' - else - PRE15_STATUS_RESULT=RUNNER_FAIL - PRE15_ORIGIN_RESULT=runner - PRE15_REASON_RESULT='command exited zero without a target marker' - fi - ;; - "$PRE15_RC_DUT_FAIL") - if test "$pre15_classify_target" = reached; then - PRE15_STATUS_RESULT=DUT_FAIL - PRE15_ORIGIN_RESULT=dut - PRE15_REASON_RESULT=$(pre15_reason_text 'DUT oracle mismatch') - else - PRE15_STATUS_RESULT=RUNNER_FAIL - PRE15_ORIGIN_RESULT=runner - PRE15_REASON_RESULT='DUT_FAIL was requested before the target marker' - fi - ;; - "$PRE15_RC_INFRA_BLOCKED") - PRE15_STATUS_RESULT=INFRA_BLOCKED - PRE15_ORIGIN_RESULT=infrastructure - PRE15_REASON_RESULT=$(pre15_reason_text 'infrastructure prerequisite failed') - ;; - "$PRE15_RC_STOP") - PRE15_STATUS_RESULT=STOP - PRE15_ORIGIN_RESULT=gate - PRE15_REASON_RESULT=$(pre15_reason_text 'gate oracle returned STOP') - ;; - 124|137) - PRE15_STATUS_RESULT=RUNNER_FAIL - PRE15_ORIGIN_RESULT=runner - PRE15_REASON_RESULT="hard timeout after ${PRE15_DEADLINE}s" - PRE15_TIMED_OUT_RESULT=1 - ;; - *) - PRE15_STATUS_RESULT=RUNNER_FAIL - PRE15_ORIGIN_RESULT=runner - PRE15_REASON_RESULT=$(pre15_reason_text "command exited $pre15_classify_rc") - ;; - esac -} - -pre15_write_identity() -{ - PRE15_DUT_HEAD=$(git -C "$PRE15_ROOT" rev-parse HEAD) - PRE15_DUT_TREE=$(git -C "$PRE15_ROOT" rev-parse HEAD:repo-pre-15) - PRE15_LINUX_SOURCE=$(git -C "$PRE15_ROOT" rev-parse HEAD:src-linux) - pre15_freebsd_git_root=$(git -C "$PRE15_FREEBSD_SRC" \ - rev-parse --show-toplevel 2>/dev/null || true) - if test -n "$pre15_freebsd_git_root" && \ - test "$(CDPATH= cd -- "$pre15_freebsd_git_root" && pwd -P)" = \ - "$(CDPATH= cd -- "$PRE15_FREEBSD_SRC" && pwd -P)"; then - pre15_freebsd_head=$(git -C "$PRE15_FREEBSD_SRC" rev-parse HEAD) - pre15_freebsd_dirty=$(git -C "$PRE15_FREEBSD_SRC" status --short | sha256sum | awk '{print $1}') - PRE15_FREEBSD_SOURCE="$pre15_freebsd_head dirty-sha256:$pre15_freebsd_dirty" - else - pre15_freebsd_metadata=$(pre15_tree_metadata_sha256 "$PRE15_FREEBSD_SRC") - if test -n "${PRE15_FREEBSD_TREE_SHA256:-}" || \ - test -n "${PRE15_FREEBSD_TREE_METADATA_SHA256:-}"; then - case "${PRE15_FREEBSD_TREE_SHA256:-}" in - ????????????????????????????????????????????????????????????????) - case "$PRE15_FREEBSD_TREE_SHA256" in - *[!0-9a-f]*) pre15_fail_usage 'cached FreeBSD tree SHA256 is invalid' ;; - esac - ;; - *) pre15_fail_usage 'cached FreeBSD tree SHA256 is invalid' ;; - esac - case "${PRE15_FREEBSD_TREE_METADATA_SHA256:-}" in - ????????????????????????????????????????????????????????????????) - case "$PRE15_FREEBSD_TREE_METADATA_SHA256" in - *[!0-9a-f]*) pre15_fail_usage 'cached FreeBSD metadata SHA256 is invalid' ;; - esac - ;; - *) pre15_fail_usage 'cached FreeBSD metadata SHA256 is invalid' ;; - esac - test "$pre15_freebsd_metadata" = \ - "$PRE15_FREEBSD_TREE_METADATA_SHA256" || \ - pre15_fail_usage 'FreeBSD source changed after its content snapshot' - pre15_freebsd_tree=$PRE15_FREEBSD_TREE_SHA256 - else - pre15_freebsd_tree=$(pre15_tree_sha256 "$PRE15_FREEBSD_SRC") - fi - PRE15_FREEBSD_SOURCE="tree-sha256:$pre15_freebsd_tree metadata-sha256:$pre15_freebsd_metadata non-git-snapshot" - fi - PRE15_WORKTREE_DIFF_SHA256=$(git -C "$PRE15_ROOT" diff --binary HEAD -- repo-pre-15 | sha256sum | awk '{print $1}') - export PRE15_DUT_HEAD PRE15_DUT_TREE PRE15_LINUX_SOURCE PRE15_FREEBSD_SOURCE - export PRE15_WORKTREE_DIFF_SHA256 - PRE15_IDENTITY_HOST=$(uname -a) - PRE15_IDENTITY_GIT_STATUS=$(git -C "$PRE15_ROOT" status --short | sha256sum | awk '{print $1}') - export PRE15_IDENTITY_HOST PRE15_IDENTITY_GIT_STATUS - python3 - "$PRE15_IDENTITY_PATH" <<'PY' -import json -import os -from pathlib import Path -import platform -import sys - -identity = { - "host_uname": os.environ["PRE15_IDENTITY_HOST"], - "python": platform.python_version(), - "git_status_sha256": os.environ["PRE15_IDENTITY_GIT_STATUS"], - "freebsd_source_path": os.environ["PRE15_FREEBSD_SRC"], - "freebsd_source_identity": os.environ["PRE15_FREEBSD_SOURCE"], -} -with Path(sys.argv[1]).open("x", encoding="ascii") as output: - json.dump(identity, output, ensure_ascii=True, indent=2, sort_keys=True) - output.write("\n") -PY -} - -pre15_new_run() -{ - umask 077 - mkdir -p "$PRE15_EVIDENCE_ROOT" - pre15_run_stamp=$(date -u '+%Y%m%dT%H%M%SZ') - pre15_run_attempt=0 - while :; do - PRE15_RUN_ID="$pre15_run_stamp-$PRE15_MODE-$PRE15_CASE_ID-$$-$pre15_run_attempt" - PRE15_RUN_DIR=$PRE15_EVIDENCE_ROOT/$PRE15_RUN_ID - if mkdir "$PRE15_RUN_DIR" 2>/dev/null; then - break - fi - pre15_run_attempt=$((pre15_run_attempt + 1)) - test "$pre15_run_attempt" -lt 100 || pre15_fail_usage \ - "could not allocate immutable run directory" - done - PRE15_STDOUT_PATH=$PRE15_RUN_DIR/stdout.log - PRE15_STDERR_PATH=$PRE15_RUN_DIR/stderr.log - PRE15_TARGET_MARKER_FILE=$PRE15_RUN_DIR/target.marker - PRE15_REASON_FILE=$PRE15_RUN_DIR/reason.txt - PRE15_OWNERSHIP_FILE=$PRE15_RUN_DIR/ownership.tsv - PRE15_CLEANUP_LOG=$PRE15_RUN_DIR/cleanup.log - PRE15_FIXTURE_HASHES=$PRE15_RUN_DIR/fixture-hashes.tsv - PRE15_MODULE_HASH=$PRE15_RUN_DIR/module-hash.tsv - PRE15_ARGV_PATH=$PRE15_RUN_DIR/command-argv.json - PRE15_IDENTITY_PATH=$PRE15_RUN_DIR/identity.json - PRE15_MANIFEST_PATH=$PRE15_RUN_DIR/manifest.json - PRE15_CASE_TMP=$PRE15_RUN_DIR/case-tmp - : > "$PRE15_STDOUT_PATH" - : > "$PRE15_STDERR_PATH" - : > "$PRE15_OWNERSHIP_FILE" - : > "$PRE15_CLEANUP_LOG" - : > "$PRE15_FIXTURE_HASHES" - : > "$PRE15_MODULE_HASH" - mkdir "$PRE15_CASE_TMP" - pre15_own_path "$PRE15_CASE_TMP" 'case temporary directory' - export PRE15_RUN_ID PRE15_RUN_DIR PRE15_STDOUT_PATH PRE15_STDERR_PATH - export PRE15_TARGET_MARKER_FILE PRE15_REASON_FILE PRE15_OWNERSHIP_FILE - export PRE15_CLEANUP_LOG PRE15_FIXTURE_HASHES PRE15_MODULE_HASH - export PRE15_ARGV_PATH PRE15_IDENTITY_PATH PRE15_MANIFEST_PATH PRE15_CASE_TMP -} - -pre15_freeze_run() -{ - find "$PRE15_RUN_DIR" -type f -exec chmod 0444 {} + - find "$PRE15_RUN_DIR" -type d -exec chmod 0555 {} + -} - -pre15_parent_signal() -{ - pre15_cleanup_owned - exit "$PRE15_RC_RUNNER_FAIL" -} - -pre15_run_command() -{ - pre15_require_paths - PRE15_MODE=$1 - PRE15_CASE_ID=$2 - PRE15_DEADLINE=$3 - shift 3 - case "$PRE15_CASE_ID" in - ''|*[!A-Za-z0-9._-]*) pre15_fail_usage "invalid case ID: $PRE15_CASE_ID" ;; - esac - case "$PRE15_DEADLINE" in - ''|*[!0-9]*|0) pre15_fail_usage "invalid deadline: $PRE15_DEADLINE" ;; - esac - command -v timeout >/dev/null 2>&1 || pre15_fail_usage 'timeout is required' - command -v python3 >/dev/null 2>&1 || pre15_fail_usage 'python3 is required' - test -f "$PRE15_SCHEMA" || pre15_fail_usage "schema is absent: $PRE15_SCHEMA" - pre15_new_run - pre15_write_identity - pre15_write_argv_json "$PRE15_ARGV_PATH" "$@" - PRE15_START_UTC=$(pre15_utc_now) - export PRE15_MODE PRE15_CASE_ID PRE15_DEADLINE PRE15_START_UTC - trap pre15_parent_signal HUP INT TERM - if timeout -k "${PRE15_TIMEOUT_KILL_AFTER:-10}" "$PRE15_DEADLINE" "$@" \ - > "$PRE15_STDOUT_PATH" 2> "$PRE15_STDERR_PATH"; then - PRE15_EXIT_CODE=0 - else - PRE15_EXIT_CODE=$? - fi - trap - HUP INT TERM - if test -f "$PRE15_TARGET_MARKER_FILE" && \ - test "$(cat "$PRE15_TARGET_MARKER_FILE")" = reached; then - PRE15_TARGET_STATE=reached - else - PRE15_TARGET_STATE=not_reached - fi - pre15_cleanup_owned - PRE15_CLEANUP_STATUS=$PRE15_CLEANUP_STATUS_RESULT - pre15_classify "$PRE15_EXIT_CODE" "$PRE15_TARGET_STATE" "$PRE15_CLEANUP_STATUS" - PRE15_STATUS=$PRE15_STATUS_RESULT - PRE15_FAILURE_ORIGIN=$PRE15_ORIGIN_RESULT - PRE15_REASON=$PRE15_REASON_RESULT - PRE15_TIMED_OUT=$PRE15_TIMED_OUT_RESULT - PRE15_END_UTC=$(pre15_utc_now) - export PRE15_EXIT_CODE PRE15_TARGET_STATE PRE15_CLEANUP_STATUS PRE15_STATUS - export PRE15_FAILURE_ORIGIN PRE15_REASON PRE15_TIMED_OUT PRE15_END_UTC - pre15_write_manifest - pre15_validate_manifest "$PRE15_MANIFEST_PATH" "$PRE15_SCHEMA" - printf 'PRE15_RESULT status=%s cleanup=%s run_id=%s evidence=%s\n' \ - "$PRE15_STATUS" "$PRE15_CLEANUP_STATUS" "$PRE15_RUN_ID" \ - "$PRE15_MANIFEST_PATH" - pre15_freeze_run - case "$PRE15_STATUS" in - PASS) return 0 ;; - DUT_FAIL) return "$PRE15_RC_DUT_FAIL" ;; - INFRA_BLOCKED) return "$PRE15_RC_INFRA_BLOCKED" ;; - STOP) return "$PRE15_RC_STOP" ;; - *) return "$PRE15_RC_RUNNER_FAIL" ;; - esac -} - -pre15_control_run() -{ - pre15_control_dir=$1 - pre15_control_deadline=$2 - shift 2 - ( - PRE15_RUN_DIR=$pre15_control_dir - PRE15_TARGET_MARKER_FILE=$pre15_control_dir/target.marker - PRE15_REASON_FILE=$pre15_control_dir/reason.txt - PRE15_OWNERSHIP_FILE=$pre15_control_dir/ownership.tsv - PRE15_CLEANUP_LOG=$pre15_control_dir/cleanup.log - PRE15_DEADLINE=$pre15_control_deadline - export PRE15_RUN_DIR PRE15_TARGET_MARKER_FILE PRE15_REASON_FILE - export PRE15_OWNERSHIP_FILE PRE15_CLEANUP_LOG PRE15_DEADLINE - mkdir "$pre15_control_dir" - : > "$PRE15_OWNERSHIP_FILE" - : > "$PRE15_CLEANUP_LOG" - if timeout -k 2 "$pre15_control_deadline" "$@" \ - > "$pre15_control_dir/stdout.log" \ - 2> "$pre15_control_dir/stderr.log"; then - pre15_control_rc=0 - else - pre15_control_rc=$? - fi - if test -f "$PRE15_TARGET_MARKER_FILE" && \ - test "$(cat "$PRE15_TARGET_MARKER_FILE")" = reached; then - pre15_control_target=reached - else - pre15_control_target=not_reached - fi - pre15_cleanup_owned - pre15_classify "$pre15_control_rc" "$pre15_control_target" \ - "$PRE15_CLEANUP_STATUS_RESULT" - printf '%s\t%s\t%s\t%s\n' "$PRE15_STATUS_RESULT" \ - "$PRE15_ORIGIN_RESULT" "$PRE15_CLEANUP_STATUS_RESULT" \ - "$PRE15_TIMED_OUT_RESULT" > "$pre15_control_dir/result.tsv" - ) -} - -pre15_find_case() -{ - pre15_case_id=$1 - case "$pre15_case_id" in - ''|*[!A-Za-z0-9._-]*) pre15_fail_usage "invalid case ID: $pre15_case_id" ;; - esac - pre15_case_path=$PRE15_DUT/tests/pre15/cases/$pre15_case_id.sh - test -f "$pre15_case_path" || pre15_fail_usage "unknown case: $pre15_case_id" - printf '%s\n' "$pre15_case_path" -} diff --git a/tests/pre15/probes/ondisk_layout.c b/tests/pre15/probes/ondisk_layout.c deleted file mode 100644 index 5f53fa9..0000000 --- a/tests/pre15/probes/ondisk_layout.c +++ /dev/null @@ -1,318 +0,0 @@ -#include -#include -#include - -#define B02_ALIGN(value, alignment) \ - (((value) + (alignment) - 1) & ~((alignment) - 1)) -#ifndef roundup2 -#define roundup2(value, alignment) B02_ALIGN(value, alignment) -#endif - -#ifdef B02_LINUX_REFERENCE -typedef uint8_t u8; -typedef uint8_t __u8; -typedef uint32_t __u32; -typedef uint16_t __le16; -typedef uint32_t __le32; -typedef uint64_t __le64; -#define __packed __attribute__((__packed__)) -#define BIT(nr) (1UL << (nr)) -#define BIT_ULL(nr) (1ULL << (nr)) -#define ALIGN(value, alignment) B02_ALIGN(value, alignment) -#define round_up(value, alignment) ALIGN(value, alignment) -#define le16_to_cpu(value) (value) -#define cpu_to_le64(value) (value) -#define BUILD_BUG_ON(condition) ((void)sizeof(condition)) -#include "../../../../src-linux/erofs_fs.h" -#define EXPECTED_ALIGN(freebsd, linux) (linux) -#else -#include "../../../src/erofs_fs.h" -#define EXPECTED_ALIGN(freebsd, linux) (freebsd) -#endif - -#define ASSERT_RECORD(type, size, freebsd_align, linux_align) \ - _Static_assert(sizeof(type) == (size), "size " #type); \ - _Static_assert(_Alignof(type) == \ - EXPECTED_ALIGN(freebsd_align, linux_align), "alignment " #type) -#define ASSERT_OFFSET(type, field, offset) \ - _Static_assert(offsetof(type, field) == (offset), \ - "offset " #type "." #field) -#define ASSERT_TYPE(type, field, expected) \ - _Static_assert(__builtin_types_compatible_p( \ - __typeof__(((type *)0)->field), expected), \ - "underlying type " #type "." #field) -#define ASSERT_FIELD(type, field, offset, expected) \ - ASSERT_OFFSET(type, field, offset); \ - ASSERT_TYPE(type, field, expected) - -ASSERT_RECORD(struct erofs_deviceslot, 128, 1, 4); -ASSERT_FIELD(struct erofs_deviceslot, tag, 0, uint8_t[64]); -ASSERT_FIELD(struct erofs_deviceslot, blocks_lo, 64, __le32); -ASSERT_FIELD(struct erofs_deviceslot, uniaddr_lo, 68, __le32); -ASSERT_FIELD(struct erofs_deviceslot, blocks_hi, 72, __le16); -ASSERT_FIELD(struct erofs_deviceslot, uniaddr_hi, 74, __le16); -ASSERT_FIELD(struct erofs_deviceslot, reserved, 76, uint8_t[52]); - -ASSERT_RECORD(struct erofs_super_block, 144, 1, 8); -ASSERT_FIELD(struct erofs_super_block, magic, 0, __le32); -ASSERT_FIELD(struct erofs_super_block, checksum, 4, __le32); -ASSERT_FIELD(struct erofs_super_block, feature_compat, 8, __le32); -ASSERT_FIELD(struct erofs_super_block, blkszbits, 12, uint8_t); -ASSERT_FIELD(struct erofs_super_block, sb_extslots, 13, uint8_t); -ASSERT_OFFSET(struct erofs_super_block, rb, 14); -ASSERT_FIELD(struct erofs_super_block, rb.rootnid_2b, 14, __le16); -ASSERT_FIELD(struct erofs_super_block, rb.blocks_hi, 14, __le16); -ASSERT_FIELD(struct erofs_super_block, inos, 16, __le64); -ASSERT_FIELD(struct erofs_super_block, epoch, 24, __le64); -ASSERT_FIELD(struct erofs_super_block, fixed_nsec, 32, __le32); -ASSERT_FIELD(struct erofs_super_block, blocks_lo, 36, __le32); -ASSERT_FIELD(struct erofs_super_block, meta_blkaddr, 40, __le32); -ASSERT_FIELD(struct erofs_super_block, xattr_blkaddr, 44, __le32); -ASSERT_FIELD(struct erofs_super_block, uuid, 48, uint8_t[16]); -ASSERT_FIELD(struct erofs_super_block, volume_name, 64, uint8_t[16]); -ASSERT_FIELD(struct erofs_super_block, feature_incompat, 80, __le32); -ASSERT_OFFSET(struct erofs_super_block, u1, 84); -ASSERT_FIELD(struct erofs_super_block, u1.available_compr_algs, 84, __le16); -ASSERT_FIELD(struct erofs_super_block, u1.lz4_max_distance, 84, __le16); -ASSERT_FIELD(struct erofs_super_block, extra_devices, 86, __le16); -ASSERT_FIELD(struct erofs_super_block, devt_slotoff, 88, __le16); -ASSERT_FIELD(struct erofs_super_block, dirblkbits, 90, uint8_t); -ASSERT_FIELD(struct erofs_super_block, xattr_prefix_count, 91, uint8_t); -ASSERT_FIELD(struct erofs_super_block, xattr_prefix_start, 92, __le32); -ASSERT_FIELD(struct erofs_super_block, packed_nid, 96, __le64); -ASSERT_FIELD(struct erofs_super_block, xattr_filter_reserved, 104, uint8_t); -ASSERT_FIELD(struct erofs_super_block, ishare_xattr_prefix_id, 105, uint8_t); -ASSERT_FIELD(struct erofs_super_block, reserved, 106, uint8_t[2]); -ASSERT_FIELD(struct erofs_super_block, build_time, 108, __le32); -ASSERT_FIELD(struct erofs_super_block, rootnid_8b, 112, __le64); -ASSERT_FIELD(struct erofs_super_block, reserved2, 120, __le64); -ASSERT_FIELD(struct erofs_super_block, metabox_nid, 128, __le64); -ASSERT_FIELD(struct erofs_super_block, reserved3, 136, __le64); - -ASSERT_RECORD(struct erofs_inode_chunk_info, 4, 1, 2); -ASSERT_FIELD(struct erofs_inode_chunk_info, format, 0, __le16); -ASSERT_FIELD(struct erofs_inode_chunk_info, reserved, 2, __le16); - -ASSERT_RECORD(union erofs_inode_i_u, 4, 4, 4); -ASSERT_FIELD(union erofs_inode_i_u, blocks_lo, 0, __le32); -ASSERT_FIELD(union erofs_inode_i_u, startblk_lo, 0, __le32); -ASSERT_FIELD(union erofs_inode_i_u, rdev, 0, __le32); -ASSERT_FIELD(union erofs_inode_i_u, c, 0, struct erofs_inode_chunk_info); - -ASSERT_RECORD(union erofs_inode_i_nb, 2, 1, 1); -ASSERT_FIELD(union erofs_inode_i_nb, nlink, 0, __le16); -ASSERT_FIELD(union erofs_inode_i_nb, blocks_hi, 0, __le16); -ASSERT_FIELD(union erofs_inode_i_nb, startblk_hi, 0, __le16); - -ASSERT_RECORD(struct erofs_inode_compact, 32, 1, 4); -ASSERT_FIELD(struct erofs_inode_compact, i_format, 0, __le16); -ASSERT_FIELD(struct erofs_inode_compact, i_xattr_icount, 2, __le16); -ASSERT_FIELD(struct erofs_inode_compact, i_mode, 4, __le16); -ASSERT_FIELD(struct erofs_inode_compact, i_nb, 6, union erofs_inode_i_nb); -ASSERT_FIELD(struct erofs_inode_compact, i_size, 8, __le32); -ASSERT_FIELD(struct erofs_inode_compact, i_mtime, 12, __le32); -ASSERT_FIELD(struct erofs_inode_compact, i_u, 16, union erofs_inode_i_u); -ASSERT_FIELD(struct erofs_inode_compact, i_ino, 20, __le32); -ASSERT_FIELD(struct erofs_inode_compact, i_uid, 24, __le16); -ASSERT_FIELD(struct erofs_inode_compact, i_gid, 26, __le16); -ASSERT_FIELD(struct erofs_inode_compact, i_reserved, 28, __le32); - -ASSERT_RECORD(struct erofs_inode_extended, 64, 1, 8); -ASSERT_FIELD(struct erofs_inode_extended, i_format, 0, __le16); -ASSERT_FIELD(struct erofs_inode_extended, i_xattr_icount, 2, __le16); -ASSERT_FIELD(struct erofs_inode_extended, i_mode, 4, __le16); -ASSERT_FIELD(struct erofs_inode_extended, i_nb, 6, union erofs_inode_i_nb); -ASSERT_FIELD(struct erofs_inode_extended, i_size, 8, __le64); -ASSERT_FIELD(struct erofs_inode_extended, i_u, 16, union erofs_inode_i_u); -ASSERT_FIELD(struct erofs_inode_extended, i_ino, 20, __le32); -ASSERT_FIELD(struct erofs_inode_extended, i_uid, 24, __le32); -ASSERT_FIELD(struct erofs_inode_extended, i_gid, 28, __le32); -ASSERT_FIELD(struct erofs_inode_extended, i_mtime, 32, __le64); -ASSERT_FIELD(struct erofs_inode_extended, i_mtime_nsec, 40, __le32); -ASSERT_FIELD(struct erofs_inode_extended, i_nlink, 44, __le32); -ASSERT_FIELD(struct erofs_inode_extended, i_reserved2, 48, uint8_t[16]); - -ASSERT_RECORD(struct erofs_xattr_ibody_header, 12, 1, 4); -ASSERT_FIELD(struct erofs_xattr_ibody_header, h_name_filter, 0, __le32); -ASSERT_FIELD(struct erofs_xattr_ibody_header, h_shared_count, 4, uint8_t); -ASSERT_FIELD(struct erofs_xattr_ibody_header, h_reserved2, 5, uint8_t[7]); -ASSERT_FIELD(struct erofs_xattr_ibody_header, h_shared_xattrs, 12, __le32[]); - -ASSERT_RECORD(struct erofs_xattr_entry, 4, 1, 2); -ASSERT_FIELD(struct erofs_xattr_entry, e_name_len, 0, uint8_t); -ASSERT_FIELD(struct erofs_xattr_entry, e_name_index, 1, uint8_t); -ASSERT_FIELD(struct erofs_xattr_entry, e_value_size, 2, __le16); -ASSERT_FIELD(struct erofs_xattr_entry, e_name, 4, char[]); - -ASSERT_RECORD(struct erofs_xattr_long_prefix, 1, 1, 1); -ASSERT_FIELD(struct erofs_xattr_long_prefix, base_index, 0, uint8_t); -ASSERT_FIELD(struct erofs_xattr_long_prefix, infix, 1, char[]); - -ASSERT_RECORD(struct erofs_inode_chunk_index, 8, 1, 4); -ASSERT_FIELD(struct erofs_inode_chunk_index, startblk_hi, 0, __le16); -ASSERT_FIELD(struct erofs_inode_chunk_index, device_id, 2, __le16); -ASSERT_FIELD(struct erofs_inode_chunk_index, startblk_lo, 4, __le32); - -ASSERT_RECORD(struct erofs_dirent, 12, 1, 1); -ASSERT_FIELD(struct erofs_dirent, nid, 0, __le64); -ASSERT_FIELD(struct erofs_dirent, nameoff, 8, __le16); -ASSERT_FIELD(struct erofs_dirent, file_type, 10, uint8_t); -ASSERT_FIELD(struct erofs_dirent, reserved, 11, uint8_t); - -ASSERT_RECORD(struct z_erofs_lz4_cfgs, 14, 1, 1); -ASSERT_FIELD(struct z_erofs_lz4_cfgs, max_distance, 0, __le16); -ASSERT_FIELD(struct z_erofs_lz4_cfgs, max_pclusterblks, 2, __le16); -ASSERT_FIELD(struct z_erofs_lz4_cfgs, reserved, 4, uint8_t[10]); - -ASSERT_RECORD(struct z_erofs_lzma_cfgs, 14, 1, 1); -ASSERT_FIELD(struct z_erofs_lzma_cfgs, dict_size, 0, __le32); -ASSERT_FIELD(struct z_erofs_lzma_cfgs, format, 4, __le16); -ASSERT_FIELD(struct z_erofs_lzma_cfgs, reserved, 6, uint8_t[8]); - -ASSERT_RECORD(struct z_erofs_deflate_cfgs, 6, 1, 1); -ASSERT_FIELD(struct z_erofs_deflate_cfgs, windowbits, 0, uint8_t); -ASSERT_FIELD(struct z_erofs_deflate_cfgs, reserved, 1, uint8_t[5]); - -ASSERT_RECORD(struct z_erofs_zstd_cfgs, 6, 1, 1); -ASSERT_FIELD(struct z_erofs_zstd_cfgs, format, 0, uint8_t); -ASSERT_FIELD(struct z_erofs_zstd_cfgs, windowlog, 1, uint8_t); -ASSERT_FIELD(struct z_erofs_zstd_cfgs, reserved, 2, uint8_t[4]); - -ASSERT_RECORD(struct z_erofs_map_header, 8, 1, 4); -ASSERT_FIELD(struct z_erofs_map_header, h_fragmentoff, 0, __le32); -ASSERT_FIELD(struct z_erofs_map_header, h_reserved1, 0, __le16); -ASSERT_FIELD(struct z_erofs_map_header, h_idata_size, 2, __le16); -ASSERT_FIELD(struct z_erofs_map_header, h_extents_lo, 0, __le32); -ASSERT_FIELD(struct z_erofs_map_header, h_advise, 4, __le16); -ASSERT_FIELD(struct z_erofs_map_header, h_algorithmtype, 6, uint8_t); -ASSERT_FIELD(struct z_erofs_map_header, h_clusterbits, 7, uint8_t); -ASSERT_FIELD(struct z_erofs_map_header, h_extents_hi, 6, __le16); - -ASSERT_RECORD(struct z_erofs_lcluster_index, 8, 1, 4); -ASSERT_FIELD(struct z_erofs_lcluster_index, di_advise, 0, __le16); -ASSERT_FIELD(struct z_erofs_lcluster_index, di_clusterofs, 2, __le16); -ASSERT_FIELD(struct z_erofs_lcluster_index, di_u.blkaddr, 4, __le32); -ASSERT_FIELD(struct z_erofs_lcluster_index, di_u.delta, 4, __le16[2]); - -ASSERT_RECORD(struct z_erofs_extent, 32, 1, 4); -ASSERT_FIELD(struct z_erofs_extent, plen, 0, __le32); -ASSERT_FIELD(struct z_erofs_extent, pstart_lo, 4, __le32); -ASSERT_FIELD(struct z_erofs_extent, pstart_hi, 8, __le32); -ASSERT_FIELD(struct z_erofs_extent, lstart_lo, 12, __le32); -ASSERT_FIELD(struct z_erofs_extent, lstart_hi, 16, __le32); -ASSERT_FIELD(struct z_erofs_extent, reserved, 20, uint8_t[12]); - -#define ASSERT_VALUE(expression, value) \ - _Static_assert((expression) == (value), "value " #expression) - -ASSERT_VALUE(EROFS_SUPER_OFFSET, 1024); -ASSERT_VALUE(EROFS_SB_EXTSLOT_SIZE, 16); -ASSERT_VALUE(EROFS_DEVT_SLOT_SIZE, 128); -ASSERT_VALUE(EROFS_FEATURE_COMPAT_SB_CHKSUM, 0x00000001); -ASSERT_VALUE(EROFS_FEATURE_COMPAT_MTIME, 0x00000002); -ASSERT_VALUE(EROFS_FEATURE_COMPAT_XATTR_FILTER, 0x00000004); -ASSERT_VALUE(EROFS_FEATURE_COMPAT_SHARED_EA_IN_METABOX, 0x00000008); -ASSERT_VALUE(EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX, 0x00000010); -ASSERT_VALUE(EROFS_FEATURE_COMPAT_ISHARE_XATTRS, 0x00000020); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING, 0x00000001); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_COMPR_CFGS, 0x00000002); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER, 0x00000002); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_CHUNKED_FILE, 0x00000004); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_DEVICE_TABLE, 0x00000008); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_COMPR_HEAD2, 0x00000008); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_ZTAILPACKING, 0x00000010); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_FRAGMENTS, 0x00000020); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_DEDUPE, 0x00000020); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES, 0x00000040); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_48BIT, 0x00000080); -ASSERT_VALUE(EROFS_FEATURE_INCOMPAT_METABOX, 0x00000100); -ASSERT_VALUE(EROFS_ALL_FEATURE_INCOMPAT, 0x000001ff); -ASSERT_VALUE(EROFS_INODE_FLAT_PLAIN, 0); -ASSERT_VALUE(EROFS_INODE_COMPRESSED_FULL, 1); -ASSERT_VALUE(EROFS_INODE_FLAT_INLINE, 2); -ASSERT_VALUE(EROFS_INODE_COMPRESSED_COMPACT, 3); -ASSERT_VALUE(EROFS_INODE_CHUNK_BASED, 4); -ASSERT_VALUE(EROFS_INODE_DATALAYOUT_MAX, 5); -ASSERT_VALUE(EROFS_I_VERSION_MASK, 0x01); -ASSERT_VALUE(EROFS_I_DATALAYOUT_MASK, 0x07); -ASSERT_VALUE(EROFS_I_VERSION_BIT, 0); -ASSERT_VALUE(EROFS_I_DATALAYOUT_BIT, 1); -ASSERT_VALUE(EROFS_I_NLINK_1_BIT, 4); -ASSERT_VALUE(EROFS_I_DOT_OMITTED_BIT, 4); -ASSERT_VALUE(EROFS_I_ALL, 31); -ASSERT_VALUE(EROFS_CHUNK_FORMAT_BLKBITS_MASK, 0x001f); -ASSERT_VALUE(EROFS_CHUNK_FORMAT_INDEXES, 0x0020); -ASSERT_VALUE(EROFS_CHUNK_FORMAT_48BIT, 0x0040); -ASSERT_VALUE(EROFS_CHUNK_FORMAT_ALL, 0x007f); -ASSERT_VALUE(EROFS_INODE_LAYOUT_COMPACT, 0); -ASSERT_VALUE(EROFS_INODE_LAYOUT_EXTENDED, 1); -ASSERT_VALUE(EROFS_XATTR_INDEX_USER, 1); -ASSERT_VALUE(EROFS_XATTR_INDEX_POSIX_ACL_ACCESS, 2); -ASSERT_VALUE(EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT, 3); -ASSERT_VALUE(EROFS_XATTR_INDEX_TRUSTED, 4); -ASSERT_VALUE(EROFS_XATTR_INDEX_LUSTRE, 5); -ASSERT_VALUE(EROFS_XATTR_INDEX_SECURITY, 6); -ASSERT_VALUE(EROFS_XATTR_LONG_PREFIX, 0x80); -ASSERT_VALUE(EROFS_XATTR_LONG_PREFIX_MASK, 0x7f); -ASSERT_VALUE(EROFS_XATTR_ALIGN(0), 0); -ASSERT_VALUE(EROFS_XATTR_ALIGN(1), 4); -ASSERT_VALUE(EROFS_XATTR_ALIGN(5), 8); -ASSERT_VALUE((uint64_t)EROFS_NULL_ADDR, UINT64_MAX); -ASSERT_VALUE(EROFS_BLOCK_MAP_ENTRY_SIZE, 4); -ASSERT_VALUE(EROFS_DIRENT_NID_METABOX_BIT, 63); -ASSERT_VALUE(EROFS_DIRENT_NID_MASK, 0x7fffffffffffffffULL); -ASSERT_VALUE(EROFS_NAME_LEN, 255); -ASSERT_VALUE(Z_EROFS_PCLUSTER_MAX_SIZE, 1048576); -ASSERT_VALUE(Z_EROFS_PCLUSTER_MAX_DSIZE, 12582912); -ASSERT_VALUE(Z_EROFS_COMPRESSION_LZ4, 0); -ASSERT_VALUE(Z_EROFS_COMPRESSION_LZMA, 1); -ASSERT_VALUE(Z_EROFS_COMPRESSION_DEFLATE, 2); -ASSERT_VALUE(Z_EROFS_COMPRESSION_ZSTD, 3); -ASSERT_VALUE(Z_EROFS_COMPRESSION_MAX, 4); -ASSERT_VALUE(Z_EROFS_ALL_COMPR_ALGS, 15); -ASSERT_VALUE(Z_EROFS_LZMA_MAX_DICT_SIZE, 8388608); -ASSERT_VALUE(Z_EROFS_ZSTD_MAX_DICT_SIZE, 1048576); -ASSERT_VALUE(Z_EROFS_ADVISE_COMPACTED_2B, 0x0001); -ASSERT_VALUE(Z_EROFS_ADVISE_EXTENTS, 0x0001); -ASSERT_VALUE(Z_EROFS_ADVISE_BIG_PCLUSTER_1, 0x0002); -ASSERT_VALUE(Z_EROFS_ADVISE_BIG_PCLUSTER_2, 0x0004); -ASSERT_VALUE(Z_EROFS_ADVISE_INLINE_PCLUSTER, 0x0008); -ASSERT_VALUE(Z_EROFS_ADVISE_INTERLACED_PCLUSTER, 0x0010); -ASSERT_VALUE(Z_EROFS_ADVISE_FRAGMENT_PCLUSTER, 0x0020); -ASSERT_VALUE(Z_EROFS_ADVISE_EXTRECSZ_BIT, 1); -ASSERT_VALUE(Z_EROFS_ADVISE_EXTRECSZ_MASK, 3); -ASSERT_VALUE(Z_EROFS_FRAGMENT_INODE_BIT, 7); -ASSERT_VALUE(Z_EROFS_LCLUSTER_TYPE_PLAIN, 0); -ASSERT_VALUE(Z_EROFS_LCLUSTER_TYPE_HEAD1, 1); -ASSERT_VALUE(Z_EROFS_LCLUSTER_TYPE_NONHEAD, 2); -ASSERT_VALUE(Z_EROFS_LCLUSTER_TYPE_HEAD2, 3); -ASSERT_VALUE(Z_EROFS_LCLUSTER_TYPE_MAX, 4); -ASSERT_VALUE(Z_EROFS_LI_LCLUSTER_TYPE_MASK, 3); -ASSERT_VALUE(Z_EROFS_LI_PARTIAL_REF, 0x8000); -ASSERT_VALUE(Z_EROFS_LI_D0_CBLKCNT, 0x0800); -ASSERT_VALUE(Z_EROFS_MAP_HEADER_END(0), 8); -ASSERT_VALUE(Z_EROFS_MAP_HEADER_END(1), 16); -ASSERT_VALUE(Z_EROFS_FULL_INDEX_START(1), 24); -ASSERT_VALUE(Z_EROFS_EXTENT_PLEN_PARTIAL, 0x08000000); -ASSERT_VALUE(Z_EROFS_EXTENT_PLEN_FMT_BIT, 28); -ASSERT_VALUE(Z_EROFS_EXTENT_PLEN_MASK, 0x001fffff); - -#ifndef B02_LINUX_REFERENCE -ASSERT_VALUE(EROFS_SUPER_MAGIC_V1, 0xE0F5E1E2); -ASSERT_VALUE(EROFS_INODE_LAYOUT_PLAIN, EROFS_INODE_FLAT_PLAIN); -ASSERT_VALUE(EROFS_DIRENT_NID_METABOX, 0x8000000000000000ULL); -ASSERT_VALUE(EROFS_FT_UNKNOWN, 0); -ASSERT_VALUE(EROFS_FT_REG_FILE, 1); -ASSERT_VALUE(EROFS_FT_DIR, 2); -ASSERT_VALUE(EROFS_FT_CHRDEV, 3); -ASSERT_VALUE(EROFS_FT_BLKDEV, 4); -ASSERT_VALUE(EROFS_FT_FIFO, 5); -ASSERT_VALUE(EROFS_FT_SOCK, 6); -ASSERT_VALUE(EROFS_FT_SYMLINK, 7); -#endif - -int -main(void) -{ - return (0); -} diff --git a/tests/pre15/run-build.sh b/tests/pre15/run-build.sh deleted file mode 100755 index ebc3d51..0000000 --- a/tests/pre15/run-build.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -set -eu - -PRE15_LIB_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/lib" && pwd -P) -PRE15_BUILD_SCRIPT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/run-build.sh -export PRE15_LIB_DIR PRE15_BUILD_SCRIPT -. "$PRE15_LIB_DIR/runner.sh" -. "$PRE15_LIB_DIR/manifest.sh" - -pre15_build_worker() -{ - pre15_build_config=$1 - case "$pre15_build_config" in - zstdio0) pre15_zstdio=0 ;; - zstdio1) pre15_zstdio=1 ;; - *) pre15_fail_usage "unknown build configuration: $pre15_build_config" ;; - esac - pre15_require_paths - test "$(uname -s)" = FreeBSD || \ - pre15_infra_blocked 'run-build.sh requires a native FreeBSD build host' - test -d "$PRE15_FREEBSD_SRC/sys" || \ - pre15_infra_blocked "FreeBSD source tree is absent: $PRE15_FREEBSD_SRC" - pre15_write_argv_json "$PRE15_RUN_DIR/build-argv.json" \ - env "FREEBSD_SRC=$PRE15_FREEBSD_SRC" "WITH_ZSTDIO=$pre15_zstdio" \ - "$PRE15_DUT/build.sh" - pre15_target_reached - if env FREEBSD_SRC="$PRE15_FREEBSD_SRC" WITH_ZSTDIO="$pre15_zstdio" \ - "$PRE15_DUT/build.sh"; then - : - else - pre15_dut_fail "native FreeBSD build failed for $pre15_build_config" - fi - pre15_record_module "$PRE15_DUT/build/erofs.ko" -} - -if test "${1:-}" = --worker; then - shift - test "$#" -eq 1 || pre15_fail_usage 'internal build worker requires one config' - pre15_build_worker "$1" - exit 0 -fi - -test "$#" -eq 1 || pre15_fail_usage 'usage: run-build.sh zstdio0|zstdio1' -case "$1" in -zstdio0|zstdio1) ;; -*) pre15_fail_usage "unknown build configuration: $1" ;; -esac -pre15_run_command build "$1" "${PRE15_BUILD_TIMEOUT:-1200}" \ - "$PRE15_BUILD_SCRIPT" --worker "$1" diff --git a/tests/pre15/run-host.sh b/tests/pre15/run-host.sh deleted file mode 100755 index 1654272..0000000 --- a/tests/pre15/run-host.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -eu - -PRE15_LIB_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/lib" && pwd -P) -export PRE15_LIB_DIR -. "$PRE15_LIB_DIR/runner.sh" -. "$PRE15_LIB_DIR/manifest.sh" - -test "$#" -ge 1 || pre15_fail_usage 'usage: run-host.sh CASE [ARG ...]' -pre15_require_paths -pre15_host_case=$1 -shift -pre15_host_script=$(pre15_find_case "$pre15_host_case") -pre15_run_command host "$pre15_host_case" "${PRE15_HOST_TIMEOUT:-240}" \ - /bin/sh "$pre15_host_script" "$@" diff --git a/tests/pre15/run-qemu.sh b/tests/pre15/run-qemu.sh deleted file mode 100755 index 3eabbd0..0000000 --- a/tests/pre15/run-qemu.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/sh -set -eu - -PRE15_LIB_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/lib" && pwd -P) -PRE15_QEMU_SCRIPT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/run-qemu.sh -export PRE15_LIB_DIR PRE15_QEMU_SCRIPT -. "$PRE15_LIB_DIR/runner.sh" -. "$PRE15_LIB_DIR/manifest.sh" - -pre15_pick_port() -{ - python3 - <<'PY' -import socket - -with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - print(sock.getsockname()[1]) -PY -} - -pre15_wait_for_ssh() -{ - pre15_ssh_wait=0 - while test "$pre15_ssh_wait" -lt "${PRE15_QEMU_BOOT_TIMEOUT:-180}"; do - if ! kill -0 "$PRE15_QEMU_PID" 2>/dev/null; then - pre15_infra_blocked 'QEMU exited before SSH became ready' - fi - if pre15_guest_ssh true >/dev/null 2>&1; then - return 0 - fi - sleep 1 - pre15_ssh_wait=$((pre15_ssh_wait + 1)) - done - pre15_infra_blocked 'guest SSH did not become ready before the boot deadline' -} - -pre15_qemu_worker() -{ - pre15_qemu_case=$1 - pre15_require_paths - pre15_qemu_case_script=$(pre15_find_case "$pre15_qemu_case") - : "${PRE15_QEMU_BASE_IMAGE:?PRE15_QEMU_BASE_IMAGE is required}" - : "${PRE15_QEMU_SSH_KEY:?PRE15_QEMU_SSH_KEY is required}" - PRE15_QEMU_SSH_USER=${PRE15_QEMU_SSH_USER:-root} - PRE15_QEMU_BASE_FORMAT=${PRE15_QEMU_BASE_FORMAT:-qcow2} - export PRE15_QEMU_SSH_USER PRE15_QEMU_BASE_FORMAT - for pre15_qemu_tool in qemu-img qemu-system-x86_64 ssh; do - command -v "$pre15_qemu_tool" >/dev/null 2>&1 || \ - pre15_infra_blocked "missing QEMU runner tool: $pre15_qemu_tool" - done - test -f "$PRE15_QEMU_SSH_KEY" || \ - pre15_infra_blocked "SSH key is absent: $PRE15_QEMU_SSH_KEY" - pre15_own_base_image "$PRE15_QEMU_BASE_IMAGE" - pre15_qemu_overlay=$PRE15_RUN_DIR/guest-overlay.qcow2 - qemu-img create -f qcow2 -F "$PRE15_QEMU_BASE_FORMAT" \ - -b "$PRE15_QEMU_BASE_IMAGE" "$pre15_qemu_overlay" - pre15_own_path "$pre15_qemu_overlay" 'QEMU overlay' - PRE15_QEMU_SSH_PORT=$(pre15_pick_port) - export PRE15_QEMU_SSH_PORT - pre15_own_port "$PRE15_QEMU_SSH_PORT" 'QEMU SSH forwarding' - PRE15_QEMU_CONTROL_PATH=$PRE15_RUN_DIR/ssh-control - PRE15_QEMU_SERIAL=$PRE15_RUN_DIR/serial.log - PRE15_QEMU_DEBUG=$PRE15_RUN_DIR/qemu.log - export PRE15_QEMU_CONTROL_PATH PRE15_QEMU_SERIAL PRE15_QEMU_DEBUG - pre15_write_argv_json "$PRE15_RUN_DIR/qemu-argv.json" \ - qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 \ - -m "${PRE15_QEMU_MEMORY_MB:-6144}" -smp "${PRE15_QEMU_CPUS:-4}" \ - -drive "file=$pre15_qemu_overlay,if=virtio,format=qcow2" \ - -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$PRE15_QEMU_SSH_PORT-:22" \ - -device virtio-net-pci,netdev=net0 -display none \ - -serial "file:$PRE15_QEMU_SERIAL" -monitor none - qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 \ - -m "${PRE15_QEMU_MEMORY_MB:-6144}" -smp "${PRE15_QEMU_CPUS:-4}" \ - -drive "file=$pre15_qemu_overlay,if=virtio,format=qcow2" \ - -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$PRE15_QEMU_SSH_PORT-:22" \ - -device virtio-net-pci,netdev=net0 -display none \ - -serial "file:$PRE15_QEMU_SERIAL" -monitor none & - PRE15_QEMU_PID=$! - export PRE15_QEMU_PID - pre15_own_pid "$PRE15_QEMU_PID" QEMU - pre15_wait_for_ssh - if ssh -MNf -o BatchMode=yes -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - -o ControlMaster=yes -o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \ - -i "$PRE15_QEMU_SSH_KEY" -p "$PRE15_QEMU_SSH_PORT" \ - "$PRE15_QEMU_SSH_USER@127.0.0.1"; then - pre15_own_ssh_control "$PRE15_QEMU_CONTROL_PATH" 'QEMU SSH ControlMaster' - else - pre15_infra_blocked 'could not establish SSH ControlMaster' - fi - pre15_guest_ssh_bounded uname -a > "$PRE15_RUN_DIR/guest-uname.txt" - pre15_guest_ssh_bounded kldstat > "$PRE15_RUN_DIR/guest-kldstat-before.txt" - pre15_guest_ssh_bounded dmesg > "$PRE15_RUN_DIR/guest-dmesg-before.txt" - /bin/sh "$pre15_qemu_case_script" - pre15_guest_ssh_bounded dmesg > "$PRE15_RUN_DIR/guest-dmesg-after.txt" -} - -if test "${1:-}" = --worker; then - shift - test "$#" -eq 1 || pre15_fail_usage 'internal QEMU worker requires one case' - pre15_qemu_worker "$1" - exit 0 -fi - -test "$#" -eq 1 || pre15_fail_usage 'usage: run-qemu.sh CASE' -pre15_require_paths -pre15_find_case "$1" >/dev/null -pre15_run_command qemu "$1" "${PRE15_QEMU_TIMEOUT:-1200}" \ - "$PRE15_QEMU_SCRIPT" --worker "$1" diff --git a/tests/pre15/run-smoke.sh b/tests/pre15/run-smoke.sh deleted file mode 100755 index 277177e..0000000 --- a/tests/pre15/run-smoke.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -set -eu - -PRE15_LIB_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/lib" && pwd -P) -PRE15_SMOKE_SCRIPT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/run-smoke.sh -export PRE15_LIB_DIR PRE15_SMOKE_SCRIPT -. "$PRE15_LIB_DIR/runner.sh" -. "$PRE15_LIB_DIR/manifest.sh" - -pre15_smoke_worker() -{ - pre15_require_paths - pre15_target_reached - for pre15_smoke_case in SMOKE-PLAIN SMOKE-LZ4 SMOKE-LZMA SMOKE-ZSTD; do - if PRE15_EVIDENCE_ROOT="$PRE15_RUN_DIR/subruns" \ - "$PRE15_DUT/tests/pre15/run-qemu.sh" "$pre15_smoke_case"; then - : - else - pre15_smoke_rc=$? - case "$pre15_smoke_rc" in - "$PRE15_RC_DUT_FAIL") pre15_dut_fail "$pre15_smoke_case failed its DUT oracle" ;; - "$PRE15_RC_INFRA_BLOCKED") pre15_infra_blocked "$pre15_smoke_case was infrastructure-blocked" ;; - *) pre15_runner_fail "$pre15_smoke_case runner failed" ;; - esac - fi - done -} - -if test "${1:-}" = --worker; then - shift - test "$#" -eq 0 || pre15_fail_usage 'internal smoke worker takes no arguments' - pre15_smoke_worker - exit 0 -fi - -test "$#" -eq 1 && test "$1" = final-four-codec || \ - pre15_fail_usage 'usage: run-smoke.sh final-four-codec' -pre15_run_command smoke final-four-codec "${PRE15_SMOKE_TIMEOUT:-1500}" \ - "$PRE15_SMOKE_SCRIPT" --worker diff --git a/tests/prepare_directory_fixtures.sh b/tests/prepare_directory_fixtures.sh deleted file mode 100755 index 049d6a3..0000000 --- a/tests/prepare_directory_fixtures.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -set -eu - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -output_dir=${1:?usage: prepare_directory_fixtures.sh OUTPUT-DIRECTORY} - -for tool in mkfs.erofs fsck.erofs dump.erofs python3; do - command -v "$tool" >/dev/null 2>&1 || { - echo "missing tool: $tool" >&2 - exit 1 - } -done -test ! -e "$output_dir" || { - echo "output already exists: $output_dir" >&2 - exit 1 -} - -work_dir=$(mktemp -d "${TMPDIR:-/tmp}/repo22-directory.XXXXXX") -trap 'rm -rf "$work_dir"' EXIT HUP INT TERM -source_dir="$work_dir/source" -mkdir -p "$source_dir/alpha/bravo/charlie" \ - "$source_dir/alpha/sibling" "$source_dir/wide" - -printf 'cold nested lookup payload\n' \ - > "$source_dir/alpha/bravo/charlie/payload.txt" -printf 'repeat lookup payload\n' > "$source_dir/alpha/bravo/repeat.txt" -printf 'sibling marker\n' > "$source_dir/alpha/sibling/marker.txt" -: > "$work_dir/expected-wide.txt" -index=0 -while [ "$index" -lt 320 ]; do - name=$(printf 'entry-%03d-abcdefghijklmnopqrstuvwxyz.txt' "$index") - printf 'wide entry %03d\n' "$index" > "$source_dir/wide/$name" - printf '%s\n' "$name" >> "$work_dir/expected-wide.txt" - index=$((index + 1)) -done -find "$source_dir" -exec touch -h -t 197001010000.00 {} + - -mkfs.erofs --quiet -d0 -x-1 -T0 --all-time --all-root --workers=1 \ - -U 11111111-2222-3333-4444-555555555554 \ - "$work_dir/namei-base.erofs" "$source_dir" -python3 "$script_dir/erofs_fixture.py" make-directory-fixtures \ - --base "$work_dir/namei-base.erofs" --output "$output_dir" -cp -R "$source_dir" "$output_dir/source" -cp "$work_dir/expected-wide.txt" "$output_dir/expected-wide.txt" - -fsck.erofs -d1 "$output_dir/namei-padding-nonzero.erofs" -dump.erofs --cat \ - --path=/wide/entry-079-abcdefghijklmnopqrstuvwxyz.txt \ - "$output_dir/namei-padding-nonzero.erofs" \ - > "$output_dir/padding-file.txt" -cmp "$source_dir/wide/entry-079-abcdefghijklmnopqrstuvwxyz.txt" \ - "$output_dir/padding-file.txt" -cat "$output_dir/fixture-evidence.txt" -cat "$output_dir/SHA256SUMS" diff --git a/tests/prepare_error_fixtures.sh b/tests/prepare_error_fixtures.sh deleted file mode 100755 index aea11eb..0000000 --- a/tests/prepare_error_fixtures.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/sh -set -eu - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -output_dir=${1:?usage: prepare_error_fixtures.sh OUTPUT-DIRECTORY} - -for tool in mkfs.erofs dump.erofs fsck.erofs python3; do - command -v "$tool" >/dev/null 2>&1 || { - echo "missing tool: $tool" >&2 - exit 1 - } -done -test ! -e "$output_dir" || { - echo "output already exists: $output_dir" >&2 - exit 1 -} - -work_dir=$(mktemp -d "${TMPDIR:-/tmp}/repo22-errors.XXXXXX") -trap 'rm -rf "$work_dir"' EXIT HUP INT TERM -source_dir="$work_dir/source" -mkdir -p "$source_dir" - -printf 'directory entry target\n' > "$source_dir/bad-entry.txt" -printf 'inode format target\n' > "$source_dir/inode-target.txt" -printf 'high offset data path\n' > "$source_dir/high-offset.txt" -printf 'control file\n' > "$source_dir/control.txt" -dd if=/dev/zero of="$source_dir/plain.bin" bs=1048576 count=2 status=none -awk 'BEGIN { for (i = 0; i < 131072; i++) - printf "%08d deterministic compressed payload\n", i }' \ - > "$source_dir/compressed.bin" -find "$source_dir" -exec touch -h -t 197001010000.00 {} + - -mkfs.erofs --quiet -d0 -x-1 -T0 --all-time --all-root --workers=1 \ - -U 33333333-4444-5555-6666-777777777771 \ - -E noinline_data,force-inode-extended \ - "$work_dir/plain.erofs" "$source_dir" -mkfs.erofs --quiet -d0 -x-1 -T0 --all-time --all-root --workers=1 \ - -U 33333333-4444-5555-6666-777777777772 \ - -E legacy-compress,force-inode-extended -zlz4 -C65536 \ - "$work_dir/lz4.erofs" "$source_dir" -mkfs.erofs --quiet -d0 -x-1 -T0 --all-time --all-root --workers=1 \ - -U 33333333-4444-5555-6666-777777777773 \ - -E legacy-compress,force-inode-extended -zdeflate,level=1 -C65536 \ - "$work_dir/deflate.erofs" "$source_dir" - -dump.erofs --path=/compressed.bin -e "$work_dir/lz4.erofs" \ - > "$work_dir/compressed.extents" -set -- $(awk '/^[[:space:]]*0:/ { - gsub(/\.\./, "", $7); print $7, $NF; exit -}' "$work_dir/compressed.extents") -test "$#" -eq 2 || { - echo "could not parse first compressed extent" >&2 - exit 1 -} -compressed_offset=$1 -compressed_length=$2 - -python3 "$script_dir/erofs_fixture.py" make-error-fixtures \ - --plain "$work_dir/plain.erofs" \ - --compressed "$work_dir/lz4.erofs" \ - --deflate "$work_dir/deflate.erofs" \ - --compressed-offset "$compressed_offset" \ - --compressed-length "$compressed_length" \ - --output "$output_dir" -cp -R "$source_dir" "$output_dir/source" -cp "$work_dir/compressed.extents" "$output_dir/compressed.extents" - -extract_dir="$work_dir/extract-control" -mkdir "$extract_dir" -fsck.erofs --extract="$extract_dir" "$output_dir/valid-lz4.erofs" -cmp "$source_dir/compressed.bin" "$extract_dir/compressed.bin" -mkdir "$work_dir/extract-deflate" -fsck.erofs --extract="$work_dir/extract-deflate" \ - "$output_dir/valid-deflate-level1.erofs" -cmp "$source_dir/compressed.bin" "$work_dir/extract-deflate/compressed.bin" -mkdir "$work_dir/extract-corrupt" -set +e -fsck.erofs --extract="$work_dir/extract-corrupt" \ - "$output_dir/compressed-stream-corrupt.erofs" \ - > "$output_dir/compressed-corrupt-fsck.txt" 2>&1 -fsck_status=$? -set -e -test "$fsck_status" -ne 0 || { - echo "corrupted compressed stream unexpectedly extracted" >&2 - exit 1 -} -printf 'compressed_corrupt_fsck_status=%s\n' "$fsck_status" \ - >> "$output_dir/fixture-evidence.txt" -cat "$output_dir/fixture-evidence.txt" -cat "$output_dir/SHA256SUMS" diff --git a/tests/prepare_g1_fixtures.sh b/tests/prepare_g1_fixtures.sh deleted file mode 100755 index 658eb39..0000000 --- a/tests/prepare_g1_fixtures.sh +++ /dev/null @@ -1,196 +0,0 @@ -#!/bin/sh -set -eu - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -output=${1:?usage: prepare_g1_fixtures.sh OUTPUT} -source_dir="$output/source" -base_dir="$output/base" -image_dir="$output/images" -expected_dir="$output/expected" - -for tool in mkfs.erofs fsck.erofs dump.erofs python3 sha256sum stat; do - command -v "$tool" >/dev/null 2>&1 || { - echo "missing tool: $tool" >&2 - exit 1 - } -done -test "$(id -u)" -eq 0 || { - echo "root is required to create device-node fixtures" >&2 - exit 1 -} -test ! -e "$output" || { - echo "output already exists: $output" >&2 - exit 1 -} - -mkdir -p \ - "$source_dir/compact/dir" \ - "$source_dir/compact/dotdir" \ - "$source_dir/extended" \ - "$source_dir/flat/very/long/path/to/a/deeply/nested/deterministic/target/directory" \ - "$source_dir/inline" \ - "$base_dir" "$expected_dir" - -SOURCE_DIR="$source_dir" EXPECTED_DIR="$expected_dir" python3 <<'PY' -from pathlib import Path -import os - -source = Path(os.environ["SOURCE_DIR"]) -expected = Path(os.environ["EXPECTED_DIR"]) - - -def write_pattern(path: Path, size: int, seed: int) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("wb") as out: - offset = 0 - while offset < size: - count = min(65536, size - offset) - out.write(bytes( - (((offset + index) * 131 + seed) ^ ((offset + index) >> 5)) & 0xff - for index in range(count) - )) - offset += count - - -compact = source / "compact" -(compact / "root.txt").write_text("repo22 G1 root payload\n", encoding="ascii") -write_pattern(compact / "testfile.txt", 65537, 1) -(compact / "small.txt").write_text("compact inode payload\n", encoding="ascii") -(compact / "single.txt").write_text("single-link payload\n", encoding="ascii") -(compact / "hard-a.txt").write_text("hard-link payload\n", encoding="ascii") -(compact / "dotdir" / "child.txt").write_text("dot directory child\n", encoding="ascii") -(compact / "invalid-target.txt").write_text("invalid nid target\n", encoding="ascii") -(compact / "special-target.txt").write_text("special target\n", encoding="ascii") - -extended = source / "extended" -write_pattern(extended / "large-file.bin", 2 * 1024 * 1024 + 731, 2) -(extended / "huge-sparse.dat").write_bytes(b"x") -(extended / "oversize.dat").write_bytes(b"x") - -flat = source / "flat" -(flat / "small.txt").write_text("flat plain small payload\n", encoding="ascii") -write_pattern(flat / "medium.dat", 100 * 1024 + 17, 3) -write_pattern(flat / "large.bin", 10 * 1024 * 1024 + 31, 4) -write_pattern(flat / "data.txt", 256 * 1024 + 19, 5) -write_pattern(flat / "random-test.bin", 1024 * 1024, 6) -write_pattern(flat / "huge-data.bin", 100 * 1024 * 1024 + 257, 7) -(flat / "very" / "long" / "path" / "to" / "a" / "deeply" / "nested" / - "deterministic" / "target" / "directory" / "file.txt").write_text( - "long symlink target payload\n", encoding="ascii" -) - -inline = source / "inline" -(inline / "tiny.txt").write_text("inline data\n", encoding="ascii") -(inline / "empty.txt").write_bytes(b"") -(inline / "inline.txt").write_bytes(b"inline-tail-payload-0123456789\n") -write_pattern(inline / "tailpacked.dat", 5000, 8) -write_pattern(inline / "file-4095.dat", 4095, 9) -write_pattern(inline / "file-4096.dat", 4096, 10) -write_pattern(inline / "file-4097.dat", 4097, 11) -(inline / "target.txt").write_text("short symlink target payload\n", encoding="ascii") - -for source_path, offset, length, output_name in ( - (flat / "medium.dat", 10 * 4096, 5 * 4096, "medium-10-5.bin"), - (flat / "random-test.bin", 0, 10 * 1024, "random-start.bin"), - (flat / "random-test.bin", 500 * 1024, 10 * 1024, "random-mid.bin"), - (flat / "random-test.bin", 1000 * 1024, 24 * 1024, "random-end.bin"), - (flat / "huge-data.bin", 50 * 1024 * 1024, 5 * 1024 * 1024, - "huge-sample.bin"), -): - with source_path.open("rb") as handle: - handle.seek(offset) - data = handle.read(length) - if len(data) != length: - raise RuntimeError(f"short expected range from {source_path}") - (expected / output_name).write_bytes(data) -(expected / "large-hole-zero-64k.bin").write_bytes(bytes(65536)) -(expected / "large-hole-description.txt").write_text( - "path=/huge-sparse.dat size=4294971393 content=all-zero\n", encoding="ascii" -) -PY - -ln "$source_dir/compact/testfile.txt" "$source_dir/compact/dir/testfile.txt" -ln "$source_dir/compact/hard-a.txt" "$source_dir/compact/hard-b.txt" -ln -s special-target.txt "$source_dir/compact/special-link" -mknod "$source_dir/compact/char-large" c 2748 344865 -mknod "$source_dir/compact/block-large" b 2748 344865 -mkfifo "$source_dir/compact/fifo" -ln -s target.txt "$source_dir/inline/link1" -ln -s /nonexistent/repo22-g1-target "$source_dir/inline/brokenlink" -ln -s very/long/path/to/a/deeply/nested/deterministic/target/directory/file.txt \ - "$source_dir/flat/longlink" -find "$source_dir" -exec touch -h -t 197001010000.00 {} + - -build_image() -{ - uuid=$1 - image=$2 - source=$3 - shift 3 - mkfs.erofs -d0 -x-1 -T0 --all-time --all-root --workers=1 \ - -U "$uuid" "$@" "$base_dir/$image" "$source_dir/$source" -} - -build_image 31000000-0000-0000-0000-000000000001 \ - compact.erofs compact -E force-inode-compact -build_image 31000000-0000-0000-0000-000000000002 \ - extended.erofs extended -E force-inode-extended -build_image 31000000-0000-0000-0000-000000000003 \ - flat.erofs flat -E noinline_data,force-inode-compact -build_image 31000000-0000-0000-0000-000000000004 \ - inline.erofs inline -E force-inode-compact - -python3 "$script_dir/g1_fixtures.py" \ - --compact "$base_dir/compact.erofs" \ - --extended "$base_dir/extended.erofs" \ - --flat "$base_dir/flat.erofs" \ - --inline "$base_dir/inline.erofs" \ - --output "$image_dir" - -for image in compact.erofs extended.erofs flat.erofs inline.erofs \ - compact-nlink1.erofs inline-zero.erofs extended-large-hole.erofs; do - fsck.erofs "$image_dir/$image" >/dev/null -done - -printf '%s\n' \ - '48-bit fixtures are field/CRC checked by g1_fixtures.py;' \ - 'production fsck.erofs 1.8.6 does not recognize incompat feature 0x80.' - -dump.erofs --cat --path=/small.txt "$image_dir/compact.erofs" | - cmp - "$source_dir/compact/small.txt" -dump.erofs --cat --path=/large-file.bin "$image_dir/extended.erofs" | - cmp - "$source_dir/extended/large-file.bin" -dump.erofs --cat --path=/medium.dat "$image_dir/flat.erofs" | - cmp - "$source_dir/flat/medium.dat" -dump.erofs --cat --path=/tailpacked.dat "$image_dir/inline.erofs" | - cmp - "$source_dir/inline/tailpacked.dat" - -( - cd "$output" - find source expected -type f -print0 | sort -z | xargs -0 sha256sum -) > "$output/SOURCE-SHA256SUMS" -( - cd "$output" - find source -type l -print | sort | while IFS= read -r path; do - printf '%s -> %s\n' "$path" "$(readlink "$path")" - done - find source -type p -printf '%p type=fifo\n' - for kind in block:b char:c; do - label=${kind%:*} - type=${kind#*:} - find source -type "$type" -print | sort | while IFS= read -r path; do - set -- $(stat -c '%t %T' "$path") - printf '%s type=%s major=%d minor=%d\n' \ - "$path" "$label" "$((0x$1))" "$((0x$2))" - done - done -) > "$output/SOURCE-METADATA" -( - cd "$output" - sha256sum SOURCE-METADATA -) > "$output/SOURCE-METADATA.sha256" - -cat "$image_dir/fixture-evidence.txt" -cat "$output/SOURCE-SHA256SUMS" -cat "$output/SOURCE-METADATA" -cat "$image_dir/IMAGE-SHA256SUMS" diff --git a/tests/prepare_g3_fixtures.sh b/tests/prepare_g3_fixtures.sh deleted file mode 100755 index 736e80d..0000000 --- a/tests/prepare_g3_fixtures.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -set -eu -umask 022 - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) -output_dir=${1:?usage: prepare_g3_fixtures.sh OUTPUT-DIRECTORY} - -for tool in mkfs.erofs fsck.erofs dump.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - printf 'missing tool: %s\n' "$tool" >&2 - exit 1 - } -done -test ! -e "$output_dir" || { - printf 'output already exists: %s\n' "$output_dir" >&2 - exit 1 -} - -python3 -B "$script_dir/pre15/fixtures/g3.py" make-all --output "$output_dir" -python3 -B "$script_dir/pre15/fixtures/g3.py" verify --output "$output_dir" -printf 'G3_MANIFEST=%s\n' "$output_dir/G3-MANIFEST.json" diff --git a/tests/read_probe.c b/tests/read_probe.c deleted file mode 100644 index e6d06c1..0000000 --- a/tests/read_probe.c +++ /dev/null @@ -1,128 +0,0 @@ -#define _POSIX_C_SOURCE 200809L - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static uint64_t -parse_number(const char *text, const char *what) -{ - char *end; - uintmax_t value; - - errno = 0; - value = strtoumax(text, &end, 0); - if (errno != 0 || *text == '\0' || *end != '\0' || value > INT64_MAX) - errx(2, "invalid %s: %s", what, text); - return ((uint64_t)value); -} - -static void -write_all(int fd, const unsigned char *buffer, size_t length) -{ - size_t done; - ssize_t written; - - for (done = 0; done < length; done += (size_t)written) { - written = write(fd, buffer + done, length - done); - if (written < 0) - err(1, "write output"); - } -} - -static void -pread_range(const char *path, uint64_t raw_offset, uint64_t raw_length, - const char *output) -{ - unsigned char buffer[65536]; - uint64_t done; - ssize_t count; - size_t request; - int fd, outfd; - - if (raw_length > INT64_MAX - raw_offset) - errx(2, "offset plus length overflows off_t"); - fd = open(path, O_RDONLY); - if (fd < 0) - err(1, "open %s", path); - outfd = open(output, O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (outfd < 0) - err(1, "open %s", output); - for (done = 0; done < raw_length; done += (uint64_t)count) { - request = raw_length - done < sizeof(buffer) ? - (size_t)(raw_length - done) : sizeof(buffer); - count = pread(fd, buffer, request, (off_t)(raw_offset + done)); - if (count < 0) - err(1, "pread %s at %ju", path, - (uintmax_t)(raw_offset + done)); - if (count == 0) - errx(1, "unexpected EOF at %ju", (uintmax_t)(raw_offset + done)); - write_all(outfd, buffer, (size_t)count); - } - if (close(outfd) != 0 || close(fd) != 0) - err(1, "close"); - printf("bytes=%ju offset=%ju\n", (uintmax_t)raw_length, - (uintmax_t)raw_offset); -} - -static void -expect_error(const char *path, int expected) -{ - unsigned char buffer[65536]; - ssize_t count; - int fd; - - fd = open(path, O_RDONLY); - if (fd < 0) { - if (errno != expected) - errx(1, "open returned %s, expected errno %d", - strerror(errno), expected); - printf("expected_errno=%d stage=open\n", expected); - return; - } - for (;;) { - errno = 0; - count = read(fd, buffer, sizeof(buffer)); - if (count < 0) - break; - if (count == 0) - errx(1, "read reached EOF without expected errno %d", expected); - } - if (errno != expected) - errx(1, "read returned %s, expected errno %d", - strerror(errno), expected); - if (close(fd) != 0) - err(1, "close"); - printf("expected_errno=%d stage=read\n", expected); -} - -int -main(int argc, char **argv) -{ - uint64_t offset, length, error_number; - - if (argc == 6 && strcmp(argv[1], "pread") == 0) { - offset = parse_number(argv[3], "offset"); - length = parse_number(argv[4], "length"); - pread_range(argv[2], offset, length, argv[5]); - return (0); - } - if (argc == 4 && strcmp(argv[1], "expect-error") == 0) { - error_number = parse_number(argv[3], "errno"); - if (error_number == 0 || error_number > INT_MAX) - errx(2, "errno is out of range"); - expect_error(argv[2], (int)error_number); - return (0); - } - errx(2, "usage: %s pread FILE OFFSET LENGTH OUTPUT | " - "expect-error FILE ERRNO", argv[0]); -} diff --git a/tests/readdir_probe.c b/tests/readdir_probe.c deleted file mode 100644 index 7cb773f..0000000 --- a/tests/readdir_probe.c +++ /dev/null @@ -1,329 +0,0 @@ -#ifndef __FreeBSD__ -#define _DEFAULT_SOURCE -#endif - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct observed_entry { - char *name; - off_t cookie; - unsigned char type; -}; - -struct observed_list { - struct observed_entry *entries; - size_t count; - size_t capacity; -}; - -static void -append_entry(struct observed_list *list, const char *name, off_t cookie, - unsigned char type) -{ - struct observed_entry *entries; - - if (list->count == list->capacity) { - list->capacity = list->capacity == 0 ? 64 : list->capacity * 2; - entries = realloc(list->entries, - list->capacity * sizeof(*list->entries)); - if (entries == NULL) - err(1, "realloc entries"); - list->entries = entries; - } - list->entries[list->count].name = strdup(name); - if (list->entries[list->count].name == NULL) - err(1, "strdup"); - list->entries[list->count].cookie = cookie; - list->entries[list->count].type = type; - list->count++; -} - -static uint64_t -entry_hash(const struct observed_list *list) -{ - uint64_t hash; - - hash = UINT64_C(14695981039346656037); - for (size_t index = 0; index < list->count; index++) { - for (const unsigned char *name = - (const unsigned char *)list->entries[index].name; - *name != '\0'; name++) { - hash ^= *name; - hash *= UINT64_C(1099511628211); - } - hash ^= 0; - hash *= UINT64_C(1099511628211); - hash ^= list->entries[index].type; - hash *= UINT64_C(1099511628211); - } - return (hash); -} - -static void -print_type_counts(const struct observed_list *list) -{ - size_t block, character, directory, fifo, link, other, regular, socket, - unknown; - - block = character = directory = fifo = link = other = regular = socket = - unknown = 0; - for (size_t index = 0; index < list->count; index++) { - switch (list->entries[index].type) { - case DT_BLK: - block++; - break; - case DT_CHR: - character++; - break; - case DT_DIR: - directory++; - break; - case DT_FIFO: - fifo++; - break; - case DT_LNK: - link++; - break; - case DT_REG: - regular++; - break; - case DT_SOCK: - socket++; - break; - case DT_UNKNOWN: - unknown++; - break; - default: - other++; - break; - } - } - printf("types=dir:%zu,reg:%zu,lnk:%zu,chr:%zu,blk:%zu,fifo:%zu," - "sock:%zu,unknown:%zu,other:%zu ", directory, regular, link, - character, block, fifo, socket, unknown, other); -} - -static void -free_list(struct observed_list *list) -{ - for (size_t index = 0; index < list->count; index++) - free(list->entries[index].name); - free(list->entries); -} - -static void -check_unique_names(const struct observed_list *list) -{ - for (size_t left = 0; left < list->count; left++) { - for (size_t right = left + 1; right < list->count; right++) { - if (strcmp(list->entries[left].name, - list->entries[right].name) == 0) - errx(1, "duplicate name: %s", - list->entries[left].name); - } - } -} - -static struct observed_list -scan_getdirentries(const char *path, size_t buffer_size) -{ - struct observed_list list = { 0 }; - struct dirent *entry; - char *buffer, *end; - off_t base, previous; - ssize_t bytes; - int fd; - - fd = open(path, O_RDONLY | O_DIRECTORY); - if (fd < 0) - err(1, "open %s", path); - buffer = malloc(buffer_size); - if (buffer == NULL) - err(1, "malloc"); - previous = 0; - for (;;) { - base = -1; - bytes = getdirentries(fd, buffer, buffer_size, &base); - if (bytes < 0) - err(1, "getdirentries %s", path); - if (bytes == 0) - break; - end = buffer + bytes; - for (entry = (struct dirent *)buffer; (char *)entry < end; - entry = (struct dirent *)((char *)entry + entry->d_reclen)) { - if (entry->d_reclen == 0 || - (char *)entry + entry->d_reclen > end) - errx(1, "invalid dirent record"); - if (entry->d_off <= previous) - errx(1, "non-increasing d_off %jd after %jd", - (intmax_t)entry->d_off, (intmax_t)previous); - append_entry(&list, entry->d_name, entry->d_off, - entry->d_type); - previous = entry->d_off; - } - } - free(buffer); - if (close(fd) != 0) - err(1, "close %s", path); - check_unique_names(&list); - return (list); -} - -static void -verify_kernel_cookies(const char *path, size_t buffer_size, - const struct observed_list *list) -{ - struct dirent *entry; - char *buffer; - off_t base; - ssize_t bytes; - int fd; - - buffer = malloc(buffer_size); - if (buffer == NULL) - err(1, "malloc restart buffer"); - for (size_t index = 0; index < list->count; index++) { - fd = open(path, O_RDONLY | O_DIRECTORY); - if (fd < 0) - err(1, "open restart %s", path); - if (lseek(fd, list->entries[index].cookie, SEEK_SET) < 0) - err(1, "lseek cookie %jd", - (intmax_t)list->entries[index].cookie); - base = -1; - bytes = getdirentries(fd, buffer, buffer_size, &base); - if (bytes < 0) - err(1, "getdirentries restart"); - if (index + 1 == list->count) { - if (bytes != 0) - errx(1, "final d_off cookie did not reach EOF"); - } else { - if (bytes <= 0) - errx(1, "restart cookie returned no dirent"); - entry = (struct dirent *)buffer; - if (entry->d_reclen == 0 || entry->d_reclen > (size_t)bytes) - errx(1, "restart cookie returned a truncated dirent"); - if (strcmp(entry->d_name, list->entries[index + 1].name) != 0) - errx(1, "cookie %jd resumed at %s, expected %s", - (intmax_t)list->entries[index].cookie, - entry->d_name, list->entries[index + 1].name); - } - if (close(fd) != 0) - err(1, "close restart"); - } - free(buffer); -} - -static struct observed_list -scan_readdir(const char *path) -{ - struct observed_list list = { 0 }; - struct dirent *entry; - DIR *directory; - long cookie, previous; - - directory = opendir(path); - if (directory == NULL) - err(1, "opendir %s", path); - previous = 0; - while ((entry = readdir(directory)) != NULL) { - cookie = telldir(directory); - if (cookie <= previous) - errx(1, "non-increasing telldir cookie %ld after %ld", - cookie, previous); - append_entry(&list, entry->d_name, (off_t)cookie, - entry->d_type); - previous = cookie; - } - if (closedir(directory) != 0) - err(1, "closedir %s", path); - check_unique_names(&list); - return (list); -} - -static void -verify_seekdir(const char *path, const struct observed_list *list) -{ - struct dirent *entry; - DIR *directory; - long cookie; - - for (size_t index = 0; index < list->count; index++) { - directory = opendir(path); - if (directory == NULL) - err(1, "opendir restart %s", path); - for (size_t position = 0; position <= index; position++) { - entry = readdir(directory); - if (entry == NULL || - strcmp(entry->d_name, list->entries[position].name) != 0) - errx(1, "seekdir setup differs at entry %zu", - position); - } - cookie = telldir(directory); - if ((off_t)cookie != list->entries[index].cookie) - errx(1, "telldir cookie differs at entry %zu", index); - seekdir(directory, cookie); - entry = readdir(directory); - if (index + 1 == list->count) { - if (entry != NULL) - errx(1, "final telldir cookie did not reach EOF"); - } else if (entry == NULL || - strcmp(entry->d_name, list->entries[index + 1].name) != 0) { - errx(1, "seekdir cookie %jd resumed at %s, expected %s", - (intmax_t)list->entries[index].cookie, - entry == NULL ? "EOF" : entry->d_name, - list->entries[index + 1].name); - } - if (closedir(directory) != 0) - err(1, "closedir restart"); - } -} - -int -main(int argc, char **argv) -{ - struct observed_list kernel, libc; - char *end; - unsigned long raw_size; - size_t buffer_size; - - if (argc < 2 || argc > 3) - errx(2, "usage: %s directory [buffer-size]", argv[0]); - buffer_size = 128; - if (argc == 3) { - raw_size = strtoul(argv[2], &end, 0); - if (*argv[2] == '\0' || *end != '\0' || raw_size < 64 || - raw_size > 1024 * 1024) - errx(2, "invalid buffer size: %s", argv[2]); - buffer_size = (size_t)raw_size; - } - kernel = scan_getdirentries(argv[1], buffer_size); - verify_kernel_cookies(argv[1], buffer_size, &kernel); - libc = scan_readdir(argv[1]); - verify_seekdir(argv[1], &libc); - if (kernel.count != libc.count) - errx(1, "getdirentries count %zu differs from readdir count %zu", - kernel.count, libc.count); - for (size_t index = 0; index < kernel.count; index++) { - if (strcmp(kernel.entries[index].name, libc.entries[index].name) != 0) - errx(1, "API order differs at entry %zu", index); - if (kernel.entries[index].type != libc.entries[index].type) - errx(1, "API type differs at entry %zu", index); - } - print_type_counts(&kernel); - printf("entries=%zu d_off_restarts=%zu seekdir_restarts=%zu " - "buffer=%zu fnv1a64=%016" PRIx64 "\n", kernel.count, - kernel.count, libc.count, buffer_size, entry_hash(&kernel)); - free_list(&libc); - free_list(&kernel); - return (0); -} diff --git a/tests/results/manual/2026-08-08T1037Z/manual-test-report.md b/tests/results/manual/2026-08-08T1037Z/manual-test-report.md deleted file mode 100644 index 75d92bf..0000000 --- a/tests/results/manual/2026-08-08T1037Z/manual-test-report.md +++ /dev/null @@ -1,188 +0,0 @@ -# repo22 手工测试执行记录 - -- 执行时间：2026-08-08 10:36-10:38 UTC -- 工作区：`/work` -- 仓库：`/work/repo-community/repo22` -- 执行方式：严格按 Markdown 用例进行前置条件检查；未生成 wrapper 或 CI；未修改代码；未 commit/push -- 总结：TC001、TC002、TC007、TC008、TC009、TC010 均为 **BLOCKED**。没有用例进入 FreeBSD 客体内的正式测试步骤，因此没有伪造 PASS/FAIL。 - -## 环境版本 - -实际命令： - -```sh -date -u '+%Y-%m-%dT%H:%M:%SZ' -uname -a -cat /etc/os-release -``` - -实际输出： - -```text -2026-08-08T10:37:36Z -Linux da8f2da26d77 6.12.74+deb13+1-cloud-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.12.74-2 (2026-03-08) x86_64 GNU/Linux -PRETTY_NAME="Debian GNU/Linux 13 (trixie)" -VERSION_ID="13" -DEBIAN_VERSION_FULL=13.5 -``` - -结论：当前 shell 位于 Linux 容器，不是 FreeBSD；`kldload`、`kldstat`、`mdconfig` 等用例命令不能在此环境代替执行。 - -## FreeBSD VM 可访问性 - -实际命令： - -```sh -timeout 3 bash -c ' /tmp/repo22-ssh-askpass.sh <<'EOF' -#!/bin/sh -printf '%s\n' '<固定测试密码>' -EOF -chmod 700 /tmp/repo22-ssh-askpass.sh -``` - -随后所有 SSH/SCP 命令使用以下认证前缀和选项： - -```sh -DISPLAY=:0 SSH_ASKPASS=/tmp/repo22-ssh-askpass.sh \ -SSH_ASKPASS_REQUIRE=force setsid -w ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o PreferredAuthentications=keyboard-interactive,password \ - -o PubkeyAuthentication=no \ - -p 9222 root@127.0.0.1 '' -``` - -## 2. Guest 环境验证 - -实际 guest 命令： - -```sh -echo "== UTC date =="; date -u "+%Y-%m-%dT%H:%M:%SZ" -echo "== identity =="; id -echo "== uname =="; uname -a -echo "== freebsd-version =="; freebsd-version -ku 2>&1 -echo "== hostname =="; hostname -echo "== architecture =="; uname -m; sysctl -n hw.machine_arch 2>&1 -echo "== securelevel =="; sysctl kern.securelevel -echo "== module tools =="; command -v kldload; command -v kldstat; command -v kldunload -echo "== filesystem tools =="; command -v mdconfig; command -v mount; command -v umount; command -v sha256; command -v stat -echo "== existing erofs module/mount/md =="; kldstat | grep -i erofs || true; mount | grep -i erofs || true; mdconfig -l -echo "== temp space =="; df -h /tmp /root -``` - -实际输出： - -```text -Warning: Permanently added '[127.0.0.1]:9222' (ED25519) to the list of known hosts. -== UTC date == -2026-08-08T11:10:13Z -== identity == -uid=0(root) gid=0(wheel) groups=0(wheel),5(operator) -== uname == -FreeBSD freebsd-build 15.0-RELEASE-p8 FreeBSD 15.0-RELEASE-p8 releng/15.0-n281036-53054229dcb3 GENERIC amd64 -== freebsd-version == -15.0-RELEASE-p8 -15.0-RELEASE-p8 -== hostname == -freebsd-build -== architecture == -amd64 -amd64 -== securelevel == -kern.securelevel: -1 -== module tools == -/sbin/kldload -/sbin/kldstat -/sbin/kldunload -== filesystem tools == -/sbin/mdconfig -/sbin/mount -/sbin/umount -/sbin/sha256 -/usr/bin/stat -== existing erofs module/mount/md == -== temp space == -Filesystem Size Used Avail Capacity Mounted on -/dev/vtbd0p2 112G 12G 91G 12% / -/dev/vtbd0p2 112G 12G 91G 12% / -``` - -结论：guest 连接及基础环境 PASS；开始时没有已加载 EROFS 模块、EROFS 挂载或 md 设备。 - -## 3. 等待 ABI 提交 - -轮询条件：HEAD 必须晚于测试开始时的 `db524d69f`，且 -`git status --porcelain -- repo-community/repo22` 行数必须为 0。 - -实际命令： - -```sh -baseline=db524d69f -for attempt in $(seq 1 40); do - now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') - head=$(git rev-parse --short HEAD) - subject=$(git log -1 --pretty=%s) - dirty=$(git status --porcelain -- repo-community/repo22 | wc -l) - printf '%s attempt=%02d HEAD=%s dirty_repo22=%s subject=%s\n' \ - "$now" "$attempt" "$head" "$dirty" "$subject" - if [ "$head" != "$baseline" ] && [ "$dirty" -eq 0 ]; then - echo 'READY: repo22 has a newer commit and its scoped worktree is clean' - git log --date=iso-strict -5 --pretty=format:'%h %ad %an %s' - echo - exit 0 - fi - sleep 15 -done -``` - -实际输出： - -```text -2026-08-08T11:10:33Z attempt=01 HEAD=db524d69f dirty_repo22=6 subject=repo22: record initial manual test blockers -2026-08-08T11:10:49Z attempt=02 HEAD=db524d69f dirty_repo22=6 subject=repo22: record initial manual test blockers -2026-08-08T11:11:04Z attempt=03 HEAD=db524d69f dirty_repo22=6 subject=repo22: record initial manual test blockers -2026-08-08T11:11:20Z attempt=04 HEAD=db524d69f dirty_repo22=6 subject=repo22: record initial manual test blockers -2026-08-08T11:11:35Z attempt=05 HEAD=db524d69f dirty_repo22=19 subject=repo22: record initial manual test blockers -2026-08-08T11:11:50Z attempt=06 HEAD=db524d69f dirty_repo22=19 subject=repo22: record initial manual test blockers -2026-08-08T11:12:05Z attempt=07 HEAD=db524d69f dirty_repo22=19 subject=repo22: record initial manual test blockers -2026-08-08T11:12:21Z attempt=08 HEAD=db524d69f dirty_repo22=19 subject=repo22: record initial manual test blockers -2026-08-08T11:12:36Z attempt=09 HEAD=f2caaae28 dirty_repo22=0 subject=repo22: align core on-disk structures -READY: repo22 has a newer commit and its scoped worktree is clean -f2caaae28 2026-08-08T11:12:36Z Ruicheng Pan repo22: align core on-disk structures -db524d69f 2026-08-08T10:46:37Z Ruicheng Pan repo22: record initial manual test blockers -524dcfac5 2026-08-08T10:43:55Z Ruicheng Pan repo22: harden core I/O and build path -173385f79 2026-08-08T10:09:38Z Ruicheng Pan Add repo22 based on repo21 for comprehensive validation and review -c782c1dc8 2026-06-18T20:01:35Z Ruicheng Pan 补充和完善 repo21_explain/src 中文注释 -``` - -## 4. 宿主构建 repo22 erofs.ko - -实际命令（工作目录 `/work/repo-community/repo22`）： - -```sh -date -u '+%Y-%m-%dT%H:%M:%SZ' -git rev-parse HEAD -clang --version | head -1 -FREEBSD_SRC=/work/dev-freebsd-releng ./build.sh -echo "build_exit=$?" -ls -l --full-time build/erofs.ko -sha256sum build/erofs.ko -``` - -实际输出（编译器对 FreeBSD 内核头文件产生了多组重复的 builtin redeclaration -warning；构建未使用 `-Werror`，没有编译或链接 error）： - -```text -2026-08-08T11:13:13Z -f2caaae2890a910d0f11547d9ff5f7cf92c062a2 -Debian clang version 19.1.7 (3+b1) -==> Building repo22 erofs.ko -[data.c、dir.c、erofs_vnops.c、inode.c、namei.c、super.c、xattr.c、lz4.c、deflate.c、zstd.c、lzma.c 编译期间重复出现以下 warning：] -warning: incompatible redeclaration of library function 'log' [-Wincompatible-library-redeclaration] -warning: incompatible redeclaration of library function 'strdup' [-Wincompatible-library-redeclaration] -warning: incompatible redeclaration of library function 'strndup' [-Wincompatible-library-redeclaration] -warning: incompatible redeclaration of library function 'free' [-Wincompatible-library-redeclaration] -warning: incompatible redeclaration of library function 'malloc' [-Wincompatible-library-redeclaration] -warning: incompatible redeclaration of library function 'realloc' [-Wincompatible-library-redeclaration] -/work/repo-community/repo22/src/lzma.c:133:19: warning: result of comparison of constant 256 with expression of type 'uint8_t' (aka 'unsigned char') is always true [-Wtautological-constant-out-of-range-compare] -/work/repo-community/repo22/src/lzma.c:135:16: warning: result of comparison of constant 256 with expression of type 'uint8_t' (aka 'unsigned char') is always true [-Wtautological-constant-out-of-range-compare] -==> SUCCESS: /work/repo-community/repo22/build/erofs.ko -build_exit=0 --rw-r--r-- 1 root root 45088 2026-08-08 11:13:16.704187568 +0000 build/erofs.ko -f119bb2c902bf6e221b532b1c8e7668caaa5c744c46b4eac8f9f105cd7494e9f build/erofs.ko -``` - -说明：原始构建输出包含每个源文件的同类完整 warning 展开；本报告逐类保留了实际 -warning 文本和两个非重复的 `lzma.c` warning，并保留了完整的成功、退出码、尺寸和哈希证据。 - -## 5. SCP 模块到 guest - -实际命令（密码由临时 askpass 提供）： - -```sh -DISPLAY=:0 SSH_ASKPASS=/tmp/repo22-ssh-askpass.sh \ -SSH_ASKPASS_REQUIRE=force setsid -w scp -O \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o PreferredAuthentications=keyboard-interactive,password \ - -o PubkeyAuthentication=no \ - -P 9222 build/erofs.ko root@127.0.0.1:/root/repo22-erofs.ko -echo "scp_exit=$?" -``` - -实际输出： - -```text -Warning: Permanently added '[127.0.0.1]:9222' (ED25519) to the list of known hosts. -scp_exit=0 -``` - -guest 校验命令： - -```sh -ls -l /root/repo22-erofs.ko -sha256 /root/repo22-erofs.ko -``` - -实际输出： - -```text --rw-r--r-- 1 root wheel 45088 Aug 8 11:13 /root/repo22-erofs.ko -SHA256 (/root/repo22-erofs.ko) = f119bb2c902bf6e221b532b1c8e7668caaa5c744c46b4eac8f9f105cd7494e9f -``` - -结论：复制成功且字节身份一致。 - -## 6. Guest 手工模块加载测试 - -### 6.1 加载前状态与 dmesg - -实际命令： - -```sh -kldstat | grep -i erofs -echo "pre_kldstat_grep_exit=$?" -dmesg | tail -30 -``` - -实际输出： - -```text -pre_kldstat_grep_exit=1 -device_attach: fdc0 attach returned 6 -ppc0: port 0x378-0x37f irq 7 on acpi0 -ppc0: Generic chipset (NIBBLE-only) in COMPATIBLE mode -ppbus0: on ppc0 -lpt0: on ppbus0 -lpt0: Interrupt-driven port -ppi0: on ppbus0 -uart0: <16550 or compatible> port 0x3f8-0x3ff irq 4 flags 0x10 on acpi0 -orm0: at iomem 0xe7800-0xeffff pnpid ORM0000 on isa0 -vga0: at port 0x3c0-0x3df iomem 0xa0000-0xbffff pnpid PNP0900 on isa0 -attimer0: at port 0x40 on isa0 -Timecounter "i8254" frequency 1193182 Hz quality 0 -Event timer "i8254" frequency 1193182 Hz quality 100 -attimer0: non-PNP ISA device will be removed from GENERIC in FreeBSD 16. -Timecounters tick every 10.000 msec -usb_needs_explore_all: no devclass -Trying to mount root from ufs:/dev/vtbd0p2 [rw]... -WARNING: / was not properly dismounted -WARNING: /: mount pending error: blocks 0 files 32 -cd0 at ata1 bus 0 scbus1 target 0 lun 0 -cd0: Removable CD-ROM SCSI device -cd0: Serial Number QM00003 -cd0: 16.700MB/s transfers (WDMA2, ATAPI 12bytes, PIO 65534bytes) -cd0: Attempt to query device size failed: NOT READY, Medium not present -intsmb0: irq 9 at device 1.3 on pci0 -intsmb0: intr IRQ 9 enabled revision 0 -smbus0: on intsmb0 -lo0: link state changed to UP -vtnet0: link state changed to UP -Security policy loaded: MAC/ntpd (mac_ntpd) -``` - -### 6.2 kldload - -实际命令： - -```sh -kldload /root/repo22-erofs.ko -echo "kldload_exit=$?" -``` - -实际输出： - -```text -kldload: an error occurred while loading module /root/repo22-erofs.ko. Please check dmesg(8) for more details. -kldload_exit=1 -``` - -失败后的实际 dmesg 命令： - -```sh -dmesg | tail -80 -``` - -实际输出末尾（失败相关行）： - -```text -link_elf_obj: symbol bcmp undefined -linker_load_file: /root/repo22-erofs.ko - unsupported file type -``` - -最终复核命令： - -```sh -date -u "+%Y-%m-%dT%H:%M:%SZ" -kldstat | grep -i erofs -echo "kldstat_grep_exit=$?" -mount | grep -i erofs -echo "mount_grep_exit=$?" -mdconfig -l -dmesg | grep -E "bcmp|repo22-erofs|linker_load_file" | tail -20 -ls -l /root/repo22-erofs.ko -sha256 /root/repo22-erofs.ko -``` - -实际输出： - -```text -2026-08-08T11:14:44Z -kldstat_grep_exit=1 -mount_grep_exit=1 -link_elf_obj: symbol bcmp undefined -linker_load_file: /root/repo22-erofs.ko - unsupported file type --rw-r--r-- 1 root wheel 45088 Aug 8 11:13 /root/repo22-erofs.ko -SHA256 (/root/repo22-erofs.ko) = f119bb2c902bf6e221b532b1c8e7668caaa5c744c46b4eac8f9f105cd7494e9f -``` - -结论：**模块加载 FAIL**。失败不是 SCP 损坏导致，因为宿主和 guest 哈希完全一致。 -FreeBSD 内核链接器无法解析模块引用的 `bcmp` 符号。 - -### 6.3 kldstat 与 kldunload - -- 加载后的 `kldstat` 验证：**BLOCKED**。加载操作返回 1，最终复核也确认没有 EROFS 模块。 -- `kldunload erofs`：**BLOCKED / NOT RUN**。模块从未成功加载；执行卸载不能构成有效卸载测试。 -- 已按“任何步骤失败立即记录并停止”要求停止，没有尝试替换模块、修改 guest、绕过未解析符号或继续文件系统测试。 - -## 7. mkfs.erofs 与 plain 镜像 - -宿主工具前置检查实际命令： - -```sh -command -v mkfs.erofs -mkfs.erofs -V -``` - -实际输出： - -```text -/usr/bin/mkfs.erofs -mkfs.erofs (erofs-utils) 1.8.6 -available compressors: lz4, lz4hc, lzma, deflate, libdeflate, zstd -``` - -状态：**NOT RUN**。工具版本满足要求，但 `kldload` 已失败，因此没有生成源数据、 -没有生成 plain 镜像、没有复制镜像到 guest。这样避免在首个失败后继续推进并产生误导性结果。 - -## 8. TC001-mount-basic - -状态：**BLOCKED / NOT RUN**。 - -阻塞原因：`TC001-mount-basic.md` 的前置条件要求 FreeBSD 系统已加载 EROFS 内核模块； -本轮 `kldload /root/repo22-erofs.ko` 返回 1。因而以下命令均未执行： - -```sh -mdconfig -a -t vnode -f test.erofs -u 0 -mkdir -p /mnt/test -mount -t erofs /dev/md0 /mnt/test -mount | grep erofs -df -h /mnt/test -sha256 /mnt/test/<测试文件> -umount /mnt/test -mdconfig -d -u md0 -``` - -没有把 BLOCKED 记录为 PASS 或文件系统行为 FAIL。 - -## 9. TC009-statfs-basic - -状态：**BLOCKED / NOT RUN**。 - -用户要求仅在 TC001 通过后执行 TC009。TC001 因模块加载失败没有进入测试步骤，故 -`df -h`、`df -i`、`stat -f` 和只读标志检查均未执行。 - -## 10. 工作树与提交状态 - -ABI 提交完成后的实际确认： - -```text -HEAD=f2caaae2890a910d0f11547d9ff5f7cf92c062a2 -git status --porcelain -- repo-community/repo22 -<无输出> -``` - -本报告是本轮唯一预期新增文件。未修改 repo22 源码，未 commit，未 push。 - -## 最终判定 - -- 手工测试准备环境：PASS -- repo22 模块宿主构建：PASS -- 模块传输与完整性：PASS -- FreeBSD guest 模块加载：**FAIL** -- 失败根因证据：`link_elf_obj: symbol bcmp undefined` -- TC001：BLOCKED / NOT RUN -- TC009：BLOCKED / NOT RUN -- 总体：**FAIL / BLOCKED** - diff --git a/tests/results/manual/2026-08-08T1138Z/manual-test-report.md b/tests/results/manual/2026-08-08T1138Z/manual-test-report.md deleted file mode 100644 index 81695f2..0000000 --- a/tests/results/manual/2026-08-08T1138Z/manual-test-report.md +++ /dev/null @@ -1,513 +0,0 @@ -# repo22 手工基础测试报告 - -- 执行时间：2026-08-08 11:38-11:43 UTC -- 工作区：`/work` -- repo22：`/work/repo-community/repo22` -- Git 分支：`main` -- 测试源码提交：`0c173fa5c07a0c2af7dafcf8f89f0167ae3e1f9d` -- FreeBSD VM：SSH `127.0.0.1:9222` -- 执行方式：手工构建、SSH/SCP 和逐条 shell 命令；未生成测试 wrapper 或 CI；未修改源码 -- 前一轮失败报告：保留 `tests/results/manual/2026-08-08T1114Z/manual-test-report.md` -- 总体结论：**TC001 PASS；TC009 FAIL（用例中的 GNU 风格 `stat` 命令与 FreeBSD `stat(1)` 不兼容）** - -## 结果摘要 - -| 项目 | 状态 | 实际结果 | -|---|---|---| -| repo22 模块重建 | PASS | `build/erofs.ko`，41712 字节 | -| `bcmp` 构建检查 | PASS | `nm -u` 中没有 `bcmp` | -| 模块传输完整性 | PASS | host/guest SHA-256 均为 `6c13819ee36b1f7fe62ea084640fdefb2e871f245462291ce4c18f1ee93a684f` | -| `kldload` / `kldstat` / `kldunload` | PASS | canonical `/root/erofs.ko` 三步退出码均为 0 | -| 固定 plain 镜像 | PASS | 10 MiB、2560×4096 blocks、重复构建逐字节相同 | -| TC001-mount-basic | **PASS** | 挂载、只读标志、容量、读取、卸载均符合预期 | -| TC009-statfs-basic | **FAIL** | `df`/只读/容量正确；原文步骤 3 未显示文件系统信息，步骤 4 返回 1 | -| 清理 | PASS | 无 EROFS 模块、挂载和 md 设备残留；guest 临时文件已删除 | - -## 1. Host 环境与源码状态 - -实际命令： - -```sh -date -u '+%Y-%m-%dT%H:%M:%SZ' -uname -a -cat /etc/os-release -git rev-parse HEAD -git branch --show-current -git show HEAD:repo-community/repo22/build.sh | sha256sum -sha256sum repo-community/repo22/build.sh -git diff -- repo-community/repo22/src repo-community/repo22/build.sh -``` - -实际输出： - -```text -2026-08-08T11:38:49Z -Linux da8f2da26d77 6.12.74+deb13+1-cloud-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.12.74-2 (2026-03-08) x86_64 GNU/Linux -PRETTY_NAME="Debian GNU/Linux 13 (trixie)" -VERSION_ID="13" -DEBIAN_VERSION_FULL=13.5 -0c173fa5c07a0c2af7dafcf8f89f0167ae3e1f9d -main -70b6c015b33479c271e11abf36108c7f9e7ca9e2b9aa9d931fe7ae21f27f59ff - -70b6c015b33479c271e11abf36108c7f9e7ca9e2b9aa9d931fe7ae21f27f59ff repo-community/repo22/build.sh - -``` - -## 2. 重新构建模块与 `bcmp` 检查 - -实际命令： - -```sh -cd /work/repo-community/repo22 -FREEBSD_SRC=/work/dev-freebsd-releng ./build.sh -ls -l --full-time build/erofs.ko -sha256sum build/erofs.ko -nm -u build/erofs.ko | awk '$NF == "bcmp" {print; found=1} END {if (!found) print "none"}' -``` - -实际输出： - -```text -==> Building repo22 erofs.ko -/work/repo-community/repo22/src/lzma.c:133:19: warning: result of comparison of constant 256 with expression of type 'uint8_t' (aka 'unsigned char') is always true [-Wtautological-constant-out-of-range-compare] - 133 | } while (byte < 0x100); - | ~~~~ ^ ~~~~~ -/work/repo-community/repo22/src/lzma.c:135:16: warning: result of comparison of constant 256 with expression of type 'uint8_t' (aka 'unsigned char') is always true [-Wtautological-constant-out-of-range-compare] - 135 | while (byte < 0x100) - | ~~~~ ^ ~~~~~ -2 warnings generated. -==> SUCCESS: /work/repo-community/repo22/build/erofs.ko --rw-r--r-- 1 root root 41712 2026-08-08 11:38:52.056076393 +0000 build/erofs.ko -6c13819ee36b1f7fe62ea084640fdefb2e871f245462291ce4c18f1ee93a684f build/erofs.ko -none -``` - -结论：构建退出 0，上一轮导致 guest 加载失败的未解析 `bcmp` 已不在模块中。 - -## 3. SSH 与 Guest 基线 - -普通公钥探测： - -```sh -timeout 5 bash -c ' /tmp/repo22-ssh-askpass.sh <<'ASKPASS' -#!/bin/sh -printf '%s\n' "$REPO22_VM_PASS" -ASKPASS -chmod 700 /tmp/repo22-ssh-askpass.sh -export REPO22_VM_PASS='' -export DISPLAY=:0 SSH_ASKPASS=/tmp/repo22-ssh-askpass.sh SSH_ASKPASS_REQUIRE=force -setsid -w ssh \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - -o PreferredAuthentications=keyboard-interactive,password \ - -o PubkeyAuthentication=no -p 9222 root@127.0.0.1 ' -date -u "+%Y-%m-%dT%H:%M:%SZ" -uname -a -freebsd-version -ku -id -sysctl kern.securelevel -command -v kldload; command -v kldstat; command -v kldunload -command -v mdconfig; command -v mount; command -v umount -command -v stat; command -v sha256 -kldstat | grep -i erofs; echo "kldstat_erofs_exit=$?" -mount | grep -i erofs; echo "mount_erofs_exit=$?" -mdconfig -l -' -``` - -```text -2026-08-08T11:39:48Z -FreeBSD freebsd-build 15.0-RELEASE-p8 FreeBSD 15.0-RELEASE-p8 releng/15.0-n281036-53054229dcb3 GENERIC amd64 -15.0-RELEASE-p8 -15.0-RELEASE-p8 -uid=0(root) gid=0(wheel) groups=0(wheel),5(operator) -kern.securelevel: -1 -/sbin/kldload -/sbin/kldstat -/sbin/kldunload -/sbin/mdconfig -/sbin/mount -/sbin/umount -/usr/bin/stat -/sbin/sha256 -kldstat_erofs_exit=1 -mount_erofs_exit=1 - -``` - -结论：VM 是 FreeBSD 15.0-RELEASE-p8 amd64；初始无 EROFS 模块、挂载或 md 设备。 - -## 4. 模块复制与生命周期回归 - -```sh -setsid -w scp -O \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - -o PreferredAuthentications=keyboard-interactive,password \ - -o PubkeyAuthentication=no -P 9222 \ - repo-community/repo22/build/erofs.ko \ - root@127.0.0.1:/root/repo22-erofs.ko -``` - -首次使用非 canonical 文件名加载的实际命令与输出： - -```sh -ls -l /root/repo22-erofs.ko -sha256 /root/repo22-erofs.ko -dmesg | grep -E "bcmp|repo22-erofs|linker_load_file|erofs" | tail -20 -kldload /root/repo22-erofs.ko; echo "kldload_exit=$?" -kldstat | grep -i erofs; echo "kldstat_grep_exit=$?" -dmesg | grep -E "bcmp|repo22-erofs|linker_load_file|erofs" | tail -20 -kldunload erofs; echo "kldunload_exit=$?" -``` - -```text --rw-r--r-- 1 root wheel 41712 Aug 8 11:40 /root/repo22-erofs.ko -SHA256 (/root/repo22-erofs.ko) = 6c13819ee36b1f7fe62ea084640fdefb2e871f245462291ce4c18f1ee93a684f -link_elf_obj: symbol bcmp undefined -linker_load_file: /root/repo22-erofs.ko - unsupported file type -kldload_exit=0 - 5 1 0xffffffff82822000 5d30 repo22-erofs.ko -kldstat_grep_exit=0 -link_elf_obj: symbol bcmp undefined -linker_load_file: /root/repo22-erofs.ko - unsupported file type -kldunload: can't find file erofs -kldunload_exit=1 -``` - -两条 `bcmp`/`unsupported file type` 是本轮加载前已存在的上一轮 dmesg 历史行；加载前后输出相同，本轮 `kldload_exit=0`，没有新增链接错误。首次卸载失败仅因加载文件名是 `repo22-erofs.ko`。 - -按模块 ID 清理后，用 canonical 文件名重新执行完整生命周期： - -```sh -module_id=$(kldstat | awk '/repo22-erofs[.]ko/ {print $1}') -echo "module_id=$module_id" -kldunload -i "$module_id"; echo "kldunload_by_id_exit=$?" -kldstat | grep -i erofs; echo "post_id_unload_kldstat_exit=$?" -cp -f /root/repo22-erofs.ko /root/erofs.ko -sha256 /root/erofs.ko -kldload /root/erofs.ko; echo "kldload_exit=$?" -kldstat | grep -i erofs; echo "kldstat_grep_exit=$?" -kldunload erofs; echo "kldunload_exit=$?" -kldstat | grep -i erofs; echo "post_unload_kldstat_grep_exit=$?" -``` - -```text -module_id=5 -kldunload_by_id_exit=0 -post_id_unload_kldstat_exit=1 -SHA256 (/root/erofs.ko) = 6c13819ee36b1f7fe62ea084640fdefb2e871f245462291ce4c18f1ee93a684f -kldload_exit=0 - 5 1 0xffffffff82822000 5d30 erofs.ko -kldstat_grep_exit=0 -kldunload_exit=0 -post_unload_kldstat_grep_exit=1 -``` - -结论：`bcmp` 修复后的 `kldload`、`kldstat`、`kldunload` 回归 **PASS**。 - -## 5. 固定 plain 镜像 - -工具检查： - -```sh -command -v mkfs.erofs -mkfs.erofs -V -command -v dump.erofs -dump.erofs -V -``` - -```text -/usr/bin/mkfs.erofs -mkfs.erofs (erofs-utils) 1.8.6 -available compressors: lz4, lz4hc, lzma, deflate, libdeflate, zstd -/usr/bin/dump.erofs -dump.erofs (erofs-utils) 1.8.6 -``` - -首次尝试显式禁用 inline： - -```sh -mkfs.erofs -b4096 -x-1 -T0 --all-root --ignore-mtime \ - -Enoinline -L repo22-manual \ - /tmp/repo22-manual-plain/test.erofs \ - /tmp/repo22-manual-plain/src -``` - -```text - erofs: unknown extended option noinline -Try 'mkfs.erofs --help' for more information. -mkfs.erofs 1.8.6 -``` - -该命令退出 1 且未产出镜像。随后用整块大小、无压缩的主文件，并固定时间戳、UUID、label、属主和 xattr 设置： - -```sh -rm -rf /tmp/repo22-manual-plain -mkdir -p /tmp/repo22-manual-plain/src -printf 'repo22 FreeBSD EROFS manual test\n' > /tmp/repo22-manual-plain/src/README.txt -dd if=/dev/zero of=/tmp/repo22-manual-plain/src/plain.bin bs=4096 count=2559 status=none -touch -d @0 /tmp/repo22-manual-plain/src \ - /tmp/repo22-manual-plain/src/README.txt \ - /tmp/repo22-manual-plain/src/plain.bin -mkfs.erofs -b4096 -x-1 -T0 --all-root --ignore-mtime \ - -U 00000000-0000-0000-0000-000000000022 -L repo22-manual \ - /tmp/repo22-manual-plain/test.erofs /tmp/repo22-manual-plain/src -mkfs.erofs -b4096 -x-1 -T0 --all-root --ignore-mtime \ - -U 00000000-0000-0000-0000-000000000022 -L repo22-manual \ - /tmp/repo22-manual-plain/test-repeat.erofs /tmp/repo22-manual-plain/src -cmp /tmp/repo22-manual-plain/test.erofs /tmp/repo22-manual-plain/test-repeat.erofs -echo "cmp_exit=$?" -sha256sum /tmp/repo22-manual-plain/src/README.txt \ - /tmp/repo22-manual-plain/src/plain.bin \ - /tmp/repo22-manual-plain/test.erofs \ - /tmp/repo22-manual-plain/test-repeat.erofs -dump.erofs -s /tmp/repo22-manual-plain/test.erofs -``` - -关键实际输出： - -```text -Filesystem UUID: 00000000-0000-0000-0000-000000000022 -Filesystem total blocks: 2560 (of 4096-byte blocks) -Filesystem total inodes: 3 -Filesystem total metadata blocks: 1 -cmp_exit=0 -c1f6e23b161735085a00a36cb72926bbe187a63eabcb4425a4f5acad5fe3ff4a /tmp/repo22-manual-plain/src/README.txt -20bf72d8cbc4dbc7e214b671f2547e3e8d57d455d10ac6b645656ef26afc6880 /tmp/repo22-manual-plain/src/plain.bin -fec83f76b8b7eb1d83a9a51246e093c319061838b04633184e9a0c601a386ff3 /tmp/repo22-manual-plain/test.erofs -fec83f76b8b7eb1d83a9a51246e093c319061838b04633184e9a0c601a386ff3 /tmp/repo22-manual-plain/test-repeat.erofs -Filesystem magic number: 0xE0F5E1E2 -Filesystem blocksize: 4096 -Filesystem blocks: 2560 -Filesystem inode count: 3 -Filesystem created: Thu Jan 1 00:00:00 1970 -Filesystem features: sb_csum mtime -Filesystem UUID: 00000000-0000-0000-0000-000000000022 -``` - -镜像大小为 10485760 字节，即 10 MiB。两次独立输出逐字节相同。 - -复制并验证： - -```sh -setsid -w scp -O <同上 SSH 认证选项> -P 9222 \ - /tmp/repo22-manual-plain/test.erofs root@127.0.0.1:/root/test.erofs -ls -l /root/test.erofs -sha256 /root/test.erofs -``` - -```text --rw-r--r-- 1 root wheel 10485760 Aug 8 11:42 /root/test.erofs -SHA256 (/root/test.erofs) = fec83f76b8b7eb1d83a9a51246e093c319061838b04633184e9a0c601a386ff3 -``` - -## 6. TC001-mount-basic - -原文步骤前仅做规定前置条件：把 canonical 模块放入标准搜索目录，并附加镜像为 `/dev/md0`。 - -```sh -ls -l /boot/modules/erofs.ko -cp -f /root/erofs.ko /boot/modules/erofs.ko -sha256 /boot/modules/erofs.ko -umount /mnt/test 2>/dev/null -mdconfig -d -u md0 2>/dev/null -mdconfig -a -t vnode -f /root/test.erofs -u 0 -echo "mdconfig_attach_exit=$?" -mdconfig -lv -``` - -```text -ls: /boot/modules/erofs.ko: No such file or directory -SHA256 (/boot/modules/erofs.ko) = 6c13819ee36b1f7fe62ea084640fdefb2e871f245462291ce4c18f1ee93a684f -mdconfig_attach_exit=0 -md0 vnode 10M /root/test.erofs - -``` - -原文步骤 1-5： - -```sh -kldload erofs; echo "tc001_step1_exit=$?" -kldstat | grep erofs; echo "tc001_step2_pipeline_exit=$?" -mkdir -p /mnt/test; echo "tc001_step3_exit=$?" -mount -t erofs /dev/md0 /mnt/test; echo "tc001_step4_exit=$?" -mount | grep erofs; echo "tc001_step5a_pipeline_exit=$?" -df -h /mnt/test; echo "tc001_step5b_exit=$?" -``` - -```text -tc001_step1_exit=0 - 5 1 0xffffffff82822000 5d30 erofs.ko -tc001_step2_pipeline_exit=0 -tc001_step3_exit=0 -tc001_step4_exit=0 -/dev/md0 on /mnt/test (erofs, local, read-only) -tc001_step5a_pipeline_exit=0 -Filesystem Size Used Avail Capacity Mounted on -/dev/md0 10M 10M 0B 100% /mnt/test -tc001_step5b_exit=0 -``` - -补充读取验证： - -```sh -ls -la /mnt/test -sha256 /mnt/test/README.txt /mnt/test/plain.bin -cat /mnt/test/README.txt -echo "tc001_content_exit=$?" -dmesg | tail -20 -``` - -```text -ls: /mnt/test/.: Operation not supported -ls: /mnt/test/README.txt: Operation not supported -ls: /mnt/test/plain.bin: Operation not supported -total 10248 -drwxr-xr-x 2 root wheel 70 Jan 1 1970 . -drwxr-xr-x 3 root wheel 512 Aug 8 11:42 .. --rw-r--r-- 1 root wheel 33 Jan 1 1970 README.txt --rw-r--r-- 1 root wheel 10481664 Jan 1 1970 plain.bin -SHA256 (/mnt/test/README.txt) = c1f6e23b161735085a00a36cb72926bbe187a63eabcb4425a4f5acad5fe3ff4a -SHA256 (/mnt/test/plain.bin) = 20bf72d8cbc4dbc7e214b671f2547e3e8d57d455d10ac6b645656ef26afc6880 -repo22 FreeBSD EROFS manual test -tc001_content_exit=0 -``` - -`ls -la` 在读取附加 stat/pathconf 信息时打印 `Operation not supported`，但仍列出目录；两个文件均完整可读且 SHA-256 与 host 一致。该补充现象不属于 TC001 规定判定项，单独保留为观察。`dmesg | tail -20` 没有本轮 mount 错误；末尾两条 `bcmp` 行是上一轮历史记录。 - -原文步骤 6-7： - -```sh -umount /mnt/test; echo "tc001_step6_exit=$?" -mount | grep erofs; echo "tc001_step7_pipeline_exit=$?" -``` - -```text -tc001_step6_exit=0 - -tc001_step7_pipeline_exit=1 -``` - -步骤 7 的 grep 返回 1 正是“没有 EROFS 挂载”的预期。 - -**TC001 最终判定：PASS。** - -## 7. TC009-statfs-basic - -原文步骤 1-2： - -```sh -mount -t erofs /dev/md0 /mnt/test; echo "tc009_step1_exit=$?" -df -h /mnt/test; echo "tc009_step2a_exit=$?" -df -i /mnt/test; echo "tc009_step2b_exit=$?" -``` - -```text -tc009_step1_exit=0 -Filesystem Size Used Avail Capacity Mounted on -/dev/md0 10M 10M 0B 100% /mnt/test -tc009_step2a_exit=0 -Filesystem 1K-blocks Used Avail Capacity iused ifree %iused Mounted on -/dev/md0 10240 10240 0 100% 3 0 100% /mnt/test -tc009_step2b_exit=0 -``` - -总容量 10 MiB、可用块 0、inode 总数 3、可用 inode 0。 - -严格执行原文步骤 3： - -```sh -stat -f /mnt/test -echo "tc009_step3_exit=$?" -``` - -```text -/mnt/test -tc009_step3_exit=0 -``` - -FreeBSD `stat(1)` 的 `-f` 表示“后一个参数是格式字符串”，不是 GNU `stat -f` 的“显示文件系统状态”。命令只回显 `/mnt/test`，没有达到用例要求的“Shows EROFS filesystem type”。 - -严格执行原文步骤 4： - -```sh -stat -f -f "%b %f %c %a %d %i %t %n" /mnt/test -echo "tc009_step4_exit=$?" -``` - -```text -stat: %b %f %c %a %d %i %t %n: No such file or directory --f -tc009_step4_exit=1 -``` - -原文步骤 5 与补充容量核对： - -```sh -mount | grep /mnt/test | grep read-only -echo "tc009_step5_pipeline_exit=$?" -mount | grep /mnt/test -df -k /mnt/test -``` - -```text -/dev/md0 on /mnt/test (erofs, local, read-only) -tc009_step5_pipeline_exit=0 -/dev/md0 on /mnt/test (erofs, local, read-only) -Filesystem 1024-blocks Used Avail Capacity Mounted on -/dev/md0 10240 10240 0 100% /mnt/test -``` - -`10240` 个 1 KiB 块等于 `2560` 个 4096-byte 块；只读标志、文件系统类型和 free=0 均正确。但严格按用例判定，步骤 3 未产生要求的信息，步骤 4 退出 1。 - -**TC009 最终判定：FAIL。失败原因是测试文档中的 `stat` 命令与 FreeBSD `stat(1)` CLI 语义不兼容；本轮未修改测试文档或源码。** - -## 8. 清理与最终状态 - -```sh -umount /mnt/test; echo "cleanup_umount_exit=$?" -mdconfig -d -u md0; echo "cleanup_mdconfig_exit=$?" -kldunload erofs; echo "cleanup_kldunload_exit=$?" -rmdir /mnt/test; echo "cleanup_rmdir_exit=$?" -rm -f /boot/modules/erofs.ko /root/test.erofs /root/erofs.ko /root/repo22-erofs.ko -echo "cleanup_rm_exit=$?" -kldstat | grep -i erofs; echo "final_kldstat_grep_exit=$?" -mount | grep -i erofs; echo "final_mount_grep_exit=$?" -mdconfig -l -``` - -```text -cleanup_umount_exit=0 -cleanup_mdconfig_exit=0 -cleanup_kldunload_exit=0 -cleanup_rmdir_exit=0 -cleanup_rm_exit=0 -final_kldstat_grep_exit=1 -final_mount_grep_exit=1 - -``` - -结论：guest 清理完成，无模块、挂载、md 设备或本轮临时文件残留。 - -## 最终判定 - -- `bcmp` 修复后模块构建与 host 检查：**PASS** -- guest `kldload` / `kldstat` / `kldunload`：**PASS** -- 固定 10 MiB plain 镜像生成、重复性与 SHA-256：**PASS** -- TC001-mount-basic：**PASS** -- TC009-statfs-basic：**FAIL（测试命令不兼容 FreeBSD `stat(1)`）** -- BLOCKED：**0** -- 源码修改：**无** -- wrapper/CI：**未生成** diff --git a/tests/results/manual/2026-08-08T1307Z/manual-test-report.md b/tests/results/manual/2026-08-08T1307Z/manual-test-report.md deleted file mode 100644 index 1ad6712..0000000 --- a/tests/results/manual/2026-08-08T1307Z/manual-test-report.md +++ /dev/null @@ -1,141 +0,0 @@ -# repo22 压缩功能手工测试报告 - -- 执行日期：2026-08-08（UTC） -- 执行时段：约 12:58-13:07 UTC -- 测试对象：`/work/repo-community/repo22` -- 测试提交：`f230129565677e602510347a55a80856237fe909` -- 远端基线：执行前 `xdm/main` 指向同一提交 -- FreeBSD VM：15.0-RELEASE-p8 amd64，QEMU TCG，SSH `127.0.0.1:9222` -- 镜像工具：erofs-utils 1.8.6 -- 执行方式：逐条 host/guest shell 命令、SSH/SCP、`mdconfig` 和真实内核挂载；未实现 CI 或测试 wrapper -- 总体结论：**压缩功能不可验收。真实 compact LZ4 镜像可挂载，但首个 4 KiB 数据读取返回 `EINTEGRITY`，输出 0 字节。** - -## 状态定义 - -| 状态 | 含义 | -|---|---| -| PASS | 该步骤已经在本轮实际执行，结果符合预期 | -| KERNEL-FAIL | 真实镜像进入 FreeBSD 内核路径后失败 | -| MKFS-UNAVAILABLE | erofs-utils 1.8.6 不能直接生成测试要求的镜像 | -| TEST-DOC | 当前 Markdown 的命令或预期不能可靠验证目标行为 | -| NOT RUN | 已准备 fixture，但收到立即停止新增测试的指令后未进入对应内核路径 | - -## 最先出现的内核失败 - -使用 erofs-utils 1.8.6 生成根目录单文件 compact LZ4 镜像： - -```sh -mkfs.erofs -T0 --all-root --ignore-mtime -x-1 -Uclear \ - -zlz4 -C4096 lz4-root-compact-4k.erofs src-root-lz4 -``` - -`dump.erofs` 确认该文件不是 plain fallback： - -```text -Filesystem features: sb_csum mtime 0padding -Path : /test-lz4.txt -Size: 1048576 On-disk size: 8192 regular file -NID: 39 Layout: 3 Compression ratio: 0.78% -Ext 0: logical 0..639004, physical 4096..8192, physical length 4096 -Ext 1: logical 639004..1048576, physical 8192..12288, physical length 4096 -``` - -Guest 中模块加载和挂载成功： - -```text -SHA256 (/root/erofs.ko) = c81957971e670c8b8cc45d5119f568eb84adb6114699538588e5a842ea4d97b3 -kldload_rc=0 -5 1 0xffffffff82822000 8468 erofs.ko -/dev/md0 on /mnt/repo22-test (erofs, local, read-only) -``` - -文件 lookup 和 `stat` 成功后，第一次读取即失败： - -```text -$ sha256 /mnt/repo22-test/test-lz4.txt -sha256: /mnt/repo22-test/test-lz4.txt: Integrity check failed -read_rc=1 - -$ dd if=/mnt/repo22-test/test-lz4.txt of=/tmp/repo22-first4k bs=4096 count=1 -dd: /mnt/repo22-test/test-lz4.txt: Integrity check failed -0+0 records in -0+0 records out -0 bytes transferred -first_read_rc=1 -``` - -因此首个确定性压缩内核失败是： - -- fixture：真实 `mkfs.erofs 1.8.6` compact LZ4，Layout 3； -- mount：PASS； -- lookup/stat：PASS； -- 首个 4 KiB read：**KERNEL-FAIL / `EINTEGRITY`**； -- 用户缓冲区：0 字节，没有把部分输出误判为成功； -- 稳定性：命令后 VM 仍响应，时间为 `2026-08-08T13:06:43Z`，未观察到 panic 或 hang。 - -并发静态审查已经指出 compact 索引位置按 filesystem block size 计算，而不是按 compact index pack 大小计算；本轮失败与该问题相符，但手工测试本身只证明“真实 compact LZ4 首读失败”，不单独宣称完成了代码级根因证明。 - -## 已尝试 TC 结果 - -| TC | 状态 | 本轮证据 | -|---|---|---| -| TC003-lz4-compressed-read | **KERNEL-FAIL** | 真实 compact LZ4 镜像 mount PASS；完整 hash 与首个 4 KiB read 都返回 `EINTEGRITY` | -| TC084-lz4-basic | **KERNEL-FAIL** | 与 TC003 共用的最小真实 LZ4 基础读路径失败，无法进行透明解压和内容比较 | -| TC086-lz4-sequential-read | **TEST-DOC** | 用例要求不存在的 `vfs:erofs::read` DTrace provider，并把 TCG VM 上固定吞吐量当功能判据 | -| TC088-lz4-config-handling | **TEST-DOC** | 4K/64K/256K fixture 均生成；“pcluster 越大则镜像必然更小、随机读必然更慢”不是对任意语料都成立的功能断言 | -| TC090-lz4-pcluster-64k | **TEST-DOC** | 步骤 4 把 `dd ... of=/tmp/out` 再 pipe 给 `tail/head`，pipe 中没有文件数据，无法验证跨边界读取 | -| TC098-fragments-support | **TEST-DOC** | EROFS fragments 使用 packed inode，不是 Markdown 所写的独立 fragment device；`-o device=/dev/md1` 验证的是多设备而非 fragments | -| TC101-unified-address-mapping | **TEST-DOC** | 前置要求多设备，但 mount 命令只提供 device0；`ktrace` 也不能直接证明驱动选中了正确 EROFS device id | -| TC110-lzma-corrupt | **TEST-DOC** | 固定覆盖 image offset 8192 没有先定位目标文件的 compressed extent，可能破坏无关数据或元数据，不能确定性验证 LZMA 错误路径 | -| TC115-unsupported-algorithm | **MKFS-UNAVAILABLE** | mkfs 1.8.6 只提供 lz4/lz4hc/lzma/deflate/libdeflate/zstd，不能直接生成 future algorithm ID；需要有校验意识的定向 ABI patch fixture | -| TC116-truncated-compressed | **TEST-DOC** | 盲目从镜像末尾截去 4096 字节不保证命中目标 compressed extent；可能只移除 padding、其他文件或元数据 | - -## 已生成但按停止指令未运行的 fixture - -以下 fixture 的 `mkfs.erofs` 和目标布局检查已经成功，但没有据此把对应 TC 标为 PASS： - -| TC 范围 | Fixture 准备结果 | TC 状态 | -|---|---|---| -| TC004、TC108、TC109 | 101 MiB MicroLZMA/LZMA level 6；Layout 3；19 个真实 compressed extents | NOT RUN | -| TC085、TC087、TC089 | 256 MiB/50 MiB/4K LZ4 corpus；compact 4K 与 legacy full 镜像均生成 | NOT RUN | -| TC090 | compact 64K big-pcluster 镜像生成，superblock 含 `compr_cfgs big_pcluster` | TEST-DOC，未运行内核读 | -| TC091、TC092 | `-Eztailpacking` 镜像生成，superblock 含 `ztailpacking` | NOT RUN | -| TC093、TC095 | single-device 1 MiB chunk 与 512 KiB chunk-index 镜像生成，Layout 4 | NOT RUN | -| TC094、TC096、TC099 | main image + 5 MiB external blob 生成；首次必须预创建 blob 文件后 mkfs 才成功 | NOT RUN | -| TC097、TC100 | 没有生成确定性 invalid table 或四外部设备 fixture | NOT RUN | -| TC102-TC104 | DEFLATE level 1/6/9 镜像均生成，file1m 为真实 Layout 3 | NOT RUN | -| TC105-TC107 | ZSTD level 1/15/22 镜像均生成，file1m 为真实 Layout 3 | NOT RUN | - -收到“立即停止新增测试”指令后，没有继续 mount/read 上述镜像，也没有继续制作 corruption、unsupported algorithm、interlaced、extent 或 partial-ref 变体。 - -## Fixture 审核摘要 - -本轮生成并由 `dump.erofs` 确认的主要布局： - -| 镜像 | 关键特征 | -|---|---| -| `lz4-compact-4k.erofs` | Layout 3 compact；`sb_csum mtime 0padding` | -| `lz4-full-4k.erofs` | Layout 1 full；`-Elegacy-compress` | -| `lz4-compact-64k.erofs` | Layout 3；`compr_cfgs big_pcluster`；真实物理 extent 最大 53248 字节 | -| `lz4-compact-256k.erofs` | Layout 3；`compr_cfgs big_pcluster` | -| `lz4-ztailpacking.erofs` | `ztailpacking` incompat feature | -| `lz4-fragments.erofs` | `fragments dedupe`，目标文件物理长度由 packed inode 提供 | -| `lzma-level6.erofs` | 101 MiB 逻辑文件，126976 字节 on-disk，Layout 3 | -| `deflate-level{1,6,9}.erofs` | 三个等级均为真实 compressed Layout 3 | -| `zstd-level{1,15,22}.erofs` | 三个等级均为真实 compressed Layout 3 | -| `chunk-single-1m.erofs` | `chunked_file`，Layout 4，三个 1 MiB extent | -| `chunk-index-512k.erofs` | `chunked_file`，Layout 4，8-byte chunk index fixture | -| `chunk-multidev-main.erofs` + `.blob` | 4 KiB metadata image + 5 MiB external blob | - -所有镜像、source corpus、VM overlay 和模块 build 产物仅位于 `/work/build`、guest `/root` 或 repo22 忽略的 `build/`，没有加入 Git。 - -## 其他观察 - -1. 多文件嵌套目录镜像首次直接 lookup `/files/test-lz4.txt` 偶发返回 `ENOENT`；先执行目录枚举后 `stat` 可成功。根目录单文件 fixture 消除了该前置干扰，并稳定进入压缩 read 后返回 `EINTEGRITY`。 -2. FreeBSD `ls -l` 对 EROFS 条目打印 `Operation not supported`，同时仍能列出项目；这看起来来自 ACL/扩展属性查询，不应当被误记为目录读取失败。 -3. dmesg 中的 `bcmp undefined` 是前一轮旧模块的历史日志；本轮 canonical `/root/erofs.ko` 的 `kldload` 返回 0，且模块 SHA-256 为新的 `c819...97b3`。 -4. 并发静态审查另报 incompat `0x8`、64K `clusterofs` 截断、通用剥前导零和短输出泄漏。本轮遵照停止指令没有再构造独立运行时 fixture 复核这些问题。 - -## 结论 - -`f2301295` 的压缩重构已经达到“模块可加载、真实压缩镜像可挂载”的阶段，但尚未达到“可读取最基本 compact LZ4 文件”的最低 feature gate。应先修复 compact zmap/read 闭环，并针对静态审查列出的五个确定性阻断补足定向手测，再恢复其余算法和高级布局验证。 diff --git a/tests/results/manual/2026-08-08T1332Z/manual-test-report.md b/tests/results/manual/2026-08-08T1332Z/manual-test-report.md deleted file mode 100644 index 598d04b..0000000 --- a/tests/results/manual/2026-08-08T1332Z/manual-test-report.md +++ /dev/null @@ -1,84 +0,0 @@ -# repo22 compact LZ4 root-cause verification - -- Date: 2026-08-08 UTC -- Source parent: `c939c472134c69fa4665f52ab1aec9a17cd583b9` -- FreeBSD guest: 15.0-RELEASE-p8 amd64, QEMU TCG -- Image tool: erofs-utils 1.8.6 -- Scope: the existing minimal compact LZ4 fixture only; no CI or wrapper was added - -## Changes under verification - -1. Accept incompat bit `0x8` for compressed HEAD2 while rejecting images with - external device slots until true multi-device I/O exists. -2. Widen zmap `clusterofs` so the 64 KiB NONHEAD sentinel is representable. -3. Count compact indexes using the logical cluster size. -4. Apply leading zero padding according to the algorithm and the LZ4 - `0padding` feature instead of stripping it unconditionally for all streams. -5. Zero decompression targets and require exact output lengths. -6. Apply DEFLATE `windowbits` and ZSTD `windowlog` to the decoder calls. -7. Separate LZ4 full and partial completion rules and validate input/output - boundaries. -8. Permit valid compact NONHEAD deltas to cross compact-pack boundaries. The - previous BSD-only restriction caused the first real image read to fail at - the second compressed extent head. - -## Build and module lifecycle - -`./build.sh` completed successfully. The resulting module had no unresolved -`bcmp` symbol and contained no temporary zmap/zdata diagnostic strings. - -On the guest, all lifecycle operations succeeded: - -```text -kldload_rc=0 -mount_rc=0 -umount_rc=0 -md_detach_rc=0 -kldunload_rc=0 -``` - -## Real image result - -The fixture was generated by the existing manual-test command: - -```sh -mkfs.erofs -T0 --all-root --ignore-mtime -x-1 -Uclear \ - -zlz4 -C4096 lz4-root-compact-4k.erofs src-root-lz4 -``` - -`dump.erofs` identifies `/test-lz4.txt` as layout 3 (compact), 1 MiB logical, -8 KiB on disk, with two compressed extents. The same image previously failed -its first read with `EINTEGRITY` on `f2301295`. - -Final FreeBSD results: - -```text -SHA256 (/mnt/repo22-rootfix/test-lz4.txt) = 370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52 -SHA256 (source test-lz4.txt) = 370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52 -SHA256 (first 4096 bytes from EROFS) = 3d6d283a95f80b3e4a9c243cf82dc0644283910b6a3be931fede5d771e132b14 -SHA256 (first 4096 bytes from source) = 3d6d283a95f80b3e4a9c243cf82dc0644283910b6a3be931fede5d771e132b14 -full_read_rc=0 -first_4k_read_rc=0 -``` - -Result: the minimal real compact LZ4 mount/read gate passes. - -## Integration follow-up - -The integration review after `6c9543892` found that the leading-padding -predicate still selected every non-LZ4 algorithm. The predicate was narrowed -to LZ4 streams with the `0padding` incompat feature, matching the stated -algorithm-specific behavior. `./build.sh` and the same FreeBSD 15 compact LZ4 -mount/read lifecycle were rerun after this correction; the full-file SHA256 -remained `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52`, -the first 4096-byte read returned 4096 bytes, and module unload succeeded. - -## Remaining gaps - -This batch does not claim runtime completion for legacy full indexes, 64 KiB -or larger pclusters, ztailpacking, interlaced pclusters, fragments, extents, -partial references, MicroLZMA, DEFLATE, or ZSTD images. DEFLATE and ZSTD -configuration handling was corrected and the module loaded on this FreeBSD 15 -kernel, but those algorithms were not image-tested in this converged batch. -The LZ4 partial-decoding branch was code-reviewed but is not exercised by the -current synchronous full-extent read path. diff --git a/tests/results/manual/2026-08-08T1356Z/manual-test-report.md b/tests/results/manual/2026-08-08T1356Z/manual-test-report.md deleted file mode 100644 index dceadfb..0000000 --- a/tests/results/manual/2026-08-08T1356Z/manual-test-report.md +++ /dev/null @@ -1,88 +0,0 @@ -# repo22 non-LZ4 leading-zero verification - -- Date: 2026-08-08 UTC -- Source baseline: `7cba92c0ec434b96180200b09e5dddfa213e67f5` -- FreeBSD guest: 15.0-RELEASE-p8 amd64, QEMU TCG -- Image tool: erofs-utils 1.8.6 -- Scope: MicroLZMA, DEFLATE, and ZSTD leading-zero handling only - -## Code change - -Linux 7.1-rc1 calls `z_erofs_fixup_insize()` for every MicroLZMA, DEFLATE, -and ZSTD stream, while LZ4 only requires it when the `LZ4_0PADDING` feature is -enabled. The FreeBSD dispatcher now applies the same predicate. Existing -first-block limits, empty-input rejection, and input-length subtraction remain -unchanged. - -## Build and module lifecycle - -`./build.sh` completed successfully and produced `build/erofs.ko` with SHA256: - -```text -4b3459a02da29642cb8e61f58e34ff5266e668e86256937c91b8af7e0e31fdcb -``` - -FreeBSD 15 results: - -```text -lifecycle_kldload_rc=0 -lifecycle_kldunload_rc=0 -dmesg_delta_lines=0 -dmesg_error_grep_rc=1 -``` - -The last two lines mean the isolated lifecycle and read pass added no kernel -messages, so no decompression, linker, panic, or integrity error was present. - -## Deterministic images - -The source file was 1 MiB and had SHA256: - -```text -370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52 -``` - -The images were regenerated with erofs-utils 1.8.6: - -```sh -mkfs.erofs -T0 --all-root --ignore-mtime -x-1 -Uclear \ - -zlzma,level=6 -C4096 lzma-root.erofs src-root -mkfs.erofs -T0 --all-root --ignore-mtime -x-1 -Uclear \ - -zdeflate,level=6 -C4096 deflate-root.erofs src-root -mkfs.erofs -T0 --all-root --ignore-mtime -x-1 -Uclear \ - -zzstd,level=3 -C4096 zstd-root.erofs src-root -``` - -`dump.erofs` reported layout 3 for `/test-lz4.txt` in all images. MicroLZMA -and ZSTD used one 4096-byte physical extent; DEFLATE used two 4096-byte -physical extents. - -## FreeBSD read results - -Each image was attached with `mdconfig`, mounted read-only, read completely by -`sha256`, unmounted, and detached before testing the next image. - -```text -lzma mount_rc=0 read_rc=0 sha256=370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52 -lzma umount_rc=0 detach_rc=0 -deflate mount_rc=0 read_rc=0 sha256=370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52 -deflate umount_rc=0 detach_rc=0 -zstd mount_rc=0 read_rc=0 sha256=370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52 -zstd umount_rc=0 detach_rc=0 -``` - -Result: MicroLZMA, DEFLATE, and ZSTD all pass real-image full-file integrity -verification on this FreeBSD 15 kernel configuration. - -## Notes and limits - -An exploratory image with the file below a `/files` directory mounted, but the -target lookup failed. Root-level fixtures were used to isolate decompression -from the separately tracked directory lookup work. This does not change the -three successful compressed read results above. - -This batch does not claim coverage for other compression levels, large files, -random access, corrupt streams, alternative FreeBSD kernel configurations, or -other compression mapping features. DEFLATE and ZSTD kernel symbols were -available on the tested FreeBSD 15 GENERIC kernel; no unsupported algorithm or -kernel-symbol limitation was encountered. diff --git a/tests/results/manual/2026-08-08T1413Z/manual-test-report.md b/tests/results/manual/2026-08-08T1413Z/manual-test-report.md deleted file mode 100644 index 26d68f0..0000000 --- a/tests/results/manual/2026-08-08T1413Z/manual-test-report.md +++ /dev/null @@ -1,99 +0,0 @@ -# repo22 compressed mapping shape manual test report - -- Date: 2026-08-08 UTC -- Source baseline: `b1f9e7c0ef5ffb9c4aab38e4047534ad2071af6c` -- Remote baseline before testing: `xdm/main` at the same commit -- FreeBSD guest: 15.0-RELEASE-p8 amd64, QEMU TCG -- Image tool: erofs-utils 1.8.6 -- Module SHA256: `4b3459a02da29642cb8e61f58e34ff5266e668e86256937c91b8af7e0e31fdcb` -- Scope: manual image generation, layout inspection, and the completed compact-index kernel reads; no CI or test wrapper was added - -## Status definitions - -| Status | Meaning | -|---|---| -| PASS | The stated mkfs/layout or kernel operation was actually run and matched its deterministic expectation | -| KERNEL-FAIL | A real image reached the FreeBSD kernel path and failed | -| MKFS-UNAVAILABLE | erofs-utils 1.8.6 cannot generate the required format directly | -| TEST-DOC | The existing Markdown command or assertion does not deterministically test the stated feature | -| NOT RUN | The fixture was generated, but kernel execution stopped on user request and no feature pass is claimed | - -## Generated fixture matrix - -All images were regenerated under `/work/build/repo22-shape-manual-20260808T1407Z` with fixed timestamps, root ownership, disabled xattrs, and a cleared UUID. Images, sources, VM state, and module build products were not staged. - -| Shape | mkfs/layout status | Evidence | Kernel status | -|---|---|---|---| -| Compact indexes | PASS | default `-zlz4 -C4096`; `/shape.dat` is Layout 3 with two compressed extents | PASS | -| Legacy full indexes | PASS | `-Elegacy-compress`; `/shape.dat` is Layout 1 with two compressed extents | NOT RUN | -| 64 KiB big pcluster | PASS | `-C65536`; superblock reports `compr_cfgs big_pcluster`; Layout 3 has one 1 MiB logical extent backed by 8192 bytes | NOT RUN | -| Inline ztailpacking | PASS | `-Eztailpacking`; superblock reports `ztailpacking`; `/inline.dat` has on-disk size 0 and physical bytes `1352..1787` inside the metadata block | NOT RUN | -| Fragments / packed inode | PASS | `-Eall-fragments`; superblock reports packed NID 42 plus `fragments dedupe`; `/fragment.dat` has on-disk size 0 | NOT RUN | -| Partial reference | PASS | `-Ededupe`; mkfs reports `Dedupe 409572 compressed data (delta 315374)` and `/b.dat` reuses the physical extent at `8192..12288` | NOT RUN | -| Extent metadata format | MKFS-UNAVAILABLE | erofs-utils v1.8.6 has no extent-map advise/structure or mkfs option for the newer extent metadata format; `--max-extent-bytes` only limits decompressed extent size | NOT RUN | - -Host-side `fsck.erofs --extract` succeeded for the initially generated compact, full, big-pcluster, ztailpacking-option, and all-fragments images. This is fixture validation only, not a FreeBSD kernel feature pass. - -## Completed FreeBSD compact-index verification - -The compact image used a deterministic 1 MiB file. `dump.erofs` reported: - -```text -Layout: 3 -Ext 0: logical 0..639004, physical 4096..8192, physical length 4096 -Ext 1: logical 639004..1048576, physical 8192..12288, physical length 4096 -``` - -Guest operations completed as follows: - -```text -kldload_rc=0 -md_attach_rc=0 -mount_rc=0 -stat_size=1048576 -full_sha256=370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52 -expected_full_sha256=370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52 -cross_extent_offset=638972 -cross_extent_length=128 -cross_extent_sha256=941b6e3cb9a7384428d93ca248eb9630782538e04caba30dbfbe31baacca8f60 -expected_cross_extent_sha256=941b6e3cb9a7384428d93ca248eb9630782538e04caba30dbfbe31baacca8f60 -``` - -Result: compact full-file integrity and a read crossing the logical boundary immediately before the second compressed extent both pass. - -## First failure and dmesg - -No KERNEL-FAIL occurred in this batch. The first incomplete command was: - -```sh -dd if=/mnt/repo22-shape/shape.dat bs=1 skip=777777 count=8192 | sha256 -q -``` - -It was operator-terminated when the user requested immediate convergence. Byte-sized `dd` caused repeated page reads under QEMU TCG and was an inefficient manual command; it did not return a kernel error and is not classified as KERNEL-FAIL. - -The guest was responsive after termination, and the mount and md device were cleaned up. The dmesg tail contained only successful mapping diagnostics such as: - -```text -erofs zread: nid=39 la=1044480 mapla=639004 llen=409572 pa=8192 plen=4096 flags=1 alg=0 -erofs zread: nid=39 la=0 mapla=0 llen=639004 pa=4096 plen=4096 flags=1 alg=0 -``` - -There was no new `EINTEGRITY`, decompression error, panic, trap, or hang message. The volume of unconditional `erofs zread` diagnostics should be reviewed separately as a style/logging issue. - -## TC disposition - -| TC area | Status | Result | -|---|---|---| -| TC084 compact LZ4 basic | PASS | Complete SHA256 and cross-extent boundary read match the source | -| TC088 pcluster configuration | TEST-DOC | Previous monotonic ratio/performance assertions were invalid; commands and pass criteria were corrected | -| TC090 64 KiB pcluster | TEST-DOC | Previous pipeline read from files redirected away from the pipe; deterministic cross-boundary command was corrected; kernel execution remains NOT RUN | -| TC091 ztailpacking | TEST-DOC | Enabling the option alone did not tail-pack the first 1 MiB target; the TC now requires `dump.erofs` proof of an inline extent; kernel execution remains NOT RUN | -| TC098 fragments | TEST-DOC | Previous TC incorrectly described fragments as an external device; it now tests the same-image packed inode; kernel execution remains NOT RUN | -| TC115 unsupported algorithm | MKFS-UNAVAILABLE | mkfs.erofs 1.8.6 only emits its supported algorithm IDs; a checksum-aware ABI patch fixture is required | -| TC116 truncated compressed data | TEST-DOC | Blind `truncate -s -4096` was nondeterministic; the TC now truncates inside a located target extent; kernel execution remains NOT RUN | - -TC085-TC087, TC089, TC092-TC101 other than TC098 were read for dependencies but were not executed in this converged batch. Multi-device and chunk tests are independent follow-up work and are not implied by the packed-inode fragment fixture. - -## Conclusion - -The completed kernel evidence establishes compact-index LZ4 full-read and compressed-extent-boundary correctness on the tested FreeBSD 15 guest. Legacy full indexes, big pclusters, inline ztailpacking, fragments, and partial references are confirmed as real erofs-utils 1.8.6 fixtures but remain explicitly NOT RUN in the FreeBSD kernel after the stop instruction. No kernel failure was observed. diff --git a/tests/results/manual/2026-08-08T1427Z/manual-test-report.md b/tests/results/manual/2026-08-08T1427Z/manual-test-report.md deleted file mode 100644 index afa346a..0000000 --- a/tests/results/manual/2026-08-08T1427Z/manual-test-report.md +++ /dev/null @@ -1,53 +0,0 @@ -# repo22 remaining compressed mapping manual read report - -- Date: 2026-08-08 UTC -- Source baseline: `df5b92c902295dc74a08aef89fd93d698bbddf21` -- Remote baseline before testing: `xdm/main` at the same commit -- FreeBSD guest: 15.0-RELEASE-p8 amd64, QEMU TCG -- Image tool: erofs-utils 1.8.6 -- Scope: kernel mount/read verification for the five previously generated compression layouts only; no source, CI, or test-wrapper changes - -## Result matrix - -| Mode | Fixture proof | Mount | Full SHA256 | Offset/boundary read | Isolated dmesg | Result | -|---|---|---:|---|---|---|---| -| Legacy full index | Layout 1, two compressed extents | PASS | `EIO`, no digest | `EIO` at offset 638972 across the 639004 extent boundary | No panic, trap, integrity, corruption, or decompression-error signature; guest responsive | **KERNEL-FAIL** | -| 64 KiB pcluster | Layout 3; `compr_cfgs big_pcluster`; one 1 MiB logical extent backed by 8192 bytes | PASS | Match | Match at offset 65504, length 64 | No error signature; guest responsive | **PASS** | -| Inline ztailpacking | `ztailpacking`; compressed extent at physical 1352..1787 inside metadata block | PASS | Match | Match at offset 31744, length 2048 | No error signature; guest responsive | **PASS** | -| Fragments / packed inode | `fragments`; packed NID; whole-file extent has zero direct on-disk bytes | PASS | Match | Match at offset 65536, length 8192 | No error signature; guest responsive | **PASS** | -| Partial reference | `/b.dat` Layout 1 reuses `/a.dat` physical extent 8192..12288 at logical 641052..1050624 | PASS | `/b.dat` returns `EIO` | `EIO` at offset 641020 crossing into the reused extent; companion Layout 3 `/a.dat` matches | No error signature; guest responsive | **KERNEL-FAIL** | - -All five fixtures contain the requested layout. No mode is classified as `FIXTURE-NOT-PROVEN` or `NOT RUN`. - -## Integrity evidence - -| Target | Full SHA256 | -|---|---| -| Legacy full reference | `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` | -| 64 KiB pcluster, actual and expected | `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` | -| Inline ztailpacking, actual and expected | `e2aa4a0a0cbcf422f397c7069a38ae0f073781386958e7db0dfa3ff2ca075513` | -| Packed fragment, actual and expected | `a3a83e5c524b5ed446a06ce78cf407192a0c80119f15d2bc3489a50515eb49e0` | -| Partial-ref `/b.dat` reference | `61b17076c2dfae88da7912d00df534b894cf6d27178863c6d8e91f8617ebb91e` | -| Partial-ref companion `/a.dat`, actual and expected | `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` | - -Offset-read evidence: - -| Mode | Actual | Expected | -|---|---|---| -| Legacy full | `f387a3a74488528803454f0c05601fb632d919f1bc5281538f687a3a6231ebea` from partial failed output | `941b6e3cb9a7384428d93ca248eb9630782538e04caba30dbfbe31baacca8f60` | -| 64 KiB pcluster | `b6ebba880cfcc438044f943370b9936a130ce1cecf13d784a1eb871f0a126182` | same | -| Inline ztailpacking | `3a630b7e9618c7e27efb1483058baaa0475ca325c3ecafb42778871d700afa96` | same | -| Packed fragment | `a826cc36ad8c9eaa1606713117fcf118b7bae5fee7503b5619dbf024213059db` | same | -| Partial reference | empty output `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` | `941b6e3cb9a7384428d93ca248eb9630782538e04caba30dbfbe31baacca8f60` | - -## Failure disposition - -The legacy full-index image mounts, but both the complete read and a read crossing its compressed-extent boundary fail with `EIO`. The fixture and its source remain under `/work/build/repo22-shape-manual-20260808T1407Z`; copies plus per-mode dmesg files remain in the guest under `/root/repo22-short-20260808`. - -The partial-reference fixture is proven: `/b.dat` reuses the final physical compressed extent of `/a.dat`. However, `/b.dat` itself uses Layout 1 and fails before partial-reference semantics can be accepted. The same image's Layout 3 `/a.dat` reads correctly. This result is therefore a kernel failure with the legacy full-index path as the immediate blocker, not a fixture failure. - -Each mode used a separate md attachment and mount lifecycle. Cleanup succeeded after every run, the module remained loaded, and the guest answered a responsiveness check after every failure and pass. The isolated dmesg scans contained no panic, trap, explicit integrity, corruption, I/O-error, or decompression-error message; the two read failures surfaced only as user-visible `EIO`. - -## Conclusion - -The 64 KiB big-pcluster, inline ztailpacking, and packed-inode fragment read paths pass the requested kernel verification. Legacy full indexes remain broken. The proven partial-reference fixture also fails because its target file traverses the broken Layout 1 path, so partial-reference support cannot be accepted until that blocker is fixed and the same fixture is rerun. diff --git a/tests/results/manual/2026-08-08T1444Z/manual-test-report.md b/tests/results/manual/2026-08-08T1444Z/manual-test-report.md deleted file mode 100644 index 6bb08b2..0000000 --- a/tests/results/manual/2026-08-08T1444Z/manual-test-report.md +++ /dev/null @@ -1,41 +0,0 @@ -# repo22 legacy full-index fix manual report - -- Date: 2026-08-08 UTC -- Source baseline: `377d467a652beb2124d4adc0d1f8e735fca22344` -- FreeBSD guest: 15.0-RELEASE-p8 amd64, QEMU TCG -- Module SHA256: `f3b9fa70bfb8e98e2cef436583fd13cb91787514a5be76f9f108698d7586ca3c` -- Scope: legacy full-index and partial-reference minimum regression only - -## Root cause and fix - -The proven legacy Layout 1 fixture maps its second extent as `HEAD1` with -`clusterofs=28`, `pblk=2`, `m_la=639004`, `m_llen=409572`, and -`m_plen=4096`. The LZ4 stream completes after consuming 2627 bytes; the -remaining 1469 bytes of the pcluster are zero padding. The repo22 decoder -incorrectly required all 4096 input bytes to be consumed and returned `EIO`. - -The decoder now accepts complete output only when the remaining pcluster bytes -are all zero, while continuing to reject nonzero trailing data. The mapping -recorder also no longer clears `compressedblks` and `partialref` on every -lcluster load, matching the Linux state machine and preserving lookback state. - -## Results - -| Test | Mount | Full SHA256 | Boundary read | Result | -|---|---:|---|---|---| -| Legacy full index `/shape.dat` | PASS | `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` matches | offset 638972, 128 bytes: `941b6e3cb9a7384428d93ca248eb9630782538e04caba30dbfbe31baacca8f60` matches | **PASS** | -| Partial reference `/b.dat` | PASS | read returns `EIO` | offset 641020 read returns no data | **KERNEL-FAIL** | - -`./build.sh`, `kldload`, and exact-name `kldunload` passed. The isolated dmesg -delta was empty, with no panic, trap, or integrity diagnostic. - -## Remaining partial-reference hypothesis - -The minimal reproducer is the existing -`lz4-partial-ref.erofs` fixture: mount it and read `/b.dat`; the companion -compact `/a.dat` remains the known-good source extent. The full-index mapping -blocker is removed, but `zdata.c` still invokes `z_erofs_decompress()` with -`partial=false` for every mapped extent even when `EROFS_MAP_PARTIAL_REF` is -set. Linux propagates this state to partial LZ4 decoding. Per the convergence -instruction, this batch records that hypothesis without further debugging or -additional source changes. diff --git a/tests/results/manual/2026-08-08T1455Z/manual-test-report.md b/tests/results/manual/2026-08-08T1455Z/manual-test-report.md deleted file mode 100644 index af8bc81..0000000 --- a/tests/results/manual/2026-08-08T1455Z/manual-test-report.md +++ /dev/null @@ -1,33 +0,0 @@ -# repo22 partial-reference fix manual report - -- Date: 2026-08-08 UTC -- Source baseline: `616d23e59b2bc16d9735ca5ad07502a21153bc0a` -- FreeBSD guest: 15.0-RELEASE-p8 amd64, QEMU TCG -- Module SHA256: `0270671fd73c4ee604294a5f94863f1fc8cc9a63083a4c83f89887891f06f1b2` -- Scope: partial-reference output range and LZ4 regression only - -## Root cause and fix - -`zmap.c` correctly marked the reused extent with `EROFS_MAP_PARTIAL_REF`, but -`zdata.c` always decompressed the complete mapped logical length and passed -`partial=false`. Linux keeps such pclusters in partial-decoding mode and sets -the decompressor output size to the highest byte needed by the current read. - -The synchronous FreeBSD path now preserves that behavior. For a partial -reference, it decodes only through `mapoff + want`, passes `partial=true`, and -copies exactly the requested `want` bytes beginning at `mapoff`. Non-partial -extents still require full logical output and retain strict LZ4 trailing-data -validation. - -## Results - -| Test | Mount | Full SHA256 | Boundary read | Result | -|---|---:|---|---|---| -| Partial reference `/b.dat` | PASS | `61b17076c2dfae88da7912d00df534b894cf6d27178863c6d8e91f8617ebb91e` matches | offset 641020, 128 bytes: `941b6e3cb9a7384428d93ca248eb9630782538e04caba30dbfbe31baacca8f60` matches | **PASS** | -| Legacy full index `/shape.dat` | PASS | `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` matches | smoke only | **PASS** | -| Compact index `/shape.dat` | PASS | `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` matches | smoke only | **PASS** | - -`./build.sh`, `kldload`, and exact-name `kldunload` passed. The isolated dmesg -delta was empty, with no panic, trap, integrity diagnostic, or decompression -error. Build objects, kernel module output, images, and VM overlays were not -staged for commit. diff --git a/tests/results/manual/2026-08-08T1705Z/manual-test-report.md b/tests/results/manual/2026-08-08T1705Z/manual-test-report.md deleted file mode 100644 index 7ef90a2..0000000 --- a/tests/results/manual/2026-08-08T1705Z/manual-test-report.md +++ /dev/null @@ -1,46 +0,0 @@ -# repo22 ACL/xattr integration smoke report - -- Date: 2026-08-08 UTC -- Source baseline before this batch: `8e629bb9f798f0c76497bd2b623329a0aff430b9` -- FreeBSD guest: 15.0-RELEASE-p8 amd64, QEMU TCG -- Module SHA256: `18967bd32e89b08a2363241ed4b9b67344b7bfb555f1f77daa64f382925e4fda` -- Scope: ACL/xattr worktree integration, build/link validation, and minimal - Markdown-directed manual smoke tests - -## Results - -| Check | Result | Evidence | -|---|---|---| -| `./build.sh` | PASS | `build/erofs.ko` generated without errors | -| unresolved `bcmp` | PASS | `nm -u build/erofs.ko` contains no `bcmp` | -| ACL kernel dependency | PASS | module declares `acl_posix1e`; `kldload` succeeds | -| module lifecycle | PASS | `kldload`, `kldstat`, and exact module-ID `kldunload` succeed | -| TC005 user xattr smoke | PASS | `user.comment` lists and reads as `repo22-user-xattr` | -| TC082 access ACL smoke | PASS | system xattr lists `posix_acl_access`; raw ACL reads successfully | -| FreeBSD ACL decode | PASS | `getfacl` reports named user, group, mask, and other entries | -| TC060 ACL capability | PASS | `getconf ACL_EXTENDED` returns `1` | -| read-only access smoke | PASS | read succeeds and write access is denied | -| cleanup | PASS | no EROFS module, mount, or md device remains | - -The ACL fixture contained a real extended POSIX.1e access ACL with a named -user and mask, so Linux did not collapse it into mode bits. `dump.erofs` -reported 88 bytes of inode xattrs. FreeBSD returned: - -```text -user::rw- -user:ntpd:r-- -group::r-- -mask::r-- -other::--- -``` - -The fixture image, source directory, kernel module, VM overlay, and generated -objects remained under `/work/build` or `repo22/build` and were not staged. - -## Remaining scope - -- Default-directory ACL retrieval and access checks under multiple credentials - were not exercised in this smoke batch. -- Shared-xattr base calculation, long/packed prefixes, xattr filters, malformed - xattr bounds, and metabox-backed shared xattrs still require dedicated tests. -- Multi-device work is outside this ACL/xattr integration batch. diff --git a/tests/results/manual/2026-08-08T1800Z-namei/manual-test-report.md b/tests/results/manual/2026-08-08T1800Z-namei/manual-test-report.md deleted file mode 100644 index af74156..0000000 --- a/tests/results/manual/2026-08-08T1800Z-namei/manual-test-report.md +++ /dev/null @@ -1,112 +0,0 @@ -# repo22 Cold Nested Namei Regression Report - -- Date: 2026-08-08 UTC -- Baseline: `29a215dd4519579f6313b3df67d524a6b4bdf3ca` -- Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG -- Final module SHA-256: - `50a19ea96c7414f73d92049671e66a9eacc9ff5abe27950a46d16c8e6c763baf` -- Result: PASS - -## Root Cause - -FreeBSD pathname lookup passes a component as `cn_nameptr` plus -`cn_namelen`. An intermediate component is followed by `/` in the pathname -buffer and is not NUL-terminated at `cn_namelen`. The imported Linux EROFS -comparison assumed Linux dentry-name termination, ignored the supplied length, -and tested `qn_name[i] == '\0'` after matching the on-disk name. Therefore a -final component worked, while the same name used as an intermediate component -compared greater than the on-disk entry and returned `ENOENT`. Looking up the -parent as a final component first populated the FreeBSD namecache and hid the -bug on the next nested lookup. - -The old error path also inserted a negative cache entry for every lookup -error, including integrity and I/O errors, which could mask later corruption -as `ENOENT`. - -## Implementation - -- `src/namei.c` - - Compares pathname components by explicit length without reading beyond - `cn_namelen`. - - Uses unsigned-byte ordering compatible with EROFS directory sorting. - - Validates the minimum block size before reading the first dirent. - - Validates the dirent-array boundary, strictly increasing name offsets, - name-slot bounds, name length, and zero-only NUL padding. - - Inserts negative namecache entries only for real `ENOENT` misses. -- `src/dir.c` - - Applies the same directory-block and name-padding validation to `readdir`. - - Determines the actual last-name length before enforcing `EROFS_NAME_LEN`, - so valid full-block zero padding is accepted. - - Keeps on-disk offsets unchanged and appends a synthetic `.` at `i_size` - for `dot_omitted`, matching Linux EROFS. - - Aligns restart positions relative to each directory block and preserves - the `i_size` cookie needed to resume the synthetic dot entry. - - Initializes returned cookie-array outputs before allocation. - -## Deterministic Fixture - -`prepare-fixtures.sh` creates the same image twice and requires `cmp` success. -The base image contains a cold multi-level path and a 320-file, multi-block -directory. Fixed timestamp, ownership, UUID, worker count, and uncompressed -layout are used. - -Final image hashes: - -```text -2e9fd75159011ced31646a38f942fa0822d5b3e8e19b7df55b844575fba991ea corrupt-nameoff.erofs -63d12232f9ea0dc7f7ff477d20b95a8412d191fc10bf4b67dacc47e050c379fb corrupt-padding.erofs -77f8a56f969e173d291ccbd957821f1ba284d3e54d875dbcdb9bc398024dfa5a corrupt-short-block.erofs -11064ffbb814ff41027aebc8f36e56d2f24affb7b50836948b3ffbf6d79dee0e dot-omitted.erofs -7a2c244ec10e9a43531b0e8b92e26b11f52d67bab00528b8ba2f584e20e57420 nested-repeat.erofs -7a2c244ec10e9a43531b0e8b92e26b11f52d67bab00528b8ba2f584e20e57420 nested.erofs -``` - -## Build Results - -- `./build.sh`: PASS. -- Final `build/erofs.ko` SHA-256 remained - `50a19ea96c7414f73d92049671e66a9eacc9ff5abe27950a46d16c8e6c763baf`. -- `nm -u build/erofs.ko | grep -w bcmp`: no match. -- `git diff --check` for all scoped source and test files: PASS. - -The final integration rerun used the same combined module and these guest -commands: - -```sh -cd /root/repo22-namei -cc -Wall -Wextra -O2 -o readdir_probe readdir_probe.c -./vm-regression.sh -``` - -`vm-regression.sh` performs `kldload`, creates each vnode-backed md device, -mounts it with `mount -t erofs`, executes the cold lookup and readdir probes, -unmounts and detaches each image, and finishes with exact module unload. - -## FreeBSD VM Results - -- `kldload`: PASS. -- Cold direct read of - `/alpha/bravo/charlie/payload.txt` without parent lookup or `readdir`: PASS. -- Repeated lookup and sibling nested lookup: PASS. -- Two negative lookups followed by an existing nested lookup: PASS. -- Multi-block `wide` readdir: 322 dirents (`.`, `..`, 320 files), PASS. -- Resume from every one of the 322 returned `d_off` cookies: PASS. -- `dot_omitted` root cookies: `12`, `24`, `47`, `48`; resume at `47` - returns only `.`, and resume at `48` returns EOF: PASS. -- Short directory block: two lookups and direct `getdirentries` all return - `EINTEGRITY`, PASS. -- Non-monotonic `nameoff`: two lookups and direct `getdirentries` all return - `EINTEGRITY`, PASS. -- Nonzero data after NUL padding: two lookups and direct `getdirentries` all - return `EINTEGRITY`, PASS. -- `kldunload`: PASS. -- Post-test EROFS module, mount, and md-device state: clean. -- Final explicit state check: zero EROFS modules, zero EROFS mounts, zero md - devices, and no recent panic or fatal trap in dmesg. - -## Remaining Scope - -No unresolved issue remains for the requested cold lookup and directory -regression. The NFS-specific `a_cookies` consumer path was not exercised by an -NFS export; the tested `d_off` restart-cookie sequence uses the same generated -cookie values. diff --git a/tests/results/manual/2026-08-08T1800Z-namei/prepare-fixtures.sh b/tests/results/manual/2026-08-08T1800Z-namei/prepare-fixtures.sh deleted file mode 100755 index 06f434c..0000000 --- a/tests/results/manual/2026-08-08T1800Z-namei/prepare-fixtures.sh +++ /dev/null @@ -1,199 +0,0 @@ -#!/bin/sh -set -eu - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -fixture_dir="$script_dir/fixture" -artifact_dir="$script_dir/artifacts" - -mkdir -p \ - "$fixture_dir/alpha/bravo/charlie" \ - "$fixture_dir/alpha/sibling" \ - "$fixture_dir/wide" \ - "$artifact_dir" - -printf '%s\n' 'cold nested lookup payload' > \ - "$fixture_dir/alpha/bravo/charlie/payload.txt" -printf '%s\n' 'repeat lookup payload' > \ - "$fixture_dir/alpha/bravo/repeat.txt" -printf '%s\n' 'sibling marker' > \ - "$fixture_dir/alpha/sibling/marker.txt" - -index=0 -while [ "$index" -lt 320 ]; do - name=$(printf 'entry-%03d-abcdefghijklmnopqrstuvwxyz.txt' "$index") - printf 'wide entry %03d\n' "$index" > "$fixture_dir/wide/$name" - index=$((index + 1)) -done - -find "$fixture_dir" -exec touch -h -t 197001010000.00 {} + - -build_image() -{ - image=$1 - mkfs.erofs -d0 -x-1 -T0 --all-time --all-root --workers=1 \ - -U 11111111-2222-3333-4444-555555555555 \ - "$image" "$fixture_dir" -} - -build_image "$artifact_dir/nested.erofs" -build_image "$artifact_dir/nested-repeat.erofs" -cmp "$artifact_dir/nested.erofs" "$artifact_dir/nested-repeat.erofs" - -ARTIFACT_DIR="$artifact_dir" python3 <<'PY' -import hashlib -import os -import shutil -import struct - -artifact_dir = os.environ["ARTIFACT_DIR"] -base_path = os.path.join(artifact_dir, "nested.erofs") - - -def u16(image, offset): - return struct.unpack_from("> 1) ^ (polynomial if checksum & 1 else 0) - put_u32(image, 1024 + 4, checksum & 0xFFFFFFFF) - - -def inode_offset(nid): - return nid << 5 - - -def compact_inline_dir(image, nid): - offset = inode_offset(nid) - inode_format = u16(image, offset) - layout = (inode_format >> 1) & 0x7 - assert (inode_format & 0x1) == 0 - assert layout == 2 - assert u16(image, offset + 2) == 0 - return offset, offset + 32, u32(image, offset + 8) - - -def dir_entries(image, data_offset, size): - first_nameoff = u16(image, data_offset + 8) - assert first_nameoff >= 12 - assert first_nameoff % 12 == 0 - assert first_nameoff < size - count = first_nameoff // 12 - entries = [] - for index in range(count): - entry_offset = data_offset + index * 12 - nid, nameoff, file_type, reserved = struct.unpack_from( - "> 1) & 0x7) == 2 -wide_startblk = u32(base, wide_inode + 16) -wide_block = wide_startblk << 12 -wide_first_nameoff = u16(base, wide_block + 8) -wide_count = wide_first_nameoff // 12 -wide_last_nameoff = u16(base, wide_block + (wide_count - 1) * 12 + 8) -padding_nul = base.index(0, wide_block + wide_last_nameoff, wide_block + 4096) -assert padding_nul + 1 < wide_block + 4096 -assert base[padding_nul + 1] == 0 -bad_padding = bytearray(base) -bad_padding[padding_nul + 1] = ord("X") - -outputs = { - "dot-omitted.erofs": dot_omitted, - "corrupt-short-block.erofs": short_block, - "corrupt-nameoff.erofs": nonmonotonic, - "corrupt-padding.erofs": bad_padding, -} -for filename, image in outputs.items(): - update_superblock_checksum(image) - path = os.path.join(artifact_dir, filename) - with open(path, "wb") as output: - output.write(image) - -with open(os.path.join(artifact_dir, "SHA256SUMS"), "w", encoding="ascii") as sums: - for filename in sorted(["nested.erofs", "nested-repeat.erofs", *outputs]): - path = os.path.join(artifact_dir, filename) - with open(path, "rb") as image_file: - digest = hashlib.sha256(image_file.read()).hexdigest() - sums.write(f"{digest} {filename}\n") -PY - -cat "$artifact_dir/SHA256SUMS" diff --git a/tests/results/manual/2026-08-08T1800Z-namei/readdir_probe.c b/tests/results/manual/2026-08-08T1800Z-namei/readdir_probe.c deleted file mode 100644 index 0b0a0ee..0000000 --- a/tests/results/manual/2026-08-08T1800Z-namei/readdir_probe.c +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -int -main(int argc, char **argv) -{ - struct dirent *entry; - off_t base, before, start; - char *buffer; - char *end; - size_t buffer_size; - ssize_t bytes; - int calls, fd; - - if (argc < 2 || argc > 5) - errx(2, "usage: %s directory [offset [buffer-size [calls]]]", - argv[0]); - start = argc >= 3 ? strtoll(argv[2], NULL, 0) : 0; - buffer_size = argc >= 4 ? strtoul(argv[3], NULL, 0) : 128; - calls = argc >= 5 ? strtol(argv[4], NULL, 0) : 32; - if (buffer_size < 32 || calls < 1) - errx(2, "invalid buffer size or call count"); - - fd = open(argv[1], O_RDONLY | O_DIRECTORY); - if (fd < 0) - err(1, "open %s", argv[1]); - if (lseek(fd, start, SEEK_SET) < 0) - err(1, "lseek %jd", (intmax_t)start); - buffer = malloc(buffer_size); - if (buffer == NULL) - err(1, "malloc"); - - for (int call = 0; call < calls; call++) { - before = lseek(fd, 0, SEEK_CUR); - if (before < 0) - err(1, "lseek current"); - base = -1; - bytes = getdirentries(fd, buffer, buffer_size, &base); - if (bytes < 0) - err(1, "getdirentries"); - printf("call=%d before=%jd after=%jd base=%jd bytes=%zd\n", - call, (intmax_t)before, - (intmax_t)lseek(fd, 0, SEEK_CUR), (intmax_t)base, bytes); - if (bytes == 0) - break; - end = buffer + bytes; - for (entry = (struct dirent *)buffer; - (char *)entry < end; - entry = (struct dirent *)((char *)entry + entry->d_reclen)) { - if (entry->d_reclen == 0 || - (char *)entry + entry->d_reclen > end) - errx(1, "invalid dirent record"); - printf(" off=%jd ino=%ju reclen=%u type=%u name=%.*s\n", - (intmax_t)entry->d_off, (uintmax_t)entry->d_fileno, - entry->d_reclen, entry->d_type, entry->d_namlen, - entry->d_name); - } - } - - free(buffer); - close(fd); - return (0); -} diff --git a/tests/results/manual/2026-08-08T1800Z-namei/vm-regression.sh b/tests/results/manual/2026-08-08T1800Z-namei/vm-regression.sh deleted file mode 100755 index 97f0de4..0000000 --- a/tests/results/manual/2026-08-08T1800Z-namei/vm-regression.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/bin/sh -set -u - -test_dir=/root/repo22-namei -mount_dir=/mnt/repo22-namei -module_id= -md_device= - -fail() -{ - echo "FAIL: $*" >&2 - exit 1 -} - -cleanup() -{ - set +e - if mount | grep -q " on $mount_dir "; then - umount "$mount_dir" - fi - if [ -n "$md_device" ]; then - mdconfig -d -u "${md_device#md}" - fi - if [ -n "$module_id" ] && kldstat -q -i "$module_id"; then - kldunload -i "$module_id" - fi -} -trap cleanup EXIT INT TERM - -mount_image() -{ - image=$1 - md_device=$(mdconfig -a -t vnode -f "$test_dir/$image") || - fail "mdconfig $image" - mount -t erofs "/dev/$md_device" "$mount_dir" || - fail "mount $image" - echo "mounted image=$image device=$md_device" -} - -unmount_image() -{ - umount "$mount_dir" || fail "umount $md_device" - mdconfig -d -u "${md_device#md}" || fail "detach $md_device" - md_device= -} - -expect_integrity_failure() -{ - image=$1 - lookup_path=$2 - readdir_path=$3 - label=${image%.erofs} - - mount_image "$image" - attempt=1 - while [ "$attempt" -le 2 ]; do - output="$test_dir/$label-lookup-$attempt.txt" - if stat "$mount_dir/$lookup_path" >"$output" 2>&1; then - fail "$image lookup attempt $attempt unexpectedly succeeded" - fi - if grep -qi "No such file" "$output"; then - fail "$image lookup attempt $attempt became ENOENT" - fi - attempt=$((attempt + 1)) - done - output="$test_dir/$label-readdir.txt" - if ./readdir_probe "$mount_dir/$readdir_path" 0 512 2 \ - >"$output" 2>&1; then - fail "$image readdir unexpectedly succeeded" - fi - unmount_image - echo "integrity image=$image repeated-lookup=error readdir=error" -} - -cd "$test_dir" || exit 1 -mkdir -p "$mount_dir" - -if mount | grep -qi erofs; then - fail "pre-existing EROFS mount" -fi -if kldstat | grep -qi erofs; then - fail "pre-existing EROFS module" -fi -if [ -n "$(mdconfig -l)" ]; then - fail "pre-existing md device" -fi - -echo "== guest ==" -uname -a -date -u - -echo "== module load ==" -kldload "$test_dir/erofs.ko" || fail "kldload" -module_id=$(kldstat | awk '$NF == "erofs.ko" { print $1 }') -[ -n "$module_id" ] || fail "loaded module not found" -kldstat -v -i "$module_id" - -echo "== cold nested lookup ==" -mount_image nested.erofs -payload=$(cat "$mount_dir/alpha/bravo/charlie/payload.txt") || - fail "cold nested lookup" -[ "$payload" = "cold nested lookup payload" ] || fail "cold payload mismatch" -payload=$(cat "$mount_dir/alpha/bravo/charlie/payload.txt") || - fail "repeated nested lookup" -[ "$payload" = "cold nested lookup payload" ] || fail "repeat payload mismatch" -payload=$(cat "$mount_dir/alpha/bravo/repeat.txt") || - fail "sibling nested lookup" -[ "$payload" = "repeat lookup payload" ] || fail "sibling payload mismatch" - -attempt=1 -while [ "$attempt" -le 2 ]; do - output="$test_dir/missing-$attempt.txt" - if stat "$mount_dir/alpha/bravo/missing" >"$output" 2>&1; then - fail "missing lookup attempt $attempt unexpectedly succeeded" - fi - grep -qi "No such file" "$output" || fail "missing lookup was not ENOENT" - attempt=$((attempt + 1)) -done -payload=$(cat "$mount_dir/alpha/bravo/charlie/payload.txt") || - fail "existing lookup after negative cache" -[ "$payload" = "cold nested lookup payload" ] || fail "post-negative payload mismatch" -echo "cold lookup=pass repeat=pass negative-cache=pass" - -echo "== large readdir and cookies ==" -wide_count=$(ls -A1 "$mount_dir/wide" | wc -l | tr -d ' ') -[ "$wide_count" = 320 ] || fail "wide entry count $wide_count" -./readdir_probe "$mount_dir/wide" 0 128 400 > wide-probe.txt || - fail "wide readdir probe" -awk '/^ off=/ { - off = $1; sub(/^off=/, "", off); - name = $5; sub(/^name=/, "", name); - print off, name; -}' wide-probe.txt > wide-cookies.txt -wide_dirents=$(wc -l < wide-cookies.txt | tr -d ' ') -[ "$wide_dirents" = 322 ] || fail "wide dirent count $wide_dirents" -awk '{ cookie[NR] = $1; name[NR] = $2 } - END { - for (i = 1; i <= NR; i++) - print cookie[i], (i < NR ? name[i + 1] : ""); - }' wide-cookies.txt > wide-resume-cases.txt -while read -r cookie expected; do - ./readdir_probe "$mount_dir/wide" "$cookie" 128 1 > wide-resume.txt || - fail "resume cookie $cookie" - if [ "$expected" = "" ]; then - grep -q 'bytes=0$' wide-resume.txt || fail "cookie $cookie not EOF" - else - actual=$(awk '/^ off=/ { - name = $5; sub(/^name=/, "", name); print name; exit; - }' wide-resume.txt) - [ "$actual" = "$expected" ] || - fail "cookie $cookie expected $expected got $actual" - fi -done < wide-resume-cases.txt -echo "wide entries=$wide_dirents all-resume-cookies=pass" -unmount_image - -echo "== dot omitted cookies ==" -mount_image dot-omitted.erofs -./readdir_probe "$mount_dir" 0 512 8 > dot-probe.txt || fail "dot probe" -awk '/^ off=/ { - off = $1; sub(/^off=/, "", off); - name = $5; sub(/^name=/, "", name); - print off, name; -}' dot-probe.txt > dot-cookies.txt -cat > dot-expected.txt <<'EOF' -12 .. -24 alpha -47 wide -48 . -EOF -cmp dot-cookies.txt dot-expected.txt || fail "dot cookie sequence" -./readdir_probe "$mount_dir" 47 128 1 > dot-resume-47.txt || fail "dot resume 47" -grep -q '^ off=48 .* name=\.$' dot-resume-47.txt || fail "dot resume 47 result" -./readdir_probe "$mount_dir" 48 128 1 > dot-resume-48.txt || fail "dot resume 48" -grep -q 'bytes=0$' dot-resume-48.txt || fail "dot resume 48 not EOF" -echo "dot cookies=12,24,47,48 resume=pass" -unmount_image - -echo "== corrupted directories ==" -expect_integrity_failure corrupt-short-block.erofs alpha . -expect_integrity_failure corrupt-nameoff.erofs alpha . -expect_integrity_failure \ - corrupt-padding.erofs \ - wide/entry-000-abcdefghijklmnopqrstuvwxyz.txt \ - wide - -echo "== module unload ==" -kldunload -i "$module_id" || fail "kldunload" -module_id= -if kldstat | grep -qi erofs; then - fail "module remains loaded" -fi -if mount | grep -qi erofs; then - fail "mount remains" -fi -if [ -n "$(mdconfig -l)" ]; then - fail "md remains" -fi - -dmesg | tail -120 > dmesg-tail.txt -echo "PASS: all VM regressions" diff --git a/tests/results/manual/2026-08-08T1812Z/manual-test-report.md b/tests/results/manual/2026-08-08T1812Z/manual-test-report.md deleted file mode 100644 index a4bb7fc..0000000 --- a/tests/results/manual/2026-08-08T1812Z/manual-test-report.md +++ /dev/null @@ -1,113 +0,0 @@ -# repo22 xattr/ACL root-cause manual test report - -Date: 2026-08-08 18:12 UTC; final integration rerun on 2026-08-08 -Baseline: `29a215dd4519579f6313b3df67d524a6b4bdf3ca` plus the scoped xattr/ACL fixes -Guest: FreeBSD 15.0-RELEASE-p8 amd64 -Host tools: erofs-utils 1.8.6 plus deterministic transformed fixtures - -## Build and module - -- `git diff --check`: PASS. -- `./build.sh`: PASS; final `build/erofs.ko` SHA-256 is - `50a19ea96c7414f73d92049671e66a9eacc9ff5abe27950a46d16c8e6c763baf`. -- `nm -u build/erofs.ko | grep -w bcmp`: no match, PASS. -- `kldload /tmp/repo22-integration-20260808/erofs.ko`: PASS. -- Repeated `mdconfig -a -t vnode -f IMAGE`, `mount -t erofs`, `umount`, and - `mdconfig -d`: PASS for every fixture below. -- `kldunload erofs`: PASS; final module, EROFS mount, and md-device counts were - all zero. -- New dmesg errors, traps, or panics: none. - -## Existing xattr and ACL regression - -- Inline user, trusted, and security xattrs: PASS. -- Shared user xattrs and shared POSIX access ACL: PASS. -- Packed long-prefix table and xattr-name-filter: PASS. -- Access/default ACL decode and inherited child ACL: PASS. -- Owner, owning-group, named-user, and other access matrix: PASS. -- Read-only ACL/xattr mutation rejection: PASS. -- TC117 malformed name length and value size: PASS. Xattr operations returned - `EINTEGRITY`; directory metadata and file contents remained readable. -- An invalid long-prefix reference without a declared prefix table is now - skipped like Linux. Other valid xattrs and file data remain readable. - -## New regression cases - -- TC134 metabox shared nonzero base: PASS with - `metabox-shared-nonzero-base.erofs`. The image has - `SHARED_EA_IN_METABOX`, `xattr_blkaddr=1`, and a plain metabox carrier. - `getextattr -qq user metaboxshared /mnt/repo22-integration/hello.txt` - returned exactly `nonzero-base`. -- TC135 metabox long-prefix backing: PASS with - `metabox-prefix-shared.erofs`. - `getextattr -qq user repo22.application.component.setting` on - `/mnt/repo22-integration/long/file2.txt` returned `long-prefix-value`. - Primary-image long-prefix fallback also passed with - `long-prefix-primary-fallback.erofs` and the same expected value. -- TC136 truncated metabox extension: PASS; mount returned `EINTEGRITY`. - Out-of-range ishare prefix ID: PASS; mount returned `EINTEGRITY`. -- TC137 shared xattr outside declared image: PASS; normal data and an in-bounds - xattr remained readable, while the redirected entry returned `EINTEGRITY`. - Prefix record outside declared image: PASS; mount failed. The valid primary - fallback control mounted and returned the expected value. -- TC138 unordered unique UID qualifiers: PASS and preserved order 3002, 2002. - Header-only ACL: PASS and fell back to mode. Duplicate UID qualifier: PASS - negative test and returned `EINTEGRITY`. -- TC139 FIFO access ACL, list/get system xattr: PASS. `setfacl`, `setextattr`, - and `rmextattr` all returned read-only filesystem errors. -- TC140 compressed metabox: PASS with `metabox-large-shared.erofs`. On-disk - qualification found `METABOX`, metabox inode datalayout 1 - (`EROFS_INODE_COMPRESSED_FULL`), and no fragment pcluster. A cold file read, - the metabox long-prefix lookup, and a 231-byte xattr value all matched the - source fixture. - -## Final metabox and metadata-boundary smoke - -The final combined module was exercised with these exact guest operations: - -```sh -kldload /tmp/repo22-integration-20260808/erofs.ko -mdconfig -a -t vnode -f /tmp/repo22-integration-20260808/IMAGE.erofs -mount -t erofs /dev/md0 /mnt/repo22-integration -getextattr -qq user NAME /mnt/repo22-integration/PATH -umount /mnt/repo22-integration -mdconfig -d -u 0 -kldunload erofs -``` - -- Compressed metabox and metabox long-prefix: - `metabox-large-shared.erofs` returned `long-prefix-value` for - `repo22.application.component.setting`; its regular file content also - matched `long prefix fixture 1`. -- Primary shared fallback while `METABOX` is enabled: - `metabox-large-shared.erofs` has `SHARED_EA_IN_METABOX` clear and - `xattr_blkaddr=1`; `shared_key` returned `repo22-shared-value` from the - primary shared-xattr area. -- `SHARED_EA_IN_METABOX` nonzero base: - `metabox-shared-nonzero-base.erofs` returned `nonzero-base` from the shared - entry addressed relative to metabox block 1. -- Cross-metadata shared xattr: `shared-cross-metadata.erofs` returned - `repo22-shared-value` for `shared_key` on `shared/file3.txt`. -- Cross-metadata inline xattr: `xattr-cross-metadata.erofs` returned the exact - expected `alpha`, `gamma`, and `delta` values on `file000`, `file090`, and - `file179`. The metabox-specific inline boundary fixture also returned - `valid-inline-boundary` for `crossboundary`. - -The fragment-backed compressed metabox negative layout was **not generated and -was not executed**. The passing TC140 fixture is compressed but non-fragment. -The code still rejects a compressed metabox inode whose fragment flag is set; -that rejection remains layout-reviewed rather than fixture-verified. - -## Fixture qualification - -The old `namespace-shared.erofs` contains an xattr entry with name index `0x80` -but advertises no `XATTR_PREFIXES` feature and contains no prefix table. It is -not a valid long-prefix fixture. The driver now follows Linux behavior by -skipping that unresolved entry while preserving all valid xattrs and file data. - -## Separate namei write set - -The previously observed cold nested lookup issue was fixed by the separate -`src/namei.c` and `src/dir.c` write set and passed TC141 in the same final -integration build. Those source files are intentionally excluded from the -xattr/ACL commit. diff --git a/tests/results/manual/2026-08-08T1937Z-multidev/manual-test-report.md b/tests/results/manual/2026-08-08T1937Z-multidev/manual-test-report.md deleted file mode 100644 index 3725489..0000000 --- a/tests/results/manual/2026-08-08T1937Z-multidev/manual-test-report.md +++ /dev/null @@ -1,232 +0,0 @@ -# repo22 real multi-device manual test report - -Date: 2026-08-08 19:37 UTC -Baseline: `96e22cc713cc57180ce3ecb2b98852f090e3868b` plus this multi-device batch -Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG -Host tools: erofs-utils 1.8.6 - -## Result - -**PASS**. Real external providers, explicit slot mapping, flatdev, unified -addresses, 48-bit fields, failure rollback, forced GEOM orphaning, and the -single-device regressions all passed. There were no new dmesg lines, traps, or -panics. Final EROFS mount, md-provider, and loaded-module counts were zero. - -## Build and module - -Commands: - -```sh -git diff --check -./build.sh -nm -u build/erofs.ko | awk '$NF == "bcmp" {bad=1} END {exit bad}' -sha256sum build/erofs.ko -``` - -Actual: - -- `git diff --check`: PASS. -- Cross-build against `/work/dev-freebsd-releng`: PASS. -- Unresolved `bcmp`: none, PASS. -- Final `build/erofs.ko` SHA256: - `aa6708256246b303f4be2a8d516bf503f8a232df1ad3f466f5685573f221c180`. -- Final `kldload` and `kldunload`: PASS. - -## Fixture commands - -The legal baseline was generated by erofs-utils 1.8.6, not by synthesizing an -EROFS image from scratch: - -```sh -WORK=/work/build/repo22-multidev-fixtures-20260808 -mkdir -p "$WORK/src/cross" -python3 - <<'PY' -from pathlib import Path -import hashlib -root = Path('/work/build/repo22-multidev-fixtures-20260808/src') -def content(label, blocks, tail=0): - out = bytearray() - for block in range(blocks): - seed = f'{label}:block:{block}'.encode() - chunk = bytearray() - counter = 0 - while len(chunk) < 4096: - chunk += hashlib.sha256( - seed + counter.to_bytes(4, 'little')).digest() - counter += 1 - out += chunk[:4096] - if tail: - out += hashlib.sha256(f'{label}:tail'.encode()).digest()[:tail] - return bytes(out) -root.joinpath('cross/striped.bin').write_bytes(content('striped', 24, 173)) -root.joinpath('cross/second.bin').write_bytes(content('second', 17, 29)) -root.joinpath('control.txt').write_text( - 'repo22 multidev deterministic control\n', encoding='ascii') -PY -truncate -s 0 "$WORK/base.blob" -mkfs.erofs -T0 --all-root --chunksize=4096 \ - --blobdev="$WORK/base.blob" "$WORK/base.primary" "$WORK/src" -``` - -erofs-utils 1.8.6 accepts one `--blobdev`; direct generation of two or more -external slots is therefore **MKFS-UNAVAILABLE**. Multi-slot fixtures were made -by a structured, assertion-driven patch of the legal baseline: - -1. `dump.erofs --path=PATH -e base.primary` supplied each NID and file size. -2. The patch asserted compact chunk layout, zero xattr size, index format, old - device ID 1, and every old block address. -3. A complete physical chunk, rounded to 4096 bytes for the final partial - chunk, was copied into a deterministic round-robin blob. -4. Each index was rewritten as little-endian - `(startblk_hi, device_id, startblk_lo)`. -5. One zeroed primary block was appended for the enlarged device table; - `blocks_lo`, `extra_devices`, and `devt_slotoff` were updated. This avoids - overwriting inode metadata merely to add slots. -6. Each 128-byte slot was written at offsets 64/68/72/74 with `blocks_lo`, - `uniaddr_lo`, `blocks_hi`, and `uniaddr_hi`. -7. The EROFS CRC32C was recomputed over bytes 1024 through 4095 with polynomial - `0x82f63b78` and initial value `0xffffffff`. - -The asserted patch manifest was: - -```text -/control.txt nid=44 size=38 chunk=4096 indexes=1 base=1440 -/cross/second.bin nid=50 size=69661 chunk=4096 indexes=18 base=1632 -/cross/striped.bin nid=56 size=98336 chunk=32768 indexes=4 base=1824 -``` - -Pre-kernel qualification: - -```sh -fsck.erofs --device=two.blob1 --device=two.blob2 \ - --extract=two.extract two.primary -fsck.erofs --device=three.blob1 --device=three.blob2 \ - --device=three.blob3 --extract=three.extract three.primary -fsck.erofs --device=four.blob1 --device=four.blob2 \ - --device=four.blob3 --device=four.blob4 \ - --extract=four.extract four.primary -``` - -All extracted source SHA256 values matched. The 48-bit feature and patched -`device_id=0 + uniaddr` extraction are not understood by erofs-utils 1.8.6; -those layouts were qualified by the FreeBSD kernel SHA256 tests below and the -Linux 7.1 `erofs_map_dev` semantics. - -FreeBSD attachment and mount commands used one md provider per image/blob: - -```sh -mdconfig -a -t vnode -f primary -u 90 -mdconfig -a -t vnode -f blob1 -u 91 -mdconfig -a -t vnode -f blob2 -u 92 -mount -t erofs -o ro -o device.2=/dev/md92 \ - -o device.1=/dev/md91 /dev/md90 /mnt/repo22-multidev -``` - -The reversed option order was intentional. Flatdev used only: - -```sh -mdconfig -a -t vnode -f two.flat -u 90 -mount -t erofs -o ro /dev/md90 /mnt/repo22-multidev -``` - -## Fixture SHA256 - -| Fixture | SHA256 | -|---|---| -| one-blob primary | `fa2f35f59f63cc4a56d230ba960a6c1d27748506fccce081447cab0dc0d540fd` | -| one-blob slot 1 | `3aeb19c519636b61dc3ebd47c6286098deaa0990ff618ce3facbff4fd65907cd` | -| two-slot primary | `46d76d5cc311f97f263ea4c20a2510338f5b11da9d71e0c73f69ae3751afd696` | -| two-slot blob 1 | `53853d033adf154a956192d9813726888c3f9575ecd13fa45a26aea5c4f07eaf` | -| two-slot blob 2 | `9ee3e3ad420024ff1945d92469d3d337817977f28c5b745c4ca7b1664f4d6584` | -| two-slot flatdev | `4b561db1eb0048989256f3c51504b4611881069bf283fbfe2051847e8162301e` | -| three-slot primary | `4f2dc4ee6385f31139151e1b04279a8fb90fcd2cbba928ee6949df62f1bb9585` | -| three-slot blobs 1/2/3 | `12c5d73cb12e1bb5fd1a9d9715ec88d2132dbaa8df62e2b5aad3cfb454116a3a`, `3d1574254a97e572fca323f153bb2a46ebdfb060a0765298700768e39a7f91d8`, `ba42a2a3724771975c27a43ac8d64a2568515fb4e8ad14b29cc09a7a865ffba0` | -| four-slot primary | `d3366dfb5f4c98db2db669ead3209a0abbba5df989d9aea7e1217aff46f0154b` | -| four-slot blobs 1/2/3/4 | `ad24e257baf97f855f756cc917878a9616858860ddbfab4ffe3cc362e9233662`, `eb55bd05f873dd8732bcbe35a0818ef232a9eff6fedec618f5f949253d7cd2d1`, `59f9a3aed34cc8598e6a96381b55a18f6a0e6674e3e26921834948ad333cb3c5`, `d5080743a85e47752cd33e86e26690b8978d314e685b27d449d5394c44e54244` | -| device ID 0 + uniaddr | `e4cf71f76f2a2e690e905e4135c21ce29d647be5dd2b6ce567ac19f82d1875d2` | -| 48-bit high uniaddr | `00d49beea1237125886f13912bf062153b992582c1cf3a2494b44679285b192e` | -| overlapping-slot negative | `6ecfc77b688ac1931f3e904f48fa43a81c05a6047c21ccfc4636e9e07c17d5cf` | -| mapped-ID-3 negative | `dd6ddc6943b7edd2009a48999d03a2d5b69a9ce7d820357527dbcdb52adb2afc` | - -Source SHA256 values for the cross-device fixture: - -```text -6a38f93ca44de4affcef23ad5979505225f21e5b6912b5c886866170c007b16c ./control.txt -45b7df16a5833013042694c2e8171d782102bf20abbc6c1ec2789518d5a65125 ./cross/second.bin -7ab83ca35cc4bb361bf60b5619d3f55b717fb8e2a7a81a2f069dce859efe8de9 ./cross/striped.bin -``` - -## Test matrix - -| Test | Expected | Actual | Status | -|---|---|---|---| -| TC093 plain single device | full SHA256 match | both files matched | PASS | -| TC093 chunk single device | device ID 0 reads primary | both files matched | PASS | -| compression regression | LZ4 data unchanged | both files matched | PASS | -| TC098 fragments | packed-inode file matches | `65b9a59e...e7cd` | PASS | -| metabox regression | cold metabox-backed files match | `f1` and `f10` matched known hashes | PASS | -| TC006/TC099 primary + one blob | all external chunks use slot 1 | three files matched | PASS | -| TC094 two external slots | reversed option order, cross-slot SHA256 | all three files matched | PASS | -| TC100 four providers total | slots 1-3, order 3/1/2 | all three files matched | PASS | -| additional four external slots | file crosses four blobs | SHA256 matched | PASS | -| statfs combined blocks | sum primary and external blocks | 184 KiB reported for 46 blocks | PASS | -| TC101 flatdev | non-zero IDs add `uniaddr` on primary | SHA256 matched | PASS | -| TC101 device ID 0 + uniaddr | unified range selects external provider | SHA256 matched | PASS | -| 48-bit root/index/uniaddr | high fields are not truncated | source SHA256 matched | PASS | - -## Error and rollback matrix - -| Case | Expected | Actual | Status | -|---|---|---|---| -| split primary, no external options | `ENXIO` flatdev media bound | `Device not configured` | PASS | -| missing slot 2 | `ENXIO` | `external devices don't match ... Device not configured` | PASS | -| short slot 2 | `ENXIO` | `Device not configured` | PASS | -| swapped short/long providers | fail before root | `Device not configured` | PASS | -| same provider in two slots | `EINVAL` | `Invalid argument` | PASS | -| overlapping unified ranges | `EINTEGRITY` | `Integrity check failed` | PASS | -| explicit writable request | `EROFS` | `Read-only file system` | PASS | -| mapped device ID 3 with two slots | read-time `ENODEV` | `Operation not supported by device` | PASS | -| second concurrent reuse mount | `EBUSY` | `Device busy`; first mount remained readable | PASS | -| normal detach while mounted | `EBUSY` | `mdconfig ... Device busy` | PASS | -| forced orphan then cold slot-2 read | `ENXIO` | `Device not configured` | PASS | -| unmount after forced orphan | clean close of detached consumer | succeeded | PASS | - -Every mount failure was followed immediately by successful md detach. Opened -external devices were therefore rolled back, including failures after one or -more earlier slots had opened. - -## dmesg and cleanup - -The final matrix captured `dmesg` before and after all tests: - -```text -before_lines=2 -after_lines=2 -no-new-lines -``` - -The two pre-existing lines were unchanged historical messages: - -```text -interface erofs.1 already present in the KLD 'erofs-repo22-xattrfix.ko'! -md0: truncating fractional last sector by 14 bytes -``` - -Final state: - -```text -kldstat -n erofs: empty -mdconfig -l: empty -mount -p | grep erofs: empty -``` - -## Tool limitations - -- **MKFS-UNAVAILABLE**: erofs-utils 1.8.6 cannot directly emit more than one - external blob option in this environment. Multi-slot images were derived - from a legal mkfs image with deterministic, asserted structural patches. -- **MKFS-UNAVAILABLE**: erofs-utils 1.8.6 rejects the experimental 48-bit - incompat feature and does not correctly extract the patched device-ID-0 - unified-address control. Both were verified by complete FreeBSD kernel - SHA256 reads. -- **KERNEL-FAIL**: none. diff --git a/tests/results/manual/2026-08-08T2037Z-nfs/manual-test-report.md b/tests/results/manual/2026-08-08T2037Z-nfs/manual-test-report.md deleted file mode 100644 index 194ef71..0000000 --- a/tests/results/manual/2026-08-08T2037Z-nfs/manual-test-report.md +++ /dev/null @@ -1,559 +0,0 @@ -# repo22 FreeBSD 15 NFS export manual test report - -Date: 2026-08-08 20:08-20:37 UTC - -Baseline: `4ef680df6dcbfad329c0b2b1c0af8fcb21cbed4a` plus this NFS export batch - -Guest: FreeBSD 15.0-RELEASE-p8 amd64, -`releng/15.0-n281036-53054229dcb3`, QEMU TCG, 4 vCPUs, 6144 MB RAM - -Host tools: erofs-utils 1.8.6, clang cross-target build - -## Result - -**PASS** for the implementation and all reasonably executable NFS tests. - -FreeBSD mountd installed real EROFS exports through export-only `MNT_UPDATE`; -NFSv3 clients resolved 16-byte EROFS handles for regular files, directories, -symlinks, FIFOs, metabox bit-63 inodes, and multidevice files. Handle mutation, -same-md remount stability, nfsd restart, 12,050-entry READDIRPLUS pagination, -cold remounts, concurrent traversal/read/stat load, read-only enforcement, and -device-boundary errors all behaved as required. - -The only non-code issue was a cleanup-order mistake after the completed test -matrix: one hard loopback NFS mount remained while nfsd and the direct EROFS -mount were stopped. This blocked `mount -p` and later produced five client -`fileid changed` messages when nfsd was temporarily restarted without the -underlying export. The functional-test dmesg snapshot taken before that cleanup -mistake was byte-identical to its baseline. The VM was rebooted to clear the -blocked cleanup process, and a final correctly ordered NFS rerun completed with -byte-identical pre/post dmesg and zero remaining resources. - -## Repository and build - -Initial verification: - -```sh -git -C /work/repo-community/repo22 fetch xdm main -git -C /work/repo-community/repo22 rev-parse HEAD -git -C /work/repo-community/repo22 rev-parse xdm/main -git -C /work/repo-community/repo22 diff --quiet -git -C /work/repo-community/repo22 diff --cached --quiet -``` - -Both revisions were: - -```text -4ef680df6dcbfad329c0b2b1c0af8fcb21cbed4a -``` - -Tracked files were clean. Existing untracked `repo22/build`, -`tests/results/manual/2026-08-08T1800Z-namei/artifacts`, and files outside -repo22 were not deleted, modified, staged, or committed. - -To execute the required `./build.sh` without overwriting the pre-existing -untracked build products, the current source was copied to an isolated build -directory: - -```sh -BUILD_ROOT=$(mktemp -d /work/build/repo22-nfs-build.XXXXXX) -mkdir -p "$BUILD_ROOT/src" -cp repo-community/repo22/build.sh "$BUILD_ROOT/build.sh" -cp repo-community/repo22/src/* "$BUILD_ROOT/src/" -chmod +x "$BUILD_ROOT/build.sh" -cd "$BUILD_ROOT" -./build.sh -nm -u build/erofs.ko | \ - awk '$NF == "bcmp" { found = 1 } END { if (found) exit 1 }' -sha256sum build/erofs.ko -``` - -Result: - -```text -==> SUCCESS: /work/build/repo22-nfs-build.eFulVf/build/erofs.ko -48776485d1679f00100bc7b2be5f891c8f03a3e8b53ac7c00da8c2fd5f74cc70 erofs.ko -``` - -There was no unresolved `bcmp` reference. - -The FreeBSD helper compiled without warnings: - -```sh -cc -O2 -Wall -Wextra -std=c17 nfs_fh_tool.c -o nfs_fh_tool -``` - -## Fixtures - -The primary fixture was generated with: - -```sh -mkdir -p plain-src/basic/subdir plain-src/bigdir plain-src/concurrent -printf 'repo22-nfs-regular\nline-two\n' > plain-src/basic/regular.txt -printf 'nested-directory-file\n' > plain-src/basic/subdir/nested.txt -ln -s regular.txt plain-src/basic/link-to-regular -mkfifo plain-src/basic/test.fifo -for i in $(seq -w 0 12049); do - printf 'entry-%s\n' "$i" > plain-src/bigdir/file-$i -done -for i in $(seq -w 0 255); do - awk -v n="$i" 'BEGIN { - for (j = 0; j < 256; j++) - printf "worker-%s-line-%03d-abcdefghijklmnopqrstuvwxyz0123456789\\n", n, j - }' > plain-src/concurrent/file-$i.dat -done -awk 'BEGIN { - for (i = 0; i < 262144; i++) - printf "throughput-%08d-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\\n", i -}' > plain-src/throughput.dat -mkfs.erofs -T0 --all-root --ignore-mtime -x-1 -Uclear \ - plain-nfs.erofs plain-src -``` - -Fixture and manifest hashes: - -| Object | SHA256 | -|---|---| -| `plain-nfs.erofs` | `6928f05b58ce462596f9c7bf1d3442ad50d555dc5c31ed01d28e1a921a448bf2` | -| `metabox-nfs.erofs` | `8f493ed70d4be8096ab1adf8208622734d61a64903cf96a1ca1a1cc1854b1985` | -| multidevice primary | `fa2f35f59f63cc4a56d230ba960a6c1d27748506fccce081447cab0dc0d540fd` | -| multidevice blob 1 | `3aeb19c519636b61dc3ebd47c6286098deaa0990ff618ce3facbff4fd65907cd` | -| source manifest | `0eda6f88c3b6c4b7c8d81d940a237d1f63c958a7f6fe7bdd4f77d6b293736fa9` | -| `basic/regular.txt` | `93e7f327676f582356e9de2fcca7b714910227008fa1c5e7820e942c471edb6f` | -| `basic/subdir/nested.txt` | `6da159d7ea19e9d1253ccb0f2c67586e6e075459954e740ff8405546683d87c3` | -| `concurrent/file-000.dat` | `8435866622642b6dadf502b09f95af32cb01907ba3376e4f1fe96278e5ada692` | -| `throughput.dat` (21,757,952 bytes) | `0841effed82d1adf394b6834ce30d4d9eb5fc9427527e854ec1cb6bb4b86c119` | -| expected 12,050-name sorted list | `4fbf1b6103e8884223629c3fd03f4ec36061e65cd2a1becfc96670daa96f9d71` | - -The metabox image was the previously qualified compressed-metabox fixture. The -multidevice fixture was the previously qualified primary-plus-one-blob image. - -## FreeBSD 15 setup commands - -The artifacts were copied to the existing VM through SSH port 9222. Password -material and askpass script contents were neither printed nor recorded. - -```sh -scp -P 9222 erofs.ko plain-nfs.erofs metabox-nfs.erofs \ - tests/nfs_fh_tool.c root@127.0.0.1:/tmp/repo22-nfs-test/ -ssh -p 9222 root@127.0.0.1 -cd /tmp/repo22-nfs-test -sha256 erofs.ko plain-nfs.erofs metabox-nfs.erofs -cc -O2 -Wall -Wextra -std=c17 nfs_fh_tool.c -o nfs_fh_tool -``` - -The guest hashes matched the host hashes. - -EROFS and NFS setup: - -```sh -kldload /tmp/repo22-nfs-test/erofs.ko -mdconfig -a -t vnode -f /tmp/repo22-nfs-test/plain-nfs.erofs -u 42 -mkdir -p /mnt/repo22-erofs -mount -t erofs -o ro /dev/md42 /mnt/repo22-erofs -mount -v | grep /mnt/repo22-erofs - -printf '%s\n' \ - '/mnt/repo22-erofs -ro -maproot=root -network 127.0.0.0 -mask 255.0.0.0' \ - > /etc/exports -service rpcbind onestart -service mountd onestart -service nfsd onestart -service mountd onereload -rpcinfo -p 127.0.0.1 -showmount -e 127.0.0.1 -mount -v | grep /mnt/repo22-erofs -``` - -Before mountd reload the EROFS line did not contain `NFS exported`. Afterwards: - -```text -/dev/md42 on /mnt/repo22-erofs (erofs, NFS exported, local, read-only, acls, ...) -``` - -NFSv2/v3 and mountd were registered over TCP and UDP. NFS components are built -into this GENERIC kernel: `kldload nfsd` and `kldload nfscl` reported -`already loaded or in kernel`, while rpcbind/mountd/nfsd operated normally. - -## File-handle validation commands - -Normal vnode types: - -```sh -./nfs_fh_tool capture /mnt/repo22-erofs/basic/regular.txt regular.before.fh -./nfs_fh_tool capture /mnt/repo22-erofs/basic/subdir directory.fh -./nfs_fh_tool lcapture /mnt/repo22-erofs/basic/link-to-regular symlink.fh -./nfs_fh_tool capture /mnt/repo22-erofs/basic/test.fifo fifo.fh -./nfs_fh_tool describe regular.before.fh -./nfs_fh_tool describe directory.fh -./nfs_fh_tool describe symlink.fh -./nfs_fh_tool describe fifo.fh -./nfs_fh_tool stat regular.before.fh -./nfs_fh_tool stat directory.fh -./nfs_fh_tool stat symlink.fh -./nfs_fh_tool stat fifo.fh -./nfs_fh_tool cat regular.before.fh regular.fhopen.out -``` - -Observed handles: - -| Type | NID | Generation | Result | -|---|---:|---:|---| -| regular | `0x74` | 1 | `fhstat` and `fhopen` PASS | -| directory | `0x76` | 1 | `fhstat` PASS | -| symlink (`lgetfh`) | `0x72` | 1 | `fhstat` PASS | -| FIFO | `0x79` | 1 | `fhstat` PASS | - -All had `len=16` and `pad=0`. The regular full-handle SHA256 was: - -```text -fb1778fc6bdc979e10d003dd6131a961e7532baa16273c5465935a665bf990f6 -``` - -Malformed and stale classification: - -```sh -./nfs_fh_tool mutate regular.before.fh bad-len.fh len 15 -./nfs_fh_tool mutate regular.before.fh bad-pad.fh pad 1 -./nfs_fh_tool mutate regular.before.fh bad-gen.fh gen 2 -./nfs_fh_tool mutate regular.before.fh bad-nid.fh nid_hi 0xffffffff -./nfs_fh_tool expect-stat bad-len.fh EINVAL -./nfs_fh_tool expect-open bad-len.fh EINVAL -./nfs_fh_tool expect-stat bad-pad.fh EINVAL -./nfs_fh_tool expect-open bad-pad.fh EINVAL -./nfs_fh_tool expect-stat bad-gen.fh ESTALE -./nfs_fh_tool expect-open bad-gen.fh ESTALE -./nfs_fh_tool expect-stat bad-nid.fh ESTALE -./nfs_fh_tool expect-open bad-nid.fh ESTALE -``` - -All eight checks passed. `fhstat` exercised the shared-lock `VFS_FHTOVP` path; -`fhopen` exercised the exclusive-lock path. - -Same explicit md-unit remount: - -```sh -umount /mnt/repo22-erofs -mdconfig -d -u 42 -mdconfig -a -t vnode -f plain-nfs.erofs -u 42 -mount -t erofs -o ro /dev/md42 /mnt/repo22-erofs -./nfs_fh_tool capture /mnt/repo22-erofs/basic/regular.txt regular.after.fh -./nfs_fh_tool compare regular.before.fh regular.after.fh -./nfs_fh_tool cat regular.before.fh regular.remount.out -``` - -The complete handles were byte-identical and the pre-remount handle still read -the correct file after remount. - -Non-export mount update rejection: - -```sh -mount -u -o noexec /mnt/repo22-erofs -``` - -Result: `Operation not supported`; the mount remained exported, local, -read-only, and did not gain `noexec`. - -## NFSv3 basic and restart tests - -Four NFSv3 TCP READDIRPLUS clients were mounted: - -```sh -mount_nfs -o nfsv3,tcp,rdirplus,readdirsize=512 \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-512 -mount_nfs -o nfsv3,tcp,rdirplus,readdirsize=1024 \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-1024 -mount_nfs -o nfsv3,tcp,rdirplus,readdirsize=4096 \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-4096 -mount_nfs -o nfsv3,tcp,rdirplus \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-default -nfsstat -m -``` - -FreeBSD clamped the requested 512/1024/4096 values to an effective -`readdirsize=8192`; the default client used 65536. This was recorded as a client -environment limit, not a failure. - -For each client: - -```sh -cmp NFS/basic/regular.txt DIRECT/basic/regular.txt -test -d NFS/basic/subdir -test "$(readlink NFS/basic/link-to-regular)" = regular.txt -test -p NFS/basic/test.fifo -test "$(stat -f %i NFS/basic/regular.txt)" = \ - "$(stat -f %i DIRECT/basic/regular.txt)" -! touch NFS/write-must-fail -``` - -All passed. Writes failed with `Read-only file system`. - -nfsd restart with an open descriptor: - -```sh -exec 3< /mnt/repo22-nfs-default/basic/regular.txt -service nfsd onerestart -cat <&3 > open-fd-after-nfsd-restart.out -exec 3<&- -cmp open-fd-after-nfsd-restart.out /mnt/repo22-erofs/basic/regular.txt -``` - -The open descriptor and subsequent path reopen both passed. Direct file handles -captured before and after restart were byte-identical. - -## Metabox bit-63 test - -Commands: - -```sh -mdconfig -a -t vnode -f metabox-nfs.erofs -u 43 -mount -t erofs -o ro /dev/md43 /mnt/repo22-metabox -./nfs_fh_tool capture /mnt/repo22-metabox/long/file1.txt metabox.before.fh -./nfs_fh_tool describe metabox.before.fh -./nfs_fh_tool stat metabox.before.fh -./nfs_fh_tool cat metabox.before.fh metabox.fhopen.out -./nfs_fh_tool mutate metabox.before.fh metabox.bad-nid.fh \ - nid_lo 0xffffffff -./nfs_fh_tool expect-stat metabox.bad-nid.fh ESTALE -./nfs_fh_tool expect-open metabox.bad-nid.fh ESTALE -``` - -Observed handle: - -```text -len=16 pad=0 nid=800000000000010a gen=1 -ino=9223372036854776074 -``` - -The metabox mount was exported with mountd and mounted over NFSv3. Direct, -`fhopen`, and NFS contents had SHA256: - -```text -661b22d2a7bd94a7da35834d4cc3647eb527d48eb08acc75bde2e3a7691ed45e -``` - -The NFS and direct inode numbers matched. The out-of-range metabox mutation -returned `ESTALE` through both shared and exclusive paths. - -## Multidevice boundary test - -Normal NFS commands: - -```sh -mdconfig -a -t vnode -f primary.img -u 44 -mdconfig -a -t vnode -f blob1.img -u 45 -mount -t erofs -o ro -o device.1=/dev/md45 /dev/md44 \ - /mnt/repo22-multidev -mount_nfs -o nfsv3,tcp,rdirplus,readdirsize=512 \ - 127.0.0.1:/mnt/repo22-multidev /mnt/repo22-nfs-multidev -cmp /mnt/repo22-nfs-multidev/alpha.bin /mnt/repo22-multidev/alpha.bin -cmp /mnt/repo22-nfs-multidev/small.txt /mnt/repo22-multidev/small.txt -``` - -Normal multidevice NFS reads passed. Representative hashes: - -```text -f779dbe3aeaa609beb32212579f3aea441f244621d13c977cb8b52d7ec3016d9 alpha.bin -5990b055a3c6d27681185a25e15ff21ba3cbdca7632faf4332233238e5f45aad small.txt -001bcf4626e7d52f1bd76dee9a6c6e7d3a718c03602c5f9b07840a6c98b8fdc3 tree/beta.bin -``` - -Cold detached-provider test: - -```sh -umount /mnt/repo22-multidev -mdconfig -d -u 44 -mdconfig -a -t vnode -f primary.img -u 44 -mdconfig -a -t vnode -f blob1.img -u 45 -mount -t erofs -o ro -o device.1=/dev/md45 /dev/md44 \ - /mnt/repo22-multidev -./nfs_fh_tool capture /mnt/repo22-multidev/tree/beta.bin multidev-beta.fh -./nfs_fh_tool stat multidev-beta.fh -mdconfig -d -o force -u 45 -! ./nfs_fh_tool cat multidev-beta.fh beta.after-orphan -``` - -The handle resolved to NID `0x36`, then the cold read returned: - -```text -Device not configured -``` - -This preserved the multidevice `ENXIO` boundary error instead of converting it -to `ESTALE`, reading past the provider, or panicking. - -## READDIRPLUS and stress - -Every one of the four initial listings returned exactly 12,050 unique expected -names. The expected list and all four sorted outputs had SHA256: - -```text -4fbf1b6103e8884223629c3fd03f4ec36061e65cd2a1becfc96670daa96f9d71 -``` - -Dot handling: - -```sh -ls -a1 /mnt/repo22-nfs-default/bigdir > nfs-dot-list -test "$(wc -l < nfs-dot-list)" -eq 12052 -test "$(grep -cx '\.' nfs-dot-list)" -eq 1 -test "$(grep -cx '\.\.' nfs-dot-list)" -eq 1 -``` - -All passed. - -Cold pagination used five unmount/remount/list/compare cycles on the small -client. Concurrency then used eight workers, three complete 12,050-entry rounds -each, distributed across the four mounts. All 29 cold/concurrent sorted outputs -had one unique SHA256 value, matching the expected list. - -Parallel data/metadata load: - -```sh -for M in 512 1024 4096 default; do - find /mnt/repo22-nfs-$M/concurrent -type f -maxdepth 1 -print0 | \ - xargs -0 -n 1 -P 8 cat > /dev/null & - find /mnt/repo22-nfs-$M/concurrent -type f -maxdepth 1 -print0 | \ - xargs -0 -n 1 -P 8 stat -f '%i %z' > /dev/null & -done -wait -``` - -All jobs completed successfully. - -Final stress counters: - -| Counter | Client | Server | -|---|---:|---:| -| requests / cache misses | 270,854 | 271,174 | -| lookup | 264,989 | 265,109 | -| read | 1,360 | 1,360 | -| READDIRPLUS | 2,512 | 2,512 | -| write | 0 | 0 | -| timed out | 0 | n/a | -| retries | 0 | n/a | - -There were no READDIR RPCs because the clients negotiated READDIRPLUS. - -Informational throughput: - -```text -21757952 bytes transferred in 5.843937 seconds -3723167 bytes/second -``` - -No fixed throughput threshold was used. - -## Test matrix - -| Test | Actual | Status | -|---|---|---| -| isolated `./build.sh` | success; module hash recorded | PASS | -| unresolved `bcmp` | none | PASS | -| FreeBSD helper compile | `-Wall -Wextra`, no warnings | PASS | -| final `kldload` / `kldunload` | load, unload, reload, unload succeeded | PASS | -| initial mount export flag | absent before mountd | PASS | -| export-only `MNT_UPDATE` | mountd installed export | PASS | -| non-export `MNT_UPDATE` | `EOPNOTSUPP`, flags unchanged | PASS | -| regular file | direct, `fhopen`, NFS content match | PASS | -| directory | handle and NFS traversal | PASS | -| symlink | `lgetfh`, target and NFS lookup | PASS | -| FIFO | handle metadata and NFS type | PASS | -| read-only behavior | direct and all NFS mounts rejected writes | PASS | -| 16-byte handle ABI | len/pad/NID hi/NID lo/gen observed | PASS | -| generation | `1`, matching `va_gen` | PASS | -| shared/exclusive lock contract | `fhstat` and `fhopen` passed | PASS | -| malformed len/pad | `EINVAL` | PASS | -| stale generation/NID | `ESTALE` | PASS | -| same `md42` remount | complete handle byte-identical | PASS | -| nfsd restart | open descriptor and reopen passed | PASS | -| metabox bit 63 | real `0x800000000000010a` round-trip | PASS | -| metabox boundary mutation | `ESTALE` | PASS | -| multidevice NFS | external-data reads matched | PASS | -| detached external device | cold handle read preserved `ENXIO` | PASS | -| 12,050-entry listings | exact count and names on four clients | PASS | -| cookie pagination | five cold remounts, no duplicate/omission | PASS | -| dot omitted | one `.` and one `..`, 12,052 total | PASS | -| concurrent traversal | 8 workers x 3 rounds, all hashes equal | PASS | -| concurrent read/stat | all jobs completed | PASS | -| READDIRPLUS | client/server 2,512 | PASS | -| NFS timeouts/retries | 0 / 0 | PASS | -| successful NFS writes | 0 | PASS | -| throughput | 3,723,167 B/s, informational | INFO | - -## dmesg and cleanup - -The functional matrix captured dmesg before loading the test module and after -all functional tests plus a load/unload cycle. Both files contained the same -two pre-existing historical lines and compared byte-for-byte equal: - -```text -interface erofs.1 already present in the KLD 'erofs-repo22-xattrfix.ko'! -md0: truncating fractional last sector by 14 bytes -``` - -No EROFS error, stale-handle message, trap, panic, or new kernel line occurred -during the functional matrix. - -The first cleanup attempt used the wrong order: it stopped nfsd and unmounted -the direct EROFS mounts before one hard loopback client was fully gone. This -left a `mount -p` process in uninterruptible NFS wait. When rpcbind/nfsd were -temporarily restarted without the underlying EROFS export to release those -clients, the NFS client logged five `fileid changed` messages. These messages -occurred after the byte-identical functional dmesg snapshot and are classified -as **environment cleanup artifact**, not an EROFS functional failure. - -The VM was rebooted to remove the blocked cleanup process. Post-reboot audit: - -```text -erofs_mounts=0 -nfs_mounts=0 -md_units=0 -erofs_modules=0 -nfsd_pids=0 mountd_pids=0 rpcbind_pids=0 -exports_exists=no -``` - -A final correctly ordered clean rerun then performed: - -```sh -kldload erofs.ko -mdconfig -a -t vnode -f plain-nfs.erofs -u 42 -mount -t erofs -o ro /dev/md42 /mnt/repo22-erofs -service rpcbind onestart -service mountd onestart -service nfsd onestart -mount_nfs -o nfsv3,tcp,rdirplus,readdirsize=512 \ - 127.0.0.1:/mnt/repo22-erofs /mnt/repo22-nfs-clean -cmp /mnt/repo22-nfs-clean/basic/regular.txt \ - /mnt/repo22-erofs/basic/regular.txt -test "$(find /mnt/repo22-nfs-clean/bigdir -type f -maxdepth 1 | wc -l)" \ - -eq 12050 -umount /mnt/repo22-nfs-clean -: > /etc/exports -service mountd onereload -service nfsd onestop -service mountd onestop -service rpcbind onestop -umount /mnt/repo22-erofs -mdconfig -d -u 42 -kldunload erofs -``` - -The clean rerun's pre/post dmesg files compared byte-for-byte equal. Final -state after that rerun: - -```text -clean_rerun_erofs=0 nfs=0 md=0 module=0 services=0 exports=absent -``` - -## Limitations - -- **ENVIRONMENT-LIMIT**: FreeBSD clamped requested readdir sizes 512, 1024, and - 4096 to an effective minimum of 8192. The test still exercised a much smaller - value than the 65536 default and produced 2,512 real READDIRPLUS calls. -- **ENVIRONMENT-CLEANUP-ARTIFACT**: the five post-test `fileid changed` lines - were caused by the explicitly documented wrong cleanup order. A reboot and a - correctly ordered rerun left no new dmesg lines or resources. -- **KERNEL-FAIL**: none. -- **NOT RUN**: none of the required functional cases were omitted. diff --git a/tests/results/manual/2026-08-08T2114Z-multidev-review/manual-test-report.md b/tests/results/manual/2026-08-08T2114Z-multidev-review/manual-test-report.md deleted file mode 100644 index b99b168..0000000 --- a/tests/results/manual/2026-08-08T2114Z-multidev-review/manual-test-report.md +++ /dev/null @@ -1,142 +0,0 @@ -# repo22 multi-device review findings manual test report - -Date: 2026-08-08 21:14 UTC -Baseline: `2354b4487c84458c1e447f4d0bb7b55daebcca26` plus this review fix -Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG -Linux reference: 7.1-rc1 `/work/dev-src-linux/fs/erofs` -Host tools: erofs-utils 1.8.6 - -## Result - -**PASS**. All five multi-device review findings were fixed and exercised with -deterministic positive and negative fixtures. A real two-block LZ4 pcluster -was read from an external provider and the complete 1 MiB output matched the -source SHA256. Multi-device NFS reads and forced-device-removal behavior also -passed. - -## Source changes validated - -- Non-zero device IDs accept a slot with `uniaddr=0`; unified device-ID-0 - lookup skips such slots. -- `devt_slotoff=0` reads the table from byte offset zero, as Linux 7.1 does. -- Device mapping carries the requested extent length. Flatdev reads must fit - wholly in the primary range or one non-zero unified slot; gaps, adjacent-slot - crossings, and pcluster crossings fail closed. -- Declared image and slot bounds are checked before physical media size, so - format corruption is `EINTEGRITY` and a genuinely short provider is - `ENXIO`. -- Primary and external `namei()` failures preserve `ENOENT`; omitted required - `device.N` options still return `ENXIO`. - -The NFS file-handle ABI, `VFS_FHTOVP`, inode bounds, readdir cookies, and GEOM -orphan lifecycle from `2354b448` were not changed. - -## Build - -Commands: - -```sh -git diff --check -./build.sh -nm -u build/erofs.ko | awk '$NF == "bcmp" {bad=1} END {exit bad}' -sha256sum build/erofs.ko -``` - -Results: - -- Cross-build: PASS. -- `git diff --check`: PASS. -- Unresolved `bcmp`: none. -- Module SHA256: - `b85c429fa73e6cebc10e6e3941575a6ca170306ab4fa9ccca4006c922971acb8`. -- FreeBSD `kldload` and `kldunload`: PASS. - -## Fixture construction - -The chunk fixtures derive from the checksum-valid TC094 images recorded in the -19:37 multi-device report. The patcher asserted the original table values -`slot1=(blocks=19, uniaddr=2)`, `slot2=(blocks=25, uniaddr=21)`, and the first -32 KiB chunk index `(device_id=2, startblk=9)` before each change. - -The compressed baseline was generated from the deterministic `shape.dat`: - -```sh -mkfs.erofs -T0 -U00000000-0000-0000-0000-000000000000 \ - --all-root -E legacy-compress -zlz4 -C65536 \ - lz4-full-64k.erofs src-big64k -dump.erofs --path=/shape.dat -e lz4-full-64k.erofs -``` - -The baseline contains one 1 MiB logical extent backed by an 8192-byte physical -pcluster. Its HEAD pblk was changed from block 1 to unified block 4; the two -compressed blocks were copied to a two-block external provider and zeroed in -the primary image. A checksum-valid device table declared -`slot1=(blocks=2, uniaddr=4)`. - -For the crossing negative, the same pcluster was placed across adjacent -`slot1=[4,5)` and `slot2=[5,6)`. The flatdev contains both real compressed -blocks, proving that failure is caused by the declared slot boundary rather -than absent data. - -erofs-utils 1.8.6 parses the external compressed extent but its userspace -compressed read path does not propagate the unified slot ID to `erofs_dev_read`; -therefore userspace extraction of this layout is tool-unavailable. The -FreeBSD kernel full-file SHA256 is the authoritative read validation. - -## Fixture hashes - -| Fixture | SHA256 | -|---|---| -| zero-uniaddr primary | `d38935291fb2ee756e0c192d1b0d39b4f4f5b4047aa37c2de44cad91a55fa018` | -| zero-uniaddr device-ID-0 control | `fac9db7631cd06d17212cbe3e7b1d681766f031d1e41e443f71f1bad39f9b960` | -| slotoff-zero primary | `39e3e8c60bea962f1200550316be87ec7da7b15c0ba11c6e5edf05f0715db451` | -| cross-slot primary | `edd0cba92a26bc9a7a59561732fb8e045b2e1565d18d9f9b3143f86f64649dee` | -| cross-slot flatdev | `6e2834450081d778cee37fa3e050249f04927def205d6816aab814a2912744bc` | -| gap flatdev | `5bc07b66f7b0f83e9568ba781dcc78a08d45e617dc27bb365ed3ee2c12353218` | -| external-compressed primary | `5625612426d68624c77dd94b09754da583fc6bd90767e9e9d958e4fc89131a4d` | -| external-compressed blob | `0d71102ce471af0a30d4d61a1fbeef1fc33437c85651d7c7f9c60ebd305ec9d0` | -| cross-pcluster primary | `2bce4a91c1aedc4bab2e65582ef9cb465094f825a051c0203d52b3d18bb17ed8` | -| cross-pcluster flatdev | `bc07fb06ca5c6bb60ef1a989eecbd3d714450092ba756f85f5b060c8b8b2ec3a` | - -Source `shape.dat` SHA256: -`370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52`. - -## FreeBSD result matrix - -| Test | Actual | Status | -|---|---|---| -| slot 1 `uniaddr=0`, explicit device ID 1 | complete striped and second-file SHA256 matched | PASS | -| slot 1 `uniaddr=0`, device ID 0 control | slot skipped; primary-bound `EINTEGRITY` | PASS | -| `devt_slotoff=0` | mount and complete striped SHA256 matched | PASS | -| nonexistent primary path | `ENOENT` / `No such file or directory` | PASS | -| nonexistent external path | `ENOENT` / `No such file or directory` | PASS | -| omitted slot option | `ENXIO` / `Device not configured` | PASS | -| flatdev chunk crosses adjacent slots | read returned `EINTEGRITY` | PASS | -| device-ID-0 chunk starts in a gap | read returned `EINTEGRITY` | PASS | -| explicit chunk exceeds declared slot and media | `EINTEGRITY`, not `ENXIO` | PASS | -| real external 64 KiB LZ4 pcluster | complete 1 MiB SHA256 matched | PASS | -| flatdev pcluster crosses adjacent slots | read returned `EINTEGRITY` | PASS | -| explicit pcluster crosses two providers | read returned `EINTEGRITY` | PASS | - -## NFS smoke - -The external-compressed image was exported read-only over local NFSv3/TCP. -The NFS client read the complete `shape.dat`; SHA256 matched the source. - -The zero-uniaddr two-slot image was then exported. The client obtained stable -inode and size data for a cold slot-2 file, slot 2 was forcibly orphaned, and -the same NFS pathname still resolved to identical metadata. Its first cold data -read failed with the underlying device error. This preserves the `2354b448` -contract: file-handle resolution is not misreported as `ESTALE`, while missing -external data remains an I/O/device failure. - -## Cleanup - -All EROFS and NFS mounts were unmounted, NFS services stopped, `/etc/exports` -cleared, md providers detached, and the module unloaded. No image, overlay, -fixture, or build artifact is part of this commit. - -A final representative rerun covered the external-compressed positive, the -flatdev-gap negative, and the offset-zero table positive. The dmesg SHA256 was -unchanged before and after: -`897003051a6a50875d49715bfa27907611cedac9e42e0a06ca7377105ac695ae`. diff --git a/tests/results/manual/2026-08-08T2306Z-compression-p0/manual-test-report.md b/tests/results/manual/2026-08-08T2306Z-compression-p0/manual-test-report.md deleted file mode 100644 index ac478a6..0000000 --- a/tests/results/manual/2026-08-08T2306Z-compression-p0/manual-test-report.md +++ /dev/null @@ -1,406 +0,0 @@ -# repo22 compression P0 WIP integration manual report - -Date: 2026-08-08 23:06 UTC -Baseline: `78182686e968c659932458cbe7a1e0889397f20b` -Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG -Build reference: `/work/dev-freebsd-releng` (`REVISION=15.0`, source branch -`RELEASE-p9`) -Linux behavior reference: `/work/dev-src-linux/fs/erofs` -Host production tool: erofs-utils 1.8.6 - -## Result - -**PASS**, with the extent-record guest case accurately recorded as -**MKFS-UNAVAILABLE** rather than PASS. - -The retained ten-file WIP was reviewed and integrated without reset, checkout, -stash, or revert. Both module configurations build and load. DEFLATE, ZSTD, -and MicroLZMA full and partial references read correctly; targeted corruption -returns `EIO`. A fragment-backed compressed metabox now works after packed -inode initialization, while loop, recursive-NID, and range mutations fail -closed. HEAD2 and interlaced images pass real FreeBSD kernel reads. LZ4, -xattr/metabox, chunk, external compressed multi-device, and NFS regressions -also pass. - -No CI or test runner was added. Only source, Markdown manual tests, and this -report are intended for the commit. - -## Instruction and baseline audit - -- `find /work -name AGENTS.md -type f -print` returned no paths. There were no - applicable `AGENTS.md` files. -- Initial local `HEAD` and `xdm/main` both resolved to - `78182686e968c659932458cbe7a1e0889397f20b`. -- The exact ten tracked WIP files were present: - `build.sh`, `src/Makefile`, `src/decompressor.c`, `src/deflate.c`, - `src/internal.h`, `src/lzma.c`, `src/super.c`, `src/zdata.c`, `src/zmap.c`, - and `src/zstd.c`. -- Untracked build, fixture, overlay, artifact, and other-repository paths were - not staged. - -## Initial guest module cleanup - -Before any build under test was loaded, the guest reported: - -```text -Id Refs Address Size Name - 5 1 0xffffffff82822000 a690 erofs-nozstd.ko -``` - -`kldstat -v -i 5` proved the path was `./erofs-nozstd.ko` and the contained -module name was `erofs`. The cleanup used the observed KLD ID, not a guessed -filename: - -```sh -kldunload -i 5 -kldstat -kldstat | grep -i erofs -``` - -The second `kldstat` contained no EROFS entry and the final grep printed -`none`. - -## Source review - -### Packed inode and metabox order - -- `packed_nid` and `metabox_nid` are decoded and checked before carrier loads. -- A packed NID with the metabox selector bit is rejected before any metadata - recursion can begin. -- The packed inode is loaded before the metabox inode, matching the Linux - dependency order and allowing a compressed metabox to terminate in a packed - fragment. -- The packed inode must be a regular, non-fragment inode. This rejects a packed - carrier that would recurse back through itself. -- A fragment-backed metabox must have a distinct loaded packed inode, non-zero - size, a real fragment tail mapping, and a range wholly inside the packed - inode. -- Fragment recursion checks compare inode NIDs rather than object pointers, - which also catches separately allocated `erofs_node` objects for the same - on-disk inode. - -### DEFLATE, ZSTD, and MicroLZMA - -- DEFLATE uses `inflate(..., Z_SYNC_FLUSH)` until the requested output is full, - detects no-progress loops, accepts `Z_OK` for partial output, and requires - `Z_STREAM_END` plus complete input consumption for full output. -- ZSTD uses FreeBSD's formal `` API with - `ZSTD_createDCtx_advanced`, `ZSTD_DCtx_setParameter`, - `ZSTD_decompressStream`, `ZSTD_isError`, and `ZSTD_freeDCtx`. -- ZSTD partial decoding stops after the requested output is produced. Full - decoding requires frame completion and complete input consumption. -- MicroLZMA partial decoding accepts `XZ_OK` or `XZ_STREAM_END` after exact - requested output. Full decoding requires `XZ_STREAM_END` and - `buffer.in_pos == srclen`. -- All decoder failures are translated to the filesystem read error `EIO` and - all allocated decoder/output buffers are released. - -### ZSTDIO build gate - -- `src/Makefile` consumes `opt_zstdio.h` and adds FreeBSD's zstd compatibility - include directory for `zstd.c`. -- `build.sh` accepts only `EROFS_ZSTDIO=0` or `1` and creates the corresponding - option header. -- The disabled translation unit contains only the availability result and a - local stub; it references no `ZSTD_*` symbol. -- A filesystem advertising ZSTD is rejected during compression-config parsing - when the module lacks ZSTDIO, before any file read can reach the stub. - -### HEAD2, interlaced, and extent records - -- HEAD2 selects `z_algorithmtype[1]` and respects the HEAD2 big-pcluster bit. -- Plain records with interlaced advise use the interlaced byte rotation before - returning data. -- Extent records retain Linux ordering and responsibilities for 4-, 8-, 16-, - and 32-byte records, partial references, explicit algorithm format, shifted - and interlaced plain data, and final fragments. -- Explicit extent tables with a non-empty file and zero extent count fail - closed. -- Final fragment mappings are checked against the loaded packed inode before - reads. - -## Build matrix - -Host commands: - -```sh -EROFS_ZSTDIO=0 ./build.sh -cp build/erofs.ko /tmp/repo22-erofs-nozstd-probe.ko -nm -u /tmp/repo22-erofs-nozstd-probe.ko - -EROFS_ZSTDIO=1 ./build.sh -cp build/erofs.ko /tmp/repo22-erofs-zstd-probe.ko -nm -u /tmp/repo22-erofs-zstd-probe.ko -``` - -| Build | Size | SHA256 | `ZSTD_*` undefined | `bcmp` undefined | Guest KLD | -|---|---:|---|---|---|---| -| no ZSTDIO | 68744 | `c96f4e1620f005a19d7ed581f98f1894969b38a34d7021f54dbce99919888ef9` | none | none | load/unload PASS | -| ZSTDIO | 69984 | `7f8cb03afb9c76b535709af1a06bf993cbab74bb5899b2f4646a6fd20b00d15c` | five formal API names | none | load/unload PASS | - -The enabled module's unresolved ZSTD set was exactly: - -```text -ZSTD_DCtx_setParameter -ZSTD_createDCtx_advanced -ZSTD_decompressStream -ZSTD_freeDCtx -ZSTD_isError -``` - -Both were loaded by path, identified with `kldstat -v -i ID`, and unloaded by -that exact ID. - -## ZSTDIO behavior - -With the disabled module, a normal LZ4 image still read successfully: - -```text -nozstd_lz4_sha256=370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52 -nozstd_lz4_dmesg_delta=empty -``` - -The same module rejected the ZSTD image: - -```text -nozstd_mount_rc=1 -mount: /dev/md0: erofs: ZSTD compression requires ZSTDIO support: Operation not supported -``` - -The ZSTDIO module mounted the image and produced the expected complete SHA256 -shown below. - -## Fixture construction and proof - -Fixtures and build outputs remained outside the repository commit. -Authentication used the existing private askpass workflow; its contents and -credentials were not printed or recorded. - -### Partial-reference images - -`dump.erofs -s` and `dump.erofs --path=PATH -e` established compressor, -layout, logical lengths, physical starts, and pcluster lengths. Byte-level -patching asserted the original inode NID, map-header offset, HEAD record, and -pblk before setting `Z_EROFS_LI_PARTIAL_REF` or redirecting a reused pblk. -CRC32C was recomputed for every superblock mutation. - -| Image | SHA256 | -|---|---| -| `deflate-partial-ref.erofs` | `8bc720a1250ef28794d091b6264e76060fbf01d66ea706c59c24c1960fa347ac` | -| `deflate-partial-ref-corrupt.erofs` | `6ff9dff9b3b5aba7da9b4a93f5b068270b4916267d2fe0fd3702c02be4c19fae` | -| `lzma-partial-ref.erofs` | `2f4bd89d2340273dd4052ea73aa9ac43802db431258c118c5dfede7727be9408` | -| `lzma-partial-ref-corrupt.erofs` | `bfbbd304a401c091d63e8b8e42b2b1984d9750760e940aded89acc909551d838` | -| `zstd-partial-ref.erofs` | `7f5ee4f8a20f20d32eaf8780a7081229f7dbf6b896ad23986da36e1a1c580519` | -| `zstd-partial-ref-corrupt.erofs` | `0c6e6b312b879297e2f71f406dcd35236654800e7cd115df0cb9a748297c48c1` | - -The LZMA and ZSTD full file was 1048576 bytes. Their partial file was 700000 -bytes and reused the complete source pcluster. The DEFLATE fixture contained -multiple real compressed extents, including reused physical extents in the -partial file. - -### Fragment-backed compressed metabox - -A METABOX-capable inspector proved: - -```text -packed_nid=40 -metabox_nid=38 -metabox inode: regular, compressed-full, size=20480 -metabox fragment header=0x800000000004e800 -fragmentoff=321536 -packed inode: regular, plain, size=342016 -321536 + 20480 = 342016 -``` - -Positive image SHA256: -`8a9a62bd203994711b8272192915d811e6c3de23e07ad9607dd63e66cc109bcd`. - -The negative images changed one proven field each: - -| Negative | Exact mutation | SHA256 | -|---|---|---| -| self-loop | `packed_nid: 40 -> 38` | `e9501ed9d149e2d735d95156669d144e733bf4be620bb69ea8a0c41f996b436c` | -| range | `fragmentoff: 321536 -> 342016` | `3fdcf2a41f50da93a5931edcef0d86ff2f576ecba781833036eaa930d99197d5` | -| metabox recursion | set bit 63 in `metabox_nid` | `b873e0cf4d2892d2d154a5769f494660f47f4429161e7be001b833da6dd0d705` | -| packed recursion | set bit 63 in `packed_nid` | `4fa368d1b1d47bb56b82132e6055d105ed2508b179a8df2a98ec5a728f91c9ac` | - -### HEAD2 - -The HEAD2 fixture patch asserted the original bytes and made only these semantic -changes, plus the resulting CRC32C bytes: - -```text -feature_incompat: 0x00000003 -> 0x0000000b -map h_advise: 0x0002 -> 0x0006 -first di_advise: HEAD1 (1) -> HEAD2 (3) -``` - -Image SHA256: -`fc70cbef0442a86ac2f507aebd7ac7bcfbdfcc3d0b9b3cf2ff5231334583d816`. -The targeted corrupted copy SHA256 was -`f0d9ad645804565c5aca7a92128df3ae9c81717a08ad7e8692489cd2ae606700`. - -### Interlaced - -This image was generated specifically with installed erofs-utils 1.8.6: - -```sh -mkfs.erofs -zlz4 -C4096 -Efragments -T0 \ - interlaced-1.8.6.erofs source -``` - -`dump.erofs -e` reported 105 real extents, with 4096-byte plain extents -interspersed with compressed extents. Image SHA256: -`d77d86f874361bae86cae9e8f6d05d9b6c68ef04cd5c8779368aef033fa485c3`. - -### Extent metadata - -Installed `mkfs.erofs -V` reported 1.8.6. Exact source-tree search found no -on-disk `Z_EROFS_ADVISE_EXTENTS`, `z_erofs_extent_recsize`, or -`struct z_erofs_extent {` definition in the 1.8.6 include/lib tree. Its -internal `struct z_erofs_extent_item` is an in-memory compressor item, not the -new on-disk extent-record ABI. - -Although a newer-tool extent image existed in the WIP build area, it was not -mounted or scored. The required result is therefore: - -```text -extent metadata guest result: MKFS-UNAVAILABLE -``` - -The independent static review compared `src/erofs_fs.h` and `src/zmap.c` -against `/work/dev-src-linux/fs/erofs/erofs_fs.h` and `zmap.c`, covering record -sizes 4/8/16/32, implicit physical bases, explicit-count binary search, -physical/logical high words, format bits, partial references, interlaced data, -fragments, metabox metadata reads, and malformed explicit zero counts. - -## Core FreeBSD result matrix - -The guest test used repeated `mdconfig -a -t vnode -f IMAGE`, read-only EROFS -mounts, `sha256 -q`, byte-exact `dd`/`cmp`, and a small C helper that printed -`errno` on read failure. - -| Case | Complete SHA256 / result | Boundary or random proof | Status | -|---|---|---|---| -| DEFLATE full | `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` | full stream completion | PASS | -| DEFLATE partial | `61b17076c2dfae88da7912d00df534b894cf6d27178863c6d8e91f8617ebb91e` | offset 122900, 512 bytes: `fa381301af1b62fa259addbe7ae427fd54486abc7604ea7619e7a9c47965606d`; offset 736700, 1024 bytes: `be1d2941b054626376fa58155ce0ef8d6357dd0defcbf9eaa37d19ff6098873c` | PASS | -| DEFLATE corrupt | `read_errno=5`, failure after 65536 output bytes | target extent only | PASS | -| MicroLZMA full | `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` | full input consumed | PASS | -| MicroLZMA partial | `5a840803f5372b7be1db70713fed7705bf6e982ca2fd319f722c21f094f6c8dd` | offset 65500, 2048 bytes: `49c231f92dde0b0104d7e5b3a01d918dde818c3a6dca05393e2a424d01c214f2`; offset 524287, 4097 bytes: `b8e80c144eacd1f8863c72eab66272379923f0d738f63576ab8286455e0dde8c` | PASS | -| MicroLZMA corrupt | `read_errno=5`, zero output bytes | target pcluster only | PASS | -| ZSTD full | `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` | formal FreeBSD API | PASS | -| ZSTD partial | `5a840803f5372b7be1db70713fed7705bf6e982ca2fd319f722c21f094f6c8dd` | same two boundary/random hashes as MicroLZMA | PASS | -| ZSTD corrupt | `read_errno=5`, zero output bytes | target pcluster only | PASS | -| HEAD2 full | `7e2f40362554f4460e80ec2f8d91f4e6c04a8e58f2980d637b2d383a7aa3b3f8` | offset 65500, 4096 bytes: `f0a80e3217f54897eae271b6e570c862abcffad3a4ffd19ffec0444ee06ae721` | PASS | -| HEAD2 corrupt | `read_errno=5`, zero output bytes | patched HEAD2 pcluster | PASS | -| interlaced full | `7e2f40362554f4460e80ec2f8d91f4e6c04a8e58f2980d637b2d383a7aa3b3f8` | offset 16240, 8192 bytes across first compressed/plain transition: `b1387900e55e5672944f8299fe66ac9542007f2b0cbed81d5974831d6040cdae` | PASS | -| extent records | static format/control-flow review only | erofs-utils 1.8.6 cannot emit | MKFS-UNAVAILABLE | - -## Fragment-backed metabox results - -| Case | Actual | Status | -|---|---|---| -| positive file | SHA256 `4536c1d7121f48829475f29179f54baa57154b4ef817cf0776f81782585d29ad` | PASS | -| shared-prefix xattr | `shared-value` | PASS | -| per-file xattr | `value-000` | PASS | -| self-loop | mount exit 1; `packed inode nid=38 is not a non-recursive regular file: Integrity check failed` | PASS | -| out-of-range | mount exit 1; `Integrity check failed` | PASS | -| metabox NID bit 63 | mount exit 1; `Integrity check failed` | PASS | -| packed NID bit 63 | mount exit 1; `Integrity check failed` | PASS | - -The `vmstat -m` EROFS row after the matrix showed zero active allocations: - -```text -erofs 0 0 240695 16,32,64,128,256,384,1024,2048,4096,8192,16384,32768,65536 -``` - -The cumulative allocation count increased as expected; the active allocation -and active-byte columns were both zero. - -## Regression matrix - -| Regression | Actual | Status | -|---|---|---| -| LZ4 legacy full index | SHA256 `370eb0a8df86868c4842ca535ed64670f0277ea2ed47a703f089bbb13ee4ac52` | PASS | -| LZ4 compact index | same SHA256 | PASS | -| LZ4 64 KiB big pcluster | same SHA256 | PASS | -| LZ4 all-fragments | SHA256 `a3a83e5c524b5ed446a06ce78cf407192a0c80119f15d2bc3489a50515eb49e0` | PASS | -| LZ4 ztailpacking | SHA256 `e2aa4a0a0cbcf422f397c7069a38ae0f073781386958e7db0dfa3ff2ca075513` | PASS | -| xattr/metabox | file SHA256 `4536c1...`; xattr `value-000` | PASS | -| single-device chunk | SHA256 `de29abbd47ecd9136f64f22f73fcb40bce1718a8865e6282e74953aa1df80c44` | PASS | -| external compressed LZ4 extent | complete SHA256 `370eb0...` | PASS | -| external compressed boundary | offset 65500, 4096 bytes: `0d66627577218a39a620e8b28d8c8d64b974b184063c52cb0cceeaf0c297c0db` | PASS | -| local NFSv3/TCP over external compressed extent | complete SHA256 `370eb0...` | PASS | - -The external compressed fixture declared slot 1 at unified block 4, stored the -real two-block LZ4 pcluster in the external provider, and zeroed the -corresponding bytes in the primary image. Primary SHA256 was -`5625612426d68624c77dd94b09754da583fc6bd90767e9e9d958e4fc89131a4d`; -blob SHA256 was -`0d71102ce471af0a30d4d61a1fbeef1fc33437c85651d7c7f9c60ebd305ec9d0`. - -## NFS startup-race investigation - -The first regression run started rpcbind, mountd, and nfsd and immediately -called `mount_nfs`. The client printed one transient -`RPCPROG_NFS: RPC: Program not registered`, then retried successfully and read -the correct complete SHA256. No dmesg line changed. - -A separate clean rerun waited for: - -```sh -rpcinfo -t 127.0.0.1 nfs 3 -``` - -Readiness succeeded on attempt 2: - -```text -program 100003 version 3 ready and waiting -``` - -The subsequent NFS mount emitted no RPC warning and produced the same complete -SHA256. This proves the first message was a user-space service-registration -race, not an EROFS or NFS data-path failure. - -## dmesg - -Core, regression, and clean NFS snapshots were each byte-identical before and -after their respective test matrices. All six snapshot files had SHA256: - -```text -2aa3500d33a7cfbe0db87854d3427231cda9cf92b7f85e553f957231f7359680 -``` - -There was no new panic, trap, decompression diagnostic, integrity message, GEOM -orphan warning, or NFS kernel message. - -## Cleanup - -Every test used a trap that unmounted the current EROFS/NFS mount, detached the -specific md unit, stopped NFS services in client-first order, restored or -removed `/etc/exports`, and unloaded the module by the observed KLD ID. - -Final audits after the core, regression, disabled-ZSTD LZ4, and clean NFS runs -all reported: - -```text -erofs mounts=0 -NFS mounts=0 -md units=0 -erofs modules=0 -nfsd/mountd/rpcbind processes=0 -``` - -## Limitations - -- **MKFS-UNAVAILABLE**: erofs-utils 1.8.6 cannot generate the new on-disk extent - record format. Extent coverage is independent static ABI/control-flow review - only. No newer-tool extent image is counted as a FreeBSD PASS. -- METABOX and the transformed HEAD2/partial-reference fixtures require newer - format-aware tooling plus byte-level assertions. Their guest reads are real - FreeBSD kernel tests; fixture creation and mutations are documented and - checksum-validated. -- No CI, automated runner, benchmark threshold, memory-pressure run, or forced - OOM scenario was added. The requested functional manual matrix and active - allocation checks were completed. -- Kernel failures: none. diff --git a/tests/results/manual/2026-08-08T2337Z-metadata-vfs/manual-test-report.md b/tests/results/manual/2026-08-08T2337Z-metadata-vfs/manual-test-report.md deleted file mode 100644 index 29f5ce1..0000000 --- a/tests/results/manual/2026-08-08T2337Z-metadata-vfs/manual-test-report.md +++ /dev/null @@ -1,358 +0,0 @@ -# repo22 Metadata and VFS Manual Test Report - -Started: 2026-08-08 23:37 UTC -Completed: 2026-08-09 UTC -Baseline: `c208bf1f4b8d7a85777f7fe45e8c6e8d3a9f2d1a` -Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG -FreeBSD source reference: `/work/dev-freebsd-releng`, releng/15.0 -Linux source reference: `/work/dev-src-linux/fs/erofs` -Host production tool: erofs-utils 1.8.6 - -## Result - -**PASS** for the implemented metadata/VFS changes and every runnable required -FreeBSD 15 regression. - -Two limitations are recorded rather than misreported as PASS: - -1. The Linux host had no loop provider/EROFS kernel mount path, so the - deterministic nonzero-padding fixture passed Linux erofs-utils 1.8.6 - `fsck.erofs` and `dump.erofs`, while the Linux kernel mount is - **ENVIRONMENT-UNAVAILABLE**. -2. A real TC010 48-bit `statfs` mount needs a provider as large as the declared - multi-terabyte image. The corrected test records this as a provider/tool - requirement; no small-media mount is called a positive PASS. - -An initial NFS stress run completed all data and metadata assertions but failed -cleanup because `service nfsd onerestart` inherited the deliberately open NFS -client descriptor. The procedure was corrected to run the service command with -`3<&-`; the complete stress/background/cleanup sequence then passed. The first -cleanup failure is retained here as evidence for the TC132 ordering fix. - -## Baseline and Scope Audit - -- `HEAD` and `FETCH_HEAD` both resolved to the required baseline. -- `find /work -name AGENTS.md -type f -print` returned no paths. -- Only `repo-community/repo22` was modified. -- Existing untracked `build/` objects and historical manual-test artifacts were - preserved and excluded from staging. -- No CI implementation, binary fixture, overlay, image, or VM artifact is part - of the intended commit. - -## Build and ABI Validation - -Commands: - -```sh -EROFS_ZSTDIO=0 ./build.sh -EROFS_ZSTDIO=1 ./build.sh -nm -u module.ko | awk '$NF == "bcmp" { n++ } END { print n + 0 }' -git diff --check -- repo-community/repo22 -``` - -Results: - -| Build | SHA256 | `bcmp` count | Result | -|---|---|---:|---| -| ZSTDIO disabled | `031038ef195497dc6a1d840d55b293292e051fb888c679c89c8cfbd19b56d525` | 0 | PASS | -| ZSTDIO enabled | `16166b9ffc96a532bda9925513e10df11f4cf8a06b3c15afbc09b1a4e4b9d2df` | 0 | PASS | - -Both modules loaded and unloaded on FreeBSD 15. The final unload used the -observed KLD ID so arbitrary copied filenames did not affect cleanup. - -FreeBSD 15 source inspection confirmed the exact current ABI: - -```text -vnode_pager_local_getpages(struct vop_getpages_args *) -vnode_pager_local_getpages_async(struct vop_getpages_async_args *) -``` - -The local ext2 vnode vector registers both functions directly. EROFS now does -the same. - -## Deterministic Fixture Evidence - -The checked-in `prepare-fixtures.sh` was syntax-checked and rerun into separate -untracked `repro3-*` directories. It reproduced the expected byte-identical -inline, special, pager, NFS, and nlink images and generated the current namei -variants. - -Key structural evidence: - -```text -inline_nid=39 inode_off=1248 inode_blockoff=1248 xattr_icount=0->695 -inline_data_blockoff=4068 inline_size=31 -wide_nid=42 block=6 dirents=80 last_nameoff=4043 padding_patch=4084:4092 -nlink1_nid=43 i_format_bit4=1 i_nb=0x1234 -special compact rdev raw_u=0x543abc21 -special extended rdev raw_u=0x543abc21 -``` - -Generated image hashes: - -| Image | SHA256 | -|---|---| -| `inline.erofs` | `0435b3ea748a88cccbdc6390dec4285a3706bec3dc09de58aa1808544ebc63d0` | -| `inline-cross-block.erofs` | `63ebb7632687b564beb4c9dd8036eb4ac63c4495f63061ad7eec1c134c656932` | -| `special-compact.erofs` | `fd78256dd83d9d6d957e5f843c7a8e8a175a4b3243d528bebd299b0226853237` | -| `special-extended.erofs` | `e73e9b84d9ceb8c2b07e9c2732733b0fd607736c68c09522a2402fbeef6ba8d5` | -| `namei-base.erofs` | `d1730ff23836797c6c09e1b39b1cf23efc16e27f85ab07fdcab577bb82871659` | -| `namei-padding-nonzero.erofs` | `9a94e9af2cab264b9c11975a20d78e615d6fe1e6f86267173cb5ac86aecb2b17` | -| `namei-corrupt-short.erofs` | `fb89f74795a5569ed3a85d63836dd75a06e1f17823d0508048c37da710ea6e75` | -| `namei-corrupt-nameoff.erofs` | `b0b70ee615f163430f04edb91ab76c27ee8cc9a75b8ebb2008834f5b550e3933` | -| `namei-corrupt-name.erofs` | `6a980ad3e241603eda2ef71a470c82975291c614366a401e4e89c17c9adf9b91` | -| `pager-plain.erofs` | `36596edea5bfbaa1157f6c142095a7ee9949b5df1b1a35620c0d22ec853f3c09` | -| `pager-lz4.erofs` | `b06daee6b02a6ebe967655be760b496c8a9d922cf47acb5c840e82a48a34c51d` | -| `nfs-a.erofs` | `6d86dcf620b007d069895e3e94a21a74dadcf35e84c92dc4f7c21a2ec23bd901` | -| `nfs-b.erofs` | `18913fd319daca20b3e4d30a89c05c416b4d3ca396394509e5112d548674f4c2` | -| `nlink.erofs` | `798eb81b3ba7270ee653b00adeba47a6e03c982adef1982bf5b4e0935669ae83` | -| `nlink1-patched.erofs` | `19fd85f32ed89117d8e02bc19ca09655dd3bae9152cf693a399eda2135233042` | - -The padding patch was followed by a rigorous CRC32C recomputation over the -superblock block. Linux erofs-utils produced the exact payload: - -```text -wide entry 079 -``` - -The Linux kernel comparison command failed before mount with “failed to setup -loop device”; it is therefore not labeled PASS. - -## TC147: FLAT_INLINE Bounds - -Commands included fresh md attach/mount for the base and cross-block images, -then cold `cat`/`stat` access. - -Observed on the final module: - -```text -positive hash=0347f272ba395aff6df5fd824a7c552fa26d7f017283f0544136385abef31b01 stat=31 8 -corrupt stat: Integrity check failed -corrupt stat: Integrity check failed -corrupt cat: Integrity check failed -dmesg_before=122 dmesg_after=122 mounts=0 mds= module_rc=1 -``` - -The positive file read exactly. The checksum-valid corrupt inode failed with -`EINTEGRITY` before its 31-byte inline range could cross the metadata block. -The mapping path also uses checked additions, and primary/metabox declared -bounds are validated at inode decode. - -Self-review first rejected the old `i_xattr_icount=1020` mutation because it -merely moved the inline data into the next block without crossing that block. -The corrected value `695` places the tail at block offset 4068. Its first -genuine rerun exposed a stale constructing vnode: the second `stat` returned -`EBADF`. The failure path now calls `vgone()` before `vput()`, and the complete -final rerun above returned `EINTEGRITY` for every repeated access. - -## TC022/TC056: Special `st_rdev` - -The source fixture used real Linux char/block nodes with major `2748`, minor -`344865`, plus a FIFO. `stat_special.c` checked `st_rdev` directly because -FreeBSD `stat -f %Lr` truncates before `minor()` for large values. - -Compact and extended results were identical: - -```text -PASS char rdev=0xa430005bc21 major=2748 minor=344865 -PASS block rdev=0xa430005bc21 major=2748 minor=344865 -PASS fifo rdev=0xffffffffffffffff -``` - -This proves Linux `new_decode_dev(0x543abc21)` followed by FreeBSD `makedev()`; -a little-endian integer cast would not produce this FreeBSD `dev_t`. - -## TC055: Real Compressed Allocation - -For every row, full FreeBSD kernel reads matched the expected SHA256 and -`st_blocks * 512` matched the inode's real on-disk compressed size: - -| Shape | File | Size | Allocated bytes | SHA256/result | -|---|---|---:|---:|---| -| LZ4 full | `shape.dat` | 1048576 | 8192 | `370eb0a8...` PASS | -| LZ4 compact | `shape.dat` | 1048576 | 8192 | `370eb0a8...` PASS | -| LZ4 fragment | `fragment.dat` | 1048699 | 0 | `a3a83e5c...` PASS | -| LZ4 ztailpacking | `inline.dat` | 65536 | 0 | `e2aa4a0a...` PASS | -| MicroLZMA partial A | `a.dat` | 1048576 | 4096 | `370eb0a8...` PASS | -| MicroLZMA partial B | `b.dat` | 700000 | 4096 | `5a840803...` PASS | -| DEFLATE compact/partial | `a.dat` | 1048576 | 36864 | `370eb0a8...` PASS | -| DEFLATE full/partial | `b.dat` | 1050624 | 8192 | `61b17076...` PASS | -| ZSTD partial A | `a.dat` | 1048576 | 4096 | `370eb0a8...` PASS | -| ZSTD partial B | `b.dat` | 700000 | 4096 | `5a840803...` PASS | - -Uncompressed regressions: - -```text -inline size=31 st_blocks=8 -plain size=21211 st_blocks=48 -chunk size=90017 st_blocks=176 -``` - -Representative compression image hashes were: - -```text -LZ4 full d784f8dc... LZ4 compact 37942ef1... -LZ4 fragment a4e5d40e... LZ4 ztail 035069eb... -MicroLZMA 2f4bd89d... DEFLATE 8bc720a1... ZSTD 7f5ee4f8... -``` - -## TC141/TC148: Directory Compatibility and Strictness - -FreeBSD results: - -- Cold `/alpha/bravo/charlie/payload.txt`: PASS without parent warming. -- Repeated cold lookup and post-negative-cache existing lookup: PASS. -- Patched nonzero tail bytes: accepted. -- `wide` enumeration: 320 files plus `.` and `..`; 322 dirents through a - 128-byte buffer and restart cookies. -- Short block, non-monotonic `nameoff`, and `/` in an on-disk name: lookup and - readdir both returned `EINTEGRITY`. -- Repeated corrupted lookup remained `EINTEGRITY`, not cached `ENOENT`. - -The duplicate validators were removed; lookup and readdir use the shared -helper in `dir.c`. - -## TC132/TC133: NFS Generation and Stress - -Direct handle validation after the final superblock-hash implementation: - -```text -nfs-a: fsid=00000034:000000e0 nid=0x2e gen=849213208 -nfs-b: fsid=00000034:000000e0 nid=0x2e gen=4011239099 -``` - -- `nfs-a` remount on the same md unit produced byte-identical complete handles. -- `va_gen` equaled handle generation. -- Replacing `nfs-a` with `nfs-b` on the same md unit made the old handle return - `ESTALE` through both `fhstat` and `fhopen`. -- Bad length/pad returned `EINVAL`; bad generation/NID returned `ESTALE`. -- Handle generation mutation used `gen_xor 1`, not a hard-coded value. - -NFSv3/TCP READDIRPLUS stress: - -- Four clients mounted; requested 512/1024/4096 readdir sizes were clamped by - FreeBSD to 8192, while default was 65536. -- All four 12,050-name sorted listings had SHA256 - `4fbf1b6103e8884223629c3fd03f4ec36061e65cd2a1becfc96670daa96f9d71`. -- Twelve traversal workers and eight cat/stat controller jobs were waited by - PID; every exit status was zero. -- READDIRPLUS client/server count was 1304 in the recorded full run. -- RPC timeouts, invalid replies, retries, and server write RPCs were zero. -- The 21,757,952-byte throughput file hash was - `0841effed82d1adf394b6834ce30d4d9eb5fc9427527e854ec1cb6bb4b86c119`. -- nfsd restart with an open descriptor passed after the service command closed - inherited fd 3. -- Corrected full rerun ended `NFS-STRESS-CLEAN-PASS`. - -## TC149: Real Pager Faults - -`tests/mmap_fault.c` was compiled natively on FreeBSD 15 and run against both -plain and LZ4 images. - -Both runs printed: - -```text -PASS size=21211 pages=6 fnv1a64=a1890a1c216724be random-faults=6 \ -eof-zero=PASS sigbus=PASS private-cow=PASS -``` - -The helper verified deterministic random faults after `MADV_DONTNEED`, full -mapping equality with `pread`, partial EOF-page zeroes, child `SIGBUS` on the -next full page, `MAP_SHARED` write denial, FreeBSD private COW semantics, and -`O_RDWR -> EROFS`. - -dmesg added exactly the two expected child exits on signal 10 (`SIGBUS`). No -parent crash, VM assertion, trap, panic, or dirty writeback appeared. - -## Additional Regressions - -| Area | Evidence | Result | -|---|---|---| -| Inline/system xattr | trusted, security, long-prefix, user values exact | PASS | -| Shared xattr | shared and per-inode values exact | PASS | -| Metabox xattr | `dirA/nested.txt` and `hello.txt` enumerated/read | PASS | -| Fragment-backed metabox | positive file/xattrs exact; self-loop and range images rejected | PASS | -| Single-device chunk | `plain.bin`, `deep/payload.bin` hashes exact | PASS | -| Chunk multidevice | block-map and indexed files hashes exact | PASS | -| External compressed multidevice | complete 1 MiB LZ4 SHA256 `370eb0a8...` | PASS | -| Compact nlink rules | explicit nlink 1/2 plus flagged nlink-one image | PASS | -| ZSTD enabled final load/read | final module and `zstd-partial-ref` read | PASS | - -Supporting fixture hashes: - -```text -system-inline-four 6f521b62... -shared xattr 208b61ca... -metabox xattr ef8d619c... -chunk single 7aa8db22... -chunk multidev 73343b70... + a36c2b9b... -external LZ4 56256124... + 0d71102c... -NFS stress image 6928f05b... -``` - -The final-module fragment-backed metabox rerun used the previously qualified, -checksum-valid image with SHA256 -`8a9a62bd203994711b8272192915d811e6c3de23e07ad9607dd63e66cc109bcd`. -`/tree/d00/file000.txt` produced SHA256 -`4536c1d7121f48829475f29179f54baa57154b4ef817cf0776f81782585d29ad`; -the shared and per-file xattrs were `shared-value` and `value-000`. The -self-loop and out-of-range images returned `Integrity check failed`. dmesg -remained at 122 lines, and the post-run audit showed zero mounts, md providers, -and loaded EROFS modules. - -## Final-Binary Closure Rerun - -After the `vgone()` error-path fix, the final ZSTDIO module SHA256 -`16166b9ffc96a532bda9925513e10df11f4cf8a06b3c15afbc09b1a4e4b9d2df` -was used for one continuous closure matrix: - -- LZ4 full/compact/fragment/ztailpacking, MicroLZMA partial, DEFLATE - compact/full partial, and ZSTD partial hashes and `st_blocks` all matched. -- Inline trusted/security/long-prefix/user xattrs matched exact values. -- Metabox shared xattrs returned `answer=forty-two`; the fragment carrier file - and `repo22.item-000=value-000` matched. -- Single-device chunk, external chunk provider, and external compressed LZ4 - hashes matched; compressed `st_blocks=16`. -- The final module preserved the same NFS handle across same-md remount, made - the old handle `ESTALE` after image replacement, synchronized `va_gen`, and - completed a real local NFSv3/TCP client read with matching SHA256. - -The first closure script stopped after the system-xattr row because it queried -an obsolete metabox attribute name. Its trap left zero mounts, md providers, -modules, and services. The corrected `answer`/`metaboxshared` queries and all -remaining rows passed. The complete final audit was: - -```text -dmesg_before=122 dmesg_after=122 -erofs_mounts=0 nfs_mounts=0 mds= module_rc=1 services=0/0/0 -FINAL-REGRESSION-REST-PASS -``` - -## Final Environment and Cleanup - -Final guest audit: - -```text -mounts=0 -mds= -modules=0 -services=0 -final_dmesg_before=122 final_dmesg_after=122 -``` - -The final dmesg tail contained only the historical pre-test duplicate-module -diagnostic and the two intentional pager-child SIGBUS exits. The final -compression, xattr, chunk, multidevice, generation, and NFS closure pass added -no dmesg lines. - -Guest helper hashes: - -| Helper | SHA256 | -|---|---| -| `mmap_fault` | `b19c9c28a7abdfebe4243e3f4876b711eebc1c02b9856c15cb8bf15ae2da3a65` | -| `nfs_fh_tool` | `0dcad233cc8768ca84569c8d030939b526d50c64190fc54d5f0de013191f11d6` | -| `stat_special` | `b5c0eabd06541268e799df4f72fa7f37a460617bd8960f918b27710483db3841` | -| `readdir_probe` | `44747ccd8d9a285a37d8049b7d61d00946db88482c08a3570e59b5a2fa1b4ea2` | - -No password material was printed into this report or written into tracked -files. diff --git a/tests/results/manual/2026-08-08T2337Z-metadata-vfs/prepare-fixtures.sh b/tests/results/manual/2026-08-08T2337Z-metadata-vfs/prepare-fixtures.sh deleted file mode 100755 index 925f194..0000000 --- a/tests/results/manual/2026-08-08T2337Z-metadata-vfs/prepare-fixtures.sh +++ /dev/null @@ -1,277 +0,0 @@ -#!/bin/sh -set -eu - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -fixture_dir=${FIXTURE_DIR:-"$script_dir/fixture"} -artifact_dir=${ARTIFACT_DIR:-"$script_dir/artifacts"} - -for tool in mkfs.erofs fsck.erofs dump.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - echo "missing tool: $tool" >&2 - exit 1 - } -done -test "$(id -u)" -eq 0 || { - echo "root is required to create device-node fixtures" >&2 - exit 1 -} -test ! -e "$fixture_dir" || { - echo "fixture directory already exists: $fixture_dir" >&2 - exit 1 -} -test ! -e "$artifact_dir" || { - echo "artifact directory already exists: $artifact_dir" >&2 - exit 1 -} - -mkdir -p \ - "$fixture_dir/inline" \ - "$fixture_dir/special" \ - "$fixture_dir/namei/alpha/bravo/charlie" \ - "$fixture_dir/namei/alpha/sibling" \ - "$fixture_dir/namei/wide" \ - "$fixture_dir/pager" \ - "$fixture_dir/nfs/basic/subdir" \ - "$fixture_dir/nlink/subdir" \ - "$artifact_dir" - -printf 'inline-tail-payload-0123456789\n' > "$fixture_dir/inline/inline.txt" -printf 'special target\n' > "$fixture_dir/special/target" -ln -s target "$fixture_dir/special/link" -mknod "$fixture_dir/special/char-large" c 2748 344865 -mknod "$fixture_dir/special/block-large" b 2748 344865 -mkfifo "$fixture_dir/special/fifo" - -printf 'cold nested lookup payload\n' > \ - "$fixture_dir/namei/alpha/bravo/charlie/payload.txt" -printf 'repeat lookup payload\n' > \ - "$fixture_dir/namei/alpha/bravo/repeat.txt" -printf 'sibling marker\n' > "$fixture_dir/namei/alpha/sibling/marker.txt" -index=0 -while [ "$index" -lt 320 ]; do - name=$(printf 'entry-%03d-abcdefghijklmnopqrstuvwxyz.txt' "$index") - printf 'wide entry %03d\n' "$index" > "$fixture_dir/namei/wide/$name" - index=$((index + 1)) -done - -python3 - "$fixture_dir" <<'PY' -from pathlib import Path -import sys - -root = Path(sys.argv[1]) -data = bytes(((index * 131 + 17) ^ (index >> 3)) & 0xff - for index in range(5 * 4096 + 731)) -(root / "pager" / "pager.bin").write_bytes(data) -(root / "nfs" / "basic" / "regular.txt").write_text( - "stable file handle payload\n", encoding="ascii") -(root / "nfs" / "basic" / "subdir" / "child.txt").write_text( - "child\n", encoding="ascii") -PY -ln -s regular.txt "$fixture_dir/nfs/basic/link-to-regular" -mkfifo "$fixture_dir/nfs/basic/test.fifo" - -printf 'single\n' > "$fixture_dir/nlink/single.txt" -printf 'hardlinked\n' > "$fixture_dir/nlink/hard-a.txt" -ln "$fixture_dir/nlink/hard-a.txt" "$fixture_dir/nlink/hard-b.txt" -printf 'child\n' > "$fixture_dir/nlink/subdir/child.txt" - -find "$fixture_dir" -exec touch -h -t 197001010000.00 {} + - -build_image() -{ - uuid=$1 - image=$2 - source=$3 - shift 3 - mkfs.erofs -d0 -x-1 -T0 --all-time --all-root --workers=1 \ - -U "$uuid" "$@" "$artifact_dir/$image" "$fixture_dir/$source" -} - -build_image 11111111-2222-3333-4444-555555555551 inline.erofs inline -build_image 11111111-2222-3333-4444-555555555552 \ - special-compact.erofs special -E force-inode-compact -build_image 11111111-2222-3333-4444-555555555553 \ - special-extended.erofs special -E force-inode-extended -build_image 11111111-2222-3333-4444-555555555554 namei-base.erofs namei -build_image 11111111-2222-3333-4444-555555555555 \ - pager-plain.erofs pager -E noinline_data -build_image 11111111-2222-3333-4444-555555555556 \ - pager-lz4.erofs pager -z lz4 -build_image aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee1 nfs-a.erofs nfs -build_image aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee2 nfs-b.erofs nfs -build_image 11111111-2222-3333-4444-555555555557 \ - nlink.erofs nlink -E force-inode-compact - -ARTIFACT_DIR="$artifact_dir" python3 <<'PY' -from pathlib import Path -import hashlib -import os -import struct - -artifact_dir = Path(os.environ["ARTIFACT_DIR"]) - - -def u16(image, offset): - return struct.unpack_from("> 1) ^ ( - polynomial if checksum & 1 else 0) - put_u32(image, 1028, checksum & 0xFFFFFFFF) - - -def inode_offset(image, nid): - block_bits = image[1036] - meta_blkaddr = u32(image, 1064) - return (meta_blkaddr << block_bits) + (nid << 5) - - -def compact_inode(image, nid): - offset = inode_offset(image, nid) - inode_format = u16(image, offset) - assert (inode_format & 1) == 0 - return offset, (inode_format >> 1) & 7, u32(image, offset + 8) - - -def inline_dir_entries(image, nid): - inode = inode_offset(image, nid) - inode_format = u16(image, inode) - inode_size = 64 if inode_format & 1 else 32 - layout = (inode_format >> 1) & 7 - size = (struct.unpack_from("= 12 and first_nameoff % 12 == 0 - count = first_nameoff // 12 - entries = [] - for index in range(count): - entry = data + index * 12 - child_nid, nameoff = struct.unpack_from(" block_size -inline_cross = bytearray(inline) -put_u16(inline_cross, inline_inode + 2, inline_cross_xattr_icount) -write_image("inline-cross-block.erofs", inline_cross) - -namei = bytearray((artifact_dir / "namei-base.erofs").read_bytes()) -root_nid = u16(namei, 1038) -root_inode, root_data, root_size, root_entries = inline_dir_entries( - namei, root_nid) -wide_nid = next(nid for nid, _, _, name in root_entries if name == b"wide") -wide_inode, wide_layout, _ = compact_inode(namei, wide_nid) -assert wide_layout == 2 -block_bits = namei[1036] -wide_block = u32(namei, wide_inode + 16) << block_bits -wide_first_nameoff = u16(namei, wide_block + 8) -wide_count = wide_first_nameoff // 12 -wide_last_nameoff = u16(namei, wide_block + (wide_count - 1) * 12 + 8) -padding_nul = namei.index(0, wide_block + wide_last_nameoff, - wide_block + (1 << block_bits)) - -nonzero_padding = bytearray(namei) -nonzero_padding[padding_nul + 1:padding_nul + 9] = b"PAD!ERO!" -write_image("namei-padding-nonzero.erofs", nonzero_padding) - -short_block = bytearray(namei) -put_u32(short_block, root_inode + 8, 8) -write_image("namei-corrupt-short.erofs", short_block) - -bad_nameoff = bytearray(namei) -first_nameoff = u16(bad_nameoff, root_data + 8) -put_u16(bad_nameoff, root_data + 12 + 8, first_nameoff) -write_image("namei-corrupt-nameoff.erofs", bad_nameoff) - -bad_name = bytearray(namei) -slash_offset = wide_block + wide_last_nameoff + 5 -assert bad_name[slash_offset] not in (0, ord("/")) -bad_name[slash_offset] = ord("/") -write_image("namei-corrupt-name.erofs", bad_name) - -nlink = bytearray((artifact_dir / "nlink.erofs").read_bytes()) -nlink_root_nid = u16(nlink, 1038) -_, _, _, nlink_entries = inline_dir_entries(nlink, nlink_root_nid) -single_nid = next(nid for nid, _, _, name in nlink_entries - if name == b"single.txt") -single_inode, _, _ = compact_inode(nlink, single_nid) -nlink1 = bytearray(nlink) -put_u16(nlink1, single_inode, u16(nlink1, single_inode) | 0x10) -put_u16(nlink1, single_inode + 6, 0x1234) -write_image("nlink1-patched.erofs", nlink1) - -with (artifact_dir / "fixture-evidence.txt").open("w", encoding="ascii") as out: - out.write(f"inline_nid={inline_nid} inode_off={inline_inode} " - f"inode_blockoff={inline_inode % block_size} " - f"xattr_icount=0->{inline_cross_xattr_icount} " - f"inline_data_blockoff={inline_data_off % block_size} " - f"inline_size={inline_size}\n") - out.write(f"wide_nid={wide_nid} block={wide_block >> block_bits} " - f"dirents={wide_count} last_nameoff={wide_last_nameoff} " - f"padding_patch={padding_nul + 1 - wide_block}:" - f"{padding_nul + 9 - wide_block}\n") - out.write(f"nlink1_nid={single_nid} i_format_bit4=1 i_nb=0x1234\n") - for image_name in ("special-compact.erofs", "special-extended.erofs"): - image = bytearray((artifact_dir / image_name).read_bytes()) - special_root = u16(image, 1038) - _, _, _, entries = inline_dir_entries(image, special_root) - out.write(image_name + "\n") - for nid, _, _, name in entries: - if name not in (b"char-large", b"block-large", b"fifo"): - continue - offset = inode_offset(image, nid) - out.write(f" {name.decode()} nid={nid} inode_size=" - f"{64 if u16(image, offset) & 1 else 32} " - f"raw_u={u32(image, offset + 16):#010x}\n") - -with (artifact_dir / "SHA256SUMS").open("w", encoding="ascii") as sums: - for path in sorted(artifact_dir.glob("*.erofs")): - sums.write(f"{hashlib.sha256(path.read_bytes()).hexdigest()} " - f"{path.name}\n") -PY - -fsck.erofs -d1 "$artifact_dir/namei-padding-nonzero.erofs" -dump.erofs --cat --path=/wide/entry-079-abcdefghijklmnopqrstuvwxyz.txt \ - "$artifact_dir/namei-padding-nonzero.erofs" -cat "$artifact_dir/fixture-evidence.txt" -cat "$artifact_dir/SHA256SUMS" diff --git a/tests/results/manual/2026-08-09T0124Z-final-review/manual-test-report.md b/tests/results/manual/2026-08-09T0124Z-final-review/manual-test-report.md deleted file mode 100644 index ba7de89..0000000 --- a/tests/results/manual/2026-08-09T0124Z-final-review/manual-test-report.md +++ /dev/null @@ -1,166 +0,0 @@ -# repo22 Final Review WIP Manual Test Report - -Started: 2026-08-09 01:24 UTC - -Completed: 2026-08-09 03:33 UTC - -Parent commit: `9a3604fba32cfe2ff263d954d80fe588f99db88f` - -Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG - -Linux reference: `/work/dev-src-linux/fs/erofs` - -FreeBSD reference: `/work/dev-freebsd-releng` - -## Scope - -This run reviews and validates four final correctness findings: - -- Linux-compatible 48-bit superblock union selection; -- rejection of extended inode sizes above FreeBSD `OFF_MAX`; -- bounded reads from multi-GiB explicit extent holes; -- 64-bit directory block-search indexes. - -No CI work, binary fixture, guest overlay, build object, or raw VM artifact is -part of the intended commit. - -## Result - -| Test | Result | Evidence | -|---|---|---| -| TC150 48-bit fallback root | PASS | Real FreeBSD mount, root read, inode and `df` | -| TC151 size above `OFF_MAX` | PASS | Six direct failures, rc 1, syscall errno 97 | -| TC152 bounded extent hole | PASS | Two pread/mmap probes, stable active allocation | -| TC153 large directory index | SHELVED | Host fixture reproducible; kernel run incomplete | - -TC153 is deliberately not marked PASS. See -`issues/TC153-large-directory-block-index-validation.md`. - -## Source Review - -Linux `super.c` initializes `blocks_lo` and uses `rb.blocks_hi` only inside -`48BIT && rootnid_8b`. repo22 now decodes blocks and root NID together under -that same selector. - -FreeBSD exposes signed `off_t` pager and vnode interfaces bounded by -`OFF_MAX`; repo22 rejects a larger decoded inode before vnode/pager setup. - -The extent-hole path now zeroes only the current requested span in -`z_erofs_do_read()` instead of allocating the complete logical extent. - -The directory block search now uses 64-bit bounds and a checked block-offset -multiplication. The within-block search remains 32-bit, matching the validated -block-sized domain. - -## Build Results - -Commands: - -```sh -EROFS_ZSTDIO=0 ./build.sh -EROFS_ZSTDIO=1 ./build.sh -nm -u build/erofs.ko -git diff --check -- repo-community/repo22 -``` - -| Configuration | Module SHA256 | Result | -|---|---|---| -| `EROFS_ZSTDIO=0` | `65bc19d53a2a7f0525bfacadb441dab37ee318f5f5b4c62b5ecd1b5090fe46d2` | PASS | -| `EROFS_ZSTDIO=1` | `d46ca4dfc858deaf4840fad8b8589b16d4b71b34afa96dc7acf1255890c5cef7` | PASS | - -Both freestanding builds completed. Neither module had an unresolved `bcmp` -reference. The ZSTDIO-enabled module was transferred to the guest, loaded as -KLD ID 7, and unloaded after testing. - -## Fixture Evidence - -| Fixture | SHA256 | -|---|---| -| `fallback-48bit-root2.erofs` | `bed3be4dfb8499d4e794b03eddd5b9cda95bc8e5571ae7d765c4852ece0d95a3` | -| `extended-size-bit63.erofs` | `e4a0f550168f1a2911603863d0074d474e61adc787c14c0278c83a060643ee38` | -| `extent-hole-5g.erofs` | `50014a24493918247e36511ad34a2fe8ab47ae09ea46d7fd62a1bed445a6c65f` | -| `large-dir-intmax.erofs` | `0f90d3d57adbbbd946e41b225c1f6c464915c6abb0b13478ec9b2a318def1f72` | - -The TC153 hash is host generator evidence only. - -## TC150 - -The fixture encoded `rootnid_2b=36`, `rootnid_8b=0`, `blocks_lo=1`, and the -48-bit incompat bit. The 4 KiB image mounted on `/dev/md0` and produced: - -```text -content=48-bit fallback root -sha256=d361f537492113ca93cfbf91c06ebc06e2b8b695d8b6e63ad3666013eaa029f0 -root inode=36 -df total=4 one-KiB blocks -``` - -This proves the union was not shifted into a high block count. It does not -replace the large-provider TC010 test. - -## TC151 - -The fixture used extended inode NID 40 at byte 1280 with -`i_size=0x8000000000000000`. The image mounted, but every access to `big.dat` -failed before open or pager setup: - -| Access | Attempts | Direct rc | `truss` result | -|---|---:|---:|---| -| `stat` | 2 | 1, 1 | `fstatat ... ERR#97` | -| `cat` | 2 | 1, 1 | `openat ... ERR#97` | -| `mmap_fault` | 2 | 1, 1 | `openat ... ERR#97` | - -All six errors were `Integrity check failed`. No file descriptor reached the -read or mmap phase. - -## TC152 - -The mounted file size was `5368713216` bytes. The native helper probed offset -`3221225472` with a one-byte pread and one-page private mmap: - -```text -attempt 1: PASS, real 0.03 s -attempt 2: PASS, real 0.02 s -``` - -The `erofs` malloc row was `3` active allocations and `768` active bytes both -before and after. The cumulative allocation counter moved from 89 to 95, as -expected for temporary request buffers; active memory did not scale with the -5 GiB hole. - -## TC153 - -The first 4096-byte fixture was Layout 2 and failed before the target namei -path. A second 65536-byte base with 400 entries was still Layout 2. The tracked -generator now converts the directory to Layout 0 by moving its complete data -to appended blocks and produced a 90112-byte patched image with raw block 16, -22 image blocks, and the hash listed above. - -The corrected fixture was not executed in the FreeBSD kernel during this run. -TC153 remains SHELVED, and the complete attempt history and acceptance criteria -are in its issue document. - -## Cleanup - -The qualified rerun used dmesg line count 123 before and after. It ended with: - -```text -EROFS mounts: 0 -md providers: none -EROFS modules: 0 -``` - -An earlier unqualified probe encountered a preloaded differently named EROFS -KLD and a `truss` exit-status ambiguity. It was discarded. The qualified run -first unloaded that KLD, loaded the exact module hash above, captured direct -command exit codes separately from syscall traces, and then cleaned up. - -## Deferred Issues - -- `issues/TC153-large-directory-block-index-validation.md` -- `issues/TC010-48bit-statfs-large-provider.md` -- `issues/extent-metadata-fixture-unavailable.md` - -The raw guest transcripts remain untracked under `/work/build`. This report, -the deterministic generator, and source/test documentation are the tracked -evidence. diff --git a/tests/results/manual/2026-08-09T0124Z-final-review/prepare-fixtures.sh b/tests/results/manual/2026-08-09T0124Z-final-review/prepare-fixtures.sh deleted file mode 100755 index dee2f81..0000000 --- a/tests/results/manual/2026-08-09T0124Z-final-review/prepare-fixtures.sh +++ /dev/null @@ -1,271 +0,0 @@ -#!/bin/sh -set -eu - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -fixture_dir=${FIXTURE_DIR:-"$script_dir/fixture"} -artifact_dir=${ARTIFACT_DIR:-"$script_dir/artifacts"} - -for tool in mkfs.erofs python3 sha256sum; do - command -v "$tool" >/dev/null 2>&1 || { - echo "missing tool: $tool" >&2 - exit 1 - } -done -test ! -e "$fixture_dir" || { - echo "fixture directory already exists: $fixture_dir" >&2 - exit 1 -} -test ! -e "$artifact_dir" || { - echo "artifact directory already exists: $artifact_dir" >&2 - exit 1 -} - -mkdir -p "$fixture_dir/fallback" "$fixture_dir/oversize" \ - "$fixture_dir/extent" "$fixture_dir/large/huge" "$artifact_dir" - -FIXTURE_DIR="$fixture_dir" python3 <<'PY' -from pathlib import Path -import os - -root = Path(os.environ["FIXTURE_DIR"]) -(root / "fallback" / "root.txt").write_text( - "48-bit fallback root\n", encoding="ascii") -(root / "oversize" / "big.dat").write_bytes(b"x") -(root / "extent" / "hole.dat").write_bytes(b"\0" * (1024 * 1024)) -(root / "large" / "huge" / "anchor.txt").write_text( - "large directory anchor\n", encoding="ascii") -for index in range(400): - (root / "large" / "huge" / - f"entry-{index:03d}-abcdefghijklmnopqrstuvwxyz.txt").write_text( - f"entry {index:03d}\n", encoding="ascii") -PY -find "$fixture_dir" -exec touch -h -t 197001010000.00 {} + - -build_image() -{ - uuid=$1 - image=$2 - source=$3 - shift 3 - mkfs.erofs -d0 -x-1 -T0 --all-time --all-root --workers=1 \ - -U "$uuid" "$@" "$artifact_dir/$image" "$fixture_dir/$source" -} - -build_image 22222222-3333-4444-5555-666666666650 \ - fallback-base.erofs fallback -E force-inode-compact -build_image 22222222-3333-4444-5555-666666666651 \ - oversize-base.erofs oversize -E force-inode-extended -build_image 22222222-3333-4444-5555-666666666652 \ - extent-base.erofs extent -E legacy-compress,force-inode-extended -z lz4 -build_image 22222222-3333-4444-5555-666666666653 \ - large-dir-base.erofs large -E force-inode-extended - -ARTIFACT_DIR="$artifact_dir" python3 <<'PY' -from pathlib import Path -import hashlib -import os -import struct - -artifact_dir = Path(os.environ["ARTIFACT_DIR"]) -SUPER = 1024 -FEATURE_INCOMPAT = SUPER + 80 -ROOTNID_2B = SUPER + 14 -BLOCKS_LO = SUPER + 36 -META_BLKADDR = SUPER + 40 -ROOTNID_8B = SUPER + 112 -EROFS_FEATURE_INCOMPAT_48BIT = 0x80 -EROFS_INODE_COMPRESSED_FULL = 1 -Z_EROFS_ADVISE_EXTENTS = 0x1 -Z_EROFS_EXTENT_RECSZ_16 = 0x4 -HOLE_SIZE = 5 * 1024 * 1024 * 1024 + 4096 -HUGE_DIR_BLOCKS = (1 << 31) + 1 - - -def u16(image, offset): - return struct.unpack_from("> 1) ^ ( - polynomial if checksum & 1 else 0) - put_u32(image, SUPER + 4, checksum & 0xFFFFFFFF) - - -def inode_offset(image, nid): - block_bits = image[SUPER + 12] - return (u32(image, META_BLKADDR) << block_bits) + (nid << 5) - - -def inode_info(image, nid): - offset = inode_offset(image, nid) - inode_format = u16(image, offset) - inode_size = 64 if inode_format & 1 else 32 - size = u64(image, offset + 8) if inode_size == 64 else u32( - image, offset + 8) - xattr_count = u16(image, offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) - layout = (inode_format >> 1) & 7 - return offset, inode_format, inode_size, xattr_size, layout, size - - -def directory_entries(image, nid): - offset, _, inode_size, xattr_size, layout, size = inode_info(image, nid) - block_size = 1 << image[SUPER + 12] - assert 0 < size <= block_size - if layout == 2: - data = offset + inode_size + xattr_size - elif layout == 0: - data = u32(image, offset + 16) << image[SUPER + 12] - else: - raise AssertionError(f"unsupported directory layout {layout}") - first_nameoff = u16(image, data + 8) - assert first_nameoff >= 12 and first_nameoff % 12 == 0 - count = first_nameoff // 12 - entries = [] - for index in range(count): - entry = data + index * 12 - child_nid = u64(image, entry) - nameoff = u16(image, entry + 8) - endoff = (u16(image, entry + 20) - if index + 1 < count else size) - name = bytes(image[data + nameoff:data + endoff]).split(b"\0", 1)[0] - assert name - entries.append((name, child_nid)) - return entries - - -def child_nid(image, parent_nid, name): - return next(nid for entry_name, nid in directory_entries(image, parent_nid) - if entry_name == name) - - -def convert_inline_directory_to_plain(image, nid): - offset, inode_format, inode_size, xattr_size, layout, size = inode_info( - image, nid) - assert layout == 2 and inode_size == 64 and size > 0 - block_size = 1 << image[SUPER + 12] - tail_size = size % block_size - assert tail_size > 0 - raw_block = u32(image, offset + 16) - full_size = size - tail_size - inline_offset = offset + inode_size + xattr_size - directory_data = bytes( - image[raw_block * block_size:raw_block * block_size + full_size] + - image[inline_offset:inline_offset + tail_size]) - assert len(directory_data) == size - - new_raw_block = (len(image) + block_size - 1) // block_size - image.extend(b"\0" * (new_raw_block * block_size - len(image))) - image.extend(directory_data) - image.extend(b"\0" * (-len(image) % block_size)) - put_u16(image, offset, inode_format & ~(7 << 1)) - put_u32(image, offset + 16, new_raw_block) - put_u32(image, BLOCKS_LO, len(image) // block_size) - assert inode_info(image, nid)[4] == 0 - return new_raw_block, len(image) // block_size - - -def write_image(name, image): - update_superblock_checksum(image) - (artifact_dir / name).write_bytes(image) - - -evidence = [] - -fallback = bytearray((artifact_dir / "fallback-base.erofs").read_bytes()) -fallback_root = u16(fallback, ROOTNID_2B) -fallback_blocks = u32(fallback, BLOCKS_LO) -assert fallback_root != 0 and u64(fallback, ROOTNID_8B) == 0 -put_u32(fallback, FEATURE_INCOMPAT, - u32(fallback, FEATURE_INCOMPAT) | EROFS_FEATURE_INCOMPAT_48BIT) -put_u64(fallback, ROOTNID_8B, 0) -write_image("fallback-48bit-root2.erofs", fallback) -evidence.append( - f"fallback rootnid_2b={fallback_root} rootnid_8b=0 " - f"blocks_lo={fallback_blocks} union=0x{u16(fallback, ROOTNID_2B):04x}") - -oversize = bytearray((artifact_dir / "oversize-base.erofs").read_bytes()) -oversize_root = u16(oversize, ROOTNID_2B) -oversize_nid = child_nid(oversize, oversize_root, b"big.dat") -oversize_off, oversize_format, oversize_isize, _, _, _ = inode_info( - oversize, oversize_nid) -assert oversize_format & 1 and oversize_isize == 64 -put_u64(oversize, oversize_off + 8, 1 << 63) -write_image("extended-size-bit63.erofs", oversize) -evidence.append( - f"oversize nid={oversize_nid} inode_off={oversize_off} " - f"i_size=0x{u64(oversize, oversize_off + 8):016x}") - -extent = bytearray((artifact_dir / "extent-base.erofs").read_bytes()) -extent_root = u16(extent, ROOTNID_2B) -extent_nid = child_nid(extent, extent_root, b"hole.dat") -extent_off, extent_format, extent_isize, extent_xattr, extent_layout, _ = \ - inode_info(extent, extent_nid) -assert extent_format & 1 and extent_isize == 64 -assert extent_layout == EROFS_INODE_COMPRESSED_FULL -header = (extent_off + extent_isize + extent_xattr + 7) & ~7 -record = (header + 8 + 15) & ~15 -root_off = inode_offset(extent, extent_root) -assert not (header < root_off + 64 and root_off < record + 16) -put_u64(extent, extent_off + 8, HOLE_SIZE) -struct.pack_into(" 123`), zero -EROFS mounts, no md provider, and zero EROFS modules. Raw transcripts, build -objects, modules, and generated images remain outside the repository under -`/work/build` and guest `/tmp` only. diff --git a/tests/results/manual/2026-08-09T0710Z-g1/manual-test-report.md b/tests/results/manual/2026-08-09T0710Z-g1/manual-test-report.md deleted file mode 100644 index 412fe15..0000000 --- a/tests/results/manual/2026-08-09T0710Z-g1/manual-test-report.md +++ /dev/null @@ -1,159 +0,0 @@ -# repo22 G1 Full Manual Regression Report - -Execution window: 2026-08-09 06:30-07:10 UTC - -Code baseline: `cd0e985b5ac54a4b7acb7042422329ad1729fb3e` - -Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG, FreeBSD clang 19.1.7 - -## Scope and Method - -This run covers exactly these 32 test cases and no others: - -`TC001, TC007, TC009, TC011-TC014, TC019-TC040, TC147, TC150, TC151` - -Every test was executed directly from its Markdown procedure. No CI, runner, -or wrapper supplied a result, and no historical run was accepted as current -evidence. Each test used a fresh dynamically allocated md provider and was -cleaned before the next test. - -Set audit: requested 32, selected 32, unique 32, duplicate 0, missing 0, -extra 0. - -## Exact Build - -The source archive exported from the code baseline had SHA256 -`5b7fd2ac32ecf791f71c74849a6e8c1775e442d1be3c7852c654d074b2c8351b`. -The module was built natively in the guest with: - -```text -FREEBSD_SRC=/tmp/repo22-freebsd15-src WITH_ZSTDIO=0 ./build.sh -``` - -The build completed with kernel `-Werror`. The module had no unresolved -`bcmp` or `ZSTD_*` symbols. - -| Item | Value | -|---|---| -| Source commit | `cd0e985b5ac54a4b7acb7042422329ad1729fb3e` | -| `WITH_ZSTDIO` | `0` | -| KLD SHA256 | `023ddf0ea2205977f5921715d2a6206b607806840180ddf470103fd1d6deba84` | -| Loaded module | KLD ID 20, size `0xb8a8`, `erofs.ko` | -| Guest userland/kernel | `15.0-RELEASE-p8` / `15.0-RELEASE-p8` | - -The later documentation-only progress commit `573aaab7ee0b47cb6aed7f76c52c68dd4041326b` -did not alter `src/`; the KLD therefore remains an exact build of the stated -code baseline. - -## Fixture Qualification - -`tests/prepare_g1_fixtures.sh` generated deterministic sources and production -erofs-utils 1.8.6 images. `tests/g1_fixtures.py` applied structured, -field-checked transforms and recalculated checksums. Independent empty output -directories `repo22-g1-repro-c.6IK7SM` and `repo22-g1-repro-d.egVQgj` produced -byte-identical source manifests, metadata, image manifests, and structured -evidence. - -| Manifest | SHA256 | -|---|---| -| `SOURCE-SHA256SUMS` | `5ce53780f455208cb1efbf2738429a1f6f5c4eb3af1663779815c6afc0fea2bc` | -| `SOURCE-METADATA` | `bd27480c238ffff441d328ab71775989611d057ffaed691dae0238f738371e46` | -| `images/IMAGE-SHA256SUMS` | `54fb256b3847acc8b1e735eacaace56c1747f65237ee5730b2b5fa3e3ab2b0f3` | -| `images/fixture-evidence.txt` | `84e8e2e595a88e034ea0a05777207d4e3430e4e764a7fbc4d4a4770defcca3db` | - -| Image | SHA256 | -|---|---| -| `compact-dot-omitted.erofs` | `bbd1d3d9f619a71ed7a874feb8c437aa9bf3e86f671abfa243a85de57e809d54` | -| `compact-nlink1.erofs` | `c9ecfc29fbd45a2c37c6422d689ac340943dbc23ee4428ffd4698ef88e0b331d` | -| `compact.erofs` | `41004ed19d58e698f277cb268b570b87203c2ba08ea4a68a72f6e3b1c7aec47a` | -| `extended-large-hole.erofs` | `7a69d791742096ddfece20c7a793d828ec7e887352d0b62f22839f66ecb26a6d` | -| `extended-size-bit63.erofs` | `9413a4197f5b2c43876e257690a5f96e82f61b2b62084e2df688144070f0825b` | -| `extended.erofs` | `3d96fd292749f1ef7ae72f8e370fc8a94231ed575fb36a6d66784a59d8c4ffa7` | -| `fallback-48bit-root2.erofs` | `8217da2ae664358e12d357524ef0bb52a897f1ed206b2353c2a2033b76d4ae79` | -| `flat.erofs` | `3785ca07e7bd16f6c611191596ae0314253c0ae9b7217a4b22de25799b0f08ac` | -| `inline-cross-block.erofs` | `354b674fd144cab93403d47ad95135968c24d41b34a8c908aa6717882243e936` | -| `inline-zero.erofs` | `c894f249ca6d4a7a16c7eb728687c6aba1f948630c4f72e20680655c0fca70c0` | -| `inline.erofs` | `1474c310a80766666c1d578174ca03972fb9e03dff8c2357a37883151d4acc47` | -| `invalid-dirent-nid.erofs` | `309d6829ebc87c22ae6ba23f07ae0b00a62c7e657c3cc2ec46fe6a64d24eea69` | -| `root8-48bit.erofs` | `2f48797586395ff7d2204d43e67d6e2aaaa04c9bdfb162c5c5e2c24dc6e88893` | - -The corrected dot-omitted directory has size 35 and on-disk names -`..,child.txt`. Its encoding was independently checked with erofs-utils 1.9.3, -including a passing `fsck.erofs`; erofs-utils 1.8.6 does not recognize the -48-bit incompat feature and was not used to claim support for that image. - -## Per-Test Results - -| Test | Result | Actual command summary and observable evidence | Cleanup | -|---|---|---|---| -| TC001 | PASS | `mdconfig`, `mount -t erofs -o ro`, `cmp`, `sha256`, `statfs_probe`; root source/mount SHA `04690aad...3105`, blocks 17, files 15, readonly 1. | mount 0, md 0 | -| TC007 | PASS | Two fresh providers and concurrent mounts; both waits returned 0, and source/mount1/mount2 `testfile` SHA was `0e5303ad...e3b1`. | mounts 0, md units 0 | -| TC009 | PASS | `statfs_probe`, `df -kT`, `df -iT`; bsize 4096, blocks 17, free 0, files 15, ffree 0, 68 KiB used, readonly 1. | mount 0, md 0 | -| TC011 | PASS | `stat`, complete top-level name comparison, root payload `cmp`; root inode 36, directory mode 0755, nlink 4, root SHA `04690aad...3105`. | mount 0, md 0 | -| TC012 | PASS | `getfh`, `fhstat`, `fhopen`, repeated handle comparison; fsid `00000063:000000e0`, NID 71, generation 2262931519, data SHA `0e5303ad...e3b1`. | mount 0, md 0, outputs 0 | -| TC013 | PASS | Patched dirent NID 59 to 3200 with valid CRC; two `read_probe expect-error` calls returned errno 97 and truss showed `fstatat ERR#97`; dmesg `123->123`. | mount 0, md 0, outputs 0 | -| TC014 | PASS | Structured superblock inspect plus mount/statfs; magic `0xe0f5e1e2`, block bits 12, root NID 36, blocks 17, inodes 15, dmesg `123->123`. | mount 0, md 0 | -| TC019 | PASS | Mounted actual `feature=0x80 rootnid_8b=36 blocks_hi=0` image; root inode 36, blocks 17, root SHA `04690aad...3105`. | mount 0, md 0 | -| TC020 | PASS | `stat`, `cmp`, `sha256` on compact `small.txt`; NID 65, size 22, nlink 1, blocks 8, SHA `6e0d152a...`. | mount 0, md 0 | -| TC021 | PASS | Field evidence `i_format=0x14 i_nb=0x1234`; mounted NID 63 reported nlink 1 and source-exact SHA `ff151c...`. | mount 0, md 0 | -| TC022 | PASS | Structured raw rdev `0x543abc21`; native special-node probe reported char/block major 2748, minor 344865, while FIFO rdev was `NODEV`. | mount 0, md 0 | -| TC023 | PASS | Extended inode NID 45, 64-byte inode, size 2097883; `stat`, full `cmp`, SHA `eed000b0...` all matched source. | mount 0, md 0 | -| TC024 | PASS | Size 4294971393 sparse fixture; three 65536-byte reads at offsets 0, 2147483648, and 4294905857 matched deterministic zero source SHA `de2f2560...`. | mount 0, md 0, outputs 0 | -| TC025 | PASS | Baseline and patched mounts checked `single` NID 63/nlink 1, `hard-a` and `hard-b` NID 57/nlink 2, and `dotdir` NID 53/nlink 2; all source comparisons passed. | mounts 0, md units 0 | -| TC026 | PASS | Valid bit-4 fixture evidence: size 35, on-disk `..,child.txt`, root8 36; listing synthesized `.`, retained `..`, and child SHA matched `de256820...41f0`. | mount 0, md 0 | -| TC027 | PASS | Host `dump.erofs --ls --path=/dotdir` showed explicit `.,..,child.txt`; guest listing and child `cmp` matched SHA `de256820...41f0`. | mount 0, md 0 | -| TC028 | PASS | `flat.erofs` layout 0 small file; size 25, blocks 8, source/mount SHA `bb9c1e27...f486`. | mount 0, md 0 | -| TC029 | PASS | Full medium-file `cmp` SHA `471108a5...12e6`; `pread` offset 40960 length 20480 matched independent range SHA `72142972...5fb`. | mount 0, md 0, output 0 | -| TC030 | PASS | 10485791-byte full `dd`/`cmp` SHA `0b39b8a4...c56e`; 1 MiB read at 5 MiB matched SHA `c26bb16b...ba5c`. | mount 0, md 0, outputs 0 | -| TC031 | PASS | Inline `tiny.txt` evidence layout 2; size 12, blocks 8, source/mount SHA `863f7088...73f38`. | mount 0, md 0 | -| TC032 | PASS | Explicit layout-2 zero-length inode; size 0, blocks 0, EOF clean, both hashes `e3b0c442...b855`. | mount 0, md 0 | -| TC033 | PASS | 5000-byte tailpacked file full SHA `e4c7fbec...b2a6`; 64-byte range crossing offset 4096 matched SHA `6ff6b010...c0e`. | mount 0, md 0, outputs 0 | -| TC034 | PASS | Evidence layouts for 4095/4096/4097 were 0/0/2; full SHAs `5255d8c8...`, `677802c8...`, `eda39df0...`; nine-byte boundary range SHA `9e67b4e5...182c`. | mount 0, md 0, outputs 0 | -| TC035 | PASS | Sequential 4096-byte `dd` to exact EOF at size 262163; source/output SHA `056f8f75...3433`. | mount 0, md 0, output 0 | -| TC036 | PASS | Independent `pread` start/mid/end ranges returned 10240/10240/24576 bytes at exact offsets; SHAs `5b6a742e...d858`, `29fa7799...849`, `6477f5a2...163d`. | mount 0, md 0, outputs 0 | -| TC037 | PASS | 104857857-byte full `dd` took 61.44 s and matched SHA `1c8daa29...c661`; 5 MiB sample at 50 MiB matched SHA `f1b60e95...25c7`. | mount 0, md 0, outputs 0 | -| TC038 | PASS | `readlink` returned exact `target.txt`; followed target source/mount SHA `887f519c...8ea3`. | mount 0, md 0, outputs 0 | -| TC039 | PASS | Evidence size 73/layout 2; source and mounted long targets matched, output was 74 bytes including newline, followed file SHA `06539545...0f42`. | mount 0, md 0, outputs 0 | -| TC040 | PASS | Source/mounted target was `/nonexistent/repo22-g1-target`; two follow attempts returned exact errno 2 at open. | mount 0, md 0, outputs 0 | -| TC147 | PASS | Positive inline file SHA `0347f272...b01`; checksum-valid cross-block image returned errno 97 twice and truss `fstatat ERR#97`; dmesg `123->123`. | mounts 0, md units 0, outputs 0 | -| TC150 | PASS | Current-KLD rerun with `feature=0x80 rootnid_2b=36 rootnid_8b=0 blocks_lo=17`; root inode 36, root SHA `04690aad...3105`, statfs blocks 17. | mount 0, md 0 | -| TC151 | PASS | Current-KLD rerun: two direct opens returned errno 97, truss showed `fstatat ERR#97`, both mmap helpers exited 1 at open with integrity failure; dmesg `123->123`. | mount 0, md 0, outputs 0 | - -## Outcome - -| Result | Count | -|---|---:| -| PASS | 32 | -| KERNEL-FAIL | 0 | -| SHELVED/ISSUE | 0 | -| ENVIRONMENT-UNAVAILABLE | 0 | - -No G1 issue file was created. Existing issue files concern other groups or -previously documented limitations and were not changed by this run. - -## Discarded Attempts - -- TC013 initially asserted a nonportable `stat(1)` wrapper exit status. The - qualified run used two direct syscall probes and truss errno instead. -- TC026 first exposed an invalid transformer assumption: a NUL had been added - after non-trailing `..`. That image was discarded, the transformer was fixed, - independent 1.9.3 validation and two-build reproduction passed, and TC026 was - rerun with the corrected image. -- TC037's first long-running remote stream did not preserve its final output. - It was not counted; the complete qualified rerun retained elapsed time, both - hashes, and cleanup evidence. -- TC151 had two pre-mount/post-evidence command assertions with incorrect grep - keys. Neither was counted; the complete qualified rerun used syscall errno and - per-output integrity counts. - -## Final Cleanup - -Before final unload, `kldstat` showed ID 20 and the exact KLD SHA256 above. -After `kldunload erofs`, the guest reported: - -```text -erofs_mounts=0 md_units=0 kld_present=0 -``` - -Generated fixtures, build objects, and native probes remain only in `/work/build` -and guest `/tmp`; none is staged for the repository. diff --git a/tests/results/manual/2026-08-09T0710Z-g2/manual-test-report.md b/tests/results/manual/2026-08-09T0710Z-g2/manual-test-report.md deleted file mode 100644 index e5b6128..0000000 --- a/tests/results/manual/2026-08-09T0710Z-g2/manual-test-report.md +++ /dev/null @@ -1,269 +0,0 @@ -# repo22 G2 Full Manual Regression Report - -Started: 2026-08-09 06:07 UTC - -Completed: 2026-08-09 07:10 UTC - -Baseline: `cd0e985b5ac54a4b7acb7042422329ad1729fb3e` - -Branch: `manual-g2-20260809T060709Z` - -Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG, isolated port 9223 - -Overlay: `/work/build/repo22-manual-g2-20260809T060709Z-freebsd15-overlay.qcow2` - -## Scope and Result - -The exact assigned set was executed manually from the numbered Markdown -procedures. No CI, runner, result wrapper, or unassigned TC was used. - -| Test | Result | Primary evidence | -|---|---|---| -| TC002 | PASS | Real CRC32C ranges; valid mount; bad covered byte `ERR#97` | -| TC008 | PASS | Bad magic `ERR#22`; 1023-byte provider `ERR#6`; empty md `EINVAL` | -| TC010 | PASS | Qualified 16 TiB sparse provider; exact 64-bit `df` total | -| TC015 | PASS | Bad block size/root/feature returned 22/97/45 | -| TC016 | PASS | Checksum-field-only mutation returned `ERR#97` | -| TC017 | PASS | Raw union fields; short `ERR#6`; qualified sparse mount | -| TC018 | PASS | Exact read at physical offset 17592191561728 | -| TC112 | PASS | Bad magic field returned `ERR#22` before root load | -| TC113 | PASS | Resolved inode `openat` returned `ERR#45` | -| TC114 | PASS | OOB FLAT_PLAIN `read` returned `ERR#97` | -| TC115 | PASS | Future algorithm bit returned `ERR#45` | -| TC116 | PASS | FBT decompressor entry count 1; `read` returned `ERR#5` | -| TC119 | PASS | FLAT_PLAIN changed byte/hash returned without kernel error | - -Count: 13 assigned, 13 PASS, 0 KERNEL-FAIL, 0 SHELVED, 0 ENV, -0 duplicate, 0 omitted, 0 extra. - -## Build and Guest - -Both configurations were built natively in the isolated guest with kernel -`-Werror`, `FREEBSD_SRC=/tmp/repo22-freebsd15-src`, and FreeBSD source -`15.0-RELEASE-p9`. The running guest was `15.0-RELEASE-p8`. - -| Configuration | KLD SHA256 | Result | -|---|---|---| -| `WITH_ZSTDIO=0` | `482e9c072c949f0459c422aa1efd9b35a2d99cf90b25bb01e59e00b440b6d0fe` | PASS | -| `WITH_ZSTDIO=1` | `8349c97c7ced253fff32a9e706f29313f309cb63a270e4e39a4501426f2b95c6` | PASS | - -The ZSTDIO KLD was loaded as ID 5 and was the only repo22 test KLD. It had the -expected FreeBSD `ZSTD_*` references and no unresolved `bcmp`. The disabled KLD -had neither `ZSTD_*` nor `bcmp` unresolved symbols. - -## Fixture Qualification - -`tests/prepare_error_fixtures.sh` ran twice in independent empty host -directories with erofs-utils 1.8.6. Both `SHA256SUMS` files were identical and -self-verified. Every field mutation records its exact field/path and old/new -value. Same-size corruption variants preserve provider length; short and empty -providers record their intentionally different lengths. - -The helper independently proved the 4096-byte-block checksum forms: - -```text -canonical: field 1028 cleared, CRC32C [1024,4096), initial 0xffffffff -kernel: CRC32C [1032,4096), seed 0x5045b54a -valid control: stored=canonical=kernel=0xf7429681 -``` - -Raw transcripts, build logs, KLDs, guest logs, fixture trees, and the overlay -remain untracked under `/work/build/repo22-manual-g2-20260809T060709Z*`. - -## TC002 - -Result: **PASS**. - -Commands: helper `inspect`; `sha256`; dynamic `mdconfig`; valid `mount`/`cmp`; -direct negative `mount`; independent `truss`. - -Hashes: valid `eb97860671931c76a171d13caa70ddc2c9731c6cdef1e99b9a3deb580baf70ae`; -bad CRC `3f684e8cb03e3cb20a1920dccf23d9b7b19bc09d5745be04608e195c541ebf73`. -Both were 7622656 bytes. The corruption changed byte 1088 without recomputing -CRC; calculated checksum became `0x202160ab` while stored remained `0xf7429681`. -Valid data matched; bad direct mount rc was 1 and `nmount` returned errno 97. -Cleanup: mount absent and exact md detached. - -## TC008 - -Result: **PASS**. - -Commands: `wc -c`, `sha256`, dynamic `mdconfig`, direct `mount`, and `truss` -for each attachable provider. - -Hashes: bad magic `d1bc3aec03c58dae1b74b54d895c0a72e5b3088f05907ab2f6a0cb8dc38a615f`; -1023-byte provider `5724796860baa23469b3118eff2567c96a0b64b7dadaf99eafb7ba3c65b9aa56`; -empty provider `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`. -Bad magic returned errno 22. The 1023-byte md attached and mount returned errno -6. The zero-byte provider was rejected by `mdconfig` with `EINVAL`. Cleanup: -zero mounts and md units after all three subcases. - -## TC010 - -Result: **PASS**, not SHELVED. - -Commands: copy qualified prefix, `truncate -s 17592193667072`, `stat`, dynamic -`mdconfig`, `diskinfo`, `mount`, `df -kT`, `df -iT`, and `cmp`. - -Prefix hash: `d3ffadc6fc9e73f85a8a5973b4ac5ee2a2da57b4e8baeccd259fa4325a2146cf`. -Fields were `blocks_hi=1`, `blocks_lo=1861`, `rootnid_8b=36`, and -`total_blocks=4294969157`. The sparse file allocated 15232 UFS blocks while -GEOM reported all 17592193667072 bytes. `df -k` total and Used were exactly -17179876628; Avail was zero. Cleanup: unmounted, md detached, sparse provider -removed. The former environmental issue is now RESOLVED. - -## TC015 - -Result: **PASS**. - -Commands: evidence/hash/length checks; three dynamic md attachments; direct -mount status followed by independent `truss` errno capture. - -Hashes: bad block size `b5c52c44ace6f693e39161102acde157efec2fce34eb4fef198681dfc7942b58`; -bad root `9cdd61eb99b2e7f45d2505b3b95d6427f8f636e117b5ffabed064a3f83997d55`; -unknown feature `95d19a490c38657a8c579a10f1f12088ab43c1819aa9b37ae86b841bb26a592a`. -All were 7622656 bytes with recomputed valid CRC. Direct mount rc was 1 for -each; `nmount` returned 22, 97, and 45 respectively. Cleanup: zero mounts/md. - -## TC016 - -Result: **PASS**. - -Commands: field evidence, `sha256`, `wc -c`, direct mount, and `truss`. - -Hash: `04249f6b7d9b7322c171f1727408130d23cb0b83709aad7a4b77b39358a8dfe6`. -The 7622656-byte image changed only checksum `0xf7429681 -> 0xf7429680`. -Direct mount rc was 1 and `nmount` returned errno 97. Cleanup: no mount/md. - -## TC017 - -Result: **PASS** with both negative guard and positive dynamic coverage. - -Commands: raw `od` at offsets 1038/1060/1136; small-prefix mount; sparse -provider creation; `diskinfo`; positive mount; `cmp`; `df -kT`. - -Hash: `d3ffadc6fc9e73f85a8a5973b4ac5ee2a2da57b4e8baeccd259fa4325a2146cf`. -Raw values were high 1, low 1861, root 36. The short prefix returned errno 6. -The qualified provider mounted and reported exactly 17179876628 one-KiB -blocks. Cleanup: unmounted, detached, sparse provider removed. - -## TC018 - -Result: **PASS** with real I/O above 16 TiB. - -Commands: sparse provider creation; high-offset `dd`; two raw `read_probe` -calls; low-decoy and high-source `cmp`/hash; dynamic md mount and mounted hash. - -Prefix hash: `cfd86ed1e39528d958e32819282c7fd87191f80a25d9575078dad473f40563a8`. -The inode encoded `startblk_hi=1`, low 1347, combined block 4294968643, and -physical offset 17592191561728. The original low payload at 5517312 was zeroed: -its hash was `6a4875ddaceaa91fb3369f0f6d962f77442daf1b1d97733457d12bcabdf79441`. -Raw high and mounted hashes both matched source -`25eb31e024bc60745d68a5f7318753951804c7858b576cb698a4260af4a334a3`. -Cleanup: unmounted, detached, sparse provider and probe outputs removed. - -## TC112 - -Result: **PASS**. - -Commands: raw magic `od`, hash/length, direct mount, and `truss`. - -Hash: `d1bc3aec03c58dae1b74b54d895c0a72e5b3088f05907ab2f6a0cb8dc38a615f`. -Magic was `0x21444142`; provider length remained 7622656. The canonical CRC was -invalid as required after changing magic, while the unchanged production -suffix checksum still matched `0xf7429681`. Direct rc was 1 and `nmount` -returned errno 22. Cleanup: no mount/md. - -## TC113 - -Result: **PASS**. - -Commands: structured path inspection; mount/control `cmp`; direct and trussed -`read_probe expect-error ... 45`; second control `cmp`. - -Hash: `28d972c4065927326607fcd51dabff81cc88a28e2343f6056e55d04418172814`. -Resolved NID 52 at inode offset 1664 changed format `0x0001 -> 0x8001` with -valid CRC and equal provider length. `openat` returned errno 45; unaffected data -remained exact. Cleanup: clean unmount and md detach. - -## TC114 - -Result: **PASS**. - -Commands: path inspection; mount/control `cmp`; direct and trussed -`read_probe expect-error ... 97`; second control `cmp`. - -Hash: `2afcb9dde4c8eaf30bb2541d00be36f726f5cf151d887e0d999803b3993285ed`. -NID 54 field offset 1744 changed start block `1349 -> 1861`, equal to declared -blocks, with valid CRC and equal media size. Target `read` returned errno 97; -control remained exact. Cleanup: clean unmount and md detach. - -## TC115 - -Result: **PASS**. - -Commands: raw field `od`, hash/length, direct mount, and `truss`. - -Hash: `a8d7d9cf0abcf34a5cd2366059c0294e08fdd20d02ed3b5b21005ddf78c69029`. -`available_compr_algs` at byte 1106 changed `0x0000 -> 0x8000`; CRC was -recomputed and media remained 716800 bytes. Direct rc was 1 and `nmount` -returned errno 45. Cleanup: no mount/md. - -## TC116 - -Result: **PASS** with direct decompressor-path proof. - -Commands: parsed `dump.erofs -e` evidence; failed userspace extraction record; -mount/control `cmp`; FBT enumeration and count; direct/trussed `read_probe`. - -Hash: `f442785ea7cdbdc3f64b2ff89ea04942460cbdc5bea5615cbed237029edda8d8`. -The equal-length 716800-byte image changed 64 bytes at 4128, wholly inside the -parsed first physical extent `[4096,69632)`. Mount succeeded, excluding the -media-size precheck. `fbt:erofs-zstdio:z_erofs_decompress:entry` counted one -call for the target command, and `read` returned errno 5. Control remained -exact. Cleanup: unmounted/detached; `dtraceall` and dependencies unloaded. - -## TC119 - -Result: **PASS** with the revised real integrity semantics. - -Commands: path/field evidence; image/source hashes; mount; complete mounted -hash; two one-byte `read_probe` calls; byte compare; control `cmp`. - -Hash: `74b54f998481f9e55b30f2393ee780e3b46144157a54b4dc0fc0c044c160bb5a`. -The equal-length image flipped `/plain.bin` file offset 257 at provider byte -5525761 from `0x00` to `0x80`; the valid superblock CRC remained unchanged. -Source hash was `5647f05ec18958947d32874eeb788fa396a05d0bab7c1b71f112ceb7e9b31eee`; -mounted hash was `e98bb2acaecb24ab86112478bc8cc1e3668c127b449fe2e664dc4804f9477d81`. -Reads succeeded, the target byte differed, and control matched. Cleanup: clean -unmount and md detach. - -## Harness Notes - -Four incomplete attempts were discarded and are not PASS evidence: - -1. TC002 repeated host Python inspection in a guest without Python and exited - before md attachment. -2. TC002 initially trusted `truss` wrapper status; the qualified run separated - direct mount rc from syscall tracing because FreeBSD `truss -o` returned 0 - around a failing mount. -3. TC008 had a stray patch marker after its second subcase; trap cleanup ran and - the complete three-subcase command was repeated. -4. TC010 initially asserted the `df` Used column was free blocks; the qualified - run used the correct `f_bfree=0` interpretation and repeated all steps. - -## Final Cleanup - -Per-TC cleanup ended with zero EROFS mounts and zero md providers. Global dmesg -changed from 102 to 104 lines; both new lines were the expected -`md0: truncating fractional last sector by 511 bytes` from TC008. New panic, -fatal trap, page fault, general-protection fault, and double-fault count was -zero. The exact test KLD unloaded successfully, leaving zero EROFS modules. -The guest shut down normally and QEMU PID 285696 exited after 12 seconds. Port -9223 no longer has a VM process. - -## Issues - -No kernel issue was found and no TC remains SHELVED. The former TC010 provider -issue is retained as a resolved record in -`issues/TC010-48bit-statfs-large-provider.md`. diff --git a/tests/results/manual/2026-08-09T0839Z-g4/manual-test-report.md b/tests/results/manual/2026-08-09T0839Z-g4/manual-test-report.md deleted file mode 100644 index acc479a..0000000 --- a/tests/results/manual/2026-08-09T0839Z-g4/manual-test-report.md +++ /dev/null @@ -1,132 +0,0 @@ -# repo22 G4 full manual regression report - -## Scope and result - -- Exact scope: TC005, TC067-TC083, TC117, TC134-TC140, TC142. -- Count check: 27 requested, 27 executed, 0 duplicate, 0 omitted. -- Result: **PASS 27, FAIL 0, SHELVE 0, NOT RUN 0**. -- Kernel source baseline: `9ae22009f23a65320730072a780998e80aa9b728`. -- No repo22 kernel source was changed during this regression. -- No kernel failure was found, so no new `issues/` entry was required. - -## Isolated environment - -- Worktree: `/work/build/repo22-manual-g4-20260809T072432Z`. -- Branch: `manual-g4-20260809T072432Z`. -- Dedicated qcow2 overlay backed by the FreeBSD 15 development base. -- Dedicated SSH forward: `127.0.0.1:9224`; port 9222 was not used. -- Guest: FreeBSD `15.0-RELEASE-p8`, amd64, GENERIC, - `releng/15.0-n281036-53054229dcb3`. -- Final guest cleanup: zero matching mounts, zero md units, EROFS malloc - active count 0, KLD unloaded, guest responsive, QEMU stopped. - -## Exact KLD build - -The module used only `repo-community/repo22/src` from the baseline worktree. -It was built for `x86_64-unknown-freebsd15` with all 14 Makefile source files, -FreeBSD `15.0 RELEASE-p9` headers, clang/LLD 19.1.7, and `WITH_ZSTDIO=0`. -The guest is p8; both header and guest are the same FreeBSD 15 release KBI. - -```text -erofs.ko SHA256 = ee0b2e7c4ebf3602de8be8d47ddcfd021bc90a5265e7e9bf449bb9de2fbf4f04 -guest kldstat = erofs.ko, module erofs, id 507 -``` - -## Fixture provenance - -Installed `mkfs.erofs`, `dump.erofs`, and `fsck.erofs` reported erofs-utils -1.8.6. `tests/g4_fixtures.py` created both source trees without consuming old -images, ran four deterministic mkfs commands, transformed structured fields, -and reopened every result. A second fresh output directory produced identical -source and image checksum files. - -```text -source inventory SHA256 = c48f72777b0a04514fcfc7ea9c4ed77089ab0ed4537eac9cb7430e1acafe3d0a -metabox payload SHA256 = 50c5740b5fe5630ee919022d7ee1ce59d3e7d089c9d66e2baf82bf9de8233d40 -``` - -| Image | SHA256 | -|---|---| -| `basic.erofs` | `b375a3ca60de64b2c635b3ec45c9285c840e0a05bec105d5d8f68308300e7670` | -| `prefix-primary.erofs` | `49fc472a26412d154df729d8b0079801b35e97af2eb7fc3fa226dc65ee1963f9` | -| `metabox-plain.erofs` | `dd7b04097d1bfe283b65c95d759acd2176beb7cb0f1fe9d5ddbc0694b5acba9d` | -| `metabox-compressed.erofs` | `682b9dc1e0b492d935c03deba2ab644c1f0bcbf8e1ec623c15fb4990d25fe584` | -| `metabox-fragment.erofs` | `c93f4c281ed621519ced42a45cdb21b543047b7058dab31ad0def851cc19f4f1` | -| `bad-inline-entry.erofs` | `f18693c880bac0a273f2497cb0f81c0c52e19edc3ad45354eb38a39624adafd2` | -| `bad-shared-entry.erofs` | `875c27df56de2336fded3ef1db1c9fdc035db618096aa9269bfe70ea6f59c664` | -| `bad-shared-declared-bounds.erofs` | `19fc69cbe21051de2cf1cc19454cf64cc4a0ece444c6607956e066c2d4d332b3` | -| `bad-prefix-declared-bounds.erofs` | `d24b7a171aa28c3a46e3893749a4d08ab5d467c103f47aefcc0ca0f4e153d365` | -| `bad-metabox-truncated-extension.erofs` | `0bad23b1ad77eb8f2ffee83f89b7ef91b1c49ba2a20661adca21cd9efce0911e` | -| `bad-ishare-prefix-id.erofs` | `10357f28a11c2a8bbc329e01a414248415719ef088734a6ad75b934c0f65fc43` | -| `bad-fragment-self-loop.erofs` | `3b61228041591716c650a6613de30b2d9abe8b01148a9ae513b7de4d9b958d78` | -| `bad-fragment-range.erofs` | `432618f98a6e32351448120d8c694a4f31b45cb3078e08dad39b85fa01b47039` | -| `bad-metabox-recursive-nid.erofs` | `ab28a5478f569eb8a59e59e94448a85918885fcd328dadc2557ba6222dcd5aaf` | -| `bad-packed-recursive-nid.erofs` | `ad9a0ed1598c8a8e16911190c9042d58315fbcdc23af824ea106af55ffe6f2f4` | - -The fragment positive self-check found carrier NID 46, compressed-full layout, -size 32768, fragment header `0x8000000000000064` at image offset 9696, -fragment offset 100, packed NID 48, and packed size 32868. Thus -`100 + 32768 == 32868`. Shared entries were at metabox offsets 4096, 4128, -and 4156; prefix records were at 8192 and 8224 with base indexes 1 and 4. - -## FreeBSD namespace semantics - -Linux `user.*` was queried through FreeBSD namespace `user` with the leading -`user.` removed. Linux `trusted.*`, `security.*`, and ACL indexes were queried -through namespace `system`. Trusted/security list names retained their full -prefixes; ACL names were `posix_acl_access`/`posix_acl_default`. This is the -FreeBSD extattr ABI and intentionally does not copy Linux `getfattr` commands. - -`getextattr` does not reliably communicate every kernel failure through its -process status, so errno evidence came from guest `truss`: `ENOATTR=87`, -`EINTEGRITY=97`, and `EROFS=30`. Every corruption in this scope returned -`EINTEGRITY`; no case required normalization to `EIO=5`. - -## Per-TC results - -| TC | Guest evidence | Result | -|---|---|---| -| TC005 | inline user list; exact NUL-containing value; miss `ENOATTR 87` | PASS | -| TC067 | shared `shared_key` exact on two files; inline `local` exact | PASS | -| TC068 | missing name and wrong namespace both `ENOATTR 87` | PASS | -| TC069 | three shared names once; text, NUL, and binary values exact | PASS | -| TC070 | shared `trusted.config` exact; unprivileged system access denied | PASS | -| TC071 | shared `security.selinux` exact; miss `ENOATTR 87` | PASS | -| TC072 | system lists trusted, security, ACL names; user subset empty | PASS | -| TC073 | plain metabox mounted; bit-63 inodes, file hashes, xattrs exact | PASS | -| TC074 | three inline user values exact, including `00..1f` | PASS | -| TC075 | inline `trusted.admin` exact; user lookup `ENOATTR 87` | PASS | -| TC076 | capability and SELinux binary bytes exact | PASS | -| TC077 | packed user long-prefix names and values exact | PASS | -| TC078 | packed trusted long-prefix names and values exact | PASS | -| TC079 | packed NID 1201; two qualified records; both namespaces exact | PASS | -| TC080 | four repeated prefix lookups exact; miss `ENOATTR 87` | PASS | -| TC081 | metabox shared, inline, and long-prefix shared values exact | PASS | -| TC082 | raw ACL bytes and `getfacl -n` agree; user lookup `ENOATTR 87` | PASS | -| TC083 | four user attributes exact; system list empty | PASS | -| TC117 | malformed inline/shared gets both `EINTEGRITY 97`; mount stable | PASS | -| TC134 | nonzero xattr base shared value exact on two bit-63 inodes | PASS | -| TC135 | metabox, packed, and primary prefix backings all exact | PASS | -| TC136 | truncated extension and bad ishare ID mounts `EINTEGRITY 97` | PASS | -| TC137 | shared get and prefix mount bounded at declared image, errno 97 | PASS | -| TC138 | UID order 3002/2002 accepted; empty fallback; 3 negatives errno 97 | PASS | -| TC139 | FIFO stat/ACL/xattr reads; set/delete operations `EROFS 30` | PASS | -| TC140 | compressed and fragment metabox positive; 4 negatives errno 97 | PASS | -| TC142 | independent fragment rerun positive; 4 negatives errno 97 | PASS | - -## Stability and cleanup - -Positive values were compared as `getextattr -qq -x` bytes, not display text. -Negative mount and VOP calls were traced at the kernel syscall boundary. Each -case recorded zero matching mounts and zero matching md providers after its -cleanup. TC140 and TC142 additionally showed: - -```text -erofs active allocations = 0 -guest responsiveness = ok -panic/trap/hang = none -``` - -The final KLD unload succeeded. The dedicated guest powered off, QEMU exited, -and TCP port 9224 no longer had a listener. Generated images, KLDs, overlays, -and raw logs remained outside Git and were not included in the commit. diff --git a/tests/results/manual/2026-08-09T1059Z-g3/manual-test-report.md b/tests/results/manual/2026-08-09T1059Z-g3/manual-test-report.md deleted file mode 100644 index 34a48d6..0000000 --- a/tests/results/manual/2026-08-09T1059Z-g3/manual-test-report.md +++ /dev/null @@ -1,154 +0,0 @@ -# repo22 G3 Full Manual Regression Report - -Execution window: 2026-08-09 10:42-10:59 UTC - -Exact source baseline: 9ae22009f23a65320730072a780998e80aa9b728 - -Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG, port 9222 - -## Scope and Method - -This run covers exactly these 31 test cases and no others: - -TC041-TC066, TC141, TC148, TC149, TC152, TC153 - -Every test was executed directly from its numbered Markdown procedure with a -fresh dynamic md provider. No CI, runner, result wrapper, historical PASS, or -unassigned TC supplied a result. - -Set audit: requested 31, selected 31, unique 31, duplicate 0, missing 0, -extra 0. - -## Exact Build - -The source archive was exported with git archive from the exact baseline. Its -SHA256 was: - - c7f36610a523427dc406023e1c27fc87a03076bbde2e6b1406311fa5ddb06b0f - -The module was built natively in the guest with: - - FREEBSD_SRC=/tmp/repo22-freebsd15-src WITH_ZSTDIO=0 ./build.sh - -The build completed with kernel -Werror. The KLD loaded as ID 20, size 0xb8a8, -and remained the only loaded EROFS module for the test interval. - -| Item | SHA256 | -|---|---| -| erofs.ko | 19ad086bd2508cbb4c57b1a51f38c93057d438b84fbcfa2e637ff2519f5bc590 | -| readdir_probe | 2387964fd0c153f9cba0e41dd57fbdc1d48c2f2b779540f27387fb168cd35128 | -| g3_vfs_probe | 08fe3999600dd1826805c00a17af3d893d247a593c0812f767d85a2633302068 | -| stat_special | 1c9b4e8fea449d2fc7b985c468a0267448db5998dd413c6c4ed36a1ab0c78ea4 | -| nfs_fh_tool | db704f67d718d04b84ed32b5cdbecb93a99c5fe3b4ab238d603a10e005d790e3 | -| mmap_fault | ba32712f9a4a01f4d85d8f5f6fafe2741acbd6b185a85fa759da73dbc416233c | -| sparse_hole_probe | e5e1d8750fb671f0fccafdf51b61f4b0e73837718906dbd5fd5e8ab01cc790aa | - -The KLD unresolved-symbol list contained both -vnode_pager_local_getpages and vnode_pager_local_getpages_async; both resolved -at load. - -## Fixture Qualification - -tests/prepare_g3_fixtures.sh completed its structured assertions and all four -checksum-manifest checks. The relevant image hashes are: - -| Alias | Image | SHA256 | -|---|---|---| -| P | vfs-plain.erofs | 79f0b5f4aa8e7ea532b711ee8fb466f14f35d0952446d41ba4bbc4d6e008b8ff | -| Z | vfs-lz4.erofs | 07e2d1fbda0e6dc1233e6943a20f85e1f6652b580632c4c8264641a9a8805743 | -| N | namei-base.erofs | d1730ff23836797c6c09e1b39b1cf23efc16e27f85ab07fdcab577bb82871659 | -| Pad | namei-padding-nonzero.erofs | 9a94e9af2cab264b9c11975a20d78e615d6fe1e6f86267173cb5ac86aecb2b17 | -| Cshort | namei-corrupt-short.erofs | fb89f74795a5569ed3a85d63836dd75a06e1f17823d0508048c37da710ea6e75 | -| Coff | namei-corrupt-nameoff.erofs | b0b70ee615f163430f04edb91ab76c27ee8cc9a75b8ebb2008834f5b550e3933 | -| Cname | namei-corrupt-name.erofs | 6a980ad3e241603eda2ef71a470c82975291c614366a401e4e89c17c9adf9b91 | -| SC | special-compact.erofs | fd78256dd83d9d6d957e5f843c7a8e8a175a4b3243d528bebd299b0226853237 | -| SE | special-extended.erofs | e73e9b84d9ceb8c2b07e9c2732733b0fd607736c68c09522a2402fbeef6ba8d5 | -| H | extent-hole-5g.erofs | 50014a24493918247e36511ad34a2fe8ab47ae09ea46d7fd62a1bed445a6c65f | -| L | large-dir-intmax.erofs | 0f90d3d57adbbbd946e41b225c1f6c464915c6abb0b13478ec9b2a318def1f72 | -| LS | TC153 sparse prefix | f9337f83b1f568f6d904331a7749e6362a691055cd5af95b59bc89772f04e3b0 | - -The wide directory has 320 real files. The structured padding mutation records -wide NID 42, directory block 6, 80 dirents in the patched block, final-name -offset 4043, and the eight changed bytes at offsets 4084:4092. - -The 5 GiB fixture records size 5368713216 and one 16-byte extent with plen=0. -TC153 records extended FLAT_PLAIN/Layout 0, NID 40, start block 16, -2147483649 directory blocks, and final block index 2147483648. - -## Per-Test Results - -Every cleanup cell means the literal umount and mdconfig detach commands -returned zero; a following mount -t erofs and mdconfig -l produced no rows. - -| Test | Result | Actual command, errno/behavior, and hash evidence | Cleanup | -|---|---|---|---| -| TC041 | PASS | readdir_probe testdir 512; errno 0; 36 entries/restarts, types 8 dir/24 reg/4 link, FNV a3b969b5aa90cdbe, exact 34-name manifest and 255-byte name; image P. | umount 0; md 0 | -| TC042 | PASS | readdir_probe wide 128 plus three stat calls; errno 0; 322 entries/restarts, 320 files, FNV f3bfa2c15b231a59, sample NIDs 72/390/710; image N. | umount 0; md 0 | -| TC043 | PASS | four g3_vfs_probe stat calls, cmp, sha256; errno 0; regular/dir/link metadata stable and mounted file matched source; image P. | umount 0; md 0 | -| TC044 | PASS | direct stat probes; missing/missing-parent errno 2, positive errno 0; image P. | umount 0; md 0 | -| TC045 | PASS | stat four case variants plus direct miss; NIDs 101/88/86/87, unmatched spelling errno 2; image P. | umount 0; md 0 | -| TC046 | PASS | readdir_probe testdir 4096 and exact manifest cmp; errno 0; 36 entries/restarts, FNV a3b969b5aa90cdbe; image P. | umount 0; md 0 | -| TC047 | PASS | readdir_probe wide 128, sorted count/duplicate checks; errno 0; 322 entries/restarts and 320 unique files; image N. | umount 0; md 0 | -| TC048 | PASS | readdir_probe wide 128; every kernel d_off reopened/lseeked, every libc cookie seeked on its producing DIR stream; 322/322 restarts, FNV f3bfa2c15b231a59; image N. | umount 0; md 0 | -| TC049 | PASS | readdir_probe plus two dot stat calls and cmp; errno 0; dot and directory both NID 44, source-exact path; image P. | umount 0; md 0 | -| TC050 | PASS | readdir_probe child plus parent/dotdot stat and cmp; errno 0; parent NID stable and multiple dotdot path source-exact; image P. | umount 0; md 0 | -| TC051 | PASS | repeated stat and cmp; errno 0; byte-identical NID/mode/size/generation and data; behavior-level only; image P. | umount 0; md 0 | -| TC052 | PASS | repeated negative stat then positive stat; errno 2/2/0; no positive poisoning; behavior-level only; image P. | umount 0; md 0 | -| TC053 | PASS | repeated stat, getfh, compare, describe; errno 0; NID 101, gen 3444757833, identical handle SHA 5000984ddaeca26d90e8fd59568a55afcb074f51aaa00eed8d422d698a4aefc6; image P. | umount 0; md 0 | -| TC054 | PASS | parent and two dotdot stat calls; errno 0; identical NID/generation/type, no lock diagnostic; behavior-level only; image P. | umount 0; md 0 | -| TC055 | PASS | g3_vfs_probe stat, native stat, sha256 on plain/LZ4; errno 0; size 21211, blocks 48/8, data SHA d09ed1cee6520e36e54d5f2fd8c3bc74bd46cd8a58426744cc301b638c83e9d4; images P/Z. | two umount 0; md 0 | -| TC056 | PASS | stat_special char/block/fifo plus stat on compact/extended images; errno 0; rdev 2748:344865, FIFO NODEV, size/blocks 0; images SC/SE. | two umount 0; md 0 | -| TC057 | PASS | nobody direct access-read, access-exec directory/file, open-read; errno 0 for all; image P. | umount 0; md 0 | -| TC058 | PASS | nobody direct access probes; restricted read/read/exec errno 13, regular-file write access errno 30; image P. | umount 0; md 0 | -| TC059 | PASS | four readlink probes, cmp, broken follow; target lengths 8/16/14/119 and FNV hashes recorded, follow errno 0/2; image P. | umount 0; md 0 | -| TC060 | KERNEL-FAIL | g3_vfs_probe pathconf; values 255/1024/64/2147483647, but NO_TRUNC and CHOWN_RESTRICTED each errno 22; probe status 1; image P; issue TC060-pathconf-standard-values.md. | umount 0; md 0 | -| TC061 | PASS | direct chmod probe and before/after stat cmp; errno 30, metadata unchanged; image P. | umount 0; md 0 | -| TC062 | PASS | direct chown probe and before/after stat cmp; errno 30, metadata unchanged; image P. | umount 0; md 0 | -| TC063 | PASS | direct O_RDWR/truncate/create probes plus before/after sha256; errno 30/30/30, hash unchanged and no new entry; image P. | umount 0; md 0 | -| TC064 | PASS | nm -u plus mmap_fault on plain/LZ4; helper errno assertions passed; size 21211, six faults, FNV a1890a1c216724be, EOF/SIGBUS/COW PASS; images P/Z. | two umount 0; md 0 | -| TC065 | PASS | real mmap_fault on plain/LZ4; MAP_SHARED EACCES and O_RDWR EROFS asserted, same FNV a1890a1c216724be; images P/Z. | two umount 0; md 0 | -| TC066 | PASS | exact source erofs_bmap EOPNOTSUPP audit, KLD symbols, real mmap fallback on plain/LZ4; both FNV a1890a1c216724be, no strategy diagnostic; images P/Z. | two umount 0; md 0 | -| TC141 | PASS | fresh open-read/cmp, two ENOENT lookups, 322-cookie probe on N/Pad; two lookup and readdir attempts on Cshort/Coff/Cname all errno 97; all five hashes above. | five umount 0; md 0 | -| TC148 | PASS | actual Pad image cmp/count/readdir_probe; errno 0; 320 files, 322 restarts, FNV f3bfa2c15b231a59; hash Pad and patch offsets recorded. | umount 0; md 0 | -| TC149 | PASS | real mmap_fault on plain/LZ4; same size/fault/FNV/EOF/SIGBUS/COW evidence as TC064 with fresh mounts; images P/Z. | two umount 0; md 0 | -| TC152 | PASS | stat, vmstat -m, two timed sparse_hole_probe calls at 3221225472; errno 0, pread/mmap zero, 0.02 s each, RSS 2652/2656 KiB, erofs memory 768 bytes; image H. | umount 0; md 0 | -| TC153 | PASS | 8796093091840-byte sparse provider, diskinfo, stat, two cold and one post-readdir lookup; /huge size 8796093026304, all lookup errno 97, root 3-cookie FNV cd20cb6ae9fe01ba; prefix LS/KLD hashes. | umount 0; md 0; sparse file removed | - -## Outcome - -| Result | Count | -|---|---:| -| PASS | 30 | -| KERNEL-FAIL | 1 | -| SHELVED/ISSUE | 0 | -| ENVIRONMENT-UNAVAILABLE | 0 | - -The only new open issue is issues/TC060-pathconf-standard-values.md. The former -TC153 validation issue is resolved by the qualified sparse-provider run. - -## Discarded Attempts - -- The first TC041 helper run passed telldir cookies across a closed and newly - opened DIR stream. POSIX only guarantees a cookie on its producing stream. - The helper was corrected and rebuilt with -Werror; this run was not counted. -- TC041 was then attempted with a 128-byte getdirentries buffer, too small for - the fixture's 255-byte name record. The 512-byte qualified rerun passed. -- TC047 was first attempted with a 64-byte buffer, too small for its long - records. The 128-byte qualified rerun still forced repeated reads and passed. - -No discarded command was counted as a kernel result. - -## Dmesg and Final Cleanup - -The dmesg baseline had 123 lines. The only eight new lines were expected -mmap_fault child exits on SIGBUS: two images for each of TC064, TC065, TC066, -and TC149. There was no new EROFS warning, integrity diagnostic, assertion, -trap, panic, OOM, dirty writeback, or "No strategy for buffer" line. - -Before unload, the exact KLD hash was rechecked. KLD unload returned zero. -The guest then reported: - - erofs_mounts=0 md_units=0 kld_present=0 guest_artifacts=0 - -The TC153 sparse provider and all transferred/generated guest files were -removed. diff --git a/tests/results/manual/2026-08-09T1120Z-tc060-fix/manual-test-report.md b/tests/results/manual/2026-08-09T1120Z-tc060-fix/manual-test-report.md deleted file mode 100644 index 40455c3..0000000 --- a/tests/results/manual/2026-08-09T1120Z-tc060-fix/manual-test-report.md +++ /dev/null @@ -1,157 +0,0 @@ -# repo22 TC060 pathconf fix manual test report - -Execution window: 2026-08-09 11:20-11:35 UTC - -Exact source commit: `cdba7e54fb9e9d82980a06f178e68d0bbc1663ac` - -Guest: FreeBSD 15.0-RELEASE-p8 amd64, QEMU TCG, port 9222 - -## Scope and Result - -TC060 is **PASS**. The original six-key helper returned the required values -with errno 0, compatibility queries retained their previous values, an unknown -name returned EINVAL, mount/read/read-only smoke passed, dmesg did not change, -and final mount/md/KLD/artifact counts were zero. - -This was a direct manual run. No CI, result wrapper, or historical PASS supplied -the result. `WITH_ZSTDIO=1` was build-verified; runtime TC060 used -`WITH_ZSTDIO=0` because pathconf is compression-independent and this avoids an -unnecessary optional runtime dependency. - -## Implementation Basis - -FreeBSD 15 `sys/kern/vfs_default.c` shows that `vop_stdpathconf()` handles -generic `_PC_ASYNC_IO`, `_PC_PATH_MAX`, and several zero-valued optional -features, then returns EINVAL for other names. The same source tree's UFS, -tmpfs, and ext2fs vnode operations explicitly return 1 for -`_PC_CHOWN_RESTRICTED` and `_PC_NO_TRUNC` before delegating the remaining names -to `vop_stdpathconf()`. - -The EROFS fix adds only those two switch labels and retains the existing -default delegation. `src/namei.c` rejects components longer than -`EROFS_NAME_LEN` with ENAMETOOLONG, while `src/erofs_vnops.c` rejects uid/gid -mutation with EROFS. Linux EROFS's 255-byte name limit and ENAMETOOLONG lookup -check were reviewed only for maintenance similarity; FreeBSD VOP behavior was -authoritative. - -## Exact Build - -The repo22 subtree was exported directly from the exact source commit. Later -`current/`-only reporting changes did not alter `src/`. - -| Item | Value | -|---|---| -| Source archive SHA256 | `a4c871ca7cefc49470ef0003f876792380a8d8db6cbeaefb96456d06ad946d53` | -| FreeBSD source path | `/tmp/repo22-freebsd15-src` | -| Compiler | FreeBSD clang 19.1.7 | -| Build command | `FREEBSD_SRC=/tmp/repo22-freebsd15-src WITH_ZSTDIO=N ./build.sh` | - -Both clean-object builds completed with kernel `-Werror`: - -| Configuration | Module SHA256 | Size | Undefined-symbol audit | Result | -|---|---|---:|---|---| -| `WITH_ZSTDIO=0` | `68536e03ce93c6c04aab9cf29dab81ee5802d4b94401bd1a4bd752de40c0e504` | 73896 | no `bcmp`, no `ZSTD_*` | PASS | -| `WITH_ZSTDIO=1` | `598f171d355af78c64407a6d513d20f053530620da92df7f8ccfdf9a804e463d` | 78728 | no `bcmp`; only five expected FreeBSD `ZSTD_*` APIs | PASS | - -The enabled module referenced `ZSTD_DCtx_setParameter`, -`ZSTD_createDCtx_advanced`, `ZSTD_decompressStream`, `ZSTD_freeDCtx`, and -`ZSTD_isError`. - -## Fixture and Probes - -| Artifact | SHA256 | -|---|---| -| `vfs-plain.erofs` | `79f0b5f4aa8e7ea532b711ee8fb466f14f35d0952446d41ba4bbc4d6e008b8ff` | -| tracked `g3_vfs_probe` binary | `08fe3999600dd1826805c00a17af3d893d247a593c0812f767d85a2633302068` | -| one-run `pathconf_audit` source | `c2cc69297848cad8c806d47585ef670ae5b378827450110e2cf47cf82ed527b3` | -| one-run `pathconf_audit` binary | `17aa9d738871ac424900ed7a928cb72cc2e5a288f2c2f9744906fa139a78835b` | - -Both probes compiled natively with `-O2 -Wall -Wextra -Werror -std=c17`. -The one-run helper was kept outside the repository and only printed each -value/errno plus assertions for unknown-name and overlong-component behavior. - -## TC060 Values - -The exact original command was: - -```text -./g3_vfs_probe pathconf /tmp/repo22-tc060-fix/mnt/testdir -``` - -It exited 0 and printed: - -```text -name_max=255 path_max=1024 filesizebits=64 link_max=2147483647 -no_trunc=1 chown_restricted=1 -``` - -The expanded value/errno capture was: - -| Query | Actual value | errno | -|---|---:|---:| -| `_PC_NAME_MAX` | 255 | 0 | -| `_PC_PATH_MAX` | 1024 | 0 | -| `_PC_FILESIZEBITS` | 64 | 0 | -| `_PC_LINK_MAX` | 2147483647 | 0 | -| `_PC_NO_TRUNC` | 1 | 0 | -| `_PC_CHOWN_RESTRICTED` | 1 | 0 | -| `_PC_ASYNC_IO` | 200112 | 0 | -| `_PC_ACL_EXTENDED` | 1 | 0 | -| `_PC_ACL_PATH_MAX` | 254 | 0 | -| `_PC_ACL_NFS4` | 0 | 0 | -| unknown name `INT_MAX` | -1 | 22 (`EINVAL`) | - -A lookup using a 256-byte path component returned -1 with errno 63 -(`ENAMETOOLONG`), confirming the reported no-truncation behavior. - -## Mount and Read Smoke - -The module loaded as KLD file ID 20, module `erofs`, and the fixture attached -as dynamic `md0`. The mount line was: - -```text -/dev/md0 on /tmp/repo22-tc060-fix/mnt (erofs, local, read-only, acls) -``` - -`testdir` reported inode 44, mode `drwxr-xr-x`, and size 1039. Reading -`testdir/file.txt` produced `repo22 G3 file payload` and SHA256 -`523af4c899ba3b8f4fa4d854fe609590be271905b63932cf067ca9630e612687`. -Opening the same file with `O_RDWR` returned errno 30 (`EROFS`) as expected. - -## Dmesg and Cleanup - -The qualified run's dmesg had 131 lines before and after, with identical -SHA256 `91bf74a3607f432b4802f143bcf9d050c2f9e741da0a54f3e6a1b47bcc996a06`. -There was no new EROFS diagnostic, assertion, trap, panic, or resource warning. - -Qualified cleanup returned zero for umount, md detach, and KLD unload. After -removing the transferred archive and test directory, the final guest audit was: - -```text -erofs_mounts=0 -md_units= -erofs_klds=0 -guest_artifacts=0 -``` - -## Discarded Attempt - -The first otherwise successful runtime pass loaded the KLD from the -non-canonical filename `erofs-zstdio0.ko`. Its explicit `kldunload erofs` -command could not resolve that filename, so the run stopped before final dmesg -and cleanup qualification. Its mount and md were already clean; KLD file ID 20 -was then unloaded directly and the guest returned to zero resources. The full -test was repeated with canonical `erofs.ko`, and only that rerun is counted. - -## Independent Review - -The post-test review found no lock, resource, or error-path issue: - -- `VOP_PATHCONF` is called with the vnode shared-locked; the function does not - change lock state. -- The new constant-return branches allocate nothing and acquire no references. -- Both normal and FIFO vnode vectors use the same filesystem-wide result. -- Existing value branches are unchanged, and unsupported names still reach - `vop_stdpathconf()` and return EINVAL. -- The syscall copies `retval` only on error 0, so the unknown-name error path - cannot expose a stale value. diff --git a/tests/results/manual/2026-08-09T1236Z-g5/manual-test-report.md b/tests/results/manual/2026-08-09T1236Z-g5/manual-test-report.md deleted file mode 100644 index 4963ae6..0000000 --- a/tests/results/manual/2026-08-09T1236Z-g5/manual-test-report.md +++ /dev/null @@ -1,212 +0,0 @@ -# repo22 G5 compression full manual regression report - -## Scope and result - -- Exact scope: TC003, TC004, TC084-TC092, TC102-TC110, TC143-TC146. -- Count check: 24 requested, 24 executed, 0 duplicate, 0 omitted. -- Result: **PASS 23, PARTIAL 1, FAIL 0, NOT RUN 0**. -- SHELVED subscenario: TC146 explicit mapped extent only. -- TC146 overall: **PARTIAL**; HEAD2 PASS, interlaced PASS, explicit extent - SHELVED. -- Kernel baseline: `f383bbbbff301a6bde18894f03ab88a8c0cc885a`. -- Initial and pre-push `xdm/main` baseline: the same commit. -- No repo22 kernel source was changed. No kernel failure issue was opened. - -## Isolated environment - -- Worktree: `/work/build/repo22-manual-g5-20260809T085450Z`. -- Branch: `manual-g5-20260809T085450Z`. -- Artifact root: `/work/build/repo22-g5-20260809T085450Z`. -- Dedicated qcow2 overlay backed by the FreeBSD development base. -- Dedicated SSH forward: `127.0.0.1:9225`. -- Guest: FreeBSD `15.0-RELEASE-p8`, amd64, - `releng/15.0-n281036-53054229dcb3`. -- Guest toolchain: FreeBSD clang/LLD 19.1.7. -- Host tools: mkfs/dump/fsck erofs-utils 1.8.6. -- No CI, runner, or wrapper was used. Each Markdown test was issued manually. - -## Reproducible fixture - -`tests/g5_fixtures.py` generated source content, invoked mkfs with fixed UUID, -timestamp, owner, sort order, and one worker, transformed structured records, -reopened every image, and wrote a field manifest. Two final fresh output -directories produced byte-identical source and image checksum inventories. - -```text -SHA256SUMS SHA256 = 8535696f187f818df8b3b30ee35cd8837aceadef9de7a85be1475732dcdf0596 -SOURCE-SHA256SUMS SHA256 = 145bd663d7552025b486ec15c0b4c1c0ce99affa7728dc4d762134bdc92420c3 -fixture manifest SHA256 = 63ecdb46f8cc3c04e348e24e0f1306ad5e4606bf1fa108919f946515d7b41d63 -image count = 25 -source count = 19 -guest hash verification = 25 images, 19 sources -``` - -### Image hashes - -| Image | SHA256 | -|---|---| -| `deflate-level1.erofs` | `58dc78da2ec5a751688f53c4eee34f94b7f828d68b7225aea006d08374187bfe` | -| `deflate-level6.erofs` | `4a3b5d2ce6b8d1bad23aa2c4d4ce38f83bc0e6642523e9165e58afe03e030c28` | -| `deflate-level9.erofs` | `c4bf219a5130d475c26452a1a5346c63ed1f434afd4315972ee30ebc66f9c751` | -| `deflate-partial-ref.erofs` | `fccb2f4038dca8b7277984e255a304bb6eee5de51fedd7b7261ef4f879362c81` | -| `deflate-partial-ref-corrupt.erofs` | `063e1bc07d2c3a3bf3b17c0cce0237eca7000eb16abdf8a1b4428d8cc7880811` | -| `extent-attempt.erofs` | `ee7472c23ccffd5eef6f4a3171ea53ae2e840f8a1ea417b2630065175ecf3225` | -| `head2.erofs` | `4948335059e605ff70da4f6e516c4ba24d58313d10e836cc6fe9c9c468775a23` | -| `head2-corrupt.erofs` | `7c25e9c8c767d2717c06b1efc854f0bb3b1f370cf3e2a8e2d53e767918bfe50f` | -| `interlaced.erofs` | `7b77055dfae301a0acc3202cf0d7cf7062eb82c6101d16b68c91f70615e1638b` | -| `lz4-compact-4k.erofs` | `0701a37430372f1240ad6485c83f0048d97bb49ae1ca85e1423dd05101bf5578` | -| `lz4-compact-64k.erofs` | `967cc1b625546f9f9f881472e71cc3d849c65feb4e98e67fbfdc9b90a4be6dda` | -| `lz4-compact-256k.erofs` | `6fbded24756b557c7ce19a5936dcde01fb75d4b1c91b0236fca46ee0c6331db0` | -| `lz4-full-4k.erofs` | `3bc6eb538752673e2b9f905a0382887bccf417574a0d04901932cfeaac258bc7` | -| `lz4-large.erofs` | `0340026fe5fea5fd7c6c4c03f288815bc86c7517c2c2a9a2ca21c61dfb96a973` | -| `lz4-ztail.erofs` | `18cd045d7f2b8e542a0dda6fb0063c266919928e6e485f498ed93c8992d9ae3a` | -| `lzma-level6.erofs` | `32107a084b27362a093768b88746c37c2e99a74b9f1301a9d4046988479defe9` | -| `lzma-large.erofs` | `6534870a686c3c5ba58fc665ea4995c07427a15c9d5ac7747690114138adf47d` | -| `lzma-partial-ref.erofs` | `fbbcbcb8a178370a8f7665c3e0cff8df5bb6f81c3ba38a0e44c137e8ed165402` | -| `lzma-partial-ref-corrupt.erofs` | `23981af2f4e36f88e382fff975f3c4fb00cb4b02c0f40f10948602ab4ce39f23` | -| `microlzma-edge.erofs` | `27a19758141d64430d7536774fadf34ba7f8befa0bc36442b789e494b4b37d4c` | -| `zstd-level1.erofs` | `34f51d5b1273cc3fc9efa458c200cf134dc7dca2c583e577a8d06ad563cfbff6` | -| `zstd-level15.erofs` | `8ca4ba52561cc8903f85048edfd958ef692583479748dcb8aa57e2c7aade1034` | -| `zstd-level22.erofs` | `99f79807ffd9f1216dbd624a2d8ecfcf17c9b42689fef5b5a69d0250ed84e5` | -| `zstd-partial-ref.erofs` | `cdef7c2a397722723dcc762044612a179358f9dbe4dcd879ac08e462cf9628d1` | -| `zstd-partial-ref-corrupt.erofs` | `ed6f4b23f6b6426e6af9180e892f03097d16a6b969edd68ad42d9a1ff8d2325e` | - -### Source hashes - -| Source | Size | SHA256 | -|---|---:|---| -| `large/large.bin` | 268435456 | `78f61f8eb37b5aeee026b579d244d766935d5655c0ac5383a96f631048611cfd` | -| `levels/level.dat` | 8388608 | `ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461` | -| `lz4/compressed.bin` | 8388608 | `3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880` | -| `lzma-large/large.bin` | 104857601 | `de846af6c7e47fac72c0576c449fd0fc1fe9587286c9158ae08c81804c77f66b` | -| `microlzma/one-byte.bin` | 1 | `333e0a1e27815d0ceee55c473fe3dc93d56c63e3bee2b3b4aee8eed6d70191a3` | -| `microlzma/block-4k.bin` | 4096 | `42b2e4ac3afeb366a6407573d5a91a961775e0fe374b0941f19681c4e3f9ea96` | -| `microlzma/boundary-16k.bin` | 16384 | `1b5a7306ca67b75c18228d08f281eb7474b31a1a1ccbe4d680b7942fc81265b9` | -| `partial/a.dat` | 1048576 | `c7ac0fce9c56d732e4b8328c1ca48ae33ee4d449166bd25d2c88322c1a852d9f` | -| `partial/b.dat` | 700000 | `926a9bb05b20cb3da745eaff6e7fec38fb43d7fa155276474367550b67b97589` | -| `partial/control.bin` | 32768 | `7544a26039c5257ee07c1280468edb1f61f0b0edaa25273b15e0252a9cc5c90a` | -| `partial-deflate/a.dat` | 1048576 | `c7ac0fce9c56d732e4b8328c1ca48ae33ee4d449166bd25d2c88322c1a852d9f` | -| `partial-deflate/b.dat` | 100000 | `1fbe057da0cb994652ef664a4aa00628e948578055665160fc63e7079469da21` | -| `partial-deflate/control.bin` | 32768 | `7544a26039c5257ee07c1280468edb1f61f0b0edaa25273b15e0252a9cc5c90a` | -| `shape/shape.dat` | 1048576 | `5be510e6b43f1ed0cda4261288ea70df11607b6ced29bc90d32ed2849ecc5e35` | -| `ztail/inline.dat` | 131071 | `e54a4bac3b6b1c01ee846dbccdf234ddb30ed06dc13f1af1df0fbbc397431ce5` | -| `ztail/exact-pcluster.dat` | 4096 | `14587a494e72ecdc7d26b4c0b947d91d44eba4365b252d1a4ec10ba979978c52` | -| `ztail/one-byte-tail.dat` | 4097 | `718f4c27896edf8925300c50572df92b9127e56b2b84a3a86ced4d67c3be9596` | -| `ztail/max-tail.dat` | 8191 | `5e1e898dba6a6bbcc8beb0b0b097bc04ecaaf4ad505b77bfb8646a3011628df6` | -| `ztail/zero-tail.dat` | 0 | `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` | - -## Key layout fields - -| Fixture | Structured proof | -|---|---| -| LZ4 compact 4K | layout 3, map offset 2183200, `h_advise=1` | -| LZ4 full 4K | layout 1, map offset 2183200, `h_advise=0` | -| LZ4 compact 64K | layout 3, `h_advise=7`, max physical pcluster 65536 | -| LZ4 compact 256K | layout 3, `h_advise=7`, max physical pcluster 262144 | -| ztailpacking | layout 3, `h_advise=9`, `h_idata_size=536`, physical bytes 1512-2048 | -| HEAD2 | incompat `2 -> 10`, map advise `2 -> 6`, first record `1 -> 3`, pblk 1 | -| interlaced | layout 3, `h_advise=49`, 52 compressed, 1 plain, first transition 999593 | -| explicit attempt | layout 1, map offset 1312, `h_advise=2`, explicit bit absent | - -The first HEAD2 logical pcluster ends at 326114; the boundary read started at -326000. The interlaced boundary read started at 999000 and crossed the proven -transition at 999593. - -Partial-reference fields after reopen: - -| Algorithm | a/b size | a/b pblk | b partial advise | b compressed blocks | -|---|---|---|---:|---| -| DEFLATE | 1048576 / 100000 | 1 / 1 | 32769 | 2 -> 17 | -| ZSTD | 1048576 / 700000 | 1 / 1 | 32769 | 1 -> 1 | -| MicroLZMA | 1048576 / 700000 | 1 / 1 | 32769 | 1 -> 1 | - -Targeted corruption fields: - -| Target | Algorithm | Pcluster | Patch | -|---|---:|---|---| -| DEFLATE `/a.dat` | 2 | 4096 + 69632 | offset 8055, length 64 | -| ZSTD `/a.dat` | 3 | 4096 + 4096 | offset 8073, length 64 | -| MicroLZMA `/a.dat` | 1 | 4096 + 4096 | offset 7957, length 64 | -| HEAD2 `/shape.dat` | 0 | 4096 + 65536 | offset 4096, length 64 | - -Every corrupted image retained a valid superblock checksum. Each read was -bounded by `timeout 20`, returned errno 5, and left `control.bin` readable -where the fixture included one. - -## Build and TC145 - -The invalid build value failed with rc 1 and the exact message -`WITH_ZSTDIO must be 0 or 1`. - -| Build | Size | SHA256 | Undefined-symbol gate | Guest | -|---|---:|---|---|---| -| `WITH_ZSTDIO=0` | 73896 | `e103ebbaf7faf8041448d93022be44e7c93822a45a36a69bca2b7a2107461d09` | no `ZSTD_*`, no `bcmp` | file ID 5 load/unload PASS | -| `WITH_ZSTDIO=1` | 78728 | `56ffcce006eed1e555121d9b8f523578272ac3d82c27a19b635d0a0a664c8370` | five formal API names, no `bcmp` | file ID 5 load/unload PASS | - -The enabled undefined ZSTD set was exactly -`ZSTD_DCtx_setParameter`, `ZSTD_createDCtx_advanced`, -`ZSTD_decompressStream`, `ZSTD_freeDCtx`, and `ZSTD_isError`. - -The disabled module read the LZ4 control hash -`3ff012b76087c4da65ce0b69813a76f47ea367e95782edf8b8cd6cd3ec4d1880`. -A direct ZSTD mount returned rc 1 with the required message; `truss` proved -`nmount` returned `ERR#45 EOPNOTSUPP`. The enabled module read ZSTD hash -`ddda39737f0f6093e828a032ec161511fefbb1fa361bc6cbffdbc91e48e4c461`. - -## Per-TC results - -| TC | Manual evidence | Result | -|---|---|---| -| TC003 | compact LZ4 full hash/cmp; first 1024 bytes cmp | PASS | -| TC004 | LZMA level 6 full hash/cmp; 102400/10240 and EOF 10000 cmp | PASS | -| TC084 | full-index LZ4 full hash/cmp; 4000/8192 cross-extent cmp | PASS | -| TC085 | 256 MiB full hash/cmp; 50/200 MiB ranges and two 4K offsets cmp | PASS | -| TC086 | deterministic sequential 4K and 1 MiB block reads both full cmp | PASS | -| TC087 | offsets 0, 4096, 65536, 1048576, and 8384512 source cmp | PASS | -| TC088 | 4K/64K/256K images full hash and 409600/40960 source cmp | PASS | -| TC089 | 0/4096, 4096/4096, 2048/4096, and 0/16384 source cmp | PASS | -| TC090 | 0/65536, 4096/4096, 65504/64, and 0/262144 source cmp | PASS | -| TC091 | ztail full hash/cmp and final 4096 bytes cmp | PASS | -| TC092 | actual 4096, 4097, 8191, and 0-byte files full hash/cmp | PASS | -| TC102 | DEFLATE level 1 full hash/cmp and two fixed ranges | PASS | -| TC103 | DEFLATE level 6 full hash/cmp and two fixed ranges | PASS | -| TC104 | DEFLATE level 9 full hash/cmp and two fixed ranges | PASS | -| TC105 | ZSTD level 1 full hash/cmp and two fixed ranges | PASS | -| TC106 | ZSTD level 15 full hash/cmp and two fixed ranges | PASS | -| TC107 | ZSTD level 22 full hash/cmp and two fixed ranges | PASS | -| TC108 | 104857601-byte LZMA level 6 full hash/cmp; 0/50/99 MiB ranges; active 0 | PASS | -| TC109 | actual 1B/4K/16K files match; 16K inode proven compressed LZMA | PASS | -| TC110 | valid LZMA control; targeted read EIO; same-image control; active 0 | PASS | -| TC143 | DEFLATE/ZSTD a+b full, partial ranges, targeted EIO, controls, active 0 | PASS | -| TC144 | MicroLZMA a+b full, partial ranges, targeted EIO, control, active 0 | PASS | -| TC145 | invalid gate, both builds/symbol sets/KLDs, disabled rejection, enabled read | PASS | -| TC146 | HEAD2 PASS; interlaced PASS; explicit mapped payload SHELVED | PARTIAL | - -TC108 used a 1800-second hard timeout for each complete read under QEMU TCG. -It completed without timeout; the long duration was diagnostic only. - -## Explicit extent disposition - -erofs-utils 1.8.6 source had zero matches for -`Z_EROFS_ADVISE_EXTENTS`, `z_erofs_extent_recsize`, and the on-disk -`struct z_erofs_extent`. The fresh `--max-extent-bytes=65536` attempt remained -ordinary full-index metadata. The structured transformer refused an -unverifiable metadata/payload relocation. Existing `review_fixtures.py` only -builds a negative `pa + plen` overflow extent table, not a valid mapped -payload. - -FreeBSD and Linux ABI/control flow for 4/8/16/32-byte records was reviewed. -That is static support, not dynamic positive coverage. Full details and -acceptance criteria are in `issues/extent-metadata-fixture-unavailable.md`. - -## Stability and cleanup - -- Every mount was read-only and every md provider was detached. -- All corruption probes returned errno 5 before their 20-second timeout. -- No dmesg delta, panic, trap, or OOM was observed. -- EROFS active allocations returned to zero after each corruption/large test. -- Final guest verification: 25 image hashes and 19 source hashes matched. -- Final guest: zero mounts, zero md providers, no EROFS allocator row, no EROFS - KLD, and responsive FreeBSD 15.0-RELEASE-p8. -- The dedicated VM was powered off and port 9225 was released after evidence - collection. diff --git a/tests/results/manual/2026-08-09T1244Z-g6/manual-test-report.md b/tests/results/manual/2026-08-09T1244Z-g6/manual-test-report.md deleted file mode 100644 index 4b2f06e..0000000 --- a/tests/results/manual/2026-08-09T1244Z-g6/manual-test-report.md +++ /dev/null @@ -1,139 +0,0 @@ -# repo22 G6 Chunk and Multi-Device Manual Regression - -Date: 2026-08-09 -Executor: G6 manual agent -Source baseline: `6b33b4afb490be7d6fec70e499469c306a58435d` -Scope: `TC006`, `TC093`-`TC101`, `TC118` (11 exact IDs) - -## Result - -PASS. All 11 requested test cases passed. There are no KFAIL, SHELVED, or -ENV results, no duplicate IDs, and no omitted IDs. - -| Status | Count | -| --- | ---: | -| PASS | 11 | -| KFAIL | 0 | -| SHELVED | 0 | -| ENV | 0 | -| Duplicate | 0 | -| Omitted | 0 | - -No kernel source was changed. `git diff -- src` was empty after testing. - -## Environment and build identity - -- Worktree: dedicated sparse worktree and branch - `manual-g6-20260809T114414Z`, based directly on `xdm/main` at the source - commit above. -- VM: dedicated qcow2 overlay, QEMU TCG, 6144 MB, 4 vCPUs, SSH forward - `127.0.0.1:9226` only. -- Guest: FreeBSD `15.0-RELEASE-p8` amd64, - `releng/15.0-n281036-53054229dcb3`, OSREL `1500068`. -- Guest kernel SHA256: - `b9abf7b58f9dd4d87f14d2fbc306cf255e6eadc47b8ae885a230f65e28be4562`. -- Tracked build sys source: `REVISION=15.0`, `BRANCH=RELEASE-p9`, archived - from the same repository commit; archive SHA256 - `66c457159b758ab1a5d298292b81bd29ab439ca4ab396091602e030da4721cb9`. -- Build configuration: native guest build, - `WITH_ZSTDIO=1 FREEBSD_SRC=/root/freebsd-src`. -- KLD SHA256: - `d5ff5ceef1ad76eb5b608d362b04f3bc40379e58a73d91ed3ae747c035d7b8af`. -- repo22 source archive SHA256: - `7562e230d165f0c0caa050684a9c1e2bd1bbb178e54a1d5273428435ae512455`. - -The latest tracked sys source is p9 while the clean guest kernel/userland is -p8. Both use OSREL 1500068; the module built and loaded successfully. This -source/guest patch-level distinction is recorded rather than hidden. - -## Fresh fixture qualification - -`tests/g6_multidev_fixtures.py` generated every source, primary image, blob, -flatdev, and negative image from zero with erofs-utils 1.8.6. No old test image -or report was an input. - -- Generator SHA256: - `18638aee056e05099b39477fa6ad4d678ab0a74a027cfb45146ae225ff35d950`. -- Final manifest SHA256: - `b636a52c7b3ec7f2343f07e2d389783c8383ec07d9d9468f7552e817891c3a81`. -- Final `SHA256SUMS` SHA256: - `ed81d63469b40f1aba691e54a08d39737c71e66d2ff4fef963a0101fa2ad9854`. -- Final fixture archive SHA256: - `28be1a5cdcebb163710433c9e7c30162c1170312c34d0b286f4143480881d62f`. -- Generated set: 62 artifacts, 17 parsed fixtures, 7 table/index negative - images, and 10 explicit mutation assertions. -- Reproducibility: two clean output directories had byte-identical - `manifest.json` and `SHA256SUMS`. -- Host fsck extraction qualified the mkfs split image, single-index image, - explicit two/three-slot images, table-at-zero, `uniaddr=0`, packed fragment, - and the original two-block LZ4 pcluster. -- Guest final `sha256 -c SHA256SUMS` returned 0. - -During qualification, an initial maximum-48-bit negative used all ones. Field -review identified that value as the legal chunk hole sentinel. The generator -was corrected to the largest non-NULL address `0xfffffffffffe`, regenerated -twice, and reverified. Only the intended primary artifact changed; its final -SHA256 is -`d6f113cb5d2cb7da7f892632930eedf1c84fb494f5e1e4f0d2a852496ee92951`. -The corrected kernel read returned `EINTEGRITY`. - -## Exact results - -| Test ID | Status | Manual evidence | -| --- | --- | --- | -| TC006 | PASS | Direct mkfs split primary/blob; full and cross-32K range `cmp`; external `MDIOCDETACH` returned `ERR#16 EBUSY`. | -| TC093 | PASS | 8-byte device-ID-0 indexes with no table; full and cross-32K range `cmp` on the primary provider. | -| TC094 | PASS | Reversed slot option order; 1/2/1 full and per-index ranges; `uniaddr=0` with nonzero ID; swapped media `ERR#6 ENXIO`; real external 8192-byte LZ4 pcluster produced the complete 131072-byte source SHA256. | -| TC095 | PASS | First/middle/last/cross-index reads; zero-high 48-bit control and nonzero-high unified read; mapped ID 3 returned `read ERR#19 ENODEV`, unaffected file remained readable. | -| TC096 | PASS | Parsed normal, table-at-zero, `uniaddr=0`, and high 48-bit tables; `df -k` reported 916 KiB = 229 blocks x 4096; table outside primary returned `ERR#97 EINTEGRITY`. | -| TC097 | PASS | Six structural variants each returned `nmount ERR#97 EINTEGRITY`; after every failure md/KLD/consumer state was zero; slot-2 `ENOENT` after slot 1 open released slot 1 immediately. | -| TC098 | PASS | Packed NID 42 whole-file fragment; size 100000; full SHA/`cmp` and 8192-byte offset range passed. | -| TC099 | PASS | Direct primary+one-blob layout; `df -k` 900 KiB = 225 blocks x 4096; full/range compare; omitted blob option returned `ERR#6 ENXIO`. | -| TC100 | PASS | Four providers with option order 3/1/2; full read, 262144-byte range across both transitions, and three concurrent reads passed; missing/duplicate/short returned `ERR#6/22/6`. | -| TC101 | PASS | Explicit unified, nonzero-ID flatdev, device-ID-0 flatdev, and nonzero-high 48-bit reads matched source; gap, explicit/unified cross-slot chunk, explicit/flatdev two-block pcluster, and largest non-NULL 48-bit address all returned `read ERR#97 EINTEGRITY`. | -| TC118 | PASS | No options/one option/short returned `ERR#6`; duplicate/primary-as-slot returned `ERR#22`; normal detach `ERR#16`; forced orphan cold slot-2 read `ERR#6`; second mount `ERR#16`; primary and external missing paths preserved `ERR#2`. | - -## External-data proof - -The TC094 compressed primary is 8192 bytes and contains only metadata plus its -device table. The original pcluster blocks are absent. Its HEAD pblk is the -slot-1 unified address, and the 8192 compressed bytes exist only in the -external provider. FreeBSD returned the complete source SHA256 -`8dd7830946e7154d9feaaa87df6d1d39e47d51da22a690c5469dc4f2955ea0f1`; -therefore this was an external compressed read, not table-only parsing. - -TC101 also mounted generated combined flatdev providers with no `device.N` -options. Both nonzero device-ID mapping and device-ID-0 unified mapping -returned the same source SHA256 values as explicit providers. erofs-utils -1.8.6 does not qualify these two forms, so the FreeBSD full/range comparisons -are the runtime qualification. - -## Errno summary - -- `EBUSY` = 16: live md detach and second concurrent mount. -- `ENXIO` = 6: missing/short media and forced-orphan cold I/O. -- `ENODEV` = 19: mapped but undeclared chunk device ID 3. -- `EINVAL` = 22: duplicate provider and primary reused as an external slot. -- `ENOENT` = 2: nonexistent primary or external pathname. -- `EINTEGRITY` = 97: malformed tables and complete-extent range failures. - -## Lifecycle and platform behavior - -Every TC ended with zero EROFS mounts, zero md units, no loaded EROFS KLD, and -no matching GEOM consumer. TC097 repeated this after each individual malformed -table. TC118 forced md92 orphaning, observed cold `read ERR#6 ENXIO`, then -unmounted and released the remaining providers without panic or hang. The -final dmesg anomaly scan for panic, fatal trap, assertion, watchdog, or EROFS -errors was empty. - -Linux device tables allow table offset zero and skip `uniaddr=0` during -device-ID-0 lookup while retaining explicit nonzero-ID selection. FreeBSD -matches those on-disk semantics but receives external providers through -explicit `device.` mount options and owns read-only GEOM consumers. -Consequently FreeBSD-specific `EBUSY` detach and forced-orphan `ENXIO` behavior -was tested directly rather than inferred from Linux loop devices. - -## Issues - -None. No kernel failure remained after correcting the fixture's reserved hole -sentinel, so no issue file and no source change were created. diff --git a/tests/results/manual/2026-08-09T1343Z-g8/manual-test-report.md b/tests/results/manual/2026-08-09T1343Z-g8/manual-test-report.md deleted file mode 100644 index fc81720..0000000 --- a/tests/results/manual/2026-08-09T1343Z-g8/manual-test-report.md +++ /dev/null @@ -1,269 +0,0 @@ -# repo22 G8 Documentation, NFS, and Integrity Manual Regression - -Date: 2026-08-09 13:07-13:56 UTC - -Executor: G8 manual agent - -Source baseline: `f11fff5b8e8050e1017ed86f0bcf71042b2b45aa` - -Exact scope: `TC111`, `TC131`-`TC133`, `TC154`-`TC156` (7 IDs) - -## Result - -All seven requested test cases passed. There are no KFAIL, SHELVED, or ENV -results, no duplicate IDs, and no omitted IDs. - -| Status | Count | -| --- | ---: | -| PASS | 7 | -| KFAIL | 0 | -| SHELVED | 0 | -| ENV | 0 | -| Duplicate | 0 | -| Omitted | 0 | - -| Test ID | Status | Manual evidence | -| --- | --- | --- | -| TC111 | PASS | Audited current source and documentation against real generic `/sbin/mount`; corrected helper and writable-request claims. | -| TC131 | PASS | Real mountd/nfsd NFSv3 export, four vnode types, read-only behavior, 16-byte handle resolution, and RPC statistics. | -| TC132 | PASS | Handle ABI/classes, stable remount, replacement `ESTALE`, metabox and external-provider regressions, and nfsd restart. | -| TC133 | PASS | Exact 12,050-name READDIRPLUS lists, five cold remounts, 20 individually waited workers, and zero RPC errors. | -| TC154 | PASS | Same-superblock/UUID/NID replacement changed per-inode generation; unchanged remount stayed stable. | -| TC155 | PASS | Explicit-extent physical-address wrap returned `EINTEGRITY` on both direct reads and in `truss`. | -| TC156 | PASS | Compact/extended epoch, `time_t`, and nanosecond boundary corruptions returned `EINTEGRITY`. | - -No kernel source changed. No kernel failure remained, so no issue file was -created. - -## Isolated Environment - -- Worktree: `/work/build/repo22-manual-g8-20260809T130723Z`, branch - `manual-g8-20260809T130723Z`, created directly from the source baseline. -- VM: independent qcow2 overlay backed by the clean FreeBSD development base; - QEMU TCG, 6144 MB, 4 vCPUs, SSH only on `127.0.0.1:9228`. -- Guest: FreeBSD `15.0-RELEASE-p8` amd64, - `releng/15.0-n281036-53054229dcb3`, OSREL `1500068`. -- Guest kernel SHA256: - `b9abf7b58f9dd4d87f14d2fbc306cf255e6eadc47b8ae885a230f65e28be4562`. -- Host erofs-utils: 1.8.6. -- No CI, runner, or test wrapper was used. Markdown procedures were issued - manually through SSH, with direct command status kept separate from tracing - tool status. - -## Exact Build - -The repo22 source archive came from the exact baseline commit. The matching -tracked FreeBSD build input was `dev-freebsd-releng/sys`, identified by -`REVISION="15.0"` and `BRANCH="RELEASE-p9"`. The build ran natively in the -guest as: - -```sh -FREEBSD_SRC=/root/freebsd-src WITH_ZSTDIO=1 ./build.sh -``` - -The compile command contained `-DZSTDIO` and kernel `-Werror`. The module had -the five expected `ZSTD_*` kernel references and no unresolved `bcmp`. - -| Object | SHA256 | -| --- | --- | -| repo22 source archive | `2f76ef7df9117c6e7d62ad41809638ad306e676191346083bf95fe551f7df515` | -| FreeBSD 15 `sys` archive | `cd806ac4d6aee5d7ceb6a2f9603020b48b5012835bc8c7f2ec0985572fcd0262` | -| `erofs.ko` | `e8cd4839328de089355505efb1629c835d2f93faad2e14f63c6e6cb844ce9589` | -| `nfs_fh_tool` | `db704f67d718d04b84ed32b5cdbecb93a99c5fe3b4ab238d603a10e005d790e3` | - -The guest kernel is p8 and the tracked source is p9, but both use the FreeBSD -15.0 OSREL ABI. The exact module loaded and unloaded successfully. - -## Fixture Qualification - -`tests/review_fixtures.py` generated two independent output directories. Their -manifests and checksum inventories were byte-identical, and all nine image -checksums self-verified. Its structured assertions recorded: - -- target NID 452 at inode offset 14464 after checksum block 4096; -- identical complete superblock block and UUID for NFS images A and B; -- generations `3895653226 -> 548470773` from the changed raw inode; -- explicit extent base `0xfffffffffffff000`, `plen0=8192`, `plen1=4096`, and - wrap before logical cluster 4096; -- compact epoch wrap/range, compact nanoseconds 1000000000, extended - nanoseconds 1000000000, and extended seconds `INT64_MAX+1`. - -Review generator SHA256 was -`b92fd6b6122d55882998138efc0109e24b236f9e877405dbff7914284a18c124`; -manifest SHA256 was -`be0836043d172b34eb625e5259f7e6cb9a8992ff74e7d1a5cdc10d69282e2335`. - -The NFS stress image was generated from a fresh deterministic tree containing -12,050 `bigdir` files, 256 concurrent-read files, four basic vnode types, and a -22,020,096-byte throughput file. Image A SHA256 was -`88721877dc3762c8eafdf3bcca2653787cdcf41983978ab159705761abefac37`; -image B was -`964b3d75579740c9b08bc9cae8329e3d641d3c505e9391de3ab9f5d24fbecd17`. -The expected sorted-name list SHA256 was -`4fbf1b6103e8884223629c3fd03f4ec36061e65cd2a1becfc96670daa96f9d71`. - -Fresh structured metabox and multidevice helpers also self-verified the two -TC132 regression inputs. The metabox image SHA256 was -`dd7b04097d1bfe283b65c95d759acd2176beb7cb0f1fe9d5ddbc0694b5acba9d`; -the multidevice primary/slot hashes were `918aa3a64e8861011914d967912909ce9de98418cff711cfbf8f91c0c7717f72`, -`4e31d95a12405f2f1396fb926a6854c591fc335eade8944f73fdc7fe6758aee6`, -and `355a74880bd3648dea22ad18e19f809cee12303ceed2a99f53094c2b66563a11`. - -## TC111 - -`/sbin/mount_erofs` was absent and `command -v mount_erofs` returned 127. -Generic `/sbin/mount -t erofs` mounted the image successfully without `-o ro`. -The following real behavior was recorded: - -- the default mount was read-only and `touch` failed with `EROFS`; -- `-o rw` returned 0 but the resulting mount was still read-only; -- `-o ro,noexec,nosuid` returned 0 and all three flags appeared; -- non-export `mount -u` returned `EOPNOTSUPP`; -- an unknown filesystem option returned direct rc 1, with - `nmount(..., MNT_RDONLY) ERR#22`, and created no mount. - -Current `src`, `README.md`, `docs/features.md`, `docs/architecture.md`, -`docs/erofs.5`, and current progress material were checked. `README.md`, -`docs/erofs.5`, and the test procedure now describe the generic frontend and -the forced-read-only `rw` behavior accurately. The man page's split-device -example was also exercised by the TC132 multidevice mount. SEE ALSO entries -absent from the qualified guest were removed; `mandoc -Tlint -Werror` and -ASCII rendering both returned 0 in that guest. - -## TC131 - -The direct EROFS line lacked `NFS exported` before mountd. After installing the -loopback export and reloading mountd, `showmount -e` listed the exact path and -the direct mount gained `NFS exported`. `rpcinfo` showed NFSv3 and mountd over -TCP and UDP. - -The NFSv3 TCP client negotiated `rdirplus`. Regular file content, directory, -symlink target, FIFO type, and inode number matched the direct EROFS mount. -`touch` failed with `Read-only file system`. Local `fhstat` and `fhopen` -resolved the regular handle with `len=16`, `pad=0`, NID 116, and generation -2068234290. - -After basic operations client READDIRPLUS was 2. Server Write and Create were -both 0. TimedOut, Invalid, X Replies, and Retries were all 0. - -## TC132 - -Regular, directory, symlink, and FIFO handles all had `len=16`, `pad=0`, full -64-bit NIDs, and nonzero generations matching `st_gen`. Bad length/pad returned -`EINVAL`; bad generation/NID returned `ESTALE` through both `fhstat` and -`fhopen`. - -The unchanged image remounted on md80 preserved the complete handle SHA256 -`9e1df1952b43d383fea295301525ede7a69b7d57e5f60a75a6d7f1531e13df0a`, -and the old handle still read the file. Replacement B retained fsid and NID -116 but changed generation `2068234290 -> 857800581`; both old-handle paths -returned `ESTALE` while B resolved. - -The metabox handle preserved NID `0x8000000000000010`; invalid metabox NID and -generation returned `ESTALE`. After forced orphaning of external slot 2, the -valid file handle still resolved metadata and the read returned `ENXIO(6)`, -not `ESTALE`. - -An NFS client descriptor remained open across `service nfsd onerestart 3<&-`. -The old descriptor and a new path read both returned SHA256 -`3e2d0b4971e1f4bf00c3562a8bf9c5d85378c9de054f3da89541f49f2b449a7b`; -the export and all RPC registrations remained live. - -## TC133 - -Requested readdir sizes 512, 1024, and 4096 were all clamped by the FreeBSD -client to 8192; the default remained 65536. This is the recorded client floor. - -Every one of the four mounts returned exactly 12,050 unique expected names. -Every sorted list, plus all five cold-remount lists, had SHA256 -`4fbf1b6103e8884223629c3fd03f4ec36061e65cd2a1becfc96670daa96f9d71`. -The `ls -a1` list contained 12,052 entries, with `.` and `..` exactly once. - -All 12 traversal workers were waited by PID and returned 0. All four parallel -cat workers and four parallel stat workers were also waited by PID and returned -0. The informational 22,020,096-byte read completed in 14.51 seconds. - -Final client/server READDIRPLUS was 3420. TimedOut, Invalid, X Replies, and -Retries remained 0. Server Write, WriteRPC, Create, and Commit remained 0. -There were no new stale-handle, timeout, retry, panic, trap, assertion, -watchdog, or EROFS error lines. - -## TC154 - -Image A produced fsid `00000034:000000e0`, `len=16`, `pad=0`, NID 452, and -generation 3895653226. `nfs_fh_tool`, `stat st_gen`, and the fixture manifest -agreed. The unchanged remount preserved complete handle SHA256 -`79f2f4b8e354af0ae369849d7f66a739162ffadc4a5689c09643c9949b548de1` -and the old handle remained readable. - -Image B retained the complete superblock block, UUID, fsid, NID, and content, -but generation became 548470773. Both old-handle paths returned `ESTALE`; B's -handle resolved. The malformed same-NID replacement returned positive errno 45 -through both paths. - -## TC155 - -The self-checked fixture SHA256 was -`1e49ecf1917265fde9b606e80f7dbae8b14aca8c6f149128b31d6c825da546dd`. -The image mounted and target `stat` reported inode 40, size 1,048,576. Two -direct one-byte reads at logical offset 4096 returned rc 1 and `Integrity check -failed`; the traced read was `ERR#97`. The repeat exercised the cached vnode. -A DTrace `io:::start` positive control observed one raw -`md90 offset=4096 bytes=4096` event. A separate trace around the failing EROFS -read observed zero md90 events, proving the wrapped address did not reach the -provider. No panic or resource remained. - -## TC156 - -Both compact root corruptions failed `nmount` with `ERR#97`. The compact range -image mounted with root mtime `INT64_MAX`; target stat returned `ERR#97` for -`INT64_MAX+1`. Both extended images mounted, and target stat returned `ERR#97` -for nanoseconds 1000000000 and seconds `INT64_MAX+1`. Every failure was run -directly and again under `truss`; no panic or resource remained. - -## Cleanup and Restoration - -Before NFS setup, `/etc/exports` did not exist; rpcbind, mountd, and nfsd were -stopped; and `rpcbind_enable`, `mountd_enable`, and `nfs_server_enable` were all -`NO`. Cleanup unmounted every loopback client while nfsd was alive, cleared and -reloaded exports, stopped nfsd/mountd/rpcbind, then removed the direct EROFS -mount, md42, and the module. - -Final state exactly matched the baseline: `/etc/exports` absent, all three -services stopped with rc 1, all three rc settings `NO`, zero NFS/EROFS mounts, -zero md providers, no EROFS KLD, no service process, and `rpcinfo` refused the -connection because rpcbind was stopped. The complete NFS-period dmesg delta had -zero anomaly lines. - -The retained guest evidence archive is outside the repository at -`/work/build/repo22-manual-g8-20260809T130723Z-vm/evidence/`: - -| Artifact | SHA256 | -| --- | --- | -| `final-guest-evidence.txt` | `eb67210818e663c15e4670ada9ba63b82b44e9ed3a9486c575df3a71ed3f0522` | -| `TC155-dtrace-assert.txt` | `f8b52985a74c8c777b3b830c39749be65c613165ffbcde6d0018dc471018c433` | -| `final-post-dtrace-cleanup.txt` | `90f04d8f5a4ae35c87cc3adb1aeb939c7cc5534233ed63e9ff4501c83fd857d2` | -| `repo22-g8-evidence.tar.gz` | `9a378fd619f9bca21f40689d746308933aa22e440f7201104732f91634c21290` | - -## Non-Qualified Attempts - -One TC131 setup command stopped at the expected nonzero service-status probe -because that probe was mistakenly under `set -e`; it occurred before module, -exports, or service changes. One TC133 attempt stopped after the four exact -lists because a shell-quoted dot assertion matched zero; the three added -clients were unmounted and the complete procedure was rerun from the start. -Tracing wrappers returned 0 even when their child failed, so all error claims -use separate direct statuses plus syscall errno. None of these discarded -attempts is counted as PASS evidence. - -Two initial DTrace specifications used field names from other provider ABIs -and failed at compile time before any probe I/O. The qualified specification -used FreeBSD `struct devstat` and `struct bio` fields. Its first assertion used -a line anchor even though the format emitted a literal `\\n`; substring counts -then proved one positive md90 event and zero failing-read md90 events. DTrace's -automatically loaded module set was unloaded as a unit and the original KLD -set was reverified. - -## Issues - -None. diff --git a/tests/results/manual/2026-08-09T1359Z-g7/manual-test-report.md b/tests/results/manual/2026-08-09T1359Z-g7/manual-test-report.md deleted file mode 100644 index 682a3f0..0000000 --- a/tests/results/manual/2026-08-09T1359Z-g7/manual-test-report.md +++ /dev/null @@ -1,183 +0,0 @@ -# repo22 G7 boundary and stress full manual regression report - -## Scope and result - -- Exact scope: TC120-TC130 inclusive. -- Count check: 11 requested, 11 executed, 0 duplicate, 0 omitted. -- Result: **PASS 11, KFAIL 0, SHELVED 0, ENV 0**. -- No repo22 kernel source was changed. -- No kernel failure occurred, so no G7 issue file was opened. -- No CI, runner, or test wrapper was used. Each test's Markdown commands were - issued manually in the dedicated FreeBSD guest. - -## Baseline and source identity - -- Requested initial `xdm/main`: `c566d8ac6bf8e801082bbccf108451f6ee46ad40`. -- G6 pushed first; G7 fetched and rebased onto - `f11fff5b8e8050e1017ed86f0bcf71042b2b45aa` before editing. -- Exact build commit: - `e44e24c2d7e4955498fb42b06bcf376d6c452190`. -- The equivalent G7 test/source commit after the final remote rebase is - `896683f14af2ba36705498f85d0d8af02b513d31`; its repo22 source tree is - byte-identical to the build input. -- Exact `repo-community/repo22/src` tree: - `34a56a583b3b43e7e103b26a30dea1d400c2bc6c`. -- G8 later advanced `xdm/main` to `6f336d0a7`; its repo22 source tree was the - same `34a56a58...`. G7 rebased before final push without force. -- Build configuration: `WITH_ZSTDIO=1`. -- Tracked FreeBSD sys source: `REVISION="15.0"`, `BRANCH="RELEASE-p9"`. -- repo22 source archive SHA256: - `e715e9d323be1448e0395add99cf017dea3c1bbb6579915cf3d07f7265ca2891`. -- FreeBSD sys archive SHA256: - `71f535ab5a9ef686c6adbe123a1d1e65fd9bfa9cd19724f9fcef222a8efbe9d8`. - -## Isolated environment - -- Worktree: `/work/build/repo22-manual-g7-20260809T125507Z`. -- Branch: `manual-g7-20260809T125507Z`. -- Artifact root: `/work/build/repo22-g7-20260809T125507Z`. -- Dedicated qcow2 overlay backed by `/work/build/vm-freebsd-dev-base.qcow2`. -- Dedicated SSH forward: `127.0.0.1:9227`. -- QEMU: TCG multi-thread, `qemu64`, 6144 MiB RAM, 4 vCPUs, virtio disk/network. -- Guest: FreeBSD `15.0-RELEASE-p8`, amd64, - `releng/15.0-n281036-53054229dcb3`, OSREL `1500068`. -- Guest kernel SHA256: - `b9abf7b58f9dd4d87f14d2fbc306cf255e6eadc47b8ae885a230f65e28be4562`. -- Host erofs-utils: 1.8.6. -- RACCT/RCTL was initially present but disabled. The dedicated overlay set - loader tunable `kern.racct.enable="1"`, rebooted, then reported enabled with - an empty initial rule set. - -## Exact KLD and probe - -The KLD built successfully from the archived commit with the tracked p9 sys -tree and loaded by full pathname. `kldstat -v` assigned file ID 5 and showed -module ID 507 for `erofs`. - -| Artifact | SHA256 | -|---|---| -| `/root/repo22-g7/erofs.ko` | `06dfa530efb2d494950672d60453d86a6e70f06ca2ce7b4eff3396ca8018c76d` | -| `/root/repo22-g7/g7_probe` | `a0bca6ea596cec6063bc058aebee12929a1814e6d343283ffbe051551bf75c57` | - -The KLD was an amd64 FreeBSD relocatable object with build ID -`d4fff529ae8e4c5395e1e867fd4ea5d57a9570b6`. The expected formal FreeBSD ZSTD -undefined symbols were present; no build or load error occurred. - -## Reproducible fixture - -Two fresh generations produced byte-identical source inventories, source -checksum lists, and image checksum lists. Each generation independently ran -the helper verification and erofs fsck checks. - -```text -source file count = 24023 -image count = 2 -SOURCE-INVENTORY.tsv SHA256 = b9be1745900a8cf6755068b6a78b93bc294cf8857539159b7771e5170bc1b9ed -SOURCE-SHA256SUMS SHA256 = 445929465a3031e84de9887fc5b02d72cb797df7ca512ea1cc874d11f77747d3 -SHA256SUMS SHA256 = f9cf45c1e3fe814290d9cd0b96d066509309cc3f764c545d5e7d7ddae4a9eac3 -fixture-manifest.json SHA256 = 0ba86269962b00329b99177494cfb099c0f58fe5de77d2dc1f12da684dedf09d -guest image/source verification = 2 images and 24023 sources, all OK -``` - -| Image | Bytes | SHA256 | -|---|---:|---| -| `boundaries.erofs` | 37752832 | `ff7e804dc309039fa5dfa9b6e75416d16224137c8f6d6d96bf04cfa0ba8a1a8c` | -| `workloads.erofs` | 155111424 | `5f21a045114c5956b80752736c9833fe6a975a296cd28324734c9b9e423f3984` | - -The sparse source was exactly 4294971393 bytes. It occupied 48 Linux -512-byte blocks before transfer and 512 FreeBSD blocks with a reported -32768-byte file-system block size after sparse-aware extraction. All five -markers and six boundary ranges passed the helper's source self-check. - -## Per-TC results - -| TC | Manual evidence | Result | -|---|---|---| -| TC120 | Exact size 0; source/mount empty SHA256 `e3b0c442...`; read returned 0; SET/END seeks returned 0 | PASS | -| TC121 | Exact 4294971393-byte size; six source-compared ranges across block, hole, 2 GiB, 4 GiB, and EOF boundaries; EOF read 0 | PASS | -| TC122 | Exactly 128 levels; deepest source/mount hash `ef151b18...`; full cmp and exact `pwd -P` | PASS | -| TC123 | Exact 255-byte name; one exact readdir row; size 4096; source/mount hash `0724316b...` | PASS | -| TC124 | Exactly 12000 one-KiB files and exact names; complete source/target hash-list digest `be93ef11...` | PASS | -| TC125 | Exactly 12000 direct entries and exact names/hashes; five boundary lookups matched; readdir timing recorded | PASS | -| TC126 | Exactly 16 saved PIDs, 16 waits, all rc 0; every expected/actual full-file hash matched; no PID remained | PASS | -| TC127 | Three RCTL-capped allocators reached ENOMEM, released all touched bytes, recovered allocation, and wait=0; pressure read hash/cmp matched | PASS | -| TC128 | 268435456-byte source/mount hash and full cmp matched; three complete sequential reads rc 0; metrics recording-only | PASS | -| TC129 | Three deterministic 16384-operation runs, all source-compared, zero mismatch, identical digest; metrics recording-only | PASS | -| TC130 | Seven saved PIDs and seven wait=0 results; sequential, hash, two random, 16-file, names, and stat workloads all matched; full cleanup | PASS | - -TC121 range digests were `7775e9e82783e06f`, `c86358c1fb3fabf7`, -`b43a063055adc383`, `a1bd19f7b12c60b2`, `5a378d4efc494af4`, and -`1873eaea77354aa6`, in manifest order. - -TC124's complete 12000-row source and mounted hash-list files both had SHA256 -`be93ef11d2e4cd0cd42ef057397b89978b624e42674f9181e5dd151b30636171`. -TC125's complete 12000-row files both had SHA256 -`8cde4ea887f9b21b265ac199bcbb447a656aa6d718a36424f4fa03a6c538ebe7`. - -TC126 persisted each worker's PID, source hash, mounted hash, and wait status. -The PID table SHA256 was `fc67656d...`; the 16-row wait table SHA256 was -`888f026d...`. Every wait code was 0 and every hash pair was equal. - -## Pressure evidence - -TC127 used three synchronized `g7_probe` processes and valid FreeBSD RCTL -rules `process::vmemoryuse:deny=640M`. The first process utilization -sample showed `memoryuse=532M` and `vmemoryuse=636M`. - -| Worker | Allocated before ENOMEM | errno | Released | Recovery | wait rc | -|---|---:|---:|---:|---|---:| -| 1 | 553648128 | 12 | 553648128 | ok | 0 | -| 2 | 545259520 | 12 | 545259520 | ok | 0 | -| 3 | 553648128 | 12 | 553648128 | ok | 0 | - -Free memory recorded by `vmstat -H` was about 4.01 GB before, 2.35 GB while -the allocations were held, and 3.89 GB after release. The 96 MiB source and -mounted file both hashed to -`e4baaad480721bfb69a0315f0092fcf1ad75426fda0b9242a750b5685ad75152` -and full `cmp` exited 0. RCTL rules and synchronization files were removed. - -## Recording-only metrics - -TC125's full 12000-entry sorted readdir scan under TCG recorded `real 11.48`, -`user 0.81`, and `sys 9.76` seconds. It would have failed the old invalid -10-second fixed threshold despite complete correctness, so timing was -correctly treated as recording-only. - -TC128 sequential reads: - -| Run | Bytes | Seconds | Bytes/s | wait rc | -|---|---:|---:|---:|---:| -| 1 | 268435456 | 6.764841 | 39680970 | 0 | -| 2 | 268435456 | 9.207558 | 29153816 | 0 | -| 3 | 268435456 | 10.603392 | 25315998 | 0 | - -TC129 used seed `0x6a09e667f3bcc909`, 16384 operations, and 4096-byte -blocks. All runs produced digest `908bc2905f48695d` and zero mismatches. - -| Run | Seconds | IOPS | Mean us | -|---|---:|---:|---:| -| 1 | 2.236047 | 7327.22 | 136.48 | -| 2 | 1.062087 | 15426.24 | 64.82 | -| 3 | 1.026698 | 15957.95 | 62.66 | - -TC130 random workers ran concurrently with the other five workloads. Seed -`0xbb67ae8584caa73b` recorded 442.73 IOPS, 2258.72 us mean, digest -`78e9d2c7f86ed20e`; seed `0x3c6ef372fe94f82b` recorded 455.91 IOPS, -2193.42 us mean, digest `ee76a5d094a0c6f2`. Both had zero mismatches. -The exact names files shared SHA256 `8d1cf816...`; exact stat files shared -SHA256 `ca8937b1...`. - -## Stability and cleanup - -- Final dmesg delta was empty; its SHA256 was the empty-file hash. No panic, - trap, assertion, watchdog, OOM, or EROFS error appeared. -- Every test detached its exact md provider after unmounting. -- Final guest state had zero EROFS mounts, md providers, G7 child processes, - RCTL rules, synchronization files, and Python bytecode caches. -- KLD file ID 5 was unloaded with `kldunload -i 5`; no EROFS allocator row or - KLD remained. -- The guest remained responsive, powered off normally, QEMU PID 344535 exited, - port 9227 was released, and the dedicated overlay/log/PID files were deleted. -- Compressed evidence archive SHA256: - `f18246076e5ff93309187a1826afe9f2dfa4a0bcdf7a958419d2890d05727ccd`. -- No issues were created because all correctness gates passed. diff --git a/tests/results/manual/2026-08-09T1804Z-final-review-independent/manual-test-report.md b/tests/results/manual/2026-08-09T1804Z-final-review-independent/manual-test-report.md deleted file mode 100644 index 70921bd..0000000 --- a/tests/results/manual/2026-08-09T1804Z-final-review-independent/manual-test-report.md +++ /dev/null @@ -1,119 +0,0 @@ -# repo22 Final Review Independent Manual Regression - -Execution window: 2026-08-09 18:04-18:29 UTC - -Source baseline: `fcc85b93d5f8fd9671686bd68bf3b086c8bd25cf` - -Scope: TC157-TC161, final dual-configuration build and mount smoke, TC111 -coverage-matrix follow-up, and resource hygiene. No CI runner or automated -kernel-test harness was used. - -## Result - -All five assigned test cases passed. - -| Test ID | Status | Manual evidence | -| --- | --- | --- | -| TC157 | PASS | Six 16/32-byte descending, duplicate, and cross-branch tables returned positive `EINTEGRITY`; repeated lookup agreed; a raw offset-4096 control was observable while failed lookups produced no payload read. | -| TC158 | PASS | A 17,592,186,056,704-byte sparse provider produced direct and file-handle `st_blocks=34359738384`; two FBT returns recorded `va_bytes=17592186052608`; the file remained readable. | -| TC159 | PASS | FIFO size-only setattr returned 0; size plus mode, owner, times, and all fields returned `EROFS(30)`; metadata remained `10640:0:0:0:0`. | -| TC160 | PASS | Both `OFF_MAX` and zero-offset `getdirentries` calls returned `EINTEGRITY(97)` and preserved their offsets; independent truss SHA256 was `6df9c594bfba8aaf849fa7105fda09a5cab99d79f04c42901a438321c90700e0`. | -| TC161 | PASS | Both normal configurations built, the private `nm` shim failed before publication without a SUCCESS line or temporary file, the previous KLD hash stayed unchanged, and a final normal build used `/usr/bin/nm` and exited 0. | - -The TC157 fixtures do not cover a first nonzero `lstart` or an extreme -extent-count performance boundary. These are residual coverage limits, not -observed failures. Positive mapped-payload coverage remains the separate TC146 -PARTIAL issue. - -## Exact Inputs - -The source archive exported from the baseline had SHA256 -`a0c330601f9d7b2aa246a17e67bcd67914d457480cb69df025d09bc34d8b7b31`. -The independent fixture archive had SHA256 -`bf8ab5430f06dfc067f041002cc7861b0b110dc00d9c447a82877f21b67b09ed`. -Its manifest and evidence hashes were -`b880be5d214985e6a7f75cc77fea333f046a30963a9d310d57a6cdc67104e740` -and `ace71446671b3d17e6a46997f43fc605a7a6bb5993c9de0e6e12cd223746b317`. - -The fixture evidence recorded: - -- complete TC157 logical-start arrays and old binary-search visit indices; -- TC158 `blocks_lo=2`, `blocks_hi=1`, and 4,294,967,298 data blocks; -- TC159 compact FIFO mode `010640`; -- TC160 `i_size=9223372036854775807` and expected errno 97. - -## TC161 Build Qualification - -The shim log SHA256 was -`a1a83797ba2c0111b66bb8783f3a67087420ef7a9bccc93fd42fb94c9c24c593`. -It contained `ERROR: nm failed while checking erofs.ko`, no SUCCESS line, and -left no `nm-undef.*` file. The post-shim status file recorded `exit=0` and -`nm=/usr/bin/nm`; its build log SHA256 was -`5ce2519a85de7a4846f95fdb858fd7818182aad3fa55a7ea8d06cf7687143f11`. - -The post-shim build used an explicit normal path and environment: - -```sh -env PATH=/usr/bin:/bin:/usr/sbin:/sbin \ - FREEBSD_SRC=/tmp/repo22-freebsd15-src \ - WITH_ZSTDIO=0 ./build.sh -``` - -Its key log lines were `==> Building repo22 erofs.ko` and -`==> SUCCESS: .../build/erofs.ko`. - -Final exact-source builds used FreeBSD 15 kernel `-Werror`: - -| Configuration | KLD SHA256 | Result | -| --- | --- | --- | -| `WITH_ZSTDIO=0` | `15fda9d334132cd81769ce4dff4f8411a6e2b4cf7531c352530d0de85f42a2d2` | PASS | -| `WITH_ZSTDIO=1` | `23782dc0ce7da188807d35020bf2d8c6044b796c5c9a8746b398ba784cd6ad4e` | PASS | - -The final ZSTD build log SHA256 was -`e58bc73d1b0b8295d8baba0cf0ce07bf4e9c994837e65c7ccdd731dc91edb283`. - -## Final Module Smoke and Cleanup - -Each final KLD loaded independently, mounted `special-setattr.erofs` through a -fresh vnode md provider, exposed the expected FIFO, unmounted, detached the md -unit, and unloaded. The smoke log SHA256 was -`7ced7329da9f39527cf903213dee4c99dbe6d8d6fa96e55bca3427c4f4a56420`. - -Final guest state was: - -```text -residual-mount-count=0 -residual-md= -residual-kld-count=0 -smoke-exit=0 -``` - -The host repo22 ignored `build/`, old ignored generated fixture/artifact trees, -and `__pycache__` content were removed before aggregation. No generated KLD, -object, image, or Python bytecode is acceptance evidence in Git. - -## TC111 Coverage-Matrix Follow-up - -A bounded filename audit found 162 specifications, TC000 through TC161, with -162 unique IDs, no duplicate, and no missing ID. TC000 is a template and is not -counted as executable. - -The eight canonical G1-G8 report tables contained exactly 156 rows and 156 -unique IDs covering TC001-TC156, with no duplicate or omission. TC157-TC161 -add five unique executable cases. TC060's original G3 failure is superseded by -its dated fixed-source PASS report; TC153's G3 result is PASS. - -The canonical final result is therefore: - -```text -Executable test cases: 161 -PASS: 160 -PARTIAL: 1 (TC146) -FAIL/KERNEL-FAIL/ENV/SHELVED_TC: 0 -``` - -TC146's shelved positive mapped-payload subitem is not counted as an additional -test case. G1-G8 evidence was collected across multiple source commits; this -report does not claim that every historical test ran on the final KLD. The -final `fcc85b93d` code baseline received the independent TC157-TC161 checks and -the affected final build/load/mount regressions recorded here. diff --git a/tests/review_fixtures.py b/tests/review_fixtures.py deleted file mode 100644 index 01b76d5..0000000 --- a/tests/review_fixtures.py +++ /dev/null @@ -1,608 +0,0 @@ -#!/usr/bin/env python3 -"""Build and self-check fixtures for TC154-TC156.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import shutil -import struct -import subprocess - - -SUPER = 1024 -EROFS_MAGIC = 0xE0F5E1E2 -FEATURE_COMPAT_SB_CHKSUM = 0x1 -EROFS_INODE_FLAT_PLAIN = 0 -EROFS_INODE_FLAT_INLINE = 2 -EROFS_INODE_COMPRESSED_FULL = 1 -Z_EROFS_ADVISE_EXTENTS = 0x1 -UINT64_MAX = (1 << 64) - 1 -INT64_MAX = (1 << 63) - 1 -FNV1_32_INIT = 33554467 -FNV_32_PRIME = 0x01000193 - - -class FixtureError(RuntimeError): - pass - - -class Inode: - def __init__( - self, - nid: int, - offset: int, - inode_format: int, - inode_size: int, - xattr_size: int, - layout: int, - size: int, - start_block: int, - ) -> None: - self.nid = nid - self.offset = offset - self.inode_format = inode_format - self.inode_size = inode_size - self.xattr_size = xattr_size - self.layout = layout - self.size = size - self.start_block = start_block - - -class ErofsImage: - def __init__(self, data: bytes | bytearray) -> None: - self.data = bytearray(data) - self.validate_superblock() - - @classmethod - def load(cls, path: Path) -> "ErofsImage": - return cls(path.read_bytes()) - - def clone(self) -> "ErofsImage": - return ErofsImage(self.data) - - def u16(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - struct.pack_into(" None: - struct.pack_into(" None: - struct.pack_into(" int: - return self.data[SUPER + 12] - - @property - def block_size(self) -> int: - if not 9 <= self.block_bits <= 16: - raise FixtureError(f"invalid block bits: {self.block_bits}") - return 1 << self.block_bits - - @property - def super_size(self) -> int: - return 128 + self.data[SUPER + 13] * 16 - - @property - def checksum_end(self) -> int: - length = self.block_size - if length > SUPER: - length -= SUPER - return SUPER + length - - @property - def feature_compat(self) -> int: - return self.u32(SUPER + 8) - - @property - def root_nid(self) -> int: - feature_incompat = self.u32(SUPER + 80) - root_nid_8b = self.u64(SUPER + 112) - if feature_incompat & 0x80 and root_nid_8b != 0: - return root_nid_8b - return self.u16(SUPER + 14) - - def calculated_checksum(self) -> int: - window = bytearray(self.data[SUPER : self.checksum_end]) - struct.pack_into("> 1) ^ ( - polynomial if checksum & 1 else 0 - ) - return checksum & 0xFFFFFFFF - - def checksum_valid(self) -> bool: - if not self.feature_compat & FEATURE_COMPAT_SB_CHKSUM: - return True - return self.u32(SUPER + 4) == self.calculated_checksum() - - def update_checksum(self) -> None: - if not self.feature_compat & FEATURE_COMPAT_SB_CHKSUM: - raise FixtureError("base image lacks superblock checksum support") - self.put_u32(SUPER + 4, 0) - self.put_u32(SUPER + 4, self.calculated_checksum()) - if not self.checksum_valid(): - raise FixtureError("updated superblock checksum does not verify") - - def validate_superblock(self) -> None: - if len(self.data) < SUPER + 144: - raise FixtureError("image is too short for an EROFS superblock") - if self.u32(SUPER) != EROFS_MAGIC: - raise FixtureError(f"bad EROFS magic: {self.u32(SUPER):#x}") - if SUPER + self.super_size > len(self.data): - raise FixtureError("declared superblock is truncated") - if not self.checksum_valid(): - raise FixtureError("superblock checksum is invalid") - - def inode(self, nid: int) -> Inode: - metadata = self.u32(SUPER + 40) << self.block_bits - offset = metadata + (nid << 5) - if offset > len(self.data) - 32: - raise FixtureError(f"nid {nid} lies outside the image") - inode_format = self.u16(offset) - inode_size = 64 if inode_format & 1 else 32 - if offset > len(self.data) - inode_size: - raise FixtureError(f"nid {nid} has a truncated inode") - xattr_count = self.u16(offset + 2) - xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1) - size = ( - self.u64(offset + 8) - if inode_size == 64 - else self.u32(offset + 8) - ) - return Inode( - nid=nid, - offset=offset, - inode_format=inode_format, - inode_size=inode_size, - xattr_size=xattr_size, - layout=(inode_format >> 1) & 7, - size=size, - start_block=self.u32(offset + 16), - ) - - def directory_data(self, inode: Inode) -> bytes: - if inode.size == 0: - raise FixtureError("empty directory cannot contain a target") - if inode.layout == EROFS_INODE_FLAT_PLAIN: - offset = inode.start_block << self.block_bits - end = offset + inode.size - if end > len(self.data): - raise FixtureError("plain directory data is truncated") - return bytes(self.data[offset:end]) - if inode.layout != EROFS_INODE_FLAT_INLINE: - raise FixtureError(f"unsupported directory layout {inode.layout}") - tail_start = ((inode.size + self.block_size - 1) // self.block_size - 1) - tail_start *= self.block_size - full_offset = inode.start_block << self.block_bits - inline_offset = inode.offset + inode.inode_size + inode.xattr_size - full = self.data[full_offset : full_offset + tail_start] - tail = self.data[inline_offset : inline_offset + inode.size - tail_start] - if len(full) + len(tail) != inode.size: - raise FixtureError("inline directory data is truncated") - return bytes(full + tail) - - def directory_entries(self, inode: Inode) -> list[tuple[bytes, int]]: - data = self.directory_data(inode) - entries: list[tuple[bytes, int]] = [] - for block_start in range(0, len(data), self.block_size): - block = data[block_start : block_start + self.block_size] - if len(block) < 12: - raise FixtureError("short directory block") - first_nameoff = struct.unpack_from("= len(block) - ): - raise FixtureError("invalid first directory name offset") - count = first_nameoff // 12 - previous = 0 - for index in range(count): - entry_offset = index * 12 - nid = struct.unpack_from(" len(block) - ): - raise FixtureError("invalid directory name range") - name = block[nameoff:endoff].split(b"\0", 1)[0] - if not name: - raise FixtureError("empty directory name") - entries.append((name, nid)) - previous = nameoff - return entries - - def resolve_root_entry(self, name: str) -> Inode: - encoded = name.encode("ascii") - root = self.inode(self.root_nid) - for entry_name, nid in self.directory_entries(root): - if entry_name == encoded: - return self.inode(nid) - raise FixtureError(f"root entry not found: {name}") - - def inode_bytes(self, inode: Inode) -> bytes: - return bytes(self.data[inode.offset : inode.offset + inode.inode_size]) - - def generation(self, inode: Inode) -> int: - seed = fnv1_32( - bytes(self.data[SUPER : SUPER + self.super_size]), FNV1_32_INIT - ) - generation = fnv1_32(struct.pack(" None: - self.validate_superblock() - path.write_bytes(self.data) - - -def fnv1_32(data: bytes, initial: int) -> int: - value = initial - for byte in data: - value = (value * FNV_32_PRIME) & 0xFFFFFFFF - value ^= byte - return value - - -def align(value: int, alignment: int) -> int: - return (value + alignment - 1) & -alignment - - -def run_mkfs(source: Path, image: Path, uuid: str, *extra: str) -> None: - command = [ - "mkfs.erofs", - "-d0", - "-x-1", - "-T0", - "--all-time", - "--all-root", - "--workers=1", - "--sort=path", - "-U", - uuid, - *extra, - str(image), - str(source), - ] - subprocess.run(command, check=True) - - -def normalize_times(root: Path) -> None: - for path in sorted(root.rglob("*"), reverse=True): - os.utime(path, (0, 0), follow_symlinks=False) - os.utime(root, (0, 0), follow_symlinks=False) - - -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def make_fixtures(output: Path) -> None: - if shutil.which("mkfs.erofs") is None: - raise FixtureError("mkfs.erofs is required") - if output.exists() and any(output.iterdir()): - raise FixtureError(f"output directory is not empty: {output}") - output.mkdir(parents=True, exist_ok=True) - source = output / "source" - nfs_source = source / "nfs" - extent_source = source / "extent" - extended_source = source / "extended" - nfs_source.mkdir(parents=True) - extent_source.mkdir(parents=True) - extended_source.mkdir(parents=True) - - for index in range(192): - (nfs_source / f"filler-{index:03d}.txt").write_text( - f"filler {index:03d}\n", encoding="ascii" - ) - (nfs_source / "zz-identity.txt").write_text( - "stable inode identity\n", encoding="ascii" - ) - (extent_source / "extent.bin").write_bytes(b"E" * (1024 * 1024)) - (extended_source / "extended-time.txt").write_text( - "extended timestamp\n", encoding="ascii" - ) - normalize_times(source) - - nfs_base_path = output / ".nfs-base.erofs" - extent_base_path = output / ".extent-base.erofs" - extended_base_path = output / ".extended-base.erofs" - run_mkfs( - nfs_source, - nfs_base_path, - "66666666-7777-4888-9999-aaaaaaaaaa54", - "-E", - "force-inode-compact", - ) - run_mkfs( - extent_source, - extent_base_path, - "66666666-7777-4888-9999-aaaaaaaaaa55", - "-E", - "legacy-compress,force-inode-extended", - "-z", - "lz4", - "-C4096", - ) - run_mkfs( - extended_source, - extended_base_path, - "66666666-7777-4888-9999-aaaaaaaaaa56", - "-E", - "force-inode-extended", - ) - - evidence: list[str] = [] - manifest: dict[str, object] = {} - - nfs_a = ErofsImage.load(nfs_base_path) - identity_a = nfs_a.resolve_root_entry("zz-identity.txt") - if identity_a.inode_size != 32: - raise FixtureError("NFS identity target is not a compact inode") - if identity_a.offset < nfs_a.checksum_end: - raise FixtureError("NFS identity inode overlaps the checksum window") - nfs_a_path = output / "nfs-inode-a.erofs" - nfs_a.save(nfs_a_path) - - nfs_b = nfs_a.clone() - identity_b = nfs_b.resolve_root_entry("zz-identity.txt") - old_mtime = nfs_b.u32(identity_b.offset + 12) - nfs_b.put_u32(identity_b.offset + 12, old_mtime + 1) - nfs_b_path = output / "nfs-inode-b.erofs" - nfs_b.save(nfs_b_path) - nfs_differences = [ - offset - for offset, (left, right) in enumerate(zip(nfs_a.data, nfs_b.data)) - if left != right - ] - mtime_offsets = range(identity_b.offset + 12, identity_b.offset + 16) - if not nfs_differences or any( - offset not in mtime_offsets for offset in nfs_differences - ): - raise FixtureError("NFS replacement changed data outside i_mtime") - if nfs_a.data[: nfs_a.checksum_end] != nfs_b.data[: nfs_b.checksum_end]: - raise FixtureError("NFS replacement changed the superblock block") - if nfs_a.data[SUPER : SUPER + nfs_a.super_size] != nfs_b.data[ - SUPER : SUPER + nfs_b.super_size - ]: - raise FixtureError("NFS replacement changed the declared superblock") - if nfs_a.inode_bytes(identity_a) == nfs_b.inode_bytes(identity_b): - raise FixtureError("NFS replacement did not change inode metadata") - generation_a = nfs_a.generation(identity_a) - generation_b = nfs_b.generation(identity_b) - if generation_a == generation_b: - raise FixtureError("per-inode generation did not change") - - nfs_corrupt = nfs_a.clone() - corrupt_inode = nfs_corrupt.resolve_root_entry("zz-identity.txt") - nfs_corrupt.put_u16( - corrupt_inode.offset, nfs_corrupt.u16(corrupt_inode.offset) | 0x8000 - ) - nfs_corrupt_path = output / "nfs-inode-corrupt.erofs" - nfs_corrupt.save(nfs_corrupt_path) - if nfs_a.data[: nfs_a.checksum_end] != nfs_corrupt.data[: nfs_a.checksum_end]: - raise FixtureError("corrupt inode fixture changed the superblock block") - - evidence.append( - "nfs " - f"nid={identity_a.nid} inode_offset={identity_a.offset} " - f"checksum_end={nfs_a.checksum_end} mtime={old_mtime}->{old_mtime + 1} " - f"generation={generation_a}->{generation_b} superblock_block=identical" - ) - manifest["nfs"] = { - "path": "/zz-identity.txt", - "nid": identity_a.nid, - "inode_offset": identity_a.offset, - "generation_a": generation_a, - "generation_b": generation_b, - "super_size": nfs_a.super_size, - "checksum_end": nfs_a.checksum_end, - } - - extent = ErofsImage.load(extent_base_path) - extent_inode = extent.resolve_root_entry("extent.bin") - if ( - extent_inode.inode_size != 64 - or extent_inode.layout != EROFS_INODE_COMPRESSED_FULL - ): - raise FixtureError("extent target is not an extended full-index inode") - header = align( - extent_inode.offset + extent_inode.inode_size + extent_inode.xattr_size, - 8, - ) - physical_base_offset = align(header + 8, 4) - record_offset = physical_base_offset + 8 - if record_offset + 8 > len(extent.data): - raise FixtureError("extent records lie outside the base image") - root_inode = extent.inode(extent.root_nid) - if not ( - record_offset + 8 <= root_inode.offset - or root_inode.offset + root_inode.inode_size <= header - ): - raise FixtureError("extent conversion overlaps the root inode") - struct.pack_into( - " argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - make_parser = subparsers.add_parser("make") - make_parser.add_argument("--output", type=Path, required=True) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - if args.command == "make": - make_fixtures(args.output) - - -if __name__ == "__main__": - try: - main() - except (FixtureError, OSError, subprocess.CalledProcessError) as error: - raise SystemExit(f"review_fixtures.py: {error}") from error diff --git a/tests/sparse_hole_probe.c b/tests/sparse_hole_probe.c deleted file mode 100644 index c9bf434..0000000 --- a/tests/sparse_hole_probe.c +++ /dev/null @@ -1,63 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -int -main(int argc, char **argv) -{ - unsigned char byte; - char *end; - void *map; - uintmax_t raw_offset; - off_t map_offset, offset; - size_t delta, pagesize; - struct stat sb; - int fd; - - if (argc != 3) - errx(2, "usage: sparse_hole_probe file offset"); - raw_offset = strtoumax(argv[2], &end, 0); - if (*argv[2] == '\0' || *end != '\0' || raw_offset > INT64_MAX) - errx(2, "invalid offset: %s", argv[2]); - offset = (off_t)raw_offset; - pagesize = (size_t)getpagesize(); - map_offset = offset & ~((off_t)pagesize - 1); - delta = (size_t)(offset - map_offset); - - fd = open(argv[1], O_RDONLY); - if (fd < 0) - err(1, "open %s", argv[1]); - if (fstat(fd, &sb) != 0) - err(1, "fstat %s", argv[1]); - if (offset < 0 || offset >= sb.st_size || - sb.st_size - offset < (off_t)(pagesize - delta)) - errx(1, "probe page is outside the file"); - if (pread(fd, &byte, 1, offset) != 1) - err(1, "pread at %jd", (intmax_t)offset); - if (byte != 0) - errx(1, "pread returned non-zero byte %#x", byte); - - map = mmap(NULL, pagesize, PROT_READ, MAP_PRIVATE, fd, map_offset); - if (map == MAP_FAILED) - err(1, "mmap at %jd", (intmax_t)map_offset); - if (madvise(map, pagesize, MADV_DONTNEED) != 0) - err(1, "madvise"); - if (((const unsigned char *)map)[delta] != 0 || - ((const unsigned char *)map)[pagesize - 1] != 0) - errx(1, "mmap returned non-zero hole data"); - if (munmap(map, pagesize) != 0) - err(1, "munmap"); - if (close(fd) != 0) - err(1, "close"); - - printf("PASS size=%jd offset=%jd pread=zero mmap-page=zero\n", - (intmax_t)sb.st_size, (intmax_t)offset); - return (0); -} diff --git a/tests/stat_special.c b/tests/stat_special.c deleted file mode 100644 index a0135fc..0000000 --- a/tests/stat_special.c +++ /dev/null @@ -1,70 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -static unsigned int -parse_number(const char *text) -{ - char *end; - unsigned long value; - - errno = 0; - value = strtoul(text, &end, 0); - if (errno != 0 || *end != '\0' || value > UINT_MAX) - errx(2, "invalid number: %s", text); - return ((unsigned int)value); -} - -int -main(int argc, char **argv) -{ - struct stat sb; - unsigned int expected_major, expected_minor; - bool is_device; - - if (argc != 3 && argc != 5) - errx(2, "usage: stat_special char|block path major minor | " - "stat_special fifo path"); - if (lstat(argv[2], &sb) != 0) - err(1, "lstat %s", argv[2]); - is_device = strcmp(argv[1], "char") == 0 || strcmp(argv[1], "block") == 0; - if (is_device) { - if (argc != 5) - errx(2, "device checks require major and minor"); - expected_major = parse_number(argv[3]); - expected_minor = parse_number(argv[4]); - if ((strcmp(argv[1], "char") == 0 && !S_ISCHR(sb.st_mode)) || - (strcmp(argv[1], "block") == 0 && !S_ISBLK(sb.st_mode))) - errx(1, "%s has the wrong file type", argv[2]); - if ((unsigned int)major(sb.st_rdev) != expected_major || - (unsigned int)minor(sb.st_rdev) != expected_minor) - errx(1, "%s rdev=%#jx major=%u minor=%u, expected %u:%u", - argv[2], (uintmax_t)sb.st_rdev, - (unsigned int)major(sb.st_rdev), - (unsigned int)minor(sb.st_rdev), expected_major, - expected_minor); - } else if (strcmp(argv[1], "fifo") == 0) { - if (argc != 3) - errx(2, "FIFO checks do not take major and minor"); - if (!S_ISFIFO(sb.st_mode)) - errx(1, "%s is not a FIFO", argv[2]); - if (sb.st_rdev != NODEV) - errx(1, "%s FIFO st_rdev=%#jx, expected NODEV", argv[2], - (uintmax_t)sb.st_rdev); - } else { - errx(2, "unknown type: %s", argv[1]); - } - printf("PASS path=%s type=%s rdev=%#jx major=%u minor=%u\n", argv[2], - argv[1], (uintmax_t)sb.st_rdev, (unsigned int)major(sb.st_rdev), - (unsigned int)minor(sb.st_rdev)); - return (0); -} diff --git a/tests/statfs_probe.c b/tests/statfs_probe.c deleted file mode 100644 index c8b7e93..0000000 --- a/tests/statfs_probe.c +++ /dev/null @@ -1,24 +0,0 @@ -#include - -#include -#include -#include - -int -main(int argc, char **argv) -{ - struct statfs sb; - - if (argc != 2) - errx(2, "usage: statfs_probe path"); - if (statfs(argv[1], &sb) != 0) - err(1, "statfs %s", argv[1]); - printf("fstype=%s bsize=%ju iosize=%ju blocks=%ju bfree=%ju " - "bavail=%jd files=%ju ffree=%ju readonly=%d from=%s on=%s\n", - sb.f_fstypename, (uintmax_t)sb.f_bsize, (uintmax_t)sb.f_iosize, - (uintmax_t)sb.f_blocks, (uintmax_t)sb.f_bfree, - (intmax_t)sb.f_bavail, (uintmax_t)sb.f_files, - (uintmax_t)sb.f_ffree, (sb.f_flags & MNT_RDONLY) != 0, - sb.f_mntfromname, sb.f_mntonname); - return (0); -} diff --git a/src/xattr.c b/xattr.c similarity index 100% rename from src/xattr.c rename to xattr.c diff --git a/src/xattr.h b/xattr.h similarity index 100% rename from src/xattr.h rename to xattr.h diff --git a/src/zdata.c b/zdata.c similarity index 100% rename from src/zdata.c rename to zdata.c diff --git a/src/zmap.c b/zmap.c similarity index 100% rename from src/zmap.c rename to zmap.c